pyhdf5_handler.src.object_handler

View Source

  1from __future__ import annotations
  2
  3import numpy as np
  4import numbers
  5import pandas as pd
  6import datetime
  7from ..src import constant
  8
  9
 10def _isinstance_pandas(value):
 11    pandas_classes = [
 12        getattr(pd, item)
 13        for item in dir(pd)
 14        if isinstance(getattr(pd, item), type)
 15    ]
 16    for cls in pandas_classes:
 17        if isinstance(value, cls):
 18            return True
 19    return False
 20
 21
 22def _isinstance_numpy(value):
 23    numpy_classes = [
 24        getattr(np, item)
 25        for item in dir(np)
 26        if isinstance(getattr(np, item), type)
 27    ]
 28    for cls in numpy_classes:
 29        if isinstance(value, cls):
 30            return True
 31    return False
 32
 33
 34def _isinstance_datetime(value):
 35    datetime_classes = [
 36        getattr(datetime, item)
 37        for item in dir(datetime)
 38        if isinstance(getattr(datetime, item), type)
 39    ]
 40    for cls in datetime_classes:
 41        if isinstance(value, cls):
 42            return True
 43    return False
 44
 45
 46def _isinstance_exclude_obj(value):
 47
 48    type_str = str(type(value))
 49    module_name = type_str.split("'")[1].split(".")[0]
 50
 51    if module_name in constant.EXCLUDE_PYTHON_OBJ:
 52        return True
 53    else:
 54        return False
 55
 56
 57def generate_dict_structure(
 58    dictionary, recursion_counter=0, recursion_limit=100, include_method=True
 59):
 60    """
 61
 62    this function create a full dictionnary containing all the structure of an dictionnary in order to save it to an hdf5
 63
 64    Parameters
 65    ----------
 66
 67    instance : python dictionary
 68        a custom dictionary.
 69    recursion_limit : int
 70        max recursion limit to dig inside `instance`
 71    recursion_counter: int
 72        current recursion value
 73    include_method: bool
 74        Include methods/functions in the object structure, if False, only data will be included.
 75
 76    Returns
 77    -------
 78
 79    list or dict :
 80        A list or dictionary matching the structure of the python object.
 81
 82    """
 83    key_data = {}
 84    key_list = list()
 85    recursion_counter = 0
 86    for attr, value in dictionary.items():
 87
 88        try:
 89            if _isinstance_exclude_obj(value):
 90                next(attrs)
 91            elif isinstance(value, dict):
 92                subkey_data = generate_dict_structure(value)
 93                if len(subkey_data) > 0:
 94                    key_data.update({attr: subkey_data})
 95
 96            elif isinstance(value, (list, tuple, numbers.Number, str)):
 97                key_list.append(attr)
 98
 99            elif _isinstance_pandas(value):
100                key_list.append(attr)
101
102            elif _isinstance_datetime(value):
103                key_list.append(attr)
104
105            elif _isinstance_numpy(value):
106                key_list.append(attr)
107
108            elif type(value) == "method":
109                if include_method:
110                    key_list.append(attr)
111                else:
112                    next(attr)
113            else:
114
115                recursion_counter = recursion_counter + 1
116
117                if recursion_counter > recursion_limit:
118                    print(
119                        f"recursion counter exceed the limit of {recursion_limit}... return"
120                    )
121                    return
122
123                subkey_data = generate_object_structure(
124                    value,
125                    recursion_counter=recursion_counter,
126                    recursion_limit=recursion_limit,
127                    include_method=include_method,
128                )
129                if len(subkey_data) > 0:
130                    key_data.update({attr: subkey_data})
131
132        except:
133            pass
134
135    for attr, value in key_data.items():
136        key_list.append({attr: value})
137
138    return key_list
139
140
141def generate_object_from_dictionary(dictionary):
142
143    class Object:
144        def __init__(self):
145            pass
146
147    obj = Object()
148
149    if isinstance(dictionary, dict):
150        for attr, value in dictionary.items():
151
152            if isinstance(value, (dict, list, tuple)):
153                setattr(obj, attr, generate_object_from_dictionary(value))
154            else:
155                setattr(obj, attr, value)
156    elif isinstance(dictionary, (list, tuple)):
157        return dictionary
158        # for attr in dictionary:
159        #    setattr(obj, attr, None)
160
161    return obj
162
163
164def map_dict_to_object(structure, dictionary, obj):
165    """
166    Map a dictionary to an object knowing its structure.
167    Structure must be generated with generate_dict_structure or generate_object_structure. If the object or an attribute of this object is
168    None, a empty object is created to be able to set new attribute.
169    """
170
171    class Object:
172        def __init__(self):
173            pass
174
175    if obj is None:
176        obj = Object()
177
178    if isinstance(structure, dict):
179
180        for attr, value in structure.items():
181            if isinstance(value, (dict, list, tuple)):
182
183                if not hasattr(obj, attr):
184                    setattr(obj, str(attr), Object())
185                else:
186                    if getattr(obj, attr) is None:
187                        setattr(obj, str(attr), Object())
188
189                map_dict_to_object(value, dictionary[attr], getattr(obj, attr))
190            else:
191                v = None
192                if attr in dictionary.keys():
193                    v = dictionary[attr]
194
195                setattr(obj, str(attr), v)
196
197    elif isinstance(structure, (list, tuple)):
198
199        for attr in structure:
200            if isinstance(attr, (dict, list, tuple)):
201                map_dict_to_object(attr, dictionary, obj)
202            else:
203                v = None
204                if attr in dictionary.keys():
205                    v = dictionary[attr]
206
207                setattr(obj, str(attr), v)
208
209
210def generate_object_structure(
211    instance, recursion_counter=0, recursion_limit=100, include_method=True
212):
213    """
214
215    this function create a full dictionnary containing all the structure of an object in order to save it to an hdf5
216
217    Parameters
218    ----------
219
220    instance : object
221        a custom python object.
222    recursion_limit : int
223        max recursion limit to dig inside `instance`
224    recursion_counter: int
225        current recursion value
226    include_method: bool
227        Include methods/functions in the object structure, if False, only data will be included.
228
229    Returns
230    -------
231
232    list or dict :
233        A list or dictionary matching the structure of the python object.
234
235    """
236    key_data = {}
237    key_list = list()
238    return_list = False
239    # recursion_counter += 1
240    for attr in dir(instance):
241
242        if not attr.startswith("_") and not attr in ["from_handle", "copy"]:
243
244            try:
245                value = getattr(instance, attr)
246
247                if _isinstance_exclude_obj(value):
248                    next(attr)
249
250                elif isinstance(value, (list, tuple)):
251                    key_list.append(attr)
252                    return_list = True
253
254                elif _isinstance_numpy(value):
255                    key_list.append(attr)
256                    return_list = True
257
258                elif isinstance(value, dict):
259
260                    depp_key_data = generate_dict_structure(value)
261                    if len(depp_key_data) > 0:
262                        key_data.update({attr: depp_key_data})
263
264                elif isinstance(value, numbers.Number):
265                    key_list.append(attr)
266                    return_list = True
267
268                elif isinstance(value, str):
269                    key_list.append(attr)
270                    return_list = True
271
272                elif "<class 'method" in str(type(value)):
273                    if include_method:
274                        key_list.append(attr)
275                        return_list = True
276                    else:
277                        next(attr)
278
279                elif _isinstance_pandas(value):
280                    key_list.append(attr)
281                    return_list = True
282
283                elif _isinstance_datetime(value):
284                    key_list.append(attr)
285                    return_list = True
286
287                # ~ elif value is None:
288                # ~ key_list.append(attr)
289                # ~ return_list = True
290
291                else:
292
293                    recursion_counter = recursion_counter + 1
294
295                    if recursion_counter > recursion_limit:
296                        print(
297                            f"recursion counter exceed the limit of {recursion_limit}... return"
298                        )
299                        return
300
301                    depp_key_data = generate_object_structure(
302                        value,
303                        recursion_counter=recursion_counter,
304                        recursion_limit=recursion_limit,
305                        include_method=include_method,
306                    )
307
308                    if len(depp_key_data) > 0:
309                        key_data.update({attr: depp_key_data})
310
311            except:
312                # raise ValueError("unable to parse attr", attr)
313                # ~ print("unable to parse attr", attr, "skip it...")
314                pass
315
316    if return_list:
317        for attr, value in key_data.items():
318            key_list.append({attr: value})
319
320        return key_list
321
322    else:
323        return key_data
324
325
326def read_object_as_dict(instance, recursion_counter=0, recursion_limit=100):
327    """
328
329    create a dictionary from a custom python object
330
331    Parameters
332    ----------
333
334    instance : object
335        an custom python object
336    recursion_limit : int
337        max recursion limit to dig inside `instance`
338    recursion_counter: int
339        current recursion value
340
341    Return
342    ------
343
344    key_data: dict
345        an dictionary containing all keys and atributes of the object
346
347    """
348    key_data = {}
349    # recursion_counter = 0
350    for attr in dir(instance):
351        # print(attr)
352        if not attr.startswith("_") and not attr in ["from_handle", "copy"]:
353            try:
354                value = getattr(instance, attr)
355
356                if _isinstance_exclude_obj(value):
357                    next(attr)
358
359                elif isinstance(value, (list, tuple)):
360
361                    if isinstance(value, list):
362                        value = np.array(value).astype("U")
363
364                    if value.dtype == "object" or value.dtype.char == "U":
365                        value = value.astype("U")
366
367                    key_data.update({attr: value})
368
369                elif isinstance(value, dict):
370                    key_data.update({attr: value})
371
372                elif isinstance(value, numbers.Number):
373                    key_data.update({attr: value})
374
375                elif isinstance(value, str):
376                    key_data.update({attr: value})
377
378                elif type(value) == "method":
379                    next(attr)
380
381                elif _isinstance_pandas(value):
382                    if value.dtype == "object" or value.dtype.char == "U":
383                        value = value.astype("U")
384                    key_data.update({attr: value})
385
386                elif _isinstance_datetime(value):
387                    if value.dtype == "object" or value.dtype.char == "U":
388                        value = value.astype("U")
389                    key_data.update({attr: value})
390
391                elif _isinstance_numpy(value):
392                    if value.dtype == "object" or value.dtype.char == "U":
393                        value = value.astype("U")
394                    key_data.update({attr: value})
395
396                else:
397
398                    recursion_counter = recursion_counter + 1
399
400                    if recursion_counter > recursion_limit:
401                        print(
402                            f"recursion counter exceed the limit of {recursion_limit}... return"
403                        )
404                        return
405
406                    depp_key_data = read_object_as_dict(
407                        value,
408                        recursion_counter=recursion_counter,
409                        recursion_limit=recursion_limit,
410                    )
411
412                    if len(depp_key_data) > 0:
413                        key_data.update({attr: depp_key_data})
414
415            except:
416                pass
417
418    return key_data

def generate_dict_structure( dictionary, recursion_counter=0, recursion_limit=100, include_method=True): View Source

 58def generate_dict_structure(
 59    dictionary, recursion_counter=0, recursion_limit=100, include_method=True
 60):
 61    """
 62
 63    this function create a full dictionnary containing all the structure of an dictionnary in order to save it to an hdf5
 64
 65    Parameters
 66    ----------
 67
 68    instance : python dictionary
 69        a custom dictionary.
 70    recursion_limit : int
 71        max recursion limit to dig inside `instance`
 72    recursion_counter: int
 73        current recursion value
 74    include_method: bool
 75        Include methods/functions in the object structure, if False, only data will be included.
 76
 77    Returns
 78    -------
 79
 80    list or dict :
 81        A list or dictionary matching the structure of the python object.
 82
 83    """
 84    key_data = {}
 85    key_list = list()
 86    recursion_counter = 0
 87    for attr, value in dictionary.items():
 88
 89        try:
 90            if _isinstance_exclude_obj(value):
 91                next(attrs)
 92            elif isinstance(value, dict):
 93                subkey_data = generate_dict_structure(value)
 94                if len(subkey_data) > 0:
 95                    key_data.update({attr: subkey_data})
 96
 97            elif isinstance(value, (list, tuple, numbers.Number, str)):
 98                key_list.append(attr)
 99
100            elif _isinstance_pandas(value):
101                key_list.append(attr)
102
103            elif _isinstance_datetime(value):
104                key_list.append(attr)
105
106            elif _isinstance_numpy(value):
107                key_list.append(attr)
108
109            elif type(value) == "method":
110                if include_method:
111                    key_list.append(attr)
112                else:
113                    next(attr)
114            else:
115
116                recursion_counter = recursion_counter + 1
117
118                if recursion_counter > recursion_limit:
119                    print(
120                        f"recursion counter exceed the limit of {recursion_limit}... return"
121                    )
122                    return
123
124                subkey_data = generate_object_structure(
125                    value,
126                    recursion_counter=recursion_counter,
127                    recursion_limit=recursion_limit,
128                    include_method=include_method,
129                )
130                if len(subkey_data) > 0:
131                    key_data.update({attr: subkey_data})
132
133        except:
134            pass
135
136    for attr, value in key_data.items():
137        key_list.append({attr: value})
138
139    return key_list

this function create a full dictionnary containing all the structure of an dictionnary in order to save it to an hdf5

Parameters

instance : python dictionary a custom dictionary. recursion_limit : int max recursion limit to dig inside instance recursion_counter: int current recursion value include_method: bool Include methods/functions in the object structure, if False, only data will be included.

Returns

list or dict : A list or dictionary matching the structure of the python object.

def generate_object_from_dictionary(dictionary): View Source

142def generate_object_from_dictionary(dictionary):
143
144    class Object:
145        def __init__(self):
146            pass
147
148    obj = Object()
149
150    if isinstance(dictionary, dict):
151        for attr, value in dictionary.items():
152
153            if isinstance(value, (dict, list, tuple)):
154                setattr(obj, attr, generate_object_from_dictionary(value))
155            else:
156                setattr(obj, attr, value)
157    elif isinstance(dictionary, (list, tuple)):
158        return dictionary
159        # for attr in dictionary:
160        #    setattr(obj, attr, None)
161
162    return obj

def map_dict_to_object(structure, dictionary, obj): View Source

165def map_dict_to_object(structure, dictionary, obj):
166    """
167    Map a dictionary to an object knowing its structure.
168    Structure must be generated with generate_dict_structure or generate_object_structure. If the object or an attribute of this object is
169    None, a empty object is created to be able to set new attribute.
170    """
171
172    class Object:
173        def __init__(self):
174            pass
175
176    if obj is None:
177        obj = Object()
178
179    if isinstance(structure, dict):
180
181        for attr, value in structure.items():
182            if isinstance(value, (dict, list, tuple)):
183
184                if not hasattr(obj, attr):
185                    setattr(obj, str(attr), Object())
186                else:
187                    if getattr(obj, attr) is None:
188                        setattr(obj, str(attr), Object())
189
190                map_dict_to_object(value, dictionary[attr], getattr(obj, attr))
191            else:
192                v = None
193                if attr in dictionary.keys():
194                    v = dictionary[attr]
195
196                setattr(obj, str(attr), v)
197
198    elif isinstance(structure, (list, tuple)):
199
200        for attr in structure:
201            if isinstance(attr, (dict, list, tuple)):
202                map_dict_to_object(attr, dictionary, obj)
203            else:
204                v = None
205                if attr in dictionary.keys():
206                    v = dictionary[attr]
207
208                setattr(obj, str(attr), v)

Map a dictionary to an object knowing its structure. Structure must be generated with generate_dict_structure or generate_object_structure. If the object or an attribute of this object is None, a empty object is created to be able to set new attribute.

def generate_object_structure( instance, recursion_counter=0, recursion_limit=100, include_method=True): View Source

211def generate_object_structure(
212    instance, recursion_counter=0, recursion_limit=100, include_method=True
213):
214    """
215
216    this function create a full dictionnary containing all the structure of an object in order to save it to an hdf5
217
218    Parameters
219    ----------
220
221    instance : object
222        a custom python object.
223    recursion_limit : int
224        max recursion limit to dig inside `instance`
225    recursion_counter: int
226        current recursion value
227    include_method: bool
228        Include methods/functions in the object structure, if False, only data will be included.
229
230    Returns
231    -------
232
233    list or dict :
234        A list or dictionary matching the structure of the python object.
235
236    """
237    key_data = {}
238    key_list = list()
239    return_list = False
240    # recursion_counter += 1
241    for attr in dir(instance):
242
243        if not attr.startswith("_") and not attr in ["from_handle", "copy"]:
244
245            try:
246                value = getattr(instance, attr)
247
248                if _isinstance_exclude_obj(value):
249                    next(attr)
250
251                elif isinstance(value, (list, tuple)):
252                    key_list.append(attr)
253                    return_list = True
254
255                elif _isinstance_numpy(value):
256                    key_list.append(attr)
257                    return_list = True
258
259                elif isinstance(value, dict):
260
261                    depp_key_data = generate_dict_structure(value)
262                    if len(depp_key_data) > 0:
263                        key_data.update({attr: depp_key_data})
264
265                elif isinstance(value, numbers.Number):
266                    key_list.append(attr)
267                    return_list = True
268
269                elif isinstance(value, str):
270                    key_list.append(attr)
271                    return_list = True
272
273                elif "<class 'method" in str(type(value)):
274                    if include_method:
275                        key_list.append(attr)
276                        return_list = True
277                    else:
278                        next(attr)
279
280                elif _isinstance_pandas(value):
281                    key_list.append(attr)
282                    return_list = True
283
284                elif _isinstance_datetime(value):
285                    key_list.append(attr)
286                    return_list = True
287
288                # ~ elif value is None:
289                # ~ key_list.append(attr)
290                # ~ return_list = True
291
292                else:
293
294                    recursion_counter = recursion_counter + 1
295
296                    if recursion_counter > recursion_limit:
297                        print(
298                            f"recursion counter exceed the limit of {recursion_limit}... return"
299                        )
300                        return
301
302                    depp_key_data = generate_object_structure(
303                        value,
304                        recursion_counter=recursion_counter,
305                        recursion_limit=recursion_limit,
306                        include_method=include_method,
307                    )
308
309                    if len(depp_key_data) > 0:
310                        key_data.update({attr: depp_key_data})
311
312            except:
313                # raise ValueError("unable to parse attr", attr)
314                # ~ print("unable to parse attr", attr, "skip it...")
315                pass
316
317    if return_list:
318        for attr, value in key_data.items():
319            key_list.append({attr: value})
320
321        return key_list
322
323    else:
324        return key_data

this function create a full dictionnary containing all the structure of an object in order to save it to an hdf5

Parameters

instance : object a custom python object. recursion_limit : int max recursion limit to dig inside instance recursion_counter: int current recursion value include_method: bool Include methods/functions in the object structure, if False, only data will be included.

Returns

list or dict : A list or dictionary matching the structure of the python object.

def read_object_as_dict(instance, recursion_counter=0, recursion_limit=100): View Source

327def read_object_as_dict(instance, recursion_counter=0, recursion_limit=100):
328    """
329
330    create a dictionary from a custom python object
331
332    Parameters
333    ----------
334
335    instance : object
336        an custom python object
337    recursion_limit : int
338        max recursion limit to dig inside `instance`
339    recursion_counter: int
340        current recursion value
341
342    Return
343    ------
344
345    key_data: dict
346        an dictionary containing all keys and atributes of the object
347
348    """
349    key_data = {}
350    # recursion_counter = 0
351    for attr in dir(instance):
352        # print(attr)
353        if not attr.startswith("_") and not attr in ["from_handle", "copy"]:
354            try:
355                value = getattr(instance, attr)
356
357                if _isinstance_exclude_obj(value):
358                    next(attr)
359
360                elif isinstance(value, (list, tuple)):
361
362                    if isinstance(value, list):
363                        value = np.array(value).astype("U")
364
365                    if value.dtype == "object" or value.dtype.char == "U":
366                        value = value.astype("U")
367
368                    key_data.update({attr: value})
369
370                elif isinstance(value, dict):
371                    key_data.update({attr: value})
372
373                elif isinstance(value, numbers.Number):
374                    key_data.update({attr: value})
375
376                elif isinstance(value, str):
377                    key_data.update({attr: value})
378
379                elif type(value) == "method":
380                    next(attr)
381
382                elif _isinstance_pandas(value):
383                    if value.dtype == "object" or value.dtype.char == "U":
384                        value = value.astype("U")
385                    key_data.update({attr: value})
386
387                elif _isinstance_datetime(value):
388                    if value.dtype == "object" or value.dtype.char == "U":
389                        value = value.astype("U")
390                    key_data.update({attr: value})
391
392                elif _isinstance_numpy(value):
393                    if value.dtype == "object" or value.dtype.char == "U":
394                        value = value.astype("U")
395                    key_data.update({attr: value})
396
397                else:
398
399                    recursion_counter = recursion_counter + 1
400
401                    if recursion_counter > recursion_limit:
402                        print(
403                            f"recursion counter exceed the limit of {recursion_limit}... return"
404                        )
405                        return
406
407                    depp_key_data = read_object_as_dict(
408                        value,
409                        recursion_counter=recursion_counter,
410                        recursion_limit=recursion_limit,
411                    )
412
413                    if len(depp_key_data) > 0:
414                        key_data.update({attr: depp_key_data})
415
416            except:
417                pass
418
419    return key_data

create a dictionary from a custom python object

Parameters

instance : object an custom python object recursion_limit : int max recursion limit to dig inside instance recursion_counter: int current recursion value

Return

key_data: dict an dictionary containing all keys and atributes of the object