pyhdf5_handler.src.hdf5_handler

   1from __future__ import annotations
   2
   3import os
   4import h5py
   5import numpy as np
   6import numbers
   7import pandas as pd
   8import datetime
   9import time
  10import importlib
  11
  12from ..src import object_handler
  13from ..src import constant
  14
  15import gc
  16import re
  17
  18
  19def close_all_hdf5_file():
  20    """
  21    Close all hdf5 file opened in the current session
  22    """
  23
  24    for obj in gc.get_objects():  # Browse through ALL objects
  25        if isinstance(obj, h5py.File):  # Just HDF5 files
  26            try:
  27                print(f"try closing {obj}")
  28                obj.close()
  29            except:
  30                pass  # Was already closed
  31
  32
  33def open_hdf5(path, read_only=False, replace=False, wait_time=0):
  34    """
  35
  36    Open or create an HDF5 file.
  37
  38    Parameters
  39    ----------
  40
  41    path : str
  42        The file path.
  43
  44    read_only : boolean
  45        If true the access to the hdf5 fil is in read-only mode. Multi process can read the same hdf5 file simulteneously. This is not possible when access mode are append 'a' or write 'w'.
  46
  47    replace: Boolean
  48        If true, the existing hdf5file is erased
  49
  50    wait_time: int
  51        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
  52
  53    Returns
  54    -------
  55
  56    f :
  57        A h5py object.
  58
  59    Examples
  60    --------
  61
  62    >>> hdf5=pyhdf5_handler.open_hdf5("./my_hdf5.hdf5")
  63    >>> hdf5.keys()
  64    >>> hdf5.attrs.keys()
  65
  66    """
  67    f = None
  68    wait = 0
  69    while wait <= wait_time:
  70
  71        f = None
  72        exist_file = True
  73
  74        try:
  75
  76            if read_only:
  77                if os.path.isfile(path):
  78                    f = h5py.File(path, "r")
  79
  80                else:
  81                    exist_file = False
  82                    raise ValueError(f"File {path} does not exist.")
  83
  84            else:
  85                if replace:
  86                    f = h5py.File(path, "w")
  87
  88                else:
  89                    if os.path.isfile(path):
  90                        f = h5py.File(path, "a")
  91
  92                    else:
  93                        f = h5py.File(path, "w")
  94        except:
  95            pass
  96
  97        if f is None:
  98            if not exist_file:
  99                print(f"File {path} does not exist.")
 100                return f
 101            else:
 102                print(
 103                    f"The file {path} is unvailable, waiting {wait}/{wait_time}s"
 104                )
 105
 106            wait = wait + 1
 107
 108            if wait_time > 0:
 109                time.sleep(1)
 110
 111        else:
 112            break
 113
 114    return f
 115
 116
 117def add_hdf5_sub_group(hdf5, subgroup=None):
 118    """
 119    Create a new subgroup in a HDF5 object
 120
 121    Parameters
 122    ----------
 123
 124    hdf5 : h5py.File
 125        An hdf5 object opened with open_hdf5()
 126
 127    subgroup: str
 128        Path to a subgroub that must be created
 129
 130    Returns
 131    -------
 132
 133    hdf5 :
 134        the h5py object.
 135
 136    Examples
 137    --------
 138
 139    >>> hdf5=pyhdf5_handler.open_hdf5("./model_subgroup.hdf5", replace=True)
 140    >>> hdf5=pyhdf5_handler.add_hdf5_sub_group(hdf5, subgroup="mygroup")
 141    >>> hdf5.keys()
 142    >>> hdf5.attrs.keys()
 143
 144    """
 145    if subgroup is not None:
 146        if subgroup == "":
 147            subgroup = "./"
 148
 149        hdf5.require_group(subgroup)
 150
 151    return hdf5
 152
 153
 154def _dump_object_to_hdf5_from_list_attribute(hdf5, instance, list_attr):
 155    """
 156    dump a object to a hdf5 file from a list of attributes
 157
 158    Parameters
 159    ----------
 160    hdf5 : h5py.File
 161        an hdf5 object
 162
 163    instance : object
 164        a custom python object.
 165
 166    list_attr : list
 167        a list of attribute
 168
 169    """
 170    if isinstance(list_attr, list):
 171        for attr in list_attr:
 172            if isinstance(attr, str):
 173                _dump_object_to_hdf5_from_str_attribute(hdf5, instance, attr)
 174
 175            elif isinstance(attr, list):
 176                _dump_object_to_hdf5_from_list_attribute(hdf5, instance, attr)
 177
 178            elif isinstance(attr, dict):
 179                _dump_object_to_hdf5_from_dict_attribute(hdf5, instance, attr)
 180
 181            else:
 182                raise ValueError(
 183                    f"inconsistent {attr} in {list_attr}. {attr} must be a an instance of dict, list or str"
 184                )
 185
 186    else:
 187        raise ValueError(f"{list_attr} must be a instance of list.")
 188
 189
 190def _dump_object_to_hdf5_from_dict_attribute(hdf5, instance, dict_attr):
 191    """
 192    dump a object to a hdf5 file from a dictionary of attributes
 193
 194    Parameters
 195    ----------
 196
 197    hdf5 : h5py.File
 198        an hdf5 object
 199
 200    instance : object
 201        a custom python object.
 202
 203    dict_attr : dict
 204        a dictionary of attribute
 205
 206    """
 207    if isinstance(dict_attr, dict):
 208        for attr, value in dict_attr.items():
 209            hdf5 = add_hdf5_sub_group(hdf5, subgroup=attr)
 210
 211            try:
 212                sub_instance = getattr(instance, attr)
 213
 214            except:
 215                if isinstance(instance, dict):
 216                    sub_instance = instance[attr]
 217                else:
 218                    sub_instance = instance
 219
 220            if isinstance(value, dict):
 221                _dump_object_to_hdf5_from_dict_attribute(
 222                    hdf5[attr], sub_instance, value
 223                )
 224
 225            elif isinstance(value, list):
 226                _dump_object_to_hdf5_from_list_attribute(
 227                    hdf5[attr], sub_instance, value
 228                )
 229
 230            elif isinstance(value, str):
 231                _dump_object_to_hdf5_from_str_attribute(
 232                    hdf5[attr], sub_instance, value
 233                )
 234
 235            else:
 236
 237                raise ValueError(
 238                    f"inconsistent '{attr}' in '{dict_attr}'. Dict({attr}) must be a instance of dict, list or str"
 239                )
 240
 241    else:
 242        raise ValueError(f"{dict_attr} must be a instance of dict.")
 243
 244
 245def _dump_object_to_hdf5_from_str_attribute(hdf5, instance, str_attr):
 246    """
 247    dump a object to a hdf5 file from a string attribute
 248
 249    Parameters
 250    ----------
 251
 252    hdf5 : h5py.File
 253        an hdf5 object
 254
 255    instance : object
 256        a custom python object.
 257
 258    str_attr : str
 259        a string attribute
 260
 261    """
 262
 263    if isinstance(str_attr, str):
 264
 265        try:
 266            value = getattr(instance, str_attr)
 267
 268        except:
 269            if isinstance(instance, dict):
 270                value = instance[str_attr]
 271            else:
 272                value = instance
 273
 274        try:
 275
 276            attribute_name = str(str_attr)
 277            for character in "/ ":
 278                attribute_name = attribute_name.replace(character, "_")
 279
 280            if isinstance(value, dict):
 281
 282                # print("---> dictionary: ", str_attr, value)
 283
 284                hdf5 = add_hdf5_sub_group(hdf5, subgroup=attribute_name)
 285                save_dict_to_hdf5(hdf5[attribute_name], value)
 286
 287            else:
 288
 289                hdf5_dataset_creator(hdf5, attribute_name, value)
 290
 291        except:
 292            raise ValueError(
 293                f"Unable to dump attribute {str_attr} with value {value} from {instance}"
 294            )
 295
 296    else:
 297        raise ValueError(f"{str_attr} must be a instance of str.")
 298
 299
 300def _dump_object_to_hdf5_from_iteratable(hdf5, instance, iteratable=None):
 301    """
 302       dump a object to a hdf5 file from a iteratable object list or dict
 303
 304       Parameters
 305       ----------
 306
 307       hdf5 : h5py.File
 308           an hdf5 object
 309       instance : object
 310           a custom python object.
 311       iteratable : list | dict
 312           a list or a dict of attribute
 313
 314       Examples
 315       --------
 316
 317       >>> setup, mesh = smash.load_dataset("cance")
 318       >>> model = smash.Model(setup, mesh)
 319       >>> model.run(inplace=True)
 320       >>>
 321       >>> hdf5=pyhdf5_handler.open_hdf5("./model.hdf5", replace=True)
 322       >>> hdf5=pyhdf5_handler.add_hdf5_sub_group(hdf5, subgroup="model1")
 323    pyhdf5_handler._dump_object_to_hdf5_from_iteratable(hdf5["model1"], model)
 324
 325    """
 326    if isinstance(iteratable, list):
 327        _dump_object_to_hdf5_from_list_attribute(hdf5, instance, iteratable)
 328
 329    elif isinstance(iteratable, dict):
 330        _dump_object_to_hdf5_from_dict_attribute(hdf5, instance, iteratable)
 331
 332    else:
 333        raise ValueError(f"{iteratable} must be a instance of list or dict.")
 334
 335
 336def _hdf5_handle_str(name, value):
 337
 338    dataset = {
 339        "name": name,
 340        "attr_value": str(type(value)),
 341        "dataset_value": value,
 342        "shape": 1,
 343        "dtype": h5py.string_dtype(encoding="utf-8"),
 344    }
 345
 346    return dataset
 347
 348
 349def _hdf5_handle_numbers(name: str, value: numbers.Number):
 350
 351    arr = np.array([value])
 352    dataset = {
 353        "name": name,
 354        "attr_value": str(type(value)),
 355        "dataset_value": arr,
 356        "shape": arr.shape,
 357        "dtype": arr.dtype,
 358    }
 359
 360    return dataset
 361
 362
 363def _hdf5_handle_none(name: str, value: None):
 364
 365    dataset = {
 366        "name": name,
 367        "attr_value": "_None_",
 368        "dataset_value": "_None_",
 369        "shape": 1,
 370        "dtype": h5py.string_dtype(encoding="utf-8"),
 371    }
 372
 373    return dataset
 374
 375
 376def _hdf5_handle_timestamp(
 377    name: str, value: pd.Timestamp | np.datetime64 | datetime.date
 378):
 379
 380    dtype = type(value)
 381
 382    if isinstance(value, (np.datetime64)):
 383        value = value.tolist()
 384
 385    dataset = {
 386        "name": name,
 387        "attr_value": str(dtype),
 388        "dataset_value": value.strftime("%Y-%m-%d %H:%M"),
 389        "shape": 1,
 390        "dtype": h5py.string_dtype(encoding="utf-8"),
 391    }
 392
 393    return dataset
 394
 395
 396def _hdf5_handle_DatetimeIndex(name: str, value: pd.DatetimeIndex):
 397
 398    dataset = _hdf5_handle_array(name, value)
 399
 400    return dataset
 401
 402
 403def _hdf5_handle_PandaDataFrame(
 404    hdf5: h5py.File, name: str, value: pd.DataFrame
 405):
 406
 407    hdf5 = add_hdf5_sub_group(hdf5, subgroup=name)
 408    hdf5_data = hdf5[name]
 409
 410    hdf5_data.attrs["_Pandas_DataFrame"] = 1
 411
 412    keys = _hdf5_handle_array("columns", np.array(list(value.columns)))
 413    _hdf5_create_dataset(hdf5_data, keys)
 414
 415    dataset = _hdf5_handle_array("array", value.to_numpy())
 416    _hdf5_create_dataset(hdf5_data, dataset)
 417
 418    dtype = []
 419    for col in value.columns:
 420        if pd.api.types.is_string_dtype(value[col]):
 421            dtype_name = "str"
 422        else:
 423            dtype_name = value[col].dtype.name
 424
 425        dtype.append(dtype_name)
 426
 427    dataset = _hdf5_handle_list("dtype", dtype)
 428    _hdf5_create_dataset(hdf5_data, dataset)
 429
 430    return
 431
 432
 433def _hdf5_handle_list(name: str, value: list | tuple):
 434
 435    arr = np.array(value)
 436
 437    dataset = _hdf5_handle_array(name, arr)
 438
 439    return dataset
 440
 441
 442def _hdf5_handle_exclude_obj(name: str, value: list | tuple):
 443
 444    dtype = type(value)
 445
 446    dataset = {
 447        "name": name,
 448        "attr_value": str(dtype),
 449        "dataset_value": f"excluded data type {str(dtype)}",
 450        "shape": 1,
 451        "dtype": h5py.string_dtype(encoding="utf-8"),
 452    }
 453
 454    return dataset
 455
 456
 457def _hdf5_skip_cls(value):
 458
 459    type_str = str(type(value))
 460    module_name = type_str.split("'")[1].split(".")[0]
 461
 462    if module_name in constant.EXCLUDE_PYTHON_OBJ:
 463        return True
 464    else:
 465        return False
 466
 467
 468def _hdf5_handle_array(name: str, value: np.ndarray):
 469
 470    dtype_attr = type(value)
 471    dtype = value.dtype
 472
 473    if value.dtype.char == "M":
 474
 475        ListDate = value.tolist()
 476        ListDateStr = list()
 477        for date in ListDate:
 478            ListDateStr.append(date.strftime("%Y-%m-%d %H:%M"))
 479        value = np.array(ListDateStr)
 480        value = value.astype("O")
 481        dtype = h5py.string_dtype(encoding="utf-8")
 482
 483    elif value.dtype == "object":
 484
 485        value = value.astype("S")
 486        dtype = h5py.string_dtype(encoding="utf-8")
 487
 488    elif value.dtype.char == "U":
 489        value = value.astype("S")
 490        dtype = h5py.string_dtype(encoding="utf-8")
 491
 492    dataset = {
 493        "name": name,
 494        "attr_value": str(dtype_attr),
 495        "dataset_value": value,
 496        "shape": value.shape,
 497        "dtype": dtype,
 498    }
 499
 500    return dataset
 501
 502
 503def _hdf5_handle_ndarray(hdf5: h5py.File, name: str, value: np.ndarray):
 504
 505    hdf5 = add_hdf5_sub_group(hdf5, subgroup=name)
 506    hdf5_data = hdf5[name]
 507
 508    hdf5_data.attrs["_numpy_ndarray"] = 1
 509    _dump_ndarray_to_hdf5(hdf5_data, value)
 510
 511
 512def _hdf5_create_dataset(hdf5: h5py.File, dataset: dict):
 513
 514    if dataset["name"] in hdf5.keys():
 515        del hdf5[dataset["name"]]
 516
 517    hdf5.create_dataset(
 518        dataset["name"],
 519        shape=dataset["shape"],
 520        dtype=dataset["dtype"],
 521        data=dataset["dataset_value"],
 522        compression="gzip",
 523        chunks=True,
 524    )
 525
 526    if "_" + dataset["name"] in list(hdf5.attrs.keys()):
 527        del hdf5.attrs["_" + dataset["name"]]
 528
 529    hdf5.attrs["_" + dataset["name"]] = dataset["attr_value"]
 530
 531
 532def hdf5_dataset_creator(hdf5: h5py.File, name: str, value):
 533    """
 534    Write any value in an hdf5 object
 535
 536    Parameters
 537    ----------
 538
 539    hdf5 : h5py.File
 540        an hdf5 object
 541
 542    name : str
 543        name of the dataset
 544
 545    value : any
 546        value to write in the hdf5
 547
 548    """
 549
 550    if _hdf5_skip_cls(value):
 551        dataset = _hdf5_handle_exclude_obj(name, value)
 552
 553    elif isinstance(value, str):
 554        dataset = _hdf5_handle_str(name, value)
 555
 556    elif isinstance(value, numbers.Number):
 557        dataset = _hdf5_handle_numbers(name, value)
 558
 559    elif value is None:
 560        dataset = _hdf5_handle_none(name, value)
 561
 562    elif isinstance(value, (pd.Timestamp, np.datetime64, datetime.date)):
 563        dataset = _hdf5_handle_timestamp(name, value)
 564
 565    elif isinstance(value, pd.DatetimeIndex):
 566        dataset = _hdf5_handle_DatetimeIndex(name, value)
 567
 568    # TODO : To be tested
 569    elif isinstance(value, pd.DataFrame):
 570
 571        _hdf5_handle_PandaDataFrame(hdf5, name, value)
 572        return
 573
 574    elif isinstance(value, list):
 575        dataset = _hdf5_handle_list(name, value)
 576
 577    elif isinstance(value, tuple):
 578        dataset = _hdf5_handle_list(name, value)
 579
 580    elif isinstance(value, np.ndarray):
 581
 582        if len(value.dtype) > 0 and len(value.dtype.names) > 0:
 583            _hdf5_handle_ndarray(hdf5, name, value)
 584            return
 585        else:
 586            dataset = _hdf5_handle_array(name, value)
 587
 588    else:
 589
 590        hdf5 = add_hdf5_sub_group(hdf5, subgroup=name)
 591
 592        newdict = object_handler.read_object_as_dict(value)
 593
 594        save_dict_to_hdf5(hdf5[name], newdict)
 595
 596        return
 597
 598    _hdf5_create_dataset(hdf5, dataset)
 599
 600
 601def _dump_ndarray_to_hdf5(hdf5, value):
 602    """
 603    dump a ndarray data structure to an hdf5 file: this functions create a group ndarray_ds and store each component of the ndarray as a dataset. Plus it add 2 datasets which store the dtypes (ndarray_dtype) and labels (ndarray_indexes).
 604
 605    Parameters
 606    ----------
 607
 608    hdf5 : h5py.File
 609        an hdf5 object
 610
 611    value : ndarray
 612        an ndarray data structure with different datatype
 613
 614    """
 615    # save ndarray datastructure
 616    # hdf5 = add_hdf5_sub_group(hdf5, subgroup="ndarray_ds")
 617    # hdf5_data = hdf5["ndarray_ds"]
 618
 619    for item in value.dtype.names:
 620
 621        hdf5_dataset_creator(hdf5=hdf5, name=item, value=value[item])
 622
 623    index = np.array(value.dtype.descr)[:, 0]
 624    dtype = np.array(value.dtype.descr)[:, 1]
 625    index = index.astype("O")
 626    dtype = dtype.astype("O")
 627    data_type = h5py.string_dtype(encoding="utf-8")
 628
 629    if "ndarray_dtype" in hdf5.keys():
 630        del hdf5["ndarray_dtype"]
 631
 632    hdf5.create_dataset(
 633        "ndarray_dtype",
 634        shape=dtype.shape,
 635        dtype=data_type,
 636        data=dtype,
 637        compression="gzip",
 638        chunks=True,
 639    )
 640
 641    if "ndarray_indexes" in hdf5.keys():
 642        del hdf5["ndarray_indexes"]
 643
 644    hdf5.create_dataset(
 645        "ndarray_indexes",
 646        shape=index.shape,
 647        dtype=data_type,
 648        data=index,
 649        compression="gzip",
 650        chunks=True,
 651    )
 652
 653
 654def _read_pd_dataframe(hdf5):
 655    """
 656    read a pandas dataframe data structure from hdf5 file
 657
 658    Parameters
 659    ----------
 660
 661    hdf5 : h5py.File
 662        an hdf5 object at the roots of the ndarray datastructure
 663
 664    Return
 665    ------
 666
 667    pd.DataFrame : the pandas dataframe
 668
 669    """
 670
 671    if "_Pandas_DataFrame" in list(hdf5.attrs.keys()):
 672        columns = hdf5["columns"][:]
 673        array = hdf5["array"][:]
 674        dtype = hdf5["dtype"][:]
 675
 676    newdict = {}
 677    for i, col in enumerate(columns):
 678        newdict.update({col.decode(): array[:, i].astype(dtype[i])})
 679
 680    return pd.DataFrame(newdict)
 681
 682
 683def _read_ndarray_datastructure(hdf5):
 684    """
 685    read a ndarray data structure from hdf5 file
 686
 687    Parameters
 688    ----------
 689
 690    hdf5 : h5py.File
 691        an hdf5 object at the roots of the ndarray datastructure
 692
 693    Return
 694    ------
 695
 696    ndarray : the ndarray
 697
 698    """
 699
 700    # if "ndarray_ds" in list(hdf5.keys()):
 701    decoded_item = list()
 702    for it in hdf5["ndarray_dtype"][:]:
 703        decoded_item.append(it.decode())
 704    list_dtypes = decoded_item
 705
 706    decoded_item = list()
 707    for it in hdf5["ndarray_indexes"][:]:
 708        decoded_item.append(it.decode())
 709    list_indexes = decoded_item
 710
 711    len_data = len(hdf5[f"{list_indexes[0]}"][:])
 712
 713    list_datatype = list()
 714    for i in range(len(list_indexes)):
 715        list_datatype.append((list_indexes[i], list_dtypes[i]))
 716
 717    datatype = np.dtype(list_datatype)
 718
 719    ndarray = np.zeros(len_data, dtype=datatype)
 720
 721    for i in range(len(list_indexes)):
 722
 723        expected_type = list_dtypes[i]
 724
 725        values = hdf5_read_dataset(hdf5[f"{list_indexes[i]}"], expected_type)
 726
 727        ndarray[list_indexes[i]] = values
 728
 729    return ndarray
 730
 731
 732def save_dict_to_hdf5(hdf5, dictionary):
 733    """
 734
 735    dump a dictionary to an hdf5 file
 736
 737    Parameters
 738    ----------
 739
 740    hdf5 : h5py.File
 741        an hdf5 object
 742
 743    dictionary : dict
 744        a custom python dictionary
 745
 746    """
 747    if isinstance(dictionary, dict):
 748        for attr, value in dictionary.items():
 749            # print("looping:",attr,value)
 750            try:
 751
 752                attribute_name = str(attr)
 753                for character in "/ ":
 754                    attribute_name = attribute_name.replace(character, "_")
 755
 756                if isinstance(value, dict):
 757                    # print("---> dictionary: ",attr, value)
 758
 759                    hdf5 = add_hdf5_sub_group(hdf5, subgroup=attribute_name)
 760                    save_dict_to_hdf5(hdf5[attribute_name], value)
 761
 762                else:
 763
 764                    hdf5_dataset_creator(hdf5, attribute_name, value)
 765
 766            except:
 767
 768                raise ValueError(
 769                    f"Unable to save attribute {str(attr)} with value {value}"
 770                )
 771
 772    else:
 773
 774        raise ValueError(f"{dictionary} must be a instance of dict.")
 775
 776
 777def save_dict_to_hdf5file(
 778    path_to_hdf5, dictionary=None, location="./", replace=False, wait_time=0
 779):
 780    """
 781
 782    dump a dictionary to an hdf5 file
 783
 784    Parameters
 785    ----------
 786
 787    path_to_hdf5 : str
 788        path to the hdf5 file
 789
 790    dictionary : dict | None
 791        a dictionary containing the data to be saved
 792
 793    location : str
 794        path location or subgroup where to write data in the hdf5 file
 795
 796    replace : Boolean
 797        replace an existing hdf5 file. Default is False
 798
 799    wait_time: int
 800        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened and the funciton will return None. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
 801
 802    Examples
 803    --------
 804
 805    >>> dict={"a":1,"b":2}
 806    >>> pyhdf5_handler.save_dict_to_hdf5("saved_dictionary.hdf5",dict)
 807
 808    """
 809    if isinstance(dictionary, dict):
 810        hdf5 = open_hdf5(path_to_hdf5, replace=replace, wait_time=wait_time)
 811
 812        if hdf5 is None:
 813            return
 814
 815        hdf5 = add_hdf5_sub_group(hdf5, subgroup=location)
 816        save_dict_to_hdf5(hdf5[location], dictionary)
 817
 818    else:
 819        raise ValueError(f"The input {dictionary} must be a instance of dict.")
 820
 821    hdf5.close()
 822
 823
 824def save_object_to_hdf5(
 825    hdf5,
 826    instance,
 827    keys_data=None,
 828    location="./",
 829    sub_data=None,
 830    replace=False,
 831    wait_time=0,
 832):
 833    """
 834
 835    dump an object to an hdf5 file
 836
 837    Parameters
 838    ----------
 839
 840    hdf5 : instance of h5py
 841        An opened hdf5 file
 842
 843    instance : object
 844        A custom python object to be saved into an hdf5
 845
 846    keys_data : list | dict
 847        optional, a list or a dictionary of the attribute to be saved
 848
 849    location : str
 850        path location or subgroup where to write data in the hdf5 file
 851
 852    sub_data : dict | None
 853        optional, a extra dictionary containing extra-data to be saved along the object
 854
 855    replace : Boolean
 856        replace an existing hdf5 file. Default is False
 857
 858    wait_time: int
 859        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened and the funciton will return None. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
 860
 861    """
 862
 863    if keys_data is None:
 864        keys_data = object_handler.generate_object_structure(
 865            instance, include_method=False
 866        )
 867
 868    if hdf5 is None:
 869        return None
 870
 871    hdf5 = add_hdf5_sub_group(hdf5, subgroup=location)
 872
 873    _dump_object_to_hdf5_from_iteratable(hdf5[location], instance, keys_data)
 874
 875    if isinstance(sub_data, dict):
 876        save_dict_to_hdf5(hdf5[location], sub_data)
 877
 878    hdf5.close()
 879
 880
 881def save_object_to_hdf5file(
 882    path_to_hdf5,
 883    instance,
 884    keys_data=None,
 885    location="./",
 886    sub_data=None,
 887    replace=False,
 888    wait_time=0,
 889):
 890    """
 891
 892    dump an object to an hdf5 file
 893
 894    Parameters
 895    ----------
 896
 897    path_to_hdf5 : str
 898        path to the hdf5 file
 899
 900    instance : object
 901        A custom python object to be saved into an hdf5
 902
 903    keys_data : list | dict
 904        optional, a list or a dictionary of the attribute to be saved
 905
 906    location : str
 907        path location or subgroup where to write data in the hdf5 file
 908
 909    sub_data : dict | None
 910        optional, a extra dictionary containing extra-data to be saved along the object
 911
 912    replace : Boolean
 913        replace an existing hdf5 file. Default is False
 914
 915    wait_time: int
 916        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened and the funciton will return None. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
 917
 918    """
 919
 920    hdf5 = open_hdf5(path_to_hdf5, replace=replace, wait_time=wait_time)
 921
 922    save_object_to_hdf5(
 923        hdf5,
 924        instance,
 925        keys_data=keys_data,
 926        location=location,
 927        sub_data=sub_data,
 928        replace=replace,
 929        wait_time=wait_time,
 930    )
 931
 932
 933def read_hdf5file_as_dict(
 934    path_to_hdf5,
 935    location="./",
 936    wait_time=0,
 937    read_attrs=True,
 938    read_dataset_attrs=False,
 939):
 940    """
 941
 942    Open, read and close an hdf5 file
 943
 944    Parameters
 945    ----------
 946
 947    path_to_hdf5 : str
 948        path to the hdf5 file
 949
 950    location: str
 951        place in the hdf5 from which we start reading the file
 952
 953    read_attrs : bool
 954        read and import attributes in the dicitonnary.
 955
 956    read_dataset_attrs : bool
 957        read and import special attributes linked to any dataset and created by pyhdf5_handler. These attributes only store the original dataype of the data stored in the dataset.
 958
 959    Return
 960    --------
 961
 962    dictionary : dict, a dictionary of all keys and attribute included in the hdf5 file
 963
 964    wait_time: int
 965        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened and the funciton will return None. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
 966
 967    Examples
 968    --------
 969
 970    read an hdf5 file
 971    dictionary=hdf5_handler.read_hdf5file_as_dict(hdf5["model1"])
 972    """
 973
 974    hdf5 = open_hdf5(path_to_hdf5, read_only=True, wait_time=wait_time)
 975
 976    if hdf5 is None:
 977        return None
 978
 979    dictionary = read_hdf5_as_dict(
 980        hdf5[location],
 981        read_attrs=read_attrs,
 982        read_dataset_attrs=read_dataset_attrs,
 983    )
 984
 985    hdf5.close()
 986
 987    return dictionary
 988
 989
 990def read_hdf5_as_dict(hdf5, read_attrs=True, read_dataset_attrs=False):
 991    """
 992    Load an hdf5 file
 993
 994    Parameters
 995    ----------
 996
 997    hdf5 : h5py.File
 998        an instance of hdf5, open with the function open_hdf5()
 999
1000    read_attrs : bool
1001        read and import attributes in the dicitonnary.
1002
1003    read_dataset_attrs : bool
1004        read and import special attributes linked to any dataset and created by pyhdf5_handler. These attributes only store the original datatype of the data stored in the dataset.
1005
1006    Return
1007    --------
1008
1009    dictionary : dict, a dictionary of all keys and attribute included in the hdf5 file
1010
1011    Examples
1012    --------
1013
1014    read only a part of an hdf5 file
1015    >>> hdf5=hdf5_handler.open_hdf5("./multi_model.hdf5")
1016    >>> dictionary=hdf5_handler.read_hdf5_as_dict(hdf5["model1"])
1017    >>> dictionary.keys()
1018
1019    """
1020
1021    if not isinstance(
1022        hdf5, (h5py.File, h5py.Group, h5py.Dataset, h5py.Datatype)
1023    ):
1024        print("Error: input arg is not an instance of hdf5.File()")
1025        return {}
1026
1027    dictionary = {}
1028
1029    for key, item in hdf5.items():
1030
1031        if str(type(item)).find("group") != -1:
1032
1033            if "_Pandas_DataFrame" in list(item.attrs.keys()):
1034                values = _read_pd_dataframe(item)
1035                dictionary.update({key: values})
1036
1037            elif "_numpy_ndarray" in list(item.attrs.keys()):
1038                values = _read_ndarray_datastructure(item)
1039                # values = _read_ndarray_datastructure(hdf5)
1040                dictionary.update({key: values})
1041
1042            else:
1043
1044                dictionary.update({key: read_hdf5_as_dict(item)})
1045
1046        if str(type(item)).find("dataset") != -1:
1047
1048            if "_" + key in hdf5.attrs.keys():
1049                expected_type = hdf5.attrs["_" + key]
1050                values = hdf5_read_dataset(item, expected_type)
1051
1052            else:
1053
1054                values = item[:]
1055
1056            dictionary.update({key: values})
1057
1058    list_attribute = []
1059    if read_attrs or read_dataset_attrs:
1060        tmp_list_attribute = list(hdf5.attrs.keys())
1061        hdf5_item_matching_attributes = [
1062            "_" + element for element in list(hdf5.keys())
1063        ]
1064
1065    if read_attrs:
1066
1067        list_attribute.extend(
1068            list(
1069                filter(
1070                    lambda l: l not in hdf5_item_matching_attributes,
1071                    tmp_list_attribute,
1072                )
1073            )
1074        )
1075
1076    if read_dataset_attrs:
1077
1078        list_attribute.extend(
1079            list(
1080                filter(
1081                    lambda l: l in hdf5_item_matching_attributes,
1082                    tmp_list_attribute,
1083                )
1084            )
1085        )
1086
1087    for key in list_attribute:
1088        dictionary.update({key: hdf5.attrs[key]})
1089
1090    return dictionary
1091
1092
1093def _is_numeric_str_class(class_str):
1094    """
1095    check if the input string is a representation of a python class and if it is a subclass of numbers.Numbers.
1096
1097    Args:
1098        class_str (str): string representation of a class like "<class 'module.ClassName'>" or "<class 'ClassName'>"
1099
1100    Returns:
1101        bool: True if the class is a subclass of numbers.Number, False sinon.
1102    """
1103    # Expression régulière pour extraire le nom complet de la classe
1104    match = re.match(r"<class '(?:([^']+)\.)?([^']+)'>", class_str.strip())
1105    if not match:
1106        return False
1107
1108    module_name, class_name = match.groups()
1109    # full_name = f"{module_name}.{class_name}" if module_name else class_name
1110
1111    try:
1112        # On essaie d'importer le module et de récupérer la classe
1113        if module_name:
1114            module = __import__(module_name, fromlist=[class_name])
1115            cls = getattr(module, class_name)
1116        else:
1117            # Cas des types built-in (int, float, etc.)
1118            cls = globals().get(class_name, None)
1119            if cls is None:
1120                cls = getattr(__builtins__, class_name, None)
1121
1122        if cls is None:
1123            return False
1124
1125        return issubclass(cls, numbers.Number)
1126    except (ImportError, AttributeError, TypeError):
1127        return False
1128
1129
1130# def _is_numeric_str_class(str_class):
1131#     if not str_class.startwith("<class"):
1132#         raise ValueError(f"'{str_class}' is not a string class representation.")
1133
1134#     path = str_class[8:-2]
1135
1136#     path_list_splitted = path.rsplit(".", 1)
1137
1138#     if len(path_list_splitted) == 2:
1139#         module, name = [*path_list_splitted]
1140#         result = issubclass(
1141#             getattr(importlib.import_module(module), name), numbers.Number
1142#         )
1143#     else:
1144#         name = path_list_splitted[0]
1145#         result = issubclass(
1146#             getattr(importlib.import_module("builtins"), name), numbers.Number
1147#         )
1148
1149#     return result
1150
1151
1152def hdf5_read_dataset(item, expected_type=None):
1153    """
1154    Read a dataset stored in an hdf5 database
1155
1156    Parameters
1157    ----------
1158
1159    item : h5py.File
1160        an hdf5 dataset/item
1161
1162    expected_type: str
1163        the expected dtype as string str(type())
1164
1165    Return
1166    --------
1167
1168    value : the value read from the hdf5, any type matching the expected type
1169
1170
1171    """
1172
1173    if expected_type == str(type("str")):
1174
1175        values = item[0].decode()
1176
1177    elif expected_type == str(type(1)):
1178
1179        values = int(item[0])  # buildin int type
1180
1181    elif expected_type == str(type(1.0)):
1182
1183        values = float(item[0])  # buildin float type
1184
1185    elif _is_numeric_str_class(expected_type):
1186
1187        values = item[0]  # other int/float type like np.int64/np.float64
1188
1189    elif expected_type == "_None_":
1190
1191        values = None
1192
1193    elif expected_type in (
1194        str(pd.Timestamp),
1195        str(np.datetime64),
1196        str(datetime.datetime),
1197    ):
1198
1199        if expected_type == str(pd.Timestamp):
1200            values = pd.Timestamp(item[0].decode())
1201
1202        elif expected_type == str(np.datetime64):
1203            values = np.datetime64(item[0].decode())
1204
1205        elif expected_type == str(datetime.datetime):
1206            values = datetime.datetime.fromisoformat(item[0].decode())
1207
1208        else:
1209            values = item[0].decode()
1210
1211    else:
1212
1213        if item[:].dtype.char == "S":
1214
1215            values = item[:].astype("U")
1216
1217        elif item[:].dtype.char == "O":
1218
1219            # decode list if required
1220            decoded_item = list()
1221            for it in item[:]:
1222
1223                decoded_item.append(it.decode())
1224
1225            values = decoded_item
1226
1227        else:
1228            values = item[:]
1229
1230    return values
1231
1232
1233def get_hdf5file_attribute(
1234    path_to_hdf5=str(), location="./", attribute=None, wait_time=0
1235):
1236    """
1237    Get the value of an attribute in the hdf5file
1238
1239    Parameters
1240    ----------
1241
1242    path_to_hdf5 : str
1243        the path to the hdf5file
1244
1245    location : str
1246        path inside the hdf5 where the attribute is stored
1247
1248    attribute: str
1249        attribute name
1250
1251    wait_time: int
1252        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened and the funciton will return None. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
1253
1254    Return
1255    --------
1256
1257    return_attribute : the value of the attribute
1258
1259    Examples
1260    --------
1261
1262    get an attribute
1263    >>> attribute=hdf5_handler.get_hdf5_attribute("./multi_model.hdf5",attribute=my_attribute_name)
1264
1265    """
1266
1267    hdf5_base = open_hdf5(path_to_hdf5, read_only=True, wait_time=wait_time)
1268
1269    if hdf5_base is None:
1270        return None
1271
1272    hdf5 = hdf5_base[location]
1273
1274    return_attribute = hdf5.attrs[attribute]
1275
1276    hdf5_base.close()
1277
1278    return return_attribute
1279
1280
1281def get_hdf5file_dataset(
1282    path_to_hdf5=str(), location="./", dataset=None, wait_time=0
1283):
1284    """
1285    Get the value of an attribute in the hdf5file
1286
1287    Parameters
1288    ----------
1289
1290    path_to_hdf5 : str
1291        the path to the hdf5file
1292
1293    location : str
1294        path inside the hdf5 where the attribute is stored
1295
1296    dataset: str
1297        dataset name
1298
1299    wait_time: int
1300        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened and the funciton will return None. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
1301
1302    Return
1303    --------
1304
1305    return_dataset : the value of the attribute
1306
1307    Examples
1308    --------
1309
1310    get a dataset
1311    >>> dataset=hdf5_handler.get_hdf5_dataset("./multi_model.hdf5",dataset=my_dataset_name)
1312
1313    """
1314
1315    hdf5_base = open_hdf5(path_to_hdf5, read_only=True, wait_time=wait_time)
1316
1317    if hdf5_base is None:
1318        return None
1319
1320    hdf5 = hdf5_base[location]
1321
1322    if "_" + dataset in hdf5.attrs.keys():
1323        expected_type = hdf5.attrs["_" + dataset]
1324        return_dataset = hdf5_read_dataset(hdf5[dataset], expected_type)
1325
1326    else:
1327        return_dataset = hdf5[dataset][:]
1328
1329    hdf5_base.close()
1330
1331    return return_dataset
1332
1333
1334def get_hdf5file_item(
1335    path_to_hdf5=str(),
1336    location="./",
1337    item=None,
1338    wait_time=0,
1339    search_attrs=False,
1340):
1341    """
1342
1343    Get a custom item in an hdf5file
1344
1345    Parameters
1346    ----------
1347
1348    path_to_hdf5 : str
1349        the path to the hdf5file
1350
1351    location : str
1352        path inside the hdf5 where the attribute is stored. If item is None, item is set to basename(location)
1353
1354    item: str
1355        item name
1356
1357    wait_time: int
1358        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened and the funciton will return None. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
1359
1360    search_attrs: bool
1361        Default is False. If True, the function will also search in the item in the attribute first.
1362
1363    Return
1364    --------
1365
1366    return : custom value. can be an hdf5 object (group), an numpy array, a string, a float, an int ...
1367
1368    Examples
1369    --------
1370
1371    get the dataset 'dataset'
1372    >>> dataset=hdf5_handler.get_hdf5_item("./multi_model.hdf5",location="path/in/hdf5/dataset")
1373
1374    """
1375
1376    hdf5 = open_hdf5(path_to_hdf5, read_only=True, wait_time=wait_time)
1377
1378    if hdf5 is None:
1379        return None
1380
1381    hdf5_item = get_hdf5_item(
1382        hdf5_instance=hdf5,
1383        location=location,
1384        item=item,
1385        search_attrs=search_attrs,
1386    )
1387
1388    hdf5.close()
1389
1390    return hdf5_item
1391
1392
1393def get_hdf5_item(
1394    hdf5_instance=None, location="./", item=None, search_attrs=False
1395):
1396    """
1397
1398    Get a custom item in an hdf5file
1399
1400    Parameters
1401    ----------
1402
1403    hdf5_instance : h5py.File
1404        an instance of an hdf5
1405
1406    location : str
1407        path inside the hdf5 where the attribute is stored. If item is None, item is set to basename(location)
1408
1409    item: str
1410        item name
1411
1412    search_attrs: bool
1413        Default is False. If True, the function will search in the item in the attribute first.
1414
1415    Return
1416    ------
1417
1418    return : custom value. can be an hdf5 object (group), an numpy array, a string, a float, an int ...
1419
1420    Examples
1421    --------
1422
1423    get the dataset 'dataset'
1424    >>> dataset=hdf5_handler.get_hdf5_item("./multi_model.hdf5",location="path/in/hdf5/dataset")
1425
1426    """
1427
1428    if item is None and isinstance(location, str):
1429        head, tail = os.path.split(location)
1430        if len(tail) > 0:
1431            item = tail
1432        location = head
1433
1434    if not isinstance(item, str):
1435        print(f"Bad search item:{item}")
1436        return None
1437
1438        return None
1439
1440    # print(f"Getting item '{item}' at location '{location}'")
1441    hdf5 = hdf5_instance[location]
1442
1443    # first search in the attribute
1444    if search_attrs:
1445        list_attribute = hdf5.attrs.keys()
1446        if item in list_attribute:
1447            return hdf5.attrs[item]
1448
1449    # then search in groups and dataset
1450    list_keys = hdf5.keys()
1451    if item in list_keys:
1452
1453        hdf5_item = hdf5[item]
1454
1455        # print("Got Item ", hdf5_item)
1456
1457        if str(type(hdf5_item)).find("group") != -1:
1458
1459            # if item == "ndarray_ds":
1460            if "_numpy_ndarray" in list(hdf5_item.attrs.keys()):
1461
1462                return _read_ndarray_datastructure(hdf5_item)
1463
1464            elif "_Pandas_DataFrame" in list(hdf5_item.attrs.keys()):
1465                return _read_pd_dataframe(hdf5_item)
1466
1467            else:
1468
1469                returned_dict = read_hdf5_as_dict(hdf5_item)
1470
1471                return returned_dict
1472
1473        elif str(type(hdf5_item)).find("dataset") != -1:
1474
1475            if "_" + item in hdf5.attrs.keys():
1476                expected_type = hdf5.attrs["_" + item]
1477                values = hdf5_read_dataset(hdf5_item, expected_type)
1478            else:
1479                values = hdf5_item[:]
1480
1481            return values
1482
1483        else:
1484
1485            return hdf5_item
1486
1487    else:
1488
1489        return None
1490
1491
1492def search_in_hdf5file(
1493    path_to_hdf5, key=None, location="./", wait_time=0, search_attrs=False
1494):
1495    """
1496
1497    Search key in an hdf5 and return a list of [locations, datatype, key name, values]. Value and key are returned only if the key is an attribute or a dataset (None otherwise)
1498
1499    Parameters
1500    ----------
1501
1502    path_to_hdf5 : str
1503        the path to the hdf5file
1504
1505    key: str
1506        key to search in the hdf5file
1507
1508    location : str
1509        path inside the hdf5 where to start the research
1510
1511    wait_time: int
1512        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened and the funciton will return None. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
1513
1514    search_attrs : Bool
1515        Default false, search in the attributes
1516
1517    Return
1518    ------
1519
1520    return_dataset : the value of the attribute
1521
1522    Examples
1523    --------
1524
1525    search in a hdf5file
1526    >>> matchkey=hdf5_handler.search_in_hdf5file(hdf5filename, key='Nom_du_BV',location="./")
1527
1528    """
1529    if key is None:
1530        print("Nothing to search, use key=")
1531        return []
1532
1533    hdf5 = open_hdf5(path_to_hdf5, read_only=True, wait_time=wait_time)
1534
1535    if hdf5 is None:
1536        return None
1537
1538    results = search_in_hdf5(
1539        hdf5, key, location=location, search_attrs=search_attrs
1540    )
1541
1542    hdf5.close()
1543
1544    return results
1545
1546
1547def search_in_hdf5(hdf5_base, key=None, location="./", search_attrs=False):
1548    """
1549
1550    Search key in an hdf5 and return a list of [locations, datatype, key name, values]. Value and key are returned only if the key is an attribute or a dataset (None otherwise)
1551
1552    Parameters
1553    ----------
1554
1555    hdf5_base : h5py.File
1556        opened instance of the hdf5
1557
1558    key: str
1559        key to search in the hdf5file
1560
1561    location : str
1562        path inside the hdf5 where to start the research
1563
1564    search_attrs : Bool
1565        Default false, search in the attributes
1566
1567    Return
1568    ------
1569
1570    return_dataset : the value of the attribute
1571
1572    Examples
1573    --------
1574
1575    search in a hdf5
1576    >>> hdf5=hdf5_handler.open_hdf5(hdf5_file)
1577    >>> matchkey=hdf5_handler.search_in_hdf5(hdf5, key='Nom_du_BV',location="./")
1578    >>> hdf5.close()
1579
1580    """
1581    if key is None:
1582        print("Nothing to search, use key=")
1583        return []
1584
1585    result = []
1586
1587    hdf5 = hdf5_base[location]
1588
1589    if search_attrs:
1590        list_attribute = hdf5.attrs.keys()
1591
1592        if key in list_attribute:
1593            result.append(
1594                {
1595                    "path": location,
1596                    "key": key,
1597                    "datatype": "attribute",
1598                    "value": hdf5.attrs[key],
1599                }
1600            )
1601
1602    for hdf5_key, item in hdf5.items():
1603
1604        if str(type(item)).find("group") != -1:
1605
1606            sub_location = os.path.join(location, hdf5_key)
1607
1608            # print(hdf5_key,sub_location,list(hdf5.keys()))
1609
1610            if hdf5_key == key:
1611
1612                if "ndarray_ds" in item.keys():
1613
1614                    result.append(
1615                        {
1616                            "path": sub_location,
1617                            "key": None,
1618                            "datatype": "ndarray",
1619                            "value": _read_ndarray_datastructure(item),
1620                        }
1621                    )
1622
1623                else:
1624
1625                    result.append(
1626                        {
1627                            "path": sub_location,
1628                            "key": None,
1629                            "datatype": "group",
1630                            "value": None,
1631                        }
1632                    )
1633
1634            res = search_in_hdf5(hdf5_base, key, sub_location)
1635
1636            if len(res) > 0:
1637                for element in res:
1638                    result.append(element)
1639
1640        if str(type(item)).find("dataset") != -1:
1641
1642            if hdf5_key == key:
1643
1644                if item[:].dtype.char == "S":
1645
1646                    values = item[:].astype("U")
1647
1648                elif item[:].dtype.char == "O":
1649
1650                    # decode list if required
1651                    decoded_item = list()
1652                    for it in item[:]:
1653                        decoded_item.append(it.decode())
1654
1655                    values = decoded_item
1656
1657                else:
1658
1659                    values = item[:]
1660
1661                result.append(
1662                    {
1663                        "path": location,
1664                        "key": key,
1665                        "datatype": "dataset",
1666                        "value": values,
1667                    }
1668                )
1669
1670    return result
1671
1672
1673def hdf5file_view(
1674    path_to_hdf5,
1675    location="./",
1676    max_depth=None,
1677    level_base=">",
1678    level_sep="--",
1679    depth=None,
1680    wait_time=0,
1681    list_attrs=True,
1682    list_dataset_attrs=False,
1683    return_view=False,
1684):
1685    """
1686
1687    Search key in an hdf5 and return a list of [locations, datatype, key name, values]. Value and key are returned only if the key is an attribute or a dataset (None otherwise)
1688
1689    Parameters
1690    ----------
1691
1692
1693    path_to_hdf5 : str
1694        Path to an hdf5 database
1695
1696    location : str
1697        path inside the hdf5 where to start the research
1698
1699    max_depth: str
1700        Max deph of the search in the hdf5
1701
1702    level_base: str
1703        string used as separator at the lower level (default '>')
1704
1705    level_sep: str
1706        string used as separator at higher level (default '--')
1707
1708    depth: int
1709        current depth level
1710
1711    list_attrs: bool
1712        default is True, list the attributes
1713
1714    list_dataset_attrs: bool
1715        default is False, list the special attributes defined for each dataset by pyhdf5_handler
1716
1717    return_view: bool
1718        retrun the object view in a dictionnary (do not print at screen)
1719
1720    wait_time: int
1721        If the hdf5 is unavailable, the function will try to access serveral time and will wait wait_time seconds maximum. If this time is elapsed, the file won't be opened and the funciton will return None. This parameter is usefull if several program or threads need to read/write simultaneously in the same hdf5 database.
1722
1723    Return
1724    --------
1725
1726    dictionnary : optional, the view of the hdf5
1727
1728    Examples
1729    --------
1730
1731    search in a hdf5file
1732    >>> matchkey=hdf5_handler.search_in_hdf5file(hdf5filename, key='Nom_du_BV',location="./")
1733
1734    """
1735
1736    hdf5 = open_hdf5(path_to_hdf5, read_only=True, wait_time=wait_time)
1737
1738    if hdf5 is None:
1739        return None
1740
1741    results = hdf5_view(
1742        hdf5,
1743        location=location,
1744        max_depth=max_depth,
1745        level_base=level_base,
1746        level_sep=level_sep,
1747        depth=depth,
1748        list_attrs=list_attrs,
1749        list_dataset_attrs=list_dataset_attrs,
1750        return_view=return_view,
1751    )
1752
1753    hdf5.close()
1754
1755    return results
1756
1757
1758def hdf5file_ls(path_to_hdf5, location="./"):
1759    """
1760    List dataset in an hdf5file.
1761
1762    Parameters
1763    ----------
1764
1765    path_to_hdf5 : str
1766        path to a hdf5file
1767
1768    location: str
1769        path inside the hdf5 where to start the research
1770
1771    Example
1772    -------
1773
1774    >>> hdf5file_ls(test.hdf5)
1775
1776    """
1777
1778    hdf5 = open_hdf5(path_to_hdf5, read_only=True)
1779
1780    hdf5_view(
1781        hdf5,
1782        location=location,
1783        max_depth=0,
1784        level_base=">",
1785        level_sep="--",
1786        list_attrs=False,
1787        return_view=False,
1788    )
1789
1790
1791def hdf5_ls(hdf5):
1792    """
1793    List dataset in an hdf5 instance.
1794
1795    Parameters
1796    ----------
1797
1798    hdf5 : h5py.File
1799        hdf5 instance
1800
1801    location: str
1802        path inside the hdf5 where to start the research
1803
1804    Example
1805    -------
1806
1807    >>> hdf5 = open_hdf5(path_to_hdf5, read_only=True)
1808    >>> hdf5_ls(hdf5)
1809
1810    """
1811
1812    hdf5_view(
1813        hdf5,
1814        location="./",
1815        max_depth=0,
1816        level_base=">",
1817        level_sep="--",
1818        list_attrs=False,
1819        return_view=False,
1820    )
1821
1822
1823def hdf5_view(
1824    hdf5_obj,
1825    location="./",
1826    max_depth=None,
1827    level_base=">",
1828    level_sep="--",
1829    depth=None,
1830    list_attrs=True,
1831    list_dataset_attrs=False,
1832    return_view=False,
1833):
1834    """
1835    List recursively all dataset (and attributes) in an hdf5 object.
1836
1837    Parameters
1838    ----------
1839
1840    hdf5_obj : h5py.File
1841        opened instance of the hdf5
1842
1843    location : str
1844        path inside the hdf5 where to start the research
1845
1846    max_depth: str
1847        Max deph of the search in the hdf5
1848
1849    level_base: str
1850        string used as separator at the lower level (default '>')
1851
1852    level_sep: str
1853        string used as separator at higher level (default '--')
1854
1855    depth: int
1856        current level depth
1857
1858    list_attrs: bool
1859        default is True, list the attributes
1860
1861    list_dataset_attrs: bool
1862        default is False, list the special attributes defined for each dataset by pyhdf5_handler
1863
1864    return_view: bool
1865        retrun the object view in a dictionnary
1866
1867    Return
1868    --------
1869
1870    dictionnary : optional, the view of the hdf5
1871
1872    Examples
1873    --------
1874
1875    search in a hdf5
1876    >>> hdf5=hdf5_handler.open_hdf5(hdf5_file)
1877    >>> matchkey=hdf5_handler.search_in_hdf5(hdf5, key='Nom_du_BV',location="./")
1878    >>> hdf5.close()
1879
1880    """
1881
1882    result = []
1883
1884    if max_depth is not None:
1885
1886        if depth is not None:
1887            depth = depth + 1
1888        else:
1889            depth = 0
1890
1891        if depth > max_depth:
1892            return result
1893
1894    hdf5 = hdf5_obj[location]
1895
1896    list_attribute = []
1897    if list_attrs or list_dataset_attrs:
1898        tmp_list_attribute = list(hdf5.attrs.keys())
1899        list_keys_matching_attributes = [
1900            "_" + element for element in list(hdf5.keys())
1901        ]
1902
1903    if list_attrs:
1904
1905        list_attribute.extend(
1906            list(
1907                filter(
1908                    lambda l: l not in list_keys_matching_attributes,
1909                    tmp_list_attribute,
1910                )
1911            )
1912        )
1913
1914    if list_dataset_attrs:
1915
1916        list_attribute.extend(
1917            list(
1918                filter(
1919                    lambda l: l in list_keys_matching_attributes,
1920                    tmp_list_attribute,
1921                )
1922            )
1923        )
1924
1925    for key in list_attribute:
1926        values = hdf5.attrs[key]
1927        sub_location = os.path.join(location, key)
1928        if isinstance(
1929            values,
1930            (int, float, np.int64, np.float64, np.int32, np.float32, np.bool),
1931        ):
1932            result.append(
1933                f"{level_base}| {sub_location}, attribute, type={type(hdf5.attrs[key])}, value={values}"
1934            )
1935        elif isinstance(values, (str)) and len(values) < 20:
1936            result.append(
1937                f"{level_base}| {sub_location}, attribute, type={type(hdf5.attrs[key])}, len={len(values)}, value={values}"
1938            )
1939        else:
1940            result.append(
1941                f"{level_base}| {sub_location}, attribute, type={type(hdf5.attrs[key])}, len={len(values)}, value={values[0:20]}..."
1942            )
1943
1944    for hdf5_key, item in hdf5.items():
1945
1946        if str(type(item)).find("group") != -1:
1947
1948            sub_location = os.path.join(location, hdf5_key)
1949
1950            if "ndarray_ds" in item.keys():
1951                result.append(f"{level_base}| {sub_location}, ndarray")
1952            else:
1953                result.append(f"{level_base}| {sub_location}, group")
1954
1955            res = hdf5_view(
1956                hdf5_obj,
1957                sub_location,
1958                max_depth=max_depth,
1959                level_base=level_base + level_sep,
1960                depth=depth,
1961                return_view=True,
1962            )
1963
1964            # if len(res)>0:
1965            for key, item in enumerate(res):
1966                result.append(item)
1967
1968        if str(type(item)).find("dataset") != -1:
1969
1970            if item[:].dtype.char == "S":
1971                values = item[:].astype("U")
1972            else:
1973                values = item[:]
1974
1975            sub_location = os.path.join(location, hdf5_key)
1976
1977            result.append(
1978                f"{level_base}| {sub_location}, dataset, type={type(values)}, shape={values.shape}"
1979            )
1980
1981    if return_view:
1982        return result
1983    else:
1984        for res in result:
1985            print(res)