1if __name__ == "__main__":
2 import numpy as np
3 import pyhdf5_handler
4 import datetime
5 import pandas as pd
6 import geopandas
7 import shapely
8
9 # open an hdf5 database, test.hdf5.
10 hdf5 = pyhdf5_handler.open_hdf5("./test.hdf5")
11
12 # Create a group in the hdf5
13 hdf5 = pyhdf5_handler.add_hdf5_sub_group(hdf5, subgroup="my_group")
14 hdf5["my_group"]
15
16 # save any data in the hdf5 database
17 pyhdf5_handler.hdf5_dataset_creator(hdf5, "str", "str")
18 pyhdf5_handler.hdf5_dataset_creator(hdf5, "numbers", 1.0)
19 pyhdf5_handler.hdf5_dataset_creator(hdf5, "numpy_numbers", np.float64(1.0))
20 pyhdf5_handler.hdf5_dataset_creator(hdf5, "none", None)
21 pyhdf5_handler.hdf5_dataset_creator(
22 hdf5, "timestamp_numpy", np.datetime64("2019-09-22T17:38:30")
23 )
24 pyhdf5_handler.hdf5_dataset_creator(
25 hdf5,
26 "timestamp_datetime",
27 datetime.datetime.fromisoformat("2019-09-22T17:38:30"),
28 )
29 pyhdf5_handler.hdf5_dataset_creator(
30 hdf5, "timestamp_pandas", pd.Timestamp("2019-09-22T17:38:30")
31 )
32 pyhdf5_handler.hdf5_dataset_creator(hdf5, "list_num", [1.0, 2.0])
33 pyhdf5_handler.hdf5_dataset_creator(hdf5, "list_str", ["a", "b"])
34 pyhdf5_handler.hdf5_dataset_creator(hdf5, "list_mixte", [1.0, "a"])
35 pyhdf5_handler.hdf5_dataset_creator(
36 hdf5,
37 "list_date_numpy",
38 [
39 np.datetime64("2019-09-22 17:38:30"),
40 np.datetime64("2019-09-22 18:38:30"),
41 ],
42 )
43 pyhdf5_handler.hdf5_dataset_creator(
44 hdf5,
45 "list_date_datetime",
46 [
47 datetime.datetime.fromisoformat("2019-09-22 17:38:30"),
48 datetime.datetime.fromisoformat("2019-09-22T18:38:30"),
49 ],
50 )
51 pyhdf5_handler.hdf5_dataset_creator(
52 hdf5,
53 "list_date_pandas",
54 [
55 pd.Timestamp("2019-09-22 17:38:30"),
56 pd.Timestamp("2019-09-22 17:38:30"),
57 ],
58 )
59 pyhdf5_handler.hdf5_dataset_creator(
60 hdf5,
61 "list_date_range_pandas",
62 pd.date_range(start="1/1/2018", end="1/08/2018"),
63 )
64
65 pyhdf5_handler.hdf5_dataset_creator(
66 hdf5,
67 "panda_dataframe_onecolumn",
68 pd.DataFrame({"column1": np.array([1, 2, 3])}),
69 )
70 pyhdf5_handler.hdf5_dataset_creator(
71 hdf5,
72 "panda_dataframe",
73 pd.DataFrame(
74 {"column1": np.array([1, 2, 3]), "column2": np.array([4, 5, 6])}
75 ),
76 )
77 pyhdf5_handler.hdf5_dataset_creator(
78 hdf5,
79 "mix_dtype_panda_dataframe",
80 pd.DataFrame(
81 {
82 "column1": np.array(["A", "B", "C"]),
83 "column2": np.array([4, 5, 6]),
84 }
85 ),
86 )
87 pyhdf5_handler.hdf5_dataset_creator(
88 hdf5,
89 "empty_dataframe",
90 pd.DataFrame({}),
91 )
92
93 # write a python dictionary in the hdf5 database
94 dictionary = {
95 "dict": {
96 "int": 1,
97 "float": 2.0,
98 "none": None,
99 "timestamp": pd.Timestamp("2019-09-22 17:38:30"),
100 "list": [1, 2, 3, 4],
101 "array": np.array([1, 2, 3, 4]),
102 "date_range": pd.date_range(start="1/1/2018", end="1/08/2018"),
103 "list_mixte": [1.0, np.datetime64("2019-09-22 17:38:30")],
104 "pandas_df": pd.DataFrame(
105 {
106 "column1": np.array([1, 2, 3]),
107 "column2": np.array([4, 5, 6]),
108 }
109 ),
110 }
111 }
112
113 hdf5.attrs["attribute"] = "myattribute"
114
115 pyhdf5_handler.save_dict_to_hdf5(hdf5, dictionary)
116
117 # handle structured ndarray
118 data = [("Alice", 25, 55.0), ("Bob", 32, 60.5)]
119 dtypes = [("name", "U10"), ("age", "i4"), ("weight", "f4")]
120 people = np.array(data, dtype=dtypes)
121
122 pyhdf5_handler.hdf5_dataset_creator(hdf5, "structured_array", people)
123
124 # viewing data stored in the hdf5 (recursive)
125 pyhdf5_handler.hdf5_view(hdf5)
126 pyhdf5_handler.hdf5file_view("./test.hdf5")
127
128 # viwing element stored in the hdf5 (at the current level)
129 pyhdf5_handler.hdf5_ls(hdf5)
130
131 # read an hdf5 and import it as a python dictionary
132 data = pyhdf5_handler.read_hdf5_as_dict(hdf5, read_attrs=True)
133
134 # read a specific item
135 pyhdf5_handler.hdf5_read_dataset(
136 item=hdf5["str"], expected_type=hdf5.attrs["_str"]
137 )
138 pyhdf5_handler.hdf5_read_dataset(
139 item=hdf5["numpy_numbers"], expected_type=hdf5.attrs["_numpy_numbers"]
140 )
141 pyhdf5_handler.hdf5_read_dataset(
142 item=hdf5["numbers"], expected_type=hdf5.attrs["_numbers"]
143 )
144 pyhdf5_handler.hdf5_read_dataset(
145 item=hdf5["list_date_numpy"],
146 expected_type=hdf5.attrs["_list_date_numpy"],
147 )
148
149 # getting specific item
150 pyhdf5_handler.get_hdf5_item(
151 hdf5_instance=hdf5,
152 location="./panda_dataframe",
153 )
154 pyhdf5_handler.get_hdf5_item(
155 hdf5_instance=hdf5,
156 location="./mix_dtype_panda_dataframe",
157 )
158 pyhdf5_handler.get_hdf5_item(
159 hdf5_instance=hdf5,
160 location="./structured_array",
161 )
162
163 # close the hdf5
164 hdf5.close()
165
166 # handle file directly
167 pyhdf5_handler.hdf5file_ls("./test.hdf5")
168 pyhdf5_handler.hdf5file_ls("./test.hdf5", location="structured_array")
169
170 pyhdf5_handler.save_dict_to_hdf5file(
171 "./panda.hdf5",
172 {
173 "mypandas": pd.DataFrame(
174 {
175 "column1": np.array(["A", "B", "C"]),
176 "column2": np.array([4, 5, 6]),
177 }
178 )
179 },
180 )
181
182 pd_data = pyhdf5_handler.read_hdf5file_as_dict(
183 "./panda.hdf5", read_attrs=False
184 )
185
186 pyhdf5_handler.save_dict_to_hdf5file(
187 "./ndarray.hdf5",
188 {"myndarray": people},
189 )
190
191 ndarray = pyhdf5_handler.read_hdf5file_as_dict(
192 "./ndarray.hdf5", read_attrs=False
193 )
194
195 pyhdf5_handler.get_hdf5file_item(
196 path_to_hdf5="./test.hdf5",
197 location="./mix_dtype_panda_dataframe",
198 )
199
200 pyhdf5_handler.save_dict_to_hdf5file("./test.hdf5", data)
201
202 # complex class with exluded datatype
203 pyhdf5_handler.EXCLUDE_PYTHON_OBJ.append("numpy") # exclude numpy also
204 mydict = {
205 "g": geopandas.GeoDataFrame(),
206 "s": shapely.Polygon(),
207 "n_exclude": np.zeros(0),
208 }
209 pyhdf5_handler.save_dict_to_hdf5file("./test.hdf5", mydict)
210 pyhdf5_handler.get_hdf5file_item(
211 path_to_hdf5="./test.hdf5",
212 location="./",
213 item="g",
214 search_attrs=False,
215 )
216 pyhdf5_handler.get_hdf5file_item(
217 path_to_hdf5="./test.hdf5",
218 location="./",
219 item="s",
220 search_attrs=False,
221 )
222 pyhdf5_handler.get_hdf5file_item(
223 path_to_hdf5="./test.hdf5",
224 location="./",
225 item="n_exclude",
226 search_attrs=False,
227 )
228 pyhdf5_handler.EXCLUDE_PYTHON_OBJ.remove("numpy")
229
230 res = pyhdf5_handler.search_in_hdf5file(
231 "./test.hdf5", key="date_range", location="./", wait_time=0
232 )
233
234 res = pyhdf5_handler.search_in_hdf5file(
235 "./test.hdf5", key="structured_array", location="./", wait_time=0
236 )
237
238 pyhdf5_handler.get_hdf5file_item(
239 path_to_hdf5="./test.hdf5",
240 location="./",
241 item="structured_array",
242 search_attrs=False,
243 )
244
245 pyhdf5_handler.get_hdf5file_item(
246 path_to_hdf5="./test.hdf5",
247 location="./",
248 item="list_mixte",
249 search_attrs=False,
250 )
251
252 pyhdf5_handler.get_hdf5file_item(
253 path_to_hdf5="./test.hdf5",
254 location="./",
255 item="attribute",
256 search_attrs=True,
257 )
258
259 pyhdf5_handler.get_hdf5file_attribute(
260 path_to_hdf5="./test.hdf5",
261 location="./",
262 attribute="_list_num",
263 wait_time=0,
264 )
265
266 pyhdf5_handler.get_hdf5file_attribute(
267 path_to_hdf5="./test.hdf5",
268 location="./structured_array/",
269 attribute="_name",
270 wait_time=0,
271 )
272
273 pyhdf5_handler.get_hdf5file_dataset(
274 path_to_hdf5="./test.hdf5", location="./dict", dataset="list_mixte"
275 )