import os import h5py import numpy as np def recursively_load_dict_contents_from_group(h5file, path): """...""" import h5py ans = {} for key, item in h5file[path].items(): if isinstance(item, h5py._hl.dataset.Dataset): ans[key] = item.value elif isinstance(item, h5py._hl.group.Group): ans[key] = recursively_load_dict_contents_from_group(h5file, path + key + '/') return ans def read_hdf5(filename): ''' this program is to read from a HDF5 file and write the data to an python dictionary ''' import h5py dic={} with h5py.File(filename, 'r') as h5file: for key, item in h5file['/'].items(): if isinstance(item, h5py._hl.dataset.Dataset): dic[key] = item.value elif isinstance(item, h5py._hl.group.Group): dic[key] = recursively_load_dict_contents_from_group(h5file, '/' + key + '/') return dic def rname(dic,labels_in=None,labels_out=None): ''' this programming is to rename the variable from labels_in into labels_out ''' import numpy as np dic_new={} for i in np.arange(np.size(labels_in)): dic_new[labels_out[i]]=dic[labels_in[i]] return dic_new def redic(dic): ''' this programing is to convert the nest dictionary into a regular dictionary. like dict={'A':{'B':2}} => dict['A_B']=2 ''' import numpy as np dic_new={} for key in dic.keys(): if isinstance(dic[key],dict): for subkey in dic[key].keys(): if isinstance(dic[key][subkey],dict): for subkey2 in dic[key][subkey].keys(): if isinstance(dic[key][subkey][subkey2],np.ndarray): key_new=key+'_'+subkey+'_'+subkey2 dic_new[key_new]=dic[key][subkey][subkey2] elif isinstance(dic[key][subkey],np.ndarray): key_new=key+'_'+subkey dic_new[key_new]=dic[key][subkey] elif isinstance(dic[key],np.ndarray): key_new=key dic_new[key_new]=dic[key] return dic_new def write_hdf5(dic,filename,path): """...""" if os.path.exists(filename): os.remove(filename) with h5py.File(filename, 'w') as h5file: recursively_save_dict_contents_to_group(h5file, path, dic) def recursively_save_dict_contents_to_group(h5file, path, dic): """...""" # argument type checking if not isinstance(dic, dict): raise ValueError("must provide a dictionary") if not isinstance(path, str): raise ValueError("path must be a string") if not isinstance(h5file, h5py._hl.files.File): raise ValueError("must be an open h5py file") # save items to the hdf5 file for key, item in dic.items(): if not isinstance(key, str): key=str(key) # raise ValueError("dict keys must be strings to save to hdf5") # save strings, numpy.int64, and numpy.float64 types if isinstance(item, (np.int64,np.int32,np.float32, np.float64, str)): h5file[path + key] = item if not h5file[path + key].value == item: raise ValueError('The data representation in the HDF5 file does not match the original dict.') # save numpy arrays elif isinstance(item, np.ndarray): dset=h5file.create_dataset(key,data=item,compression='gzip',compression_opts=9) # h5file[path + key] = item # if not np.array_equal(h5file[path + key].value, item): # raise ValueError('The data representation in the HDF5 file does not match the original dict.') # save dictionaries elif isinstance(item, dict): recursively_save_dict_contents_to_group(h5file, path + key + '/', item) # other types cannot be saved and will result in an error else: raise ValueError('Cannot save %s type.' % type(item))