5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implemented hdf5_file_reader.py and updated register.yaml and hdf5_writer.py. This replaces previous function __copy_file_in_group().

Browse Source
This commit is contained in:
florez_j
2025-02-25 12:25:15 +01:00
parent 5f9f09d288
commit 32abd4cd56
3 changed files with 87 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/hdf5_writer.py
View File

@ -22,27 +22,6 @@ import instruments.filereader_registry as filereader_registry
def __copy_file_in_group(path_to_output_file, source_file_path, dest_group_name, work_with_copy : bool = True):
# Create copy of original file to avoid possible file corruption and work with it.
with h5py.File(path_to_output_file, mode='r+', track_order=True) as dest_file_obj:
if work_with_copy:
tmp_file_path = utils.make_file_copy(source_file_path)
else:
tmp_file_path = source_file_path
# Open backup h5 file and copy complet filesystem directory onto a group in h5file
with h5py.File(tmp_file_path,'r') as src_file:
dest_file_obj.copy(source= src_file['/'], dest= dest_group_name)
if 'tmp_files' in tmp_file_path:
os.remove(tmp_file_path)
stdout = f'Completed transfer for /{dest_group_name}'
return stdout
def create_hdf5_file_from_filesystem_path(path_to_input_directory: str,
path_to_filenames_dict: dict = None,
select_dir_keywords : list = [],
@ -178,20 +157,18 @@ def create_hdf5_file_from_filesystem_path(path_to_input_directory: str,
# hdf5 path to filename group
dest_group_name = f'{group_name}/{filename}'
source_file_path = os.path.join(dirpath,filename)
dest_file_obj = h5file
if not 'h5' in filename:
#file_dict = config_file.select_file_readers(group_id)[file_ext](os.path.join(dirpath,filename))
#file_dict = ext_to_reader_dict[file_ext](os.path.join(dirpath,filename))
file_dict = filereader_registry.select_file_reader(dest_group_name)(source_file_path)
stdout = hdf5_ops.save_file_dict_to_hdf5(h5file, group_name, file_dict)
stdout = hdf5_ops.save_file_dict_to_hdf5(dest_file_obj, group_name, file_dict)
else:
dest_file_obj = h5file
#group_name +'/'+filename
#ext_to_reader_dict[file_ext](source_file_path, dest_file_obj, dest_group_name)
#g5505f_reader.select_file_reader(dest_group_name)(source_file_path, dest_file_obj, dest_group_name)
stdout = __copy_file_in_group(source_file_path, dest_file_obj, dest_group_name, False)
from instruments.readers.hdf5_file_reader import hdf5_file_reader
stdout = hdf5_file_reader(dest_file_obj, source_file_path, dest_group_name)
#stdout = __copy_file_in_group(source_file_path, dest_file_obj, dest_group_name, False)
# Update the progress bar and log the end message
instFoldermsdEnd = f'\nCompleted data transfer for instFolder: {group_name}\n'
@ -353,7 +330,7 @@ def create_hdf5_file_from_filesystem_path_new(path_to_input_directory: str,
filereader_registry.run_reader(path_to_output_file, source_file_path, dest_group_name)
else:
from instruments.readers.hdf5_file_reader import hdf5_file_reader
#try:
# # Create group if it does not exist
# if dest_group_name not in dest_file_obj:
@ -370,7 +347,7 @@ def create_hdf5_file_from_filesystem_path_new(path_to_input_directory: str,
#group_name +'/'+filename
#ext_to_reader_dict[file_ext](source_file_path, dest_file_obj, dest_group_name)
#g5505f_reader.select_file_reader(dest_group_name)(source_file_path, dest_file_obj, dest_group_name)
stdout = __copy_file_in_group(path_to_output_file, source_file_path, dest_group_name, False)
stdout = hdf5_file_reader(path_to_output_file, source_file_path, dest_group_name, False)
# Update the progress bar and log the end message
instFoldermsdEnd = f'\nCompleted data transfer for instFolder: {group_name}\n'