5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enhanced data transfer progress visualization and logging

Browse Source
This commit is contained in:
florez_j
2024-05-28 08:59:29 +02:00
parent 08d58557df
commit 41c7660be3
1 changed files with 19 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/hdf5_lib.py
View File

@ -361,11 +361,19 @@ def create_hdf5_file_from_filesystem_path(output_filename : str,
allowed_file_extensions) allowed_file_extensions)
# Set input_directory as copied input directory # Set input_directory as copied input directory
root_dir = output_dir_path root_dir = output_dir_path
with h5py.File(output_filename, 'w') as h5file: with h5py.File(output_filename, 'w') as h5file:
number_of_dirs = len(path_to_filenames_dict.keys())
dir_number = 1
for dirpath, filtered_filenames_list in path_to_filenames_dict.items(): for dirpath, filtered_filenames_list in path_to_filenames_dict.items():
start_message = f'Starting to transfer files in directory: {dirpath}'
end_message = f'\nCompleted transferring files in directory: {dirpath}'
# Print and log the start message
print(start_message)
logging.info(start_message)
# Check if dirpath is valid. TODO: This is perhaps redundant by design of path_to_filenames_dict. # Check if dirpath is valid. TODO: This is perhaps redundant by design of path_to_filenames_dict.
if not is_valid_directory_path(dirpath,select_dir_keywords): if not is_valid_directory_path(dirpath,select_dir_keywords):
continue continue
@ -392,7 +400,7 @@ def create_hdf5_file_from_filesystem_path(output_filename : str,
for filenumber, filename in enumerate(filtered_filenames_list): for filenumber, filename in enumerate(filtered_filenames_list):
file_name, file_ext = os.path.splitext(filename) file_ext = os.path.splitext(filename)[1]
#try: #try:
if not 'h5' in filename: if not 'h5' in filename:
#file_dict = config_file.select_file_readers(group_id)[file_ext](os.path.join(dirpath,filename)) #file_dict = config_file.select_file_readers(group_id)[file_ext](os.path.join(dirpath,filename))
@ -417,18 +425,20 @@ def create_hdf5_file_from_filesystem_path(output_filename : str,
# Add dataset's attributes # Add dataset's attributes
attributes = dataset.get('attributes', {}) attributes = dataset.get('attributes', {})
h5file[group_name][file_dict['name']][dataset['name']].attrs.update(attributes) h5file[group_name][file_dict['name']][dataset['name']].attrs.update(attributes)
except Exception as inst: except Exception as inst:
# TODO: log when a file could not be stored as a dataset
print(inst) print(inst)
logging.error('Fail to transfer %s into HDF5: %s',os.path.join(dirpath,filename),inst)
else: else:
ext_to_reader_dict[file_ext](source_file_path = os.path.join(dirpath,filename), ext_to_reader_dict[file_ext](source_file_path = os.path.join(dirpath,filename),
dest_file_obj = h5file, dest_file_obj = h5file,
dest_group_name = group_name +'/'+filename) dest_group_name = group_name +'/'+filename)
#print(filename,file_ext, ':)')
# Update the progress bar and log the end message
progressBar(dir_number, number_of_dirs, end_message)
progressBar(filenumber,len(filtered_filenames_list), 'Uploading files in ' + dirpath) logging.info(end_message)
dir_number = dir_number + 1
if len(root_metadata_dict.keys())>0: if len(root_metadata_dict.keys())>0: