diff --git a/src/hdf5_lib.py b/src/hdf5_lib.py index fd6fce8..bf1dc01 100644 --- a/src/hdf5_lib.py +++ b/src/hdf5_lib.py @@ -361,11 +361,19 @@ def create_hdf5_file_from_filesystem_path(output_filename : str, allowed_file_extensions) # Set input_directory as copied input directory root_dir = output_dir_path - + with h5py.File(output_filename, 'w') as h5file: + number_of_dirs = len(path_to_filenames_dict.keys()) + dir_number = 1 for dirpath, filtered_filenames_list in path_to_filenames_dict.items(): - + + start_message = f'Starting to transfer files in directory: {dirpath}' + end_message = f'\nCompleted transferring files in directory: {dirpath}' + # Print and log the start message + print(start_message) + logging.info(start_message) + # Check if dirpath is valid. TODO: This is perhaps redundant by design of path_to_filenames_dict. if not is_valid_directory_path(dirpath,select_dir_keywords): continue @@ -392,7 +400,7 @@ def create_hdf5_file_from_filesystem_path(output_filename : str, for filenumber, filename in enumerate(filtered_filenames_list): - file_name, file_ext = os.path.splitext(filename) + file_ext = os.path.splitext(filename)[1] #try: if not 'h5' in filename: #file_dict = config_file.select_file_readers(group_id)[file_ext](os.path.join(dirpath,filename)) @@ -417,18 +425,20 @@ def create_hdf5_file_from_filesystem_path(output_filename : str, # Add dataset's attributes attributes = dataset.get('attributes', {}) h5file[group_name][file_dict['name']][dataset['name']].attrs.update(attributes) - except Exception as inst: - # TODO: log when a file could not be stored as a dataset + except Exception as inst: print(inst) + logging.error('Fail to transfer %s into HDF5: %s',os.path.join(dirpath,filename),inst) else: ext_to_reader_dict[file_ext](source_file_path = os.path.join(dirpath,filename), dest_file_obj = h5file, dest_group_name = group_name +'/'+filename) - #print(filename,file_ext, ':)') - - - progressBar(filenumber,len(filtered_filenames_list), 'Uploading files in ' + dirpath) + + # Update the progress bar and log the end message + progressBar(dir_number, number_of_dirs, end_message) + logging.info(end_message) + dir_number = dir_number + 1 + if len(root_metadata_dict.keys())>0: