Source code for src.hdf5_writer

import sys
import os
root_dir = os.path.abspath(os.curdir)
sys.path.append(root_dir)

import pandas as pd
import numpy as np
import h5py
import logging

import utils.g5505_utils as utils
import instruments.readers.filereader_registry as filereader_registry

 
   
def __transfer_file_dict_to_hdf5(h5file, group_name, file_dict):
    """
    Transfers data from a file_dict to an HDF5 file.

    Parameters
    ----------
    h5file : h5py.File
        HDF5 file object where the data will be written.
    group_name : str
        Name of the HDF5 group where data will be stored.
    file_dict : dict
        Dictionary containing file data to be transferred. Required structure:
        {
            'name': str,
            'attributes_dict': dict,
            'datasets': [
                {
                    'name': str,
                    'data': array-like,
                    'shape': tuple,
                    'attributes': dict (optional)
                },
                ...
            ]
        }

    Returns
    -------
    None
    """

    if not file_dict:
        return

    try:
        # Create group and add their attributes
        filename = file_dict['name']
        group = h5file[group_name].create_group(name=filename)
        # Add group attributes                                
        group.attrs.update(file_dict['attributes_dict'])
        
        # Add datasets to the just created group
        for dataset in file_dict['datasets']:
            dataset_obj = group.create_dataset(
                name=dataset['name'], 
                data=dataset['data'],
                shape=dataset['shape']
            )
            
            # Add dataset's attributes                                
            attributes = dataset.get('attributes', {})
            dataset_obj.attrs.update(attributes)
        group.attrs['last_update_date'] = utils.created_at().encode('utf-8')

        stdout = f'Completed transfer for /{group_name}/{filename}'

    except Exception as inst: 
        stdout = inst
        logging.error('Failed to transfer data into HDF5: %s', inst)

    return stdout

def __copy_file_in_group(source_file_path, dest_file_obj : h5py.File, dest_group_name, work_with_copy : bool = True):
    # Create copy of original file to avoid possible file corruption and work with it.

    if work_with_copy:
        tmp_file_path = utils.make_file_copy(source_file_path)
    else:
        tmp_file_path = source_file_path

    # Open backup h5 file and copy complet filesystem directory onto a group in h5file
    with h5py.File(tmp_file_path,'r') as src_file:
        dest_file_obj.copy(source= src_file['/'], dest= dest_group_name)

    if 'tmp_files' in tmp_file_path:
        os.remove(tmp_file_path)

    stdout = f'Completed transfer for /{dest_group_name}'
    return stdout

[docs] def create_hdf5_file_from_filesystem_path(path_to_input_directory: str, path_to_filenames_dict: dict = None, select_dir_keywords : list = [], root_metadata_dict : dict = {}, mode = 'w'): """ Creates an .h5 file with name "output_filename" that preserves the directory tree (or folder structure) of a given filesystem path. The data integration capabilities are limited by our file reader, which can only access data from a list of admissible file formats. These, however, can be extended. Directories are groups in the resulting HDF5 file. Files are formatted as composite objects consisting of a group, file, and attributes. Parameters ---------- output_filename : str Name of the output HDF5 file. path_to_input_directory : str Path to root directory, specified with forward slashes, e.g., path/to/root. path_to_filenames_dict : dict, optional A pre-processed dictionary where keys are directory paths on the input directory's tree and values are lists of files. If provided, 'input_file_system_path' is ignored. select_dir_keywords : list List of string elements to consider or select only directory paths that contain a word in 'select_dir_keywords'. When empty, all directory paths are considered to be included in the HDF5 file group hierarchy. root_metadata_dict : dict Metadata to include at the root level of the HDF5 file. mode : str 'w' create File, truncate if it exists, or 'r+' read/write, File must exists. By default, mode = "w". Returns ------- output_filename : str Path to the created HDF5 file. """ if not mode in ['w','r+']: raise ValueError(f'Parameter mode must take values in ["w","r+"]') if not '/' in path_to_input_directory: raise ValueError('path_to_input_directory needs to be specified using forward slashes "/".' ) #path_to_output_directory = os.path.join(path_to_input_directory,'..') path_to_input_directory = os.path.normpath(path_to_input_directory).rstrip(os.sep) for i, keyword in enumerate(select_dir_keywords): select_dir_keywords[i] = keyword.replace('/',os.sep) if not path_to_filenames_dict: # On dry_run=True, returns path to files dictionary of the output directory without making a actual copy of the input directory. # Therefore, there wont be a copying conflict by setting up input and output directories the same path_to_filenames_dict = utils.copy_directory_with_contraints(input_dir_path=path_to_input_directory, output_dir_path=path_to_input_directory, dry_run=True) # Set input_directory as copied input directory root_dir = path_to_input_directory path_to_output_file = path_to_input_directory.rstrip(os.path.sep) + '.h5' start_message = f'\n[Start] Data integration :\nSource: {path_to_input_directory}\nDestination: {path_to_output_file}\n' print(start_message) logging.info(start_message) # Check if the .h5 file already exists if os.path.exists(path_to_output_file) and mode in ['w']: message = ( f"[Notice] The file '{path_to_output_file}' already exists and will not be overwritten.\n" "If you wish to replace it, please delete the existing file first and rerun the program." ) print(message) logging.error(message) else: with h5py.File(path_to_output_file, mode=mode, track_order=True) as h5file: number_of_dirs = len(path_to_filenames_dict.keys()) dir_number = 1 for dirpath, filtered_filenames_list in path_to_filenames_dict.items(): # Check if filtered_filenames_list is nonempty. TODO: This is perhaps redundant by design of path_to_filenames_dict. if not filtered_filenames_list: continue group_name = dirpath.replace(os.sep,'/') group_name = group_name.replace(root_dir.replace(os.sep,'/') + '/', '/') # Flatten group name to one level if select_dir_keywords: offset = sum([len(i.split(os.sep)) if i in dirpath else 0 for i in select_dir_keywords]) else: offset = 1 tmp_list = group_name.split('/') if len(tmp_list) > offset+1: group_name = '/'.join([tmp_list[i] for i in range(offset+1)]) # Create group called "group_name". Hierarchy of nested groups can be implicitly defined by the forward slashes if not group_name in h5file.keys(): h5file.create_group(group_name) h5file[group_name].attrs['creation_date'] = utils.created_at().encode('utf-8') #h5file[group_name].attrs.create(name='filtered_file_list',data=convert_string_to_bytes(filtered_filename_list)) #h5file[group_name].attrs.create(name='file_list',data=convert_string_to_bytes(filenames_list)) #else: #print(group_name,' was already created.') instFoldermsgStart = f'Starting data transfer from instFolder: {group_name}' print(instFoldermsgStart) for filenumber, filename in enumerate(filtered_filenames_list): #file_ext = os.path.splitext(filename)[1] #try: # hdf5 path to filename group dest_group_name = f'{group_name}/{filename}' if not 'h5' in filename: #file_dict = config_file.select_file_readers(group_id)[file_ext](os.path.join(dirpath,filename)) #file_dict = ext_to_reader_dict[file_ext](os.path.join(dirpath,filename)) file_dict = filereader_registry.select_file_reader(dest_group_name)(os.path.join(dirpath,filename)) stdout = __transfer_file_dict_to_hdf5(h5file, group_name, file_dict) else: source_file_path = os.path.join(dirpath,filename) dest_file_obj = h5file #group_name +'/'+filename #ext_to_reader_dict[file_ext](source_file_path, dest_file_obj, dest_group_name) #g5505f_reader.select_file_reader(dest_group_name)(source_file_path, dest_file_obj, dest_group_name) stdout = __copy_file_in_group(source_file_path, dest_file_obj, dest_group_name, False) # Update the progress bar and log the end message instFoldermsdEnd = f'\nCompleted data transfer for instFolder: {group_name}\n' # Print and log the start message utils.progressBar(dir_number, number_of_dirs, instFoldermsdEnd) logging.info(instFoldermsdEnd ) dir_number = dir_number + 1 print('[End] Data integration') logging.info('[End] Data integration') if len(root_metadata_dict.keys())>0: for key, value in root_metadata_dict.items(): #if key in h5file.attrs: # del h5file.attrs[key] h5file.attrs.create(key, value) #annotate_root_dir(output_filename,root_metadata_dict) #output_yml_filename_path = hdf5_vis.take_yml_snapshot_of_hdf5_file(output_filename) return path_to_output_file #, output_yml_filename_path
[docs] def create_hdf5_file_from_dataframe(ofilename, input_data, group_by_funcs: list, approach: str = None, extract_attrs_func=None): """ Creates an HDF5 file with hierarchical groups based on the specified grouping functions or columns. Parameters: ----------- ofilename (str): Path for the output HDF5 file. input_data (pd.DataFrame or str): Input data as a DataFrame or a valid file system path. group_by_funcs (list): List of callables or column names to define hierarchical grouping. approach (str): Specifies the approach ('top-down' or 'bottom-up') for creating the HDF5 file. extract_attrs_func (callable, optional): Function to extract additional attributes for HDF5 groups. Returns: -------- None """ # Check whether input_data is a valid file-system path or a DataFrame is_valid_path = lambda x: os.path.exists(x) if isinstance(x, str) else False if is_valid_path(input_data): # If input_data is a file-system path, create a DataFrame with file info file_list = os.listdir(input_data) df = pd.DataFrame(file_list, columns=['filename']) df = utils.augment_with_filetype(df) # Add filetype information if needed elif isinstance(input_data, pd.DataFrame): # If input_data is a DataFrame, make a copy df = input_data.copy() else: raise ValueError("input_data must be either a valid file-system path or a DataFrame.") # Generate grouping columns based on group_by_funcs if utils.is_callable_list(group_by_funcs): grouping_cols = [] for i, func in enumerate(group_by_funcs): col_name = f'level_{i}_groups' grouping_cols.append(col_name) df[col_name] = func(df) elif utils.is_str_list(group_by_funcs) and all([item in df.columns for item in group_by_funcs]): grouping_cols = group_by_funcs else: raise ValueError("'group_by_funcs' must be a list of callables or valid column names in the DataFrame.") # Generate group paths df['group_path'] = ['/' + '/'.join(row) for row in df[grouping_cols].values.astype(str)] # Open the HDF5 file in write mode with h5py.File(ofilename, 'w') as file: for group_path in df['group_path'].unique(): # Create groups in HDF5 group = file.create_group(group_path) # Filter the DataFrame for the current group datatable = df[df['group_path'] == group_path].copy() # Drop grouping columns and the generated 'group_path' datatable = datatable.drop(columns=grouping_cols + ['group_path']) # Add datasets to groups if data exists if not datatable.empty: dataset = utils.convert_dataframe_to_np_structured_array(datatable) group.create_dataset(name='data_table', data=dataset) # Add attributes if extract_attrs_func is provided if extract_attrs_func: attrs = extract_attrs_func(datatable) for key, value in attrs.items(): group.attrs[key] = value # Save metadata about depth of hierarchy file.attrs.create(name='depth', data=len(grouping_cols) - 1) print(f"HDF5 file created successfully at {ofilename}") return ofilename
[docs] def save_processed_dataframe_to_hdf5(df, annotator, output_filename): # src_hdf5_path, script_date, script_name): """ Save processed dataframe columns with annotations to an HDF5 file. Parameters: df (pd.DataFrame): DataFrame containing processed time series. annotator (): Annotator object with get_metadata method. output_filename (str): Path to the source HDF5 file. """ # Convert datetime columns to string datetime_cols = df.select_dtypes(include=['datetime64']).columns if list(datetime_cols): df[datetime_cols] = df[datetime_cols].map(str) # Convert dataframe to structured array icad_data_table = utils.convert_dataframe_to_np_structured_array(df) # Get metadata metadata_dict = annotator.get_metadata() # Prepare project level attributes to be added at the root level project_level_attributes = metadata_dict['metadata']['project'] # Prepare high-level attributes high_level_attributes = { 'parent_files': metadata_dict['parent_files'], **metadata_dict['metadata']['sample'], **metadata_dict['metadata']['environment'], **metadata_dict['metadata']['instruments'] } # Prepare data level attributes data_level_attributes = metadata_dict['metadata']['datasets'] for key, value in data_level_attributes.items(): if isinstance(value,dict): data_level_attributes[key] = utils.convert_attrdict_to_np_structured_array(value) # Prepare file dictionary file_dict = { 'name': project_level_attributes['processing_file'], 'attributes_dict': high_level_attributes, 'datasets': [{ 'name': "data_table", 'data': icad_data_table, 'shape': icad_data_table.shape, 'attributes': data_level_attributes }] } # Check if the file exists if os.path.exists(output_filename): mode = "a" print(f"File {output_filename} exists. Opening in append mode.") else: mode = "w" print(f"File {output_filename} does not exist. Creating a new file.") # Write to HDF5 with h5py.File(output_filename, mode) as h5file: # Add project level attributes at the root/top level h5file.attrs.update(project_level_attributes) __transfer_file_dict_to_hdf5(h5file, '/', file_dict)
#if __name__ == '__main__':