5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into 'feature/DB_for_FileReader_Repo'

Browse Source 
# Conflicts:
#   instruments/filereader_registry.py
#   pipelines/data_integration.py
#   src/hdf5_writer.py
This commit is contained in:
florez_j
2025-02-25 10:41:02 +01:00
parent 064b8b3a62 9511377883
commit 295b43a89a
2 changed files with 17 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/hdf5_writer.py
View File

@ -7,6 +7,7 @@ import pandas as pd
import numpy as np
import h5py
import logging
import json
#try:
# from dima.utils import g5505_utils as utils
@ -158,11 +159,22 @@ def create_hdf5_file_from_filesystem_path(path_to_input_directory: str,
stdout = inst
logging.error('Failed to create group %s into HDF5: %s', group_name, inst)
if 'data_lineage_metadata.json' in filtered_filenames_list:
idx = filtered_filenames_list.index('data_lineage_metadata.json')
data_lineage_file = filtered_filenames_list[idx]
try:
with open('/'.join([dirpath,data_lineage_file]),'r') as dlf:
data_lineage_dict = json.load(dlf)
filtered_filenames_list.pop(idx)
except json.JSONDecodeError:
data_lineage_dict = {} # Start fresh if file is invalid
else:
data_lineage_dict = {}
for filenumber, filename in enumerate(filtered_filenames_list):
#file_ext = os.path.splitext(filename)[1]
#try:
# hdf5 path to filename group
dest_group_name = f'{group_name}/{filename}'
source_file_path = os.path.join(dirpath,filename)