5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update unload operation to remove reference and fix logic error to dataset metadata extraction.

Browse Source
This commit is contained in:
florez_j
2025-01-24 10:28:43 +01:00
parent 1ae607f73b
commit ef66d8f1c2
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/hdf5_ops.py
View File

@ -65,6 +65,7 @@ class HDF5DataOpsManager():
self.file_obj.flush() # Ensure all data is written to disk
self.file_obj.close()
self.file_obj = None
self.dataset_metadata_df = None # maybe replace by del self.dataset_metadata_df to explicitly clear the reference as well as the memory.
def extract_and_load_dataset_metadata(self):
@ -84,9 +85,9 @@ class HDF5DataOpsManager():
self.file_obj.visititems(lambda name, obj: __get_datasets(name, obj, list_of_datasets))
dataset_metadata_df = pd.DataFrame({'dataset_name': list_of_datasets})
dataset_metadata_df['parent_instrument'] = dataset_metadata_df['dataset_name'].apply(lambda x: x.split('/')[-3])
dataset_metadata_df['parent_instrument'] = dataset_metadata_df['dataset_name'].apply(lambda x: '/'.join(x.split('/')[i] for i in range(0,len(x.split('/')) - 2)))#[-3]))
dataset_metadata_df['parent_file'] = dataset_metadata_df['dataset_name'].apply(lambda x: x.split('/')[-2])
self.dataset_metadata_df = dataset_metadata_df
except Exception as e: