From ef66d8f1c26622b583380ba1f1c1f286d829e2d6 Mon Sep 17 00:00:00 2001 From: Florez Ospina Juan Felipe Date: Fri, 24 Jan 2025 10:28:43 +0100 Subject: [PATCH] Update unload operation to remove reference and fix logic error to dataset metadata extraction. --- src/hdf5_ops.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hdf5_ops.py b/src/hdf5_ops.py index e020216..7b1003b 100644 --- a/src/hdf5_ops.py +++ b/src/hdf5_ops.py @@ -65,6 +65,7 @@ class HDF5DataOpsManager(): self.file_obj.flush() # Ensure all data is written to disk self.file_obj.close() self.file_obj = None + self.dataset_metadata_df = None # maybe replace by del self.dataset_metadata_df to explicitly clear the reference as well as the memory. def extract_and_load_dataset_metadata(self): @@ -84,9 +85,9 @@ class HDF5DataOpsManager(): self.file_obj.visititems(lambda name, obj: __get_datasets(name, obj, list_of_datasets)) dataset_metadata_df = pd.DataFrame({'dataset_name': list_of_datasets}) - dataset_metadata_df['parent_instrument'] = dataset_metadata_df['dataset_name'].apply(lambda x: x.split('/')[-3]) + dataset_metadata_df['parent_instrument'] = dataset_metadata_df['dataset_name'].apply(lambda x: '/'.join(x.split('/')[i] for i in range(0,len(x.split('/')) - 2)))#[-3])) dataset_metadata_df['parent_file'] = dataset_metadata_df['dataset_name'].apply(lambda x: x.split('/')[-2]) - + self.dataset_metadata_df = dataset_metadata_df except Exception as e: