5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Modified function to return list of paths when config_file.yaml integration mode = experimental step.

Browse Source
This commit is contained in:
florez_j
2024-05-28 11:29:32 +02:00
parent f6a46168ec
commit 804ea52583
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/data_integration_lib.py
View File

@ -18,7 +18,7 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'):
log_dir (str): Directory to save the log file. log_dir (str): Directory to save the log file.
Returns: Returns:
str: Path to the created HDF5 file. str: Path (or list of Paths) to the created HDF5 file(s).
""" """
date = utils.created_at() date = utils.created_at()
@ -63,10 +63,12 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'):
print(tmp) print(tmp)
if 'experimental_step' in config_dict['integration_mode']: if 'experimental_step' in config_dict['integration_mode']:
output_filename_path = []
for datetime_step in datetime_augment_dict.keys(): for datetime_step in datetime_augment_dict.keys():
date_str = datetime_step.strftime('%Y-%m-%d') date_str = datetime_step.strftime('%Y-%m-%d')
select_file_keywords = datetime_augment_dict[datetime_step] select_file_keywords = datetime_augment_dict[datetime_step]
output_filename_path = create_hdf5_file(date_str, select_file_keywords) dt_step_output_filename_path= create_hdf5_file(date_str, select_file_keywords)
output_filename_path.append(dt_step_output_filename_path)
elif 'collection' in config_dict['integration_mode']: elif 'collection' in config_dict['integration_mode']:
select_file_keywords = [] select_file_keywords = []