5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changed names of expected root level metadata attributes.

Browse Source
This commit is contained in:
florez_j
2024-07-17 08:48:47 +02:00
parent a06e28291c
commit 938e8e50a1
1 changed files with 18 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/data_integration_lib.py
View File

@ -34,15 +34,16 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'):
def output_filename(name, date, initials): def output_filename(name, date, initials):
return f"{name}_{date}_{initials}.h5" return f"{name}_{date}_{initials}.h5"
exp_campaign_name = config_dict['experiment_name'] exp_campaign_name = config_dict['experiment']
initials = config_dict['user_name'] initials = config_dict['contact']
input_file_dir = config_dict['input_file_directory'] input_file_dir = config_dict['input_file_directory']
output_dir = config_dict['output_file_directory'] output_dir = config_dict['output_file_directory']
select_dir_keywords = config_dict['instrument_datafolder'] select_dir_keywords = config_dict['instrument_datafolder']
root_metadata_dict = { root_metadata_dict = {
'project_name' : config_dict['project_name'], 'project' : config_dict['project'],
'experiment_name' : config_dict['experiment_name'], 'experiment' : config_dict['experiment'],
'user_name' : config_dict['user_name'], 'contact' : config_dict['contact'],
'actris_level': config_dict['actris_level']
} }
def create_hdf5_file(date_str, select_file_keywords,root_metadata): def create_hdf5_file(date_str, select_file_keywords,root_metadata):
@ -68,8 +69,8 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'):
date_str = datetime_step.strftime('%Y-%m-%d') date_str = datetime_step.strftime('%Y-%m-%d')
select_file_keywords = datetime_augment_dict[datetime_step] select_file_keywords = datetime_augment_dict[datetime_step]
root_metadata_dict.update({'experiment_startdate': date_str, root_metadata_dict.update({'dataset_startdate': date_str,
'experiment_enddate': date_str}) 'dataset_enddate': date_str})
dt_step_output_filename_path= create_hdf5_file(date_str, select_file_keywords, root_metadata_dict) dt_step_output_filename_path= create_hdf5_file(date_str, select_file_keywords, root_metadata_dict)
output_filename_path.append(dt_step_output_filename_path) output_filename_path.append(dt_step_output_filename_path)
@ -78,20 +79,20 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'):
for datetime_step in datetime_augment_dict.keys(): for datetime_step in datetime_augment_dict.keys():
select_file_keywords = select_file_keywords + datetime_augment_dict[datetime_step] select_file_keywords = select_file_keywords + datetime_augment_dict[datetime_step]
config_dict['experiment_startdate'] = min(datetime_augment_dict.keys()) config_dict['dataset_startdate'] = min(datetime_augment_dict.keys())
config_dict['experiment_enddate'] = max(datetime_augment_dict.keys()) config_dict['dataset_enddate'] = max(datetime_augment_dict.keys())
startdate = config_dict['experiment_startdate'].strftime('%Y-%m-%d') startdate = config_dict['dataset_startdate'].strftime('%Y-%m-%d')
enddate = config_dict['experiment_enddate'].strftime('%Y-%m-%d') enddate = config_dict['dataset_enddate'].strftime('%Y-%m-%d')
root_metadata_dict.update({'experiment_startdate': startdate, root_metadata_dict.update({'dataset_startdate': startdate,
'experiment_enddate': enddate}) 'dataset_enddate': enddate})
date_str = f'{startdate}_{enddate}' date_str = f'{startdate}_{enddate}'
output_filename_path = create_hdf5_file(date_str, select_file_keywords, root_metadata_dict) output_filename_path = create_hdf5_file(date_str, select_file_keywords, root_metadata_dict)
else: else:
startdate = config_dict['experiment_startdate'] startdate = config_dict['dataset_startdate']
enddate = config_dict['experiment_enddate'] enddate = config_dict['dataset_enddate']
root_metadata_dict.update({'experiment_startdate': startdate, root_metadata_dict.update({'dataset_startdate': startdate,
'experiment_enddate': enddate}) 'dataset_enddate': enddate})
date_str = f'{startdate}_{enddate}' date_str = f'{startdate}_{enddate}'
output_filename_path = create_hdf5_file(date_str, select_file_keywords = [], root_metadata = root_metadata_dict) output_filename_path = create_hdf5_file(date_str, select_file_keywords = [], root_metadata = root_metadata_dict)