5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changed names of expected root level metadata attributes.

Browse Source
This commit is contained in:
florez_j
2024-07-17 08:48:47 +02:00
parent a06e28291c
commit 938e8e50a1
1 changed files with 18 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/data_integration_lib.py
View File

@ -34,15 +34,16 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'):
def output_filename(name, date, initials):
return f"{name}_{date}_{initials}.h5"
exp_campaign_name = config_dict['experiment_name']
initials = config_dict['user_name']
exp_campaign_name = config_dict['experiment']
initials = config_dict['contact']
input_file_dir = config_dict['input_file_directory']
output_dir = config_dict['output_file_directory']
select_dir_keywords = config_dict['instrument_datafolder']
root_metadata_dict = {
'project_name' : config_dict['project_name'],
'experiment_name' : config_dict['experiment_name'],
'user_name' : config_dict['user_name'],
'project' : config_dict['project'],
'experiment' : config_dict['experiment'],
'contact' : config_dict['contact'],
'actris_level': config_dict['actris_level']
}
def create_hdf5_file(date_str, select_file_keywords,root_metadata):
@ -68,8 +69,8 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'):
date_str = datetime_step.strftime('%Y-%m-%d')
select_file_keywords = datetime_augment_dict[datetime_step]
root_metadata_dict.update({'experiment_startdate': date_str,
'experiment_enddate': date_str})
root_metadata_dict.update({'dataset_startdate': date_str,
'dataset_enddate': date_str})
dt_step_output_filename_path= create_hdf5_file(date_str, select_file_keywords, root_metadata_dict)
output_filename_path.append(dt_step_output_filename_path)
@ -78,20 +79,20 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'):
for datetime_step in datetime_augment_dict.keys():
select_file_keywords = select_file_keywords + datetime_augment_dict[datetime_step]
config_dict['experiment_startdate'] = min(datetime_augment_dict.keys())
config_dict['experiment_enddate'] = max(datetime_augment_dict.keys())
startdate = config_dict['experiment_startdate'].strftime('%Y-%m-%d')
enddate = config_dict['experiment_enddate'].strftime('%Y-%m-%d')
root_metadata_dict.update({'experiment_startdate': startdate,
'experiment_enddate': enddate})
config_dict['dataset_startdate'] = min(datetime_augment_dict.keys())
config_dict['dataset_enddate'] = max(datetime_augment_dict.keys())
startdate = config_dict['dataset_startdate'].strftime('%Y-%m-%d')
enddate = config_dict['dataset_enddate'].strftime('%Y-%m-%d')
root_metadata_dict.update({'dataset_startdate': startdate,
'dataset_enddate': enddate})
date_str = f'{startdate}_{enddate}'
output_filename_path = create_hdf5_file(date_str, select_file_keywords, root_metadata_dict)
else:
startdate = config_dict['experiment_startdate']
enddate = config_dict['experiment_enddate']
root_metadata_dict.update({'experiment_startdate': startdate,
'experiment_enddate': enddate})
startdate = config_dict['dataset_startdate']
enddate = config_dict['dataset_enddate']
root_metadata_dict.update({'dataset_startdate': startdate,
'dataset_enddate': enddate})
date_str = f'{startdate}_{enddate}'
output_filename_path = create_hdf5_file(date_str, select_file_keywords = [], root_metadata = root_metadata_dict)