diff --git a/src/data_integration_lib.py b/src/data_integration_lib.py index 9f54d2c..3188e31 100644 --- a/src/data_integration_lib.py +++ b/src/data_integration_lib.py @@ -34,15 +34,16 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'): def output_filename(name, date, initials): return f"{name}_{date}_{initials}.h5" - exp_campaign_name = config_dict['experiment_name'] - initials = config_dict['user_name'] + exp_campaign_name = config_dict['experiment'] + initials = config_dict['contact'] input_file_dir = config_dict['input_file_directory'] output_dir = config_dict['output_file_directory'] select_dir_keywords = config_dict['instrument_datafolder'] root_metadata_dict = { - 'project_name' : config_dict['project_name'], - 'experiment_name' : config_dict['experiment_name'], - 'user_name' : config_dict['user_name'], + 'project' : config_dict['project'], + 'experiment' : config_dict['experiment'], + 'contact' : config_dict['contact'], + 'actris_level': config_dict['actris_level'] } def create_hdf5_file(date_str, select_file_keywords,root_metadata): @@ -68,8 +69,8 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'): date_str = datetime_step.strftime('%Y-%m-%d') select_file_keywords = datetime_augment_dict[datetime_step] - root_metadata_dict.update({'experiment_startdate': date_str, - 'experiment_enddate': date_str}) + root_metadata_dict.update({'dataset_startdate': date_str, + 'dataset_enddate': date_str}) dt_step_output_filename_path= create_hdf5_file(date_str, select_file_keywords, root_metadata_dict) output_filename_path.append(dt_step_output_filename_path) @@ -78,20 +79,20 @@ def integrate_data_sources(yaml_config_file_path, log_dir='logs/'): for datetime_step in datetime_augment_dict.keys(): select_file_keywords = select_file_keywords + datetime_augment_dict[datetime_step] - config_dict['experiment_startdate'] = min(datetime_augment_dict.keys()) - config_dict['experiment_enddate'] = max(datetime_augment_dict.keys()) - startdate = config_dict['experiment_startdate'].strftime('%Y-%m-%d') - enddate = config_dict['experiment_enddate'].strftime('%Y-%m-%d') - root_metadata_dict.update({'experiment_startdate': startdate, - 'experiment_enddate': enddate}) + config_dict['dataset_startdate'] = min(datetime_augment_dict.keys()) + config_dict['dataset_enddate'] = max(datetime_augment_dict.keys()) + startdate = config_dict['dataset_startdate'].strftime('%Y-%m-%d') + enddate = config_dict['dataset_enddate'].strftime('%Y-%m-%d') + root_metadata_dict.update({'dataset_startdate': startdate, + 'dataset_enddate': enddate}) date_str = f'{startdate}_{enddate}' output_filename_path = create_hdf5_file(date_str, select_file_keywords, root_metadata_dict) else: - startdate = config_dict['experiment_startdate'] - enddate = config_dict['experiment_enddate'] - root_metadata_dict.update({'experiment_startdate': startdate, - 'experiment_enddate': enddate}) + startdate = config_dict['dataset_startdate'] + enddate = config_dict['dataset_enddate'] + root_metadata_dict.update({'dataset_startdate': startdate, + 'dataset_enddate': enddate}) date_str = f'{startdate}_{enddate}' output_filename_path = create_hdf5_file(date_str, select_file_keywords = [], root_metadata = root_metadata_dict)