From ecdf3687ba8c20fd02e0faab4bf395c04e9b4cb8 Mon Sep 17 00:00:00 2001 From: Florez Ospina Juan Felipe Date: Mon, 10 Mar 2025 11:38:51 +0100 Subject: [PATCH] Remove/replace def load_parameters(flag_type) with steps.utils.load_project_yaml_files(.). This is to have a more centralized treatment of yaml files inside the data chain. --- pipelines/steps/generate_flags.py | 38 ++++--------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/pipelines/steps/generate_flags.py b/pipelines/steps/generate_flags.py index 19d231f..02341c2 100644 --- a/pipelines/steps/generate_flags.py +++ b/pipelines/steps/generate_flags.py @@ -28,43 +28,13 @@ import dima.src.hdf5_ops as dataOps import pipelines.steps.utils as stepUtils import dima.utils.g5505_utils as utils import json +from pipelines.steps.utils import load_project_yaml_files def compute_cpc_flags(): # TODO: ask rob where to find this information. return 0 -def load_parameters(flag_type): - - # Implicit input - if flag_type == 'diagnostics': - flag_type_file = os.path.normpath(os.path.join(projectPath,'pipelines/params/validity_thresholds.yaml')) - error_message = f"Error accessing validation thresholds at: {flag_type_file}" - elif flag_type == 'species': - flag_type_file = os.path.normpath(os.path.join(projectPath,'pipelines/params/calibration_params.yaml')) - error_message = f"Error accessing calibration parameters at: {flag_type_file}" - - - output_dict = {} - try: - with open(flag_type_file, 'r') as stream: - output_dict = yaml.load(stream, Loader=yaml.FullLoader) - except Exception as e: - - print(error_message) - return {} - - # Get name of the specifies to flag based on diagnostics and manual flags - #path_to_calib_params = os.path.normpath(os.path.join(projectPath,'pipelines/params/calibration_params.yaml')) - - #if not os.path.exists(path_to_calib_params): - # raise FileNotFoundError(f'Calibration params file:{path_to_calib_params}') - - #with open(path_to_calib_params,'r') as stream: - # calib_param_dict = yaml.safe_load(stream) - - return output_dict - #def compute_diagnostic_variable_flags(data_table, validity_thresholds_dict): def generate_diagnostic_flags(data_table, validity_thresholds_dict): """ @@ -390,11 +360,13 @@ if __name__ == '__main__': # Compute diagnostic flags based on validity thresholds defined in configuration_file_dict if flag_type == 'diagnostics': - validity_thresholds_dict = load_parameters(flag_type) + #validity_thresholds_dict = load_parameters(flag_type) + validity_thresholds_dict = load_project_yaml_files(projectPath, "validity_thresholds.yaml") flags_table = generate_diagnostic_flags(data_table, validity_thresholds_dict) if flag_type == 'species': - calib_param_dict = load_parameters(flag_type) + #calib_param_dict = load_parameters(flag_type) + calib_param_dict = load_project_yaml_files(projectPath, "calibration_params.yaml") flags_table = generate_species_flags(data_table,calib_param_dict) metadata = {'actris_level' : 1,