From d6fa51effc1c23f59a07cf25351a3e6f27153218 Mon Sep 17 00:00:00 2001 From: Florez Ospina Juan Felipe Date: Mon, 10 Mar 2025 13:02:38 +0100 Subject: [PATCH] Fix typo and missing key. 1) Update allowed yaml files in load_project_yaml_files by adding pipelines/params/calibration_factors.yaml. 2) Correct filename access after os.path.split operation. --- pipelines/steps/apply_calibration_factors.py | 4 ++-- pipelines/steps/utils.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pipelines/steps/apply_calibration_factors.py b/pipelines/steps/apply_calibration_factors.py index 80c19a2..63b8e11 100644 --- a/pipelines/steps/apply_calibration_factors.py +++ b/pipelines/steps/apply_calibration_factors.py @@ -120,7 +120,7 @@ def compute_calibration_factors(data_table, datetime_var_name, calibration_param def load_calibration_file(calibration_factors_file): # Load and validate calibration factors structure. TODO: Make sure load_project_yaml_files implements YAML FILE VALIDATION. - filename = os.path.split(calibration_factors_file)[0] + filename = os.path.split(calibration_factors_file)[1] calibration_factors = load_project_yaml_files(projectPath,filename) # Get path to file where calibrations params are defined @@ -298,7 +298,7 @@ if __name__ == '__main__': print(f'Processing script : {processingScriptRelPath}') print(f'Output directory : {path_to_output_folder}') - + # Apply calibration factors to input data_table and generate data lineage metadata calibration_factor_table, calibrated_table = apply_calibration_factors(data_table, datetime_var, args.calibration_file) #calibration_factors) calibrated_table_err = generate_error_dataframe(calibrated_table, datetime_var) diff --git a/pipelines/steps/utils.py b/pipelines/steps/utils.py index 2341845..804a3c9 100644 --- a/pipelines/steps/utils.py +++ b/pipelines/steps/utils.py @@ -149,13 +149,14 @@ def metadata_dict_to_dataframe(metadata: dict, shape: tuple): def load_project_yaml_files(projectPath : str, filename : str): - allowed_filenames = ['acsm_to_ebas.yaml', 'calibration_params.yaml', 'limits_of_detection.yaml', 'station_params.yaml', 'validity_thresholds.yaml'] + allowed_filenames = ['acsm_to_ebas.yaml', 'calibration_params.yaml', 'calibration_factors.yaml', 'limits_of_detection.yaml', 'station_params.yaml', 'validity_thresholds.yaml'] if not filename in allowed_filenames: raise ValueError(f'Invalid filename : {filename}. The filename should be selected from the following list {allowed_filenames}.') filename_to_relpath = {"acsm_to_ebas.yaml":"pipelines/dictionaries/acsm_to_ebas.yaml", "calibration_params.yaml":"pipelines/params/calibration_params.yaml", + "calibration_factors.yaml" : "pipelines/params/calibration_factors.yaml", "limits_of_detection.yaml":"pipelines/params/limits_of_detection.yaml", "station_params.yaml":"pipelines/params/station_params.yaml", "validity_thresholds.yaml":"pipelines/params/validity_thresholds.yaml"}