From f3ec8c6acf07956f6e35c0ffdae769fdbf3b7e0a Mon Sep 17 00:00:00 2001 From: Florez Ospina Juan Felipe Date: Wed, 29 Jan 2025 19:27:18 +0100 Subject: [PATCH 1/2] Fix typo, i was not properly accessing the key variables in the yaml dictionary --- pipelines/steps/apply_calibration_factors.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pipelines/steps/apply_calibration_factors.py b/pipelines/steps/apply_calibration_factors.py index 7532034..2590c7a 100644 --- a/pipelines/steps/apply_calibration_factors.py +++ b/pipelines/steps/apply_calibration_factors.py @@ -50,9 +50,11 @@ def apply_calibration_factors(data_table, calibration_factors): for variable_name in new_data_table.select_dtypes(include=["number"]).columns: if variable_name in calibration_factors['variables'].keys(): # use standard calibration factor + + #print(variable_name) # Extract numerator and denominator values - numerator = prod(calibration_factors[variable_name]['num']) - denominator = prod(calibration_factors[variable_name]['den']) + numerator = prod(calibration_factors['variables'][variable_name]['num']) + denominator = prod(calibration_factors['variables'][variable_name]['den']) # Apply calibration to each variable new_data_table[variable_name] = new_data_table[variable_name].mul((numerator / denominator)) From d69e4182a88c665f4ad05f3ac559e6d5fdd9f900 Mon Sep 17 00:00:00 2001 From: Florez Ospina Juan Felipe Date: Wed, 29 Jan 2025 19:28:43 +0100 Subject: [PATCH 2/2] Fix alias reference pairs. YAML does not allow references to individual elements of a dictionary using *dictAlias.keyword notation --- pipelines/params/calibration_factors.yaml | 85 ++++++++++++----------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/pipelines/params/calibration_factors.yaml b/pipelines/params/calibration_factors.yaml index 0718184..b535dd7 100644 --- a/pipelines/params/calibration_factors.yaml +++ b/pipelines/params/calibration_factors.yaml @@ -1,97 +1,98 @@ # Define common factors -factors: &factors - # Get values from data///config_acsm_.r, values used in Tofware analysis. - IE: 145.9 - ABRefWave: 254000 # TODO: verify if AB_ref in the config file is the same as ABRefWave - RIE_SO4: 0.63 - RIE_NH4: 3.495 - RIE_Org : 1.4 - # Get values from data///cal.csv - IE_correct: 145.9 - AB_ref_correct: 254000 - RIE_SO4_correct: 0.63 - RIE_NH4_correct: 3.495 - RIE_Org_correct : 1.4 - flow_ref_correct : 1.36 + +# Get values from data///config_acsm_.r, values used in Tofware analysis. +IE: &IE 145.9 +ABRefWave: &ABRefWave 254000 +RIE_SO4: &RIE_SO4 0.63 +RIE_NH4: &RIE_NH4 3.495 +RIE_Org: &RIE_Org 1.4 + +# Get values from data///cal.csv +IE_correct: &IE_correct 145.9 +AB_ref_correct: &AB_ref_correct 254000 +RIE_SO4_correct: &RIE_SO4_correct 0.63 +RIE_NH4_correct: &RIE_NH4_correct 3.495 +RIE_Org_correct: &RIE_Org_correct 1.4 +flow_ref_correct: &flow_ref_correct 1.36 # Define mappings for associated variables variables: # all_dat[, NO3_correct := (NO3_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] NO3_11000: - num: [*factors.IE, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.ABRefWave] + num: [*IE, *AB_ref_correct] + den: [*IE_correct, *ABRefWave] # all_dat[, SO4_correct := (SO4_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] SO4_11000: - num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] + num: [*IE, *RIE_SO4, *AB_ref_correct] + den: [*IE_correct, *RIE_SO4_correct, *ABRefWave] # all_dat[, NH4_correct := (NH4_11000 * IE * RIE_NH4 * AB_ref_correct) / (IE_correct * RIE_NH4_correct * ABRefWave)] NH4_11000: - num: [*factors.IE, *factors.RIE_NH4, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.RIE_NH4_correct, *factors.ABRefWave] + num: [*IE, *RIE_NH4, *AB_ref_correct] + den: [*IE_correct, *RIE_NH4_correct, *ABRefWave] # all_dat[, Org_correct := (Org_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] Org_11000: - num: [*factors.IE, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.ABRefWave] + num: [*IE, *AB_ref_correct] + den: [*IE_correct, *ABRefWave] # all_dat[, Chl_correct := (Chl_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] Chl_11000: - num: [*factors.IE, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.ABRefWave] + num: [*IE, *AB_ref_correct] + den: [*IE_correct, *ABRefWave] # all_dat[, Org_44_11000_correct := (Org_44_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] Org_44_11000: - num: [*factors.IE, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.ABRefWave] + num: [*IE, *AB_ref_correct] + den: [*IE_correct, *ABRefWave] # all_dat[, Org_43_11000_correct := (Org_43_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] Org_43_11000: - num: [*factors.IE, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.ABRefWave] + num: [*IE, *AB_ref_correct] + den: [*IE_correct, *ABRefWave] # all_dat[, Org_60_11000_correct := (Org_60_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] Org_60_11000: - num: [*factors.IE, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.ABRefWave] + num: [*IE, *AB_ref_correct] + den: [*IE_correct, *ABRefWave] # all_dat[, NO3_30_11000_correct := (NO3_30_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] NO3_30_11000: - num: [*factors.IE, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.ABRefWave] + num: [*IE, *AB_ref_correct] + den: [*IE_correct, *ABRefWave] # all_dat[, SO4_98_11000_correct := (SO4_98_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] SO4_98_11000: - num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] + num: [*IE, *RIE_SO4, *AB_ref_correct] + den: [*IE_correct, *RIE_SO4_correct, *ABRefWave] # all_dat[, SO4_81_11000_correct := (SO4_81_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] SO4_81_11000: - num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] + num: [*IE, *RIE_SO4, *AB_ref_correct] + den: [*IE_correct, *RIE_SO4_correct, *ABRefWave] # all_dat[, SO4_82_11000_correct := (SO4_82_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] SO4_82_11000: - num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] + num: [*IE, *RIE_SO4, *AB_ref_correct] + den: [*IE_correct, *RIE_SO4_correct, *ABRefWave] # all_dat[, SO4_62_11000_correct := (SO4_62_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] SO4_62_11000: - num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] + num: [*IE, *RIE_SO4, *AB_ref_correct] + den: [*IE_correct, *RIE_SO4_correct, *ABRefWave] # all_dat[, SO4_48_11000_correct := (SO4_48_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] SO4_48_11000: - num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] - den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] + num: [*IE, *RIE_SO4, *AB_ref_correct] + den: [*IE_correct, *RIE_SO4_correct, *ABRefWave]