APOG_public/acsm-fairifier
0
0
Fork 0
mirror of https://gitea.psi.ch/APOG/acsm-fairifier.git synced 2025-07-06 08:04:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://gitlab.psi.ch/apog/acsmnode

Browse Source
This commit is contained in:
Florez Ospina Juan Felipe (ADM)
2025-01-30 13:29:56 +01:00
parent b19ab3d867 d69e4182a8
commit 1f119991f2
2 changed files with 47 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
pipelines/params/calibration_factors.yaml
View File

@ -1,97 +1,98 @@
# Define common factors # Define common factors
factors: &factors
# Get values from data/<station>/<year>/config_acsm_<year>.r, values used in Tofware analysis. # Get values from data/<station>/<year>/config_acsm_<year>.r, values used in Tofware analysis.
IE: 145.9 IE: &IE 145.9
ABRefWave: 254000 # TODO: verify if AB_ref in the config file is the same as ABRefWave ABRefWave: &ABRefWave 254000
RIE_SO4: 0.63 RIE_SO4: &RIE_SO4 0.63
RIE_NH4: 3.495 RIE_NH4: &RIE_NH4 3.495
RIE_Org : 1.4 RIE_Org: &RIE_Org 1.4
# Get values from data/<station>/<year>/cal.csv
IE_correct: 145.9 # Get values from data/<station>/<year>/cal.csv
AB_ref_correct: 254000 IE_correct: &IE_correct 145.9
RIE_SO4_correct: 0.63 AB_ref_correct: &AB_ref_correct 254000
RIE_NH4_correct: 3.495 RIE_SO4_correct: &RIE_SO4_correct 0.63
RIE_Org_correct : 1.4 RIE_NH4_correct: &RIE_NH4_correct 3.495
flow_ref_correct : 1.36 RIE_Org_correct: &RIE_Org_correct 1.4
flow_ref_correct: &flow_ref_correct 1.36
# Define mappings for associated variables # Define mappings for associated variables
variables: variables:
# all_dat[, NO3_correct := (NO3_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] # all_dat[, NO3_correct := (NO3_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)]
NO3_11000: NO3_11000:
num: [*factors.IE, *factors.AB_ref_correct] num: [*IE, *AB_ref_correct]
den: [*factors.IE_correct, *factors.ABRefWave] den: [*IE_correct, *ABRefWave]
# all_dat[, SO4_correct := (SO4_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] # all_dat[, SO4_correct := (SO4_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)]
SO4_11000: SO4_11000:
num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] num: [*IE, *RIE_SO4, *AB_ref_correct]
den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] den: [*IE_correct, *RIE_SO4_correct, *ABRefWave]
# all_dat[, NH4_correct := (NH4_11000 * IE * RIE_NH4 * AB_ref_correct) / (IE_correct * RIE_NH4_correct * ABRefWave)] # all_dat[, NH4_correct := (NH4_11000 * IE * RIE_NH4 * AB_ref_correct) / (IE_correct * RIE_NH4_correct * ABRefWave)]
NH4_11000: NH4_11000:
num: [*factors.IE, *factors.RIE_NH4, *factors.AB_ref_correct] num: [*IE, *RIE_NH4, *AB_ref_correct]
den: [*factors.IE_correct, *factors.RIE_NH4_correct, *factors.ABRefWave] den: [*IE_correct, *RIE_NH4_correct, *ABRefWave]
# all_dat[, Org_correct := (Org_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] # all_dat[, Org_correct := (Org_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)]
Org_11000: Org_11000:
num: [*factors.IE, *factors.AB_ref_correct] num: [*IE, *AB_ref_correct]
den: [*factors.IE_correct, *factors.ABRefWave] den: [*IE_correct, *ABRefWave]
# all_dat[, Chl_correct := (Chl_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] # all_dat[, Chl_correct := (Chl_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)]
Chl_11000: Chl_11000:
num: [*factors.IE, *factors.AB_ref_correct] num: [*IE, *AB_ref_correct]
den: [*factors.IE_correct, *factors.ABRefWave] den: [*IE_correct, *ABRefWave]
# all_dat[, Org_44_11000_correct := (Org_44_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] # all_dat[, Org_44_11000_correct := (Org_44_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)]
Org_44_11000: Org_44_11000:
num: [*factors.IE, *factors.AB_ref_correct] num: [*IE, *AB_ref_correct]
den: [*factors.IE_correct, *factors.ABRefWave] den: [*IE_correct, *ABRefWave]
# all_dat[, Org_43_11000_correct := (Org_43_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] # all_dat[, Org_43_11000_correct := (Org_43_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)]
Org_43_11000: Org_43_11000:
num: [*factors.IE, *factors.AB_ref_correct] num: [*IE, *AB_ref_correct]
den: [*factors.IE_correct, *factors.ABRefWave] den: [*IE_correct, *ABRefWave]
# all_dat[, Org_60_11000_correct := (Org_60_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] # all_dat[, Org_60_11000_correct := (Org_60_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)]
Org_60_11000: Org_60_11000:
num: [*factors.IE, *factors.AB_ref_correct] num: [*IE, *AB_ref_correct]
den: [*factors.IE_correct, *factors.ABRefWave] den: [*IE_correct, *ABRefWave]
# all_dat[, NO3_30_11000_correct := (NO3_30_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)] # all_dat[, NO3_30_11000_correct := (NO3_30_11000 * IE * AB_ref_correct) / (IE_correct * ABRefWave)]
NO3_30_11000: NO3_30_11000:
num: [*factors.IE, *factors.AB_ref_correct] num: [*IE, *AB_ref_correct]
den: [*factors.IE_correct, *factors.ABRefWave] den: [*IE_correct, *ABRefWave]
# all_dat[, SO4_98_11000_correct := (SO4_98_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] # all_dat[, SO4_98_11000_correct := (SO4_98_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)]
SO4_98_11000: SO4_98_11000:
num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] num: [*IE, *RIE_SO4, *AB_ref_correct]
den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] den: [*IE_correct, *RIE_SO4_correct, *ABRefWave]
# all_dat[, SO4_81_11000_correct := (SO4_81_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] # all_dat[, SO4_81_11000_correct := (SO4_81_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)]
SO4_81_11000: SO4_81_11000:
num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] num: [*IE, *RIE_SO4, *AB_ref_correct]
den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] den: [*IE_correct, *RIE_SO4_correct, *ABRefWave]
# all_dat[, SO4_82_11000_correct := (SO4_82_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] # all_dat[, SO4_82_11000_correct := (SO4_82_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)]
SO4_82_11000: SO4_82_11000:
num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] num: [*IE, *RIE_SO4, *AB_ref_correct]
den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] den: [*IE_correct, *RIE_SO4_correct, *ABRefWave]
# all_dat[, SO4_62_11000_correct := (SO4_62_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] # all_dat[, SO4_62_11000_correct := (SO4_62_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)]
SO4_62_11000: SO4_62_11000:
num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] num: [*IE, *RIE_SO4, *AB_ref_correct]
den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] den: [*IE_correct, *RIE_SO4_correct, *ABRefWave]
# all_dat[, SO4_48_11000_correct := (SO4_48_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)] # all_dat[, SO4_48_11000_correct := (SO4_48_11000 * IE * RIE_SO4 * AB_ref_correct) / (IE_correct * RIE_SO4_correct * ABRefWave)]
SO4_48_11000: SO4_48_11000:
num: [*factors.IE, *factors.RIE_SO4, *factors.AB_ref_correct] num: [*IE, *RIE_SO4, *AB_ref_correct]
den: [*factors.IE_correct, *factors.RIE_SO4_correct, *factors.ABRefWave] den: [*IE_correct, *RIE_SO4_correct, *ABRefWave]

6
pipelines/steps/apply_calibration_factors.py
View File

@ -50,9 +50,11 @@ def apply_calibration_factors(data_table, calibration_factors):
for variable_name in new_data_table.select_dtypes(include=["number"]).columns: for variable_name in new_data_table.select_dtypes(include=["number"]).columns:
if variable_name in calibration_factors['variables'].keys(): # use standard calibration factor if variable_name in calibration_factors['variables'].keys(): # use standard calibration factor
#print(variable_name)
# Extract numerator and denominator values # Extract numerator and denominator values
numerator = prod(calibration_factors[variable_name]['num']) numerator = prod(calibration_factors['variables'][variable_name]['num'])
denominator = prod(calibration_factors[variable_name]['den']) denominator = prod(calibration_factors['variables'][variable_name]['den'])
# Apply calibration to each variable # Apply calibration to each variable
new_data_table[variable_name] = new_data_table[variable_name].mul((numerator / denominator)) new_data_table[variable_name] = new_data_table[variable_name].mul((numerator / denominator))