APOG_public/acsmnode
0
0
Fork 0
mirror of https://gitea.psi.ch/APOG/acsmnode.git synced 2025-06-27 03:51:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Rewrote load_calib_file() in terms pipelines.steps.utils.load_project_yaml_files(). In the future we will replace it completely so we can validate and manage all yaml files in the chain in a more centralized manner.

Browse Source
This commit is contained in:
florez_j
2025-03-10 11:46:24 +01:00
parent ecdf3687ba
commit 4ca99f4f04
1 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
pipelines/steps/apply_calibration_factors.py
View File

@ -37,6 +37,7 @@ import dima.src.hdf5_ops as dataOps
import dima.utils.g5505_utils as utils import dima.utils.g5505_utils as utils
import pipelines.steps.utils as stepUtils import pipelines.steps.utils as stepUtils
from pipelines.steps.utils import generate_error_dataframe from pipelines.steps.utils import generate_error_dataframe
from pipelines.steps.utils import load_project_yaml_files
def compute_calibration_factors(data_table, datetime_var_name, calibration_params, calibration_factors): def compute_calibration_factors(data_table, datetime_var_name, calibration_params, calibration_factors):
""" """
@ -116,19 +117,18 @@ def compute_calibration_factors(data_table, datetime_var_name, calibration_param
return tmp_df return tmp_df
def load_calibration_file(calibration_file): def load_calibration_file(calibration_factors_file):
# START YAML FILE VALIDATION # Load and validate calibration factors structure. TODO: Make sure load_project_yaml_files implements YAML FILE VALIDATION.
# TODO : create a separate validation function filename = os.path.split(calibration_factors_file)[0]
with open(calibration_file, 'r') as stream: calibration_factors = load_project_yaml_files(projectPath,filename)
calibration_factors = yaml.load(stream, Loader=yaml.FullLoader)
# Get path to file where calibrations params are defined # Get path to file where calibrations params are defined
path_to_calib_params_file = calibration_factors.get("calibration_params", {}).get('path_to_file') path_to_calib_params_file = calibration_factors.get("calibration_params", {}).get('path_to_file')
# Validate # Validate
if not path_to_calib_params_file: if not path_to_calib_params_file:
raise ValueError(f'Invalid yaml file. {calibration_file} must contain "calibration_params" with a valid "path_to_file".') raise ValueError(f'Invalid yaml file. {calibration_factors_file} must contain "calibration_params" with a valid "path_to_file".')
if not os.path.exists(path_to_calib_params_file): if not os.path.exists(path_to_calib_params_file):
raise FileNotFoundError(f'Calibration parameters file not found: {path_to_calib_params_file}') raise FileNotFoundError(f'Calibration parameters file not found: {path_to_calib_params_file}')
@ -149,7 +149,7 @@ def load_calibration_file(calibration_file):
return calibration_params, calibration_factors return calibration_params, calibration_factors
def apply_calibration_factors(data_table, datetime_var_name, calibration_file): def apply_calibration_factors(data_table, datetime_var_name, calibration_factors_file : str = 'pipelines/params/calibration_factors.yaml'):
""" """
Calibrates the species data in the given data table using a calibration factor. Calibrates the species data in the given data table using a calibration factor.
@ -167,7 +167,7 @@ def apply_calibration_factors(data_table, datetime_var_name, calibration_file):
# Make a copy of the input table to avoid modifying the original # Make a copy of the input table to avoid modifying the original
new_data_table = data_table.copy() new_data_table = data_table.copy()
calibration_params, calibration_factors = load_calibration_file(calibration_file) calibration_params, calibration_factors = load_calibration_file(calibration_factors_file)
calibration_factor_table = compute_calibration_factors(new_data_table, calibration_factor_table = compute_calibration_factors(new_data_table,