APOG_public/acsmnode
0
0
Fork 0
mirror of https://gitea.psi.ch/APOG/acsmnode.git synced 2025-06-25 05:31:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Added data chain step. This step updates the current parameters with new input folder params

Browse Source
This commit is contained in:
florez_j
2025-04-11 11:23:04 +02:00
parent 4408514439
commit 0353c8492b
1 changed files with 29 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
notebooks/demo_acsm_pipeline.ipynb
View File

@ -55,10 +55,11 @@
"from pipelines.steps.prepare_ebas_submission import main as prepare_ebas_submission \n",
"from pipelines.steps.update_actris_header import main as update_actris_header\n",
"from pipelines.steps.utils import load_project_yaml_files\n",
"from pipelines.steps.update_datachain_params import main as update_datachain_params\n",
"\n",
"campaign_descriptor = load_project_yaml_files(project_path, \"campaignDescriptor.yaml\")\n",
"YEAR = campaign_descriptor['year']\n",
"STATION = campaign_descriptor['station']"
"STATION_ABBR = campaign_descriptor['station_abbr']"
]
},
{
@ -85,13 +86,30 @@
"APPEND_DATA_DIR = os.path.splitext(CAMPAIGN_DATA_FILE)[0]\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1.1: Update Data Chain Parameters with Input Data\n",
"* Ensure the data folder retreived from the network drive contains a suitably specified folder `ACSM_TOFWARE/<year>/params`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"update_datachain_params(CAMPAIGN_DATA_FILE, 'ACSM_TOFWARE/2024')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Calibrate Input Campaign Data and Save Data Products\n",
"\n",
"* Set up the input and output directory paths.\n",
"* Make sure the variable `CAMPAIGN_DATA_FILE` is properly defined in previous step. Otherwise, set the variable manually as indicated below.\n",
"* Execute the cell."
]
},
@ -101,12 +119,11 @@
"metadata": {},
"outputs": [],
"source": [
"# Define manually path to data file by uncomenting the following line, and filling the path\n",
"\n",
"# CAMPAIGN_DATA_FILE = ../data/<enter here *.h5 filename of interest inside the data directory>\n",
"path_to_data_file = CAMPAIGN_DATA_FILE\n",
"path_to_calibration_file = '../pipelines/params/calibration_factors.yaml'\n",
"#dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_timeseries.txt/data_table'\n",
"#command = ['python', 'pipelines/steps/apply_calibration_factors.py', path_to_data_file, dataset_name, path_to_calibration_file]\n",
"#status = subprocess.run(command, capture_output=True, check=True)\n",
"#print(status.stdout.decode())\n",
"\n",
"apply_calibration_factors(path_to_data_file,path_to_calibration_file)\n"
]
@ -129,7 +146,7 @@
"metadata": {},
"outputs": [],
"source": [
"dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION}_{YEAR}_meta.txt/data_table'\n",
"dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION_ABBR}_{YEAR}_meta.txt/data_table'\n",
"path_to_config_file = 'pipelines/params/validity_thresholds.yaml'\n",
"#command = ['python', 'pipelines/steps/compute_automated_flags.py', path_to_data_file, dataset_name, path_to_config_file]\n",
"#status = subprocess.run(command, capture_output=True, check=True)\n",
@ -166,7 +183,7 @@
"source": [
"#CAMPAIGN_DATA_FILE = '../data/collection_JFJ_2024_2025-04-08_2025-04-08.h5'\n",
"path_to_data_file = CAMPAIGN_DATA_FILE\n",
"dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION}_{YEAR}_meta.txt/data_table'\n",
"dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION_ABBR}_{YEAR}_meta.txt/data_table'\n",
"path_to_config_file = 'pipelines/params/validity_thresholds.yaml'\n",
"#command = ['python', 'pipelines/steps/compute_automated_flags.py', path_to_data_file, dataset_name, path_to_config_file]\n",
"#status = subprocess.run(command, capture_output=True, check=True)\n",
@ -193,10 +210,10 @@
"DATA_DIR = f\"{APPEND_DATA_DIR}/ACSM_TOFWARE_processed/{YEAR}\"\n",
"FLAGS_DIR = f\"{APPEND_DATA_DIR}/ACSM_TOFWARE_flags/{YEAR}\"\n",
"\n",
"PATH1 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibrated.csv\"\n",
"PATH2 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibrated_err.csv\"\n",
"PATH3 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibration_factors.csv\"\n",
"PATH4 = f\"{FLAGS_DIR}/ACSM_{STATION}_{YEAR}_timeseries_flags.csv\"\n",
"PATH1 = f\"{DATA_DIR}/ACSM_{STATION_ABBR}_{YEAR}_timeseries_calibrated.csv\"\n",
"PATH2 = f\"{DATA_DIR}/ACSM_{STATION_ABBR}_{YEAR}_timeseries_calibrated_err.csv\"\n",
"PATH3 = f\"{DATA_DIR}/ACSM_{STATION_ABBR}_{YEAR}_timeseries_calibration_factors.csv\"\n",
"PATH4 = f\"{FLAGS_DIR}/ACSM_{STATION_ABBR}_{YEAR}_timeseries_flags.csv\"\n",
"\n",
"month = \"2-3\"\n",
"update_actris_header('../campaignDescriptor.yaml')\n",