APOG_public/acsmnode
0
0
Fork 0
mirror of https://gitea.psi.ch/APOG/acsmnode.git synced 2025-06-24 21:21:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update data chain notebook

Browse Source
This commit is contained in:
florez_j
2025-04-08 21:26:36 +02:00
parent 881209bf82
commit 4725d85e61
1 changed files with 23 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
notebooks/demo_acsm_pipeline.ipynb
View File

@ -52,7 +52,13 @@
"from dima.pipelines.data_integration import run_pipeline as get_campaign_data\n",
"from pipelines.steps.apply_calibration_factors import main as apply_calibration_factors\n",
"from pipelines.steps.generate_flags import main as generate_flags\n",
"from pipelines.steps.prepare_ebas_submission import main as prepare_ebas_submission "
"from pipelines.steps.prepare_ebas_submission import main as prepare_ebas_submission \n",
"from pipelines.steps.update_actris_header import main as update_actris_header\n",
"from pipelines.steps.utils import load_project_yaml_files\n",
"\n",
"campaign_descriptor = load_project_yaml_files(project_path, \"campaignDescriptor.yaml\")\n",
"YEAR = campaign_descriptor['year']\n",
"STATION = campaign_descriptor['station']"
]
},
{
@ -74,10 +80,9 @@
"source": [
"path_to_config_file = '../campaignDescriptor.yaml'\n",
"paths_to_hdf5_files = get_campaign_data(path_to_config_file)\n",
"\n",
"# Select campaign data file and append directory\n",
"CAMPAIGN_DATA_FILE = paths_to_hdf5_files[0]\n",
"APPEND_DATA_DIR = os.path.splitext(CAMPAIGN_DATA_FILE)[0]"
"APPEND_DATA_DIR = os.path.splitext(CAMPAIGN_DATA_FILE)[0]\n"
]
},
{
@ -98,7 +103,7 @@
"source": [
"path_to_data_file = CAMPAIGN_DATA_FILE\n",
"path_to_calibration_file = '../pipelines/params/calibration_factors.yaml'\n",
"dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_timeseries.txt/data_table'\n",
"#dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_timeseries.txt/data_table'\n",
"#command = ['python', 'pipelines/steps/apply_calibration_factors.py', path_to_data_file, dataset_name, path_to_calibration_file]\n",
"#status = subprocess.run(command, capture_output=True, check=True)\n",
"#print(status.stdout.decode())\n",
@ -124,7 +129,7 @@
"metadata": {},
"outputs": [],
"source": [
"dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_meta.txt/data_table'\n",
"dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION}_{YEAR}_meta.txt/data_table'\n",
"path_to_config_file = 'pipelines/params/validity_thresholds.yaml'\n",
"#command = ['python', 'pipelines/steps/compute_automated_flags.py', path_to_data_file, dataset_name, path_to_config_file]\n",
"#status = subprocess.run(command, capture_output=True, check=True)\n",
@ -159,8 +164,9 @@
"metadata": {},
"outputs": [],
"source": [
"#CAMPAIGN_DATA_FILE = '../data/collection_JFJ_2024_2025-04-08_2025-04-08.h5'\n",
"path_to_data_file = CAMPAIGN_DATA_FILE\n",
"dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_meta.txt/data_table'\n",
"dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION}_{YEAR}_meta.txt/data_table'\n",
"path_to_config_file = 'pipelines/params/validity_thresholds.yaml'\n",
"#command = ['python', 'pipelines/steps/compute_automated_flags.py', path_to_data_file, dataset_name, path_to_config_file]\n",
"#status = subprocess.run(command, capture_output=True, check=True)\n",
@ -184,12 +190,17 @@
"metadata": {},
"outputs": [],
"source": [
"PATH1=\"../data/collection_JFJ_2024_2025-03-14_2025-03-14/ACSM_TOFWARE_processed/2024/ACSM_JFJ_2024_timeseries_calibrated.csv\"\n",
"PATH2=\"../data/collection_JFJ_2024_2025-03-14_2025-03-14/ACSM_TOFWARE_processed/2024/ACSM_JFJ_2024_timeseries_calibrated_err.csv\"\n",
"PATH3=\"../data/collection_JFJ_2024_2025-03-14_2025-03-14/ACSM_TOFWARE_processed/2024/ACSM_JFJ_2024_timeseries_calibration_factors.csv\"\n",
"PATH4=\"../data/collection_JFJ_2024_2025-03-14_2025-03-14/ACSM_TOFWARE_flags/2024/ACSM_JFJ_2024_timeseries_flags.csv\"\n",
"month = 4\n",
"prepare_ebas_submission([PATH1,PATH2,PATH3], PATH4, month)"
"DATA_DIR = f\"{APPEND_DATA_DIR}/ACSM_TOFWARE_processed/{YEAR}\"\n",
"FLAGS_DIR = f\"{APPEND_DATA_DIR}/ACSM_TOFWARE_flags/{YEAR}\"\n",
"\n",
"PATH1 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibrated.csv\"\n",
"PATH2 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibrated_err.csv\"\n",
"PATH3 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibration_factors.csv\"\n",
"PATH4 = f\"{FLAGS_DIR}/ACSM_{STATION}_{YEAR}_timeseries_flags.csv\"\n",
"\n",
"month = \"2-3\"\n",
"update_actris_header('../campaignDescriptor.yaml')\n",
"prepare_ebas_submission([PATH1, PATH2, PATH3], PATH4, month)\n"
]
},
{