diff --git a/notebooks/demo_acsm_pipeline.ipynb b/notebooks/demo_acsm_pipeline.ipynb index 50e5887..0da54c6 100644 --- a/notebooks/demo_acsm_pipeline.ipynb +++ b/notebooks/demo_acsm_pipeline.ipynb @@ -52,7 +52,13 @@ "from dima.pipelines.data_integration import run_pipeline as get_campaign_data\n", "from pipelines.steps.apply_calibration_factors import main as apply_calibration_factors\n", "from pipelines.steps.generate_flags import main as generate_flags\n", - "from pipelines.steps.prepare_ebas_submission import main as prepare_ebas_submission " + "from pipelines.steps.prepare_ebas_submission import main as prepare_ebas_submission \n", + "from pipelines.steps.update_actris_header import main as update_actris_header\n", + "from pipelines.steps.utils import load_project_yaml_files\n", + "\n", + "campaign_descriptor = load_project_yaml_files(project_path, \"campaignDescriptor.yaml\")\n", + "YEAR = campaign_descriptor['year']\n", + "STATION = campaign_descriptor['station']" ] }, { @@ -74,10 +80,9 @@ "source": [ "path_to_config_file = '../campaignDescriptor.yaml'\n", "paths_to_hdf5_files = get_campaign_data(path_to_config_file)\n", - "\n", "# Select campaign data file and append directory\n", "CAMPAIGN_DATA_FILE = paths_to_hdf5_files[0]\n", - "APPEND_DATA_DIR = os.path.splitext(CAMPAIGN_DATA_FILE)[0]" + "APPEND_DATA_DIR = os.path.splitext(CAMPAIGN_DATA_FILE)[0]\n" ] }, { @@ -98,7 +103,7 @@ "source": [ "path_to_data_file = CAMPAIGN_DATA_FILE\n", "path_to_calibration_file = '../pipelines/params/calibration_factors.yaml'\n", - "dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_timeseries.txt/data_table'\n", + "#dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_timeseries.txt/data_table'\n", "#command = ['python', 'pipelines/steps/apply_calibration_factors.py', path_to_data_file, dataset_name, path_to_calibration_file]\n", "#status = subprocess.run(command, capture_output=True, check=True)\n", "#print(status.stdout.decode())\n", @@ -124,7 +129,7 @@ "metadata": {}, "outputs": [], "source": [ - "dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_meta.txt/data_table'\n", + "dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION}_{YEAR}_meta.txt/data_table'\n", "path_to_config_file = 'pipelines/params/validity_thresholds.yaml'\n", "#command = ['python', 'pipelines/steps/compute_automated_flags.py', path_to_data_file, dataset_name, path_to_config_file]\n", "#status = subprocess.run(command, capture_output=True, check=True)\n", @@ -159,8 +164,9 @@ "metadata": {}, "outputs": [], "source": [ + "#CAMPAIGN_DATA_FILE = '../data/collection_JFJ_2024_2025-04-08_2025-04-08.h5'\n", "path_to_data_file = CAMPAIGN_DATA_FILE\n", - "dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_meta.txt/data_table'\n", + "dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION}_{YEAR}_meta.txt/data_table'\n", "path_to_config_file = 'pipelines/params/validity_thresholds.yaml'\n", "#command = ['python', 'pipelines/steps/compute_automated_flags.py', path_to_data_file, dataset_name, path_to_config_file]\n", "#status = subprocess.run(command, capture_output=True, check=True)\n", @@ -184,12 +190,17 @@ "metadata": {}, "outputs": [], "source": [ - "PATH1=\"../data/collection_JFJ_2024_2025-03-14_2025-03-14/ACSM_TOFWARE_processed/2024/ACSM_JFJ_2024_timeseries_calibrated.csv\"\n", - "PATH2=\"../data/collection_JFJ_2024_2025-03-14_2025-03-14/ACSM_TOFWARE_processed/2024/ACSM_JFJ_2024_timeseries_calibrated_err.csv\"\n", - "PATH3=\"../data/collection_JFJ_2024_2025-03-14_2025-03-14/ACSM_TOFWARE_processed/2024/ACSM_JFJ_2024_timeseries_calibration_factors.csv\"\n", - "PATH4=\"../data/collection_JFJ_2024_2025-03-14_2025-03-14/ACSM_TOFWARE_flags/2024/ACSM_JFJ_2024_timeseries_flags.csv\"\n", - "month = 4\n", - "prepare_ebas_submission([PATH1,PATH2,PATH3], PATH4, month)" + "DATA_DIR = f\"{APPEND_DATA_DIR}/ACSM_TOFWARE_processed/{YEAR}\"\n", + "FLAGS_DIR = f\"{APPEND_DATA_DIR}/ACSM_TOFWARE_flags/{YEAR}\"\n", + "\n", + "PATH1 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibrated.csv\"\n", + "PATH2 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibrated_err.csv\"\n", + "PATH3 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibration_factors.csv\"\n", + "PATH4 = f\"{FLAGS_DIR}/ACSM_{STATION}_{YEAR}_timeseries_flags.csv\"\n", + "\n", + "month = \"2-3\"\n", + "update_actris_header('../campaignDescriptor.yaml')\n", + "prepare_ebas_submission([PATH1, PATH2, PATH3], PATH4, month)\n" ] }, {