diff --git a/notebooks/demo_acsm_pipeline.ipynb b/notebooks/demo_acsm_pipeline.ipynb index 0da54c6..54945c5 100644 --- a/notebooks/demo_acsm_pipeline.ipynb +++ b/notebooks/demo_acsm_pipeline.ipynb @@ -55,10 +55,11 @@ "from pipelines.steps.prepare_ebas_submission import main as prepare_ebas_submission \n", "from pipelines.steps.update_actris_header import main as update_actris_header\n", "from pipelines.steps.utils import load_project_yaml_files\n", + "from pipelines.steps.update_datachain_params import main as update_datachain_params\n", "\n", "campaign_descriptor = load_project_yaml_files(project_path, \"campaignDescriptor.yaml\")\n", "YEAR = campaign_descriptor['year']\n", - "STATION = campaign_descriptor['station']" + "STATION_ABBR = campaign_descriptor['station_abbr']" ] }, { @@ -85,13 +86,30 @@ "APPEND_DATA_DIR = os.path.splitext(CAMPAIGN_DATA_FILE)[0]\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1.1: Update Data Chain Parameters with Input Data\n", + "* Ensure the data folder retreived from the network drive contains a suitably specified folder `ACSM_TOFWARE//params`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "update_datachain_params(CAMPAIGN_DATA_FILE, 'ACSM_TOFWARE/2024')" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Step 2: Calibrate Input Campaign Data and Save Data Products\n", "\n", - "* Set up the input and output directory paths.\n", + "* Make sure the variable `CAMPAIGN_DATA_FILE` is properly defined in previous step. Otherwise, set the variable manually as indicated below.\n", "* Execute the cell." ] }, @@ -101,12 +119,11 @@ "metadata": {}, "outputs": [], "source": [ + "# Define manually path to data file by uncomenting the following line, and filling the path\n", + "\n", + "# CAMPAIGN_DATA_FILE = ../data/\n", "path_to_data_file = CAMPAIGN_DATA_FILE\n", "path_to_calibration_file = '../pipelines/params/calibration_factors.yaml'\n", - "#dataset_name = 'ACSM_TOFWARE/2024/ACSM_JFJ_2024_timeseries.txt/data_table'\n", - "#command = ['python', 'pipelines/steps/apply_calibration_factors.py', path_to_data_file, dataset_name, path_to_calibration_file]\n", - "#status = subprocess.run(command, capture_output=True, check=True)\n", - "#print(status.stdout.decode())\n", "\n", "apply_calibration_factors(path_to_data_file,path_to_calibration_file)\n" ] @@ -129,7 +146,7 @@ "metadata": {}, "outputs": [], "source": [ - "dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION}_{YEAR}_meta.txt/data_table'\n", + "dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION_ABBR}_{YEAR}_meta.txt/data_table'\n", "path_to_config_file = 'pipelines/params/validity_thresholds.yaml'\n", "#command = ['python', 'pipelines/steps/compute_automated_flags.py', path_to_data_file, dataset_name, path_to_config_file]\n", "#status = subprocess.run(command, capture_output=True, check=True)\n", @@ -166,7 +183,7 @@ "source": [ "#CAMPAIGN_DATA_FILE = '../data/collection_JFJ_2024_2025-04-08_2025-04-08.h5'\n", "path_to_data_file = CAMPAIGN_DATA_FILE\n", - "dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION}_{YEAR}_meta.txt/data_table'\n", + "dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION_ABBR}_{YEAR}_meta.txt/data_table'\n", "path_to_config_file = 'pipelines/params/validity_thresholds.yaml'\n", "#command = ['python', 'pipelines/steps/compute_automated_flags.py', path_to_data_file, dataset_name, path_to_config_file]\n", "#status = subprocess.run(command, capture_output=True, check=True)\n", @@ -193,10 +210,10 @@ "DATA_DIR = f\"{APPEND_DATA_DIR}/ACSM_TOFWARE_processed/{YEAR}\"\n", "FLAGS_DIR = f\"{APPEND_DATA_DIR}/ACSM_TOFWARE_flags/{YEAR}\"\n", "\n", - "PATH1 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibrated.csv\"\n", - "PATH2 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibrated_err.csv\"\n", - "PATH3 = f\"{DATA_DIR}/ACSM_{STATION}_{YEAR}_timeseries_calibration_factors.csv\"\n", - "PATH4 = f\"{FLAGS_DIR}/ACSM_{STATION}_{YEAR}_timeseries_flags.csv\"\n", + "PATH1 = f\"{DATA_DIR}/ACSM_{STATION_ABBR}_{YEAR}_timeseries_calibrated.csv\"\n", + "PATH2 = f\"{DATA_DIR}/ACSM_{STATION_ABBR}_{YEAR}_timeseries_calibrated_err.csv\"\n", + "PATH3 = f\"{DATA_DIR}/ACSM_{STATION_ABBR}_{YEAR}_timeseries_calibration_factors.csv\"\n", + "PATH4 = f\"{FLAGS_DIR}/ACSM_{STATION_ABBR}_{YEAR}_timeseries_flags.csv\"\n", "\n", "month = \"2-3\"\n", "update_actris_header('../campaignDescriptor.yaml')\n",