APOG_public/acsmnode
0
0
Fork 0
mirror of https://gitea.psi.ch/APOG/acsmnode.git synced 2025-06-24 13:11:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

update notebooks after running them with 2024 pay data

Browse Source
This commit is contained in:
florez_j
2025-06-06 17:21:01 +02:00
parent 5d84d726b6
commit bb9625cd34
2 changed files with 516385 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

284
notebooks/demo_acsm_pipeline.ipynb
View File

@ -20,9 +20,36 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\dima\n",
"c:\\Users\\florez_j\\Documents\\Gitlab\\ecdataobjstore\\envs\\multiphase_chem_env\\python311.zip\n",
"c:\\Users\\florez_j\\Documents\\Gitlab\\ecdataobjstore\\envs\\multiphase_chem_env\\DLLs\n",
"c:\\Users\\florez_j\\Documents\\Gitlab\\ecdataobjstore\\envs\\multiphase_chem_env\\Lib\n",
"c:\\Users\\florez_j\\Documents\\Gitlab\\ecdataobjstore\\envs\\multiphase_chem_env\n",
"\n",
"c:\\Users\\florez_j\\Documents\\Gitlab\\ecdataobjstore\\envs\\multiphase_chem_env\\Lib\\site-packages\n",
"c:\\Users\\florez_j\\Documents\\Gitlab\\ecdataobjstore\\envs\\multiphase_chem_env\\Lib\\site-packages\\win32\n",
"c:\\Users\\florez_j\\Documents\\Gitlab\\ecdataobjstore\\envs\\multiphase_chem_env\\Lib\\site-packages\\win32\\lib\n",
"c:\\Users\\florez_j\\Documents\\Gitlab\\ecdataobjstore\\envs\\multiphase_chem_env\\Lib\\site-packages\\Pythonwin\n",
"c:\\Users\\florez_j\\Documents\\Gitlab\\ecdataobjstore\\envs\\multiphase_chem_env\\Lib\\site-packages\\setuptools\\_vendor\n",
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\n",
"File path: c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\pipelines\\steps\\apply_calibration_factors.py\n",
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\pipelines\\steps\\generate_flags.py\n",
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\pipelines\\steps\\prepare_ebas_submission.py\n",
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\pipelines\\steps\\update_actris_header.py\n",
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\pipelines\\steps\\update_datachain_params.py\n",
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\pipelines\\steps\\drop_column_from_nas_file.py\n",
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\pipelines\\steps\\adjust_uncertainty_column_in_nas_file.py\n",
"workflow_acsm_data_PAY_2024\n"
]
}
],
"source": [
"import sys\n",
"import os\n",
@ -81,9 +108,43 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"[Start] Data integration :\n",
"Source: ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06\n",
"Destination: ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06.h5\n",
"\n",
"Starting data transfer from instFolder: /ACSM_TOFWARE/2024\n",
"[==================================================--------------------------------------------------] 50.0% ...\n",
"Completed data transfer for instFolder: /ACSM_TOFWARE/2024\n",
"Completed transfer for //ACSM_TOFWARE/2024/ACSM_PAY_2024_meta.txt\n",
"Completed transfer for //ACSM_TOFWARE/2024/ACSM_PAY_2024_timeseries.txt\n",
"Completed transfer for //ACSM_TOFWARE/2024/Org_data_valid.csv\n",
"Completed transfer for //ACSM_TOFWARE/2024/Org_err_valid.csv\n",
"Completed transfer for //ACSM_TOFWARE/2024/Org_mz_valid.csv\n",
"Completed transfer for //ACSM_TOFWARE/2024/Org_time_valid.csv\n",
"[====================================================================================================] 100.0% ...\n",
"Completed data transfer for instFolder: /ACSM_TOFWARE/2024\n",
"[End] Data integration\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\dima\\instruments\\readers\\acsm_tofware_reader.py:112: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
" df = pd.read_csv(tmp_filename,\n",
"c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\dima\\instruments\\readers\\acsm_tofware_reader.py:112: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
" df = pd.read_csv(tmp_filename,\n"
]
}
],
"source": [
"path_to_config_file = '../campaignDescriptor.yaml'\n",
"paths_to_hdf5_files = get_campaign_data(path_to_config_file)\n",
@ -117,9 +178,30 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Synchronized: calibration_params.yaml\n",
"Synchronized: limits_of_detection.yaml\n",
"Synchronized: validity_thresholds.yaml\n",
"[Skipping] Step 'update_datachain_params' already exists. Use 'force=True' to overwrite.\n"
]
},
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"update_datachain_params(CAMPAIGN_DATA_FILE, 'ACSM_TOFWARE/2024', capture_renku_metadata=True, workflow_name=workflow_fname)"
]
@ -136,9 +218,69 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Opening data file: ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06.h5 using src.hdf5_ops.HDF5DataOpsManager().\n",
" dataset_name parent_instrument \\\n",
"0 ACSM_TOFWARE/2024/ACSM_PAY_2024_meta.txt/data_... ACSM_TOFWARE/2024 \n",
"1 ACSM_TOFWARE/2024/ACSM_PAY_2024_timeseries.txt... ACSM_TOFWARE/2024 \n",
"2 ACSM_TOFWARE/2024/Org_data_valid.csv/data_table ACSM_TOFWARE/2024 \n",
"3 ACSM_TOFWARE/2024/Org_err_valid.csv/data_table ACSM_TOFWARE/2024 \n",
"4 ACSM_TOFWARE/2024/Org_mz_valid.csv/data_table ACSM_TOFWARE/2024 \n",
"\n",
" parent_file \n",
"0 ACSM_PAY_2024_meta.txt \n",
"1 ACSM_PAY_2024_timeseries.txt \n",
"2 Org_data_valid.csv \n",
"3 Org_err_valid.csv \n",
"4 Org_mz_valid.csv \n",
"ACSM_PAY_2024_timeseries.txt\n",
"../pipelines/params/calibration_factors.yaml\n",
"Closing data file: ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06.h5 to unlock the file.\n",
"Total rows: 63742\n",
"NaT (missing) values: 0\n",
"Percentage of data loss: 0.0000%\n",
"Output directory: ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06\\ACSM_TOFWARE_processed\\2024\n",
"t_start_Buf 63742 23\n",
"NO3_11000\n",
"SO4_11000\n",
"NH4_11000\n",
"Org_11000\n",
"Chl_11000\n",
"Org_44_11000\n",
"Org_43_11000\n",
"Org_60_11000\n",
"NO3_30_11000\n",
"SO4_98_11000\n",
"SO4_81_11000\n",
"SO4_82_11000\n",
"SO4_62_11000\n",
"SO4_48_11000\n",
"Saved ACSM_PAY_2024_timeseries_calibrated.csv to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06\\ACSM_TOFWARE_processed\\2024\n",
"Metadata for calibrated data saved to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06\\ACSM_TOFWARE_processed\\2024/data_lineage_metadata.json\n",
"Saved ACSM_PAY_2024_timeseries_calibrated_err.csv to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06\\ACSM_TOFWARE_processed\\2024\n",
"Metadata for calibrated data saved to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06\\ACSM_TOFWARE_processed\\2024/data_lineage_metadata.json\n",
"Saved ACSM_PAY_2024_timeseries_calibration_factors.csv to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06\\ACSM_TOFWARE_processed\\2024\n",
"Metadata for calibrated data saved to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06\\ACSM_TOFWARE_processed\\2024/data_lineage_metadata.json\n",
"[Skipping] Step 'apply_calibration_factors' already exists. Use 'force=True' to overwrite.\n"
]
},
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Define manually path to data file by uncomenting the following line, and filling the path\n",
"\n",
@ -163,9 +305,44 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Total rows: 63742\n",
"NaT (missing) values: 0\n",
"Percentage of data loss: 0.0000%\n",
"Starting flag generation.\n",
"Processing script: pipelines\\steps\\generate_flags.py\n",
"Output directory: ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_flags/2024\n",
"Unspecified validity thresholds for variable t_base. If needed, update pipelines/params/validity_thresholds.yaml accordingly.\n",
"Unspecified validity thresholds for variable HeaterBias_V. If needed, update pipelines/params/validity_thresholds.yaml accordingly.\n",
"Unspecified validity thresholds for variable FlowRefWave. If needed, update pipelines/params/validity_thresholds.yaml accordingly.\n",
"Unspecified validity thresholds for variable FlowRate_mb. If needed, update pipelines/params/validity_thresholds.yaml accordingly.\n",
"Unspecified validity thresholds for variable Detector_V. If needed, update pipelines/params/validity_thresholds.yaml accordingly.\n",
"Unspecified validity thresholds for variable AnalogInput06_V. If needed, update pipelines/params/validity_thresholds.yaml accordingly.\n",
"Unspecified validity thresholds for variable ABRefWave. If needed, update pipelines/params/validity_thresholds.yaml accordingly.\n",
"Unspecified validity thresholds for variable ABCorrFact. If needed, update pipelines/params/validity_thresholds.yaml accordingly.\n",
"Metadata for calibrated data saved to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_flags/2024/data_lineage_metadata.json\n",
"Flags saved to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_flags/2024/ACSM_PAY_2024_meta_flags.csv\n",
"Data lineage saved to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_flags/2024\n",
"[Skipping] Step 'generate_flags_diagnostics' already exists. Use 'force=True' to overwrite.\n"
]
},
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset_name = f'ACSM_TOFWARE/{YEAR}/ACSM_{STATION_ABBR}_{YEAR}_meta.txt/data_table'\n",
"path_to_config_file = 'pipelines/params/validity_thresholds.yaml'\n",
@ -179,9 +356,27 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Error loading input files: cpc file is not uniquely identifiable: Series([], Name: parent_file, dtype: object)\n"
]
},
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"generate_flags(path_to_data_file, 'cpc', capture_renku_metadata=True, workflow_name=workflow_fname)"
@ -210,9 +405,38 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Total rows: 63742\n",
"NaT (missing) values: 0\n",
"Percentage of data loss: 0.0000%\n",
"Starting flag generation.\n",
"Processing script: pipelines\\steps\\generate_flags.py\n",
"Output directory: ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_flags/2024\n",
"Retreiving species to be flagged ...\n",
"Species to be flagged are: ['Chl_11000', 'NH4_11000', 'SO4_11000', 'NO3_11000', 'Org_11000']. If needed, update pipelines/params/calibration_params.yaml\n",
"Metadata for calibrated data saved to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_flags/2024/data_lineage_metadata.json\n",
"Flags saved to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_flags/2024/ACSM_PAY_2024_timeseries_flags.csv\n",
"Data lineage saved to ..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_flags/2024\n",
"[Skipping] Step 'generate_flags_species' already exists. Use 'force=True' to overwrite.\n"
]
},
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#CAMPAIGN_DATA_FILE = '../data/collection_JFJ_2024_2025-04-08_2025-04-08.h5'\n",
"path_to_data_file = CAMPAIGN_DATA_FILE\n",
@ -236,9 +460,37 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"..\\data\\collection_PAY_2024_2025-06-06_2025-06-06\n",
"..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_processed/2024/ACSM_PAY_2024_timeseries_calibrated.csv True\n",
"..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_processed/2024/ACSM_PAY_2024_timeseries_calibrated_err.csv True\n",
"..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_processed/2024/ACSM_PAY_2024_timeseries_calibration_factors.csv True\n",
"..\\data\\collection_PAY_2024_2025-06-06_2025-06-06/ACSM_TOFWARE_flags/2024/ACSM_PAY_2024_timeseries_flags.csv True\n",
"{'originator': 'Simon, Leïla, leila.simon@psi.ch, Paul Scherrer Institute, PSI, Laboratory of Atmospheric Chemistry, , , 5232, Villigen PSI, Switzerland', 'submitter': 'Simon, Leïla, leila.simon@psi.ch, Paul Scherrer Institute, PSI, Laboratory of Atmospheric Chemistry, , , 5232, Villigen PSI, Switzerland', 'station_abbr': 'PAY', 'originator_name': 'Simon, Leïla', 'submitter_name': 'Simon, Leïla'}\n",
"Using template: c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\pipelines/actris_header/PAY_ACSM_092.actris_header\n",
"[LIVE RUN] Target header will be updated.\n",
"Writing to: c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\third_party/acsmProcessingSoftware/src/cfg/actris_header/PAY_ACSM_092.actris_header\n",
"Total rows: 12479\n",
"NaT (missing) values: 0\n",
"Percentage of data loss: 0.00%\n",
"Total rows: 12479\n",
"NaT (missing) values: 0\n",
"Percentage of data loss: 0.00%\n",
"> processing ['c:\\\\Users\\\\florez_j\\\\Documents\\\\Gitea\\\\acsmnode\\\\data\\\\PAY_ACSM-092_2024.txt']\n",
"> processing C:\\Users\\florez_j\\AppData\\Local\\Temp\\tmp4b4ladgn\\CH0002G.20240201000309.20250606151517.aerosol_mass_spectrometer.chemistry_ACSM.pm1_non_refractory.7w.4mn.CH02L_Aerodyne_ToF-ACSM_092.CH02L_Aerodyne_ToF-ACSM_PAY.lev0.nas\n",
"> processing C:\\Users\\florez_j\\AppData\\Local\\Temp\\tmp4b4ladgn\\CH0002G.20240201000309.20250606151524.aerosol_mass_spectrometer.chemistry_ACSM.pm1_non_refractory.7w.4mn.CH02L_Aerodyne_ToF-ACSM_092.CH02L_Aerodyne_ToF-ACSM_PAY.lev0a.nas\n",
"> processing C:\\Users\\florez_j\\AppData\\Local\\Temp\\tmp4b4ladgn\\CH0002G.20240201000309.20250606151529.aerosol_mass_spectrometer.chemistry_ACSM.pm1_non_refractory.7w.4mn.CH02L_Aerodyne_ToF-ACSM_092.CH02L_Aerodyne_ToF-ACSM_PAY.lev1.nas\n",
"> move into c:\\Users\\florez_j\\Documents\\Gitea\\acsmnode\\data\n",
"[Skipping] Step 'workflow_acsm_data_PAY_2024_step' already exists. Use 'force=True' to overwrite.\n"
]
}
],
"source": [
"import warnings\n",
"print(APPEND_DATA_DIR)\n",

516126
notebooks/demo_visualize_diagnostic_flags_from_hdf5_file.ipynb
View File

File diff suppressed because it is too large Load Diff