Add new demo for acsm pipeline and clearout the other notebooks

This commit is contained in:
2025-02-10 16:15:56 +01:00
parent 3c482c1d21
commit e7ad4d2a36
3 changed files with 421 additions and 18 deletions

View File

@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -15,7 +15,8 @@
"\n",
"import data_flagging_utils as utils\n",
"\n",
"DataOpsObj = utils.FlaggingAppDataManager('data_products/collection_acsm_campaign_NoraN_2024-01-01_2024-02-29.h5')\n",
"path_to_file = 'data_products/collection_JFJ_2024_acsm_campaign_NoraN_2024-01-01_2024-02-29.h5'\n",
"DataOpsObj = utils.FlaggingAppDataManager(path_to_file)\n",
"\n"
]
},
@ -25,14 +26,105 @@
"metadata": {},
"outputs": [],
"source": [
"path_to_append_dir = 'data_products/collection_acsm_campaign_NoraN_2024-01-01_2024-02-29'\n",
"import sys\n",
"import os\n",
"import pandas as pd\n",
"# Set up project root directory\n",
"root_dir = os.path.abspath(os.curdir)\n",
"sys.path.append(root_dir)\n",
"sys.path.append(os.path.join(root_dir,'dima'))\n",
"\n",
"\n",
"from pipelines.steps.apply_calibration_factors import apply_calibration_factors\n",
"\n",
"import dima.src.hdf5_ops as dataOps\n",
"\n",
"file_path = \"data/collection_JFJ_2024_LeilaS_2025-02-03_2025-02-03.h5\"\n",
"dataOpsManager = dataOps.HDF5DataOpsManager(file_path)\n",
"\n",
"\n",
"dataOpsManager.load_file_obj()\n",
"dataOpsManager.extract_and_load_dataset_metadata()\n",
"print(dataOpsManager.dataset_metadata_df.head())\n",
"#dataOpsManager.unload_file_obj\n",
"\n",
"#dataOpsManager.unload_file_obj()\n",
"dataset_name = \"ACSM_TOFWARE/2024/ACSM_JFJ_2024_timeseries.txt/data_table\"\n",
"data_table = dataOpsManager.extract_dataset_as_dataframe(dataset_name)\n",
"datetime_var, datetime_format = dataOpsManager.infer_datetime_variable(dataset_name)\n",
"\n",
"#data_table['t_start_Buf'] = data_table['t_start_Buf'].apply(lambda x : x.decode())\n",
"dataOpsManager.unload_file_obj()\n",
"\n",
"a, b = apply_calibration_factors(data_table, datetime_var,'pipelines/params/calibration_factors.yaml')\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"a.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"apply_calib"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"import os\n",
"# Set up project root directory\n",
"root_dir = os.path.abspath(os.curdir)\n",
"sys.path.append(root_dir)\n",
"sys.path.append(os.path.join(root_dir,'dima'))\n",
"\n",
"import dima.src.hdf5_ops as data_ops\n",
"\n",
"path_to_file = 'data/collection_JFJ_2024_NoraN_2024-01-01_2024-02-29.h5'\n",
"DataOpsObj = data_ops.HDF5DataOpsManager(path_to_file)\n",
"\n",
"DataOpsObj.load_file_obj()\n",
"data_table = DataOpsObj.extract_dataset_as_dataframe('/ACSM_TOFWARE/2024/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table')\n",
"DataOpsObj.unload_file_obj()\n",
"\n",
"data_table.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"path_to_append_dir = 'data_products/collection_JFJ_2024_acsm_campaign_NoraN_2024-01-01_2024-02-29'\n",
"DataOpsObj.load_file_obj()\n",
"DataOpsObj.transfer_flags()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -61,7 +153,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -70,7 +162,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -81,7 +173,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -93,7 +185,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "multiphase_chemistry_env",
"display_name": "flaggingapp_env",
"language": "python",
"name": "python3"
},
@ -107,7 +199,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.9"
"version": "3.10.15"
}
},
"nbformat": 4,