mirror of
https://gitea.psi.ch/APOG/acsmnode.git
synced 2025-07-03 14:40:47 +02:00
Rename and move jupyter notebook demos to notebooks folder
This commit is contained in:
167
notebooks/demo_data_integration.ipynb
Normal file
167
notebooks/demo_data_integration.ipynb
Normal file
@ -0,0 +1,167 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Data integration workflow of experimental campaign\n",
|
||||
"\n",
|
||||
"In this notebook, we will go through a our data integration workflow. This involves the following steps:\n",
|
||||
"\n",
|
||||
"1. Specify data integration file through YAML configuration file.\n",
|
||||
"2. Create an integrated HDF5 file of experimental campaign from configuration file.\n",
|
||||
"3. Display the created HDF5 file using a treemap\n",
|
||||
"\n",
|
||||
"## Import libraries and modules\n",
|
||||
"\n",
|
||||
"* Excecute (or Run) the Cell below"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"# Set up project root directory\n",
|
||||
"root_dir = os.path.abspath(os.curdir)\n",
|
||||
"sys.path.append(root_dir)\n",
|
||||
"sys.path.append(os.path.join(root_dir,'dima'))\n",
|
||||
"#sys.path.append(os.path.join(root_dir,'dima','instruments'))\n",
|
||||
"#sys.path.append(os.path.join(root_dir,'dima','src'))\n",
|
||||
"#sys.path.append(os.path.join(root_dir,'dima','utils'))\n",
|
||||
"\n",
|
||||
"import dima.src.hdf5_vis as hdf5_vis\n",
|
||||
"import dima.pipelines.data_integration as dilib\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 1: Specify data integration task through YAML configuration file\n",
|
||||
"\n",
|
||||
"* Create your configuration file (i.e., *.yaml file) adhering to the example yaml file in the input folder.\n",
|
||||
"* Set up input directory and output directory paths and Excecute Cell.\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"yaml_config_file_path = 'dima_config.yaml'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Create an integrated HDF5 file of experimental campaign.\n",
|
||||
"\n",
|
||||
"* Excecute Cell. Here we run the function `integrate_data_sources` with input argument as the previously specified YAML config file."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"hdf5_file_path = dilib.integrate_data_sources(yaml_config_file_path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Display integrated HDF5 file using a treemap\n",
|
||||
"\n",
|
||||
"* Excecute Cell. A visual representation in html format of the integrated file should be displayed and stored in the output directory folder"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if isinstance(hdf5_file_path ,list):\n",
|
||||
" for path_item in hdf5_file_path :\n",
|
||||
" hdf5_vis.display_group_hierarchy_on_a_treemap(path_item)\n",
|
||||
"else:\n",
|
||||
" hdf5_vis.display_group_hierarchy_on_a_treemap(hdf5_file_path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import dima.pipelines.metadata_revision as metadata\n",
|
||||
"\n",
|
||||
"import dima.src.hdf5_ops as h5de\n",
|
||||
"\n",
|
||||
"channels1 = ['Chl_11000','NH4_11000','SO4_11000','NO3_11000','Org_11000']\n",
|
||||
"channels2 = ['FilamentEmission_mA','VaporizerTemp_C','FlowRate_mb','ABsamp']\n",
|
||||
"\n",
|
||||
"target_channels = {'location':'ACSM_TOFWARE/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table',\n",
|
||||
" 'names': ','.join(['t_start_Buf','Chl_11000','NH4_11000','SO4_11000','NO3_11000','Org_11000'])\n",
|
||||
" }\n",
|
||||
"diagnostic_channels = {'location':'ACSM_TOFWARE/ACSM_JFJ_2024_JantoFeb_meta.txt/data_table',\n",
|
||||
" 'names': ','.join(['t_base','FilamentEmission_mA','VaporizerTemp_C','FlowRate_mb','ABsamp'])}\n",
|
||||
"\n",
|
||||
"DataOpsAPI = h5de.HDF5DataOpsManager(hdf5_file_path[0])\n",
|
||||
"\n",
|
||||
"DataOpsAPI.load_file_obj()\n",
|
||||
"DataOpsAPI.append_metadata('/ACSM_TOFWARE/',{'target_channels' : target_channels, 'diagnostic_channels' : diagnostic_channels})\n",
|
||||
"\n",
|
||||
"DataOpsAPI.reformat_datetime_column('ACSM_TOFWARE/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table','t_start_Buf',src_format='%d.%m.%Y %H:%M:%S.%f')\n",
|
||||
"DataOpsAPI.reformat_datetime_column('ACSM_TOFWARE/ACSM_JFJ_2024_JantoFeb_meta.txt/data_table','t_base',src_format='%d.%m.%Y %H:%M:%S')\n",
|
||||
"\n",
|
||||
"DataOpsAPI.unload_file_obj()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "multiphase_chemistry_env",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
115
notebooks/demo_flaggingapp_manager.ipynb
Normal file
115
notebooks/demo_flaggingapp_manager.ipynb
Normal file
@ -0,0 +1,115 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"# Set up project root directory\n",
|
||||
"root_dir = os.path.abspath(os.curdir)\n",
|
||||
"sys.path.append(root_dir)\n",
|
||||
"sys.path.append(os.path.join(root_dir,'dima'))\n",
|
||||
"\n",
|
||||
"import data_flagging_utils as utils\n",
|
||||
"\n",
|
||||
"DataOpsObj = utils.FlaggingAppDataManager('data_products/collection_acsm_campaign_NoraN_2024-01-01_2024-02-29.h5')\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_append_dir = 'data_products/collection_acsm_campaign_NoraN_2024-01-01_2024-02-29'\n",
|
||||
"DataOpsObj.load_file_obj()\n",
|
||||
"DataOpsObj.transfer_flags()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DataOpsObj.load_file_obj()\n",
|
||||
"DataOpsObj._data_ops_obj.extract_and_load_dataset_metadata()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DataOpsObj._data_ops_obj.dataset_metadata_df.head(n=15)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DataOpsObj.load_file_obj()\n",
|
||||
"DataOpsObj.apply_flags('ACSM_TOFWARE')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DataOpsObj._data_ops_obj.unload_file_obj()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DataOpsObj.load_file_obj()\n",
|
||||
"DataOpsObj._data_ops_obj.append_metadata('ACSM_TOFWARE__cleaned/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table',{ 'creation_date' : b'2024-10-09_15-33-54_UTC-OFST_+0200' })\n",
|
||||
"DataOpsObj._data_ops_obj.unload_file_obj()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DataOpsObj.load_file_obj()\n",
|
||||
"DataOpsObj._data_ops_obj.delete_metadata('ACSM_TOFWARE__cleaned/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table',{ 'created_at' : {'delete': True }})\n",
|
||||
"DataOpsObj._data_ops_obj.unload_file_obj()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "multiphase_chemistry_env",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user