diff --git a/workflow_data_integration.ipynb b/workflow_data_integration.ipynb index 9c4f775..830155c 100644 --- a/workflow_data_integration.ipynb +++ b/workflow_data_integration.ipynb @@ -19,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -29,9 +29,12 @@ "root_dir = os.path.abspath(os.curdir)\n", "sys.path.append(root_dir)\n", "sys.path.append(os.path.join(root_dir,'dima'))\n", + "#sys.path.append(os.path.join(root_dir,'dima','instruments'))\n", + "#sys.path.append(os.path.join(root_dir,'dima','src'))\n", + "#sys.path.append(os.path.join(root_dir,'dima','utils'))\n", "\n", "import dima.src.hdf5_vis as hdf5_vis\n", - "import dima.src.data_integration_lib as dilib\n" + "import dima.pipelines.data_integration as dilib\n" ] }, { @@ -47,11 +50,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "yaml_config_file_path = 'dima_config.yaml'\n" + "yaml_config_file_path = 'dima_config.yaml'" ] }, { @@ -104,13 +107,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "import dima.src.metadata_review_lib as metadata\n", + "import dima.pipelines.metadata_revision as metadata\n", "\n", - "import dima.src.hdf5_data_extraction as h5de\n", + "import dima.src.hdf5_ops as h5de\n", "\n", "channels1 = ['Chl_11000','NH4_11000','SO4_11000','NO3_11000','Org_11000']\n", "channels2 = ['FilamentEmission_mA','VaporizerTemp_C','FlowRate_mb','ABsamp']\n", @@ -121,14 +124,15 @@ "diagnostic_channels = {'location':'ACSM_TOFWARE/ACSM_JFJ_2024_JantoFeb_meta.txt/data_table',\n", " 'names': ','.join(['t_base','FilamentEmission_mA','VaporizerTemp_C','FlowRate_mb','ABsamp'])}\n", "\n", - "DataOpsAPI = h5de.HDF5DataOpsManager(hdf5_file_path)\n", + "DataOpsAPI = h5de.HDF5DataOpsManager(hdf5_file_path[0])\n", "\n", - "DataOpsAPI.append_annotations('/',{'target_channels' : target_channels, 'diagnostic_channels' : diagnostic_channels})\n", + "DataOpsAPI.load_file_obj()\n", + "DataOpsAPI.append_metadata('/ACSM_TOFWARE/',{'target_channels' : target_channels, 'diagnostic_channels' : diagnostic_channels})\n", "\n", "DataOpsAPI.reformat_datetime_column('ACSM_TOFWARE/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table','t_start_Buf',src_format='%d.%m.%Y %H:%M:%S.%f')\n", "DataOpsAPI.reformat_datetime_column('ACSM_TOFWARE/ACSM_JFJ_2024_JantoFeb_meta.txt/data_table','t_base',src_format='%d.%m.%Y %H:%M:%S')\n", "\n", - "DataOpsAPI.close_file()" + "DataOpsAPI.unload_file_obj()" ] }, { diff --git a/workflow_test_flaggingapp_manager.ipynb b/workflow_test_flaggingapp_manager.ipynb new file mode 100644 index 0000000..0a61719 --- /dev/null +++ b/workflow_test_flaggingapp_manager.ipynb @@ -0,0 +1,115 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "# Set up project root directory\n", + "root_dir = os.path.abspath(os.curdir)\n", + "sys.path.append(root_dir)\n", + "sys.path.append(os.path.join(root_dir,'dima'))\n", + "\n", + "import data_flagging_utils as utils\n", + "\n", + "DataOpsObj = utils.FlaggingAppDataManager('data_products/collection_acsm_campaign_NoraN_2024-01-01_2024-02-29.h5')\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "path_to_append_dir = 'data_products/collection_acsm_campaign_NoraN_2024-01-01_2024-02-29'\n", + "DataOpsObj.load_file_obj()\n", + "DataOpsObj.transfer_flags()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "DataOpsObj.load_file_obj()\n", + "DataOpsObj._data_ops_obj.extract_and_load_dataset_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "DataOpsObj._data_ops_obj.dataset_metadata_df.head(n=15)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "DataOpsObj.load_file_obj()\n", + "DataOpsObj.apply_flags('ACSM_TOFWARE')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "DataOpsObj._data_ops_obj.unload_file_obj()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "DataOpsObj.load_file_obj()\n", + "DataOpsObj._data_ops_obj.append_metadata('ACSM_TOFWARE__cleaned/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table',{ 'creation_date' : b'2024-10-09_15-33-54_UTC-OFST_+0200' })\n", + "DataOpsObj._data_ops_obj.unload_file_obj()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "DataOpsObj.load_file_obj()\n", + "DataOpsObj._data_ops_obj.delete_metadata('ACSM_TOFWARE__cleaned/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table',{ 'created_at' : {'delete': True }})\n", + "DataOpsObj._data_ops_obj.unload_file_obj()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "multiphase_chemistry_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}