acsm-fairifier/notebooks/demo_flaggingapp_manager.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "# Set up project root directory\n",
    "root_dir = os.path.abspath(os.curdir)\n",
    "sys.path.append(root_dir)\n",
    "sys.path.append(os.path.join(root_dir,'dima'))\n",
    "\n",
    "import data_flagging_utils as utils\n",
    "\n",
    "path_to_file = 'data_products/collection_JFJ_2024_acsm_campaign_NoraN_2024-01-01_2024-02-29.h5'\n",
    "DataOpsObj = utils.FlaggingAppDataManager(path_to_file)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import pandas as pd\n",
    "# Set up project root directory\n",
    "root_dir = os.path.abspath(os.curdir)\n",
    "sys.path.append(root_dir)\n",
    "sys.path.append(os.path.join(root_dir,'dima'))\n",
    "\n",
    "\n",
    "from pipelines.steps.apply_calibration_factors import apply_calibration_factors\n",
    "\n",
    "import dima.src.hdf5_ops as dataOps\n",
    "\n",
    "file_path = \"data/collection_JFJ_2024_LeilaS_2025-02-03_2025-02-03.h5\"\n",
    "dataOpsManager = dataOps.HDF5DataOpsManager(file_path)\n",
    "\n",
    "\n",
    "dataOpsManager.load_file_obj()\n",
    "dataOpsManager.extract_and_load_dataset_metadata()\n",
    "print(dataOpsManager.dataset_metadata_df.head())\n",
    "#dataOpsManager.unload_file_obj\n",
    "\n",
    "#dataOpsManager.unload_file_obj()\n",
    "dataset_name = \"ACSM_TOFWARE/2024/ACSM_JFJ_2024_timeseries.txt/data_table\"\n",
    "data_table = dataOpsManager.extract_dataset_as_dataframe(dataset_name)\n",
    "datetime_var, datetime_format = dataOpsManager.infer_datetime_variable(dataset_name)\n",
    "\n",
    "#data_table['t_start_Buf'] = data_table['t_start_Buf'].apply(lambda x : x.decode())\n",
    "dataOpsManager.unload_file_obj()\n",
    "\n",
    "a, b = apply_calibration_factors(data_table, datetime_var,'pipelines/params/calibration_factors.yaml')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "apply_calib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "# Set up project root directory\n",
    "root_dir = os.path.abspath(os.curdir)\n",
    "sys.path.append(root_dir)\n",
    "sys.path.append(os.path.join(root_dir,'dima'))\n",
    "\n",
    "import dima.src.hdf5_ops as data_ops\n",
    "\n",
    "path_to_file = 'data/collection_JFJ_2024_NoraN_2024-01-01_2024-02-29.h5'\n",
    "DataOpsObj = data_ops.HDF5DataOpsManager(path_to_file)\n",
    "\n",
    "DataOpsObj.load_file_obj()\n",
    "data_table = DataOpsObj.extract_dataset_as_dataframe('/ACSM_TOFWARE/2024/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table')\n",
    "DataOpsObj.unload_file_obj()\n",
    "\n",
    "data_table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_append_dir = 'data_products/collection_JFJ_2024_acsm_campaign_NoraN_2024-01-01_2024-02-29'\n",
    "DataOpsObj.load_file_obj()\n",
    "DataOpsObj.transfer_flags()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataOpsObj.load_file_obj()\n",
    "DataOpsObj._data_ops_obj.extract_and_load_dataset_metadata()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataOpsObj._data_ops_obj.dataset_metadata_df.head(n=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataOpsObj.load_file_obj()\n",
    "DataOpsObj.apply_flags('ACSM_TOFWARE')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataOpsObj._data_ops_obj.unload_file_obj()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataOpsObj.load_file_obj()\n",
    "DataOpsObj._data_ops_obj.append_metadata('ACSM_TOFWARE__cleaned/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table',{ 'creation_date' : b'2024-10-09_15-33-54_UTC-OFST_+0200' })\n",
    "DataOpsObj._data_ops_obj.unload_file_obj()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataOpsObj.load_file_obj()\n",
    "DataOpsObj._data_ops_obj.delete_metadata('ACSM_TOFWARE__cleaned/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table',{ 'created_at' : {'delete': True }})\n",
    "DataOpsObj._data_ops_obj.unload_file_obj()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "flaggingapp_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}