{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import sys\n", "import os\n", "# Set up project root directory\n", "root_dir = os.path.abspath(os.curdir)\n", "sys.path.append(root_dir)\n", "sys.path.append(os.path.join(root_dir,'dima'))\n", "\n", "import data_flagging_utils as utils\n", "\n", "path_to_file = 'data_products/collection_JFJ_2024_acsm_campaign_NoraN_2024-01-01_2024-02-29.h5'\n", "DataOpsObj = utils.FlaggingAppDataManager(path_to_file)\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import sys\n", "import os\n", "import pandas as pd\n", "# Set up project root directory\n", "root_dir = os.path.abspath(os.curdir)\n", "sys.path.append(root_dir)\n", "sys.path.append(os.path.join(root_dir,'dima'))\n", "\n", "\n", "from pipelines.steps.apply_calibration_factors import apply_calibration_factors\n", "\n", "import dima.src.hdf5_ops as dataOps\n", "\n", "file_path = \"data/collection_JFJ_2024_LeilaS_2025-02-03_2025-02-03.h5\"\n", "dataOpsManager = dataOps.HDF5DataOpsManager(file_path)\n", "\n", "\n", "dataOpsManager.load_file_obj()\n", "dataOpsManager.extract_and_load_dataset_metadata()\n", "print(dataOpsManager.dataset_metadata_df.head())\n", "#dataOpsManager.unload_file_obj\n", "\n", "#dataOpsManager.unload_file_obj()\n", "dataset_name = \"ACSM_TOFWARE/2024/ACSM_JFJ_2024_timeseries.txt/data_table\"\n", "data_table = dataOpsManager.extract_dataset_as_dataframe(dataset_name)\n", "datetime_var, datetime_format = dataOpsManager.infer_datetime_variable(dataset_name)\n", "\n", "#data_table['t_start_Buf'] = data_table['t_start_Buf'].apply(lambda x : x.decode())\n", "dataOpsManager.unload_file_obj()\n", "\n", "a, b = apply_calibration_factors(data_table, datetime_var,'pipelines/params/calibration_factors.yaml')\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "a.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "apply_calib" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import sys\n", "import os\n", "# Set up project root directory\n", "root_dir = os.path.abspath(os.curdir)\n", "sys.path.append(root_dir)\n", "sys.path.append(os.path.join(root_dir,'dima'))\n", "\n", "import dima.src.hdf5_ops as data_ops\n", "\n", "path_to_file = 'data/collection_JFJ_2024_NoraN_2024-01-01_2024-02-29.h5'\n", "DataOpsObj = data_ops.HDF5DataOpsManager(path_to_file)\n", "\n", "DataOpsObj.load_file_obj()\n", "data_table = DataOpsObj.extract_dataset_as_dataframe('/ACSM_TOFWARE/2024/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table')\n", "DataOpsObj.unload_file_obj()\n", "\n", "data_table.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "path_to_append_dir = 'data_products/collection_JFJ_2024_acsm_campaign_NoraN_2024-01-01_2024-02-29'\n", "DataOpsObj.load_file_obj()\n", "DataOpsObj.transfer_flags()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "DataOpsObj.load_file_obj()\n", "DataOpsObj._data_ops_obj.extract_and_load_dataset_metadata()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "DataOpsObj._data_ops_obj.dataset_metadata_df.head(n=15)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "DataOpsObj.load_file_obj()\n", "DataOpsObj.apply_flags('ACSM_TOFWARE')\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "DataOpsObj._data_ops_obj.unload_file_obj()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "DataOpsObj.load_file_obj()\n", "DataOpsObj._data_ops_obj.append_metadata('ACSM_TOFWARE__cleaned/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table',{ 'creation_date' : b'2024-10-09_15-33-54_UTC-OFST_+0200' })\n", "DataOpsObj._data_ops_obj.unload_file_obj()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "DataOpsObj.load_file_obj()\n", "DataOpsObj._data_ops_obj.delete_metadata('ACSM_TOFWARE__cleaned/ACSM_JFJ_2024_JantoFeb_timeseries.txt/data_table',{ 'created_at' : {'delete': True }})\n", "DataOpsObj._data_ops_obj.unload_file_obj()" ] } ], "metadata": { "kernelspec": { "display_name": "flaggingapp_env", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.15" } }, "nbformat": 4, "nbformat_minor": 2 }