diff --git a/workflow_data_integration.ipynb b/workflow_data_integration.ipynb index fc82d59..45dc92c 100644 --- a/workflow_data_integration.ipynb +++ b/workflow_data_integration.ipynb @@ -1,41 +1,55 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Import python packages and modules" + ] + }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "# Set up project root directory\n", + "root_dir = os.path.abspath(os.curdir)\n", + "sys.path.append(root_dir)\n", + "\n", + "import src.hdf5_vis as hdf5_vis\n", + "import src.hdf5_lib as hdf5_lib\n", + "import input_files.config_file as config_file\n", + "\n", + "\n", + "output_dir = 'output_files/'\n", + "group_id = '5505'#'smog_chamber'#'5505'\n", + "user_initials = 'LL' #'NG' #'LL' # 'TBR'\n", + "\n", + "group_id = 'smog_chamber'#'5505'\n", + "user_initials = 'NG'#'LL' #'NG' #'LL' # 'TBR'\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Define input file path and keywords\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2 26\n", - ".dat :)\n", - "2 26\n", - ".dat :)\n", - "2 26\n", - ".dat :)\n", - "2 26\n", - ".dat :)\n", - "Humidity_Sensors\n", - "Humidity_Sensors/2022\n", - "Humidity_Sensors/2022/01_Januar\n", - "Humidity_Sensors/2022/02_Februar\n", - "Humidity_Sensors/2022/03_März\n", - "Humidity_Sensors/2022/04_April\n", - "Humidity_Sensors/2022/05_Mai\n", - "Humidity_Sensors/2022/06_Juni\n", - "Humidity_Sensors/2022/07_Juli\n", - "Humidity_Sensors/2022/10_Oktober\n", - "Humidity_Sensors/2022/11_November\n", - "Humidity_Sensors/2022/12_Dezember\n", - "ICAD\n", - "ICAD/HONO\n", - "ICAD/HONO/2022\n", - "ICAD/HONO_prototype\n", - "ICAD/HONO_prototype/2022\n", - "ICAD/NO2\n", - "ICAD/NO2/2022\n" + "[==================================================--------------------------------------------------] 50.0% ...Uploading files in \\\\fs03\\Iron_Sulphate\\smps\\20220726\r" ] }, { @@ -49,237 +63,113 @@ "branchvalues": "remainder", "customdata": [ "
", - "/Humidity_Sensors", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022/01_Januar", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables", - "/Humidity_Sensors/2022/02_Februar", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables", - "/Humidity_Sensors/2022/03_März", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables", - "/Humidity_Sensors/2022/04_April", - "/Humidity_Sensors/2022/05_Mai", - "/Humidity_Sensors/2022/06_Juni", - "/Humidity_Sensors/2022/07_Juli", - "/Humidity_Sensors/2022/10_Oktober", - "/Humidity_Sensors/2022/11_November", - "/Humidity_Sensors/2022/12_Dezember", - "/ICAD", - "/ICAD/HONO", - "/ICAD/HONO/2022", - "/ICAD/HONO/2022/10_Oct", - "/ICAD/HONO/2022/11_Nov", - "/ICAD/HONO/2022/12_Dec", - "/ICAD/HONO_prototype", - "/ICAD/HONO_prototype/2022", - "/ICAD/HONO_prototype/2022/01_Jan", - "/ICAD/HONO_prototype/2022/02_Feb", - "/ICAD/NO2", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022/01_Jan", - "/ICAD/NO2/2022/02_Feb", - "/ICAD/NO2/2022/03_Mar", - "/ICAD/NO2/2022/04_Apr", - "/ICAD/NO2/2022/05_May", - "/ICAD/NO2/2022/06_June", - "/ICAD/NO2/2022/07_July", - "/ICAD/NO2/2022/10_Oct", - "/ICAD/NO2/2022/11_Nov", - "/ICAD/NO2/2022/12_Dec" + "/gas", + "/gas/20220726_000004_MSC_gases.txt", + "/gas/20220726_000004_MSC_gases.txt/categorial_variable_names", + "/gas/20220726_000004_MSC_gases.txt/categorical_variables", + "/gas/20220726_000004_MSC_gases.txt/numerical_variable_names", + "/gas/20220726_000004_MSC_gases.txt/numerical_variables", + "/gas/20220726_101617_MSC_gases.txt", + "/gas/20220726_101617_MSC_gases.txt/categorial_variable_names", + "/gas/20220726_101617_MSC_gases.txt/categorical_variables", + "/gas/20220726_101617_MSC_gases.txt/numerical_variable_names", + "/gas/20220726_101617_MSC_gases.txt/numerical_variables", + "/smps", + "/smps/20220726", + "/smps/20220726/20220726_mass.TXT", + "/smps/20220726/20220726_mass.TXT/categorial_variable_names", + "/smps/20220726/20220726_mass.TXT/categorical_variables", + "/smps/20220726/20220726_mass.TXT/numerical_variable_names", + "/smps/20220726/20220726_mass.TXT/numerical_variables", + "/smps/20220726/20220726_num.TXT", + "/smps/20220726/20220726_num.TXT/categorial_variable_names", + "/smps/20220726/20220726_num.TXT/categorical_variables", + "/smps/20220726/20220726_num.TXT/numerical_variable_names", + "/smps/20220726/20220726_num.TXT/numerical_variables" ], "hovertemplate": "%{label}
Count: %{value}
Path: %{customdata}", "labels": [ "/", - "/Humidity_Sensors", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022/01_Januar", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables", - "/Humidity_Sensors/2022/02_Februar", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables", - "/Humidity_Sensors/2022/03_März", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables", - "/Humidity_Sensors/2022/04_April", - "/Humidity_Sensors/2022/05_Mai", - "/Humidity_Sensors/2022/06_Juni", - "/Humidity_Sensors/2022/07_Juli", - "/Humidity_Sensors/2022/10_Oktober", - "/Humidity_Sensors/2022/11_November", - "/Humidity_Sensors/2022/12_Dezember", - "/ICAD", - "/ICAD/HONO", - "/ICAD/HONO/2022", - "/ICAD/HONO/2022/10_Oct", - "/ICAD/HONO/2022/11_Nov", - "/ICAD/HONO/2022/12_Dec", - "/ICAD/HONO_prototype", - "/ICAD/HONO_prototype/2022", - "/ICAD/HONO_prototype/2022/01_Jan", - "/ICAD/HONO_prototype/2022/02_Feb", - "/ICAD/NO2", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022/01_Jan", - "/ICAD/NO2/2022/02_Feb", - "/ICAD/NO2/2022/03_Mar", - "/ICAD/NO2/2022/04_Apr", - "/ICAD/NO2/2022/05_May", - "/ICAD/NO2/2022/06_June", - "/ICAD/NO2/2022/07_July", - "/ICAD/NO2/2022/10_Oct", - "/ICAD/NO2/2022/11_Nov", - "/ICAD/NO2/2022/12_Dec" + "/gas", + "/gas/20220726_000004_MSC_gases.txt", + "/gas/20220726_000004_MSC_gases.txt/categorial_variable_names", + "/gas/20220726_000004_MSC_gases.txt/categorical_variables", + "/gas/20220726_000004_MSC_gases.txt/numerical_variable_names", + "/gas/20220726_000004_MSC_gases.txt/numerical_variables", + "/gas/20220726_101617_MSC_gases.txt", + "/gas/20220726_101617_MSC_gases.txt/categorial_variable_names", + "/gas/20220726_101617_MSC_gases.txt/categorical_variables", + "/gas/20220726_101617_MSC_gases.txt/numerical_variable_names", + "/gas/20220726_101617_MSC_gases.txt/numerical_variables", + "/smps", + "/smps/20220726", + "/smps/20220726/20220726_mass.TXT", + "/smps/20220726/20220726_mass.TXT/categorial_variable_names", + "/smps/20220726/20220726_mass.TXT/categorical_variables", + "/smps/20220726/20220726_mass.TXT/numerical_variable_names", + "/smps/20220726/20220726_mass.TXT/numerical_variables", + "/smps/20220726/20220726_num.TXT", + "/smps/20220726/20220726_num.TXT/categorial_variable_names", + "/smps/20220726/20220726_num.TXT/categorical_variables", + "/smps/20220726/20220726_num.TXT/numerical_variable_names", + "/smps/20220726/20220726_num.TXT/numerical_variables" ], "name": "", "parents": [ "", "/", - "/Humidity_Sensors", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022/01_Januar", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022/02_Februar", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022/03_März", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022", - "/Humidity_Sensors/2022", + "/gas", + "/gas/20220726_000004_MSC_gases.txt", + "/gas/20220726_000004_MSC_gases.txt", + "/gas/20220726_000004_MSC_gases.txt", + "/gas/20220726_000004_MSC_gases.txt", + "/gas", + "/gas/20220726_101617_MSC_gases.txt", + "/gas/20220726_101617_MSC_gases.txt", + "/gas/20220726_101617_MSC_gases.txt", + "/gas/20220726_101617_MSC_gases.txt", "/", - "/ICAD", - "/ICAD/HONO", - "/ICAD/HONO/2022", - "/ICAD/HONO/2022", - "/ICAD/HONO/2022", - "/ICAD", - "/ICAD/HONO_prototype", - "/ICAD/HONO_prototype/2022", - "/ICAD/HONO_prototype/2022", - "/ICAD", - "/ICAD/NO2", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022", - "/ICAD/NO2/2022" + "/smps", + "/smps/20220726", + "/smps/20220726/20220726_mass.TXT", + "/smps/20220726/20220726_mass.TXT", + "/smps/20220726/20220726_mass.TXT", + "/smps/20220726/20220726_mass.TXT", + "/smps/20220726", + "/smps/20220726/20220726_num.TXT", + "/smps/20220726/20220726_num.TXT", + "/smps/20220726/20220726_num.TXT", + "/smps/20220726/20220726_num.TXT" ], "root": { "color": "lightgrey" }, "type": "treemap", "values": [ - 1, - 1, - 0, - 5, - 1, - 1, - 1, - 1, - 1, - 14, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 6, - 1, + 2, + 2, + 4, 1, 1, 1, 1, 4, - 9, - 11, - 3, - 8, - 17, + 1, + 1, + 1, + 1, + 1, 2, - 1, - 1, - 0, - 7, - 8, - 2, - 1, - 1, - 3, - 6, - 1, - 0, - 3, - 6, - 5, - 3, 4, - 6, - 2, - 5, - 8, - 2 + 1, + 1, + 1, + 1, + 4, + 1, + 1, + 1, + 1 ] } ], @@ -1116,18 +1006,37 @@ } ], "source": [ - "import sys\n", - "import os\n", - "root_dir = os.path.abspath(os.curdir)\n", - "sys.path.append(root_dir)\n", + "#input_file_dir = '//fs101/5505/People/Juan/TypicalBeamTime'\n", + "#select_file_keywords=[]\n", + "#select_dir_keywords = ['NEXAFS', 'Notes', 'Photos', 'Pressure', 'RGA', 'SES']\n", "\n", - "import src.hdf5_vis as hdf5_vis\n", - "import src.hdf5_lib as hdf5_lib\n", "\n", - "output_filename_path, output_yml_filename_path = hdf5_lib.main()\n", + "#input_file_dir = '//fs101/5505/Data' \n", + "#select_dir_keywords = ['Lopap', 'Humidity_Sensors/2022', 'ICAD/HONO/2022', 'ICAD/NO2/2022', 'T200_NOX', 'T360U_CO2']\n", + "#select_file_keywords = ['2022-03-25','2022_03_25','20220325']\n", + "\n", + "\n", + "input_file_dir = '//fs03/Iron_Sulphate'\n", + "select_dir_keywords = ['gas','smps/20220726']#,'htof/2022.07.26','ptr/2022.07.26','ams/2022.07.26']\n", + "#select_dir_keywords = ['htof','ams', 'ptr', 'gas','smps'] \n", + "\n", + "select_file_keywords = ['20220726','2022.07.26']\n", + "\n", + "config_param = {'group_id' : group_id, 'user_initials' : user_initials, 'output_dir': output_dir}\n", + "\n", + "\n", + "output_filename_path, output_yml_filename_path = hdf5_lib.create_hdf5_file_from_filesystem_path(config_param,\n", + " input_file_dir,\n", + " select_dir_keywords,\n", + " select_file_keywords)\n", "\n", "hdf5_vis.display_group_hierarchy_on_a_treemap(output_filename_path)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": {