diff --git a/workflow_data_integration.ipynb b/workflow_data_integration.ipynb
index fc82d59..45dc92c 100644
--- a/workflow_data_integration.ipynb
+++ b/workflow_data_integration.ipynb
@@ -1,41 +1,55 @@
{
"cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Import python packages and modules"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import sys\n",
+ "import os\n",
+ "# Set up project root directory\n",
+ "root_dir = os.path.abspath(os.curdir)\n",
+ "sys.path.append(root_dir)\n",
+ "\n",
+ "import src.hdf5_vis as hdf5_vis\n",
+ "import src.hdf5_lib as hdf5_lib\n",
+ "import input_files.config_file as config_file\n",
+ "\n",
+ "\n",
+ "output_dir = 'output_files/'\n",
+ "group_id = '5505'#'smog_chamber'#'5505'\n",
+ "user_initials = 'LL' #'NG' #'LL' # 'TBR'\n",
+ "\n",
+ "group_id = 'smog_chamber'#'5505'\n",
+ "user_initials = 'NG'#'LL' #'NG' #'LL' # 'TBR'\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Define input file path and keywords\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "2 26\n",
- ".dat :)\n",
- "2 26\n",
- ".dat :)\n",
- "2 26\n",
- ".dat :)\n",
- "2 26\n",
- ".dat :)\n",
- "Humidity_Sensors\n",
- "Humidity_Sensors/2022\n",
- "Humidity_Sensors/2022/01_Januar\n",
- "Humidity_Sensors/2022/02_Februar\n",
- "Humidity_Sensors/2022/03_März\n",
- "Humidity_Sensors/2022/04_April\n",
- "Humidity_Sensors/2022/05_Mai\n",
- "Humidity_Sensors/2022/06_Juni\n",
- "Humidity_Sensors/2022/07_Juli\n",
- "Humidity_Sensors/2022/10_Oktober\n",
- "Humidity_Sensors/2022/11_November\n",
- "Humidity_Sensors/2022/12_Dezember\n",
- "ICAD\n",
- "ICAD/HONO\n",
- "ICAD/HONO/2022\n",
- "ICAD/HONO_prototype\n",
- "ICAD/HONO_prototype/2022\n",
- "ICAD/NO2\n",
- "ICAD/NO2/2022\n"
+ "[==================================================--------------------------------------------------] 50.0% ...Uploading files in \\\\fs03\\Iron_Sulphate\\smps\\20220726\r"
]
},
{
@@ -49,237 +63,113 @@
"branchvalues": "remainder",
"customdata": [
"
",
- "/Humidity_Sensors",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022/01_Januar",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables",
- "/Humidity_Sensors/2022/02_Februar",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables",
- "/Humidity_Sensors/2022/03_März",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables",
- "/Humidity_Sensors/2022/04_April",
- "/Humidity_Sensors/2022/05_Mai",
- "/Humidity_Sensors/2022/06_Juni",
- "/Humidity_Sensors/2022/07_Juli",
- "/Humidity_Sensors/2022/10_Oktober",
- "/Humidity_Sensors/2022/11_November",
- "/Humidity_Sensors/2022/12_Dezember",
- "/ICAD",
- "/ICAD/HONO",
- "/ICAD/HONO/2022",
- "/ICAD/HONO/2022/10_Oct",
- "/ICAD/HONO/2022/11_Nov",
- "/ICAD/HONO/2022/12_Dec",
- "/ICAD/HONO_prototype",
- "/ICAD/HONO_prototype/2022",
- "/ICAD/HONO_prototype/2022/01_Jan",
- "/ICAD/HONO_prototype/2022/02_Feb",
- "/ICAD/NO2",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022/01_Jan",
- "/ICAD/NO2/2022/02_Feb",
- "/ICAD/NO2/2022/03_Mar",
- "/ICAD/NO2/2022/04_Apr",
- "/ICAD/NO2/2022/05_May",
- "/ICAD/NO2/2022/06_June",
- "/ICAD/NO2/2022/07_July",
- "/ICAD/NO2/2022/10_Oct",
- "/ICAD/NO2/2022/11_Nov",
- "/ICAD/NO2/2022/12_Dec"
+ "/gas",
+ "/gas/20220726_000004_MSC_gases.txt",
+ "/gas/20220726_000004_MSC_gases.txt/categorial_variable_names",
+ "/gas/20220726_000004_MSC_gases.txt/categorical_variables",
+ "/gas/20220726_000004_MSC_gases.txt/numerical_variable_names",
+ "/gas/20220726_000004_MSC_gases.txt/numerical_variables",
+ "/gas/20220726_101617_MSC_gases.txt",
+ "/gas/20220726_101617_MSC_gases.txt/categorial_variable_names",
+ "/gas/20220726_101617_MSC_gases.txt/categorical_variables",
+ "/gas/20220726_101617_MSC_gases.txt/numerical_variable_names",
+ "/gas/20220726_101617_MSC_gases.txt/numerical_variables",
+ "/smps",
+ "/smps/20220726",
+ "/smps/20220726/20220726_mass.TXT",
+ "/smps/20220726/20220726_mass.TXT/categorial_variable_names",
+ "/smps/20220726/20220726_mass.TXT/categorical_variables",
+ "/smps/20220726/20220726_mass.TXT/numerical_variable_names",
+ "/smps/20220726/20220726_mass.TXT/numerical_variables",
+ "/smps/20220726/20220726_num.TXT",
+ "/smps/20220726/20220726_num.TXT/categorial_variable_names",
+ "/smps/20220726/20220726_num.TXT/categorical_variables",
+ "/smps/20220726/20220726_num.TXT/numerical_variable_names",
+ "/smps/20220726/20220726_num.TXT/numerical_variables"
],
"hovertemplate": "%{label}
Count: %{value}
Path: %{customdata}",
"labels": [
"/",
- "/Humidity_Sensors",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022/01_Januar",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables",
- "/Humidity_Sensors/2022/02_Februar",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables",
- "/Humidity_Sensors/2022/03_März",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables",
- "/Humidity_Sensors/2022/04_April",
- "/Humidity_Sensors/2022/05_Mai",
- "/Humidity_Sensors/2022/06_Juni",
- "/Humidity_Sensors/2022/07_Juli",
- "/Humidity_Sensors/2022/10_Oktober",
- "/Humidity_Sensors/2022/11_November",
- "/Humidity_Sensors/2022/12_Dezember",
- "/ICAD",
- "/ICAD/HONO",
- "/ICAD/HONO/2022",
- "/ICAD/HONO/2022/10_Oct",
- "/ICAD/HONO/2022/11_Nov",
- "/ICAD/HONO/2022/12_Dec",
- "/ICAD/HONO_prototype",
- "/ICAD/HONO_prototype/2022",
- "/ICAD/HONO_prototype/2022/01_Jan",
- "/ICAD/HONO_prototype/2022/02_Feb",
- "/ICAD/NO2",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022/01_Jan",
- "/ICAD/NO2/2022/02_Feb",
- "/ICAD/NO2/2022/03_Mar",
- "/ICAD/NO2/2022/04_Apr",
- "/ICAD/NO2/2022/05_May",
- "/ICAD/NO2/2022/06_June",
- "/ICAD/NO2/2022/07_July",
- "/ICAD/NO2/2022/10_Oct",
- "/ICAD/NO2/2022/11_Nov",
- "/ICAD/NO2/2022/12_Dec"
+ "/gas",
+ "/gas/20220726_000004_MSC_gases.txt",
+ "/gas/20220726_000004_MSC_gases.txt/categorial_variable_names",
+ "/gas/20220726_000004_MSC_gases.txt/categorical_variables",
+ "/gas/20220726_000004_MSC_gases.txt/numerical_variable_names",
+ "/gas/20220726_000004_MSC_gases.txt/numerical_variables",
+ "/gas/20220726_101617_MSC_gases.txt",
+ "/gas/20220726_101617_MSC_gases.txt/categorial_variable_names",
+ "/gas/20220726_101617_MSC_gases.txt/categorical_variables",
+ "/gas/20220726_101617_MSC_gases.txt/numerical_variable_names",
+ "/gas/20220726_101617_MSC_gases.txt/numerical_variables",
+ "/smps",
+ "/smps/20220726",
+ "/smps/20220726/20220726_mass.TXT",
+ "/smps/20220726/20220726_mass.TXT/categorial_variable_names",
+ "/smps/20220726/20220726_mass.TXT/categorical_variables",
+ "/smps/20220726/20220726_mass.TXT/numerical_variable_names",
+ "/smps/20220726/20220726_mass.TXT/numerical_variables",
+ "/smps/20220726/20220726_num.TXT",
+ "/smps/20220726/20220726_num.TXT/categorial_variable_names",
+ "/smps/20220726/20220726_num.TXT/categorical_variables",
+ "/smps/20220726/20220726_num.TXT/numerical_variable_names",
+ "/smps/20220726/20220726_num.TXT/numerical_variables"
],
"name": "",
"parents": [
"",
"/",
- "/Humidity_Sensors",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022/01_Januar",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022/02_Februar",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022/03_März",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022",
- "/Humidity_Sensors/2022",
+ "/gas",
+ "/gas/20220726_000004_MSC_gases.txt",
+ "/gas/20220726_000004_MSC_gases.txt",
+ "/gas/20220726_000004_MSC_gases.txt",
+ "/gas/20220726_000004_MSC_gases.txt",
+ "/gas",
+ "/gas/20220726_101617_MSC_gases.txt",
+ "/gas/20220726_101617_MSC_gases.txt",
+ "/gas/20220726_101617_MSC_gases.txt",
+ "/gas/20220726_101617_MSC_gases.txt",
"/",
- "/ICAD",
- "/ICAD/HONO",
- "/ICAD/HONO/2022",
- "/ICAD/HONO/2022",
- "/ICAD/HONO/2022",
- "/ICAD",
- "/ICAD/HONO_prototype",
- "/ICAD/HONO_prototype/2022",
- "/ICAD/HONO_prototype/2022",
- "/ICAD",
- "/ICAD/NO2",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022",
- "/ICAD/NO2/2022"
+ "/smps",
+ "/smps/20220726",
+ "/smps/20220726/20220726_mass.TXT",
+ "/smps/20220726/20220726_mass.TXT",
+ "/smps/20220726/20220726_mass.TXT",
+ "/smps/20220726/20220726_mass.TXT",
+ "/smps/20220726",
+ "/smps/20220726/20220726_num.TXT",
+ "/smps/20220726/20220726_num.TXT",
+ "/smps/20220726/20220726_num.TXT",
+ "/smps/20220726/20220726_num.TXT"
],
"root": {
"color": "lightgrey"
},
"type": "treemap",
"values": [
- 1,
- 1,
- 0,
- 5,
- 1,
- 1,
- 1,
- 1,
- 1,
- 14,
- 1,
- 1,
- 1,
- 1,
- 1,
- 1,
- 1,
- 1,
- 1,
- 1,
- 6,
- 1,
+ 2,
+ 2,
+ 4,
1,
1,
1,
1,
4,
- 9,
- 11,
- 3,
- 8,
- 17,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
2,
- 1,
- 1,
- 0,
- 7,
- 8,
- 2,
- 1,
- 1,
- 3,
- 6,
- 1,
- 0,
- 3,
- 6,
- 5,
- 3,
4,
- 6,
- 2,
- 5,
- 8,
- 2
+ 1,
+ 1,
+ 1,
+ 1,
+ 4,
+ 1,
+ 1,
+ 1,
+ 1
]
}
],
@@ -1116,18 +1006,37 @@
}
],
"source": [
- "import sys\n",
- "import os\n",
- "root_dir = os.path.abspath(os.curdir)\n",
- "sys.path.append(root_dir)\n",
+ "#input_file_dir = '//fs101/5505/People/Juan/TypicalBeamTime'\n",
+ "#select_file_keywords=[]\n",
+ "#select_dir_keywords = ['NEXAFS', 'Notes', 'Photos', 'Pressure', 'RGA', 'SES']\n",
"\n",
- "import src.hdf5_vis as hdf5_vis\n",
- "import src.hdf5_lib as hdf5_lib\n",
"\n",
- "output_filename_path, output_yml_filename_path = hdf5_lib.main()\n",
+ "#input_file_dir = '//fs101/5505/Data' \n",
+ "#select_dir_keywords = ['Lopap', 'Humidity_Sensors/2022', 'ICAD/HONO/2022', 'ICAD/NO2/2022', 'T200_NOX', 'T360U_CO2']\n",
+ "#select_file_keywords = ['2022-03-25','2022_03_25','20220325']\n",
+ "\n",
+ "\n",
+ "input_file_dir = '//fs03/Iron_Sulphate'\n",
+ "select_dir_keywords = ['gas','smps/20220726']#,'htof/2022.07.26','ptr/2022.07.26','ams/2022.07.26']\n",
+ "#select_dir_keywords = ['htof','ams', 'ptr', 'gas','smps'] \n",
+ "\n",
+ "select_file_keywords = ['20220726','2022.07.26']\n",
+ "\n",
+ "config_param = {'group_id' : group_id, 'user_initials' : user_initials, 'output_dir': output_dir}\n",
+ "\n",
+ "\n",
+ "output_filename_path, output_yml_filename_path = hdf5_lib.create_hdf5_file_from_filesystem_path(config_param,\n",
+ " input_file_dir,\n",
+ " select_dir_keywords,\n",
+ " select_file_keywords)\n",
"\n",
"hdf5_vis.display_group_hierarchy_on_a_treemap(output_filename_path)"
]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": []
}
],
"metadata": {