5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implemented first approach to data integration workflow

Browse Source
This commit is contained in:
florez_j
2024-04-03 13:51:21 +02:00
parent f9b31c06fd
commit 5cd19979b6
1 changed files with 152 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

395
workflow_data_integration.ipynb
View File

@ -1,41 +1,55 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Import python packages and modules"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"import os\n",
"# Set up project root directory\n",
"root_dir = os.path.abspath(os.curdir)\n",
"sys.path.append(root_dir)\n",
"\n",
"import src.hdf5_vis as hdf5_vis\n",
"import src.hdf5_lib as hdf5_lib\n",
"import input_files.config_file as config_file\n",
"\n",
"\n",
"output_dir = 'output_files/'\n",
"group_id = '5505'#'smog_chamber'#'5505'\n",
"user_initials = 'LL' #'NG' #'LL' # 'TBR'\n",
"\n",
"group_id = 'smog_chamber'#'5505'\n",
"user_initials = 'NG'#'LL' #'NG' #'LL' # 'TBR'\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Define input file path and keywords\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2 26\n",
".dat :)\n",
"2 26\n",
".dat :)\n",
"2 26\n",
".dat :)\n",
"2 26\n",
".dat :)\n",
"Humidity_Sensors\n",
"Humidity_Sensors/2022\n",
"Humidity_Sensors/2022/01_Januar\n",
"Humidity_Sensors/2022/02_Februar\n",
"Humidity_Sensors/2022/03_März\n",
"Humidity_Sensors/2022/04_April\n",
"Humidity_Sensors/2022/05_Mai\n",
"Humidity_Sensors/2022/06_Juni\n",
"Humidity_Sensors/2022/07_Juli\n",
"Humidity_Sensors/2022/10_Oktober\n",
"Humidity_Sensors/2022/11_November\n",
"Humidity_Sensors/2022/12_Dezember\n",
"ICAD\n",
"ICAD/HONO\n",
"ICAD/HONO/2022\n",
"ICAD/HONO_prototype\n",
"ICAD/HONO_prototype/2022\n",
"ICAD/NO2\n",
"ICAD/NO2/2022\n"
"[==================================================--------------------------------------------------] 50.0% ...Uploading files in \\\\fs03\\Iron_Sulphate\\smps\\20220726\r"
]
},
{
@ -49,237 +63,113 @@
"branchvalues": "remainder",
"customdata": [
"<br>",
"/Humidity_Sensors",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022/01_Januar",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/02_Februar",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/03_März",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/04_April",
"/Humidity_Sensors/2022/05_Mai",
"/Humidity_Sensors/2022/06_Juni",
"/Humidity_Sensors/2022/07_Juli",
"/Humidity_Sensors/2022/10_Oktober",
"/Humidity_Sensors/2022/11_November",
"/Humidity_Sensors/2022/12_Dezember",
"/ICAD",
"/ICAD/HONO",
"/ICAD/HONO/2022",
"/ICAD/HONO/2022/10_Oct",
"/ICAD/HONO/2022/11_Nov",
"/ICAD/HONO/2022/12_Dec",
"/ICAD/HONO_prototype",
"/ICAD/HONO_prototype/2022",
"/ICAD/HONO_prototype/2022/01_Jan",
"/ICAD/HONO_prototype/2022/02_Feb",
"/ICAD/NO2",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022/01_Jan",
"/ICAD/NO2/2022/02_Feb",
"/ICAD/NO2/2022/03_Mar",
"/ICAD/NO2/2022/04_Apr",
"/ICAD/NO2/2022/05_May",
"/ICAD/NO2/2022/06_June",
"/ICAD/NO2/2022/07_July",
"/ICAD/NO2/2022/10_Oct",
"/ICAD/NO2/2022/11_Nov",
"/ICAD/NO2/2022/12_Dec"
"/gas",
"/gas/20220726_000004_MSC_gases.txt",
"/gas/20220726_000004_MSC_gases.txt/categorial_variable_names",
"/gas/20220726_000004_MSC_gases.txt/categorical_variables",
"/gas/20220726_000004_MSC_gases.txt/numerical_variable_names",
"/gas/20220726_000004_MSC_gases.txt/numerical_variables",
"/gas/20220726_101617_MSC_gases.txt",
"/gas/20220726_101617_MSC_gases.txt/categorial_variable_names",
"/gas/20220726_101617_MSC_gases.txt/categorical_variables",
"/gas/20220726_101617_MSC_gases.txt/numerical_variable_names",
"/gas/20220726_101617_MSC_gases.txt/numerical_variables",
"/smps",
"/smps/20220726",
"/smps/20220726/20220726_mass.TXT",
"/smps/20220726/20220726_mass.TXT/categorial_variable_names",
"/smps/20220726/20220726_mass.TXT/categorical_variables",
"/smps/20220726/20220726_mass.TXT/numerical_variable_names",
"/smps/20220726/20220726_mass.TXT/numerical_variables",
"/smps/20220726/20220726_num.TXT",
"/smps/20220726/20220726_num.TXT/categorial_variable_names",
"/smps/20220726/20220726_num.TXT/categorical_variables",
"/smps/20220726/20220726_num.TXT/numerical_variable_names",
"/smps/20220726/20220726_num.TXT/numerical_variables"
],
"hovertemplate": "<b>%{label} </b> <br> Count: %{value} <br> Path: %{customdata}",
"labels": [
"/",
"/Humidity_Sensors",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022/01_Januar",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/02_Februar",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/03_März",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/04_April",
"/Humidity_Sensors/2022/05_Mai",
"/Humidity_Sensors/2022/06_Juni",
"/Humidity_Sensors/2022/07_Juli",
"/Humidity_Sensors/2022/10_Oktober",
"/Humidity_Sensors/2022/11_November",
"/Humidity_Sensors/2022/12_Dezember",
"/ICAD",
"/ICAD/HONO",
"/ICAD/HONO/2022",
"/ICAD/HONO/2022/10_Oct",
"/ICAD/HONO/2022/11_Nov",
"/ICAD/HONO/2022/12_Dec",
"/ICAD/HONO_prototype",
"/ICAD/HONO_prototype/2022",
"/ICAD/HONO_prototype/2022/01_Jan",
"/ICAD/HONO_prototype/2022/02_Feb",
"/ICAD/NO2",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022/01_Jan",
"/ICAD/NO2/2022/02_Feb",
"/ICAD/NO2/2022/03_Mar",
"/ICAD/NO2/2022/04_Apr",
"/ICAD/NO2/2022/05_May",
"/ICAD/NO2/2022/06_June",
"/ICAD/NO2/2022/07_July",
"/ICAD/NO2/2022/10_Oct",
"/ICAD/NO2/2022/11_Nov",
"/ICAD/NO2/2022/12_Dec"
"/gas",
"/gas/20220726_000004_MSC_gases.txt",
"/gas/20220726_000004_MSC_gases.txt/categorial_variable_names",
"/gas/20220726_000004_MSC_gases.txt/categorical_variables",
"/gas/20220726_000004_MSC_gases.txt/numerical_variable_names",
"/gas/20220726_000004_MSC_gases.txt/numerical_variables",
"/gas/20220726_101617_MSC_gases.txt",
"/gas/20220726_101617_MSC_gases.txt/categorial_variable_names",
"/gas/20220726_101617_MSC_gases.txt/categorical_variables",
"/gas/20220726_101617_MSC_gases.txt/numerical_variable_names",
"/gas/20220726_101617_MSC_gases.txt/numerical_variables",
"/smps",
"/smps/20220726",
"/smps/20220726/20220726_mass.TXT",
"/smps/20220726/20220726_mass.TXT/categorial_variable_names",
"/smps/20220726/20220726_mass.TXT/categorical_variables",
"/smps/20220726/20220726_mass.TXT/numerical_variable_names",
"/smps/20220726/20220726_mass.TXT/numerical_variables",
"/smps/20220726/20220726_num.TXT",
"/smps/20220726/20220726_num.TXT/categorial_variable_names",
"/smps/20220726/20220726_num.TXT/categorical_variables",
"/smps/20220726/20220726_num.TXT/numerical_variable_names",
"/smps/20220726/20220726_num.TXT/numerical_variables"
],
"name": "",
"parents": [
"",
"/",
"/Humidity_Sensors",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022/01_Januar",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022/02_Februar",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022/03_März",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/gas",
"/gas/20220726_000004_MSC_gases.txt",
"/gas/20220726_000004_MSC_gases.txt",
"/gas/20220726_000004_MSC_gases.txt",
"/gas/20220726_000004_MSC_gases.txt",
"/gas",
"/gas/20220726_101617_MSC_gases.txt",
"/gas/20220726_101617_MSC_gases.txt",
"/gas/20220726_101617_MSC_gases.txt",
"/gas/20220726_101617_MSC_gases.txt",
"/",
"/ICAD",
"/ICAD/HONO",
"/ICAD/HONO/2022",
"/ICAD/HONO/2022",
"/ICAD/HONO/2022",
"/ICAD",
"/ICAD/HONO_prototype",
"/ICAD/HONO_prototype/2022",
"/ICAD/HONO_prototype/2022",
"/ICAD",
"/ICAD/NO2",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022"
"/smps",
"/smps/20220726",
"/smps/20220726/20220726_mass.TXT",
"/smps/20220726/20220726_mass.TXT",
"/smps/20220726/20220726_mass.TXT",
"/smps/20220726/20220726_mass.TXT",
"/smps/20220726",
"/smps/20220726/20220726_num.TXT",
"/smps/20220726/20220726_num.TXT",
"/smps/20220726/20220726_num.TXT",
"/smps/20220726/20220726_num.TXT"
],
"root": {
"color": "lightgrey"
},
"type": "treemap",
"values": [
1,
1,
0,
5,
1,
1,
1,
1,
1,
14,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
6,
1,
2,
2,
4,
1,
1,
1,
1,
4,
9,
11,
3,
8,
17,
1,
1,
1,
1,
1,
2,
1,
1,
0,
7,
8,
2,
1,
1,
3,
6,
1,
0,
3,
6,
5,
3,
4,
6,
2,
5,
8,
2
1,
1,
1,
1,
4,
1,
1,
1,
1
]
}
],
@ -1116,18 +1006,37 @@
}
],
"source": [
"import sys\n",
"import os\n",
"root_dir = os.path.abspath(os.curdir)\n",
"sys.path.append(root_dir)\n",
"#input_file_dir = '//fs101/5505/People/Juan/TypicalBeamTime'\n",
"#select_file_keywords=[]\n",
"#select_dir_keywords = ['NEXAFS', 'Notes', 'Photos', 'Pressure', 'RGA', 'SES']\n",
"\n",
"import src.hdf5_vis as hdf5_vis\n",
"import src.hdf5_lib as hdf5_lib\n",
"\n",
"output_filename_path, output_yml_filename_path = hdf5_lib.main()\n",
"#input_file_dir = '//fs101/5505/Data' \n",
"#select_dir_keywords = ['Lopap', 'Humidity_Sensors/2022', 'ICAD/HONO/2022', 'ICAD/NO2/2022', 'T200_NOX', 'T360U_CO2']\n",
"#select_file_keywords = ['2022-03-25','2022_03_25','20220325']\n",
"\n",
"\n",
"input_file_dir = '//fs03/Iron_Sulphate'\n",
"select_dir_keywords = ['gas','smps/20220726']#,'htof/2022.07.26','ptr/2022.07.26','ams/2022.07.26']\n",
"#select_dir_keywords = ['htof','ams', 'ptr', 'gas','smps'] \n",
"\n",
"select_file_keywords = ['20220726','2022.07.26']\n",
"\n",
"config_param = {'group_id' : group_id, 'user_initials' : user_initials, 'output_dir': output_dir}\n",
"\n",
"\n",
"output_filename_path, output_yml_filename_path = hdf5_lib.create_hdf5_file_from_filesystem_path(config_param,\n",
" input_file_dir,\n",
" select_dir_keywords,\n",
" select_file_keywords)\n",
"\n",
"hdf5_vis.display_group_hierarchy_on_a_treemap(output_filename_path)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
}
],
"metadata": {