5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implemented first approach to data integration workflow

Browse Source
This commit is contained in:
florez_j
2024-04-03 13:51:21 +02:00
parent f9b31c06fd
commit 5cd19979b6
1 changed files with 152 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

395
workflow_data_integration.ipynb
View File

@ -1,41 +1,55 @@
{ {
"cells": [ "cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Import python packages and modules"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"import os\n",
"# Set up project root directory\n",
"root_dir = os.path.abspath(os.curdir)\n",
"sys.path.append(root_dir)\n",
"\n",
"import src.hdf5_vis as hdf5_vis\n",
"import src.hdf5_lib as hdf5_lib\n",
"import input_files.config_file as config_file\n",
"\n",
"\n",
"output_dir = 'output_files/'\n",
"group_id = '5505'#'smog_chamber'#'5505'\n",
"user_initials = 'LL' #'NG' #'LL' # 'TBR'\n",
"\n",
"group_id = 'smog_chamber'#'5505'\n",
"user_initials = 'NG'#'LL' #'NG' #'LL' # 'TBR'\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Define input file path and keywords\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"2 26\n", "[==================================================--------------------------------------------------] 50.0% ...Uploading files in \\\\fs03\\Iron_Sulphate\\smps\\20220726\r"
".dat :)\n",
"2 26\n",
".dat :)\n",
"2 26\n",
".dat :)\n",
"2 26\n",
".dat :)\n",
"Humidity_Sensors\n",
"Humidity_Sensors/2022\n",
"Humidity_Sensors/2022/01_Januar\n",
"Humidity_Sensors/2022/02_Februar\n",
"Humidity_Sensors/2022/03_März\n",
"Humidity_Sensors/2022/04_April\n",
"Humidity_Sensors/2022/05_Mai\n",
"Humidity_Sensors/2022/06_Juni\n",
"Humidity_Sensors/2022/07_Juli\n",
"Humidity_Sensors/2022/10_Oktober\n",
"Humidity_Sensors/2022/11_November\n",
"Humidity_Sensors/2022/12_Dezember\n",
"ICAD\n",
"ICAD/HONO\n",
"ICAD/HONO/2022\n",
"ICAD/HONO_prototype\n",
"ICAD/HONO_prototype/2022\n",
"ICAD/NO2\n",
"ICAD/NO2/2022\n"
] ]
}, },
{ {
@ -49,237 +63,113 @@
"branchvalues": "remainder", "branchvalues": "remainder",
"customdata": [ "customdata": [
"<br>", "<br>",
"/Humidity_Sensors", "/gas",
"/Humidity_Sensors/2022", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar", "/gas/20220726_000004_MSC_gases.txt/categorial_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_000004_MSC_gases.txt/categorical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/gas/20220726_000004_MSC_gases.txt/numerical_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables", "/gas/20220726_000004_MSC_gases.txt/numerical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables", "/gas/20220726_101617_MSC_gases.txt/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar", "/gas/20220726_101617_MSC_gases.txt/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", "/gas/20220726_101617_MSC_gases.txt/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/gas/20220726_101617_MSC_gases.txt/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables", "/smps",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names", "/smps/20220726",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables", "/smps/20220726/20220726_mass.TXT",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", "/smps/20220726/20220726_mass.TXT/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names", "/smps/20220726/20220726_mass.TXT/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables", "/smps/20220726/20220726_mass.TXT/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names", "/smps/20220726/20220726_mass.TXT/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables", "/smps/20220726/20220726_num.TXT",
"/Humidity_Sensors/2022/03_März", "/smps/20220726/20220726_num.TXT/categorial_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", "/smps/20220726/20220726_num.TXT/categorical_variables",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/smps/20220726/20220726_num.TXT/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables", "/smps/20220726/20220726_num.TXT/numerical_variables"
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/04_April",
"/Humidity_Sensors/2022/05_Mai",
"/Humidity_Sensors/2022/06_Juni",
"/Humidity_Sensors/2022/07_Juli",
"/Humidity_Sensors/2022/10_Oktober",
"/Humidity_Sensors/2022/11_November",
"/Humidity_Sensors/2022/12_Dezember",
"/ICAD",
"/ICAD/HONO",
"/ICAD/HONO/2022",
"/ICAD/HONO/2022/10_Oct",
"/ICAD/HONO/2022/11_Nov",
"/ICAD/HONO/2022/12_Dec",
"/ICAD/HONO_prototype",
"/ICAD/HONO_prototype/2022",
"/ICAD/HONO_prototype/2022/01_Jan",
"/ICAD/HONO_prototype/2022/02_Feb",
"/ICAD/NO2",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022/01_Jan",
"/ICAD/NO2/2022/02_Feb",
"/ICAD/NO2/2022/03_Mar",
"/ICAD/NO2/2022/04_Apr",
"/ICAD/NO2/2022/05_May",
"/ICAD/NO2/2022/06_June",
"/ICAD/NO2/2022/07_July",
"/ICAD/NO2/2022/10_Oct",
"/ICAD/NO2/2022/11_Nov",
"/ICAD/NO2/2022/12_Dec"
], ],
"hovertemplate": "<b>%{label} </b> <br> Count: %{value} <br> Path: %{customdata}", "hovertemplate": "<b>%{label} </b> <br> Count: %{value} <br> Path: %{customdata}",
"labels": [ "labels": [
"/", "/",
"/Humidity_Sensors", "/gas",
"/Humidity_Sensors/2022", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar", "/gas/20220726_000004_MSC_gases.txt/categorial_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_000004_MSC_gases.txt/categorical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/gas/20220726_000004_MSC_gases.txt/numerical_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables", "/gas/20220726_000004_MSC_gases.txt/numerical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables", "/gas/20220726_101617_MSC_gases.txt/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar", "/gas/20220726_101617_MSC_gases.txt/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", "/gas/20220726_101617_MSC_gases.txt/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/gas/20220726_101617_MSC_gases.txt/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables", "/smps",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names", "/smps/20220726",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables", "/smps/20220726/20220726_mass.TXT",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", "/smps/20220726/20220726_mass.TXT/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names", "/smps/20220726/20220726_mass.TXT/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables", "/smps/20220726/20220726_mass.TXT/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names", "/smps/20220726/20220726_mass.TXT/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables", "/smps/20220726/20220726_num.TXT",
"/Humidity_Sensors/2022/03_März", "/smps/20220726/20220726_num.TXT/categorial_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", "/smps/20220726/20220726_num.TXT/categorical_variables",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/smps/20220726/20220726_num.TXT/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables", "/smps/20220726/20220726_num.TXT/numerical_variables"
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/04_April",
"/Humidity_Sensors/2022/05_Mai",
"/Humidity_Sensors/2022/06_Juni",
"/Humidity_Sensors/2022/07_Juli",
"/Humidity_Sensors/2022/10_Oktober",
"/Humidity_Sensors/2022/11_November",
"/Humidity_Sensors/2022/12_Dezember",
"/ICAD",
"/ICAD/HONO",
"/ICAD/HONO/2022",
"/ICAD/HONO/2022/10_Oct",
"/ICAD/HONO/2022/11_Nov",
"/ICAD/HONO/2022/12_Dec",
"/ICAD/HONO_prototype",
"/ICAD/HONO_prototype/2022",
"/ICAD/HONO_prototype/2022/01_Jan",
"/ICAD/HONO_prototype/2022/02_Feb",
"/ICAD/NO2",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022/01_Jan",
"/ICAD/NO2/2022/02_Feb",
"/ICAD/NO2/2022/03_Mar",
"/ICAD/NO2/2022/04_Apr",
"/ICAD/NO2/2022/05_May",
"/ICAD/NO2/2022/06_June",
"/ICAD/NO2/2022/07_July",
"/ICAD/NO2/2022/10_Oct",
"/ICAD/NO2/2022/11_Nov",
"/ICAD/NO2/2022/12_Dec"
], ],
"name": "", "name": "",
"parents": [ "parents": [
"", "",
"/", "/",
"/Humidity_Sensors", "/gas",
"/Humidity_Sensors/2022", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/02_Februar", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022/03_März",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/", "/",
"/ICAD", "/smps",
"/ICAD/HONO", "/smps/20220726",
"/ICAD/HONO/2022", "/smps/20220726/20220726_mass.TXT",
"/ICAD/HONO/2022", "/smps/20220726/20220726_mass.TXT",
"/ICAD/HONO/2022", "/smps/20220726/20220726_mass.TXT",
"/ICAD", "/smps/20220726/20220726_mass.TXT",
"/ICAD/HONO_prototype", "/smps/20220726",
"/ICAD/HONO_prototype/2022", "/smps/20220726/20220726_num.TXT",
"/ICAD/HONO_prototype/2022", "/smps/20220726/20220726_num.TXT",
"/ICAD", "/smps/20220726/20220726_num.TXT",
"/ICAD/NO2", "/smps/20220726/20220726_num.TXT"
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022"
], ],
"root": { "root": {
"color": "lightgrey" "color": "lightgrey"
}, },
"type": "treemap", "type": "treemap",
"values": [ "values": [
1, 2,
1, 2,
0, 4,
5,
1,
1,
1,
1,
1,
14,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
6,
1,
1, 1,
1, 1,
1, 1,
1, 1,
4, 4,
9, 1,
11, 1,
3, 1,
8, 1,
17, 1,
2, 2,
1,
1,
0,
7,
8,
2,
1,
1,
3,
6,
1,
0,
3,
6,
5,
3,
4, 4,
6, 1,
2, 1,
5, 1,
8, 1,
2 4,
1,
1,
1,
1
] ]
} }
], ],
@ -1116,18 +1006,37 @@
} }
], ],
"source": [ "source": [
"import sys\n", "#input_file_dir = '//fs101/5505/People/Juan/TypicalBeamTime'\n",
"import os\n", "#select_file_keywords=[]\n",
"root_dir = os.path.abspath(os.curdir)\n", "#select_dir_keywords = ['NEXAFS', 'Notes', 'Photos', 'Pressure', 'RGA', 'SES']\n",
"sys.path.append(root_dir)\n",
"\n", "\n",
"import src.hdf5_vis as hdf5_vis\n",
"import src.hdf5_lib as hdf5_lib\n",
"\n", "\n",
"output_filename_path, output_yml_filename_path = hdf5_lib.main()\n", "#input_file_dir = '//fs101/5505/Data' \n",
"#select_dir_keywords = ['Lopap', 'Humidity_Sensors/2022', 'ICAD/HONO/2022', 'ICAD/NO2/2022', 'T200_NOX', 'T360U_CO2']\n",
"#select_file_keywords = ['2022-03-25','2022_03_25','20220325']\n",
"\n",
"\n",
"input_file_dir = '//fs03/Iron_Sulphate'\n",
"select_dir_keywords = ['gas','smps/20220726']#,'htof/2022.07.26','ptr/2022.07.26','ams/2022.07.26']\n",
"#select_dir_keywords = ['htof','ams', 'ptr', 'gas','smps'] \n",
"\n",
"select_file_keywords = ['20220726','2022.07.26']\n",
"\n",
"config_param = {'group_id' : group_id, 'user_initials' : user_initials, 'output_dir': output_dir}\n",
"\n",
"\n",
"output_filename_path, output_yml_filename_path = hdf5_lib.create_hdf5_file_from_filesystem_path(config_param,\n",
" input_file_dir,\n",
" select_dir_keywords,\n",
" select_file_keywords)\n",
"\n", "\n",
"hdf5_vis.display_group_hierarchy_on_a_treemap(output_filename_path)" "hdf5_vis.display_group_hierarchy_on_a_treemap(output_filename_path)"
] ]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
} }
], ],
"metadata": { "metadata": {