5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Resolved incoming changes from main branch

Browse Source
This commit is contained in:
florez_j
2024-04-04 13:07:43 +02:00
parent 48d3b8f492 44d4a7b29b
commit 5839c0f466
9 changed files with 557 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,4 +1,5 @@
*.pyc *.pyc
__pycache__/ __pycache__/
*.h5 *.h5
tmp_files/ tmp_files/
*.ipynb

91
input_files/config_file.py
View File

@ -14,85 +14,32 @@ import pygit2 as pygit
author = pygit.Signature('Florez Ospina Juan Felipe', 'juan.florez-ospina@psi.ch') author = pygit.Signature('Florez Ospina Juan Felipe', 'juan.florez-ospina@psi.ch')
committer = pygit.Signature('Florez Ospina Juan Felipe', 'juan.florez-ospina@psi.ch') committer = pygit.Signature('Florez Ospina Juan Felipe', 'juan.florez-ospina@psi.ch')
group_id = 'smog_chamber'
#group_id = '5505'
output_filename_tempate = lambda group_id, timestamp,user_initials : '_'.join(['unified_file',group_id,timestamp,user_initials])+'.h5' output_filename_tempate = lambda group_id, timestamp,user_initials : '_'.join(['unified_file',group_id,timestamp,user_initials])+'.h5'
def created_at():
now = datetime.now() now = datetime.now()
# Populate now object with time zone infotmation obtained from the local system # Populate now object with time zone infotmation obtained from the local system
now_tz_aware = now.astimezone() now_tz_aware = now.astimezone()
tz = now_tz_aware.strftime('%z') tz = now_tz_aware.strftime('%z')
#created_at = now_tz_aware.strftime('%Y-%m-%d_%H-%M-%S')+'_utcoffset_' + tz #created_at = now_tz_aware.strftime('%Y-%m-%d_%H-%M-%S')+'_utcoffset_' + tz
created_at = now_tz_aware.strftime('%Y-%m-%d')+'_UTC-OFST_' + tz created_at = now_tz_aware.strftime('%Y-%m-%d')+'_UTC-OFST_' + tz
# Make created at timestamp with tz information # Make created at timestamp with tz information
#created_at = now.isoformat() #created_at = now.isoformat()
return created_at
usecase = 1
if usecase == 1:
group_id == 'smog_chamber'
user_initials = 'NG'
#from smog_chamber_file_reader import read_txt_files_as_dict
#from g5505_file_reader import copy_file_in_group
#select_dir_keywords = ['htof','ams', 'ptr', 'gas','smps']
inputfile_dir = '\\\\fs03\\Iron_Sulphate'
inputfile_dir = '//fs03/Iron_Sulphate'
#select_dir_keywords = ['gas','smps\\20220726','htof\\2022.07.26','ptr\\2022.07.26','ams\\2022.07.26']
select_dir_keywords = ['gas','smps/20220726']#,'htof/2022.07.26','ptr/2022.07.26','ams/2022.07.26']
select_file_keywords = ['20220726','2022.07.26']
outputfile_dir = 'output_files'
output_filename = output_filename_tempate(group_id,created_at,user_initials) #'test_smog_chamber_v14.h5'
#output_filename = 'unified_file_smog_chamber_2024-03-19_UTC-OFST_+0100_NG.h5'
ext_to_reader_dict = {'.txt': scf_reader.read_txt_files_as_dict, def select_file_readers(group_id):
if group_id == '5505':
ext_to_reader_dict = {'.ibw': g5505f_reader.read_xps_ibw_file_as_dict,
'.txt': g5505f_reader.read_txt_files_as_dict,
'.dat': g5505f_reader.read_txt_files_as_dict,
'.h5': g5505f_reader.copy_file_in_group}
elif group_id == 'smog_chamber':
ext_to_reader_dict = {'.txt': scf_reader.read_txt_files_as_dict,
'.TXT': scf_reader.read_txt_files_as_dict, '.TXT': scf_reader.read_txt_files_as_dict,
'.h5': g5505f_reader.copy_file_in_group} '.h5': g5505f_reader.copy_file_in_group}
elif usecase == 2 :
group_id == '5505' return ext_to_reader_dict
user_initials = 'TBR'
outputfile_dir = 'output_files'
#output_filename = 'test_sls_data_v8.h5'
inputfile_dir = '//fs101/5505/People/Juan/TypicalBeamTime'
select_file_keywords=[]
select_dir_keywords = ['NEXAFS', 'Notes', 'Photos', 'Pressure', 'RGA', 'SES']
output_filename = output_filename_tempate(group_id,created_at,user_initials)
#output_filename = 'unified_file_5505_2024-03-19_UTC-OFST_+0100_TBR.h5'
ext_to_reader_dict = {'.ibw': g5505f_reader.read_xps_ibw_file_as_dict,
'.txt': g5505f_reader.read_txt_files_as_dict,
'.dat': g5505f_reader.read_txt_files_as_dict,
'.h5': g5505f_reader.copy_file_in_group}
elif usecase == 3:
user_initials = 'LL'
outputfile_dir = 'output_files'
output_filename = output_filename_tempate(group_id,created_at,user_initials)
inputfile_dir = '//fs101/5505/Data'
#select_dir_keywords = ['Lopap', 'Humidity_Sensors', 'ICAD/HONO', 'ICAD/NO2', 'T200_NOX', 'T360U_CO2']
# TODO: make sure in the code composite keywords are broken down into single keywords
##select_dir_keywords = ['Humidity_Sensors','ICAD/HONO','ICAD/NO2']
select_dir_keywords = ['Humidity_Sensors/2022','ICAD/HONO/2022','ICAD/NO2/2022', '2022/01_Jan', '2022/02_Feb', '2022/03_März']
dates = pd.read_excel(os.path.abspath(os.path.join('input_files','date_experiments_for Juan.xlsx')))
select_file_keywords=[item.strftime('%Y-%m-%d') for item in dates.loc[0:2,'experiment_date']]
select_file_keywords= select_file_keywords + [item.strftime('%Y%m%d') for item in dates.loc[0:2,'experiment_date']]
ext_to_reader_dict = {'.ibw': g5505f_reader.read_xps_ibw_file_as_dict,
'.txt': g5505f_reader.read_txt_files_as_dict,
'.dat': g5505f_reader.read_txt_files_as_dict,
'.h5': g5505f_reader.copy_file_in_group}

23
src/g5505_file_reader.py
View File

@ -93,10 +93,12 @@ def read_txt_files_as_dict(filename : str ):
file_encoding = 'latin-1' file_encoding = 'latin-1'
elif 'ICAD' in filename and 'HONO' in filename: elif 'ICAD' in filename and 'HONO' in filename:
table_header = 'Start Date/Time (UTC) Duration (s) NO2 (ppb) NO2 Uncertainty (ppb) H2O (ppb) H2O Uncertainty (ppb) CHOCHO (ppb) CHOCHO Uncertainty (ppb) File Number Light Intensity #ICEDOAS iter. Cell Pressure Ambient Pressure Cell Temp Spec Temp Lat Lon Height Speed GPSQuality 0-Air Ref. Time 0-Air Ref. Duration 0-Air Ref. File Number 0-Air Ref. Intensity 0-Air Ref. Rel Intensity 0-Air Ref. Intensity valid MeasMode SampleSource' table_header = 'Start Date/Time (UTC) Duration (s) NO2 (ppb) NO2 Uncertainty (ppb) H2O (ppb) H2O Uncertainty (ppb) CHOCHO (ppb) CHOCHO Uncertainty (ppb) File Number Light Intensity #ICEDOAS iter. Cell Pressure Ambient Pressure Cell Temp Spec Temp Lat Lon Height Speed GPSQuality 0-Air Ref. Time 0-Air Ref. Duration 0-Air Ref. File Number 0-Air Ref. Intensity 0-Air Ref. Rel Intensity 0-Air Ref. Intensity valid MeasMode SampleSource'
separator = '\t' separator = '\t'
file_encoding = 'latin-1'
elif 'ICAD' in filename and 'NO2' in filename: elif 'ICAD' in filename and 'NO2' in filename:
table_header = 'Start Date/Time (UTC) Duration (s) NO2 (ppb) NO2 Uncertainty (ppb) H2O (ppb) H2O Uncertainty (ppb) CHOCHO (ppb) CHOCHO Uncertainty (ppb) File Number Light Intensity #ICEDOAS iter. Cell Pressure Ambient Pressure Cell Temp Spec Temp Lat Lon Height Speed GPSQuality 0-Air Ref. Time 0-Air Ref. Duration 0-Air Ref. File Number 0-Air Ref. Intensity 0-Air Ref. Rel Intensity 0-Air Ref. Intensity valid MeasMode SampleSource' table_header = 'Start Date/Time (UTC) Duration (s) NO2 (ppb) NO2 Uncertainty (ppb) H2O (ppb) H2O Uncertainty (ppb) CHOCHO (ppb) CHOCHO Uncertainty (ppb) File Number Light Intensity #ICEDOAS iter. Cell Pressure Ambient Pressure Cell Temp Spec Temp Lat Lon Height Speed GPSQuality 0-Air Ref. Time 0-Air Ref. Duration 0-Air Ref. File Number 0-Air Ref. Intensity 0-Air Ref. Rel Intensity 0-Air Ref. Intensity valid MeasMode SampleSource'
separator = '\t' separator = '\t'
file_encoding = 'latin-1'
else: else:
return {} return {}
#raise ValueError('intrument_folder must be set as a either "RGA" or "Pressure"') #raise ValueError('intrument_folder must be set as a either "RGA" or "Pressure"')
@ -107,33 +109,32 @@ def read_txt_files_as_dict(filename : str ):
# Work with copy of the file for safety # Work with copy of the file for safety
tmp_filename = utils.make_file_copy(source_file_path=filename) tmp_filename = utils.make_file_copy(source_file_path=filename)
with open(tmp_filename,'r',encoding=file_encoding,errors='ignore') as f: #with open(tmp_filename,'rb',encoding=file_encoding,errors='ignore') as f:
#file_encoding = f.encoding with open(tmp_filename,'rb') as f:
#table_preamble = ""
table_preamble = [] table_preamble = []
for line_number, line in enumerate(f): for line_number, line in enumerate(f):
if table_header in line: if table_header in line.decode(file_encoding):
list_of_substrings = line.split(separator) list_of_substrings = line.decode(file_encoding).split(separator)
data_start = True data_start = True
column_names = [] column_names = []
for i, name in enumerate(list_of_substrings): for i, name in enumerate(list_of_substrings):
column_names.append(str(i)+'_'+name) column_names.append(str(i)+'_'+name)
print(line_number, len(column_names )) #print(line_number, len(column_names ),'\n')
break break
# Subdivide line into words, and join them by single space. # Subdivide line into words, and join them by single space.
# I asumme this can produce a cleaner line that contains no weird separator characters \t \r or extra spaces and so on. # I asumme this can produce a cleaner line that contains no weird separator characters \t \r or extra spaces and so on.
list_of_substrings = line.split() list_of_substrings = line.decode(file_encoding).split()
# TODO: ideally we should use a multilinear string but the yalm parser is not recognizing \n as special character # TODO: ideally we should use a multilinear string but the yalm parser is not recognizing \n as special character
#line = ' '.join(list_of_substrings+['\n']) #line = ' '.join(list_of_substrings+['\n'])
line = ' '.join(list_of_substrings) #line = ' '.join(list_of_substrings)
table_preamble.append(line)# += new_line table_preamble.append(' '.join(list_of_substrings))# += new_line
header_dict["table_preamble"] = table_preamble header_dict["table_preamble"] = table_preamble
# TODO: it does not work with separater as none :(. fix for RGA # TODO: it does not work with separator as none :(. fix for RGA
try: try:
df = pd.read_csv(tmp_filename, df = pd.read_csv(tmp_filename,
delimiter = separator, delimiter = separator,

193
src/hdf5_lib.py
View File

@ -17,6 +17,14 @@ import h5py
import yaml import yaml
def progressBar(count_value, total, suffix=''):
bar_length = 100
filled_up_Length = int(round(bar_length* count_value / float(total)))
percentage = round(100.0 * count_value/float(total),1)
bar = '=' * filled_up_Length + '-' * (bar_length - filled_up_Length)
sys.stdout.write('[%s] %s%s ...%s\r' %(bar, percentage, '%', suffix))
sys.stdout.flush()
def read_mtable_as_dataframe(filename): def read_mtable_as_dataframe(filename):
""" Reconstruct a Matlab Table encoded in a .h5 file as a Pandas DataFrame. The input .h5 file """ Reconstruct a Matlab Table encoded in a .h5 file as a Pandas DataFrame. The input .h5 file
@ -150,10 +158,11 @@ def get_parent_child_relationships(file: h5py.File):
parent = [''] parent = ['']
#values = [file.attrs['count']] #values = [file.attrs['count']]
# TODO: maybe we should make this more general and not dependent on file_list attribute? # TODO: maybe we should make this more general and not dependent on file_list attribute?
if 'file_list' in file.attrs.keys(): #if 'file_list' in file.attrs.keys():
values = [len(file.attrs['file_list'])] # values = [len(file.attrs['file_list'])]
else: #else:
values = [1] # values = [1]
values = [len(file.keys())]
def node_visitor(name,obj): def node_visitor(name,obj):
#if isinstance(obj,h5py.Group): #if isinstance(obj,h5py.Group):
@ -161,10 +170,12 @@ def get_parent_child_relationships(file: h5py.File):
parent.append(obj.parent.name) parent.append(obj.parent.name)
#nodes.append(os.path.split(obj.name)[1]) #nodes.append(os.path.split(obj.name)[1])
#parent.append(os.path.split(obj.parent.name)[1]) #parent.append(os.path.split(obj.parent.name)[1])
if isinstance(obj,h5py.Dataset) or not 'file_list' in obj.attrs.keys():
if isinstance(obj,h5py.Dataset):# or not 'file_list' in obj.attrs.keys():
values.append(1) values.append(1)
else: else:
values.append(len(obj.attrs['file_list'])) values.append(len(obj.keys()))
#values.append(len(obj.attrs['file_list']))
file.visititems(node_visitor) file.visititems(node_visitor)
return nodes, parent, values return nodes, parent, values
@ -204,15 +215,20 @@ def annotate_root_dir(filename,annotation_dict: dict):
import shutil import shutil
def create_hdf5_file_from_filesystem_path(ofilename : str, def create_hdf5_file_from_filesystem_path(config_param : dict ,
input_file_system_path : str, input_file_system_path : str,
select_dir_keywords = [], select_dir_keywords = [],
select_file_keywords =[], select_file_keywords =[],
top_sub_dir_mask : bool = True): top_sub_dir_mask : bool = True):
#def create_hdf5_file_from_filesystem_path(output_filename : str,
# input_file_system_path : str,
# select_dir_keywords = [],
# select_file_keywords =[],
# top_sub_dir_mask : bool = True):
""" """
Creates an .h5 file with name ofilename that preserves the directory tree (or folder structure) of given a filesystem path and Creates an .h5 file with name "output_filename" that preserves the directory tree (or folder structure) of given a filesystem path.
a few file and directory keywords. The keywords enable filtering of directories and files that do not contain the specified keywords. When file and directory keywords are non-empty, the keywords enable filtering of directories and files that do not contain the specified keywords.
In the .h5 file, only files that are admissible file formats will be stored in the form of datasets and attributes. In the .h5 file, only files that are admissible file formats will be stored in the form of datasets and attributes.
@ -237,35 +253,74 @@ def create_hdf5_file_from_filesystem_path(ofilename : str,
# Ensure OS compliant paths and keywords # Ensure OS compliant paths and keywords
# TODO: validate config_param dict, make sure output_filename is a valid file_path
group_id = config_param['group_id']
user_initials = config_param['user_initials']
created_at = config_file.created_at()
output_dir = config_param['output_dir']
output_filename = output_dir + config_file.output_filename_tempate(group_id,created_at,user_initials)
admissible_file_ext_list = list(config_file.select_file_readers(group_id).keys())
if '/' in input_file_system_path: if '/' in input_file_system_path:
input_file_system_path = input_file_system_path.replace('/',os.sep) input_file_system_path = input_file_system_path.replace('/',os.sep)
else: else:
raise ValueError('input_file_system_path needs to be specified using forward slashes "/".' ) raise ValueError('input_file_system_path needs to be specified using forward slashes "/".' )
for i, keyword in enumerate(select_dir_keywords): for i, keyword in enumerate(select_dir_keywords):
select_dir_keywords[i] = keyword.replace('/',os.sep) select_dir_keywords[i] = keyword.replace('/',os.sep)
# Visit each subdirectory from top to bottom, root directory defined by input_file_sytem_path to the lower
# level directories.
with h5py.File(ofilename, 'w') as h5file: # Constrain walkable paths on the specified directory tree by allowing walks that start from root
# through subdirectories specified by dir_keywords. This improves efficiency especially, in deep
# directory trees with many leaves.
paths = []
if top_sub_dir_mask:
for item in os.listdir(input_file_system_path):
if any([item in keyword for keyword in select_dir_keywords]):
paths.append(os.path.join(input_file_system_path,item))
else:
paths.append(input_file_system_path)
# Visit each subdirectory from top to bottom, root directory defined by input_file_sytem_path to the lower with h5py.File(output_filename, 'w') as h5file:
# level directories.
# Constrain walkable paths on the specified directory tree by allowing walks that start from root
# through subdirectories specified by dir_keywords. This improves efficiency especially, in deep
# directory trees with many leaves.
paths = []
if top_sub_dir_mask:
for item in os.listdir(input_file_system_path):
if any([item in keyword for keyword in select_dir_keywords]):
paths.append(os.path.join(input_file_system_path,item))
else:
paths.append(input_file_system_path)
for item in paths: for item in paths:
root_dir = input_file_system_path root_dir = input_file_system_path
# Create dictionary with directory-files pairs where files satisfy keyword and admisible type contraints
# It requires an extra pass over directory three and additional memory for dictionary, but it may be useful
# to speed up subsequent step and prune resulting directory tree.
# For each directory and/or subdirectory, keep files that satisfy file_keyword constraints, and store
# (directory_path, suitable files) relationships in a dictionary.
file_paths_dict = {}
check_file_ext = lambda filename: any([ext in filename for ext in admissible_file_ext_list])
for dirpath, _, filenames in os.walk(item,topdown=False):
file_paths_dict[dirpath] = []
# Check files that have an admissible extension and store them in admissible_filenames list
admissible_filenames = []
for fn in filenames:
if check_file_ext(fn):
admissible_filenames.append(fn)
if select_file_keywords: # when select_file_keywords = [], all files are considered
for filename in admissible_filenames:
# Do not consider files with types for which there is still no file_reader. TODO: extend file_reader library.
#if not any([ext in filename for ext in admissible_file_ext_list]):
# continue
# Add files with name, that contains any of the file_keywords
if any([keyword in filename for keyword in select_file_keywords]):
file_paths_dict[dirpath].append(filename)
else:
file_paths_dict[dirpath] = admissible_filenames
for node_number, node in enumerate(os.walk(item, topdown=True)): for node_number, node in enumerate(os.walk(item, topdown=True)):
dirpath, dirnames, filenames_list = node dirpath, dirnames, filenames_list = node
@ -277,26 +332,24 @@ def create_hdf5_file_from_filesystem_path(ofilename : str,
# When select_file_keywords is an empty, i.e., [], do not apply any filter on the filenames. # When select_file_keywords is an empty, i.e., [], do not apply any filter on the filenames.
filtered_filename_list = [] #filtered_filename_list = []
if select_file_keywords: #if select_file_keywords:
for filename in filenames_list: # for filename in filenames_list:
if any([keyword in filename for keyword in select_file_keywords]): # if any([keyword in filename for keyword in select_file_keywords]):
filtered_filename_list.append(filename) # filtered_filename_list.append(filename)
else: #else:
filtered_filename_list = filenames_list.copy() # filtered_filename_list = filenames_list.copy()
admissible_file_ext_list = list(config_file.ext_to_reader_dict.keys()) filtered_filename_list = file_paths_dict.get(dirpath,filenames_list.copy())
for filename in filtered_filename_list.copy():
if not any([ext in filename for ext in admissible_file_ext_list]):
filtered_filename_list.remove(filename)
# Skip subdirectories that do not contain a keyword in the parameter 'select_dir_keywords' when it is nonempty # Skip subdirectories that do not contain a keyword in the parameter 'select_dir_keywords' when it is nonempty
if select_dir_keywords: if select_dir_keywords:
#if (dirpath.count(os.sep) > offset) and not any([item in dirpath for item in select_dir_keywords]): #if (dirpath.count(os.sep) > offset) and not any([item in dirpath for item in select_dir_keywords]):
#tail, dirname = os.path.split(dirpath)
#if not any([item in dirname for item in select_dir_keywords]):
if not any([item in dirpath for item in select_dir_keywords]): if not any([item in dirpath for item in select_dir_keywords]):
continue continue
group_name = dirpath.replace(os.sep,'/') group_name = dirpath.replace(os.sep,'/')
group_name = group_name.replace(root_dir.replace(os.sep,'/') + '/', '/') group_name = group_name.replace(root_dir.replace(os.sep,'/') + '/', '/')
@ -308,46 +361,62 @@ def create_hdf5_file_from_filesystem_path(ofilename : str,
# TODO: for each "admissible" file in filenames, create an associated dataset in the corresponding group (subdirectory) # TODO: for each "admissible" file in filenames, create an associated dataset in the corresponding group (subdirectory)
for filename in filtered_filename_list: for filenumber, filename in enumerate(filtered_filename_list):
# Get file extension (or file type) # Get file extension (or file type)
file_name, file_ext = os.path.splitext(filename) file_name, file_ext = os.path.splitext(filename)
#print(filename)
#try: #try:
if not 'h5' in filename: if not 'h5' in filename:
file_dict = config_file.ext_to_reader_dict[file_ext](os.path.join(dirpath,filename)) file_dict = config_file.select_file_readers(group_id)[file_ext](os.path.join(dirpath,filename))
if not file_dict: if not file_dict:
continue continue
# Create group and add their attributes try:
h5file[group_name].create_group(name=file_dict['name']) # Create group and add their attributes
for key in file_dict['attributes_dict'].keys(): h5file[group_name].create_group(name=file_dict['name'])
for key in file_dict['attributes_dict'].keys():
# Represent string values as fixed length strings in the HDF5 file, which need
# to be decoded as string when we read them. It provides better control than variable strings, # Represent string values as fixed length strings in the HDF5 file, which need
# at the expense of flexibility. # to be decoded as string when we read them. It provides better control than variable strings,
# https://docs.h5py.org/en/stable/strings.html # at the expense of flexibility.
value = file_dict['attributes_dict'][key] # https://docs.h5py.org/en/stable/strings.html
if isinstance(value,str): value = file_dict['attributes_dict'][key]
utf8_type = h5py.string_dtype('utf-8', len(value)) if isinstance(value,str):
value = np.array(value.encode('utf-8'),dtype=utf8_type) utf8_type = h5py.string_dtype('utf-8', len(value))
value = np.array(value.encode('utf-8'),dtype=utf8_type)
h5file[group_name][file_dict['name']].attrs.create(name=key, h5file[group_name][file_dict['name']].attrs.create(name=key,
data=value) data=value)
# Add datasets to just created group # Add datasets to just created group
for dataset in file_dict['datasets']: for dataset in file_dict['datasets']:
h5file[group_name][file_dict['name']].create_dataset(name = dataset['name'], h5file[group_name][file_dict['name']].create_dataset(name = dataset['name'],
data = dataset['data'], data = dataset['data'],
#dtype = file_dict['dtype'], #dtype = file_dict['dtype'],
shape = dataset['shape']) shape = dataset['shape'])
except Exception as inst:
# TODO: log when a file could not be stored as a dataset
print(inst)
else: else:
config_file.ext_to_reader_dict[file_ext](source_file_path = os.path.join(dirpath,filename), config_file.select_file_readers(group_id)[file_ext](source_file_path = os.path.join(dirpath,filename),
dest_file_obj = h5file, dest_file_obj = h5file,
dest_group_name = group_name +'/'+filename) dest_group_name = group_name +'/'+filename)
print(file_ext, ':)') #print(filename,file_ext, ':)')
progressBar(filenumber,len(filtered_filename_list), 'Uploading files in ' + dirpath)
output_yml_filename_path = hdf5_vis.take_yml_snapshot_of_hdf5_file(output_filename)
return output_filename, output_yml_filename_path

2
src/hdf5_vis.py
View File

@ -140,7 +140,7 @@ def print_metadata(name, obj, folder_depth, yaml_dict):
#group_dict[obj.name]["name"] = obj.name #group_dict[obj.name]["name"] = obj.name
#group_dict[obj.name]["attributes"] = attr_dict #group_dict[obj.name]["attributes"] = attr_dict
#group_dict[obj.name]["datasets"] = {} #group_dict[obj.name]["datasets"] = {}
print(name) #print(name)
yaml_dict[obj.name] = group_dict yaml_dict[obj.name] = group_dict
elif isinstance(obj, h5py.Dataset): elif isinstance(obj, h5py.Dataset):

116
src/metadata_review_lib.py
View File

@ -35,16 +35,26 @@ def get_review_status(filename_path):
workflow_steps.append(line) workflow_steps.append(line)
return workflow_steps[-1] return workflow_steps[-1]
def checkout_review_branch(repo_obj,branch_name): def checkout_review_branch(branch_name):
# Create a new branch # Create a new branch
#branch_name = 'metadata-review-by-'+initials #branch_name = 'metadata-review-by-'+initials
head_commit = repo_obj.head.peel()# Get the commit hash associated with HEAD head_commit = repo_obj.head.peel()# Get the commit hash associated with HEAD
if not branch_name in repo_obj.branches: checkout_branch_command = lambda branch_name : ['git','checkout', branch_name]
branch = repo_obj.create_branch(branch_name, head_commit) output = subprocess.run(checkout_branch_command(branch_name), capture_output=True,text=True,check=True)
else:
branch = repo_obj.branches[branch_name] print(output.stdout)
repo_obj.checkout(branch)
#if not branch_name in repo_obj.branches:
# branch = repo_obj.create_branch(branch_name, head_commit)
#else:
# branch = repo_obj.branches[branch_name]
#repo_obj.checkout(branch)
current_branch_command = ['git','branch','--show-current']
status_command = ['git','status']
add_command = lambda add_list: ['git','add'] + add_list
commit_command = lambda message: ['git','commit','-m', message]
def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs): def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs):
@ -69,35 +79,55 @@ def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs):
raise ValueError("metadata review cannot be initialized. The associated .yaml file under review was not found. Run take_yml_snapshot_of_hdf5_file(filename_path) ") raise ValueError("metadata review cannot be initialized. The associated .yaml file under review was not found. Run take_yml_snapshot_of_hdf5_file(filename_path) ")
# Initialize metadata review workflow # Initialize metadata review workflow
print("Create branch metadata-review-by-"+initials+"\n") # print("Create branch metadata-review-by-"+initials+"\n")
#checkout_review_branch(branch_name)
# Check you are working at the right branch
current_branch_command = ['git','branch','--show-current']
curr_branch = subprocess.run(current_branch_command,capture_output=True,text=True,check=True)
if not branch_name in curr_branch.stdout:
raise ValueError("Please checkout the branch: "+branch_name+" via Git Bash Terminal while in the project's directory")
# Check if review file already exists and then check if it is still untracked # Check if review file already exists and then check if it is still untracked
review_yaml_file_path = os.path.join("review/",filename+YAML_EXT) review_yaml_file_path = os.path.join("review/",filename+YAML_EXT)
if not os.path.exists(review_yaml_file_path): if not os.path.exists(review_yaml_file_path):
review_yaml_file_path = utils.make_file_copy(os.path.join(hdf5_file_path_tail,filename+YAML_EXT), 'review') review_yaml_file_path = utils.make_file_copy(os.path.join(hdf5_file_path_tail,filename+YAML_EXT), 'review')
#else:
# raise Warning("the file " + os.path.join("review/",filename+YAML_EXT)+ " already exists. Delete this file to reinitialize the metadata review process.")
review_yaml_file_path_tail, ext = os.path.splitext(review_yaml_file_path) review_yaml_file_path_tail, ext = os.path.splitext(review_yaml_file_path)
with open(os.path.join(review_yaml_file_path_tail+"-review_status"+".txt"),'w') as f: with open(os.path.join(review_yaml_file_path_tail+"-review_status"+".txt"),'w') as f:
f.write('under review') f.write('under review')
checkout_review_branch(repo_obj, branch_name) # Stage review files and commit them to local repository
status = subprocess.run(status_command,capture_output=True,text=True,check=True)
untracked_files_for_review = []
for line in status.stdout.splitlines():
if 'review/' in line.decode('utf8'):
untracked_files_for_review.append(line)
status_dict = repo_obj.status() result = subprocess.run(add_command(untracked_files_for_review),capture_output=True,check=True)
for filepath, file_status in status_dict.items(): message = 'Initialized metadata review.'
commit_output = subprocess.run(commit_command(message),capture_output=True,check=True)
print(commit_output.stdout)
#status_dict = repo_obj.status()
#for filepath, file_status in status_dict.items():
# Identify keys associated to review files and stage them # Identify keys associated to review files and stage them
if 'review/'+filename in filepath: # if 'review/'+filename in filepath:
# Stage changes # Stage changes
repo_obj.index.add(filepath) # repo_obj.index.add(filepath)
author = config_file.author #default_signature #author = config_file.author #default_signature
committer = config_file.committer #committer = config_file.committer
message = "Initialized metadata review process." #message = "Initialized metadata review process."
tree = repo_obj.index.write_tree() #tree = repo_obj.index.write_tree()
oid = repo_obj.create_commit('HEAD', author, committer, message, tree, [repo_obj.head.peel().oid]) #oid = repo_obj.create_commit('HEAD', author, committer, message, tree, [repo_obj.head.peel().oid])
#print("Add and commit"+"\n") #print("Add and commit"+"\n")
@ -119,41 +149,33 @@ def second_submit_metadata_review(review_yaml_file_path, reviewer_attrs):
initials = reviewer_attrs['initials'] initials = reviewer_attrs['initials']
branch_name = '-'.join([reviewer_attrs['type'],'review','by',initials]) branch_name = '-'.join([reviewer_attrs['type'],'review','by',initials])
# TODO: replace with subprocess + git # TODO: replace with subprocess + git
checkout_review_branch(repo_obj, branch_name) #checkout_review_branch(repo_obj, branch_name)
# Check you are working at the right branch
curr_branch = subprocess.run(current_branch_command,capture_output=True,text=True,check=True)
if not branch_name in curr_branch.stdout:
raise ValueError('Please checkout ' + branch_name + ' via Git Bash before submitting metadata review files. ')
#if any([status in get_review_status(filename_path) for status in ['under review','submitted']]): # Collect modified review files
# filename_path_tail, filename_path_head = os.path.split(filename_path)
# filename, ext = os.path.splitext(filename_path_head)
# # TODO:
##
status_command = ['git','status']
add_command = lambda add_list: ['git','add'] + add_list
commit_command = lambda message: ['git','commit','-m', message]
#push_command = lambda repository,refspec: ['git','push',repository,refspec]
status = subprocess.run(status_command,capture_output=True,check=True) status = subprocess.run(status_command,capture_output=True,check=True)
modified_files = []
files_to_add_list = []
for line in status.stdout.splitlines(): for line in status.stdout.splitlines():
# conver line from bytes to str # conver line from bytes to str
tmp = line.decode("utf-8") tmp = line.decode("utf-8")
if 'modified' in tmp and review_yaml_file_path in tmp: if 'modified' in tmp and review_yaml_file_path in tmp:
files_to_add_list.append(tmp.split()[1]) modified_files.append(tmp.split()[1])
##
# Stage modified files and commit them to local repository
review_yaml_file_path_tail, review_yaml_file_path_head = os.path.split(review_yaml_file_path) review_yaml_file_path_tail, review_yaml_file_path_head = os.path.split(review_yaml_file_path)
filename, ext = os.path.splitext(review_yaml_file_path_head) filename, ext = os.path.splitext(review_yaml_file_path_head)
if files_to_add_list: if modified_files:
review_status_file_path = os.path.join("review/",filename+"-review_status"+TXT_EXT) review_status_file_path = os.path.join("review/",filename+"-review_status"+TXT_EXT)
with open(review_status_file_path,'a') as f: with open(review_status_file_path,'a') as f:
f.write('\nsubmitted') f.write('\nsubmitted')
files_to_add_list.append(review_status_file_path) modified_files.append(review_status_file_path)
result = subprocess.run(add_command(files_to_add_list),capture_output=True,check=True) result = subprocess.run(add_command(modified_files),capture_output=True,check=True)
message = 'Submitted metadata review.' message = 'Submitted metadata review.'
commit_output = subprocess.run(commit_command(message),capture_output=True,check=True) commit_output = subprocess.run(commit_command(message),capture_output=True,check=True)
@ -163,20 +185,6 @@ def second_submit_metadata_review(review_yaml_file_path, reviewer_attrs):
print('Nothing to commit.') print('Nothing to commit.')
#status_dict = repo_obj.status()
#for filepath, file_status in status_dict.items():
# Identify keys associated to review files and stage them
# if ('review/'+filename in filepath) and (file_status == pygit.GIT_STATUS_WT_MODIFIED):
# Stage changes
# repo_obj.index.add(filepath)
#author = config_file.author #default_signature
#committer = config_file.committer
#message = "Submitted metadata review."
#tree = repo_obj.index.write_tree()
#oid = repo_obj.create_commit('HEAD', author, committer, message, tree, [repo_obj.head.peel().oid])
def third_complete_metadata_review(reviewer_attrs): def third_complete_metadata_review(reviewer_attrs):

395
workflow_data_integration.ipynb
View File

@ -1,41 +1,55 @@
{ {
"cells": [ "cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Import python packages and modules"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"import os\n",
"# Set up project root directory\n",
"root_dir = os.path.abspath(os.curdir)\n",
"sys.path.append(root_dir)\n",
"\n",
"import src.hdf5_vis as hdf5_vis\n",
"import src.hdf5_lib as hdf5_lib\n",
"import input_files.config_file as config_file\n",
"\n",
"\n",
"output_dir = 'output_files/'\n",
"group_id = '5505'#'smog_chamber'#'5505'\n",
"user_initials = 'LL' #'NG' #'LL' # 'TBR'\n",
"\n",
"group_id = 'smog_chamber'#'5505'\n",
"user_initials = 'NG'#'LL' #'NG' #'LL' # 'TBR'\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Define input file path and keywords\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"2 26\n", "[==================================================--------------------------------------------------] 50.0% ...Uploading files in \\\\fs03\\Iron_Sulphate\\smps\\20220726\r"
".dat :)\n",
"2 26\n",
".dat :)\n",
"2 26\n",
".dat :)\n",
"2 26\n",
".dat :)\n",
"Humidity_Sensors\n",
"Humidity_Sensors/2022\n",
"Humidity_Sensors/2022/01_Januar\n",
"Humidity_Sensors/2022/02_Februar\n",
"Humidity_Sensors/2022/03_März\n",
"Humidity_Sensors/2022/04_April\n",
"Humidity_Sensors/2022/05_Mai\n",
"Humidity_Sensors/2022/06_Juni\n",
"Humidity_Sensors/2022/07_Juli\n",
"Humidity_Sensors/2022/10_Oktober\n",
"Humidity_Sensors/2022/11_November\n",
"Humidity_Sensors/2022/12_Dezember\n",
"ICAD\n",
"ICAD/HONO\n",
"ICAD/HONO/2022\n",
"ICAD/HONO_prototype\n",
"ICAD/HONO_prototype/2022\n",
"ICAD/NO2\n",
"ICAD/NO2/2022\n"
] ]
}, },
{ {
@ -49,237 +63,113 @@
"branchvalues": "remainder", "branchvalues": "remainder",
"customdata": [ "customdata": [
"<br>", "<br>",
"/Humidity_Sensors", "/gas",
"/Humidity_Sensors/2022", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar", "/gas/20220726_000004_MSC_gases.txt/categorial_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_000004_MSC_gases.txt/categorical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/gas/20220726_000004_MSC_gases.txt/numerical_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables", "/gas/20220726_000004_MSC_gases.txt/numerical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables", "/gas/20220726_101617_MSC_gases.txt/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar", "/gas/20220726_101617_MSC_gases.txt/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", "/gas/20220726_101617_MSC_gases.txt/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/gas/20220726_101617_MSC_gases.txt/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables", "/smps",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names", "/smps/20220726",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables", "/smps/20220726/20220726_mass.TXT",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", "/smps/20220726/20220726_mass.TXT/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names", "/smps/20220726/20220726_mass.TXT/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables", "/smps/20220726/20220726_mass.TXT/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names", "/smps/20220726/20220726_mass.TXT/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables", "/smps/20220726/20220726_num.TXT",
"/Humidity_Sensors/2022/03_März", "/smps/20220726/20220726_num.TXT/categorial_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", "/smps/20220726/20220726_num.TXT/categorical_variables",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/smps/20220726/20220726_num.TXT/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables", "/smps/20220726/20220726_num.TXT/numerical_variables"
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/04_April",
"/Humidity_Sensors/2022/05_Mai",
"/Humidity_Sensors/2022/06_Juni",
"/Humidity_Sensors/2022/07_Juli",
"/Humidity_Sensors/2022/10_Oktober",
"/Humidity_Sensors/2022/11_November",
"/Humidity_Sensors/2022/12_Dezember",
"/ICAD",
"/ICAD/HONO",
"/ICAD/HONO/2022",
"/ICAD/HONO/2022/10_Oct",
"/ICAD/HONO/2022/11_Nov",
"/ICAD/HONO/2022/12_Dec",
"/ICAD/HONO_prototype",
"/ICAD/HONO_prototype/2022",
"/ICAD/HONO_prototype/2022/01_Jan",
"/ICAD/HONO_prototype/2022/02_Feb",
"/ICAD/NO2",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022/01_Jan",
"/ICAD/NO2/2022/02_Feb",
"/ICAD/NO2/2022/03_Mar",
"/ICAD/NO2/2022/04_Apr",
"/ICAD/NO2/2022/05_May",
"/ICAD/NO2/2022/06_June",
"/ICAD/NO2/2022/07_July",
"/ICAD/NO2/2022/10_Oct",
"/ICAD/NO2/2022/11_Nov",
"/ICAD/NO2/2022/12_Dec"
], ],
"hovertemplate": "<b>%{label} </b> <br> Count: %{value} <br> Path: %{customdata}", "hovertemplate": "<b>%{label} </b> <br> Count: %{value} <br> Path: %{customdata}",
"labels": [ "labels": [
"/", "/",
"/Humidity_Sensors", "/gas",
"/Humidity_Sensors/2022", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar", "/gas/20220726_000004_MSC_gases.txt/categorial_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_000004_MSC_gases.txt/categorical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/gas/20220726_000004_MSC_gases.txt/numerical_variable_names",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/categorical_variables", "/gas/20220726_000004_MSC_gases.txt/numerical_variables",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variable_names", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat/numerical_variables", "/gas/20220726_101617_MSC_gases.txt/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar", "/gas/20220726_101617_MSC_gases.txt/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", "/gas/20220726_101617_MSC_gases.txt/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/gas/20220726_101617_MSC_gases.txt/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/categorical_variables", "/smps",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variable_names", "/smps/20220726",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat/numerical_variables", "/smps/20220726/20220726_mass.TXT",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat", "/smps/20220726/20220726_mass.TXT/categorial_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorial_variable_names", "/smps/20220726/20220726_mass.TXT/categorical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/categorical_variables", "/smps/20220726/20220726_mass.TXT/numerical_variable_names",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variable_names", "/smps/20220726/20220726_mass.TXT/numerical_variables",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat/numerical_variables", "/smps/20220726/20220726_num.TXT",
"/Humidity_Sensors/2022/03_März", "/smps/20220726/20220726_num.TXT/categorial_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat", "/smps/20220726/20220726_num.TXT/categorical_variables",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorial_variable_names", "/smps/20220726/20220726_num.TXT/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/categorical_variables", "/smps/20220726/20220726_num.TXT/numerical_variables"
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variable_names",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat/numerical_variables",
"/Humidity_Sensors/2022/04_April",
"/Humidity_Sensors/2022/05_Mai",
"/Humidity_Sensors/2022/06_Juni",
"/Humidity_Sensors/2022/07_Juli",
"/Humidity_Sensors/2022/10_Oktober",
"/Humidity_Sensors/2022/11_November",
"/Humidity_Sensors/2022/12_Dezember",
"/ICAD",
"/ICAD/HONO",
"/ICAD/HONO/2022",
"/ICAD/HONO/2022/10_Oct",
"/ICAD/HONO/2022/11_Nov",
"/ICAD/HONO/2022/12_Dec",
"/ICAD/HONO_prototype",
"/ICAD/HONO_prototype/2022",
"/ICAD/HONO_prototype/2022/01_Jan",
"/ICAD/HONO_prototype/2022/02_Feb",
"/ICAD/NO2",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022/01_Jan",
"/ICAD/NO2/2022/02_Feb",
"/ICAD/NO2/2022/03_Mar",
"/ICAD/NO2/2022/04_Apr",
"/ICAD/NO2/2022/05_May",
"/ICAD/NO2/2022/06_June",
"/ICAD/NO2/2022/07_July",
"/ICAD/NO2/2022/10_Oct",
"/ICAD/NO2/2022/11_Nov",
"/ICAD/NO2/2022/12_Dec"
], ],
"name": "", "name": "",
"parents": [ "parents": [
"", "",
"/", "/",
"/Humidity_Sensors", "/gas",
"/Humidity_Sensors/2022", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_000004_MSC_gases.txt",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas",
"/Humidity_Sensors/2022/01_Januar/2022-01-31_09.37.56_PC14751_Humidity_Sensors.dat", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/02_Februar", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat", "/gas/20220726_101617_MSC_gases.txt",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_09.07.50_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/02_Februar/2022-02-11_16.46.26_PC10228_Humidity_Sensors.dat",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022/03_März",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022/03_März/2022-03-14_09.05.01_PC14751_Humidity_Sensors.dat",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/Humidity_Sensors/2022",
"/", "/",
"/ICAD", "/smps",
"/ICAD/HONO", "/smps/20220726",
"/ICAD/HONO/2022", "/smps/20220726/20220726_mass.TXT",
"/ICAD/HONO/2022", "/smps/20220726/20220726_mass.TXT",
"/ICAD/HONO/2022", "/smps/20220726/20220726_mass.TXT",
"/ICAD", "/smps/20220726/20220726_mass.TXT",
"/ICAD/HONO_prototype", "/smps/20220726",
"/ICAD/HONO_prototype/2022", "/smps/20220726/20220726_num.TXT",
"/ICAD/HONO_prototype/2022", "/smps/20220726/20220726_num.TXT",
"/ICAD", "/smps/20220726/20220726_num.TXT",
"/ICAD/NO2", "/smps/20220726/20220726_num.TXT"
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022",
"/ICAD/NO2/2022"
], ],
"root": { "root": {
"color": "lightgrey" "color": "lightgrey"
}, },
"type": "treemap", "type": "treemap",
"values": [ "values": [
1, 2,
1, 2,
0, 4,
5,
1,
1,
1,
1,
1,
14,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
6,
1,
1, 1,
1, 1,
1, 1,
1, 1,
4, 4,
9, 1,
11, 1,
3, 1,
8, 1,
17, 1,
2, 2,
1,
1,
0,
7,
8,
2,
1,
1,
3,
6,
1,
0,
3,
6,
5,
3,
4, 4,
6, 1,
2, 1,
5, 1,
8, 1,
2 4,
1,
1,
1,
1
] ]
} }
], ],
@ -1116,18 +1006,37 @@
} }
], ],
"source": [ "source": [
"import sys\n", "#input_file_dir = '//fs101/5505/People/Juan/TypicalBeamTime'\n",
"import os\n", "#select_file_keywords=[]\n",
"root_dir = os.path.abspath(os.curdir)\n", "#select_dir_keywords = ['NEXAFS', 'Notes', 'Photos', 'Pressure', 'RGA', 'SES']\n",
"sys.path.append(root_dir)\n",
"\n", "\n",
"import src.hdf5_vis as hdf5_vis\n",
"import src.hdf5_lib as hdf5_lib\n",
"\n", "\n",
"output_filename_path, output_yml_filename_path = hdf5_lib.main()\n", "#input_file_dir = '//fs101/5505/Data' \n",
"#select_dir_keywords = ['Lopap', 'Humidity_Sensors/2022', 'ICAD/HONO/2022', 'ICAD/NO2/2022', 'T200_NOX', 'T360U_CO2']\n",
"#select_file_keywords = ['2022-03-25','2022_03_25','20220325']\n",
"\n",
"\n",
"input_file_dir = '//fs03/Iron_Sulphate'\n",
"select_dir_keywords = ['gas','smps/20220726']#,'htof/2022.07.26','ptr/2022.07.26','ams/2022.07.26']\n",
"#select_dir_keywords = ['htof','ams', 'ptr', 'gas','smps'] \n",
"\n",
"select_file_keywords = ['20220726','2022.07.26']\n",
"\n",
"config_param = {'group_id' : group_id, 'user_initials' : user_initials, 'output_dir': output_dir}\n",
"\n",
"\n",
"output_filename_path, output_yml_filename_path = hdf5_lib.create_hdf5_file_from_filesystem_path(config_param,\n",
" input_file_dir,\n",
" select_dir_keywords,\n",
" select_file_keywords)\n",
"\n", "\n",
"hdf5_vis.display_group_hierarchy_on_a_treemap(output_filename_path)" "hdf5_vis.display_group_hierarchy_on_a_treemap(output_filename_path)"
] ]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
} }
], ],
"metadata": { "metadata": {

160
workflow_data_owner_review.ipynb Normal file
View File

@ -0,0 +1,160 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Import libraries and modules\n",
"\n",
"* Excecute (or Run) Cell"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"import os\n",
"root_dir = os.path.abspath(os.curdir)\n",
"sys.path.append(root_dir)\n",
"\n",
"import src.metadata_review_lib as metadata_review_lib"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Step 1: initialize metadata review.\n",
"\n",
"* Specify hdf5 file whose metadata is to be reviewed by editing the string variable `hdf5_file_path`.\n",
"* Edit reviewer attributes, i.e., the dict variable `reviewer_attrs` with your own initials and role.\n",
"* Excecute Cell."
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Create branch metadata-review-by-NG\n",
"\n"
]
},
{
"ename": "ValueError",
"evalue": "('Please checkout the branch: ', 'data-owner-review-by-NG', \" via Git Bash Terminal while in the project's directory\")",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[1;32mIn[2], line 12\u001b[0m\n\u001b[0;32m 7\u001b[0m reviewer_attrs \u001b[38;5;241m=\u001b[39m {\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minitials\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mNG\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m 8\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata-owner\u001b[39m\u001b[38;5;124m'\u001b[39m}\n\u001b[0;32m 10\u001b[0m \u001b[38;5;66;03m#output_filename_path, output_yml_filename_path = hdf5_lib.main()\u001b[39;00m\n\u001b[1;32m---> 12\u001b[0m review_yaml_file_path \u001b[38;5;241m=\u001b[39m \u001b[43mmetadata_review_lib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfirst_initialize_metadata_review\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhdf5_file_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreviewer_attrs\u001b[49m\u001b[43m)\u001b[49m \n\u001b[0;32m 14\u001b[0m \u001b[38;5;28mprint\u001b[39m(review_yaml_file_path) \n",
"File \u001b[1;32mc:\\Users\\florez_j\\Documents\\GitLab\\functionspython\\src\\metadata_review_lib.py:89\u001b[0m, in \u001b[0;36mfirst_initialize_metadata_review\u001b[1;34m(hdf5_file_path, reviewer_attrs)\u001b[0m\n\u001b[0;32m 87\u001b[0m curr_branch \u001b[38;5;241m=\u001b[39m subprocess\u001b[38;5;241m.\u001b[39mrun(current_branch_command,capture_output\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,text\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,check\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m 88\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m branch_name \u001b[38;5;129;01min\u001b[39;00m curr_branch\u001b[38;5;241m.\u001b[39mstdout:\n\u001b[1;32m---> 89\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease checkout the branch: \u001b[39m\u001b[38;5;124m\"\u001b[39m,branch_name,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m via Git Bash Terminal while in the project\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms directory\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 91\u001b[0m \u001b[38;5;66;03m# Check if review file already exists and then check if it is still untracked\u001b[39;00m\n\u001b[0;32m 92\u001b[0m review_yaml_file_path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mreview/\u001b[39m\u001b[38;5;124m\"\u001b[39m,filename\u001b[38;5;241m+\u001b[39mYAML_EXT)\n",
"\u001b[1;31mValueError\u001b[0m: ('Please checkout the branch: ', 'data-owner-review-by-NG', \" via Git Bash Terminal while in the project's directory\")"
]
}
],
"source": [
"\n",
"#hdf5_file_path = \"output_files/unified_file_smog_chamber_2024-03-25_UTC-OFST_+0100_NG.h5\"\n",
"#yml_file_path = \"output_files/unified_file_smog_chamber_2024-03-25_UTC-OFST_+0100_NG.yaml\"\n",
"\n",
"hdf5_file_path = \"output_files/unified_file_smog_chamber_2024-04-03_UTC-OFST_+0200_NG.h5\"\n",
"yml_file_path = \"output_files/unified_file_smog_chamber_2024-04-03_UTC-OFST_+0200_NG.yaml\"\n",
"\n",
"reviewer_attrs = {'initials': 'NG',\n",
" 'type': 'data-owner'}\n",
"\n",
"#output_filename_path, output_yml_filename_path = hdf5_lib.main()\n",
"\n",
"review_yaml_file_path = metadata_review_lib.first_initialize_metadata_review(hdf5_file_path, reviewer_attrs) \n",
"\n",
"print(review_yaml_file_path) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Step 2: Submit metadata review. \n",
"\n",
"* Edit yaml file in review folder and save changes\n",
"* Excecute Cell."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"metadata_review_lib.second_submit_metadata_review(review_yaml_file_path,reviewer_attrs)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Step 3: Update hdf5 file metadata w/ submitted review yaml file.\n",
"\n",
"* Make sure previous step was carried out properly.\n",
"* Excecute Cell."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"metadata_review_lib.third_update_hdf5_file_with_review(hdf5_file_path, review_yaml_file_path, reviewer_attrs)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Step 4: Complete data-owner review. Update remote repository\n",
"\n",
"* Excecute Cell."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"metadata_review_lib.fourth_complete_metadata_review(reviewer_attrs)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "test_atmos_chem_env",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

18
workflow_metadata_reviewer.ipynb Normal file
View File

@ -0,0 +1,18 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 2
}