5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleaned up code by removing commented lines and so on.

Browse Source
This commit is contained in:
florez_j
2024-02-21 10:47:12 +01:00
parent 219435511b
commit 79b7428b9f
2 changed files with 35 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/config_file.py
View File

@@ -3,17 +3,20 @@ import smog_chamber_file_reader as scf_reader
import g5505_file_reader as g5505f_reader
select_file_readers = 'smog_chamber'
#select_file_readers = '5505'
select_file_readers = '5505'
if select_file_readers == 'smog_chamber':
#from smog_chamber_file_reader import read_txt_files_as_dict
#from g5505_file_reader import copy_file_in_group
select_dir_keywords = ['htof','ams', 'ptr', 'gas','smps']
#select_dir_keywords = ['htof','ams', 'ptr', 'gas','smps']
inputfile_dir = '\\\\fs03\\Iron_Sulphate'
select_dir_keywords = ['gas','smps\\20220726','htof\\2022.07.26','ptr\\2022.07.26','ams\\2022.07.26']
select_file_keywords = ['20220726','2022.07.26']
output_filename = 'test_smog_chamber_v8.h5'
outputfile_dir = './output_files'
output_filename = 'test_smog_chamber_v11.h5'
ext_to_reader_dict = {'.txt': scf_reader.read_txt_files_as_dict,
'.TXT': scf_reader.read_txt_files_as_dict,
@@ -21,13 +24,14 @@ if select_file_readers == 'smog_chamber':
elif select_file_readers == '5505':
output_filename = 'test_sls_data_v5.h5'
outputfile_dir = './output_files'
output_filename = 'test_sls_data_v8.h5'
inputfile_dir = '\\\\fs101\\5505\\People\\Juan\\TypicalBeamTime'
select_file_keywords=[]
select_dir_keywords = ['NEXAFS', 'Notes', 'Photos', 'Pressure', 'RGA', 'SES']
#from g5505_file_reader import read_xps_ibw_file_as_dict, copy_file_in_group, read_txt_files_as_dict
ext_to_reader_dict = {'.ibw': g5505f_reader.read_xps_ibw_file_as_dict,
'.txt': g5505f_reader.read_txt_files_as_dict,
#'.txt': g5505f_reader.read_txt_files_as_dict,
'.dat': g5505f_reader.read_txt_files_as_dict,
'.h5': g5505f_reader.copy_file_in_group}

119
src/hdf5_lib.py
View File

@@ -207,27 +207,29 @@ def display_group_hierarchy_on_a_treemap(filename: str):
metadata_list.append(key[key.find('_')+1::]+':'+file.attrs[key])
metadata = '<br>'.join(['<br>'] + metadata_list)
customdata_series = pd.Series(nodes)
customdata_series[0] = metadata
customdata_series = pd.Series(nodes)
customdata_series[0] = metadata
fig = make_subplots(1, 1, specs=[[{"type": "domain"}]],)
fig.add_trace(go.Treemap(
labels=nodes, #formating_df['formated_names'][nodes],
parents=parents,#formating_df['formated_names'][parents],
values=values,
branchvalues='remainder',
customdata= customdata_series,
#marker=dict(
# colors=df_all_trees['color'],
# colorscale='RdBu',
# cmid=average_score),
#hovertemplate='<b>%{label} </b> <br> Number of files: %{value}<br> Success rate: %{color:.2f}',
hovertemplate='<b>%{label} </b> <br> Count: %{value} <br> Path: %{customdata}',
name='',
root_color="lightgrey"
))
fig.update_layout(width = 800, height= 600, margin = dict(t=50, l=25, r=25, b=25))
fig.show()
fig = make_subplots(1, 1, specs=[[{"type": "domain"}]],)
fig.add_trace(go.Treemap(
labels=nodes, #formating_df['formated_names'][nodes],
parents=parents,#formating_df['formated_names'][parents],
values=values,
branchvalues='remainder',
customdata= customdata_series,
#marker=dict(
# colors=df_all_trees['color'],
# colorscale='RdBu',
# cmid=average_score),
#hovertemplate='<b>%{label} </b> <br> Number of files: %{value}<br> Success rate: %{color:.2f}',
hovertemplate='<b>%{label} </b> <br> Count: %{value} <br> Path: %{customdata}',
name='',
root_color="lightgrey"
))
fig.update_layout(width = 800, height= 600, margin = dict(t=50, l=25, r=25, b=25))
fig.show()
file_name, file_ext = os.path.splitext(filename)
fig.write_html(file_name + ".html")
def annotate_root_dir(filename,annotation_dict: dict):
with h5py.File(filename,'r+') as file:
@@ -321,11 +323,6 @@ def create_hdf5_file_from_filesystem_path(ofilename : str, input_file_system_pat
# TODO: for each "admissible" file in filenames, create an associated dataset in the corresponding group (subdirectory)
tmp_dirpath = os.path.join(os.getcwd(), 'tmp')
if not os.path.exists(tmp_dirpath):
os.mkdir(tmp_dirpath)
for filename in filtered_filename_list:
# Get file extension (or file type)
@@ -359,56 +356,6 @@ def create_hdf5_file_from_filesystem_path(ofilename : str, input_file_system_pat
#if 'ibw' in filename:
#file_dict = g5505_file_reader.read_xps_ibw_file_as_dict(os.path.join(dirpath,filename))
#h5file[group_name][file_dict['name']].dims[0] = file_dict['dimension_units']
#if 'h5' in filename:
# Create copy of original file to avoid possible file corruption and work with it.
#backup_filename = 'backup_'+filename
# Path
#shutil.copy(os.path.join(dirpath,filename), os.path.join(tmp_dirpath,backup_filename))
# Open backup h5 file and copy complet filesystem directory onto a group in h5file
#with h5py.File(os.path.join(tmp_dirpath,backup_filename),'r') as src_file:
# h5file.copy(source=src_file['/'],dest= group_name +'/'+filename)
# h5file.copy(source= file_obj, dest= group_name +'/'+filename)
# TODO: generilize to multiphase chemistry text and dat files
# TODO: include header information from files as well
# if ('txt' in filename or 'TXT' in filename) and any([item in os.path.join(dirpath,filename) for item in ['smps','gas']]):
# #if 'smps' in os.path.join(dirpath,filename):
# # file_dict = smog_chamber_file_reader.read_txt_files_as_dict(os.path.join(dirpath,filename),'smps')
# #elif 'gas' in os.path.join(dirpath,filename):
# file_dict = smog_chamber_file_reader.read_txt_files_as_dict(os.path.join(dirpath,filename),'gas')
# file_dict = file_obj
# # TODO: create datasets of compound data type to include variable/or column names and datetimestamps
# h5file[group_name].create_group(filename)
# h5file[group_name][filename].create_dataset(name = 'data',
# data = file_dict['data'],
#dtype = file_dict['dtype'],
# shape = file_dict['data'].shape)
# h5file[group_name][filename].create_dataset(name = 'data_column_names',
# data = np.array(file_dict['data_column_names']),
#dtype = file_dict['dtype'],
# shape = np.array(file_dict['data_column_names']).shape)
# for key in file_dict['categ_data_dict'].keys():
# h5file[group_name][filename].create_dataset(name=key,data=file_dict['categ_data_dict'][key])
def create_hdf5_file_from_dataframe(ofilename, input_data, approach : str, group_by_funcs : list, extract_attrs_func = None):
""" Creates an hdf5 file with as many levels as indicated by len(group_by_funcs).
@@ -510,29 +457,15 @@ def create_hdf5_file_from_dataframe(ofilename, input_data, approach : str, group
#return 0
def main_5505():
inputfile_dir = '\\\\fs101\\5505\\People\\Juan\\TypicalBeamTime'
select_file_keywords=[]
select_dir_keywords = ['NEXAFS', 'Notes', 'Photos', 'Pressure', 'RGA', 'SES']
create_hdf5_file_from_filesystem_path('test_sls_data_v3.h5',inputfile_dir,select_dir_keywords,select_file_keywords)
display_group_hierarchy_on_a_treemap('test_sls_data_v3.h5')
#create_hdf5_file('test', inputfile_dir, 'Topdown', [group_by_type], extract_attrs_func = None)
def main():
def main():
#select_dir_keywords = ['htof','ams', 'ptr', 'gas','smps']
output_filename = config_file.output_filename
inputfile_dir = config_file.inputfile_dir #'\\\\fs03\\Iron_Sulphate'
select_dir_keywords = config_file.select_dir_keywords #['gas','smps\\20220726','htof\\2022.07.26','ptr\\2022.07.26','ams\\2022.07.26']
select_file_keywords = config_file.select_file_keywords #['20220726','2022.07.26']
create_hdf5_file_from_filesystem_path(output_filename,inputfile_dir,select_dir_keywords,select_file_keywords)
display_group_hierarchy_on_a_treemap(output_filename)
output_filename_path = os.path.join(config_file.outputfile_dir,config_file.output_filename)
create_hdf5_file_from_filesystem_path(output_filename_path,inputfile_dir,select_dir_keywords,select_file_keywords)
display_group_hierarchy_on_a_treemap(output_filename_path)
def main_mtable_h5_from_dataframe():