Modified reader to output table_preamble as a dataset as opposed to attributes of the file. I believe this is better for readability of the metadata given that those preambles can sometimes contain large ammounts of text.
This commit is contained in:
@ -186,15 +186,6 @@ def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ):
|
|||||||
#line = ' '.join(list_of_substrings)
|
#line = ' '.join(list_of_substrings)
|
||||||
table_preamble.append(' '.join([item for item in list_of_substrings]))# += new_line
|
table_preamble.append(' '.join([item for item in list_of_substrings]))# += new_line
|
||||||
|
|
||||||
# Represent string values as fixed length strings in the HDF5 file, which need
|
|
||||||
# to be decoded as string when we read them. It provides better control than variable strings,
|
|
||||||
# at the expense of flexibility.
|
|
||||||
# https://docs.h5py.org/en/stable/strings.html
|
|
||||||
|
|
||||||
if table_preamble:
|
|
||||||
header_dict["table_preamble"] = utils.convert_string_to_bytes(table_preamble)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# TODO: it does not work with separator as none :(. fix for RGA
|
# TODO: it does not work with separator as none :(. fix for RGA
|
||||||
try:
|
try:
|
||||||
@ -290,6 +281,22 @@ def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ):
|
|||||||
#except ValueError as err:
|
#except ValueError as err:
|
||||||
# print(err)
|
# print(err)
|
||||||
|
|
||||||
|
# Represent string values as fixed length strings in the HDF5 file, which need
|
||||||
|
# to be decoded as string when we read them. It provides better control than variable strings,
|
||||||
|
# at the expense of flexibility.
|
||||||
|
# https://docs.h5py.org/en/stable/strings.html
|
||||||
|
|
||||||
|
|
||||||
|
if table_preamble:
|
||||||
|
#header_dict["table_preamble"] = utils.convert_string_to_bytes(table_preamble)
|
||||||
|
tp_dataset = {}
|
||||||
|
tp_dataset['name'] = "table_preamble"
|
||||||
|
tp_dataset['data'] = utils.convert_string_to_bytes(table_preamble)
|
||||||
|
tp_dataset['shape'] = tp_dataset['data'].shape
|
||||||
|
tp_dataset['dtype'] = type(tp_dataset['data'])
|
||||||
|
tp_dataset['attributes'] = {}
|
||||||
|
file_dict['datasets'].append(tp_dataset)
|
||||||
|
|
||||||
file_dict['datasets'].append(dataset)
|
file_dict['datasets'].append(dataset)
|
||||||
|
|
||||||
|
|
||||||
@ -302,10 +309,6 @@ def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ):
|
|||||||
# if timestamps_name in categorical_variables:
|
# if timestamps_name in categorical_variables:
|
||||||
# dataset['attributes'] = {timestamps_name: utils.parse_attribute({'unit':'YYYY-MM-DD HH:MM:SS.ffffff'})}
|
# dataset['attributes'] = {timestamps_name: utils.parse_attribute({'unit':'YYYY-MM-DD HH:MM:SS.ffffff'})}
|
||||||
# file_dict['datasets'].append(dataset)
|
# file_dict['datasets'].append(dataset)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
except:
|
except:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user