Removed smogchamber reader because its funtionality is now integrated into g5505_file_reader.py.
This commit is contained in:
@ -1,176 +0,0 @@
|
|||||||
import os
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pandas as pd
|
|
||||||
from igor2.binarywave import load as loadibw
|
|
||||||
|
|
||||||
import src.g5505_utils as utils
|
|
||||||
|
|
||||||
|
|
||||||
import h5py
|
|
||||||
|
|
||||||
#def read_txt_files_as_dict(filename : str ,instrument_folder : str):
|
|
||||||
def read_txt_files_as_dict(filename : str ):
|
|
||||||
|
|
||||||
#if instrument_folder == 'smps':
|
|
||||||
# Infer from filename whether txt file comes from smps or gas folder
|
|
||||||
#TODO: this may be prone to error if assumed folder structure is non compliant
|
|
||||||
if 'smps' in filename:
|
|
||||||
table_of_header = 'Sample # Date Start Time Sample Temp (C) Sample Pressure (kPa)'
|
|
||||||
separator = '\t'
|
|
||||||
file_encoding = 'latin-1'
|
|
||||||
elif 'gas' in filename:
|
|
||||||
table_of_header = 'Date_Time HoribaNO HoribaNOy Thermo42C_NO Thermo42C_NOx APHA370 CH4'
|
|
||||||
separator = '\t'
|
|
||||||
file_encoding = 'utf-8'
|
|
||||||
else:
|
|
||||||
raise ValueError('intrument_folder must be set as a either "smps" or "gas"')
|
|
||||||
|
|
||||||
tmp_file_path = utils.make_file_copy(filename)
|
|
||||||
|
|
||||||
# Read header as a dictionary and detect where data table starts
|
|
||||||
header_dict = {}
|
|
||||||
data_start = False
|
|
||||||
with open(tmp_file_path,'r', encoding=file_encoding, errors='ignore') as f:
|
|
||||||
#file_encoding = f.encoding
|
|
||||||
#table_preamble = ""
|
|
||||||
table_preamble = []
|
|
||||||
line_number = 0
|
|
||||||
for line_number, line in enumerate(f):
|
|
||||||
#print(line_number,line)
|
|
||||||
if table_of_header in line:
|
|
||||||
list_of_substrings = line.split(separator)
|
|
||||||
data_start = True
|
|
||||||
column_names = []
|
|
||||||
for i, name in enumerate(list_of_substrings):
|
|
||||||
column_names.append(str(i)+'_'+name)
|
|
||||||
|
|
||||||
#print(line_number, len(column_names ))
|
|
||||||
break
|
|
||||||
# Subdivide line into words, and join them by single space.
|
|
||||||
# I asumme this can produce a cleaner line that contains no weird separator characters \t \r or extra spaces and so on.
|
|
||||||
list_of_substrings = line.split()
|
|
||||||
# TODO: ideally we should use a multilinear string but the yalm parser is not recognizing \n as special character
|
|
||||||
#line = ' '.join(list_of_substrings+['\n'])
|
|
||||||
line = ' '.join(list_of_substrings)
|
|
||||||
table_preamble.append(line)# += new_line
|
|
||||||
|
|
||||||
header_dict["table_preamble"] = table_preamble #.replace('\n','\\n').replace('\t','\\t')
|
|
||||||
|
|
||||||
if not data_start:
|
|
||||||
raise ValueError('Invalid table header. The table header was not found and therefore table data cannot be extracted from txt or dat file.')
|
|
||||||
|
|
||||||
df = pd.read_csv(tmp_file_path,
|
|
||||||
delimiter = separator,
|
|
||||||
header=line_number,
|
|
||||||
#encoding='latin-1',
|
|
||||||
encoding= file_encoding,
|
|
||||||
names=column_names,
|
|
||||||
skip_blank_lines=True)
|
|
||||||
|
|
||||||
df_numerical_attrs = df.select_dtypes(include ='number')
|
|
||||||
df_categorical_attrs = df.select_dtypes(exclude='number')
|
|
||||||
|
|
||||||
if 'smps' in tmp_file_path:
|
|
||||||
df_categorical_attrs['timestamps'] = [ df_categorical_attrs.loc[i,'1_Date']+' '+df_categorical_attrs.loc[i,'2_Start Time'] for i in df.index]
|
|
||||||
df_categorical_attrs = df_categorical_attrs.drop(columns=['1_Date','2_Start Time'])
|
|
||||||
elif 'gas' in tmp_file_path:
|
|
||||||
df_categorical_attrs = df_categorical_attrs.rename(columns={'0_Date_Time' : 'timestamps'})
|
|
||||||
|
|
||||||
#data_column_names = [item.encode("utf-8") for item in df_numerical_attrs.columns]
|
|
||||||
numerical_variables = [item for item in df_numerical_attrs.columns]
|
|
||||||
categorical_variables = [item for item in df_categorical_attrs.columns]
|
|
||||||
|
|
||||||
###
|
|
||||||
file_dict = {}
|
|
||||||
path_tail, path_head = os.path.split(tmp_file_path)
|
|
||||||
|
|
||||||
file_dict['name'] = path_head
|
|
||||||
# TODO: review this header dictionary, it may not be the best way to represent header data
|
|
||||||
file_dict['attributes_dict'] = header_dict
|
|
||||||
file_dict['datasets'] = []
|
|
||||||
####
|
|
||||||
|
|
||||||
if numerical_variables:
|
|
||||||
dataset = {}
|
|
||||||
dataset['name'] = 'numerical_variables'
|
|
||||||
dataset['data'] = df_numerical_attrs.to_numpy()
|
|
||||||
dataset['shape'] = dataset['data'].shape
|
|
||||||
dataset['dtype'] = type(dataset['data'])
|
|
||||||
#dataset['data_units'] = file_obj['wave']['data_units']
|
|
||||||
file_dict['datasets'].append(dataset)
|
|
||||||
rows,cols = dataset['shape']
|
|
||||||
|
|
||||||
# This lines were added to test the structured array functionality
|
|
||||||
tmp = [tuple(dataset['data'][i,:]) for i in range(dataset['shape'][0])]
|
|
||||||
dtype_tmp = [(numerical_variables[i],'f4') for i in range(dataset['shape'][1])]
|
|
||||||
|
|
||||||
data = np.array(tmp, dtype=dtype_tmp)
|
|
||||||
dataset['data'] = data
|
|
||||||
dataset['shape'] = dataset['data'].shape
|
|
||||||
|
|
||||||
dataset = {}
|
|
||||||
numerical_variables= [item.encode("utf-8") for item in numerical_variables]
|
|
||||||
dataset['name'] = 'numerical_variable_names'
|
|
||||||
dataset['data'] = np.array(numerical_variables).reshape((1,cols))
|
|
||||||
dataset['shape'] = dataset['data'].shape
|
|
||||||
dataset['dtype'] = type(dataset['data'])
|
|
||||||
file_dict['datasets'].append(dataset)
|
|
||||||
|
|
||||||
#if 'timestamps' in categorical_variables:
|
|
||||||
# dataset = {}
|
|
||||||
# dataset['name'] = 'timestamps'
|
|
||||||
# dataset['data'] = df_categorical_attrs['timestamps'].to_numpy().reshape((rows,1))
|
|
||||||
# dataset['shape'] = dataset['data'].shape
|
|
||||||
# dataset['dtype'] = type(dataset['data'])
|
|
||||||
# file_dict['datasets'].append(dataset)
|
|
||||||
# categorical_variables.remove('timestamps')
|
|
||||||
|
|
||||||
if categorical_variables:
|
|
||||||
dataset = {}
|
|
||||||
dataset['name'] = 'categorical_variables'
|
|
||||||
dataset['data'] = df_categorical_attrs.loc[:,categorical_variables].to_numpy()
|
|
||||||
dataset['shape'] = dataset['data'].shape
|
|
||||||
dataset['dtype'] = type(dataset['data'])
|
|
||||||
file_dict['datasets'].append(dataset)
|
|
||||||
|
|
||||||
dataset = {}
|
|
||||||
categorical_variables = [item.encode("utf-8") for item in categorical_variables]
|
|
||||||
dataset['name'] = 'categorial_variable_names'
|
|
||||||
dataset['data'] = np.array(categorical_variables).reshape((1,len(categorical_variables)))
|
|
||||||
dataset['shape'] = dataset['data'].shape
|
|
||||||
dataset['dtype'] = type(dataset['data'])
|
|
||||||
file_dict['datasets'].append(dataset)
|
|
||||||
|
|
||||||
return file_dict
|
|
||||||
|
|
||||||
def main():
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#filename = 'M:\\gas\\20220705_000004_MSC_gases.txt' corrupted file
|
|
||||||
#filename = 'M:\\gas\\20220726_101617_MSC_gases.txt'
|
|
||||||
root_dir = '\\\\fs03\\Iron_Sulphate'
|
|
||||||
|
|
||||||
instrument_folder = 'smps'
|
|
||||||
filename_path = os.path.join(root_dir,'smps\\20220726\\20220726_num.TXT')
|
|
||||||
|
|
||||||
|
|
||||||
instrument_folder = 'gas'
|
|
||||||
filename_path = os.path.join(root_dir,'gas\\20220726_101617_MSC_gases.txt')
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
result = read_txt_files_as_dict(filename_path,instrument_folder)
|
|
||||||
|
|
||||||
print(':)')
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
output_dict = main()
|
|
||||||
|
|
||||||
print(output_dict['num_data_df'].columns, len(output_dict['num_data_df'].columns),'\n')
|
|
||||||
print(output_dict['num_data_df'].columns, len(output_dict['categ_data_df'].columns))
|
|
||||||
Reference in New Issue
Block a user