diff --git a/src/g5505_file_reader.py b/src/g5505_file_reader.py index c86a2ee..97abb1a 100644 --- a/src/g5505_file_reader.py +++ b/src/g5505_file_reader.py @@ -77,6 +77,7 @@ def read_txt_files_as_dict(filename : str ): #TODO: this may be prone to error if assumed folder structure is non compliant + file_encoding = 'utf-8' if 'RGA' in filename: #TODO: it does not work with separator as none :(. fix for RGA #end_of_header = 'Channel, Mass(amu), Name, Cal Factor, Noise Floor, CEM Status', table_header = 'Time(s) Channel#1 Channel#2 Channel#3 Channel#4 Channel#5 Channel#6 Channel#7 Channel#8' @@ -89,9 +90,10 @@ def read_txt_files_as_dict(filename : str ): elif 'Humidity_Sensors' in filename: table_header = 'Date Time RH1[%] RH2[%] RH3[%] RH4[%] RH5[%] RH6[%] RH7[%] RH8[%] T1[°C] T2[°C] T3[°C] T4[°C] T5[°C] T6[°C] T7[°C] T8[°C] DP1[°C] DP2[°C] DP3[°C] DP4[°C] DP5[°C] DP6[°C] DP7[°C] DP8[°C]' separator = '\t' + file_encoding = 'latin-1' elif 'ICAD' in filename and 'HONO' in filename: table_header = 'Start Date/Time (UTC) Duration (s) NO2 (ppb) NO2 Uncertainty (ppb) H2O (ppb) H2O Uncertainty (ppb) CHOCHO (ppb) CHOCHO Uncertainty (ppb) File Number Light Intensity #ICEDOAS iter. Cell Pressure Ambient Pressure Cell Temp Spec Temp Lat Lon Height Speed GPSQuality 0-Air Ref. Time 0-Air Ref. Duration 0-Air Ref. File Number 0-Air Ref. Intensity 0-Air Ref. Rel Intensity 0-Air Ref. Intensity valid MeasMode SampleSource' - separator = '\t' + separator = '\t' elif 'ICAD' in filename and 'NO2' in filename: table_header = 'Start Date/Time (UTC) Duration (s) NO2 (ppb) NO2 Uncertainty (ppb) H2O (ppb) H2O Uncertainty (ppb) CHOCHO (ppb) CHOCHO Uncertainty (ppb) File Number Light Intensity #ICEDOAS iter. Cell Pressure Ambient Pressure Cell Temp Spec Temp Lat Lon Height Speed GPSQuality 0-Air Ref. Time 0-Air Ref. Duration 0-Air Ref. File Number 0-Air Ref. Intensity 0-Air Ref. Rel Intensity 0-Air Ref. Intensity valid MeasMode SampleSource' separator = '\t' @@ -105,13 +107,12 @@ def read_txt_files_as_dict(filename : str ): # Work with copy of the file for safety tmp_filename = utils.make_file_copy(source_file_path=filename) - with open(tmp_filename,'r') as f: - file_encoding = f.encoding + with open(tmp_filename,'r',encoding=file_encoding,errors='ignore') as f: + #file_encoding = f.encoding #table_preamble = "" table_preamble = [] for line_number, line in enumerate(f): - table_preamble += line if table_header in line: list_of_substrings = line.split(separator) data_start = True