5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Removed data table split into categorical and numerical variables and numering is only introduce to disambiguate repeated columns.

Browse Source
This commit is contained in:
florez_j
2024-06-10 16:18:51 +02:00
parent 1ec7ad76ff
commit a410bde23e
1 changed files with 37 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/g5505_file_reader.py
View File

@ -112,6 +112,8 @@ def dataframe_to_np_structured_array(df: pd.DataFrame):
return structured_array return structured_array
from collections import Counter
def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ): def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ):
with open('src/instruments/text_data_sources.yaml','r') as stream: with open('src/instruments/text_data_sources.yaml','r') as stream:
@ -164,8 +166,14 @@ def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ):
if table_header in line.decode(file_encoding): if table_header in line.decode(file_encoding):
list_of_substrings = line.decode(file_encoding).split(separator) list_of_substrings = line.decode(file_encoding).split(separator)
# Count occurrences of each substring
substring_counts = Counter(list_of_substrings)
data_start = True data_start = True
column_names = [str(i)+'_'+name.strip() for i, name in enumerate(list_of_substrings)] # Generate column names with appended index only for repeated substrings
column_names = [f"{i}_{name.strip()}" if substring_counts[name] > 1 else name.strip() for i, name in enumerate(list_of_substrings)]
#column_names = [str(i)+'_'+name.strip() for i, name in enumerate(list_of_substrings)]
#column_names = [] #column_names = []
#for i, name in enumerate(list_of_substrings): #for i, name in enumerate(list_of_substrings):
# column_names.append(str(i)+'_'+name) # column_names.append(str(i)+'_'+name)
@ -210,25 +218,28 @@ def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ):
#df_categorical_attrs['timestamps'] = [ df_categorical_attrs.loc[i,'0_Date']+' '+df_categorical_attrs.loc[i,'1_Time'] for i in df.index] #df_categorical_attrs['timestamps'] = [ df_categorical_attrs.loc[i,'0_Date']+' '+df_categorical_attrs.loc[i,'1_Time'] for i in df.index]
df_categorical_attrs['timestamps'] = df_categorical_attrs[timestamp_variables].astype(str).agg(' '.join, axis=1) #df_categorical_attrs['timestamps'] = df_categorical_attrs[timestamp_variables].astype(str).agg(' '.join, axis=1)
timestamps_name = ' '.join(timestamp_variables)
df_categorical_attrs[ timestamps_name] = df_categorical_attrs[timestamp_variables].astype(str).agg(' '.join, axis=1)
valid_indices = [] valid_indices = []
if datetime_format: if datetime_format:
df_categorical_attrs['timestamps'] = pd.to_datetime(df_categorical_attrs['timestamps'],format=datetime_format,errors='coerce') df_categorical_attrs[ timestamps_name] = pd.to_datetime(df_categorical_attrs[ timestamps_name],format=datetime_format,errors='coerce')
valid_indices = df_categorical_attrs.dropna(subset=['timestamps']).index valid_indices = df_categorical_attrs.dropna(subset=[timestamps_name]).index
df_categorical_attrs = df_categorical_attrs.loc[valid_indices,:] df_categorical_attrs = df_categorical_attrs.loc[valid_indices,:]
df_numerical_attrs = df_numerical_attrs.loc[valid_indices,:] df_numerical_attrs = df_numerical_attrs.loc[valid_indices,:]
df_categorical_attrs['timestamps'] = df_categorical_attrs['timestamps'].dt.strftime(config_dict['default']['desired_format']) df_categorical_attrs[timestamps_name] = df_categorical_attrs[timestamps_name].dt.strftime(config_dict['default']['desired_format'])
startdate = df_categorical_attrs['timestamps'].min() startdate = df_categorical_attrs[timestamps_name].min()
enddate = df_categorical_attrs['timestamps'].max() enddate = df_categorical_attrs[timestamps_name].max()
df_categorical_attrs['timestamps'] = df_categorical_attrs['timestamps'].astype(str) df_categorical_attrs[timestamps_name] = df_categorical_attrs[timestamps_name].astype(str)
#header_dict.update({'stastrrtdate':startdate,'enddate':enddate}) #header_dict.update({'stastrrtdate':startdate,'enddate':enddate})
header_dict['startdate']= str(startdate) header_dict['startdate']= str(startdate)
header_dict['enddate']=str(enddate) header_dict['enddate']=str(enddate)
df_categorical_attrs = df_categorical_attrs.drop(columns = timestamp_variables) if len(timestamp_variables) > 1:
df_categorical_attrs = df_categorical_attrs.drop(columns = timestamp_variables)
#df_categorical_attrs.reindex(drop=True) #df_categorical_attrs.reindex(drop=True)
@ -253,12 +264,11 @@ def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ):
if numerical_variables: if numerical_variables:
dataset = {} dataset = {}
dataset['name'] = 'table_numerical_variables' dataset['name'] = 'data_table'#_numerical_variables'
dataset['data'] = dataframe_to_np_structured_array(df_numerical_attrs) #df_numerical_attrs.to_numpy() dataset['data'] = dataframe_to_np_structured_array(pd.concat((df_categorical_attrs,df_numerical_attrs),axis=1)) #df_numerical_attrs.to_numpy()
dataset['shape'] = dataset['data'].shape dataset['shape'] = dataset['data'].shape
dataset['dtype'] = type(dataset['data']) dataset['dtype'] = type(dataset['data'])
#dataset['data_units'] = file_obj['wave']['data_units'] #dataset['data_units'] = file_obj['wave']['data_units']
file_dict['datasets'].append(dataset)
try: try:
dataset['attributes'] = description_dict['table_header'].copy() dataset['attributes'] = description_dict['table_header'].copy()
@ -267,21 +277,25 @@ def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ):
dataset['attributes'].pop(key) # delete key dataset['attributes'].pop(key) # delete key
else: else:
dataset['attributes'][key] = metadata.parse_attribute(dataset['attributes'][key]) dataset['attributes'][key] = metadata.parse_attribute(dataset['attributes'][key])
if timestamps_name in categorical_variables:
dataset['attributes'][timestamps_name] = metadata.parse_attribute({'unit':'YYYY-MM-DD HH:MM:SS.ffffff'})
except ValueError as err: except ValueError as err:
print(err) print(err)
if categorical_variables:
dataset = {}
dataset['name'] = 'table_categorical_variables'
dataset['data'] = dataframe_to_np_structured_array(df_categorical_attrs) #df_categorical_attrs.loc[:,categorical_variables].to_numpy()
dataset['shape'] = dataset['data'].shape
dataset['dtype'] = type(dataset['data'])
if 'timestamps' in categorical_variables:
dataset['attributes'] = {'timestamps': metadata.parse_attribute({'unit':'YYYY-MM-DD HH:MM:SS.ffffff'})}
file_dict['datasets'].append(dataset) file_dict['datasets'].append(dataset)
#if categorical_variables:
# dataset = {}
# dataset['name'] = 'table_categorical_variables'
# dataset['data'] = dataframe_to_np_structured_array(df_categorical_attrs) #df_categorical_attrs.loc[:,categorical_variables].to_numpy()
# dataset['shape'] = dataset['data'].shape
# dataset['dtype'] = type(dataset['data'])
# if timestamps_name in categorical_variables:
# dataset['attributes'] = {timestamps_name: metadata.parse_attribute({'unit':'YYYY-MM-DD HH:MM:SS.ffffff'})}
# file_dict['datasets'].append(dataset)
except: except: