5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added the feature to activate or deactivate data copying before reading the input file. This is to avoid redundant copying when we are already working on file copies.

Browse Source
This commit is contained in:
florez_j
2024-05-28 14:40:14 +02:00
parent e0d84d7822
commit 0f505df45c
1 changed files with 10 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/g5505_file_reader.py
View File

@ -62,10 +62,13 @@ def read_xps_ibw_file_as_dict(filename):
return file_dict return file_dict
def copy_file_in_group(source_file_path, dest_file_obj : h5py.File, dest_group_name): def copy_file_in_group(source_file_path, dest_file_obj : h5py.File, dest_group_name, work_with_copy : bool = True):
# Create copy of original file to avoid possible file corruption and work with it. # Create copy of original file to avoid possible file corruption and work with it.
tmp_file_path = utils.make_file_copy(source_file_path) if work_with_copy:
tmp_file_path = utils.make_file_copy(source_file_path)
else:
tmp_file_path = source_file_path
# Open backup h5 file and copy complet filesystem directory onto a group in h5file # Open backup h5 file and copy complet filesystem directory onto a group in h5file
with h5py.File(tmp_file_path,'r') as src_file: with h5py.File(tmp_file_path,'r') as src_file:
@ -109,7 +112,7 @@ def dataframe_to_np_structured_array(df: pd.DataFrame):
return structured_array return structured_array
def read_txt_files_as_dict(filename : str ): def read_txt_files_as_dict(filename : str , work_with_copy : bool = True ):
with open('src/text_data_sources.yaml','r') as stream: with open('src/text_data_sources.yaml','r') as stream:
try: try:
@ -148,7 +151,10 @@ def read_txt_files_as_dict(filename : str ):
header_dict = {} header_dict = {}
data_start = False data_start = False
# Work with copy of the file for safety # Work with copy of the file for safety
tmp_filename = utils.make_file_copy(source_file_path=filename) if work_with_copy:
tmp_filename = utils.make_file_copy(source_file_path=filename)
else:
tmp_filename = filename
#with open(tmp_filename,'rb',encoding=file_encoding,errors='ignore') as f: #with open(tmp_filename,'rb',encoding=file_encoding,errors='ignore') as f:
with open(tmp_filename,'rb') as f: with open(tmp_filename,'rb') as f: