From 09b6e4fdcfe81eb88d9c9f071db3d2a8747fc406 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 22 Oct 2021 15:36:42 +0200 Subject: [PATCH] Skip unreadable files For #41 --- pyzebra/app/panel_ccl_integrate.py | 73 ++++++++++++++++++++---------- pyzebra/app/panel_param_study.py | 71 ++++++++++++++++++++--------- pyzebra/ccl_io.py | 1 + 3 files changed, 100 insertions(+), 45 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index cad0978..415c971 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -72,7 +72,7 @@ for (let i = 0; i < js_data.data['fname'].length; i++) { def create(): doc = curdoc() - det_data = {} + det_data = [] fit_params = {} js_data = ColumnDataSource(data=dict(content=["", ""], fname=["", ""], ext=["", ""])) @@ -132,59 +132,80 @@ def create(): def file_open_button_callback(): nonlocal det_data - for f_ind, f_path in enumerate(file_select.value): + new_data = [] + for f_path in file_select.value: with open(f_path) as file: - base, ext = os.path.splitext(os.path.basename(f_path)) - file_data = pyzebra.parse_1D(file, ext) + f_name = os.path.basename(f_path) + base, ext = os.path.splitext(f_name) + try: + file_data = pyzebra.parse_1D(file, ext) + except: + print(f"Error loading {f_name}") + continue pyzebra.normalize_dataset(file_data, monitor_spinner.value) - if f_ind == 0: # first file - det_data = file_data - pyzebra.merge_duplicates(det_data) + if not new_data: # first file + new_data = file_data + pyzebra.merge_duplicates(new_data) js_data.data.update(fname=[base, base]) else: - pyzebra.merge_datasets(det_data, file_data) + pyzebra.merge_datasets(new_data, file_data) - _init_datatable() - append_upload_button.disabled = False + if new_data: + det_data = new_data + _init_datatable() + append_upload_button.disabled = False file_open_button = Button(label="Open New", width=100, disabled=True) file_open_button.on_click(file_open_button_callback) def file_append_button_callback(): + file_data = [] for f_path in file_select.value: with open(f_path) as file: - _, ext = os.path.splitext(f_path) - file_data = pyzebra.parse_1D(file, ext) + f_name = os.path.basename(f_path) + _, ext = os.path.splitext(f_name) + try: + file_data = pyzebra.parse_1D(file, ext) + except: + print(f"Error loading {f_name}") + continue pyzebra.normalize_dataset(file_data, monitor_spinner.value) pyzebra.merge_datasets(det_data, file_data) - _init_datatable() + if file_data: + _init_datatable() file_append_button = Button(label="Append", width=100, disabled=True) file_append_button.on_click(file_append_button_callback) def upload_button_callback(_attr, _old, _new): nonlocal det_data - det_data = [] + new_data = [] for f_str, f_name in zip(upload_button.value, upload_button.filename): with io.StringIO(base64.b64decode(f_str).decode()) as file: base, ext = os.path.splitext(f_name) - file_data = pyzebra.parse_1D(file, ext) + try: + file_data = pyzebra.parse_1D(file, ext) + except: + print(f"Error loading {f_name}") + continue pyzebra.normalize_dataset(file_data, monitor_spinner.value) - if not det_data: # first file - det_data = file_data - pyzebra.merge_duplicates(det_data) + if not new_data: # first file + new_data = file_data + pyzebra.merge_duplicates(new_data) js_data.data.update(fname=[base, base]) else: - pyzebra.merge_datasets(det_data, file_data) + pyzebra.merge_datasets(new_data, file_data) - _init_datatable() - append_upload_button.disabled = False + if new_data: + det_data = new_data + _init_datatable() + append_upload_button.disabled = False upload_div = Div(text="or upload new .ccl/.dat files:", margin=(5, 5, 0, 5)) upload_button = FileInput(accept=".ccl,.dat", multiple=True, width=200) @@ -193,15 +214,21 @@ def create(): upload_button.on_change("filename", upload_button_callback) def append_upload_button_callback(_attr, _old, _new): + file_data = [] for f_str, f_name in zip(append_upload_button.value, append_upload_button.filename): with io.StringIO(base64.b64decode(f_str).decode()) as file: _, ext = os.path.splitext(f_name) - file_data = pyzebra.parse_1D(file, ext) + try: + file_data = pyzebra.parse_1D(file, ext) + except: + print(f"Error loading {f_name}") + continue pyzebra.normalize_dataset(file_data, monitor_spinner.value) pyzebra.merge_datasets(det_data, file_data) - _init_datatable() + if file_data: + _init_datatable() append_upload_div = Div(text="append extra files:", margin=(5, 5, 0, 5)) append_upload_button = FileInput(accept=".ccl,.dat", multiple=True, width=200, disabled=True) diff --git a/pyzebra/app/panel_param_study.py b/pyzebra/app/panel_param_study.py index dab3344..dcac5cf 100644 --- a/pyzebra/app/panel_param_study.py +++ b/pyzebra/app/panel_param_study.py @@ -134,59 +134,80 @@ def create(): def file_open_button_callback(): nonlocal det_data - for f_ind, f_path in enumerate(file_select.value): + new_data = [] + for f_path in file_select.value: with open(f_path) as file: - base, ext = os.path.splitext(os.path.basename(f_path)) - file_data = pyzebra.parse_1D(file, ext) + f_name = os.path.basename(f_path) + base, ext = os.path.splitext(f_name) + try: + file_data = pyzebra.parse_1D(file, ext) + except: + print(f"Error loading {f_name}") + continue pyzebra.normalize_dataset(file_data, monitor_spinner.value) - if f_ind == 0: # first file - det_data = file_data - pyzebra.merge_duplicates(det_data) + if not new_data: # first file + new_data = file_data + pyzebra.merge_duplicates(new_data) js_data.data.update(fname=[base]) else: - pyzebra.merge_datasets(det_data, file_data) + pyzebra.merge_datasets(new_data, file_data) - _init_datatable() - append_upload_button.disabled = False + if new_data: + det_data = new_data + _init_datatable() + append_upload_button.disabled = False file_open_button = Button(label="Open New", width=100, disabled=True) file_open_button.on_click(file_open_button_callback) def file_append_button_callback(): + file_data = [] for f_path in file_select.value: with open(f_path) as file: - _, ext = os.path.splitext(f_path) - file_data = pyzebra.parse_1D(file, ext) + f_name = os.path.basename(f_path) + _, ext = os.path.splitext(f_name) + try: + file_data = pyzebra.parse_1D(file, ext) + except: + print(f"Error loading {f_name}") + continue pyzebra.normalize_dataset(file_data, monitor_spinner.value) pyzebra.merge_datasets(det_data, file_data) - _init_datatable() + if file_data: + _init_datatable() file_append_button = Button(label="Append", width=100, disabled=True) file_append_button.on_click(file_append_button_callback) def upload_button_callback(_attr, _old, _new): nonlocal det_data - det_data = [] + new_data = [] for f_str, f_name in zip(upload_button.value, upload_button.filename): with io.StringIO(base64.b64decode(f_str).decode()) as file: base, ext = os.path.splitext(f_name) - file_data = pyzebra.parse_1D(file, ext) + try: + file_data = pyzebra.parse_1D(file, ext) + except: + print(f"Error loading {f_name}") + continue pyzebra.normalize_dataset(file_data, monitor_spinner.value) - if not det_data: # first file - det_data = file_data - pyzebra.merge_duplicates(det_data) + if not new_data: # first file + new_data = file_data + pyzebra.merge_duplicates(new_data) js_data.data.update(fname=[base]) else: - pyzebra.merge_datasets(det_data, file_data) + pyzebra.merge_datasets(new_data, file_data) - _init_datatable() - append_upload_button.disabled = False + if new_data: + det_data = new_data + _init_datatable() + append_upload_button.disabled = False upload_div = Div(text="or upload new .ccl/.dat files:", margin=(5, 5, 0, 5)) upload_button = FileInput(accept=".ccl,.dat", multiple=True, width=200) @@ -195,15 +216,21 @@ def create(): upload_button.on_change("filename", upload_button_callback) def append_upload_button_callback(_attr, _old, _new): + file_data = [] for f_str, f_name in zip(append_upload_button.value, append_upload_button.filename): with io.StringIO(base64.b64decode(f_str).decode()) as file: _, ext = os.path.splitext(f_name) - file_data = pyzebra.parse_1D(file, ext) + try: + file_data = pyzebra.parse_1D(file, ext) + except: + print(f"Error loading {f_name}") + continue pyzebra.normalize_dataset(file_data, monitor_spinner.value) pyzebra.merge_datasets(det_data, file_data) - _init_datatable() + if file_data: + _init_datatable() append_upload_div = Div(text="append extra files:", margin=(5, 5, 0, 5)) append_upload_button = FileInput(accept=".ccl,.dat", multiple=True, width=200, disabled=True) diff --git a/pyzebra/ccl_io.py b/pyzebra/ccl_io.py index afa22e4..34f835b 100644 --- a/pyzebra/ccl_io.py +++ b/pyzebra/ccl_io.py @@ -192,6 +192,7 @@ def parse_1D(fileobj, data_type): match = re.search("(.*) Points, Mode: (.*), Preset (.*)", next(fileobj)) if match.group(2) != "Monitor": raise Exception("Unknown mode in dat file.") + s["n_points"] = int(match.group(1)) s["monitor"] = float(match.group(3)) col_names = list(map(str.lower, next(fileobj).split()))