Skip unreadable files

For #41

pyzebra/app/panel_ccl_integrate.py

@@ -72,7 +72,7 @@ for (let i = 0; i < js_data.data['fname'].length; i++) {
 
 def create():
     doc = curdoc()
-    det_data = {}
+    det_data = []
     fit_params = {}
     js_data = ColumnDataSource(data=dict(content=["", ""], fname=["", ""], ext=["", ""]))
 
@@ -132,20 +132,28 @@ def create():
 
     def file_open_button_callback():
         nonlocal det_data
-        for f_ind, f_path in enumerate(file_select.value):
+        new_data = []
+        for f_path in file_select.value:
             with open(f_path) as file:
-                base, ext = os.path.splitext(os.path.basename(f_path))
+                f_name = os.path.basename(f_path)
+                base, ext = os.path.splitext(f_name)
+                try:
                     file_data = pyzebra.parse_1D(file, ext)
+                except:
+                    print(f"Error loading {f_name}")
+                    continue
 
             pyzebra.normalize_dataset(file_data, monitor_spinner.value)
 
-            if f_ind == 0:  # first file
+            if not new_data:  # first file
-                det_data = file_data
+                new_data = file_data
-                pyzebra.merge_duplicates(det_data)
+                pyzebra.merge_duplicates(new_data)
                 js_data.data.update(fname=[base, base])
             else:
-                pyzebra.merge_datasets(det_data, file_data)
+                pyzebra.merge_datasets(new_data, file_data)
 
+        if new_data:
+            det_data = new_data
             _init_datatable()
             append_upload_button.disabled = False
 
@@ -153,14 +161,21 @@ def create():
     file_open_button.on_click(file_open_button_callback)
 
     def file_append_button_callback():
+        file_data = []
         for f_path in file_select.value:
             with open(f_path) as file:
-                _, ext = os.path.splitext(f_path)
+                f_name = os.path.basename(f_path)
+                _, ext = os.path.splitext(f_name)
+                try:
                     file_data = pyzebra.parse_1D(file, ext)
+                except:
+                    print(f"Error loading {f_name}")
+                    continue
 
             pyzebra.normalize_dataset(file_data, monitor_spinner.value)
             pyzebra.merge_datasets(det_data, file_data)
 
+        if file_data:
             _init_datatable()
 
     file_append_button = Button(label="Append", width=100, disabled=True)
@@ -168,21 +183,27 @@ def create():
 
     def upload_button_callback(_attr, _old, _new):
         nonlocal det_data
-        det_data = []
+        new_data = []
         for f_str, f_name in zip(upload_button.value, upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 base, ext = os.path.splitext(f_name)
+                try:
                     file_data = pyzebra.parse_1D(file, ext)
+                except:
+                    print(f"Error loading {f_name}")
+                    continue
 
             pyzebra.normalize_dataset(file_data, monitor_spinner.value)
 
-            if not det_data:  # first file
+            if not new_data:  # first file
-                det_data = file_data
+                new_data = file_data
-                pyzebra.merge_duplicates(det_data)
+                pyzebra.merge_duplicates(new_data)
                 js_data.data.update(fname=[base, base])
             else:
-                pyzebra.merge_datasets(det_data, file_data)
+                pyzebra.merge_datasets(new_data, file_data)
 
+        if new_data:
+            det_data = new_data
             _init_datatable()
             append_upload_button.disabled = False
 
@@ -193,14 +214,20 @@ def create():
     upload_button.on_change("filename", upload_button_callback)
 
     def append_upload_button_callback(_attr, _old, _new):
+        file_data = []
         for f_str, f_name in zip(append_upload_button.value, append_upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 _, ext = os.path.splitext(f_name)
+                try:
                     file_data = pyzebra.parse_1D(file, ext)
+                except:
+                    print(f"Error loading {f_name}")
+                    continue
 
             pyzebra.normalize_dataset(file_data, monitor_spinner.value)
             pyzebra.merge_datasets(det_data, file_data)
 
+        if file_data:
             _init_datatable()
 
     append_upload_div = Div(text="append extra files:", margin=(5, 5, 0, 5))

pyzebra/app/panel_param_study.py

@@ -134,20 +134,28 @@ def create():
 
     def file_open_button_callback():
         nonlocal det_data
-        for f_ind, f_path in enumerate(file_select.value):
+        new_data = []
+        for f_path in file_select.value:
             with open(f_path) as file:
-                base, ext = os.path.splitext(os.path.basename(f_path))
+                f_name = os.path.basename(f_path)
+                base, ext = os.path.splitext(f_name)
+                try:
                     file_data = pyzebra.parse_1D(file, ext)
+                except:
+                    print(f"Error loading {f_name}")
+                    continue
 
             pyzebra.normalize_dataset(file_data, monitor_spinner.value)
 
-            if f_ind == 0:  # first file
+            if not new_data:  # first file
-                det_data = file_data
+                new_data = file_data
-                pyzebra.merge_duplicates(det_data)
+                pyzebra.merge_duplicates(new_data)
                 js_data.data.update(fname=[base])
             else:
-                pyzebra.merge_datasets(det_data, file_data)
+                pyzebra.merge_datasets(new_data, file_data)
 
+        if new_data:
+            det_data = new_data
             _init_datatable()
             append_upload_button.disabled = False
 
@@ -155,14 +163,21 @@ def create():
     file_open_button.on_click(file_open_button_callback)
 
     def file_append_button_callback():
+        file_data = []
         for f_path in file_select.value:
             with open(f_path) as file:
-                _, ext = os.path.splitext(f_path)
+                f_name = os.path.basename(f_path)
+                _, ext = os.path.splitext(f_name)
+                try:
                     file_data = pyzebra.parse_1D(file, ext)
+                except:
+                    print(f"Error loading {f_name}")
+                    continue
 
             pyzebra.normalize_dataset(file_data, monitor_spinner.value)
             pyzebra.merge_datasets(det_data, file_data)
 
+        if file_data:
             _init_datatable()
 
     file_append_button = Button(label="Append", width=100, disabled=True)
@@ -170,21 +185,27 @@ def create():
 
     def upload_button_callback(_attr, _old, _new):
         nonlocal det_data
-        det_data = []
+        new_data = []
         for f_str, f_name in zip(upload_button.value, upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 base, ext = os.path.splitext(f_name)
+                try:
                     file_data = pyzebra.parse_1D(file, ext)
+                except:
+                    print(f"Error loading {f_name}")
+                    continue
 
             pyzebra.normalize_dataset(file_data, monitor_spinner.value)
 
-            if not det_data:  # first file
+            if not new_data:  # first file
-                det_data = file_data
+                new_data = file_data
-                pyzebra.merge_duplicates(det_data)
+                pyzebra.merge_duplicates(new_data)
                 js_data.data.update(fname=[base])
             else:
-                pyzebra.merge_datasets(det_data, file_data)
+                pyzebra.merge_datasets(new_data, file_data)
 
+        if new_data:
+            det_data = new_data
             _init_datatable()
             append_upload_button.disabled = False
 
@@ -195,14 +216,20 @@ def create():
     upload_button.on_change("filename", upload_button_callback)
 
     def append_upload_button_callback(_attr, _old, _new):
+        file_data = []
         for f_str, f_name in zip(append_upload_button.value, append_upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 _, ext = os.path.splitext(f_name)
+                try:
                     file_data = pyzebra.parse_1D(file, ext)
+                except:
+                    print(f"Error loading {f_name}")
+                    continue
 
             pyzebra.normalize_dataset(file_data, monitor_spinner.value)
             pyzebra.merge_datasets(det_data, file_data)
 
+        if file_data:
             _init_datatable()
 
     append_upload_div = Div(text="append extra files:", margin=(5, 5, 0, 5))

pyzebra/ccl_io.py

@@ -192,6 +192,7 @@ def parse_1D(fileobj, data_type):
         match = re.search("(.*) Points, Mode: (.*), Preset (.*)", next(fileobj))
         if match.group(2) != "Monitor":
             raise Exception("Unknown mode in dat file.")
+        s["n_points"] = int(match.group(1))
         s["monitor"] = float(match.group(3))
 
         col_names = list(map(str.lower, next(fileobj).split()))