Updating for version 0.5.1

Fix #40

.github/workflows/deployment.yaml

@@ -16,7 +16,6 @@ jobs:
       run: |
         $CONDA/bin/conda install --quiet --yes conda-build anaconda-client
         $CONDA/bin/conda config --append channels conda-forge
-        $CONDA/bin/conda config --set channel_priority strict
         $CONDA/bin/conda config --set anaconda_upload yes
 
     - name: Build and upload

conda-recipe/meta.yaml

@@ -22,7 +22,7 @@ requirements:
     - numpy
     - scipy
     - h5py
-    - bokeh =2.3
+    - bokeh =2.4
     - numba
     - lmfit
 

pyzebra/__init__.py

@@ -1,11 +1,8 @@
 from pyzebra.anatric import *
 from pyzebra.ccl_io import *
-from pyzebra.h5 import *
-from pyzebra.xtal import *
 from pyzebra.ccl_process import *
+from pyzebra.h5 import *
+from pyzebra.utils import *
+from pyzebra.xtal import *
 
-ZEBRA_PROPOSALS_PATHS = [
-    f"/afs/psi.ch/project/sinqdata/{year}/zebra/" for year in (2016, 2017, 2018, 2020, 2021)
-]
-
-__version__ = "0.5.0"
+__version__ = "0.5.1"

pyzebra/app/app.py

@@ -2,9 +2,10 @@ import logging
 import sys
 from io import StringIO
 
+import pyzebra
 from bokeh.io import curdoc
 from bokeh.layouts import column, row
-from bokeh.models import Tabs, TextAreaInput
+from bokeh.models import Button, Panel, Tabs, TextAreaInput, TextInput
 
 import panel_ccl_integrate
 import panel_hdf_anatric
@@ -13,7 +14,6 @@ import panel_hdf_viewer
 import panel_param_study
 import panel_spind
 
-
 doc = curdoc()
 
 sys.stdout = StringIO()
@@ -26,11 +26,32 @@ bokeh_logger = logging.getLogger("bokeh")
 bokeh_logger.addHandler(bokeh_handler)
 bokeh_log_textareainput = TextAreaInput(title="server output:", height=150)
 
+def proposal_textinput_callback(_attr, _old, _new):
+    apply_button.disabled = False
+
+proposal_textinput = TextInput(title="Proposal number:", name="")
+proposal_textinput.on_change("value_input", proposal_textinput_callback)
+doc.proposal_textinput = proposal_textinput
+
+def apply_button_callback():
+    try:
+        proposal_path = pyzebra.find_proposal_path(proposal_textinput.value)
+    except ValueError as e:
+        print(e)
+        return
+
+    proposal_textinput.name = proposal_path
+    apply_button.disabled = True
+
+apply_button = Button(label="Apply", button_type="primary")
+apply_button.on_click(apply_button_callback)
+
 # Final layout
 doc.add_root(
     column(
         Tabs(
             tabs=[
+                Panel(child=column(proposal_textinput, apply_button), title="user config"),
                 panel_hdf_viewer.create(),
                 panel_hdf_anatric.create(),
                 panel_ccl_integrate.create(),

pyzebra/app/panel_ccl_integrate.py

@@ -38,7 +38,6 @@ from bokeh.models import (
     Spinner,
     TableColumn,
     TextAreaInput,
-    TextInput,
     WheelZoomTool,
     Whisker,
 )
@@ -77,36 +76,27 @@ def create():
     js_data = ColumnDataSource(data=dict(content=["", ""], fname=["", ""], ext=["", ""]))
 
     def file_select_update_for_proposal():
-        proposal = proposal_textinput.value.strip()
-        if not proposal:
+        proposal_path = proposal_textinput.name
+        if proposal_path:
+            file_list = []
+            for file in os.listdir(proposal_path):
+                if file.endswith((".ccl", ".dat")):
+                    file_list.append((os.path.join(proposal_path, file), file))
+            file_select.options = file_list
+            file_open_button.disabled = False
+            file_append_button.disabled = False
+        else:
             file_select.options = []
             file_open_button.disabled = True
             file_append_button.disabled = True
-            return
-
-        for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
-            proposal_path = os.path.join(zebra_proposals_path, proposal)
-            if os.path.isdir(proposal_path):
-                # found it
-                break
-        else:
-            raise ValueError(f"Can not find data for proposal '{proposal}'.")
-
-        file_list = []
-        for file in os.listdir(proposal_path):
-            if file.endswith((".ccl", ".dat")):
-                file_list.append((os.path.join(proposal_path, file), file))
-        file_select.options = file_list
-        file_open_button.disabled = False
-        file_append_button.disabled = False
 
     doc.add_periodic_callback(file_select_update_for_proposal, 5000)
 
     def proposal_textinput_callback(_attr, _old, _new):
         file_select_update_for_proposal()
 
-    proposal_textinput = TextInput(title="Proposal number:", width=210)
-    proposal_textinput.on_change("value", proposal_textinput_callback)
+    proposal_textinput = doc.proposal_textinput
+    proposal_textinput.on_change("name", proposal_textinput_callback)
 
     def _init_datatable():
         scan_list = [s["idx"] for s in det_data]
@@ -126,19 +116,19 @@ def create():
 
     def file_open_button_callback():
         nonlocal det_data
-        det_data = []
-        for f_path in file_select.value:
+        for f_ind, f_path in enumerate(file_select.value):
             with open(f_path) as file:
                 base, ext = os.path.splitext(os.path.basename(f_path))
-                if det_data:
-                    append_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-                    pyzebra.merge_datasets(det_data, append_data)
-                else:
-                    det_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(det_data, monitor_spinner.value)
-                    pyzebra.merge_duplicates(det_data)
-                    js_data.data.update(fname=[base, base])
+                file_data = pyzebra.parse_1D(file, ext)
+
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+
+            if f_ind == 0:  # first file
+                det_data = file_data
+                pyzebra.merge_duplicates(det_data)
+                js_data.data.update(fname=[base, base])
+            else:
+                pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
         append_upload_button.disabled = False
@@ -150,10 +140,10 @@ def create():
         for f_path in file_select.value:
             with open(f_path) as file:
                 _, ext = os.path.splitext(f_path)
-                append_data = pyzebra.parse_1D(file, ext)
+                file_data = pyzebra.parse_1D(file, ext)
 
-            pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-            pyzebra.merge_datasets(det_data, append_data)
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+            pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
 
@@ -163,19 +153,19 @@ def create():
     def upload_button_callback(_attr, _old, new):
         nonlocal det_data
         det_data = []
-        proposal_textinput.value = ""
         for f_str, f_name in zip(new, upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 base, ext = os.path.splitext(f_name)
-                if det_data:
-                    append_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-                    pyzebra.merge_datasets(det_data, append_data)
-                else:
-                    det_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(det_data, monitor_spinner.value)
-                    pyzebra.merge_duplicates(det_data)
-                    js_data.data.update(fname=[base, base])
+                file_data = pyzebra.parse_1D(file, ext)
+
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+
+            if not det_data:  # first file
+                det_data = file_data
+                pyzebra.merge_duplicates(det_data)
+                js_data.data.update(fname=[base, base])
+            else:
+                pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
         append_upload_button.disabled = False
@@ -188,10 +178,10 @@ def create():
         for f_str, f_name in zip(new, append_upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 _, ext = os.path.splitext(f_name)
-                append_data = pyzebra.parse_1D(file, ext)
+                file_data = pyzebra.parse_1D(file, ext)
 
-            pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-            pyzebra.merge_datasets(det_data, append_data)
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+            pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
 
@@ -202,23 +192,26 @@ def create():
     def monitor_spinner_callback(_attr, old, new):
         if det_data:
             pyzebra.normalize_dataset(det_data, new)
-            _update_plot(_get_selected_scan())
+            _update_plot()
 
     monitor_spinner = Spinner(title="Monitor:", mode="int", value=100_000, low=1, width=145)
     monitor_spinner.on_change("value", monitor_spinner_callback)
 
-    def _update_table():
+    def _update_datatable():
         fit_ok = [(1 if "fit" in scan else 0) for scan in det_data]
-        scan_table_source.data.update(fit=fit_ok)
+        export = [scan.get("active", True) for scan in det_data]
+        scan_table_source.data.update(fit=fit_ok, export=export)
 
-    def _update_plot(scan):
+    def _update_plot():
+        scan = _get_selected_scan()
         scan_motor = scan["scan_motor"]
 
         y = scan["counts"]
+        y_err = scan["counts_err"]
         x = scan[scan_motor]
 
         plot.axis[0].axis_label = scan_motor
-        plot_scatter_source.data.update(x=x, y=y, y_upper=y + np.sqrt(y), y_lower=y - np.sqrt(y))
+        plot_scatter_source.data.update(x=x, y=y, y_upper=y + y_err, y_lower=y - y_err)
 
         fit = scan.get("fit")
         if fit is not None:
@@ -321,13 +314,14 @@ def create():
             # skip unnecessary update caused by selection drop
             return
 
-        _update_plot(det_data[new[0]])
+        _update_plot()
 
     def scan_table_source_callback(_attr, _old, _new):
         _update_preview()
 
     scan_table_source = ColumnDataSource(dict(scan=[], hkl=[], fit=[], export=[]))
     scan_table_source.on_change("data", scan_table_source_callback)
+    scan_table_source.selected.on_change("indices", scan_table_select_callback)
 
     scan_table = DataTable(
         source=scan_table_source,
@@ -343,8 +337,6 @@ def create():
         editable=True,
     )
 
-    scan_table_source.selected.on_change("indices", scan_table_select_callback)
-
     def _get_selected_scan():
         return det_data[scan_table_source.selected.indices[0]]
 
@@ -359,14 +351,16 @@ def create():
             return
 
         pyzebra.merge_scans(scan_into, scan_from)
-        _update_plot(_get_selected_scan())
+        _update_datatable()
+        _update_plot()
 
     merge_button = Button(label="Merge into current", width=145)
     merge_button.on_click(merge_button_callback)
 
     def restore_button_callback():
         pyzebra.restore_scan(_get_selected_scan())
-        _update_plot(_get_selected_scan())
+        _update_datatable()
+        _update_plot()
 
     restore_button = Button(label="Restore scan", width=145)
     restore_button.on_click(restore_button_callback)
@@ -502,8 +496,8 @@ def create():
                     lorentz=lorentz_checkbox.active,
                 )
 
-        _update_plot(_get_selected_scan())
-        _update_table()
+        _update_plot()
+        _update_datatable()
 
     proc_all_button = Button(label="Process All", button_type="primary", width=145)
     proc_all_button.on_click(proc_all_button_callback)
@@ -519,8 +513,8 @@ def create():
             lorentz=lorentz_checkbox.active,
         )
 
-        _update_plot(scan)
-        _update_table()
+        _update_plot()
+        _update_datatable()
 
     proc_button = Button(label="Process Current", width=145)
     proc_button.on_click(proc_button_callback)
@@ -598,7 +592,6 @@ def create():
     )
 
     import_layout = column(
-        proposal_textinput,
         file_select,
         row(file_open_button, file_append_button),
         upload_div,

pyzebra/app/panel_hdf_param_study.py

@@ -33,7 +33,6 @@ from bokeh.models import (
     Spinner,
     TableColumn,
     Tabs,
-    TextInput,
     Title,
     WheelZoomTool,
 )
@@ -56,42 +55,51 @@ def create():
 
     num_formatter = NumberFormatter(format="0.00", nan_format="")
 
-    def file_select_update_for_proposal():
-        proposal = proposal_textinput.value.strip()
-        if not proposal:
-            return
+    def file_select_update():
+        if data_source.value == "proposal number":
+            proposal_path = proposal_textinput.name
+            if proposal_path:
+                file_list = []
+                for file in os.listdir(proposal_path):
+                    if file.endswith(".hdf"):
+                        file_list.append((os.path.join(proposal_path, file), file))
+                file_select.options = file_list
+            else:
+                file_select.options = []
 
-        for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
-            proposal_path = os.path.join(zebra_proposals_path, proposal)
-            if os.path.isdir(proposal_path):
-                # found it
-                break
-        else:
-            raise ValueError(f"Can not find data for proposal '{proposal}'.")
+        else:  # "cami file"
+            if not cami_meta:
+                file_select.options = []
+                return
 
-        file_list = []
-        for file in os.listdir(proposal_path):
-            if file.endswith(".hdf"):
-                file_list.append((os.path.join(proposal_path, file), file))
-        file_select.options = file_list
+            file_list = cami_meta["filelist"]
+            file_select.options = [(entry, os.path.basename(entry)) for entry in file_list]
 
-    doc.add_periodic_callback(file_select_update_for_proposal, 5000)
+    def data_source_callback(_attr, _old, _new):
+        file_select_update()
+
+    data_source = Select(
+        title="Data Source:",
+        value="proposal number",
+        options=["proposal number", "cami file"],
+        width=210,
+    )
+    data_source.on_change("value", data_source_callback)
+
+    doc.add_periodic_callback(file_select_update, 5000)
 
     def proposal_textinput_callback(_attr, _old, _new):
-        nonlocal cami_meta
-        cami_meta = {}
-        file_select_update_for_proposal()
+        file_select_update()
 
-    proposal_textinput = TextInput(title="Proposal number:", width=210)
-    proposal_textinput.on_change("value", proposal_textinput_callback)
+    proposal_textinput = doc.proposal_textinput
+    proposal_textinput.on_change("name", proposal_textinput_callback)
 
     def upload_button_callback(_attr, _old, new):
         nonlocal cami_meta
-        proposal_textinput.value = ""
         with io.StringIO(base64.b64decode(new).decode()) as file:
             cami_meta = pyzebra.parse_h5meta(file)
-            file_list = cami_meta["filelist"]
-            file_select.options = [(entry, os.path.basename(entry)) for entry in file_list]
+        data_source.value = "cami file"
+        file_select_update()
 
     upload_div = Div(text="or upload .cami file:", margin=(5, 5, 0, 5))
     upload_button = FileInput(accept=".cami", width=200)
@@ -180,7 +188,16 @@ def create():
         else:  # zebra_mode == "bi"
             metadata_table_source.data.update(geom=["bisecting"])
 
-        update_image(0)
+        if "mf" in det_data:
+            metadata_table_source.data.update(mf=[det_data["mf"][0]])
+        else:
+            metadata_table_source.data.update(mf=[None])
+
+        if "temp" in det_data:
+            metadata_table_source.data.update(temp=[det_data["temp"][0]])
+        else:
+            metadata_table_source.data.update(temp=[None])
+
         update_overview_plot()
 
     def scan_table_source_callback(_attr, _old, _new):
@@ -229,23 +246,17 @@ def create():
     )
     param_select.on_change("value", param_select_callback)
 
-    def update_image(index=None):
-        if "mf" in det_data:
-            metadata_table_source.data.update(mf=[det_data["mf"][index]])
-        else:
-            metadata_table_source.data.update(mf=[None])
-
-        if "temp" in det_data:
-            metadata_table_source.data.update(temp=[det_data["temp"][index]])
-        else:
-            metadata_table_source.data.update(temp=[None])
-
     def update_overview_plot():
         h5_data = det_data["data"]
         n_im, n_y, n_x = h5_data.shape
         overview_x = np.mean(h5_data, axis=1)
         overview_y = np.mean(h5_data, axis=2)
 
+        # normalize for simpler colormapping
+        overview_max_val = max(np.max(overview_x), np.max(overview_y))
+        overview_x = 1000 * overview_x / overview_max_val
+        overview_y = 1000 * overview_y / overview_max_val
+
         overview_plot_x_image_source.data.update(image=[overview_x], dw=[n_x], dh=[n_im])
         overview_plot_y_image_source.data.update(image=[overview_y], dw=[n_y], dh=[n_im])
 
@@ -381,7 +392,7 @@ def create():
     colormap.on_change("value", colormap_callback)
     colormap.value = "plasma"
 
-    PROJ_STEP = 0.1
+    PROJ_STEP = 1
 
     def proj_auto_checkbox_callback(state):
         if state:
@@ -598,7 +609,7 @@ def create():
 
     # Final layout
     import_layout = column(
-        proposal_textinput,
+        data_source,
         upload_div,
         upload_button,
         file_select,

pyzebra/app/panel_hdf_viewer.py

@@ -37,7 +37,6 @@ from bokeh.models import (
     Spacer,
     Spinner,
     TableColumn,
-    TextInput,
     Title,
     WheelZoomTool,
 )
@@ -59,42 +58,51 @@ def create():
 
     num_formatter = NumberFormatter(format="0.00", nan_format="")
 
-    def file_select_update_for_proposal():
-        proposal = proposal_textinput.value.strip()
-        if not proposal:
-            return
+    def file_select_update():
+        if data_source.value == "proposal number":
+            proposal_path = proposal_textinput.name
+            if proposal_path:
+                file_list = []
+                for file in os.listdir(proposal_path):
+                    if file.endswith(".hdf"):
+                        file_list.append((os.path.join(proposal_path, file), file))
+                file_select.options = file_list
+            else:
+                file_select.options = []
 
-        for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
-            proposal_path = os.path.join(zebra_proposals_path, proposal)
-            if os.path.isdir(proposal_path):
-                # found it
-                break
-        else:
-            raise ValueError(f"Can not find data for proposal '{proposal}'.")
+        else:  # "cami file"
+            if not cami_meta:
+                file_select.options = []
+                return
 
-        file_list = []
-        for file in os.listdir(proposal_path):
-            if file.endswith(".hdf"):
-                file_list.append((os.path.join(proposal_path, file), file))
-        file_select.options = file_list
+            file_list = cami_meta["filelist"]
+            file_select.options = [(entry, os.path.basename(entry)) for entry in file_list]
 
-    doc.add_periodic_callback(file_select_update_for_proposal, 5000)
+    def data_source_callback(_attr, _old, _new):
+        file_select_update()
+
+    data_source = Select(
+        title="Data Source:",
+        value="proposal number",
+        options=["proposal number", "cami file"],
+        width=210,
+    )
+    data_source.on_change("value", data_source_callback)
+
+    doc.add_periodic_callback(file_select_update, 5000)
 
     def proposal_textinput_callback(_attr, _old, _new):
-        nonlocal cami_meta
-        cami_meta = {}
-        file_select_update_for_proposal()
+        file_select_update()
 
-    proposal_textinput = TextInput(title="Proposal number:", width=210)
-    proposal_textinput.on_change("value", proposal_textinput_callback)
+    proposal_textinput = doc.proposal_textinput
+    proposal_textinput.on_change("name", proposal_textinput_callback)
 
     def upload_button_callback(_attr, _old, new):
         nonlocal cami_meta
-        proposal_textinput.value = ""
         with io.StringIO(base64.b64decode(new).decode()) as file:
             cami_meta = pyzebra.parse_h5meta(file)
-            file_list = cami_meta["filelist"]
-            file_select.options = [(entry, os.path.basename(entry)) for entry in file_list]
+        data_source.value = "cami file"
+        file_select_update()
 
     upload_div = Div(text="or upload .cami file:", margin=(5, 5, 0, 5))
     upload_button = FileInput(accept=".cami", width=200)
@@ -147,6 +155,11 @@ def create():
         overview_x = np.mean(h5_data, axis=1)
         overview_y = np.mean(h5_data, axis=2)
 
+        # normalize for simpler colormapping
+        overview_max_val = max(np.max(overview_x), np.max(overview_y))
+        overview_x = 1000 * overview_x / overview_max_val
+        overview_y = 1000 * overview_y / overview_max_val
+
         overview_plot_x_image_source.data.update(image=[overview_x], dw=[n_x], dh=[n_im])
         overview_plot_y_image_source.data.update(image=[overview_y], dw=[n_y], dh=[n_im])
 
@@ -536,7 +549,7 @@ def create():
     )
     display_min_spinner.on_change("value", display_min_spinner_callback)
 
-    PROJ_STEP = 0.1
+    PROJ_STEP = 1
 
     def proj_auto_checkbox_callback(state):
         if state:
@@ -726,7 +739,7 @@ def create():
     )
 
     # Final layout
-    import_layout = column(proposal_textinput, upload_div, upload_button, file_select)
+    import_layout = column(data_source, upload_div, upload_button, file_select)
     layout_image = column(gridplot([[proj_v, None], [plot, proj_h]], merge_tools=False))
     colormap_layout = column(
         colormap,

pyzebra/app/panel_param_study.py

@@ -42,7 +42,6 @@ from bokeh.models import (
     TableColumn,
     Tabs,
     TextAreaInput,
-    TextInput,
     WheelZoomTool,
     Whisker,
 )
@@ -87,36 +86,27 @@ def create():
     js_data = ColumnDataSource(data=dict(content=[""], fname=[""], ext=[""]))
 
     def file_select_update_for_proposal():
-        proposal = proposal_textinput.value.strip()
-        if not proposal:
+        proposal_path = proposal_textinput.name
+        if proposal_path:
+            file_list = []
+            for file in os.listdir(proposal_path):
+                if file.endswith((".ccl", ".dat")):
+                    file_list.append((os.path.join(proposal_path, file), file))
+            file_select.options = file_list
+            file_open_button.disabled = False
+            file_append_button.disabled = False
+        else:
             file_select.options = []
             file_open_button.disabled = True
             file_append_button.disabled = True
-            return
-
-        for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
-            proposal_path = os.path.join(zebra_proposals_path, proposal)
-            if os.path.isdir(proposal_path):
-                # found it
-                break
-        else:
-            raise ValueError(f"Can not find data for proposal '{proposal}'.")
-
-        file_list = []
-        for file in os.listdir(proposal_path):
-            if file.endswith((".ccl", ".dat")):
-                file_list.append((os.path.join(proposal_path, file), file))
-        file_select.options = file_list
-        file_open_button.disabled = False
-        file_append_button.disabled = False
 
     doc.add_periodic_callback(file_select_update_for_proposal, 5000)
 
     def proposal_textinput_callback(_attr, _old, _new):
         file_select_update_for_proposal()
 
-    proposal_textinput = TextInput(title="Proposal number:", width=210)
-    proposal_textinput.on_change("value", proposal_textinput_callback)
+    proposal_textinput = doc.proposal_textinput
+    proposal_textinput.on_change("name", proposal_textinput_callback)
 
     def _init_datatable():
         scan_list = [s["idx"] for s in det_data]
@@ -142,18 +132,19 @@ def create():
 
     def file_open_button_callback():
         nonlocal det_data
-        det_data = []
-        for f_path in file_select.value:
+        for f_ind, f_path in enumerate(file_select.value):
             with open(f_path) as file:
                 base, ext = os.path.splitext(os.path.basename(f_path))
-                if det_data:
-                    append_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-                    det_data.extend(append_data)
-                else:
-                    det_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(det_data, monitor_spinner.value)
-                    js_data.data.update(fname=[base])
+                file_data = pyzebra.parse_1D(file, ext)
+
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+
+            if f_ind == 0:  # first file
+                det_data = file_data
+                pyzebra.merge_duplicates(det_data)
+                js_data.data.update(fname=[base])
+            else:
+                pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
         append_upload_button.disabled = False
@@ -165,10 +156,10 @@ def create():
         for f_path in file_select.value:
             with open(f_path) as file:
                 _, ext = os.path.splitext(f_path)
-                append_data = pyzebra.parse_1D(file, ext)
+                file_data = pyzebra.parse_1D(file, ext)
 
-            pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-            det_data.extend(append_data)
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+            pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
 
@@ -178,18 +169,19 @@ def create():
     def upload_button_callback(_attr, _old, new):
         nonlocal det_data
         det_data = []
-        proposal_textinput.value = ""
         for f_str, f_name in zip(new, upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 base, ext = os.path.splitext(f_name)
-                if det_data:
-                    append_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-                    det_data.extend(append_data)
-                else:
-                    det_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(det_data, monitor_spinner.value)
-                    js_data.data.update(fname=[base])
+                file_data = pyzebra.parse_1D(file, ext)
+
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+
+            if not det_data:  # first file
+                det_data = file_data
+                pyzebra.merge_duplicates(det_data)
+                js_data.data.update(fname=[base])
+            else:
+                pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
         append_upload_button.disabled = False
@@ -202,10 +194,10 @@ def create():
         for f_str, f_name in zip(new, append_upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 _, ext = os.path.splitext(f_name)
-                append_data = pyzebra.parse_1D(file, ext)
+                file_data = pyzebra.parse_1D(file, ext)
 
-            pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-            det_data.extend(append_data)
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+            pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
 
@@ -235,17 +227,19 @@ def create():
         scan_table_source.data.update(fit=fit_ok)
 
     def _update_plot():
-        _update_single_scan_plot(_get_selected_scan())
+        _update_single_scan_plot()
         _update_overview()
 
-    def _update_single_scan_plot(scan):
+    def _update_single_scan_plot():
+        scan = _get_selected_scan()
         scan_motor = scan["scan_motor"]
 
         y = scan["counts"]
+        y_err = scan["counts_err"]
         x = scan[scan_motor]
 
         plot.axis[0].axis_label = scan_motor
-        plot_scatter_source.data.update(x=x, y=y, y_upper=y + np.sqrt(y), y_lower=y - np.sqrt(y))
+        plot_scatter_source.data.update(x=x, y=y, y_upper=y + y_err, y_lower=y - y_err)
 
         fit = scan.get("fit")
         if fit is not None:
@@ -325,13 +319,19 @@ def create():
     def _update_param_plot():
         x = []
         y = []
+        y_lower = []
+        y_upper = []
         fit_param = fit_param_select.value
         for s, p in zip(det_data, scan_table_source.data["param"]):
             if "fit" in s and fit_param:
                 x.append(p)
-                y.append(s["fit"].values[fit_param])
+                param_fit_val = s["fit"].params[fit_param].value
+                param_fit_std = s["fit"].params[fit_param].stderr
+                y.append(param_fit_val)
+                y_lower.append(param_fit_val - param_fit_std)
+                y_upper.append(param_fit_val + param_fit_std)
 
-        param_plot_scatter_source.data.update(x=x, y=y)
+        param_plot_scatter_source.data.update(x=x, y=y, y_lower=y_lower, y_upper=y_upper)
 
     # Main plot
     plot = Plot(
@@ -441,8 +441,11 @@ def create():
     param_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
     param_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
 
-    param_plot_scatter_source = ColumnDataSource(dict(x=[], y=[]))
+    param_plot_scatter_source = ColumnDataSource(dict(x=[], y=[], y_upper=[], y_lower=[]))
     param_plot.add_glyph(param_plot_scatter_source, Scatter(x="x", y="y"))
+    param_plot.add_layout(
+        Whisker(source=param_plot_scatter_source, base="x", upper="y_upper", lower="y_lower")
+    )
 
     param_plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
     param_plot.toolbar.logo = None
@@ -486,6 +489,7 @@ def create():
 
     scan_table_source = ColumnDataSource(dict(file=[], scan=[], param=[], fit=[], export=[]))
     scan_table_source.on_change("data", scan_table_source_callback)
+    scan_table_source.selected.on_change("indices", scan_table_select_callback)
 
     scan_table = DataTable(
         source=scan_table_source,
@@ -501,13 +505,6 @@ def create():
         autosize_mode="none",
     )
 
-    def scan_table_source_callback(_attr, _old, _new):
-        if scan_table_source.selected.indices:
-            _update_plot()
-
-    scan_table_source.selected.on_change("indices", scan_table_select_callback)
-    scan_table_source.on_change("data", scan_table_source_callback)
-
     def _get_selected_scan():
         return det_data[scan_table_source.selected.indices[0]]
 
@@ -748,7 +745,6 @@ def create():
     scan_layout = column(scan_table, row(monitor_spinner, scan_motor_select, param_select))
 
     import_layout = column(
-        proposal_textinput,
         file_select,
         row(file_open_button, file_append_button),
         upload_div,

pyzebra/ccl_io.py

@@ -169,6 +169,7 @@ def parse_1D(fileobj, data_type):
             while len(counts) < s["n_points"]:
                 counts.extend(map(float, next(fileobj).split()))
             s["counts"] = np.array(counts)
+            s["counts_err"] = np.sqrt(s["counts"])
 
             if s["h"].is_integer() and s["k"].is_integer() and s["l"].is_integer():
                 s["h"], s["k"], s["l"] = map(int, (s["h"], s["k"], s["l"]))
@@ -204,6 +205,8 @@ def parse_1D(fileobj, data_type):
         for name in col_names:
             s[name] = np.array(s[name])
 
+        s["counts_err"] = np.sqrt(s["counts"])
+
         s["scan_motors"] = []
         for motor, step in zip(motors, steps):
             if step == 0:
@@ -211,27 +214,24 @@ def parse_1D(fileobj, data_type):
                 s[motor] = np.median(s[motor])
             else:
                 s["scan_motors"].append(motor)
-        s["scan_motor"] = s["scan_motors"][0]
 
         # "om" -> "omega"
         if "om" in s["scan_motors"]:
             s["scan_motors"][s["scan_motors"].index("om")] = "omega"
-            if s["scan_motor"] == "om":
-                s["scan_motor"] = "omega"
             s["omega"] = s["om"]
             del s["om"]
 
         # "tt" -> "temp"
-        elif "tt" in s["scan_motors"]:
+        if "tt" in s["scan_motors"]:
             s["scan_motors"][s["scan_motors"].index("tt")] = "temp"
-            if s["scan_motor"] == "tt":
-                s["scan_motor"] = "temp"
             s["temp"] = s["tt"]
             del s["tt"]
 
         # "mf" stays "mf"
         # "phi" stays "phi"
 
+        s["scan_motor"] = s["scan_motors"][0]
+
         if "h" not in s:
             s["h"] = s["k"] = s["l"] = float("nan")
 

pyzebra/ccl_process.py

@@ -1,4 +1,3 @@
-import itertools
 import os
 
 import numpy as np
@@ -30,6 +29,7 @@ def normalize_dataset(dataset, monitor=100_000):
     for scan in dataset:
         monitor_ratio = monitor / scan["monitor"]
         scan["counts"] *= monitor_ratio
+        scan["counts_err"] *= monitor_ratio
         scan["monitor"] = monitor
 
 
@@ -81,17 +81,41 @@ def merge_datasets(dataset_into, dataset_from):
 
 def merge_scans(scan_into, scan_from):
     # TODO: does it need to be "scan_motor" instead of omega for a generalized solution?
-    if "init_omega" not in scan_into:
-        scan_into["init_omega"] = scan_into["omega"]
-        scan_into["init_counts"] = scan_into["counts"]
+    if "init_scan" not in scan_into:
+        scan_into["init_scan"] = scan_into.copy()
 
-    omega = np.concatenate((scan_into["omega"], scan_from["omega"]))
-    counts = np.concatenate((scan_into["counts"], scan_from["counts"]))
+    if "merged_scans" not in scan_into:
+        scan_into["merged_scans"] = []
 
-    index = np.argsort(omega)
+    if scan_from in scan_into["merged_scans"]:
+        return
 
-    scan_into["omega"] = omega[index]
-    scan_into["counts"] = counts[index]
+    scan_into["merged_scans"].append(scan_from)
+
+    if (
+        scan_into["omega"].shape == scan_from["omega"].shape
+        and np.max(np.abs(scan_into["omega"] - scan_from["omega"])) < 0.0005
+    ):
+        counts_tmp = 0
+        counts_err_tmp = 0
+
+        for scan in [scan_into["init_scan"], *scan_into["merged_scans"]]:
+            counts_tmp += scan["counts"]
+            counts_err_tmp += scan["counts_err"] ** 2
+
+        scan_into["counts"] = counts_tmp / (1 + len(scan_into["merged_scans"]))
+        scan_into["counts_err"] = np.sqrt(counts_err_tmp)
+
+    else:
+        omega = np.concatenate((scan_into["omega"], scan_from["omega"]))
+        counts = np.concatenate((scan_into["counts"], scan_from["counts"]))
+        counts_err = np.concatenate((scan_into["counts_err"], scan_from["counts_err"]))
+
+        index = np.argsort(omega)
+
+        scan_into["omega"] = omega[index]
+        scan_into["counts"] = counts[index]
+        scan_into["counts_err"] = counts_err[index]
 
     scan_from["active"] = False
 
@@ -101,11 +125,14 @@ def merge_scans(scan_into, scan_from):
 
 
 def restore_scan(scan):
-    if "init_omega" in scan:
-        scan["omega"] = scan["init_omega"]
-        scan["counts"] = scan["init_counts"]
-        del scan["init_omega"]
-        del scan["init_counts"]
+    if "merged_scans" in scan:
+        for merged_scan in scan["merged_scans"]:
+            merged_scan["active"] = True
+
+    if "init_scan" in scan:
+        tmp = scan["init_scan"]
+        scan.clear()
+        scan.update(tmp)
 
 
 def fit_scan(scan, model_dict, fit_from=None, fit_to=None):
@@ -115,11 +142,13 @@ def fit_scan(scan, model_dict, fit_from=None, fit_to=None):
         fit_to = np.inf
 
     y_fit = scan["counts"]
+    y_err = scan["counts_err"]
     x_fit = scan[scan["scan_motor"]]
 
     # apply fitting range
     fit_ind = (fit_from <= x_fit) & (x_fit <= fit_to)
     y_fit = y_fit[fit_ind]
+    y_err = y_err[fit_ind]
     x_fit = x_fit[fit_ind]
 
     model = None
@@ -167,7 +196,7 @@ def fit_scan(scan, model_dict, fit_from=None, fit_to=None):
         else:
             model += _model
 
-    weights = [1 / np.sqrt(val) if val != 0 else 1 for val in y_fit]
+    weights = [1 / y_err if y_err != 0 else 1 for y_err in y_err]
     scan["fit"] = model.fit(y_fit, x=x_fit, weights=weights)
 
 

pyzebra/utils.py

@@ -0,0 +1,20 @@
+import os
+
+ZEBRA_PROPOSALS_PATHS = [
+    f"/afs/psi.ch/project/sinqdata/{year}/zebra/" for year in (2016, 2017, 2018, 2020, 2021)
+]
+
+def find_proposal_path(proposal):
+    proposal = proposal.strip()
+    if proposal:
+        for zebra_proposals_path in ZEBRA_PROPOSALS_PATHS:
+            proposal_path = os.path.join(zebra_proposals_path, proposal)
+            if os.path.isdir(proposal_path):
+                # found it
+                break
+        else:
+            raise ValueError(f"Can not find data for proposal '{proposal}'.")
+    else:
+        proposal_path = ""
+
+    return proposal_path