Updating for version 0.5.1

Fix #40

.github/workflows/deployment.yaml

@@ -16,7 +16,6 @@ jobs:
       run: |
         $CONDA/bin/conda install --quiet --yes conda-build anaconda-client
         $CONDA/bin/conda config --append channels conda-forge
-        $CONDA/bin/conda config --set channel_priority strict
         $CONDA/bin/conda config --set anaconda_upload yes
 
     - name: Build and upload

conda-recipe/meta.yaml

@@ -22,7 +22,7 @@ requirements:
     - numpy
     - scipy
     - h5py
-    - bokeh =2.3
+    - bokeh =2.4
     - numba
     - lmfit
 

pyzebra/__init__.py

@@ -1,11 +1,8 @@
 from pyzebra.anatric import *
 from pyzebra.ccl_io import *
-from pyzebra.h5 import *
-from pyzebra.xtal import *
 from pyzebra.ccl_process import *
+from pyzebra.h5 import *
+from pyzebra.utils import *
+from pyzebra.xtal import *
 
-ZEBRA_PROPOSALS_PATHS = [
-    f"/afs/psi.ch/project/sinqdata/{year}/zebra/" for year in (2016, 2017, 2018, 2020, 2021)
-]
-
-__version__ = "0.4.0"
+__version__ = "0.5.1"

pyzebra/app/app.py

@@ -2,17 +2,18 @@ import logging
 import sys
 from io import StringIO
 
+import pyzebra
 from bokeh.io import curdoc
 from bokeh.layouts import column, row
-from bokeh.models import Tabs, TextAreaInput
+from bokeh.models import Button, Panel, Tabs, TextAreaInput, TextInput
 
 import panel_ccl_integrate
 import panel_hdf_anatric
+import panel_hdf_param_study
 import panel_hdf_viewer
 import panel_param_study
 import panel_spind
 
-
 doc = curdoc()
 
 sys.stdout = StringIO()
@@ -25,16 +26,40 @@ bokeh_logger = logging.getLogger("bokeh")
 bokeh_logger.addHandler(bokeh_handler)
 bokeh_log_textareainput = TextAreaInput(title="server output:", height=150)
 
-# Final layout
-tab_hdf_viewer = panel_hdf_viewer.create()
-tab_hdf_anatric = panel_hdf_anatric.create()
-tab_ccl_integrate = panel_ccl_integrate.create()
-tab_param_study = panel_param_study.create()
-tab_spind = panel_spind.create()
+def proposal_textinput_callback(_attr, _old, _new):
+    apply_button.disabled = False
 
+proposal_textinput = TextInput(title="Proposal number:", name="")
+proposal_textinput.on_change("value_input", proposal_textinput_callback)
+doc.proposal_textinput = proposal_textinput
+
+def apply_button_callback():
+    try:
+        proposal_path = pyzebra.find_proposal_path(proposal_textinput.value)
+    except ValueError as e:
+        print(e)
+        return
+
+    proposal_textinput.name = proposal_path
+    apply_button.disabled = True
+
+apply_button = Button(label="Apply", button_type="primary")
+apply_button.on_click(apply_button_callback)
+
+# Final layout
 doc.add_root(
     column(
-        Tabs(tabs=[tab_hdf_viewer, tab_hdf_anatric, tab_ccl_integrate, tab_param_study, tab_spind]),
+        Tabs(
+            tabs=[
+                Panel(child=column(proposal_textinput, apply_button), title="user config"),
+                panel_hdf_viewer.create(),
+                panel_hdf_anatric.create(),
+                panel_ccl_integrate.create(),
+                panel_param_study.create(),
+                panel_hdf_param_study.create(),
+                panel_spind.create(),
+            ]
+        ),
         row(stdout_textareainput, bokeh_log_textareainput, sizing_mode="scale_both"),
     )
 )

pyzebra/app/panel_ccl_integrate.py

@@ -38,7 +38,6 @@ from bokeh.models import (
     Spinner,
     TableColumn,
     TextAreaInput,
-    TextInput,
     WheelZoomTool,
     Whisker,
 )
@@ -77,36 +76,27 @@ def create():
     js_data = ColumnDataSource(data=dict(content=["", ""], fname=["", ""], ext=["", ""]))
 
     def file_select_update_for_proposal():
-        proposal = proposal_textinput.value.strip()
-        if not proposal:
+        proposal_path = proposal_textinput.name
+        if proposal_path:
+            file_list = []
+            for file in os.listdir(proposal_path):
+                if file.endswith((".ccl", ".dat")):
+                    file_list.append((os.path.join(proposal_path, file), file))
+            file_select.options = file_list
+            file_open_button.disabled = False
+            file_append_button.disabled = False
+        else:
             file_select.options = []
             file_open_button.disabled = True
             file_append_button.disabled = True
-            return
-
-        for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
-            proposal_path = os.path.join(zebra_proposals_path, proposal)
-            if os.path.isdir(proposal_path):
-                # found it
-                break
-        else:
-            raise ValueError(f"Can not find data for proposal '{proposal}'.")
-
-        file_list = []
-        for file in os.listdir(proposal_path):
-            if file.endswith((".ccl", ".dat")):
-                file_list.append((os.path.join(proposal_path, file), file))
-        file_select.options = file_list
-        file_open_button.disabled = False
-        file_append_button.disabled = False
 
     doc.add_periodic_callback(file_select_update_for_proposal, 5000)
 
     def proposal_textinput_callback(_attr, _old, _new):
         file_select_update_for_proposal()
 
-    proposal_textinput = TextInput(title="Proposal number:", width=210)
-    proposal_textinput.on_change("value", proposal_textinput_callback)
+    proposal_textinput = doc.proposal_textinput
+    proposal_textinput.on_change("name", proposal_textinput_callback)
 
     def _init_datatable():
         scan_list = [s["idx"] for s in det_data]
@@ -126,19 +116,19 @@ def create():
 
     def file_open_button_callback():
         nonlocal det_data
-        det_data = []
-        for f_name in file_select.value:
-            with open(f_name) as file:
-                base, ext = os.path.splitext(f_name)
-                if det_data:
-                    append_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-                    pyzebra.merge_datasets(det_data, append_data)
-                else:
-                    det_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(det_data, monitor_spinner.value)
-                    pyzebra.merge_duplicates(det_data)
-                    js_data.data.update(fname=[base, base])
+        for f_ind, f_path in enumerate(file_select.value):
+            with open(f_path) as file:
+                base, ext = os.path.splitext(os.path.basename(f_path))
+                file_data = pyzebra.parse_1D(file, ext)
+
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+
+            if f_ind == 0:  # first file
+                det_data = file_data
+                pyzebra.merge_duplicates(det_data)
+                js_data.data.update(fname=[base, base])
+            else:
+                pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
         append_upload_button.disabled = False
@@ -147,13 +137,13 @@ def create():
     file_open_button.on_click(file_open_button_callback)
 
     def file_append_button_callback():
-        for f_name in file_select.value:
-            with open(f_name) as file:
-                _, ext = os.path.splitext(f_name)
-                append_data = pyzebra.parse_1D(file, ext)
+        for f_path in file_select.value:
+            with open(f_path) as file:
+                _, ext = os.path.splitext(f_path)
+                file_data = pyzebra.parse_1D(file, ext)
 
-            pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-            pyzebra.merge_datasets(det_data, append_data)
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+            pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
 
@@ -163,19 +153,19 @@ def create():
     def upload_button_callback(_attr, _old, new):
         nonlocal det_data
         det_data = []
-        proposal_textinput.value = ""
         for f_str, f_name in zip(new, upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 base, ext = os.path.splitext(f_name)
-                if det_data:
-                    append_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-                    pyzebra.merge_datasets(det_data, append_data)
-                else:
-                    det_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(det_data, monitor_spinner.value)
-                    pyzebra.merge_duplicates(det_data)
-                    js_data.data.update(fname=[base, base])
+                file_data = pyzebra.parse_1D(file, ext)
+
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+
+            if not det_data:  # first file
+                det_data = file_data
+                pyzebra.merge_duplicates(det_data)
+                js_data.data.update(fname=[base, base])
+            else:
+                pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
         append_upload_button.disabled = False
@@ -188,10 +178,10 @@ def create():
         for f_str, f_name in zip(new, append_upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 _, ext = os.path.splitext(f_name)
-                append_data = pyzebra.parse_1D(file, ext)
+                file_data = pyzebra.parse_1D(file, ext)
 
-            pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-            pyzebra.merge_datasets(det_data, append_data)
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+            pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
 
@@ -202,23 +192,26 @@ def create():
     def monitor_spinner_callback(_attr, old, new):
         if det_data:
             pyzebra.normalize_dataset(det_data, new)
-            _update_plot(_get_selected_scan())
+            _update_plot()
 
     monitor_spinner = Spinner(title="Monitor:", mode="int", value=100_000, low=1, width=145)
     monitor_spinner.on_change("value", monitor_spinner_callback)
 
-    def _update_table():
+    def _update_datatable():
         fit_ok = [(1 if "fit" in scan else 0) for scan in det_data]
-        scan_table_source.data.update(fit=fit_ok)
+        export = [scan.get("active", True) for scan in det_data]
+        scan_table_source.data.update(fit=fit_ok, export=export)
 
-    def _update_plot(scan):
+    def _update_plot():
+        scan = _get_selected_scan()
         scan_motor = scan["scan_motor"]
 
         y = scan["counts"]
+        y_err = scan["counts_err"]
         x = scan[scan_motor]
 
         plot.axis[0].axis_label = scan_motor
-        plot_scatter_source.data.update(x=x, y=y, y_upper=y + np.sqrt(y), y_lower=y - np.sqrt(y))
+        plot_scatter_source.data.update(x=x, y=y, y_upper=y + y_err, y_lower=y - y_err)
 
         fit = scan.get("fit")
         if fit is not None:
@@ -321,13 +314,14 @@ def create():
             # skip unnecessary update caused by selection drop
             return
 
-        _update_plot(det_data[new[0]])
+        _update_plot()
 
     def scan_table_source_callback(_attr, _old, _new):
         _update_preview()
 
     scan_table_source = ColumnDataSource(dict(scan=[], hkl=[], fit=[], export=[]))
     scan_table_source.on_change("data", scan_table_source_callback)
+    scan_table_source.selected.on_change("indices", scan_table_select_callback)
 
     scan_table = DataTable(
         source=scan_table_source,
@@ -343,8 +337,6 @@ def create():
         editable=True,
     )
 
-    scan_table_source.selected.on_change("indices", scan_table_select_callback)
-
     def _get_selected_scan():
         return det_data[scan_table_source.selected.indices[0]]
 
@@ -359,14 +351,16 @@ def create():
             return
 
         pyzebra.merge_scans(scan_into, scan_from)
-        _update_plot(_get_selected_scan())
+        _update_datatable()
+        _update_plot()
 
     merge_button = Button(label="Merge into current", width=145)
     merge_button.on_click(merge_button_callback)
 
     def restore_button_callback():
         pyzebra.restore_scan(_get_selected_scan())
-        _update_plot(_get_selected_scan())
+        _update_datatable()
+        _update_plot()
 
     restore_button = Button(label="Restore scan", width=145)
     restore_button.on_click(restore_button_callback)
@@ -502,8 +496,8 @@ def create():
                     lorentz=lorentz_checkbox.active,
                 )
 
-        _update_plot(_get_selected_scan())
-        _update_table()
+        _update_plot()
+        _update_datatable()
 
     proc_all_button = Button(label="Process All", button_type="primary", width=145)
     proc_all_button.on_click(proc_all_button_callback)
@@ -519,8 +513,8 @@ def create():
             lorentz=lorentz_checkbox.active,
         )
 
-        _update_plot(scan)
-        _update_table()
+        _update_plot()
+        _update_datatable()
 
     proc_button = Button(label="Process Current", width=145)
     proc_button.on_click(proc_button_callback)
@@ -598,7 +592,6 @@ def create():
     )
 
     import_layout = column(
-        proposal_textinput,
         file_select,
         row(file_open_button, file_append_button),
         upload_div,

pyzebra/app/panel_hdf_param_study.py

@@ -0,0 +1,634 @@
+import base64
+import io
+import math
+import os
+
+import numpy as np
+from bokeh.io import curdoc
+from bokeh.layouts import column, gridplot, row
+from bokeh.models import (
+    BasicTicker,
+    BoxZoomTool,
+    Button,
+    CheckboxGroup,
+    ColumnDataSource,
+    DataRange1d,
+    DataTable,
+    Div,
+    FileInput,
+    Grid,
+    MultiSelect,
+    NumberEditor,
+    NumberFormatter,
+    Image,
+    LinearAxis,
+    LinearColorMapper,
+    Panel,
+    PanTool,
+    Plot,
+    Range1d,
+    ResetTool,
+    Scatter,
+    Select,
+    Spinner,
+    TableColumn,
+    Tabs,
+    Title,
+    WheelZoomTool,
+)
+from bokeh.palettes import Cividis256, Greys256, Plasma256  # pylint: disable=E0611
+from scipy.optimize import curve_fit
+
+import pyzebra
+
+IMAGE_W = 256
+IMAGE_H = 128
+IMAGE_PLOT_W = int(IMAGE_W * 2) + 52
+IMAGE_PLOT_H = int(IMAGE_H * 2) + 27
+
+
+def create():
+    doc = curdoc()
+    zebra_data = []
+    det_data = {}
+    cami_meta = {}
+
+    num_formatter = NumberFormatter(format="0.00", nan_format="")
+
+    def file_select_update():
+        if data_source.value == "proposal number":
+            proposal_path = proposal_textinput.name
+            if proposal_path:
+                file_list = []
+                for file in os.listdir(proposal_path):
+                    if file.endswith(".hdf"):
+                        file_list.append((os.path.join(proposal_path, file), file))
+                file_select.options = file_list
+            else:
+                file_select.options = []
+
+        else:  # "cami file"
+            if not cami_meta:
+                file_select.options = []
+                return
+
+            file_list = cami_meta["filelist"]
+            file_select.options = [(entry, os.path.basename(entry)) for entry in file_list]
+
+    def data_source_callback(_attr, _old, _new):
+        file_select_update()
+
+    data_source = Select(
+        title="Data Source:",
+        value="proposal number",
+        options=["proposal number", "cami file"],
+        width=210,
+    )
+    data_source.on_change("value", data_source_callback)
+
+    doc.add_periodic_callback(file_select_update, 5000)
+
+    def proposal_textinput_callback(_attr, _old, _new):
+        file_select_update()
+
+    proposal_textinput = doc.proposal_textinput
+    proposal_textinput.on_change("name", proposal_textinput_callback)
+
+    def upload_button_callback(_attr, _old, new):
+        nonlocal cami_meta
+        with io.StringIO(base64.b64decode(new).decode()) as file:
+            cami_meta = pyzebra.parse_h5meta(file)
+        data_source.value = "cami file"
+        file_select_update()
+
+    upload_div = Div(text="or upload .cami file:", margin=(5, 5, 0, 5))
+    upload_button = FileInput(accept=".cami", width=200)
+    upload_button.on_change("value", upload_button_callback)
+
+    file_select = MultiSelect(title="Available .hdf files:", width=210, height=320)
+
+    def _init_datatable():
+        file_list = []
+        for scan in zebra_data:
+            file_list.append(os.path.basename(scan["original_filename"]))
+
+        scan_table_source.data.update(
+            file=file_list,
+            param=[None] * len(zebra_data),
+            frame=[None] * len(zebra_data),
+            x_pos=[None] * len(zebra_data),
+            y_pos=[None] * len(zebra_data),
+        )
+        scan_table_source.selected.indices = []
+        scan_table_source.selected.indices = [0]
+
+        param_select.value = "user defined"
+
+    def _update_table():
+        frame = []
+        x_pos = []
+        y_pos = []
+        for scan in zebra_data:
+            if "fit" in scan:
+                framei = scan["fit"]["frame"]
+                x_posi = scan["fit"]["x_pos"]
+                y_posi = scan["fit"]["y_pos"]
+            else:
+                framei = x_posi = y_posi = None
+
+            frame.append(framei)
+            x_pos.append(x_posi)
+            y_pos.append(y_posi)
+
+        scan_table_source.data.update(frame=frame, x_pos=x_pos, y_pos=y_pos)
+
+    def file_open_button_callback():
+        nonlocal zebra_data
+        zebra_data = []
+        for f_name in file_select.value:
+            zebra_data.append(pyzebra.read_detector_data(f_name))
+
+        _init_datatable()
+
+    file_open_button = Button(label="Open New", width=100)
+    file_open_button.on_click(file_open_button_callback)
+
+    def file_append_button_callback():
+        for f_name in file_select.value:
+            zebra_data.append(pyzebra.read_detector_data(f_name))
+
+        _init_datatable()
+
+    file_append_button = Button(label="Append", width=100)
+    file_append_button.on_click(file_append_button_callback)
+
+    # Scan select
+    def scan_table_select_callback(_attr, old, new):
+        nonlocal det_data
+
+        if not new:
+            # skip empty selections
+            return
+
+        # Avoid selection of multiple indicies (via Shift+Click or Ctrl+Click)
+        if len(new) > 1:
+            # drop selection to the previous one
+            scan_table_source.selected.indices = old
+            return
+
+        if len(old) > 1:
+            # skip unnecessary update caused by selection drop
+            return
+
+        det_data = zebra_data[new[0]]
+
+        zebra_mode = det_data["zebra_mode"]
+        if zebra_mode == "nb":
+            metadata_table_source.data.update(geom=["normal beam"])
+        else:  # zebra_mode == "bi"
+            metadata_table_source.data.update(geom=["bisecting"])
+
+        if "mf" in det_data:
+            metadata_table_source.data.update(mf=[det_data["mf"][0]])
+        else:
+            metadata_table_source.data.update(mf=[None])
+
+        if "temp" in det_data:
+            metadata_table_source.data.update(temp=[det_data["temp"][0]])
+        else:
+            metadata_table_source.data.update(temp=[None])
+
+        update_overview_plot()
+
+    def scan_table_source_callback(_attr, _old, _new):
+        pass
+
+    scan_table_source = ColumnDataSource(dict(file=[], param=[], frame=[], x_pos=[], y_pos=[]))
+    scan_table_source.selected.on_change("indices", scan_table_select_callback)
+    scan_table_source.on_change("data", scan_table_source_callback)
+
+    scan_table = DataTable(
+        source=scan_table_source,
+        columns=[
+            TableColumn(field="file", title="file", width=150),
+            TableColumn(
+                field="param",
+                title="param",
+                formatter=num_formatter,
+                editor=NumberEditor(),
+                width=50,
+            ),
+            TableColumn(field="frame", title="Frame", formatter=num_formatter, width=70),
+            TableColumn(field="x_pos", title="X", formatter=num_formatter, width=70),
+            TableColumn(field="y_pos", title="Y", formatter=num_formatter, width=70),
+        ],
+        width=470,  # +60 because of the index column
+        height=420,
+        editable=True,
+        autosize_mode="none",
+    )
+
+    def param_select_callback(_attr, _old, new):
+        if new == "user defined":
+            param = [None] * len(zebra_data)
+        else:
+            # TODO: which value to take?
+            param = [scan[new][0] for scan in zebra_data]
+
+        scan_table_source.data["param"] = param
+        _update_param_plot()
+
+    param_select = Select(
+        title="Parameter:",
+        options=["user defined", "temp", "mf", "h", "k", "l"],
+        value="user defined",
+        width=145,
+    )
+    param_select.on_change("value", param_select_callback)
+
+    def update_overview_plot():
+        h5_data = det_data["data"]
+        n_im, n_y, n_x = h5_data.shape
+        overview_x = np.mean(h5_data, axis=1)
+        overview_y = np.mean(h5_data, axis=2)
+
+        # normalize for simpler colormapping
+        overview_max_val = max(np.max(overview_x), np.max(overview_y))
+        overview_x = 1000 * overview_x / overview_max_val
+        overview_y = 1000 * overview_y / overview_max_val
+
+        overview_plot_x_image_source.data.update(image=[overview_x], dw=[n_x], dh=[n_im])
+        overview_plot_y_image_source.data.update(image=[overview_y], dw=[n_y], dh=[n_im])
+
+        if proj_auto_checkbox.active:
+            im_min = min(np.min(overview_x), np.min(overview_y))
+            im_max = max(np.max(overview_x), np.max(overview_y))
+
+            proj_display_min_spinner.value = im_min
+            proj_display_max_spinner.value = im_max
+
+            overview_plot_x_image_glyph.color_mapper.low = im_min
+            overview_plot_y_image_glyph.color_mapper.low = im_min
+            overview_plot_x_image_glyph.color_mapper.high = im_max
+            overview_plot_y_image_glyph.color_mapper.high = im_max
+
+        frame_range.start = 0
+        frame_range.end = n_im
+        frame_range.reset_start = 0
+        frame_range.reset_end = n_im
+        frame_range.bounds = (0, n_im)
+
+        scan_motor = det_data["scan_motor"]
+        overview_plot_y.axis[1].axis_label = f"Scanning motor, {scan_motor}"
+
+        var = det_data[scan_motor]
+        var_start = var[0]
+        var_end = var[-1] + (var[-1] - var[0]) / (n_im - 1)
+
+        scanning_motor_range.start = var_start
+        scanning_motor_range.end = var_end
+        scanning_motor_range.reset_start = var_start
+        scanning_motor_range.reset_end = var_end
+        # handle both, ascending and descending sequences
+        scanning_motor_range.bounds = (min(var_start, var_end), max(var_start, var_end))
+
+    # shared frame ranges
+    frame_range = Range1d(0, 1, bounds=(0, 1))
+    scanning_motor_range = Range1d(0, 1, bounds=(0, 1))
+
+    det_x_range = Range1d(0, IMAGE_W, bounds=(0, IMAGE_W))
+    overview_plot_x = Plot(
+        title=Title(text="Projections on X-axis"),
+        x_range=det_x_range,
+        y_range=frame_range,
+        extra_y_ranges={"scanning_motor": scanning_motor_range},
+        plot_height=400,
+        plot_width=IMAGE_PLOT_W - 3,
+    )
+
+    # ---- tools
+    wheelzoomtool = WheelZoomTool(maintain_focus=False)
+    overview_plot_x.toolbar.logo = None
+    overview_plot_x.add_tools(
+        PanTool(), BoxZoomTool(), wheelzoomtool, ResetTool(),
+    )
+    overview_plot_x.toolbar.active_scroll = wheelzoomtool
+
+    # ---- axes
+    overview_plot_x.add_layout(LinearAxis(axis_label="Coordinate X, pix"), place="below")
+    overview_plot_x.add_layout(
+        LinearAxis(axis_label="Frame", major_label_orientation="vertical"), place="left"
+    )
+
+    # ---- grid lines
+    overview_plot_x.add_layout(Grid(dimension=0, ticker=BasicTicker()))
+    overview_plot_x.add_layout(Grid(dimension=1, ticker=BasicTicker()))
+
+    # ---- rgba image glyph
+    overview_plot_x_image_source = ColumnDataSource(
+        dict(image=[np.zeros((1, 1), dtype="float32")], x=[0], y=[0], dw=[IMAGE_W], dh=[1])
+    )
+
+    overview_plot_x_image_glyph = Image(image="image", x="x", y="y", dw="dw", dh="dh")
+    overview_plot_x.add_glyph(
+        overview_plot_x_image_source, overview_plot_x_image_glyph, name="image_glyph"
+    )
+
+    det_y_range = Range1d(0, IMAGE_H, bounds=(0, IMAGE_H))
+    overview_plot_y = Plot(
+        title=Title(text="Projections on Y-axis"),
+        x_range=det_y_range,
+        y_range=frame_range,
+        extra_y_ranges={"scanning_motor": scanning_motor_range},
+        plot_height=400,
+        plot_width=IMAGE_PLOT_H + 22,
+    )
+
+    # ---- tools
+    wheelzoomtool = WheelZoomTool(maintain_focus=False)
+    overview_plot_y.toolbar.logo = None
+    overview_plot_y.add_tools(
+        PanTool(), BoxZoomTool(), wheelzoomtool, ResetTool(),
+    )
+    overview_plot_y.toolbar.active_scroll = wheelzoomtool
+
+    # ---- axes
+    overview_plot_y.add_layout(LinearAxis(axis_label="Coordinate Y, pix"), place="below")
+    overview_plot_y.add_layout(
+        LinearAxis(
+            y_range_name="scanning_motor",
+            axis_label="Scanning motor",
+            major_label_orientation="vertical",
+        ),
+        place="right",
+    )
+
+    # ---- grid lines
+    overview_plot_y.add_layout(Grid(dimension=0, ticker=BasicTicker()))
+    overview_plot_y.add_layout(Grid(dimension=1, ticker=BasicTicker()))
+
+    # ---- rgba image glyph
+    overview_plot_y_image_source = ColumnDataSource(
+        dict(image=[np.zeros((1, 1), dtype="float32")], x=[0], y=[0], dw=[IMAGE_H], dh=[1])
+    )
+
+    overview_plot_y_image_glyph = Image(image="image", x="x", y="y", dw="dw", dh="dh")
+    overview_plot_y.add_glyph(
+        overview_plot_y_image_source, overview_plot_y_image_glyph, name="image_glyph"
+    )
+
+    cmap_dict = {
+        "gray": Greys256,
+        "gray_reversed": Greys256[::-1],
+        "plasma": Plasma256,
+        "cividis": Cividis256,
+    }
+
+    def colormap_callback(_attr, _old, new):
+        overview_plot_x_image_glyph.color_mapper = LinearColorMapper(palette=cmap_dict[new])
+        overview_plot_y_image_glyph.color_mapper = LinearColorMapper(palette=cmap_dict[new])
+
+    colormap = Select(title="Colormap:", options=list(cmap_dict.keys()), width=210)
+    colormap.on_change("value", colormap_callback)
+    colormap.value = "plasma"
+
+    PROJ_STEP = 1
+
+    def proj_auto_checkbox_callback(state):
+        if state:
+            proj_display_min_spinner.disabled = True
+            proj_display_max_spinner.disabled = True
+        else:
+            proj_display_min_spinner.disabled = False
+            proj_display_max_spinner.disabled = False
+
+        update_overview_plot()
+
+    proj_auto_checkbox = CheckboxGroup(
+        labels=["Projections Intensity Range"], active=[0], width=145, margin=[10, 5, 0, 5]
+    )
+    proj_auto_checkbox.on_click(proj_auto_checkbox_callback)
+
+    def proj_display_max_spinner_callback(_attr, _old_value, new_value):
+        proj_display_min_spinner.high = new_value - PROJ_STEP
+        overview_plot_x_image_glyph.color_mapper.high = new_value
+        overview_plot_y_image_glyph.color_mapper.high = new_value
+
+    proj_display_max_spinner = Spinner(
+        low=0 + PROJ_STEP,
+        value=1,
+        step=PROJ_STEP,
+        disabled=bool(proj_auto_checkbox.active),
+        width=100,
+        height=31,
+    )
+    proj_display_max_spinner.on_change("value", proj_display_max_spinner_callback)
+
+    def proj_display_min_spinner_callback(_attr, _old_value, new_value):
+        proj_display_max_spinner.low = new_value + PROJ_STEP
+        overview_plot_x_image_glyph.color_mapper.low = new_value
+        overview_plot_y_image_glyph.color_mapper.low = new_value
+
+    proj_display_min_spinner = Spinner(
+        low=0,
+        high=1 - PROJ_STEP,
+        value=0,
+        step=PROJ_STEP,
+        disabled=bool(proj_auto_checkbox.active),
+        width=100,
+        height=31,
+    )
+    proj_display_min_spinner.on_change("value", proj_display_min_spinner_callback)
+
+    def fit_event(scan):
+        p0 = [1.0, 0.0, 1.0]
+        maxfev = 100000
+
+        # wave = scan["wave"]
+        # ddist = scan["ddist"]
+        # cell = scan["cell"]
+
+        # gamma = scan["gamma"][0]
+        # omega = scan["omega"][0]
+        # nu = scan["nu"][0]
+        # chi = scan["chi"][0]
+        # phi = scan["phi"][0]
+
+        scan_motor = scan["scan_motor"]
+        var_angle = scan[scan_motor]
+
+        x0 = int(np.floor(det_x_range.start))
+        xN = int(np.ceil(det_x_range.end))
+        y0 = int(np.floor(det_y_range.start))
+        yN = int(np.ceil(det_y_range.end))
+        fr0 = int(np.floor(frame_range.start))
+        frN = int(np.ceil(frame_range.end))
+        data_roi = scan["data"][fr0:frN, y0:yN, x0:xN]
+
+        cnts = np.sum(data_roi, axis=(1, 2))
+        coeff, _ = curve_fit(gauss, range(len(cnts)), cnts, p0=p0, maxfev=maxfev)
+
+        # m = cnts.mean()
+        # sd = cnts.std()
+        # snr_cnts = np.where(sd == 0, 0, m / sd)
+
+        frC = fr0 + coeff[1]
+        var_F = var_angle[math.floor(frC)]
+        var_C = var_angle[math.ceil(frC)]
+        # frStep = frC - math.floor(frC)
+        var_step = var_C - var_F
+        # var_p = var_F + var_step * frStep
+
+        # if scan_motor == "gamma":
+        #     gamma = var_p
+        # elif scan_motor == "omega":
+        #     omega = var_p
+        # elif scan_motor == "nu":
+        #     nu = var_p
+        # elif scan_motor == "chi":
+        #     chi = var_p
+        # elif scan_motor == "phi":
+        #     phi = var_p
+
+        intensity = coeff[1] * abs(coeff[2] * var_step) * math.sqrt(2) * math.sqrt(np.pi)
+
+        projX = np.sum(data_roi, axis=(0, 1))
+        coeff, _ = curve_fit(gauss, range(len(projX)), projX, p0=p0, maxfev=maxfev)
+        x_pos = x0 + coeff[1]
+
+        projY = np.sum(data_roi, axis=(0, 2))
+        coeff, _ = curve_fit(gauss, range(len(projY)), projY, p0=p0, maxfev=maxfev)
+        y_pos = y0 + coeff[1]
+
+        scan["fit"] = {"frame": frC, "x_pos": x_pos, "y_pos": y_pos, "intensity": intensity}
+
+    metadata_table_source = ColumnDataSource(dict(geom=[""], temp=[None], mf=[None]))
+    metadata_table = DataTable(
+        source=metadata_table_source,
+        columns=[
+            TableColumn(field="geom", title="Geometry", width=100),
+            TableColumn(field="temp", title="Temperature", formatter=num_formatter, width=100),
+            TableColumn(field="mf", title="Magnetic Field", formatter=num_formatter, width=100),
+        ],
+        width=300,
+        height=50,
+        autosize_mode="none",
+        index_position=None,
+    )
+
+    def _update_param_plot():
+        x = []
+        y = []
+        fit_param = fit_param_select.value
+        for s, p in zip(zebra_data, scan_table_source.data["param"]):
+            if "fit" in s and fit_param:
+                x.append(p)
+                y.append(s["fit"][fit_param])
+        param_plot_scatter_source.data.update(x=x, y=y)
+
+    # Parameter plot
+    param_plot = Plot(x_range=DataRange1d(), y_range=DataRange1d(), plot_height=400, plot_width=700)
+
+    param_plot.add_layout(LinearAxis(axis_label="Fit parameter"), place="left")
+    param_plot.add_layout(LinearAxis(axis_label="Parameter"), place="below")
+
+    param_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
+    param_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
+
+    param_plot_scatter_source = ColumnDataSource(dict(x=[], y=[]))
+    param_plot.add_glyph(param_plot_scatter_source, Scatter(x="x", y="y"))
+
+    param_plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
+    param_plot.toolbar.logo = None
+
+    def fit_param_select_callback(_attr, _old, _new):
+        _update_param_plot()
+
+    fit_param_select = Select(title="Fit parameter", options=[], width=145)
+    fit_param_select.on_change("value", fit_param_select_callback)
+
+    def proc_all_button_callback():
+        for scan in zebra_data:
+            fit_event(scan)
+
+        _update_table()
+
+        for scan in zebra_data:
+            if "fit" in scan:
+                options = list(scan["fit"].keys())
+                fit_param_select.options = options
+                fit_param_select.value = options[0]
+                break
+
+        _update_param_plot()
+
+    proc_all_button = Button(label="Process All", button_type="primary", width=145)
+    proc_all_button.on_click(proc_all_button_callback)
+
+    def proc_button_callback():
+        fit_event(det_data)
+
+        _update_table()
+
+        for scan in zebra_data:
+            if "fit" in scan:
+                options = list(scan["fit"].keys())
+                fit_param_select.options = options
+                fit_param_select.value = options[0]
+                break
+
+        _update_param_plot()
+
+    proc_button = Button(label="Process Current", width=145)
+    proc_button.on_click(proc_button_callback)
+
+    layout_controls = row(
+        colormap,
+        column(proj_auto_checkbox, row(proj_display_min_spinner, proj_display_max_spinner)),
+        proc_button,
+        proc_all_button,
+    )
+
+    layout_overview = column(
+        gridplot(
+            [[overview_plot_x, overview_plot_y]],
+            toolbar_options=dict(logo=None),
+            merge_tools=True,
+            toolbar_location="left",
+        ),
+        layout_controls,
+    )
+
+    # Plot tabs
+    plots = Tabs(
+        tabs=[
+            Panel(child=layout_overview, title="single scan"),
+            Panel(child=column(param_plot, row(fit_param_select)), title="parameter plot"),
+        ]
+    )
+
+    # Final layout
+    import_layout = column(
+        data_source,
+        upload_div,
+        upload_button,
+        file_select,
+        row(file_open_button, file_append_button),
+    )
+
+    scan_layout = column(scan_table, row(param_select, metadata_table))
+
+    tab_layout = column(row(import_layout, scan_layout, plots))
+
+    return Panel(child=tab_layout, title="hdf param study")
+
+
+def gauss(x, *p):
+    """Defines Gaussian function
+    Args:
+        A - amplitude, mu - position of the center, sigma - width
+    Returns:
+        Gaussian function
+    """
+    A, mu, sigma = p
+    return A * np.exp(-((x - mu) ** 2) / (2.0 * sigma ** 2))

pyzebra/app/panel_hdf_viewer.py

@@ -37,7 +37,6 @@ from bokeh.models import (
     Spacer,
     Spinner,
     TableColumn,
-    TextInput,
     Title,
     WheelZoomTool,
 )
@@ -59,42 +58,51 @@ def create():
 
     num_formatter = NumberFormatter(format="0.00", nan_format="")
 
-    def file_select_update_for_proposal():
-        proposal = proposal_textinput.value.strip()
-        if not proposal:
-            return
+    def file_select_update():
+        if data_source.value == "proposal number":
+            proposal_path = proposal_textinput.name
+            if proposal_path:
+                file_list = []
+                for file in os.listdir(proposal_path):
+                    if file.endswith(".hdf"):
+                        file_list.append((os.path.join(proposal_path, file), file))
+                file_select.options = file_list
+            else:
+                file_select.options = []
 
-        for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
-            proposal_path = os.path.join(zebra_proposals_path, proposal)
-            if os.path.isdir(proposal_path):
-                # found it
-                break
-        else:
-            raise ValueError(f"Can not find data for proposal '{proposal}'.")
+        else:  # "cami file"
+            if not cami_meta:
+                file_select.options = []
+                return
 
-        file_list = []
-        for file in os.listdir(proposal_path):
-            if file.endswith(".hdf"):
-                file_list.append((os.path.join(proposal_path, file), file))
-        file_select.options = file_list
+            file_list = cami_meta["filelist"]
+            file_select.options = [(entry, os.path.basename(entry)) for entry in file_list]
 
-    doc.add_periodic_callback(file_select_update_for_proposal, 5000)
+    def data_source_callback(_attr, _old, _new):
+        file_select_update()
+
+    data_source = Select(
+        title="Data Source:",
+        value="proposal number",
+        options=["proposal number", "cami file"],
+        width=210,
+    )
+    data_source.on_change("value", data_source_callback)
+
+    doc.add_periodic_callback(file_select_update, 5000)
 
     def proposal_textinput_callback(_attr, _old, _new):
-        nonlocal cami_meta
-        cami_meta = {}
-        file_select_update_for_proposal()
+        file_select_update()
 
-    proposal_textinput = TextInput(title="Proposal number:", width=210)
-    proposal_textinput.on_change("value", proposal_textinput_callback)
+    proposal_textinput = doc.proposal_textinput
+    proposal_textinput.on_change("name", proposal_textinput_callback)
 
     def upload_button_callback(_attr, _old, new):
         nonlocal cami_meta
-        proposal_textinput.value = ""
         with io.StringIO(base64.b64decode(new).decode()) as file:
             cami_meta = pyzebra.parse_h5meta(file)
-            file_list = cami_meta["filelist"]
-            file_select.options = [(entry, os.path.basename(entry)) for entry in file_list]
+        data_source.value = "cami file"
+        file_select_update()
 
     upload_div = Div(text="or upload .cami file:", margin=(5, 5, 0, 5))
     upload_button = FileInput(accept=".cami", width=200)
@@ -147,6 +155,11 @@ def create():
         overview_x = np.mean(h5_data, axis=1)
         overview_y = np.mean(h5_data, axis=2)
 
+        # normalize for simpler colormapping
+        overview_max_val = max(np.max(overview_x), np.max(overview_y))
+        overview_x = 1000 * overview_x / overview_max_val
+        overview_y = 1000 * overview_y / overview_max_val
+
         overview_plot_x_image_source.data.update(image=[overview_x], dw=[n_x], dh=[n_im])
         overview_plot_y_image_source.data.update(image=[overview_y], dw=[n_y], dh=[n_im])
 
@@ -536,7 +549,7 @@ def create():
     )
     display_min_spinner.on_change("value", display_min_spinner_callback)
 
-    PROJ_STEP = 0.1
+    PROJ_STEP = 1
 
     def proj_auto_checkbox_callback(state):
         if state:
@@ -726,7 +739,7 @@ def create():
     )
 
     # Final layout
-    import_layout = column(proposal_textinput, upload_div, upload_button, file_select)
+    import_layout = column(data_source, upload_div, upload_button, file_select)
     layout_image = column(gridplot([[proj_v, None], [plot, proj_h]], merge_tools=False))
     colormap_layout = column(
         colormap,

pyzebra/app/panel_param_study.py

@@ -42,7 +42,6 @@ from bokeh.models import (
     TableColumn,
     Tabs,
     TextAreaInput,
-    TextInput,
     WheelZoomTool,
     Whisker,
 )
@@ -54,18 +53,23 @@ import pyzebra
 from pyzebra.ccl_process import AREA_METHODS
 
 javaScript = """
+let j = 0;
 for (let i = 0; i < js_data.data['fname'].length; i++) {
     if (js_data.data['content'][i] === "") continue;
 
-    const blob = new Blob([js_data.data['content'][i]], {type: 'text/plain'})
-    const link = document.createElement('a');
-    document.body.appendChild(link);
-    const url = window.URL.createObjectURL(blob);
-    link.href = url;
-    link.download = js_data.data['fname'][i];
-    link.click();
-    window.URL.revokeObjectURL(url);
-    document.body.removeChild(link);
+    setTimeout(function() {
+        const blob = new Blob([js_data.data['content'][i]], {type: 'text/plain'})
+        const link = document.createElement('a');
+        document.body.appendChild(link);
+        const url = window.URL.createObjectURL(blob);
+        link.href = url;
+        link.download = js_data.data['fname'][i] + js_data.data['ext'][i];
+        link.click();
+        window.URL.revokeObjectURL(url);
+        document.body.removeChild(link);
+    }, 100 * j)
+
+    j++;
 }
 """
 
@@ -79,39 +83,30 @@ def create():
     doc = curdoc()
     det_data = []
     fit_params = {}
-    js_data = ColumnDataSource(data=dict(content=["", ""], fname=["", ""]))
+    js_data = ColumnDataSource(data=dict(content=[""], fname=[""], ext=[""]))
 
     def file_select_update_for_proposal():
-        proposal = proposal_textinput.value.strip()
-        if not proposal:
+        proposal_path = proposal_textinput.name
+        if proposal_path:
+            file_list = []
+            for file in os.listdir(proposal_path):
+                if file.endswith((".ccl", ".dat")):
+                    file_list.append((os.path.join(proposal_path, file), file))
+            file_select.options = file_list
+            file_open_button.disabled = False
+            file_append_button.disabled = False
+        else:
             file_select.options = []
             file_open_button.disabled = True
             file_append_button.disabled = True
-            return
-
-        for zebra_proposals_path in pyzebra.ZEBRA_PROPOSALS_PATHS:
-            proposal_path = os.path.join(zebra_proposals_path, proposal)
-            if os.path.isdir(proposal_path):
-                # found it
-                break
-        else:
-            raise ValueError(f"Can not find data for proposal '{proposal}'.")
-
-        file_list = []
-        for file in os.listdir(proposal_path):
-            if file.endswith((".ccl", ".dat")):
-                file_list.append((os.path.join(proposal_path, file), file))
-        file_select.options = file_list
-        file_open_button.disabled = False
-        file_append_button.disabled = False
 
     doc.add_periodic_callback(file_select_update_for_proposal, 5000)
 
     def proposal_textinput_callback(_attr, _old, _new):
         file_select_update_for_proposal()
 
-    proposal_textinput = TextInput(title="Proposal number:", width=210)
-    proposal_textinput.on_change("value", proposal_textinput_callback)
+    proposal_textinput = doc.proposal_textinput
+    proposal_textinput.on_change("name", proposal_textinput_callback)
 
     def _init_datatable():
         scan_list = [s["idx"] for s in det_data]
@@ -137,18 +132,19 @@ def create():
 
     def file_open_button_callback():
         nonlocal det_data
-        det_data = []
-        for f_name in file_select.value:
-            with open(f_name) as file:
-                base, ext = os.path.splitext(f_name)
-                if det_data:
-                    append_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-                    det_data.extend(append_data)
-                else:
-                    det_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(det_data, monitor_spinner.value)
-                    js_data.data.update(fname=[base + ".comm", base + ".incomm"])
+        for f_ind, f_path in enumerate(file_select.value):
+            with open(f_path) as file:
+                base, ext = os.path.splitext(os.path.basename(f_path))
+                file_data = pyzebra.parse_1D(file, ext)
+
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+
+            if f_ind == 0:  # first file
+                det_data = file_data
+                pyzebra.merge_duplicates(det_data)
+                js_data.data.update(fname=[base])
+            else:
+                pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
         append_upload_button.disabled = False
@@ -157,13 +153,13 @@ def create():
     file_open_button.on_click(file_open_button_callback)
 
     def file_append_button_callback():
-        for f_name in file_select.value:
-            with open(f_name) as file:
-                _, ext = os.path.splitext(f_name)
-                append_data = pyzebra.parse_1D(file, ext)
+        for f_path in file_select.value:
+            with open(f_path) as file:
+                _, ext = os.path.splitext(f_path)
+                file_data = pyzebra.parse_1D(file, ext)
 
-            pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-            det_data.extend(append_data)
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+            pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
 
@@ -173,18 +169,19 @@ def create():
     def upload_button_callback(_attr, _old, new):
         nonlocal det_data
         det_data = []
-        proposal_textinput.value = ""
         for f_str, f_name in zip(new, upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 base, ext = os.path.splitext(f_name)
-                if det_data:
-                    append_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-                    det_data.extend(append_data)
-                else:
-                    det_data = pyzebra.parse_1D(file, ext)
-                    pyzebra.normalize_dataset(det_data, monitor_spinner.value)
-                    js_data.data.update(fname=[base + ".comm", base + ".incomm"])
+                file_data = pyzebra.parse_1D(file, ext)
+
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+
+            if not det_data:  # first file
+                det_data = file_data
+                pyzebra.merge_duplicates(det_data)
+                js_data.data.update(fname=[base])
+            else:
+                pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
         append_upload_button.disabled = False
@@ -197,10 +194,10 @@ def create():
         for f_str, f_name in zip(new, append_upload_button.filename):
             with io.StringIO(base64.b64decode(f_str).decode()) as file:
                 _, ext = os.path.splitext(f_name)
-                append_data = pyzebra.parse_1D(file, ext)
+                file_data = pyzebra.parse_1D(file, ext)
 
-            pyzebra.normalize_dataset(append_data, monitor_spinner.value)
-            det_data.extend(append_data)
+            pyzebra.normalize_dataset(file_data, monitor_spinner.value)
+            pyzebra.merge_datasets(det_data, file_data)
 
         _init_datatable()
 
@@ -230,17 +227,19 @@ def create():
         scan_table_source.data.update(fit=fit_ok)
 
     def _update_plot():
-        _update_single_scan_plot(_get_selected_scan())
+        _update_single_scan_plot()
         _update_overview()
 
-    def _update_single_scan_plot(scan):
+    def _update_single_scan_plot():
+        scan = _get_selected_scan()
         scan_motor = scan["scan_motor"]
 
         y = scan["counts"]
+        y_err = scan["counts_err"]
         x = scan[scan_motor]
 
         plot.axis[0].axis_label = scan_motor
-        plot_scatter_source.data.update(x=x, y=y, y_upper=y + np.sqrt(y), y_lower=y - np.sqrt(y))
+        plot_scatter_source.data.update(x=x, y=y, y_upper=y + y_err, y_lower=y - y_err)
 
         fit = scan.get("fit")
         if fit is not None:
@@ -320,13 +319,19 @@ def create():
     def _update_param_plot():
         x = []
         y = []
+        y_lower = []
+        y_upper = []
         fit_param = fit_param_select.value
         for s, p in zip(det_data, scan_table_source.data["param"]):
             if "fit" in s and fit_param:
                 x.append(p)
-                y.append(s["fit"].values[fit_param])
+                param_fit_val = s["fit"].params[fit_param].value
+                param_fit_std = s["fit"].params[fit_param].stderr
+                y.append(param_fit_val)
+                y_lower.append(param_fit_val - param_fit_std)
+                y_upper.append(param_fit_val + param_fit_std)
 
-        param_plot_scatter_source.data.update(x=x, y=y)
+        param_plot_scatter_source.data.update(x=x, y=y, y_lower=y_lower, y_upper=y_upper)
 
     # Main plot
     plot = Plot(
@@ -436,8 +441,11 @@ def create():
     param_plot.add_layout(Grid(dimension=0, ticker=BasicTicker()))
     param_plot.add_layout(Grid(dimension=1, ticker=BasicTicker()))
 
-    param_plot_scatter_source = ColumnDataSource(dict(x=[], y=[]))
+    param_plot_scatter_source = ColumnDataSource(dict(x=[], y=[], y_upper=[], y_lower=[]))
     param_plot.add_glyph(param_plot_scatter_source, Scatter(x="x", y="y"))
+    param_plot.add_layout(
+        Whisker(source=param_plot_scatter_source, base="x", upper="y_upper", lower="y_lower")
+    )
 
     param_plot.add_tools(PanTool(), WheelZoomTool(), ResetTool())
     param_plot.toolbar.logo = None
@@ -481,6 +489,7 @@ def create():
 
     scan_table_source = ColumnDataSource(dict(file=[], scan=[], param=[], fit=[], export=[]))
     scan_table_source.on_change("data", scan_table_source_callback)
+    scan_table_source.selected.on_change("indices", scan_table_select_callback)
 
     scan_table = DataTable(
         source=scan_table_source,
@@ -496,13 +505,6 @@ def create():
         autosize_mode="none",
     )
 
-    def scan_table_source_callback(_attr, _old, _new):
-        if scan_table_source.selected.indices:
-            _update_plot()
-
-    scan_table_source.selected.on_change("indices", scan_table_select_callback)
-    scan_table_source.on_change("data", scan_table_source_callback)
-
     def _get_selected_scan():
         return det_data[scan_table_source.selected.indices[0]]
 
@@ -698,34 +700,38 @@ def create():
 
     lorentz_checkbox = CheckboxGroup(labels=["Lorentz Correction"], width=145, margin=(13, 5, 5, 5))
 
-    export_preview_textinput = TextAreaInput(title="Export file(s) preview:", width=450, height=400)
+    export_preview_textinput = TextAreaInput(title="Export file preview:", width=450, height=400)
 
     def _update_preview():
         with tempfile.TemporaryDirectory() as temp_dir:
             temp_file = temp_dir + "/temp"
             export_data = []
-            for s, export in zip(det_data, scan_table_source.data["export"]):
+            param_data = []
+            for s, p, export in zip(
+                det_data, scan_table_source.data["param"], scan_table_source.data["export"]
+            ):
                 if export:
                     export_data.append(s)
+                    param_data.append(p)
 
-            # pyzebra.export_1D(export_data, temp_file, "fullprof")
+            pyzebra.export_param_study(export_data, param_data, temp_file)
 
             exported_content = ""
             file_content = []
-            for ext in (".comm", ".incomm"):
-                fname = temp_file + ext
-                if os.path.isfile(fname):
-                    with open(fname) as f:
-                        content = f.read()
-                        exported_content += f"{ext} file:\n" + content
-                else:
-                    content = ""
-                file_content.append(content)
+
+            fname = temp_file
+            if os.path.isfile(fname):
+                with open(fname) as f:
+                    content = f.read()
+                    exported_content += content
+            else:
+                content = ""
+            file_content.append(content)
 
             js_data.data.update(content=file_content)
             export_preview_textinput.value = exported_content
 
-    save_button = Button(label="Download File(s)", button_type="success", width=220)
+    save_button = Button(label="Download File", button_type="success", width=220)
     save_button.js_on_click(CustomJS(args={"js_data": js_data}, code=javaScript))
 
     fitpeak_controls = row(
@@ -739,7 +745,6 @@ def create():
     scan_layout = column(scan_table, row(monitor_spinner, scan_motor_select, param_select))
 
     import_layout = column(
-        proposal_textinput,
         file_select,
         row(file_open_button, file_append_button),
         upload_div,

pyzebra/app/panel_spind.py

@@ -24,7 +24,7 @@ def create():
     events_data = doc.events_data
 
     npeaks_spinner = Spinner(title="Number of peaks from hdf_view panel:", disabled=True)
-    lattice_const_textinput = TextInput(title="Lattice constants:", disabled=True)
+    lattice_const_textinput = TextInput(title="Lattice constants:")
     max_res_spinner = Spinner(title="max-res:", value=2, step=0.01, width=145)
     seed_pool_size_spinner = Spinner(title="seed-pool-size:", value=5, step=0.01, width=145)
     seed_len_tol_spinner = Spinner(title="seed-len-tol:", value=0.02, step=0.01, width=145)

pyzebra/ccl_io.py

@@ -169,6 +169,7 @@ def parse_1D(fileobj, data_type):
             while len(counts) < s["n_points"]:
                 counts.extend(map(float, next(fileobj).split()))
             s["counts"] = np.array(counts)
+            s["counts_err"] = np.sqrt(s["counts"])
 
             if s["h"].is_integer() and s["k"].is_integer() and s["l"].is_integer():
                 s["h"], s["k"], s["l"] = map(int, (s["h"], s["k"], s["l"]))
@@ -204,6 +205,8 @@ def parse_1D(fileobj, data_type):
         for name in col_names:
             s[name] = np.array(s[name])
 
+        s["counts_err"] = np.sqrt(s["counts"])
+
         s["scan_motors"] = []
         for motor, step in zip(motors, steps):
             if step == 0:
@@ -211,27 +214,24 @@ def parse_1D(fileobj, data_type):
                 s[motor] = np.median(s[motor])
             else:
                 s["scan_motors"].append(motor)
-        s["scan_motor"] = s["scan_motors"][0]
 
         # "om" -> "omega"
         if "om" in s["scan_motors"]:
             s["scan_motors"][s["scan_motors"].index("om")] = "omega"
-            if s["scan_motor"] == "om":
-                s["scan_motor"] = "omega"
             s["omega"] = s["om"]
             del s["om"]
 
         # "tt" -> "temp"
-        elif "tt" in s["scan_motors"]:
+        if "tt" in s["scan_motors"]:
             s["scan_motors"][s["scan_motors"].index("tt")] = "temp"
-            if s["scan_motor"] == "tt":
-                s["scan_motor"] = "temp"
             s["temp"] = s["tt"]
             del s["tt"]
 
         # "mf" stays "mf"
         # "phi" stays "phi"
 
+        s["scan_motor"] = s["scan_motors"][0]
+
         if "h" not in s:
             s["h"] = s["k"] = s["l"] = float("nan")
 
@@ -301,3 +301,31 @@ def export_1D(data, path, export_target, hkl_precision=2):
         if content:
             with open(path + ext, "w") as out_file:
                 out_file.writelines(content)
+
+
+def export_param_study(data, param_data, path):
+    file_content = []
+    for scan, param in zip(data, param_data):
+        if "fit" not in scan:
+            continue
+
+        if not file_content:
+            title_str = f"{'param':12}"
+            for fit_param_name in scan["fit"].params:
+                title_str = title_str + f"{fit_param_name:20}" + f"{'std_' + fit_param_name:20}"
+            title_str = title_str + "file"
+            file_content.append(title_str + "\n")
+
+        param_str = f"{param:<12.2f}"
+
+        fit_str = ""
+        for fit_param in scan["fit"].params.values():
+            fit_str = fit_str + f"{fit_param.value:<20.2f}" + f"{fit_param.stderr:<20.2f}"
+
+        _, fname_str = os.path.split(scan["original_filename"])
+
+        file_content.append(param_str + fit_str + fname_str + "\n")
+
+    if file_content:
+        with open(path, "w") as out_file:
+            out_file.writelines(file_content)

pyzebra/ccl_process.py

@@ -1,4 +1,3 @@
-import itertools
 import os
 
 import numpy as np
@@ -30,6 +29,7 @@ def normalize_dataset(dataset, monitor=100_000):
     for scan in dataset:
         monitor_ratio = monitor / scan["monitor"]
         scan["counts"] *= monitor_ratio
+        scan["counts_err"] *= monitor_ratio
         scan["monitor"] = monitor
 
 
@@ -81,17 +81,41 @@ def merge_datasets(dataset_into, dataset_from):
 
 def merge_scans(scan_into, scan_from):
     # TODO: does it need to be "scan_motor" instead of omega for a generalized solution?
-    if "init_omega" not in scan_into:
-        scan_into["init_omega"] = scan_into["omega"]
-        scan_into["init_counts"] = scan_into["counts"]
+    if "init_scan" not in scan_into:
+        scan_into["init_scan"] = scan_into.copy()
 
-    omega = np.concatenate((scan_into["omega"], scan_from["omega"]))
-    counts = np.concatenate((scan_into["counts"], scan_from["counts"]))
+    if "merged_scans" not in scan_into:
+        scan_into["merged_scans"] = []
 
-    index = np.argsort(omega)
+    if scan_from in scan_into["merged_scans"]:
+        return
 
-    scan_into["omega"] = omega[index]
-    scan_into["counts"] = counts[index]
+    scan_into["merged_scans"].append(scan_from)
+
+    if (
+        scan_into["omega"].shape == scan_from["omega"].shape
+        and np.max(np.abs(scan_into["omega"] - scan_from["omega"])) < 0.0005
+    ):
+        counts_tmp = 0
+        counts_err_tmp = 0
+
+        for scan in [scan_into["init_scan"], *scan_into["merged_scans"]]:
+            counts_tmp += scan["counts"]
+            counts_err_tmp += scan["counts_err"] ** 2
+
+        scan_into["counts"] = counts_tmp / (1 + len(scan_into["merged_scans"]))
+        scan_into["counts_err"] = np.sqrt(counts_err_tmp)
+
+    else:
+        omega = np.concatenate((scan_into["omega"], scan_from["omega"]))
+        counts = np.concatenate((scan_into["counts"], scan_from["counts"]))
+        counts_err = np.concatenate((scan_into["counts_err"], scan_from["counts_err"]))
+
+        index = np.argsort(omega)
+
+        scan_into["omega"] = omega[index]
+        scan_into["counts"] = counts[index]
+        scan_into["counts_err"] = counts_err[index]
 
     scan_from["active"] = False
 
@@ -101,11 +125,14 @@ def merge_scans(scan_into, scan_from):
 
 
 def restore_scan(scan):
-    if "init_omega" in scan:
-        scan["omega"] = scan["init_omega"]
-        scan["counts"] = scan["init_counts"]
-        del scan["init_omega"]
-        del scan["init_counts"]
+    if "merged_scans" in scan:
+        for merged_scan in scan["merged_scans"]:
+            merged_scan["active"] = True
+
+    if "init_scan" in scan:
+        tmp = scan["init_scan"]
+        scan.clear()
+        scan.update(tmp)
 
 
 def fit_scan(scan, model_dict, fit_from=None, fit_to=None):
@@ -115,11 +142,13 @@ def fit_scan(scan, model_dict, fit_from=None, fit_to=None):
         fit_to = np.inf
 
     y_fit = scan["counts"]
+    y_err = scan["counts_err"]
     x_fit = scan[scan["scan_motor"]]
 
     # apply fitting range
     fit_ind = (fit_from <= x_fit) & (x_fit <= fit_to)
     y_fit = y_fit[fit_ind]
+    y_err = y_err[fit_ind]
     x_fit = x_fit[fit_ind]
 
     model = None
@@ -167,7 +196,7 @@ def fit_scan(scan, model_dict, fit_from=None, fit_to=None):
         else:
             model += _model
 
-    weights = [1 / np.sqrt(val) if val != 0 else 1 for val in y_fit]
+    weights = [1 / y_err if y_err != 0 else 1 for y_err in y_err]
     scan["fit"] = model.fit(y_fit, x=x_fit, weights=weights)
 
 

pyzebra/h5.py

@@ -75,6 +75,7 @@ def read_detector_data(filepath, cami_meta=None):
         data = data.reshape(n, rows, cols)
 
         det_data = {"data": data}
+        det_data["original_filename"] = filepath
 
         if "/entry1/zebra_mode" in h5f:
             det_data["zebra_mode"] = h5f["/entry1/zebra_mode"][0].decode()

pyzebra/utils.py

@@ -0,0 +1,20 @@
+import os
+
+ZEBRA_PROPOSALS_PATHS = [
+    f"/afs/psi.ch/project/sinqdata/{year}/zebra/" for year in (2016, 2017, 2018, 2020, 2021)
+]
+
+def find_proposal_path(proposal):
+    proposal = proposal.strip()
+    if proposal:
+        for zebra_proposals_path in ZEBRA_PROPOSALS_PATHS:
+            proposal_path = os.path.join(zebra_proposals_path, proposal)
+            if os.path.isdir(proposal_path):
+                # found it
+                break
+        else:
+            raise ValueError(f"Can not find data for proposal '{proposal}'.")
+    else:
+        proposal_path = ""
+
+    return proposal_path