From 956c8ff043f04c2e15d8465062907de344572ca0 Mon Sep 17 00:00:00 2001 From: wakonig_k Date: Sat, 7 Nov 2026 11:26:38 +0100 Subject: [PATCH] feat(waveform): overlay successive scans --- .../settings/curve_settings/curve_tree.py | 12 +- .../widgets/plots/waveform/waveform.py | 387 +++++++++++++++--- tests/unit_tests/test_waveform.py | 203 +++++++++ 3 files changed, 542 insertions(+), 60 deletions(-) diff --git a/bec_widgets/widgets/plots/waveform/settings/curve_settings/curve_tree.py b/bec_widgets/widgets/plots/waveform/settings/curve_settings/curve_tree.py index a2ee23ff..c79f69ff 100644 --- a/bec_widgets/widgets/plots/waveform/settings/curve_settings/curve_tree.py +++ b/bec_widgets/widgets/plots/waveform/settings/curve_settings/curve_tree.py @@ -722,10 +722,12 @@ class CurveTree(BECWidget, QWidget): self.waveform.color_palette = self.color_palette data = self.export_all_curves() if self.waveform is not None: - custom_labels = {c.name() for c in self.waveform.curves if c.config.source == "custom"} + custom_labels = { + c.name() for c in self.waveform.configured_curves if c.config.source == "custom" + } data.extend( c.config.model_dump() - for c in self.waveform.curves + for c in self.waveform.configured_curves if c.config.source == "dap" and c.config.parent_label in custom_labels ) json_data = json.dumps(data, indent=2) @@ -772,8 +774,10 @@ class CurveTree(BECWidget, QWidget): self.tree.clear() self.all_items = [] - top_curves = [c for c in self.waveform.curves if c.config.source in ("device", "history")] - dap_curves = [c for c in self.waveform.curves if c.config.source == "dap"] + top_curves = [ + c for c in self.waveform.configured_curves if c.config.source in ("device", "history") + ] + dap_curves = [c for c in self.waveform.configured_curves if c.config.source == "dap"] for dev in top_curves: dr = CurveRow( self.tree, diff --git a/bec_widgets/widgets/plots/waveform/waveform.py b/bec_widgets/widgets/plots/waveform/waveform.py index 2a8459d2..432209c7 100644 --- a/bec_widgets/widgets/plots/waveform/waveform.py +++ b/bec_widgets/widgets/plots/waveform/waveform.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from dataclasses import dataclass from typing import TYPE_CHECKING, Literal import numpy as np @@ -63,6 +64,17 @@ else: lmfit = lazy_import("lmfit") +_OverlayAxis = Literal["index", "timestamp"] | tuple[str, str] + + +@dataclass +class _ScanOverlay: + auto_axis: _OverlayAxis + axis: _OverlayAxis | None + requested_axis: _OverlayAxis + is_async: bool + + # noinspection PyDataclass class WaveformConfig(ConnectionConfig): color_palette: str | None = Field( @@ -123,6 +135,7 @@ class Waveform(PlotBase): dap_summary_update = Signal(dict, dict) new_scan = Signal() new_scan_id = Signal(str) + _overlay_data_loaded = Signal(object, object, object) roi_changed = Signal(tuple) roi_active = Signal(bool) @@ -160,6 +173,8 @@ class Waveform(PlotBase): self.old_scan_id = None self.scan_id = None self.scan_item = None + self._accumulated_curves: dict[Curve, _ScanOverlay] = {} + self._curve_data_axes: dict[Curve, _OverlayAxis] = {} self.x_axis_mode = { "name": "auto", "entry": None, @@ -183,6 +198,8 @@ class Waveform(PlotBase): self._enable_roi_toolbar_action(False) # default state where are no dap curves self._init_curve_dialog() self._init_alignment_mode() + self._init_scan_accumulation() + self._overlay_data_loaded.connect(self._apply_overlay_data) self.curve_settings_dialog = None # Large‑dataset guard @@ -212,9 +229,210 @@ class Waveform(PlotBase): self._connect_viewbox_menu_actions() self.toolbar.show_bundles( - ["plot_export", "mouse_interaction", "roi", "alignment_mode", "axis_popup"] + [ + "plot_export", + "mouse_interaction", + "roi", + "alignment_mode", + "scan_accumulation", + "axis_popup", + ] ) + def _init_scan_accumulation(self) -> None: + action = MaterialIconAction( + icon_name="layers", tooltip="Accumulate Scans", checkable=True, parent=self + ) + self._accumulate_scans_action = action.action + self.toolbar.components.add_safe("accumulate_scans", action) + bundle = ToolbarBundle("scan_accumulation", self.toolbar.components) + bundle.add_action("accumulate_scans") + self.toolbar.add_bundle(bundle) + action.action.toggled.connect(self._on_accumulate_scans_toggled) + + @SafeSlot(bool) + def _on_accumulate_scans_toggled(self, checked: bool) -> None: + endpoint = MessageEndpoints.scan_history() + if checked: + self.bec_dispatcher.connect_slot(self._on_overlay_scan_saved, endpoint) + else: + self.bec_dispatcher.disconnect_slot(self._on_overlay_scan_saved, endpoint) + self._clear_accumulated_curves() + + def _clear_accumulated_curves(self) -> None: + for curve in tuple(self._accumulated_curves): + self.plot_item.removeItem(curve) + self._curve_clean_up(curve) + self._refresh_crosshair_markers() + + def _accumulate_current_scan(self) -> None: + if self.scan_id is None or self.scan_item is None: + return + if isinstance(self.scan_item, ScanDataContainer): + scan_number = self.scan_item.metadata.get("bec", {}).get("scan_number") + else: + status = getattr(self.scan_item, "status_message", None) + scan_number = getattr(status, "scan_number", None) + scan_label = self.scan_id if scan_number is None else scan_number + live_data = getattr(self.scan_item, "live_data", None) + auto_axis = self._scan_auto_axis() + for live in self.curves: + if live.config.source != "device" or not live.isVisible(): + continue + signal = live.config.signal + label = f"{signal.device}-{signal.signal}-scan-{scan_label}" + if self._check_curve_id(label): + continue + x, y = live.get_data() + has_data = x is not None and y is not None and len(y) > 0 + in_live_data = live_data is not None and signal.signal in live_data.get( + signal.device, {} + ) + if live_data is not None and live in self._sync_curves and not in_live_data: + continue + if not has_data and not in_live_data: + continue + is_async = live in self._async_curves + axis = auto_axis if self.x_mode == "auto" else self._overlay_axis(is_async) + snapshot_axis = self._curve_data_axes.get(live) + config = CurveConfig( + label=label, + widget_class="Curve", + parent_id=self.gui_id, + source="history", + signal=DeviceSignal(device=signal.device, signal=signal.signal), + scan_id=self.scan_id, + scan_number=scan_number, + current_x_mode=self.x_mode, + color=self._generate_color_from_palette(), + symbol=live.config.symbol, + symbol_size=live.config.symbol_size, + pen_width=live.config.pen_width, + pen_style=live.config.pen_style, + ) + # Seed the history curve from the displayed data. The scan file may + # still be writing when the next scan starts. + curve = self._add_curve_object(name=label, config=config) + if has_data: + curve.setData(x.copy(), y.copy()) + self._history_curves.append(curve) + self._accumulated_curves[curve] = _ScanOverlay( + auto_axis, snapshot_axis if has_data else None, axis, is_async + ) + self._load_overlay(curve, axis) + self._update_overlay_visibility() + + def _scan_auto_axis(self) -> _OverlayAxis: + if self._async_curves: + return "index" + try: + devices = self.scan_item.metadata["bec"]["scan_report_devices"] + except (AttributeError, KeyError, TypeError): + status = getattr(self.scan_item, "status_message", None) + devices = status.info.get("scan_report_devices") if status is not None else None + devices = self._ensure_str_list(devices or []) + if devices: + device = devices[0] + return device, self.entry_validator.validate_signal(device, None) + return "index" + + def _overlay_axis(self, is_async: bool = False) -> _OverlayAxis: + mode = self.x_mode + if is_async and mode == "timestamp": + return "index" + if mode == "auto": + return self._current_x_device or "index" + if mode in ("index", "timestamp"): + return mode + return mode, self.x_axis_mode["entry"] + + def _update_overlay_visibility(self) -> None: + if not self._accumulated_curves: + return + timestamp_axis = self.x_mode == "timestamp" and ( + "timestamp" in self._curve_data_axes.values() + or any(state.axis == "timestamp" for state in self._accumulated_curves.values()) + ) + axis = self._overlay_axis() + async_axis = axis if timestamp_axis else self._overlay_axis(is_async=True) + overlay_visibility = {} + for curve, state in self._accumulated_curves.items(): + visible = state.axis == (async_axis if state.is_async else axis) + curve.setVisible(visible) + overlay_visibility[curve.name()] = visible + for curve in self._dap_curves: + if curve.config.parent_label in overlay_visibility: + curve.setVisible(overlay_visibility[curve.config.parent_label]) + + @SafeSlot(dict, dict) + def _on_overlay_scan_saved(self, msg: dict, meta: dict) -> None: + for curve, state in self._accumulated_curves.items(): + if curve.config.scan_id == msg["scan_id"]: + self._load_overlay(curve, state.requested_axis, msg["file_path"]) + + def _load_overlay(self, curve: Curve, axis: _OverlayAxis, file_path: str | None = None) -> None: + scan_id = curve.config.scan_id + signal = curve.config.signal + history = self.client.history + size_limit = None if self.skip_large_dataset_check else self.config.max_dataset_size_mb + dataset_size_mb = self._dataset_size_mb + emit = self._overlay_data_loaded.emit + + def load(): + if file_path is not None: + scan = ScanDataContainer(file_path=file_path) + else: + scan = history.get_by_scan_id(scan_id) if history is not None else None + if scan is None: + return + + def get_entry(device_name, entry_name): + device = scan.devices.get(device_name) + if device is None: + return None + if size_limit is not None and dataset_size_mb(device, entry_name) > size_limit: + # Automatic refresh keeps the snapshot if loading needs user approval. + return None + return device.get(entry_name) + + entry = get_entry(signal.device, signal.signal) + if entry is None: + return + data = entry.read() + y = np.asarray(data["value"]) + if y.ndim > 1: + y = y[-1] + if not y.size: + return + if axis == "index": + x = np.arange(y.size) + elif axis == "timestamp": + x = np.asarray(data["timestamp"]) + else: + device_x, signal_x = axis + x_entry = get_entry(device_x, signal_x) + if x_entry is None: + return + x = np.asarray(x_entry.read()["value"]) + if x.shape == y.shape: + try: + emit(curve, axis, (x, y)) + except RuntimeError: + # The widget can be deleted while a file is being read. + return + + self.submit_task(load) + + @SafeSlot(object, object, object) + def _apply_overlay_data(self, curve: Curve, axis: _OverlayAxis, data: tuple) -> None: + state = self._accumulated_curves.get(curve) + if state is None or state.requested_axis != axis or curve.config.source != "history": + return + curve.set_data(*data) + curve.config.current_x_mode = self.x_mode + state.axis = axis + self._update_overlay_visibility() + def _connect_viewbox_menu_actions(self): """Connect the viewbox menu action ViewAll to the custom reset_view method.""" menu = self.plot_item.vb.menu @@ -777,13 +995,23 @@ class Waveform(PlotBase): for i, curve in enumerate(self.curves): curve.set_color(colors[i]) + @property + def configured_curves(self) -> list[Curve]: + """Curves managed by Curve Settings, excluding automatic scan overlays and their fits.""" + labels = {curve.name() for curve in self._accumulated_curves} + return [ + curve + for curve in self.curves + if curve not in self._accumulated_curves and curve.config.parent_label not in labels + ] + @SafeProperty(str, designable=False, popup_error=True) def curve_json(self) -> str: """ - A JSON string property that serializes all curves' pydantic configs. + Serialize the configured device, history and fit curves. """ raw_list = [] - for c in self.curves: + for c in self.configured_curves: if c.config.source == "custom": # Do not serialize custom curves continue cfg_dict = c.config.model_dump() @@ -795,21 +1023,24 @@ class Waveform(PlotBase): """ Load curves from a JSON string and add them to the plot, omitting custom source curves. - The serialized curves replace all device, history and DAP curves. Custom curves hold - data that cannot be serialized, so the existing ones are kept unchanged (configuration - and data) and custom entries in the JSON are not loaded. The JSON is rejected before - any curve is removed if one of its curves is invalid, refers to a scan that is not in - the history, or would get a label that a kept custom curve or another entry already uses. + The serialized curves replace configured device, history and DAP curves. Custom curves + and automatic scan overlays stay on the plot. Reject the JSON before removing any curves + if an entry is invalid, refers to missing scan history, or uses a label that is taken. """ try: curve_configs = json.loads(json_data) - custom_labels = {c.name() for c in self.curves if c.config.source == "custom"} + configured = self.configured_curves + kept_labels = { + curve.name() + for curve in self.curves + if curve not in configured or curve.config.source == "custom" + } # Build and check every serialized curve before anything is removed, so a curve # that cannot be added does not leave the plot half-loaded configs = [] for cfg_dict in curve_configs: if cfg_dict.get("source") == "custom": - if cfg_dict.get("label") not in custom_labels: + if cfg_dict.get("label") not in kept_labels: logger.warning(f"Custom source curve '{cfg_dict.get('label')}' not loaded.") continue config = CurveConfig(**cfg_dict) @@ -817,24 +1048,27 @@ class Waveform(PlotBase): self._resolve_history_config(config) configs.append(config) labels = [config.label for config in configs if config.label] - taken = custom_labels & set(labels) + taken = kept_labels & set(labels) if taken: - raise ValueError(f"Labels {sorted(taken)} are already used by custom curves.") + raise ValueError( + f"Labels {sorted(taken)} are already used by curves kept on the plot." + ) duplicates = {label for label in labels if labels.count(label) > 1} if duplicates: raise ValueError(f"Labels {sorted(duplicates)} are used by more than one curve.") - # Like clear_all, but keep custom curves and their colors (remove_curve recolors) - self._dap_curves = [] - self._history_curves = [] + # Keep automatic overlays and their fits registered while replacing configured curves. + self._dap_curves = [curve for curve in self._dap_curves if curve not in configured] + self._history_curves = list(self._accumulated_curves) self._sync_curves = [] self._async_curves = [] - for curve in self.curves: - if curve.name() not in custom_labels: + for curve in configured: + if curve.config.source != "custom": self._remove_curve_by_name(curve.name()) if self.crosshair is not None: self.crosshair.clear_markers() for config in configs: self._add_curve(config=config) + self._update_overlay_visibility() self._refresh_alignment_state(force_readback=self._alignment_panel_visible) self._refresh_dap_signals() except json.JSONDecodeError as e: @@ -1178,6 +1412,7 @@ class Waveform(PlotBase): # Actually create the Curve item curve = self._add_curve_object(name=label, config=config, scan_item=scan_item) + self._categorise_device_curves() # If custom => set initial data if config.source == "custom" and x_data is not None and y_data is not None: @@ -1191,11 +1426,13 @@ class Waveform(PlotBase): self.sync_signal_update.emit() if config.source == "dap": self._dap_curves.append(curve) + self._update_overlay_visibility() self.setup_dap_for_scan() self.roi_enable.emit(True) # Enable the ROI toolbar action self.request_dap() # Request DAP update directly without blocking proxy if config.source == "history": self._history_curves.append(curve) + self._update_overlay_visibility() # Autorange with a delay to ensure the plot is updated; skipped if the widget is closed. self._call_later(150, self.auto_range) @@ -1248,7 +1485,6 @@ class Waveform(PlotBase): self.plot_item.addItem(curve) if scan_item is not None: self._fetch_history_data_for_curve(curve, scan_item) - self._categorise_device_curves() curve.visibleChanged.connect(self._refresh_crosshair_markers) curve.visibleChanged.connect(self.auto_range) return curve @@ -1387,6 +1623,7 @@ class Waveform(PlotBase): curve.setVisible(False) return curve.set_data(x=x_data, y=y_data) + self._curve_data_axes[curve] = self._overlay_axis() return curve def _refresh_history_curves(self): @@ -1398,6 +1635,15 @@ class Waveform(PlotBase): """ self._history_curves = [c for c in self._history_curves if c.config.source == "history"] for curve in list(self._history_curves): + state = self._accumulated_curves.get(curve) + if state is not None: + axis = ( + state.auto_axis if self.x_mode == "auto" else self._overlay_axis(state.is_async) + ) + if state.requested_axis != axis: + state.requested_axis = axis + self._load_overlay(curve, axis) + continue scan_item = self.get_history_scan_item( scan_id=curve.config.scan_id, scan_index=curve.config.scan_number ) @@ -1405,6 +1651,7 @@ class Waveform(PlotBase): self._fetch_history_data_for_curve(curve, scan_item) else: logger.warning(f"Scan item for curve {curve.name()} not found.") + self._update_overlay_visibility() def _refresh_crosshair_markers(self): """ @@ -1442,6 +1689,8 @@ class Waveform(PlotBase): for c in self.curves: if c.config.source != "history": c.clear_data() + self._curve_data_axes.pop(c, None) + self._update_overlay_visibility() # X-axis compatibility helpers def _is_curve_compatible(self, curve: Curve) -> bool: @@ -1475,7 +1724,10 @@ class Waveform(PlotBase): def _update_curve_visibility(self) -> None: """Show or hide curves according to `_is_curve_compatible`.""" for c in self.curves: - c.setVisible(self._is_curve_compatible(c)) + # Accumulated history already has its X data and only reloads on axis changes. + if c not in self._accumulated_curves: + c.setVisible(self._is_curve_compatible(c)) + self._update_overlay_visibility() def clear_all(self): """ @@ -1526,6 +1778,7 @@ class Waveform(PlotBase): self._refresh_colors() self._categorise_device_curves() + self._update_overlay_visibility() self._refresh_alignment_state() def _remove_curve_by_name(self, name: str): @@ -1571,6 +1824,8 @@ class Waveform(PlotBase): curve.rpc_register.remove_rpc(curve) # Stop tracking the removed curve, so history refreshes and DAP requests skip it + self._accumulated_curves.pop(curve, None) + self._curve_data_axes.pop(curve, None) if curve in self._history_curves: self._history_curves.remove(curve) if curve in self._dap_curves: @@ -1643,6 +1898,16 @@ class Waveform(PlotBase): return if current_scan_id != self.scan_id: + if self._accumulate_scans_action.isChecked(): + self._accumulate_current_scan() + device_curves = [c for c in self.curves if c.config.source == "device"] + device_labels = {c.name() for c in device_curves} + for curve in device_curves + [ + c for c in self._dap_curves if c.config.parent_label in device_labels + ]: + curve.clear_data() + self._curve_data_axes.pop(curve, None) + self._update_overlay_visibility() self.reset() self.new_scan.emit() self.new_scan_id.emit(current_scan_id) @@ -1722,6 +1987,8 @@ class Waveform(PlotBase): logger.info("No scan executed so far; skipping device curves categorisation.") return data, access_key = self._fetch_scan_data_and_access() + timestamp_mode = self.x_mode == "timestamp" + had_timestamps = timestamp_mode and "timestamp" in self._curve_data_axes.values() for curve in self._sync_curves: device_name = curve.config.signal.device device_entry = curve.config.signal.signal @@ -1737,8 +2004,12 @@ class Waveform(PlotBase): return if device_data is not None and x_data is not None: curve.setData(x_data, device_data) + self._curve_data_axes[curve] = self._overlay_axis() if device_data is not None and x_data is None: curve.setData(device_data) + self._curve_data_axes[curve] = "index" + if timestamp_mode and had_timestamps != ("timestamp" in self._curve_data_axes.values()): + self._update_overlay_visibility() self.request_dap_update.emit() def update_async_curves(self): @@ -1756,6 +2027,8 @@ class Waveform(PlotBase): """ data, access_key = self._fetch_scan_data_and_access() + if data is None: + return for curve in self._async_curves: device_name = curve.config.signal.device @@ -1779,21 +2052,24 @@ class Waveform(PlotBase): continue # Async curves only support plotting vs index or other device - if self.x_axis_mode["name"] in ["timestamp", "index", "auto"]: + axis = "index" if self.x_mode == "auto" else self._overlay_axis(is_async=True) + if axis == "index": device_data_x = np.linspace(0, len(device_data) - 1, len(device_data)) else: # Fetch data from signal instead device_data_x = self._get_x_data(device_name, device_entry) # Fallback to 'index' in case data is not of equal length - if len(device_data_x) != len(device_data): + if device_data_x is None or len(device_data_x) != len(device_data): logger.warning( f"Async data for curve {curve.name()} and x_axis {device_entry} is not of equal length. Falling back to 'index' plotting." ) device_data_x = np.linspace(0, len(device_data) - 1, len(device_data)) + axis = "index" self._auto_adjust_async_curve_settings(curve, len(device_data)) curve.setData(device_data_x, device_data) + self._curve_data_axes[curve] = axis self.request_dap_update.emit() @@ -1832,6 +2108,7 @@ class Waveform(PlotBase): signal = curve.config.signal.signal async_signal_found, stream = self._check_async_signal_found(name, signal) + self._curve_data_axes.pop(curve, None) try: curve.clear_data() except KeyError: @@ -1957,12 +2234,14 @@ class Waveform(PlotBase): data_plot_x = np.linspace(0, len(data_plot_y) - 1, len(data_plot_y)) self._auto_adjust_async_curve_settings(curve, len(data_plot_y)) curve.setData(data_plot_x, data_plot_y) + self._curve_data_axes[curve] = "index" # Move on in the loop continue # x_axis_mode is device signal # Only consider device signals that are async for now, fallback is index signal_x = self.x_axis_mode["entry"] + axis = (plot_mode, signal_x) async_data = msg["signals"].get(signal_x, None) # Make sure the signal exists, otherwise fall back to index if async_data is None: @@ -1970,18 +2249,25 @@ class Waveform(PlotBase): data_plot_x = self._get_x_data(plot_mode, signal_x) else: data_plot_x = np.asarray(async_data["value"]) - if x_data is not None: + if x_data is not None and data_plot_x is not None: data_plot_x = np.hstack((x_data, data_plot_x)) - # Fallback incase data is not of equal length - if len(data_plot_x) != len(data_plot_y): + # Appended samples need the same X coordinates as the existing samples. + if ( + data_plot_x is None + or len(data_plot_x) != len(data_plot_y) + or (x_data is not None and self._curve_data_axes.get(curve) != axis) + ): logger.warning( - f"Async data for curve {curve.name()} and x_axis {signal_x} is not of equal length. Falling back to 'index' plotting." + f"Async data for curve {curve.name()} is incompatible with X signal " + f"{signal_x}. Falling back to index plotting." ) data_plot_x = np.linspace(0, len(data_plot_y) - 1, len(data_plot_y)) + axis = "index" # Plot the data self._auto_adjust_async_curve_settings(curve, len(data_plot_y)) curve.setData(data_plot_x, data_plot_y) + self._curve_data_axes[curve] = axis self.request_dap_update.emit() @@ -2338,35 +2624,20 @@ class Waveform(PlotBase): # 4 Best effort automatic mode if self.x_axis_mode["name"] is None or self.x_axis_mode["name"] == "auto": - # 4.1 If there are async curves, use index - if len(self._async_curves) > 0: + auto_axis = self._scan_auto_axis() + if auto_axis == "index": x_data = None new_suffix = " (auto: index)" self._current_x_device = None - # 4.2 If there are sync curves, use the first device from the scan report else: - try: - scan_report_devices = self._ensure_str_list( - self.scan_item.metadata["bec"]["scan_report_devices"] - ) - except Exception: - scan_report_devices = self.scan_item.status_message.info.get( - "scan_report_devices", [] - ) - if not scan_report_devices: - x_data = None - new_suffix = " (auto: index)" - self._current_x_device = None + device_x, signal_x = auto_axis + if access_key == "val": + x_data = data.get(device_x, {}).get(signal_x, {}).get(access_key, None) else: - device_x = scan_report_devices[0] - signal_x = self.entry_validator.validate_signal(device_x, None) - if access_key == "val": - x_data = data.get(device_x, {}).get(signal_x, {}).get(access_key, None) - else: - entry_obj = data.get(device_x, {}).get(signal_x) - x_data = entry_obj.read()["value"] if entry_obj else None - new_suffix = f" (auto: {device_x}-{signal_x})" - self._current_x_device = (device_x, signal_x) + entry_obj = data.get(device_x, {}).get(signal_x) + x_data = entry_obj.read()["value"] if entry_obj else None + new_suffix = f" (auto: {device_x}-{signal_x})" + self._current_x_device = auto_axis self._update_x_label_suffix(new_suffix) if previous_x_device != self._current_x_device: self._refresh_alignment_state(force_readback=True) @@ -2384,6 +2655,7 @@ class Waveform(PlotBase): self.x_axis_mode["label_suffix"] = new_suffix self.set_x_label_suffix(new_suffix) + self._update_overlay_visibility() def _switch_x_axis_item(self, mode: str): """ @@ -2570,6 +2842,15 @@ class Waveform(PlotBase): ################################################################################ # Large dataset handling helpers + @staticmethod + def _dataset_size_mb(dataset_obj, device_entry: str) -> float: + info = dataset_obj._info + mem_bytes = info.get(device_entry, {}).get("value", {}).get("mem_size", 0) + if mem_bytes == 0 and info: + first_key = next(iter(info)) + mem_bytes = info[first_key]["value"]["mem_size"] + return mem_bytes / (1024 * 1024) + def _check_dataset_size_and_confirm(self, dataset_obj, device_entry: str) -> bool: """ Check the size of the dataset and confirm with the user if it exceeds the limit. @@ -2583,13 +2864,7 @@ class Waveform(PlotBase): False if the dataset exceeds the size limit and user declined to load it. """ try: - info = dataset_obj._info - mem_bytes = info.get(device_entry, {}).get("value", {}).get("mem_size", 0) - # Fallback – grab first entry if lookup failed - if mem_bytes == 0 and info: - first_key = next(iter(info)) - mem_bytes = info[first_key]["value"]["mem_size"] - size_mb = mem_bytes / (1024 * 1024) + size_mb = self._dataset_size_mb(dataset_obj, device_entry) logger.info(f"Dataset size: {size_mb:.1f} MB") except Exception as exc: # noqa: BLE001 logger.error(f"Unable to evaluate dataset size: {exc}") diff --git a/tests/unit_tests/test_waveform.py b/tests/unit_tests/test_waveform.py index a17cf885..f00ab755 100644 --- a/tests/unit_tests/test_waveform.py +++ b/tests/unit_tests/test_waveform.py @@ -25,6 +25,7 @@ from bec_widgets.tests.client_mocks import ( ) from bec_widgets.widgets.plots.plot_base import UIMode from bec_widgets.widgets.plots.waveform.curve import DeviceSignal +from bec_widgets.widgets.plots.waveform.settings.curve_settings.curve_tree import CurveTree from bec_widgets.widgets.plots.waveform.waveform import Waveform from bec_widgets.widgets.services.scan_history_browser.scan_history_browser import ( ScanHistoryBrowser, @@ -592,6 +593,208 @@ def test_on_scan_status(qtbot, mocked_client, monkeypatch, mode, calls): assert async_spy.call_count == calls[1], "async_signal_update should be called exactly once" +def test_accumulate_scans_overlays_previous_device_data( + qtbot, mocked_client, monkeypatch, scan_history_factory +): + wf = create_widget(qtbot, Waveform, client=mocked_client) + inject_scan_history(wf, scan_history_factory, ("saved", 10)) + wf.x_mode = "index" + saved = wf.plot("bpm4i", scan_id="saved") + live = wf.plot("bpm4i") + static = wf.plot(x=[0, 1], y=[0, 1], label="reference") + action = wf.toolbar.components.get_action("accumulate_scans").action + scans = {str(i): create_dummy_scan_item(scan_id=str(i), scan_number=i) for i in range(1, 6)} + monkeypatch.setattr(wf.queue.scan_storage, "find_scan_by_ID", scans.get) + + def start_scan(number): + wf.on_scan_status({"scan_id": str(number)}, {}) + + def overlays(): + return [ + curve + for curve in wf.curves + if curve.config.source == "history" and curve.name() != saved.name() + ] + + assert not action.isChecked() + start_scan(1) + live.setData([0, 1], [1, 2]) + start_scan(2) + assert overlays() == [] + + action.trigger() + assert action.isChecked() + live.setData([0, 1], [2, 3]) + start_scan(3) + start_scan(3) + live.setData([0, 1], [3, 4]) + start_scan(4) + live.setData([0, 1], [4, 5]) + + first, second = overlays() + assert [first.config.scan_id, second.config.scan_id] == ["2", "3"] + assert [first.name(), second.name()] == [f"bpm4i-bpm4i-scan-{i}" for i in (2, 3)] + np.testing.assert_array_equal(first.get_data()[1], [2, 3]) + np.testing.assert_array_equal(second.get_data()[1], [3, 4]) + + # Applying the curve configuration before these scans are saved keeps their snapshots. + tree = create_widget(qtbot, CurveTree, waveform=wf) + tree.refresh_from_waveform() + assert tree.tree.topLevelItemCount() == 2 + tree.send_curve_json() + assert overlays() == [first, second] + live = wf.get_curve("bpm4i-bpm4i") + live.setData([0, 1], [4, 5]) + + action.trigger() + assert overlays() == [] + np.testing.assert_array_equal(live.get_data()[1], [4, 5]) + np.testing.assert_array_equal(static.get_data()[1], [0, 1]) + start_scan(5) + assert overlays() == [] + assert wf.get_curve(saved.name()) is not None + + +def test_accumulated_scan_uses_saved_points_and_hides_on_incompatible_auto_axis( + qtbot, mocked_client, monkeypatch, scan_history_factory +): + wf = create_widget(qtbot, Waveform, client=mocked_client) + live = wf.plot("bpm4i") + scans = {str(i): create_dummy_scan_item(scan_id=str(i), scan_number=i) for i in (1, 2, 3)} + for i, motor in ((1, "samx"), (2, "samy"), (3, "samx")): + scans[str(i)].metadata["bec"]["scan_report_devices"] = [motor] + scans[str(i)].status_message.info["scan_report_devices"] = [motor] + monkeypatch.setattr(wf.queue.scan_storage, "find_scan_by_ID", scans.get) + + wf.toolbar.components.get_action("accumulate_scans").action.trigger() + wf.on_scan_status({"scan_id": "1"}, {}) + wf.update_sync_curves() + # The next scan starts before the previous scan's final point is rendered. + live.setData([10, 20], [5, 6]) + wf.on_scan_status({"scan_id": "2"}, {}) + wf.update_sync_curves() + overlay = wf.get_curve("bpm4i-bpm4i-scan-1") + assert not overlay.isVisible() + assert len(overlay.get_data()[1]) == 2 + + saved = scan_history_factory(scan_id="1", scan_number=1, num_points=3) + wf._on_overlay_scan_saved(saved.content, {}) + qtbot.waitUntil(lambda: len(overlay.get_data()[1]) == 3) + assert not overlay.isVisible() + + wf.on_scan_status({"scan_id": "3"}, {}) + wf.update_sync_curves() + assert overlay.isVisible() + assert not wf.get_curve("bpm4i-bpm4i-scan-2").isVisible() + + +@pytest.mark.parametrize("x_mode", ["samx", "timestamp"]) +@pytest.mark.parametrize("streaming", [False, True]) +def test_accumulated_async_curve_uses_actual_x_axis( + qtbot, mocked_client, monkeypatch, scan_history_factory, x_mode, streaming +): + wf = create_widget(qtbot, Waveform, client=mocked_client) + wf.x_mode = x_mode + live = wf.plot("async_device") + scans = {str(i): create_dummy_scan_item(scan_id=str(i), scan_number=i) for i in (1, 2)} + values = [5, 6, 7, 8] + scans["1"].live_data["async_device"]["async_device"] = DummyData( + val=values, timestamps=[11, 21, 31, 41] + ) + monkeypatch.setattr(wf.queue.scan_storage, "find_scan_by_ID", scans.get) + wf.toolbar.components.get_action("accumulate_scans").action.trigger() + wf.on_scan_status({"scan_id": "1"}, {}) + + if streaming: + wf.on_async_readback( + {"signals": {"async_device": {"value": values}}}, + {"cb_info": {"scan_id": "1"}, "async_update": {"type": "replace"}}, + ) + else: + wf.update_async_curves() + np.testing.assert_array_equal(live.get_data()[0], np.arange(len(values))) + + wf.on_scan_status({"scan_id": "2"}, {}) + overlay = wf.get_curve("async_device-async_device-scan-1") + np.testing.assert_array_equal(overlay.get_data()[0], np.arange(len(values))) + np.testing.assert_array_equal(overlay.get_data()[1], values) + assert overlay.isVisible() == (x_mode == "timestamp") + + wf.x_mode = "index" + assert overlay.isVisible() + wf.x_mode = "timestamp" + assert overlay.isVisible() + + saved = scan_history_factory(scan_id="1", scan_number=1, num_points=3) + wf._on_overlay_scan_saved(saved.content, {}) + qtbot.waitUntil(lambda: len(overlay.get_data()[1]) == 30) + np.testing.assert_array_equal(overlay.get_data()[0], np.arange(30)) + assert overlay.isVisible() + + +@pytest.mark.parametrize("switch_to", [None, "samx", "auto"]) +def test_accumulated_sync_indices_are_hidden_on_motor_axis( + qtbot, mocked_client, monkeypatch, scan_history_factory, switch_to +): + wf = create_widget(qtbot, Waveform, client=mocked_client) + wf.x_mode = "auto" if switch_to is None else "index" + live = wf.plot("bpm4i") + scans = {str(i): create_dummy_scan_item(scan_id=str(i), scan_number=i) for i in (1, 2)} + if switch_to is None: + del scans["1"].live_data["samx"] + monkeypatch.setattr(wf.queue.scan_storage, "find_scan_by_ID", scans.get) + wf.toolbar.components.get_action("accumulate_scans").action.trigger() + wf.on_scan_status({"scan_id": "1"}, {}) + wf.update_sync_curves() + np.testing.assert_array_equal(live.get_data()[0], [0, 1, 2]) + + if switch_to is not None: + # The next scan arrives before the rate-limited refresh for the new X mode. + wf.x_mode = switch_to + wf.on_scan_status({"scan_id": "2"}, {}) + wf.update_sync_curves() + overlay = wf.get_curve("bpm4i-bpm4i-scan-1") + np.testing.assert_array_equal(overlay.get_data()[0], [0, 1, 2]) + assert not overlay.isVisible() + + saved = scan_history_factory(scan_id="1", scan_number=1, num_points=3) + wf._on_overlay_scan_saved(saved.content, {}) + qtbot.waitUntil(overlay.isVisible) + np.testing.assert_array_equal(overlay.get_data()[0], [-5, 0, 5]) + + +def test_mixed_timestamp_overlays_hide_index_fallback(qtbot, mocked_client_with_dap, monkeypatch): + wf = create_widget(qtbot, Waveform, client=mocked_client_with_dap) + wf.x_mode = "timestamp" + wf.plot("bpm4i") + wf.plot("async_device") + scans = {str(i): create_dummy_scan_item(scan_id=str(i), scan_number=i) for i in (1, 2)} + monkeypatch.setattr(wf.queue.scan_storage, "find_scan_by_ID", scans.get) + wf.toolbar.components.get_action("accumulate_scans").action.trigger() + wf.on_scan_status({"scan_id": "1"}, {}) + wf.update_sync_curves() + wf.update_async_curves() + wf.on_scan_status({"scan_id": "2"}, {}) + wf.update_sync_curves() + + sync_overlay = wf.get_curve("bpm4i-bpm4i-scan-1") + async_overlay = wf.get_curve("async_device-async_device-scan-1") + np.testing.assert_array_equal(sync_overlay.get_data()[0], [101, 201, 301]) + np.testing.assert_array_equal(async_overlay.get_data()[0], [0, 1, 2]) + assert sync_overlay.isVisible() + assert not async_overlay.isVisible() + fit = wf.add_dap_curve(async_overlay.name(), "GaussianModel") + fit.setData([0, 1, 2], [1, 2, 3]) + assert not fit.isVisible() + + wf.remove_curve(sync_overlay.name()) + assert not async_overlay.isVisible() + assert not fit.isVisible() + wf.remove_curve("bpm4i-bpm4i") + assert async_overlay.isVisible() + assert fit.isVisible() + + def test_on_scan_status_ignored_without_device_curves(qtbot, mocked_client_with_dap, monkeypatch): """ A widget with only custom/dap curves (no live scan to follow) must not have its