diff --git a/bec_widgets/examples/jpeg_preview/README.md b/bec_widgets/examples/jpeg_preview/README.md index b35ee46f..9d9ccfaa 100644 --- a/bec_widgets/examples/jpeg_preview/README.md +++ b/bec_widgets/examples/jpeg_preview/README.md @@ -8,12 +8,19 @@ Shows detector preview frames that arrive JPEG-compressed, next to the same fram live: `dev.jpeg_cam.configure_stream(width=4096, height=4096, encoding="jpeg_gray")`. - The Image widget decodes JPEG previews on the thread that receives Redis messages (`BECDispatcher.connect_slot(..., preprocess=decode_jpeg_preview)`) and keeps only the newest - frame when it falls behind (`latest_only=True`). Everything else in the widget is unchanged and - works on the decoded 0–255 values. + frame when it falls behind (`latest_only=True`). JPEG frames take a fast 8-bit path + (`Image.fast_8bit_preview`, on by default): no processed copy, no per-frame statistics, a fixed + 0–255 range under autorange, and the frame laid out column-first off the GUI thread so + pyqtgraph draws it without another copy. Crosshair, ROIs and the colour bar work on the decoded + 0–255 values. Rotation, transpose, log or FFT fall back to the normal path. +- Frames may carry `"sensor_shape": [H, W]`: a preview scaled down from a bigger sensor is then + drawn in sensor pixels, so coordinates and ROIs stay put when the preview resolution changes. - `jpeg_view.JpegView` is the picture-only path: Qt decodes the JPEG into a QImage off the GUI - thread and the view draws it as a pixmap. No levels, histogram, ROI or pixel values. -- `demo.py` puts both views side by side with live camera controls, a statistics table and a - benchmark sweep (sizes x encodings x views, saved as CSV and JSON). + thread and the view draws it as a pixmap on a pyqtgraph plot in sensor pixels, with a crosshair + and rectangle/circle ROIs. Coordinates only: no levels, histogram or pixel values. +- `demo.py` puts both views side by side with live camera controls, a normal/fast switch for the + Image widget, a statistics table and a benchmark sweep (sizes x encodings x views, saved as CSV + and JSON). - `test_offline_benchmark.py` measures the same path without Redis. Install `simplejpeg` (`pip install -e .[jpeg]`) for the fast decoder; without it the widgets fall diff --git a/bec_widgets/examples/jpeg_preview/demo.py b/bec_widgets/examples/jpeg_preview/demo.py index 3f00238b..e53650d8 100644 --- a/bec_widgets/examples/jpeg_preview/demo.py +++ b/bec_widgets/examples/jpeg_preview/demo.py @@ -215,7 +215,7 @@ class CameraControl(QWidget): layout.setContentsMargins(4, 4, 4, 4) for label, widget in ( (None, self.live_button), - ("Size", self.resolution_box), + ("Preview size", self.resolution_box), (None, self.width_spin), ("x", self.height_spin), ("Encoding", self.encoding_box), @@ -346,6 +346,14 @@ class JpegPreviewDemo(QMainWindow): for check in (self.image_check, self.view_check): check.setChecked(True) self.image_check.toggled.connect(lambda on: self.enable_view("Image widget", on)) + self.fast_check = QCheckBox("Fast 8-bit path", self) + self.fast_check.setToolTip( + "On: JPEG frames are drawn as 8-bit images with a fixed 0-255 range and no per-frame" + " statistics.\nOff: the normal Image widget path (statistics and mean +/- 2 std" + " autorange on every frame)." + ) + self.fast_check.setChecked(self.image.fast_8bit_preview) + self.fast_check.toggled.connect(self.set_fast_path) self.view_check.toggled.connect(lambda on: self.enable_view("Picture view", on)) left, right = QWidget(self), QWidget(self) @@ -355,7 +363,12 @@ class JpegPreviewDemo(QMainWindow): ): column = QVBoxLayout(holder) column.setContentsMargins(0, 0, 0, 0) - column.addWidget(check) + header = QHBoxLayout() + header.addWidget(check) + if view is self.image: + header.addWidget(self.fast_check) + header.addStretch(1) + column.addLayout(header) column.addWidget(view) splitter = QSplitter(Qt.Orientation.Horizontal, self) splitter.addWidget(left) @@ -412,6 +425,14 @@ class JpegPreviewDemo(QMainWindow): self.jpeg_view.bec_dispatcher.disconnect_slot(self.jpeg_view.on_frame) self.trackers[name].clear() + @SafeSlot(bool) + def set_fast_path(self, enabled: bool) -> None: + """Switch the Image widget between its normal and its fast 8-bit path.""" + self.fast_check.setChecked(enabled) + self.image.fast_8bit_preview = enabled + self.image.autorange = True + self.trackers["Image widget"].clear() + @SafeSlot() def _on_view_frame(self): md = self.jpeg_view.last_frame_metadata @@ -475,6 +496,14 @@ class JpegPreviewDemo(QMainWindow): self.benchmark.start() +# view name in the results -> (panel, Image widget fast 8-bit path) +BENCHMARK_VIEWS = { + "Image widget": ("Image widget", False), + "Image widget (fast)": ("Image widget", True), + "Picture view": ("Picture view", False), +} + + class BenchmarkRunner(QObject): """ Steps through every size, encoding and view without blocking the GUI. @@ -488,7 +517,7 @@ class BenchmarkRunner(QObject): ): super().__init__(demo) self.demo = demo - self.steps = list(itertools.product(sizes, encodings, ("Image widget", "Picture view"))) + self.steps = list(itertools.product(sizes, encodings, BENCHMARK_VIEWS)) self.frame_rate = frame_rate self.settle_s = settle_s self.measure_s = measure_s @@ -500,6 +529,7 @@ class BenchmarkRunner(QObject): self._timer = QTimer(self) self._timer.setSingleShot(True) self._previous_views = (demo.image_check.isChecked(), demo.view_check.isChecked()) + self._previous_fast = demo.fast_check.isChecked() def start(self): """Run the first step.""" @@ -515,8 +545,10 @@ class BenchmarkRunner(QObject): self.demo.benchmark_label.setText( f"Benchmark {self._index + 1}/{len(self.steps)}: {view}, {size} x {size}, {encoding}" ) - self.demo.enable_view("Image widget", view == "Image widget") - self.demo.enable_view("Picture view", view == "Picture view") + panel, fast = BENCHMARK_VIEWS[view] + self.demo.set_fast_path(fast) + self.demo.enable_view("Image widget", panel == "Image widget") + self.demo.enable_view("Picture view", panel == "Picture view") self.demo.control.set_settings( width=size, height=size, encoding=encoding, frame_rate=self.frame_rate ) @@ -533,7 +565,8 @@ class BenchmarkRunner(QObject): def _record(self): self._timer.timeout.disconnect(self._record) size, encoding, view = self.steps[self._index] - summary = self.demo.summaries(since=self._measure_start).get(view, {}) + panel = BENCHMARK_VIEWS[view][0] + summary = self.demo.summaries(since=self._measure_start).get(panel, {}) row = {"view": view, "size": f"{size}x{size}", "encoding": encoding} row.update({key: round(summary.get(key, float("nan")), 2) for key in STAT_COLUMNS}) self.rows.append(row) @@ -551,6 +584,7 @@ class BenchmarkRunner(QObject): csv_path.with_suffix(".json").write_text(json.dumps(self.rows, indent=1)) self.demo.benchmark_label.setText(f"Benchmark saved to {csv_path}") self.demo.benchmark_button.setEnabled(True) + self.demo.set_fast_path(self._previous_fast) self.demo.enable_view("Image widget", self._previous_views[0]) self.demo.enable_view("Picture view", self._previous_views[1]) print(f"Benchmark saved to {csv_path}") diff --git a/bec_widgets/examples/jpeg_preview/jpeg_view.py b/bec_widgets/examples/jpeg_preview/jpeg_view.py index 3ca3e332..b5554f64 100644 --- a/bec_widgets/examples/jpeg_preview/jpeg_view.py +++ b/bec_widgets/examples/jpeg_preview/jpeg_view.py @@ -1,26 +1,35 @@ """ -Picture-only live view for JPEG preview frames. +Picture-only live view for JPEG preview frames, with crosshair and ROIs in sensor pixels. The frame is decoded by Qt into a QImage on the thread that receives Redis messages and drawn -as a pixmap. There is no numpy data on the GUI side, so there are no levels, histogram, ROI or -pixel values; a grayscale frame can still be coloured through an 8-bit colour table. +as a pixmap. There is no numpy data on the GUI side, so there are no levels, histogram or pixel +values; a grayscale frame can still be coloured through an 8-bit colour table. + +The pixmap sits in a pyqtgraph plot, scaled onto the sensor the preview was taken from +(``"sensor_shape"`` in the frame metadata, otherwise the frame's own size). Axes, crosshair and +ROIs are therefore in sensor pixels and stay put when the preview resolution changes. As in the +Image widget, array rows run along x and columns along y. """ from __future__ import annotations import time +import pyqtgraph as pg from bec_lib.endpoints import MessageEndpoints -from qtpy.QtCore import QRectF, Qt, Signal -from qtpy.QtGui import QColor, QImage, QPainter, QPixmap +from qtpy.QtCore import QPointF, QRectF, Qt, Signal +from qtpy.QtGui import QColor, QImage, QPixmap, QTransform from qtpy.QtWidgets import ( + QAbstractItemView, QComboBox, QGraphicsPixmapItem, - QGraphicsScene, - QGraphicsView, QHBoxLayout, + QHeaderView, QLabel, QPushButton, + QSplitter, + QTableWidget, + QTableWidgetItem, QVBoxLayout, QWidget, ) @@ -28,46 +37,53 @@ from qtpy.QtWidgets import ( from bec_widgets.utils.bec_widget import BECWidget from bec_widgets.utils.colors import Colors from bec_widgets.utils.error_popups import SafeSlot -from bec_widgets.utils.jpeg_frames import qimage_preview +from bec_widgets.utils.jpeg_frames import qimage_preview, sensor_shape COLORMAPS = ("gray", "magma", "viridis", "plasma", "inferno") - - -class _ZoomView(QGraphicsView): - """Graphics view with wheel zoom around the cursor.""" - - def __init__(self, scene: QGraphicsScene, parent=None): - super().__init__(scene, parent) - self.setTransformationAnchor(QGraphicsView.ViewportAnchor.AnchorUnderMouse) - self.setDragMode(QGraphicsView.DragMode.ScrollHandDrag) - self.setRenderHint(QPainter.RenderHint.SmoothPixmapTransform, False) - self.setBackgroundBrush(QColor(20, 20, 20)) - self.fit_on_next_frame = True - - def wheelEvent(self, event): # pylint: disable=invalid-name - factor = 1.25 if event.angleDelta().y() > 0 else 0.8 - self.scale(factor, factor) - self.fit_on_next_frame = False +ROI_COLORS = ("#ffd200", "#00d0ff", "#ff5ab4", "#7dff5a", "#ff8c1a", "#b48cff") class JpegView(BECWidget, QWidget): """Live view that shows preview frames as a Qt picture, without decoding to numpy.""" frame_shown = Signal() + rois_changed = Signal(list) def __init__(self, parent=None, device: str | None = None, signal: str = "preview", **kwargs): super().__init__(parent=parent, **kwargs) self._device = None self._signal = signal self._color_table: list[int] | None = None + self._frame_size: tuple[int, int] | None = None + self._sensor: tuple[int, int] | None = None + self._encoding: str | None = None + self._fit_on_next_frame = True + self._rois: list[pg.ROI] = [] + self._roi_count = 0 self.last_frame_metadata: dict = {} self.last_update_ms = 0.0 - self._scene = QGraphicsScene(self) + self.view = pg.PlotWidget(parent=self) + self.plot_item = self.view.getPlotItem() + self.plot_item.setAspectLocked(True) + self.plot_item.setLabel("bottom", "x (sensor px)") + self.plot_item.setLabel("left", "y (sensor px)") + self.plot_item.setMenuEnabled(False) + self.plot_item.hideButtons() + self.view_box: pg.ViewBox = self.plot_item.getViewBox() + self._pixmap_item = QGraphicsPixmapItem() self._pixmap_item.setTransformationMode(Qt.TransformationMode.FastTransformation) - self._scene.addItem(self._pixmap_item) - self.view = _ZoomView(self._scene, self) + self._pixmap_item.setZValue(-100) + self.view_box.addItem(self._pixmap_item, ignoreBounds=True) + + pen = pg.mkPen("#ffd200", width=1, style=Qt.PenStyle.DashLine) + self._v_line = pg.InfiniteLine(angle=90, movable=False, pen=pen) + self._h_line = pg.InfiniteLine(angle=0, movable=False, pen=pen) + for line in (self._v_line, self._h_line): + line.setVisible(False) + self.view_box.addItem(line, ignoreBounds=True) + self.view.scene().sigMouseMoved.connect(self._on_mouse_moved) self.colormap_box = QComboBox(self) self.colormap_box.addItems(COLORMAPS) @@ -75,22 +91,58 @@ class JpegView(BECWidget, QWidget): self.colormap_box.currentTextChanged.connect(self.set_colormap) fit_button = QPushButton("Fit", self) fit_button.clicked.connect(self.fit) + self.crosshair_button = QPushButton("Crosshair", self) + self.crosshair_button.setCheckable(True) + self.crosshair_button.toggled.connect(self.set_crosshair) + rect_button = QPushButton("+ Rect ROI", self) + rect_button.clicked.connect(lambda: self.add_roi("rect")) + circle_button = QPushButton("+ Circle ROI", self) + circle_button.clicked.connect(lambda: self.add_roi("circle")) + clear_button = QPushButton("Clear ROIs", self) + clear_button.clicked.connect(self.clear_rois) self.info_label = QLabel("No frame yet", self) + self.cursor_label = QLabel("", self) + self.cursor_label.setMinimumWidth(170) + + self.roi_table = QTableWidget(0, 5, self) + self.roi_table.setHorizontalHeaderLabels(["ROI", "x from", "x to", "y from", "y to"]) + self.roi_table.verticalHeader().setVisible(False) + self.roi_table.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeMode.Stretch) + self.roi_table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers) + self.roi_table.setToolTip( + "ROI bounds in sensor pixels. Drag the ROI to move it, its handles to resize it;" + " right-click an ROI to remove it." + ) + self.roi_table.setMaximumHeight(150) + self.roi_table.setVisible(False) bar = QHBoxLayout() bar.addWidget(QLabel("Colormap", self)) bar.addWidget(self.colormap_box) bar.addWidget(fit_button) + bar.addWidget(self.crosshair_button) + bar.addWidget(rect_button) + bar.addWidget(circle_button) + bar.addWidget(clear_button) bar.addStretch(1) + bar.addWidget(self.cursor_label) bar.addWidget(self.info_label) + splitter = QSplitter(Qt.Orientation.Vertical, self) + splitter.addWidget(self.view) + splitter.addWidget(self.roi_table) + splitter.setStretchFactor(0, 5) + splitter.setStretchFactor(1, 1) layout = QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) layout.addLayout(bar) - layout.addWidget(self.view) + layout.addWidget(splitter) if device: self.set_source(device, signal) + ################################################################################ + # Source and display + def set_source(self, device: str, signal: str = "preview") -> None: """ Subscribe to a device preview signal. @@ -104,7 +156,7 @@ class JpegView(BECWidget, QWidget): self.on_frame, MessageEndpoints.device_preview(self._device, self._signal) ) self._device, self._signal = device, signal - self.view.fit_on_next_frame = True + self._fit_on_next_frame = True self.bec_dispatcher.connect_slot( self.on_frame, MessageEndpoints.device_preview(device, signal), @@ -128,8 +180,11 @@ class JpegView(BECWidget, QWidget): @SafeSlot() def fit(self) -> None: - """Fit the whole frame into the view.""" - self.view.fitInView(self._pixmap_item, Qt.AspectRatioMode.KeepAspectRatio) + """Fit the whole sensor into the view.""" + if self._sensor is None: + return + rows, cols = self._sensor + self.view_box.setRange(QRectF(0, 0, rows, cols), padding=0) @SafeSlot(dict, dict) def on_frame(self, msg: dict, metadata: dict) -> None: @@ -151,15 +206,192 @@ class JpegView(BECWidget, QWidget): pixmap = QPixmap.fromImage(indexed) else: pixmap = QPixmap.fromImage(qimage) - new_size = pixmap.size() != self._pixmap_item.pixmap().size() + frame_size = (qimage.height(), qimage.width()) + sensor = sensor_shape(metadata, frame_size) + encoding = metadata.get("encoding", "raw") + if (frame_size, sensor, encoding) != (self._frame_size, self._sensor, self._encoding): + self._set_geometry(frame_size, sensor, encoding) self._pixmap_item.setPixmap(pixmap) - if new_size: - self._scene.setSceneRect(QRectF(pixmap.rect())) - if new_size or self.view.fit_on_next_frame: - self.view.fit_on_next_frame = False - self.fit() self.last_update_ms = (time.perf_counter() - t0) * 1e3 self.last_frame_metadata = metadata - encoding = metadata.get("encoding", "raw") - self.info_label.setText(f"{qimage.width()} x {qimage.height()} {encoding}") self.frame_shown.emit() + + def _set_geometry( + self, frame_size: tuple[int, int], sensor: tuple[int, int], encoding: str + ) -> None: + """Place frame pixels on sensor pixels: pixmap (col, row) -> view (row, col).""" + sensor_changed = sensor != self._sensor + self._frame_size, self._sensor, self._encoding = frame_size, sensor, encoding + scale_r, scale_c = sensor[0] / frame_size[0], sensor[1] / frame_size[1] + self._pixmap_item.setTransform(QTransform(0, scale_c, scale_r, 0, 0, 0)) + text = f"{frame_size[1]} x {frame_size[0]} {encoding}" + if frame_size != sensor: + text += f", sensor {sensor[1]} x {sensor[0]}" + self.info_label.setText(text) + if sensor_changed or self._fit_on_next_frame: + self._fit_on_next_frame = False + self.fit() + + ################################################################################ + # Crosshair + + @SafeSlot(bool) + def set_crosshair(self, enabled: bool) -> None: + """ + Show or hide the crosshair that follows the mouse. + + Args: + enabled (bool): Whether the crosshair is shown. + """ + if self.crosshair_button.isChecked() != enabled: + self.crosshair_button.setChecked(enabled) # comes back here through toggled + return + if not enabled: + for line in (self._v_line, self._h_line): + line.setVisible(False) + self.cursor_label.setText("") + + def _on_mouse_moved(self, scene_pos: QPointF) -> None: + if not self.crosshair_button.isChecked(): + return + if not self.view_box.sceneBoundingRect().contains(scene_pos): + return + point = self.view_box.mapSceneToView(scene_pos) + self.update_crosshair(point.x(), point.y()) + + def update_crosshair(self, x: float, y: float) -> None: + """ + Move the crosshair to a point and show its sensor pixel. + + Args: + x (float): x in sensor pixels (array row). + y (float): y in sensor pixels (array column). + """ + self._v_line.setPos(x) + self._h_line.setPos(y) + for line in (self._v_line, self._h_line): + line.setVisible(True) + if self._sensor is None: + return + rows, cols = self._sensor + if 0 <= x < rows and 0 <= y < cols: + self.cursor_label.setText(f"x {int(x)} y {int(y)}") + else: + self.cursor_label.setText("outside the sensor") + + ################################################################################ + # ROIs + + def add_roi(self, shape: str = "rect") -> pg.ROI: + """ + Add a draggable ROI in the middle of the sensor. + + Args: + shape (str): "rect" or "circle". + + Returns: + pg.ROI: The new ROI. Its position and size are in sensor pixels. + """ + rows, cols = self._sensor or (100, 100) + size = max(4, int(min(rows, cols) / 5)) + pos = (int(rows / 2 - size / 2), int(cols / 2 - size / 2)) + color = ROI_COLORS[self._roi_count % len(ROI_COLORS)] + self._roi_count += 1 + options = { + "pen": pg.mkPen(color, width=2), + "removable": True, + "snapSize": 1, + "translateSnap": True, + "scaleSnap": True, + } + if shape == "circle": + roi = pg.CircleROI(pos, (size, size), **options) + elif shape == "rect": + roi = pg.RectROI(pos, (size, size), **options) + else: + raise ValueError(f"Unknown ROI shape {shape!r}; use 'rect' or 'circle'") + roi.setProperty("roi_name", f"{shape} {self._roi_count}") + roi.setProperty("roi_shape", shape) + roi.setProperty("roi_color", color) + roi.sigRegionChanged.connect(self._update_roi_table) + roi.sigRemoveRequested.connect(self.remove_roi) + self.view_box.addItem(roi, ignoreBounds=True) + self._rois.append(roi) + self._update_roi_table() + return roi + + @SafeSlot(object) + def remove_roi(self, roi: pg.ROI) -> None: + """ + Remove one ROI. + + Args: + roi (pg.ROI): The ROI to remove. + """ + if roi not in self._rois: + return + self._rois.remove(roi) + self.view_box.removeItem(roi) + # pyqtgraph gives the ROI and each of its handles a parentless menu + for item in (roi, *roi.getHandles()): + _delete_menu(getattr(item, "menu", None)) + self._update_roi_table() + + @SafeSlot() + def clear_rois(self) -> None: + """Remove all ROIs.""" + for roi in list(self._rois): + self.remove_roi(roi) + + def roi_regions(self) -> list[dict]: + """ + ROI bounds in sensor pixels. + + Returns: + list[dict]: One dict per ROI with ``name``, ``shape``, ``x`` and ``y`` (each a + ``(from, to)`` pair; x runs along array rows, y along columns) and, for circles, + ``center`` and ``radius``. + """ + regions = [] + for roi in self._rois: + (x0, y0), (w, h) = tuple(roi.pos()), tuple(roi.size()) + region = { + "name": roi.property("roi_name"), + "shape": roi.property("roi_shape"), + "x": (x0, x0 + w), + "y": (y0, y0 + h), + } + if region["shape"] == "circle": + region["center"] = (x0 + w / 2, y0 + h / 2) + region["radius"] = w / 2 + regions.append(region) + return regions + + def _update_roi_table(self) -> None: + regions = self.roi_regions() + self.roi_table.setRowCount(len(regions)) + for row, (roi, region) in enumerate(zip(self._rois, regions)): + name = QTableWidgetItem(region["name"]) + name.setForeground(QColor(roi.property("roi_color"))) + if region["shape"] == "circle": + cx, cy = region["center"] + name.setToolTip(f"centre x {cx:g}, y {cy:g}, radius {region['radius']:g}") + self.roi_table.setItem(row, 0, name) + values = (*region["x"], *region["y"]) + for col, value in enumerate(values, start=1): + self.roi_table.setItem(row, col, QTableWidgetItem(f"{value:g}")) + self.roi_table.setVisible(bool(regions)) + self.rois_changed.emit(regions) + + def cleanup(self): + """Remove ROIs and the plot's context menus, which are top-level widgets.""" + self.clear_rois() + _delete_menu(self.view_box.menu) + _delete_menu(self.plot_item.ctrlMenu) + super().cleanup() + + +def _delete_menu(menu) -> None: + if menu is not None: + menu.close() + menu.deleteLater() diff --git a/bec_widgets/utils/jpeg_frames.py b/bec_widgets/utils/jpeg_frames.py index 11fa9d60..fe82783d 100644 --- a/bec_widgets/utils/jpeg_frames.py +++ b/bec_widgets/utils/jpeg_frames.py @@ -5,6 +5,10 @@ A JPEG preview frame is a ``DevicePreviewMessage`` whose ``data`` is a flat uint the JPEG bytes, with message metadata ``{"encoding": "jpeg", "shape": [h, w], "channels": 1 | 3}``. Frames without that metadata are passed through untouched. +A preview may be scaled down from a bigger sensor. ``"sensor_shape": [H, W]`` in the metadata +says which sensor size the frame covers; ``sensor_transform`` maps frame pixels onto sensor +pixels, so crosshair and ROI coordinates do not change with the preview resolution. + The ``*_preview`` functions match the ``preprocess`` hook of ``BECDispatcher.connect_slot``: they run on the thread that receives Redis messages, so decoding never blocks the GUI thread. """ @@ -15,7 +19,7 @@ import time import numpy as np from qtpy.QtCore import QByteArray -from qtpy.QtGui import QImage +from qtpy.QtGui import QImage, QTransform try: # libjpeg-turbo bindings; much faster than Qt's own JPEG plugin import simplejpeg @@ -79,6 +83,30 @@ def decode_jpeg(data: bytes | np.ndarray, channels: int | None = None) -> np.nda return qimage_to_array(qimage) +def column_first(image: np.ndarray, tile: int = 256) -> np.ndarray: + """ + Copy a frame into column-first memory order; values and indexing stay the same. + + pyqtgraph draws array rows along x and makes a transposed, contiguous copy of the frame + before every paint. For a frame already laid out column-first that copy is a no-op, so + calling this off the GUI thread takes the copy out of the paint. The copy runs in tiles, + which is several times faster than one strided copy of a large frame. + + Args: + image (np.ndarray): (h, w) or (h, w, channels) frame. + tile (int): Tile edge in pixels. + + Returns: + np.ndarray: The same frame, with ``np.swapaxes(result, 0, 1)`` C-contiguous. + """ + h, w = image.shape[:2] + out = np.empty((w, h) + image.shape[2:], image.dtype) + for r in range(0, h, tile): + for c in range(0, w, tile): + out[c : c + tile, r : r + tile] = np.swapaxes(image[r : r + tile, c : c + tile], 0, 1) + return np.swapaxes(out, 0, 1) + + def decode_jpeg_preview(msg_content: dict, metadata: dict) -> tuple[dict, dict]: """ Dispatcher preprocess hook: replace JPEG bytes in a preview message by the decoded array. @@ -95,6 +123,7 @@ def decode_jpeg_preview(msg_content: dict, metadata: dict) -> tuple[dict, dict]: data = msg_content.get("data") t0 = time.perf_counter() image = decode_jpeg(data, metadata.get("channels")) + image = column_first(image) decode_ms = (time.perf_counter() - t0) * 1e3 metadata = {**metadata, "decode_ms": decode_ms, "transfer_nbytes": int(np.size(data))} return {**msg_content, "data": image}, metadata @@ -130,3 +159,37 @@ def qimage_preview(msg_content: dict, metadata: dict) -> tuple[dict, dict]: decode_ms = (time.perf_counter() - t0) * 1e3 metadata = {**metadata, "decode_ms": decode_ms, "transfer_nbytes": int(np.asarray(data).nbytes)} return {**msg_content, "qimage": qimage}, metadata + + +def sensor_shape(metadata: dict | None, frame_shape: tuple[int, ...]) -> tuple[int, int]: + """ + Sensor height and width a preview frame covers. + + Args: + metadata (dict | None): Preview message metadata. + frame_shape (tuple[int, ...]): Shape of the frame, rows first. + + Returns: + tuple[int, int]: ``metadata["sensor_shape"]`` if given, otherwise the frame's own size. + """ + shape = (metadata or {}).get("sensor_shape") + if shape is not None and len(shape) == 2 and min(shape) > 0: + return int(shape[0]), int(shape[1]) + return int(frame_shape[0]), int(frame_shape[1]) + + +def sensor_transform(metadata: dict | None, frame_shape: tuple[int, ...]) -> QTransform | None: + """ + Scale from frame pixels to sensor pixels for an image item with rows on the x axis. + + Args: + metadata (dict | None): Preview message metadata. + frame_shape (tuple[int, ...]): Shape of the frame, rows first. + + Returns: + QTransform | None: The scaling, or None if the frame has no ``sensor_shape``. + """ + if not (metadata or {}).get("sensor_shape"): + return None + rows, cols = sensor_shape(metadata, frame_shape) + return QTransform.fromScale(rows / frame_shape[0], cols / frame_shape[1]) diff --git a/bec_widgets/widgets/plots/image/image.py b/bec_widgets/widgets/plots/image/image.py index 6411a153..61841e35 100644 --- a/bec_widgets/widgets/plots/image/image.py +++ b/bec_widgets/widgets/plots/image/image.py @@ -12,7 +12,12 @@ from qtpy.QtWidgets import QWidget from bec_widgets.utils.bec_connector import ConnectionConfig from bec_widgets.utils.colors import Colors, apply_theme from bec_widgets.utils.error_popups import SafeProperty, SafeSlot -from bec_widgets.utils.jpeg_frames import decode_jpeg_preview +from bec_widgets.utils.jpeg_frames import ( + decode_jpeg_preview, + is_jpeg_frame, + sensor_shape, + sensor_transform, +) from bec_widgets.widgets.plots.image.image_base import ImageBase from bec_widgets.widgets.plots.image.image_item import ImageItem from bec_widgets.widgets.plots.image.toolbar_components.device_selection import ( @@ -120,6 +125,9 @@ class Image(ImageBase): self.gui_id = config.gui_id self.subscriptions: defaultdict[str, ImageLayerConfig] = defaultdict(ImageLayerConfig) self.last_frame_metadata: dict = {} + self._frame_extent: tuple[int, int] | None = None + # Draw JPEG preview frames as 8-bit images without per-frame statistics + self.fast_8bit_preview = True # Store signal configs separately (not serialized to QSettings) self._signal_configs: dict[str, dict] = {} @@ -1008,13 +1016,19 @@ class Image(ImageBase): logger.warning("No data received for image update from 2D.") return self.last_frame_metadata = metadata - previous = image.raw_data - if previous is not None and np.shape(previous) != np.shape(data): + # A preview scaled down from a bigger sensor is drawn in sensor pixels + transform = sensor_transform(metadata, np.shape(data)) + extent = sensor_shape(metadata, np.shape(data)) + if transform is None and image.image_transform is not None: + image.resetTransform() + if image.raw_data is not None and extent != self._frame_extent: # e.g. the detector changed resolution: show the whole new frame self._autorange_on_next_update = True + self._frame_extent = extent if self._color_bar is not None: self._color_bar.blockSignals(True) - image.set_data(data) + image.preview_8bit = self.fast_8bit_preview and is_jpeg_frame(metadata) + image.set_data(data, transform=transform) if self._color_bar is not None: self._color_bar.blockSignals(False) if self._autorange_on_next_update: diff --git a/bec_widgets/widgets/plots/image/image_item.py b/bec_widgets/widgets/plots/image/image_item.py index 3b20d896..1395acf3 100644 --- a/bec_widgets/widgets/plots/image/image_item.py +++ b/bec_widgets/widgets/plots/image/image_item.py @@ -88,6 +88,8 @@ class ImageItem(BECConnector, pg.ImageItem): else: self.parent_image = None self.image_transform = None + # Set by the owner for frames a camera already scaled to 8 bit (e.g. JPEG previews) + self.preview_8bit = False super().__init__(config=config, gui_id=gui_id, **kwargs) self.raw_data = None @@ -197,6 +199,9 @@ class ImageItem(BECConnector, pg.ImageItem): def apply_autorange(self): if self.raw_data is None: return + if self._draws_8bit_as_is(): + self._show_full_8bit_range() + return data = self.image if data is None: data = self.raw_data @@ -227,6 +232,16 @@ class ImageItem(BECConnector, pg.ImageItem): if self.raw_data is None: return + if self._draws_8bit_as_is(): + # The camera already scaled the frame: draw it as it is, without a processed copy + # or statistics over the whole frame. Autorange shows the full 0-255 range. + self.setImage(self.raw_data, autoLevels=False) + if self.image_transform is not None: + self.setTransform(self.image_transform) + if self.config.autorange: + self._show_full_8bit_range() + return + if np.all(np.isnan(self.raw_data)): return @@ -238,6 +253,20 @@ class ImageItem(BECConnector, pg.ImageItem): self.setTransform(self.image_transform) self.autorange = autorange + def _draws_8bit_as_is(self) -> bool: + """Whether the current frame is an 8-bit preview frame that needs no processing.""" + processing = self.config.processing + return ( + self.preview_8bit + and getattr(self.raw_data, "dtype", None) == np.uint8 + and not (processing.fft or processing.log or processing.transpose) + and not processing.num_rotation_90 + ) + + def _show_full_8bit_range(self) -> None: + if self.levels is None or tuple(self.levels) != (0, 255): + self.set_v_range((0, 255), disable_autorange=False) + @property def fft(self) -> bool: """Get or set whether FFT postprocessing is enabled.""" diff --git a/tests/unit_tests/test_jpeg_frames.py b/tests/unit_tests/test_jpeg_frames.py index 46b18d8d..81535a82 100644 --- a/tests/unit_tests/test_jpeg_frames.py +++ b/tests/unit_tests/test_jpeg_frames.py @@ -189,3 +189,145 @@ def test_jpeg_view_shows_frame_and_colormap(qtbot, mocked_client): assert view._pixmap_item.pixmap().height() == 24 view.set_colormap("gray") assert view._color_table is None + + +def test_sensor_transform_maps_frame_to_sensor_pixels(): + assert jpeg_frames.sensor_transform({}, (512, 256)) is None + assert jpeg_frames.sensor_shape({}, (512, 256)) == (512, 256) + metadata = {"sensor_shape": [4096, 2048]} + assert jpeg_frames.sensor_shape(metadata, (512, 256)) == (4096, 2048) + transform = jpeg_frames.sensor_transform(metadata, (512, 256)) + # image item axes: x = row, y = column + assert transform.map(512.0, 256.0) == (4096.0, 2048.0) + + +def test_image_draws_preview_in_sensor_pixels(qtbot, mocked_client): + view = create_widget(qtbot, Image, client=mocked_client) + metadata = {"sensor_shape": [400, 800]} + view.on_image_update_2d({"data": np.zeros((100, 200))}, metadata) + bounds = view.main_image.mapRectToView(view.main_image.boundingRect()) + assert (bounds.width(), bounds.height()) == (400, 800) + view._autorange_on_next_update = False + with mock.patch.object(view, "auto_range") as auto_range: + # a new preview resolution of the same sensor keeps the zoom + view.on_image_update_2d({"data": np.zeros((50, 100))}, metadata) + auto_range.assert_not_called() + bounds = view.main_image.mapRectToView(view.main_image.boundingRect()) + assert (bounds.width(), bounds.height()) == (400, 800) + # frames without a sensor shape are drawn in their own pixels again + view.on_image_update_2d({"data": np.zeros((10, 20))}, {}) + bounds = view.main_image.mapRectToView(view.main_image.boundingRect()) + assert (bounds.width(), bounds.height()) == (10, 20) + + +def _show_frame(view, image, metadata): + content, metadata = qimage_preview( + {"data": np.frombuffer(_jpeg_bytes(image), np.uint8)}, {"encoding": "jpeg", **metadata} + ) + view.on_frame(content, metadata) + + +def test_jpeg_view_places_preview_on_sensor(qtbot, mocked_client): + view = create_widget(qtbot, JpegView, client=mocked_client) + _show_frame(view, _gradient(h=24, w=40), {"sensor_shape": [240, 80]}) + bounds = view._pixmap_item.mapRectToParent(view._pixmap_item.boundingRect()) + # rows along x, columns along y, scaled up to the sensor + assert (bounds.width(), bounds.height()) == (240, 80) + assert "sensor 80 x 240" in view.info_label.text() + # another preview resolution covers the same sensor area + _show_frame(view, _gradient(h=12, w=20), {"sensor_shape": [240, 80]}) + bounds = view._pixmap_item.mapRectToParent(view._pixmap_item.boundingRect()) + assert (bounds.width(), bounds.height()) == (240, 80) + + +def test_jpeg_view_crosshair_reports_sensor_pixel(qtbot, mocked_client): + view = create_widget(qtbot, JpegView, client=mocked_client) + _show_frame(view, _gradient(h=24, w=40), {"sensor_shape": [240, 400]}) + view.set_crosshair(True) + assert view.crosshair_button.isChecked() + view.update_crosshair(120.7, 33.2) + assert view.cursor_label.text() == "x 120 y 33" + assert view._v_line.value() == pytest.approx(120.7) + view.update_crosshair(-1, 10) + assert view.cursor_label.text() == "outside the sensor" + view.set_crosshair(False) + assert not view._v_line.isVisible() and view.cursor_label.text() == "" + + +def test_jpeg_view_rois_in_sensor_pixels(qtbot, mocked_client): + view = create_widget(qtbot, JpegView, client=mocked_client) + _show_frame(view, _gradient(h=24, w=40), {"sensor_shape": [1000, 500]}) + with qtbot.waitSignal(view.rois_changed) as blocker: + rect = view.add_roi("rect") + assert blocker.args[0][0]["x"] == (450, 550) and blocker.args[0][0]["y"] == (200, 300) + circle = view.add_roi("circle") + rect.setPos((10, 20)) + rect.setSize((30, 40)) + regions = view.roi_regions() + assert regions[0]["x"] == (10, 40) and regions[0]["y"] == (20, 60) + assert regions[1]["shape"] == "circle" and regions[1]["radius"] == 50 + assert view.roi_table.rowCount() == 2 + assert view.roi_table.item(0, 1).text() == "10" + # ROIs stay where they are when the preview resolution changes + _show_frame(view, _gradient(h=12, w=20), {"sensor_shape": [1000, 500]}) + assert view.roi_regions()[0]["x"] == (10, 40) + view.remove_roi(circle) + assert len(view.roi_regions()) == 1 + view.clear_rois() + assert view.roi_regions() == [] and view.roi_table.isHidden() + with pytest.raises(ValueError): + view.add_roi("polygon") + + +@pytest.mark.parametrize("shape", [(5, 7), (300, 520), (300, 520, 3)]) +def test_column_first_keeps_values(shape): + image = np.random.default_rng(0).integers(0, 255, shape, dtype=np.uint8) + result = jpeg_frames.column_first(image, tile=256) + assert np.array_equal(result, image) + assert np.swapaxes(result, 0, 1).flags["C_CONTIGUOUS"] + + +def test_decode_jpeg_preview_lays_frame_out_column_first(): + payload = np.frombuffer(_jpeg_bytes(_gradient()), np.uint8) + content, _ = decode_jpeg_preview({"data": payload}, {"encoding": "jpeg", "channels": 1}) + assert content["data"].shape == (24, 40) + assert content["data"].T.flags["C_CONTIGUOUS"] + + +def _noisy_8bit(h=32, w=48): + return np.random.default_rng(1).integers(0, 60, (h, w), dtype=np.uint8) + + +def test_image_fast_path_draws_jpeg_frames_as_they_are(qtbot, mocked_client): + view = create_widget(qtbot, Image, client=mocked_client) + frame = _noisy_8bit() + with mock.patch("bec_widgets.widgets.plots.image.image_item.ImageStats.from_data") as from_data: + view.on_image_update_2d({"data": frame}, {"encoding": "jpeg"}) + from_data.assert_not_called() + image = view.main_image + assert np.shares_memory(image.image, frame) # no processed copy + assert tuple(image.levels) == (0, 255) and image.autorange + # a manual range survives later frames + image.v_range = (10, 40) + view.on_image_update_2d({"data": _noisy_8bit()}, {"encoding": "jpeg"}) + assert tuple(image.levels) == (10, 40) + # auto levels from the menu go back to the full 8-bit range + view.autorange = True + assert tuple(image.levels) == (0, 255) + + +@pytest.mark.parametrize( + "setup, metadata", + [ + (lambda view: setattr(view, "fast_8bit_preview", False), {"encoding": "jpeg"}), + (lambda view: None, {}), # uint8 frames that are not JPEG previews + (lambda view: setattr(view.main_image, "log", True), {"encoding": "jpeg"}), + ], +) +def test_image_normal_path_when_fast_path_does_not_apply(qtbot, mocked_client, setup, metadata): + view = create_widget(qtbot, Image, client=mocked_client) + setup(view) + frame = _noisy_8bit() + view.on_image_update_2d({"data": frame}, metadata) + image = view.main_image + assert tuple(image.levels) != (0, 255)