From b48aaa6ade35e4811b3edd1abbbfd66961283174 Mon Sep 17 00:00:00 2001 From: Mirko Holler Date: Mon, 7 Sep 2026 07:47:21 -0300 Subject: [PATCH] feat(vision): make edge detection tunable (thresholds + pre-Canny blur) Real captures (a photo of the gripper/sample gap against a CNC-machined metal background, and the existing flomni reference_images/ set) show dense machining scratches registering as Canny edges as strongly as the actual object boundary, at any threshold -- verified on one real capture: low=50/high=150 finds ~203k edge pixels, low=150/high=250 still ~76k, both dominated by texture rather than the gripper. Raising thresholds alone can't separate "faint real edge" from "strong but irrelevant texture" when both survive at the same gradient magnitude. vision_toolkit.detect_edges() gains a blur_ksize parameter (Gaussian blur before Canny, 0 = off) for exactly this: blurring suppresses fine texture while leaving the coarser real edge mostly intact. On the same capture, adding blur_ksize=9 on top of low=150/high=250 drops the edge count to ~5k; blur_ksize=15 to ~1.8k. VisionInterface exposes this as live device state (set_edge_params()/ get_edge_params(), used by "edges" mode), and VisionTuning gains edge low/high/ blur spin boxes (enabled only in "edges" mode) next to the existing mode/ROI controls, so tuning is visual and interactive rather than a blind parameter sweep. VISION_PIPELINE_TESTING_HOWTO.md's manual edge-inspection section is extended accordingly. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01DQpiwJcJZJKGo6Pzozfsi1 --- .../AI_docs/VISION_PIPELINE_TESTING_HOWTO.md | 37 +++++++++++- .../widgets/vision_tuning/vision_tuning.py | 59 +++++++++++++++++++ csaxs_bec/devices/vision/vision_interface.py | 52 +++++++++++++++- csaxs_bec/devices/vision/vision_toolkit.py | 16 ++++- tests/tests_devices/test_vision_interface.py | 41 +++++++++++++ tests/tests_devices/test_vision_toolkit.py | 19 ++++++ 6 files changed, 218 insertions(+), 6 deletions(-) diff --git a/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/VISION_PIPELINE_TESTING_HOWTO.md b/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/VISION_PIPELINE_TESTING_HOWTO.md index 75bb5ec9..2711392f 100644 --- a/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/VISION_PIPELINE_TESTING_HOWTO.md +++ b/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/VISION_PIPELINE_TESTING_HOWTO.md @@ -267,9 +267,40 @@ cv2.imwrite("/tmp/check_overlay.png", cv2.cvtColor(overlay, cv2.COLOR_RGB2BGR)) Open `check_overlay.png` and confirm the red contour actually traces the gripper/pin, not something else in the background, before trusting any score computed from it -- if it doesn't, adjust the ROI -and/or `edge_low_threshold`/`edge_high_threshold` and try again. This is also the way to see the -`diagnostic_image` overlay's equivalent without needing the GUI dock that doesn't exist yet (see -below). +and/or `edge_low_threshold`/`edge_high_threshold` and try again. + +**A busy real background can need more than higher thresholds.** On a CNC-machined metal surface, +fine machining marks/scratches can register as Canny edges just as strongly as the actual object +boundary, at *any* threshold -- raising `edge_high_threshold` alone may still leave the whole +background lit up (verified: on one real capture here, `low=50/high=150` found ~203k edge pixels, +and `low=150/high=250` still found ~76k -- both dominated by texture, not the gripper). `detect_edges()` +now takes a `blur_ksize` (Gaussian blur kernel size before Canny runs, `0` = off, must be odd +otherwise) for exactly this case -- pushing it up suppresses the fine texture while leaving the +coarser real edge mostly intact, in a way raising the threshold alone can't (same capture: adding +`blur_ksize=9` on top of `low=150/high=250` dropped it to ~5k; `blur_ksize=15` to ~1.8k): + +```python +edges = vt.detect_edges(crop, low_threshold=150, high_threshold=250, blur_ksize=9) +``` + +`VisionInterface` exposes the same three parameters as live, settable device state -- +`set_edge_params(low_threshold, high_threshold, blur_ksize=0)`/`get_edge_params()` -- which is what +`"edges"` mode (`set_mode("edges")`) actually uses, so tuning them affects `refresh()`'s published +`diagnostic_image` immediately: + +```python +dev.vision_test_interface.set_edge_params(150, 250, blur_ksize=9) +dev.vision_test_interface.set_mode("edges") +dev.vision_test_interface.refresh("vision_cam_a") +``` + +There's now also a GUI for this loop instead of typing every call by hand -- +`gui.test.new("VisionTuning", object_name="vision_tuning")` (`csaxs_bec/bec_widgets/widgets/vision_tuning/`) +gives a device/camera picker, mode picker, ROI apply/clear, and edge low/high/blur spin boxes (enabled +only in `"edges"` mode) next to a live `diagnostic_image` preview -- the visual side of exactly this +section, without the manual `cv2.imwrite`-and-reopen loop above. The manual toolkit-level approach +above is still the right one when you want to inspect a single frame offline without a live device +(e.g. comparing parameter choices side by side on a saved capture). ## What this does and doesn't prove diff --git a/csaxs_bec/bec_widgets/widgets/vision_tuning/vision_tuning.py b/csaxs_bec/bec_widgets/widgets/vision_tuning/vision_tuning.py index 11dbe76a..b6dd97a9 100644 --- a/csaxs_bec/bec_widgets/widgets/vision_tuning/vision_tuning.py +++ b/csaxs_bec/bec_widgets/widgets/vision_tuning/vision_tuning.py @@ -108,6 +108,30 @@ class VisionTuning(BECWidget, QWidget): roi_row.addWidget(self.clear_roi_button) root.addLayout(roi_row) + # Edge-detection tuning -- only meaningful in "edges" mode (see _update_edge_controls_enabled). + # A busy/textured real background (e.g. CNC machining marks) can register as edges at any + # threshold; blur suppresses that fine texture before Canny runs, which raising the + # thresholds alone can't do -- see vision_toolkit.detect_edges()'s docstring. + edge_row = QHBoxLayout() + self.edge_low = QSpinBox(parent=self) + self.edge_high = QSpinBox(parent=self) + self.edge_blur = QSpinBox(parent=self) + self.edge_low.setRange(0, 1000) + self.edge_high.setRange(0, 1000) + # 0 = no blur, otherwise must be odd (cv2.GaussianBlur's kernel-size requirement) -- default + # single-step of 1 so every value 0..51 is reachable via the arrows, not just even ones. + self.edge_blur.setRange(0, 51) + for label_text, spin_box in ( + ("edge low", self.edge_low), + ("edge high", self.edge_high), + ("blur", self.edge_blur), + ): + edge_row.addWidget(QLabel(label_text, parent=self)) + edge_row.addWidget(spin_box) + self.apply_edge_button = QPushButton("Apply Edge Params", parent=self) + edge_row.addWidget(self.apply_edge_button) + root.addLayout(edge_row) + control_row = QHBoxLayout() self.refresh_button = QPushButton("Refresh", parent=self) self.live_checkbox = QCheckBox("Live", parent=self) @@ -124,8 +148,10 @@ class VisionTuning(BECWidget, QWidget): self.mode_combo.currentTextChanged.connect(self._on_mode_changed) self.apply_roi_button.clicked.connect(self._on_apply_roi) self.clear_roi_button.clicked.connect(self._on_clear_roi) + self.apply_edge_button.clicked.connect(self._on_apply_edge_params) self.refresh_button.clicked.connect(self._on_refresh_clicked) self.live_checkbox.toggled.connect(self._on_live_toggled) + self._update_edge_controls_enabled(self.mode_combo.currentText()) def _populate_devices(self): """List every configured ``VisionInterface`` instance, once, at widget construction -- @@ -161,6 +187,16 @@ class VisionTuning(BECWidget, QWidget): self.camera_combo.blockSignals(False) if device_name: self.image.image(device=device_name, signal="diagnostic_image") + if interface is not None: + edge_params = interface.get_edge_params() + for spin_box, key in ( + (self.edge_low, "low_threshold"), + (self.edge_high, "high_threshold"), + (self.edge_blur, "blur_ksize"), + ): + spin_box.blockSignals(True) + spin_box.setValue(edge_params[key]) + spin_box.blockSignals(False) if camera_names: self._on_camera_changed(camera_names[0]) else: @@ -190,12 +226,21 @@ class VisionTuning(BECWidget, QWidget): @SafeSlot(str) def _on_mode_changed(self, mode: str): + self._update_edge_controls_enabled(mode) interface = self._current_interface() if interface is None or not mode: return interface.set_mode(mode) self._on_refresh_clicked() + def _update_edge_controls_enabled(self, mode: str): + """Edge threshold/blur only affect "edges" mode -- grey them out otherwise rather than + leaving controls that currently do nothing clickable (same reasoning as + z_ConsoleButtonsWidget greying out Yes/No with no message to respond to).""" + enabled = mode == "edges" + for widget in (self.edge_low, self.edge_high, self.edge_blur, self.apply_edge_button): + widget.setEnabled(enabled) + @SafeSlot() def _on_apply_roi(self): interface = self._current_interface() @@ -213,6 +258,20 @@ class VisionTuning(BECWidget, QWidget): interface.set_roi(None, camera_name=self._current_camera_name) self._on_refresh_clicked() + @SafeSlot() + def _on_apply_edge_params(self): + interface = self._current_interface() + if interface is None: + return + try: + interface.set_edge_params( + self.edge_low.value(), self.edge_high.value(), self.edge_blur.value() + ) + except Exception: + logger.exception("VisionTuning: set_edge_params failed") + return + self._on_refresh_clicked() + @SafeSlot() def _on_refresh_clicked(self): """Pull+process+publish once. Never triggers a live camera acquisition -- ``refresh()`` diff --git a/csaxs_bec/devices/vision/vision_interface.py b/csaxs_bec/devices/vision/vision_interface.py index 3240958b..1df6c2c4 100644 --- a/csaxs_bec/devices/vision/vision_interface.py +++ b/csaxs_bec/devices/vision/vision_interface.py @@ -62,6 +62,8 @@ class VisionInterface(PSIDeviceBase): "get_mode", "set_roi", "get_roi", + "set_edge_params", + "get_edge_params", "get_last_processed_image", "get_raw_frame", "publish_diagnostic_image", @@ -78,6 +80,9 @@ class VisionInterface(PSIDeviceBase): device_manager: "DeviceManagerBase | None" = None, mode: VisionMode = "raw", roi: "RoiType | dict[str, RoiType] | None" = None, + edge_low_threshold: int = 50, + edge_high_threshold: int = 150, + edge_blur_ksize: int = 0, **kwargs, ): """Initialize the vision interface. @@ -101,6 +106,10 @@ class VisionInterface(PSIDeviceBase): single ``(x, y, width, height)`` applied to every configured camera, a ``{camera_name: (x, y, width, height)}`` mapping for per-camera ROIs, or ``None`` (the default) to use each camera's full frame. + edge_low_threshold (int): Initial Canny low threshold for ``"edges"`` mode -- see + :meth:`set_edge_params`. + edge_high_threshold (int): Initial Canny high threshold for ``"edges"`` mode. + edge_blur_ksize (int): Initial pre-Canny Gaussian blur kernel size (``0`` = no blur). """ super().__init__( name=name, prefix=prefix, scan_info=scan_info, device_manager=device_manager, **kwargs @@ -110,6 +119,7 @@ class VisionInterface(PSIDeviceBase): self._camera_names = list(camera_names) self._rois: dict[str, RoiType] = {} self.set_mode(mode) + self.set_edge_params(edge_low_threshold, edge_high_threshold, edge_blur_ksize) if isinstance(roi, dict): for camera_name, camera_roi in roi.items(): self.set_roi(camera_roi, camera_name=camera_name) @@ -176,6 +186,44 @@ class VisionInterface(PSIDeviceBase): self._check_camera_name(camera_name) return self._rois.get(camera_name) + def set_edge_params( + self, low_threshold: int, high_threshold: int, blur_ksize: int = 0 + ) -> None: + """Set the Canny edge parameters used by ``"edges"`` mode (see :meth:`set_mode`). + + Not exposed as per-camera state like :meth:`set_roi` -- unlike an ROI, which frames a + physically different region per camera, edge detection tuning is a property of the scene + (background texture, lighting), so one setting per interface is enough. + + Args: + low_threshold (int): Canny low threshold (``vision_toolkit.detect_edges()``'s + ``low_threshold``). + high_threshold (int): Canny high threshold. + blur_ksize (int): Pre-Canny Gaussian blur kernel size -- must be ``0`` (no blur) or a + positive odd integer. Real, richly-textured surfaces (machining marks, scratches) + can register as edges at any threshold; blurring first suppresses that fine texture + while leaving the actual object boundary largely intact, which raising the + threshold alone cannot do (see ``detect_edges()``'s docstring). Start here (e.g. + blur_ksize=5) if a busy background is drowning out the real edge, not just by + raising the thresholds. + """ + if blur_ksize and (blur_ksize < 0 or blur_ksize % 2 == 0): + raise VisionInterfaceError( + f"{self.name}: blur_ksize must be 0 (no blur) or a positive odd integer, got" + f" {blur_ksize}." + ) + self._edge_low_threshold = low_threshold + self._edge_high_threshold = high_threshold + self._edge_blur_ksize = blur_ksize + + def get_edge_params(self) -> dict[str, int]: + """Current Canny edge parameters -- see :meth:`set_edge_params`.""" + return { + "low_threshold": self._edge_low_threshold, + "high_threshold": self._edge_high_threshold, + "blur_ksize": self._edge_blur_ksize, + } + def _resolve_camera(self, camera_name: str): self._check_camera_name(camera_name) if self.device_manager is None or not hasattr(self.device_manager, "devices"): @@ -207,7 +255,9 @@ class VisionInterface(PSIDeviceBase): if mode == "grayscale": return vt.to_grayscale(image) if mode == "edges": - return vt.detect_edges(image) + return vt.detect_edges( + image, self._edge_low_threshold, self._edge_high_threshold, self._edge_blur_ksize + ) if mode == "clahe": return vt.apply_clahe(image) raise VisionInterfaceError(f"{self.name}: unknown mode '{mode}'.") diff --git a/csaxs_bec/devices/vision/vision_toolkit.py b/csaxs_bec/devices/vision/vision_toolkit.py index 45054f50..e009772a 100644 --- a/csaxs_bec/devices/vision/vision_toolkit.py +++ b/csaxs_bec/devices/vision/vision_toolkit.py @@ -164,10 +164,22 @@ def apply_clahe( def detect_edges( - image: np.ndarray, low_threshold: int = 50, high_threshold: int = 150 + image: np.ndarray, low_threshold: int = 50, high_threshold: int = 150, blur_ksize: int = 0 ) -> np.ndarray: - """Canny edge map of a grayscale (or CLAHE-normalized) image.""" + """Canny edge map of a grayscale (or CLAHE-normalized) image. + + ``blur_ksize`` (a Gaussian blur kernel size, must be odd; ``0`` -- the default -- skips + blurring entirely) exists for real, richly-textured surfaces (e.g. a CNC-machined metal + background) where fine texture -- machining marks, scratches -- register as Canny edges just + as strongly as the actual object boundary, at any threshold: raising ``low_threshold``/ + ``high_threshold`` alone can't separate "faint but real edge" from "strong but irrelevant + texture" when both survive at the same gradient magnitude. A blur applied *before* Canny + suppresses the high-frequency texture while leaving the larger-scale object boundary largely + intact, which raising the threshold on the unblurred image cannot do on its own. + """ gray = to_grayscale(image) + if blur_ksize: + gray = cv2.GaussianBlur(gray, (blur_ksize, blur_ksize), 0) return cv2.Canny(gray, low_threshold, high_threshold) diff --git a/tests/tests_devices/test_vision_interface.py b/tests/tests_devices/test_vision_interface.py index 97d3fec7..0b0c15de 100644 --- a/tests/tests_devices/test_vision_interface.py +++ b/tests/tests_devices/test_vision_interface.py @@ -225,3 +225,44 @@ def test_publish_diagnostic_image_makes_an_externally_computed_image_available() interface.publish_diagnostic_image(overlay) published = interface.diagnostic_image.get() assert np.array_equal(published.data, overlay) + + +def test_default_edge_params(): + interface = _make_interface() + assert interface.get_edge_params() == { + "low_threshold": 50, + "high_threshold": 150, + "blur_ksize": 0, + } + + +def test_edge_params_constructor_arguments(): + interface = _make_interface( + edge_low_threshold=10, edge_high_threshold=200, edge_blur_ksize=5 + ) + assert interface.get_edge_params() == { + "low_threshold": 10, + "high_threshold": 200, + "blur_ksize": 5, + } + + +def test_set_edge_params_rejects_even_positive_blur_ksize(): + interface = _make_interface() + with pytest.raises(VisionInterfaceError): + interface.set_edge_params(50, 150, blur_ksize=4) + + +def test_set_edge_params_zero_blur_ksize_is_allowed(): + interface = _make_interface() + interface.set_edge_params(50, 150, blur_ksize=0) + assert interface.get_edge_params()["blur_ksize"] == 0 + + +def test_set_edge_params_affects_edges_mode_output(): + image = _rgb_image() + interface = _make_interface(images={"cam_a": image}, mode="edges") + default_result = interface.get_last_processed_image() + interface.set_edge_params(250, 255) # thresholds high enough to suppress the square's edges + strict_result = interface.get_last_processed_image() + assert strict_result.sum() <= default_result.sum() diff --git a/tests/tests_devices/test_vision_toolkit.py b/tests/tests_devices/test_vision_toolkit.py index ea1c0900..73db5225 100644 --- a/tests/tests_devices/test_vision_toolkit.py +++ b/tests/tests_devices/test_vision_toolkit.py @@ -84,6 +84,25 @@ def test_detect_edges_blank_image_has_no_edges(): assert edges.sum() == 0 +def test_detect_edges_blur_ksize_zero_matches_unblurred_default(): + image = _square_image() + assert np.array_equal(vt.detect_edges(image, blur_ksize=0), vt.detect_edges(image)) + + +def test_detect_edges_blur_suppresses_fine_texture_noise(): + # Simulate a busy real background (e.g. CNC machining marks): a plain square plus dense + # salt noise, which registers as edges just as strongly as the square's real boundary at any + # threshold -- only a pre-Canny blur can tell them apart (see detect_edges()'s docstring). + rng = np.random.default_rng(0) + image = _square_image(size=100, square_side=30) + noisy = image.copy() + noise_mask = rng.random(noisy.shape[:2]) < 0.05 + noisy[noise_mask] = 255 + unblurred_edges = vt.detect_edges(noisy) + blurred_edges = vt.detect_edges(noisy, blur_ksize=9) + assert blurred_edges.sum() < unblurred_edges.sum() + + def test_register_to_reference_identity_for_identical_images(): image = _square_image() result = vt.register_to_reference(image, image)