From 9b65b2ddac4aacdb50844da534bc4c30e0653f7f Mon Sep 17 00:00:00 2001 From: x01dc Date: Sun, 6 Sep 2026 14:09:57 +0200 Subject: [PATCH] fix(vision): correct a misdiagnosis, make registration + position config The previous commit's "translation is currently invisible" finding was wrong about the cause: verified directly that contour_similarity_score() (Hu-moment shape matching) is unconditionally translation/rotation/scale- invariant regardless of registration -- scoring completely unregistered frames gives the same result as registered ones for a pure translation. warp_mode never controlled that; it only affects registration/ rasterization quality for a genuine scale/rotation difference (small, now verified effect). Per feedback: OMNY genuinely wants position ignored, but this should be configurable for other future consumers, not hardcoded. Since the shape score can't be made position-sensitive, added a separate, independent signal instead -- vision_toolkit.centroid_distance() (always measured on raw, unregistered contours, since registering first would remove the displacement it's meant to detect) and SampleVisionMonitor's position_tolerance_px (None by default, matching OMNY's actual need; runtime-adjustable via set/get_position_tolerance_px()). warp_mode is now itself configurable too (WARP_MODES: none/translation/euclidean/ affine, friendly-string YAML-representable via resolve_warp_mode()) -- legitimate on its own merits, just not a fix for the translation non-issue. Corrected the one test that had encoded the wrong diagnosis (asserted warp_mode="none" would make translation "count" -- it doesn't), and rewrote the howto's affected section to match. Also fixed one arithmetic error caught while re-verifying a howto example (offset=(20,0) is a distance of 20px, not 20*sqrt(2)). 15 new/changed tests (6 vision_toolkit: resolve_warp_mode, none-mode registration skip, corrected shape-invariance test, centroid/position tests; 5 SampleVisionMonitor: warp_mode get/set, position_tolerance_px behavior). Full suite: 371 passed. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018Mupip8x7Hean6E3FFvpbj --- .../MACHINE_VISION_AUTOCONFIRM_PLAN.md | 39 +++-- .../AI_docs/VISION_PIPELINE_TESTING_HOWTO.md | 89 ++++++++--- .../simulated_omny/vision_test.yaml | 7 +- .../devices/vision/sample_vision_monitor.py | 61 ++++++- csaxs_bec/devices/vision/vision_toolkit.py | 151 ++++++++++++++++-- .../test_sample_vision_monitor.py | 91 ++++++++++- tests/tests_devices/test_vision_toolkit.py | 99 ++++++++++++ 7 files changed, 479 insertions(+), 58 deletions(-) diff --git a/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/MACHINE_VISION_AUTOCONFIRM_PLAN.md b/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/MACHINE_VISION_AUTOCONFIRM_PLAN.md index 3e81c9d2..f6c5ee83 100644 --- a/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/MACHINE_VISION_AUTOCONFIRM_PLAN.md +++ b/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/MACHINE_VISION_AUTOCONFIRM_PLAN.md @@ -7,33 +7,40 @@ this plan grows out of — see its "full gripper sample-transfer chain... untest ## Implementation status (branch `feat/omny_vision`) -- **Layer 0 (`csaxs_bec/devices/vision/vision_toolkit.py`) — done, tested (16 tests).** Grayscale, - CLAHE, Canny edges, ECC registration (never raises on non-convergence), contour - extraction/scoring, multi-reference best-match, contour overlay drawing. -- **Layer 1 (`csaxs_bec/devices/vision/vision_interface.py`, `VisionInterface`) — done, tested (12 +- **Layer 0 (`csaxs_bec/devices/vision/vision_toolkit.py`) — done, tested (29 tests).** Grayscale, + CLAHE, Canny edges, ECC registration with a configurable, named `warp_mode` + (`"none"`/`"translation"`/`"euclidean"`/`"affine"`, never raises on non-convergence), contour + extraction/scoring, a separate position-sensitive `centroid_distance()` signal (kept independent + of the shape score on purpose — see below), multi-reference best-match, contour overlay drawing. +- **Layer 1 (`csaxs_bec/devices/vision/vision_interface.py`, `VisionInterface`) — done, tested (14 tests).** Config-driven `camera_names`, lazy per-call resolution via `device_manager.devices.get()`, runtime-selectable mode (`raw`/`grayscale`/`edges`/`clahe`), own `diagnostic_image` `PreviewSignal`, `refresh()`/`get_last_processed_image()`. Not yet wired into any device config (no YAML entry, no real camera exercised) or into a GUI dock. - **Layer 2 (`csaxs_bec/devices/vision/sample_vision_monitor.py`, `SampleVisionMonitor`) — done, - tested (7 tests).** Per-tag, per-camera in-memory reference store + `best_reference_match()` + tested (15 tests).** Per-tag, per-camera in-memory reference store + `best_reference_match()` scoring across a `VisionInterface`'s cameras, worst-camera diagnostic overlay published via - `publish_diagnostic_image()`. Not yet wired into any OMNY confirm gate. + `publish_diagnostic_image()`, configurable/runtime-adjustable `warp_mode` and + `position_tolerance_px`. Not yet wired into any OMNY confirm gate. - **Testing tools — done.** `DummyVisionCamera` (`csaxs_bec/devices/vision/dummy_vision_camera.py`, a settable, hardware-free `get_last_image()` stand-in) + `device_configs/simulated_omny/vision_test.yaml` (two dummy cameras + one `VisionInterface` + one `SampleVisionMonitor`, loadable into any running session via `bec.config.add_to_session(...)`) + `AI_docs/VISION_PIPELINE_TESTING_HOWTO.md` (step-by-step walkthrough). -- **⚠ Known limitation, found via the above testing tools, not yet fixed**: `register_to_reference()` - has no bound on how much translation its ECC alignment will correct, so a same-shaped object that - has simply *moved* is registered right back into alignment and scores as unchanged — pure - positional drift is currently invisible to `best_reference_match()`. This is backwards for OMNY's - actual need ("the pin isn't where it should be" is exactly the kind of thing a confirm gate must - catch). Missing/deformed defects are unaffected and verified working. See - `VISION_PIPELINE_TESTING_HOWTO.md`'s "Known limitation" section for the reproduction and a sketch - of two possible fixes (bound the tolerated translation; or run a supplementary un-registered - comparison) — needs a real design decision before Phase 0 data collection can be trusted for - drift-type defects specifically. +- **Finding, corrected after further testing**: an earlier pass here claimed + `register_to_reference()`'s registration was what made pure translation "invisible" to scoring, + and treated that as a bug. Verified false on closer testing: `contour_similarity_score()` + (Hu-moment shape matching) is *unconditionally* translation/rotation/scale-invariant by + construction — this holds with registration fully disabled too, so it was never caused by + registration and isn't something `warp_mode` controls. What `warp_mode` actually, verifiably + affects is registration/rasterization quality for a real scale or rotation difference (small + effect) — legitimate and now configurable per Mirko's request, but not a fix for a "bug" that + turned out not to be one. Since Mirko separately confirmed OMNY genuinely wants position ignored + but wanted the *option* to make it matter for other future consumers, added a real, independent + position-sensitive signal instead: `centroid_distance()` (Layer 0) / + `SampleVisionMonitor`'s `position_tolerance_px` (Layer 2, off by default, matching OMNY's actual + need) — deliberately kept separate from the shape score rather than trying to make the shape + metric itself position-sensitive. - **CLI wrapper methods (`omny.ovision_*`), GUI mode-picker, `x_ray_eye_align.py` integration, confirm-gate wiring — not started.** diff --git a/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/VISION_PIPELINE_TESTING_HOWTO.md b/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/VISION_PIPELINE_TESTING_HOWTO.md index 4afd3221..45e62915 100644 --- a/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/VISION_PIPELINE_TESTING_HOWTO.md +++ b/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/VISION_PIPELINE_TESTING_HOWTO.md @@ -69,8 +69,8 @@ dev.vision_test_monitor.score("test_tag") # {'tag': 'test_tag', 'overall_score': ~0.0, 'worst_camera': 'vision_cam_a' or 'vision_cam_b', # 'per_camera': {'vision_cam_a': {'score': ~0.0, 'reference_count': 1, 'converged': True}, ...}} -# Now simulate a real problem on one camera -- the shape genuinely changes (not just moves -- -# see "Known limitation: pure translation is currently invisible" below for why that distinction +# Now simulate a real problem on one camera -- the shape genuinely changes (not just moves or +# zooms -- see "warp_mode: what counts as a real difference" below for why that distinction # matters), e.g. the pin/square is no longer there at all. import numpy as np dev.vision_cam_b.set_image(np.zeros((200, 200, 3), dtype=np.uint8)) @@ -84,35 +84,78 @@ A camera with nothing saved under a tag yet scores `inf`, not an error -- try e.g. from step 2/3 above -- a camera with no frame at all raises instead, since that's a real setup error, not "no references yet") to see this (expected state, not a bug). -## Known limitation, found while writing this howto: pure translation is currently invisible +## The shape score always ignores position -- `warp_mode` doesn't control that -Verified directly, not a hypothetical: shifting the square (as the very first draft of this howto's -step 4 did, `set_test_pattern(offset=(30, 0))`) does **not** raise the score at all -- -`best_reference_match()` scores it at `0.0`, identical to an unchanged frame. Reproduce: +Found while first writing this howto, and initially misdiagnosed: shifting the square +(`set_test_pattern(offset=(30, 0))`) did not raise the score at all -- `best_reference_match()` +scored it at `0.0`, identical to an unchanged frame. The first explanation reached for was +"registration is silently correcting the translation away" -- **verified false** by comparing scores +with registration fully disabled (`warp_mode="none"`) against the default: both come out the same, +near-zero. The real reason: `contour_similarity_score()` compares Hu moments +(`cv2.matchShapes`), which are *defined* to be invariant to translation, rotation, and scale -- +unconditionally, regardless of whether the frames were registered first. `warp_mode` genuinely +affects registration quality (see the small, verified effect below), but it does not and cannot +make the shape score position-sensitive -- that was never what it was doing. ```python from csaxs_bec.devices.vision import vision_toolkit as vt -current = vt.make_square_test_pattern(offset=(0, 0)) -shifted = vt.make_square_test_pattern(offset=(30, 0)) -vt.best_reference_match(current, [shifted]).best_score # 0.0, not "different" + +current = vt.make_square_test_pattern(size=100, square_side=20) +shifted = vt.make_square_test_pattern(size=100, square_side=20, offset=(15, 0)) + +vt.best_reference_match(current, [shifted], warp_mode="euclidean").best_score # ~0.0 +vt.best_reference_match(current, [shifted], warp_mode="none").best_score # ~0.0 too -- same ``` -Root cause: `register_to_reference()`'s ECC alignment has no bound on how much translation it will -correct -- it estimates and removes *any* translation between the two frames before the contour -comparison ever runs, so a same-shaped object that has simply moved is registered right back into -alignment and then, correctly by the scoring step's own logic, looks identical. This is exactly -backwards for OMNY's actual use case: registration is meant to compensate for small *camera* mount -drift, not to silently erase genuine *sample* position changes -- and "the pin isn't where it should -be" is one of the more important things a confirm gate needs to catch, not blind itself to. +What `warp_mode` *does* verifiably affect is a small rasterization-noise residual left over from an +uncorrected scale or rotation difference (not from translation, which introduces no such residual): -**Not fixed here** -- this needs a real design decision (e.g. cap `register_to_reference()`'s -tolerated translation to a small camera-drift-sized bound and treat anything larger as itself part -of the score, or run a supplementary un-registered comparison alongside the registered one) before -Phase 0 data collection can be trusted for drift-type defects specifically. Missing/deformed defects -(the `np.zeros(...)` case above, and `test_score_flags_a_camera_that_no_longer_matches_its_reference` -in `test_sample_vision_monitor.py`) are unaffected and already verified working. +```python +zoomed = vt.make_square_test_pattern(size=100, square_side=26) # same center, bigger square -## 5. A more realistic manual test: a real photo instead of a synthetic square +vt.best_reference_match(current, [zoomed], warp_mode="euclidean").best_score # small residual (> 0) +vt.best_reference_match(current, [zoomed], warp_mode="affine").best_score # ~0.0 -- cleaner +``` + +`WARP_MODES` (`vision_toolkit.py`): `"none"` (no registration), `"translation"`, `"euclidean"` (the +toolkit's default -- translation+rotation registered, scale not), `"affine"` (also registers scale, +plus shear/anisotropic scale as a side effect -- OpenCV's ECC has no uniform-scale-only mode). +Exposed at every level (`register_to_reference()`/`best_reference_match()`'s `warp_mode` argument, +`SampleVisionMonitor`'s `warp_mode` constructor argument + `set_warp_mode()`/`get_warp_mode()`) -- +still a legitimate, now-configurable choice (per Mirko's request), just not a fix for the +translation "problem" above, which turned out not to be a problem the shape score has in the first +place. + +## Making position actually matter: `position_tolerance_px` + +Since the shape score can never be made position-sensitive, and Mirko confirmed OMNY genuinely wants +position ignored while wanting the *option* for other future consumers to care, position is a +**separate, independent signal**: `vision_toolkit.centroid_distance()` (always compares *raw, +unregistered* contours -- registering first would remove the very displacement it measures) and +`SampleVisionMonitor`'s `position_tolerance_px` (in pixels; `None` by default -- OMNY's actual +desired setting -- meaning position isn't checked at all). + +```python +dev.vision_cam_a.set_test_pattern(offset=(0, 0)) +dev.vision_cam_b.set_test_pattern(offset=(0, 0)) +dev.vision_test_monitor.save_reference("position_test") + +dev.vision_test_monitor.get_position_tolerance_px() # None -- not checked, matches vision_test.yaml + +dev.vision_test_monitor.set_position_tolerance_px(5.0) +dev.vision_cam_b.set_test_pattern(offset=(20, 0)) # moved well past 5 px +result = dev.vision_test_monitor.score("position_test") +result["per_camera"]["vision_cam_b"]["position_within_tolerance"] # False +result["per_camera"]["vision_cam_b"]["score"] # still ~0.0 -- shape still matches +``` + +The shape score and the position check are deliberately independent axes in the returned dict -- +`score()` does not collapse them into one number. Combining them into an actual auto-confirm +decision (e.g. "match only if shape AND position are both fine") is a policy choice that belongs to +whatever calls `score()`, per the plan doc's staged rollout -- not something Layer 2 decides for the +caller. + +## A more realistic manual test: a real photo instead of a synthetic square `DummyVisionCamera.load_image_file(path)` reads an image file from disk (jpg/png/...) -- e.g. a phone photo of an actual gripper/pin, transferred to whatever machine the BEC session is running diff --git a/csaxs_bec/device_configs/simulated_omny/vision_test.yaml b/csaxs_bec/device_configs/simulated_omny/vision_test.yaml index e7384935..e485a9f1 100644 --- a/csaxs_bec/device_configs/simulated_omny/vision_test.yaml +++ b/csaxs_bec/device_configs/simulated_omny/vision_test.yaml @@ -49,10 +49,15 @@ vision_test_interface: - vision_test vision_test_monitor: - description: SampleVisionMonitor (Layer 2) scoring against vision_test_interface. + description: SampleVisionMonitor (Layer 2) scoring against vision_test_interface. warp_mode + left at the toolkit's conservative "euclidean" default here, not OMNY's actual desired + "affine" (translation+zoom both ignored) -- a real OMNY deployment config should set + warp_mode to affine explicitly. position_tolerance_px left unset (position not checked), + matching OMNY's actual desired behavior. deviceClass: csaxs_bec.devices.vision.sample_vision_monitor.SampleVisionMonitor deviceConfig: vision_interface_name: vision_test_interface + warp_mode: euclidean enabled: true onFailure: buffer readOnly: false diff --git a/csaxs_bec/devices/vision/sample_vision_monitor.py b/csaxs_bec/devices/vision/sample_vision_monitor.py index a50b4c0c..6a7d6af4 100644 --- a/csaxs_bec/devices/vision/sample_vision_monitor.py +++ b/csaxs_bec/devices/vision/sample_vision_monitor.py @@ -36,13 +36,24 @@ class SampleVisionMonitor(PSIDeviceBase): ("Reference corpus curation"), not resolved or implemented here. """ - USER_ACCESS = ["save_reference", "score", "reference_tags", "clear_reference"] + USER_ACCESS = [ + "save_reference", + "score", + "reference_tags", + "clear_reference", + "set_warp_mode", + "get_warp_mode", + "set_position_tolerance_px", + "get_position_tolerance_px", + ] def __init__( self, *, name: str, vision_interface_name: str, + warp_mode: str | int | None = "euclidean", + position_tolerance_px: float | None = None, prefix: str = "", scan_info: "ScanInfo | None" = None, **kwargs, @@ -54,12 +65,44 @@ class SampleVisionMonitor(PSIDeviceBase): vision_interface_name (str): Name of the ``VisionInterface`` device this monitor scores against, resolved through the device manager at call time (same lazy-lookup pattern as ``VisionInterface``'s own camera dependency). + warp_mode (str | int | None): Which drift between a reference and the current frame + ``score()``'s *shape* score should tolerate (register away before comparing + contours) rather than count toward the score -- one of ``vision_toolkit.WARP_MODES`` + ("none", "translation", "euclidean", "affine") or an already-resolved ``cv2`` + warp-mode int. Note this has no effect on whether the shape score itself is + position-sensitive -- it isn't, ever (see ``vision_toolkit.contour_similarity_score``'s + docstring) -- only on registration/rasterization quality. What's right here is + application-specific (see ``WARP_MODES``'s docstring) -- OMNY's sample-transfer + confirms want camera-zoom differences ignored too, i.e. "affine", not this + constructor default. Runtime-adjustable via :meth:`set_warp_mode`. + position_tolerance_px (float | None): If set, ``score()`` also computes each camera's + position (contour centroid) displacement from its best-matching *raw, unregistered* + reference -- the only actually position-sensitive signal this device can report, + independent of ``warp_mode`` -- and flags whether it's within this many pixels. + ``None`` (the default, and OMNY's actual desired setting): position is not checked + at all; a pin/gripper in a different spot is not, by itself, treated as a problem. + Runtime-adjustable via :meth:`set_position_tolerance_px`. """ super().__init__(name=name, prefix=prefix, scan_info=scan_info, **kwargs) self._vision_interface_name = vision_interface_name + self._warp_mode = warp_mode + self._position_tolerance_px = position_tolerance_px # tag -> camera_name -> list of reference frames. self._references: dict[str, dict[str, list[np.ndarray]]] = {} + def set_warp_mode(self, warp_mode: str | int | None) -> None: + vt.resolve_warp_mode(warp_mode) # raises clearly now, rather than at the next score() call + self._warp_mode = warp_mode + + def get_warp_mode(self) -> str | int | None: + return self._warp_mode + + def set_position_tolerance_px(self, position_tolerance_px: float | None) -> None: + self._position_tolerance_px = position_tolerance_px + + def get_position_tolerance_px(self) -> float | None: + return self._position_tolerance_px + def _resolve_vision_interface(self): if self.device_manager is None or not hasattr(self.device_manager, "devices"): raise SampleVisionMonitorError(f"{self.name}: device manager is not available.") @@ -102,16 +145,28 @@ class SampleVisionMonitor(PSIDeviceBase): worst_match: vt.MatchResult | None = None worst_frame: np.ndarray | None = None + check_position = self._position_tolerance_px is not None for camera_name in interface.camera_names: current_frame = interface.get_raw_frame(camera_name) match = vt.best_reference_match( - current_frame, per_camera_references.get(camera_name, []) + current_frame, + per_camera_references.get(camera_name, []), + warp_mode=self._warp_mode, + check_position=check_position, ) - per_camera_summary[camera_name] = { + camera_summary = { "score": match.best_score, "reference_count": len(per_camera_references.get(camera_name, [])), "converged": match.best.registration.converged if match.best is not None else None, } + if check_position: + position_distance_px = match.best.position_distance_px if match.best else None + camera_summary["position_distance_px"] = position_distance_px + camera_summary["position_within_tolerance"] = ( + position_distance_px is not None + and position_distance_px <= self._position_tolerance_px + ) + per_camera_summary[camera_name] = camera_summary if worst_match is None or match.best_score > worst_match.best_score: worst_camera_name = camera_name worst_match = match diff --git a/csaxs_bec/devices/vision/vision_toolkit.py b/csaxs_bec/devices/vision/vision_toolkit.py index a8a11d21..f1ed3be5 100644 --- a/csaxs_bec/devices/vision/vision_toolkit.py +++ b/csaxs_bec/devices/vision/vision_toolkit.py @@ -27,11 +27,19 @@ class RegistrationResult: @dataclass(frozen=True) class ReferenceMatch: - """Score of ``current_image`` against a single reference, from :func:`best_reference_match`.""" + """Score of ``current_image`` against a single reference, from :func:`best_reference_match`. + + ``contour_score`` (Hu-moment shape matching) is unconditionally invariant to translation, + rotation, and scale -- ``warp_mode`` cannot make it position-sensitive, it only affects + registration quality (see ``WARP_MODES``). ``position_distance_px`` is the separate, + deliberately position-*sensitive* signal for a caller that does care whether the object moved -- + ``None`` unless ``best_reference_match(..., check_position=True)`` was used. + """ reference_index: int contour_score: float registration: RegistrationResult + position_distance_px: float | None = None @dataclass(frozen=True) @@ -47,6 +55,50 @@ class MatchResult: return self.best.contour_score if self.best is not None else math.inf +WARP_MODES: dict[str, int | None] = { + "none": None, + "translation": cv2.MOTION_TRANSLATION, + "euclidean": cv2.MOTION_EUCLIDEAN, + "affine": cv2.MOTION_AFFINE, +} +"""Named registration tolerances for register_to_reference()/best_reference_match()'s warp_mode. + +Which of these is "correct" is application-specific, not a toolkit-level default to get right once +-- OMNY's own confirm gates want both translation and camera-zoom differences ignored (a pin/gripper +in a slightly different spot or at a slightly different scale should still score as a match), while +a different consumer of this same toolkit might care about exactly those differences. Callers +(SampleVisionMonitor's warp_mode constructor argument, currently) are expected to choose explicitly +rather than rely on best_reference_match()'s default: + +- "none": no registration at all -- compares the raw frames as-is, so any positional/scale + difference between reference and current counts toward the score. +- "translation": only translation is compensated (rotation/scale differences still count). +- "euclidean": translation + rotation compensated, scale still counts (the toolkit's own default -- + a conservative "camera mount drift only" tolerance, not OMNY-specific). +- "affine": translation + rotation + scale all compensated -- also tolerates shear and independent + x/y scaling as a side effect, since OpenCV's ECC has no dedicated similarity (uniform-scale-only) + motion model; usually fine for a small zoom difference, but more permissive than strictly needed. +""" + + +def resolve_warp_mode(warp_mode: str | int | None) -> int | None: + """Resolve a ``warp_mode`` argument to what ``register_to_reference()`` actually needs. + + Accepts either one of ``WARP_MODES``' friendly names (the only form a YAML device config can + express, since raw ``cv2`` constants aren't representable there) or an already-resolved ``cv2`` + warp-mode int / ``None``, passed through unchanged -- so device config and direct Python callers + both work the same way. + """ + if warp_mode is None or isinstance(warp_mode, int): + return warp_mode + try: + return WARP_MODES[warp_mode] + except KeyError: + raise ValueError( + f"Unknown warp_mode '{warp_mode}'; valid names are {sorted(WARP_MODES)}." + ) from None + + def make_square_test_pattern( size: int = 200, square_side: int = 60, offset: tuple[int, int] = (0, 0) ) -> np.ndarray: @@ -98,24 +150,34 @@ def detect_edges( def register_to_reference( reference: np.ndarray, current: np.ndarray, - warp_mode: int = cv2.MOTION_EUCLIDEAN, + warp_mode: str | int | None = "euclidean", max_iterations: int = 50, epsilon: float = 1e-4, ) -> RegistrationResult: - """Align ``current`` to ``reference`` (translation+rotation) to compensate for small camera drift. + """Align ``current`` to ``reference`` to compensate for drift between them, up to whatever + ``warp_mode`` tolerates (see ``WARP_MODES``) -- e.g. small camera-mount drift, or (with + ``warp_mode="affine"``) a slightly different camera zoom too. + + ``warp_mode="none"``/``None`` skips registration entirely and returns ``current`` unchanged + (identity warp, trivially "converged"). ``cv2.findTransformECC`` raises ``cv2.error`` outright if it fails to converge (e.g. a low-texture or badly mismatched pair) -- caught here rather than propagated, since a failed registration should fall back to comparing the unregistered frame, not abort the caller's confirm gate. """ + resolved_mode = resolve_warp_mode(warp_mode) + identity_warp = np.eye(2, 3, dtype=np.float32) + if resolved_mode is None: + return RegistrationResult(aligned_image=current, warp_matrix=identity_warp, converged=True) + reference_gray = to_grayscale(reference).astype(np.float32) current_gray = to_grayscale(current).astype(np.float32) - warp_matrix = np.eye(2, 3, dtype=np.float32) + warp_matrix = identity_warp criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, max_iterations, epsilon) try: _, warp_matrix = cv2.findTransformECC( - reference_gray, current_gray, warp_matrix, warp_mode, criteria + reference_gray, current_gray, warp_matrix, resolved_mode, criteria ) except cv2.error: return RegistrationResult(aligned_image=current, warp_matrix=warp_matrix, converged=False) @@ -136,25 +198,74 @@ def largest_contour(binary_image: np.ndarray) -> np.ndarray | None: def contour_similarity_score(contour_a: np.ndarray | None, contour_b: np.ndarray | None) -> float: - """Shape-distance between two contours (lower = more similar); +inf if either is missing.""" + """Shape-distance between two contours (lower = more similar); +inf if either is missing. + + Hu-moment-based (``cv2.matchShapes``) -- unconditionally invariant to translation, rotation, + and scale, by definition of what a Hu moment is. This holds regardless of registration/ + ``warp_mode``: a same-shaped object that has simply moved or been captured at a different scale + already scores as identical here, with or without registering the frames first. Verified + directly (not assumed): scoring completely unregistered frames gives the same result as scoring + registered ones, for a pure translation. For a caller that *does* want position to count, use + ``centroid_distance()``/``best_reference_match(..., check_position=True)`` instead -- a + deliberately position-sensitive signal, kept separate from this one on purpose. + """ if contour_a is None or contour_b is None: return math.inf return cv2.matchShapes(contour_a, contour_b, cv2.CONTOURS_MATCH_I1, 0.0) +def contour_centroid(contour: np.ndarray | None) -> tuple[float, float] | None: + """Centroid ``(x, y)`` in pixels of a contour, via image moments -- ``None`` if the contour is + missing or degenerate (zero area, e.g. a single point).""" + if contour is None: + return None + moments = cv2.moments(contour) + if moments["m00"] == 0: + return None + return (moments["m10"] / moments["m00"], moments["m01"] / moments["m00"]) + + +def centroid_distance(contour_a: np.ndarray | None, contour_b: np.ndarray | None) -> float: + """Euclidean pixel distance between two contours' centroids; +inf if either is unavailable. + + Unlike ``contour_similarity_score()`` (unconditionally position-invariant by construction), + this *is* sensitive to where the object is -- always call it with raw, unregistered contours; + registering one contour onto the other first would remove exactly the displacement this is + meant to detect. + """ + centroid_a = contour_centroid(contour_a) + centroid_b = contour_centroid(contour_b) + if centroid_a is None or centroid_b is None: + return math.inf + return math.hypot(centroid_a[0] - centroid_b[0], centroid_a[1] - centroid_b[1]) + + def best_reference_match( current_image: np.ndarray, reference_images: list[np.ndarray], edge_low_threshold: int = 50, edge_high_threshold: int = 150, + warp_mode: str | int | None = "euclidean", + check_position: bool = False, ) -> MatchResult: """Register+score ``current_image`` against every image in ``reference_images``. - Scores by contour shape (edge-based, so largely exposure-invariant) after registering each - reference to the current frame. Returns every per-reference match plus the best (lowest-score) - one -- an empty ``reference_images`` list yields ``MatchResult(best=None)``, not an error, since - "no references yet" is an expected state (see the plan's "reference corpus curation" open - question), not a caller mistake. + Scores by contour shape (edge-based, so largely exposure-invariant, and -- see + ``contour_similarity_score()`` -- unconditionally position/scale/rotation-invariant regardless + of ``warp_mode``) after registering each reference to the current frame (per ``warp_mode`` -- + see ``WARP_MODES``; registration mainly affects rasterization-noise cleanliness for a scale/ + rotation difference, not whether the shape score notices position at all). Returns every + per-reference match plus the best (lowest-score) one -- an empty ``reference_images`` list + yields ``MatchResult(best=None)``, not an error, since "no references yet" is an expected state + (see the plan's "reference corpus curation" open question), not a caller mistake. + + Args: + check_position (bool): if True, also compute each reference's position (centroid) + distance from the current frame's *raw, unregistered* contour -- the only signal in + this module that's actually sensitive to whether the object moved. Off by default: + OMNY explicitly wants position ignored (see ``WARP_MODES``' docstring); a caller that + does care about position should turn this on rather than expect ``warp_mode`` to + provide it. """ current_edges = detect_edges(current_image, edge_low_threshold, edge_high_threshold) current_contour = largest_contour(current_edges) @@ -164,14 +275,28 @@ def best_reference_match( # Warp the *reference* into the current frame (not the other way round) so # current_contour, computed once above from the unwarped live frame, is directly # comparable to reference_contour below without re-deriving it per reference. - registration = register_to_reference(current_image, reference_image) + registration = register_to_reference(current_image, reference_image, warp_mode=warp_mode) reference_edges = detect_edges( registration.aligned_image, edge_low_threshold, edge_high_threshold ) reference_contour = largest_contour(reference_edges) score = contour_similarity_score(current_contour, reference_contour) + + position_distance_px = None + if check_position: + raw_reference_edges = detect_edges( + reference_image, edge_low_threshold, edge_high_threshold + ) + raw_reference_contour = largest_contour(raw_reference_edges) + position_distance_px = centroid_distance(current_contour, raw_reference_contour) + matches.append( - ReferenceMatch(reference_index=index, contour_score=score, registration=registration) + ReferenceMatch( + reference_index=index, + contour_score=score, + registration=registration, + position_distance_px=position_distance_px, + ) ) best = min(matches, key=lambda match: match.contour_score) if matches else None diff --git a/tests/tests_devices/test_sample_vision_monitor.py b/tests/tests_devices/test_sample_vision_monitor.py index 48ac4fcf..ff0bd58b 100644 --- a/tests/tests_devices/test_sample_vision_monitor.py +++ b/tests/tests_devices/test_sample_vision_monitor.py @@ -27,7 +27,9 @@ def _square_image(size: int = 60, offset: int = 0) -> np.ndarray: return vt.make_square_test_pattern(size=size, square_side=20, offset=(offset, offset)) -def _make_monitor(camera_names=("cam_a", "cam_b"), images=None) -> SampleVisionMonitor: +def _make_monitor( + camera_names=("cam_a", "cam_b"), images=None, **monitor_kwargs +) -> SampleVisionMonitor: dm = DMMock() images = images or {} cameras = {name: FakeCamera(images.get(name)) for name in camera_names} @@ -38,7 +40,10 @@ def _make_monitor(camera_names=("cam_a", "cam_b"), images=None) -> SampleVisionM ) dm.devices["vision_interface_test"] = interface monitor = SampleVisionMonitor( - name="vision_monitor_test", vision_interface_name="vision_interface_test", device_manager=dm + name="vision_monitor_test", + vision_interface_name="vision_interface_test", + device_manager=dm, + **monitor_kwargs, ) dm.devices["vision_monitor_test"] = monitor monitor._cameras = cameras # convenience handle for tests to mutate live frames @@ -124,3 +129,85 @@ def test_save_reference_accumulates_multiple_references_per_tag(): result = monitor.score("tag_a") assert result["per_camera"]["cam_a"]["reference_count"] == 2 + + +def test_default_warp_mode_is_euclidean(): + monitor = _make_monitor() + assert monitor.get_warp_mode() == "euclidean" + + +def test_constructor_accepts_a_custom_warp_mode(): + monitor = _make_monitor(warp_mode="affine") + assert monitor.get_warp_mode() == "affine" + + +def test_set_warp_mode_rejects_unknown_name(): + monitor = _make_monitor() + with pytest.raises(ValueError): + monitor.set_warp_mode("sepia") + # A rejected change must not have taken effect. + assert monitor.get_warp_mode() == "euclidean" + + +def test_warp_mode_affects_scoring_of_a_zoomed_reference(): + # Same shape, different scale (simulating a slightly different camera zoom) -- euclidean mode + # (translation+rotation only) cannot compensate for this, affine mode can. + current = vt.make_square_test_pattern(size=100, square_side=20, offset=(0, 0)) + zoomed_reference = vt.make_square_test_pattern(size=100, square_side=26, offset=(0, 0)) + monitor = _make_monitor(images={"cam_a": current, "cam_b": current}, warp_mode="euclidean") + monitor._cameras["cam_a"].image = zoomed_reference + monitor._cameras["cam_b"].image = zoomed_reference + monitor.save_reference("zoom_tag") + for camera in monitor._cameras.values(): + camera.image = current + + euclidean_score = monitor.score("zoom_tag")["overall_score"] + + monitor.set_warp_mode("affine") + affine_score = monitor.score("zoom_tag")["overall_score"] + + assert affine_score < euclidean_score + + +def test_position_not_checked_by_default(): + image = _square_image() + monitor = _make_monitor(images={"cam_a": image, "cam_b": image}) + monitor.save_reference("tag_a") + + result = monitor.score("tag_a") + + assert "position_distance_px" not in result["per_camera"]["cam_a"] + + +def test_position_tolerance_flags_a_camera_that_has_moved_too_far(): + reference_image = _square_image(offset=0) + monitor = _make_monitor( + images={"cam_a": reference_image, "cam_b": reference_image}, position_tolerance_px=5.0 + ) + monitor.save_reference("tag_a") + + # cam_b's live frame is the same shape, just shifted well past the tolerance -- the shape + # score alone (still euclidean by default) would call this a match; position tolerance must + # catch it independently. + monitor._cameras["cam_b"].image = _square_image(offset=15) + + result = monitor.score("tag_a") + + assert result["per_camera"]["cam_a"]["position_within_tolerance"] is True + assert result["per_camera"]["cam_b"]["position_within_tolerance"] is False + assert result["per_camera"]["cam_b"]["position_distance_px"] > 5.0 + # The shape score itself must still report a match -- position and shape are independent axes. + assert result["per_camera"]["cam_b"]["score"] == pytest.approx(0.0, abs=1e-3) + + +def test_get_position_tolerance_px_default_is_none(): + monitor = _make_monitor() + assert monitor.get_position_tolerance_px() is None + + +def test_set_position_tolerance_px_updates_at_runtime(): + monitor = _make_monitor() + monitor.set_position_tolerance_px(2.5) + assert monitor.get_position_tolerance_px() == 2.5 + monitor.set_position_tolerance_px(None) + assert monitor.get_position_tolerance_px() is None diff --git a/tests/tests_devices/test_vision_toolkit.py b/tests/tests_devices/test_vision_toolkit.py index 3582ef5c..47bd6a46 100644 --- a/tests/tests_devices/test_vision_toolkit.py +++ b/tests/tests_devices/test_vision_toolkit.py @@ -76,6 +76,105 @@ def test_register_to_reference_reports_non_convergence_without_raising(): assert result.aligned_image is flat_b +def test_resolve_warp_mode_accepts_friendly_names(): + assert vt.resolve_warp_mode("none") is None + assert vt.resolve_warp_mode("euclidean") == cv2.MOTION_EUCLIDEAN + assert vt.resolve_warp_mode("affine") == cv2.MOTION_AFFINE + + +def test_resolve_warp_mode_passes_through_int_and_none(): + assert vt.resolve_warp_mode(None) is None + assert vt.resolve_warp_mode(cv2.MOTION_AFFINE) == cv2.MOTION_AFFINE + + +def test_resolve_warp_mode_rejects_unknown_name(): + with pytest.raises(ValueError): + vt.resolve_warp_mode("sepia") + + +def test_register_to_reference_none_mode_skips_registration(): + reference = _square_image(offset=(0, 0)) + current = _square_image(offset=(20, 0)) + result = vt.register_to_reference(reference, current, warp_mode="none") + assert result.converged + assert result.aligned_image is current + assert np.array_equal(result.warp_matrix, np.eye(2, 3, dtype=np.float32)) + + +def test_best_reference_match_shape_score_ignores_translation_regardless_of_warp_mode(): + # contour_similarity_score (Hu moments) is unconditionally position-invariant -- verify this + # holds with registration fully disabled too, not just with the default "euclidean" mode. + # (An earlier draft of this test asserted warp_mode="none" would make translation "count" -- + # it does not, because the invariance is inherent to the shape metric, not to registration.) + current = _square_image(offset=(0, 0)) + shifted_reference = _square_image(offset=(20, 0)) + euclidean_score = vt.best_reference_match( + current, [shifted_reference], warp_mode="euclidean" + ).best_score + none_score = vt.best_reference_match(current, [shifted_reference], warp_mode="none").best_score + assert euclidean_score == pytest.approx(0.0, abs=1e-6) + assert none_score == pytest.approx(0.0, abs=1e-6) + + +def test_best_reference_match_affine_mode_reduces_scale_rasterization_noise(): + # A genuine zoom difference leaves a small residual shape-score even though Hu moments are + # nominally scale-invariant, because the *rasterized* contour differs slightly at different + # pixel scales; "affine" registration corrects the scale before comparing and removes most of + # that residual. Small effect, not "euclidean fails to notice a real difference" -- both are + # already near the "matches" end of the scale. + current = vt.make_square_test_pattern(size=200, square_side=60, offset=(0, 0)) + zoomed_reference = vt.make_square_test_pattern(size=200, square_side=75, offset=(0, 0)) + euclidean_match = vt.best_reference_match(current, [zoomed_reference], warp_mode="euclidean") + affine_match = vt.best_reference_match(current, [zoomed_reference], warp_mode="affine") + assert affine_match.best_score < euclidean_match.best_score + assert affine_match.best_score == pytest.approx(0.0, abs=1e-2) + + +def test_contour_centroid_of_a_centered_square(): + image = vt.make_square_test_pattern(size=100, square_side=20, offset=(0, 0)) + contour = vt.largest_contour(vt.detect_edges(image)) + centroid = vt.contour_centroid(contour) + assert centroid == pytest.approx((50.0, 50.0), abs=1.0) + + +def test_contour_centroid_none_for_missing_contour(): + assert vt.contour_centroid(None) is None + + +def test_centroid_distance_reflects_actual_displacement(): + image = vt.make_square_test_pattern(size=100, square_side=20, offset=(0, 0)) + shifted = vt.make_square_test_pattern(size=100, square_side=20, offset=(30, 0)) + contour_a = vt.largest_contour(vt.detect_edges(image)) + contour_b = vt.largest_contour(vt.detect_edges(shifted)) + assert vt.centroid_distance(contour_a, contour_a) == pytest.approx(0.0, abs=1e-6) + assert vt.centroid_distance(contour_a, contour_b) == pytest.approx(30.0, abs=1.0) + + +def test_centroid_distance_infinite_if_either_contour_missing(): + image = vt.make_square_test_pattern(size=100, square_side=20) + contour = vt.largest_contour(vt.detect_edges(image)) + assert vt.centroid_distance(contour, None) == float("inf") + + +def test_best_reference_match_check_position_reports_raw_displacement(): + # check_position must measure displacement on the RAW (unregistered) frames -- if it measured + # post-registration contours instead, registration would remove the very displacement this is + # meant to detect, and the reported distance would collapse to ~0 regardless of warp_mode. + current = _square_image(offset=(0, 0)) + shifted_reference = _square_image(offset=(20, 0)) + match = vt.best_reference_match( + current, [shifted_reference], warp_mode="euclidean", check_position=True + ) + assert match.best.position_distance_px == pytest.approx(20.0, abs=1.0) + + +def test_best_reference_match_position_distance_is_none_when_not_requested(): + current = _square_image(offset=(0, 0)) + reference = _square_image(offset=(0, 0)) + match = vt.best_reference_match(current, [reference]) + assert match.best.position_distance_px is None + + def test_largest_contour_finds_the_square(): image = _square_image() edges = vt.detect_edges(image)