diff --git a/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/GRIPPER_LOCALIZATION_PLAN.md b/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/GRIPPER_LOCALIZATION_PLAN.md index bc2f93ed..51489df5 100644 --- a/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/GRIPPER_LOCALIZATION_PLAN.md +++ b/csaxs_bec/bec_ipython_client/plugins/omny/AI_docs/GRIPPER_LOCALIZATION_PLAN.md @@ -195,6 +195,57 @@ visual: dump `find_dark_holes()`'s raw (pre-ranking) candidate list for `get_2` photo and check by eye whether the true notch is even in the candidate set at all before assuming the shape-matching step is what's failing. +## Update 2026-09-08 (continued): resolved -- and a pivot to fiducial markers + +Mirko tested `get_2` (pin fully seated) directly: the notch genuinely doesn't exist as a feature +there -- it's not a detection failure, the gap physically closes when the pin is fully in. This +closes the open question above. It also answers "can the outer silhouette help instead" -- no: +the outer-shape approach hit the *same* background-bleed/glow-bridging problems (see the first +2026-09-08 update) regardless of pin state, so it's not a more robust fallback, just the same fight +fought again. + +**Decision: add physical fiducial markers to the gripper, rather than continuing to try to +recognize the gripper itself.** Mirko has no time to iterate on the physical hardware, so the +choice needs to be right the first time -- **ArUco markers were recommended over hand-drawn dots** +specifically for that reason: dots need custom, unverified geometry-decoding code written from +scratch (distance/angle between blobs) and have no protection against a false trigger (a scratch or +glare spot of similar size/spacing could spoof them); ArUco tags decode to a specific integer id via +a long-established, already-implemented algorithm (`cv2.aruco`, confirmed present in this +environment's OpenCV 5.0 build), so a scratch essentially never false-triggers one, and detection +returns the 4 corner points directly -- from a known physical tag size that's position, in-plane +rotation, *and* scale in one step, with no custom geometry code to get right. + +**Chosen spec**: `DICT_4X4_50`, marker id `0` for the left jaw, id `1` for the right jaw -- printed +as large as each jaw's flat outer face allows (bigger is more robust to glare/low local contrast, +don't minimize it), on matte (not glossy) material, mounted with a bit of white quiet-zone margin +left around the black border (don't cut flush to the marker edge), kept flat (no wrinkling/doming, +which would corrupt the corner-based pose estimate). Two tags, not one: redundancy if one is +occluded from some angle, and the distance between them becomes a free signal for jaw open/closed +state later. Generated marker images (`DICT_4X4_50`, ids 0/1, with a 15%-of-side white margin) were +produced and print-verified this session (round-tripped through `cv2.aruco` detection correctly, +including after aggressive downscale+blur simulating a small tag seen from a distance) -- see +`detect_aruco_markers()` below for how to regenerate them. + +**New Layer 0 primitive, already implemented and tested (not waiting on the physical markers)**: +`vision_toolkit.detect_aruco_markers(image, dictionary_name="DICT_4X4_50") -> list[ArucoDetection]` +(`ArucoDetection` = `marker_id` + `corners` + a `centroid` property). Tested against synthetically +generated marker images (`cv2.aruco.generateImageMarker`), not real gripper photos yet -- there are +none with real fiducials mounted. This is deliberately generic (any dictionary, any id) and doesn't +know anything about "gripper" or "jaw" -- that mapping (which id is which jaw, what offset from a +tag's centroid the pin-check region sits at) is a Layer 1/2 concern for once real marked captures +exist to calibrate against. + +**Next steps once the physical tags are mounted and a fresh capture exists**: +1. Confirm `detect_aruco_markers()` actually finds both tags on the real gripper (lighting/ + reflection/print-quality could differ from the synthetic round-trip test above). +2. Define the pin-search region in tag-relative coordinates (e.g. "N pixels below the midpoint of + the two tag centroids") instead of a fixed frame ROI -- this is what makes the pin-alignment + check work regardless of where the gripper is in frame, the original Phase-0 goal. +3. Re-evaluate `find_dark_holes()`/`detect_edges()` for the *pin* check specifically, now run + inside that small, reliably-anchored region rather than searched for across an unconstrained, + shifting frame -- the tools already validated this session, just no longer doing double duty as + the gripper locator too. + ## What this doesn't touch `VisionInterface`'s edge-threshold/blur tuning (this session's other change) and the `VisionTuning` diff --git a/csaxs_bec/devices/vision/vision_toolkit.py b/csaxs_bec/devices/vision/vision_toolkit.py index ba7cddfd..e0dea651 100644 --- a/csaxs_bec/devices/vision/vision_toolkit.py +++ b/csaxs_bec/devices/vision/vision_toolkit.py @@ -286,6 +286,65 @@ def find_dark_holes( return matches +@dataclass(frozen=True) +class ArucoDetection: + """One detected ArUco marker: its decoded id and its four corners, in image coordinates + (OpenCV's own corner order: top-left, top-right, bottom-right, bottom-left).""" + + marker_id: int + corners: np.ndarray # shape (4, 2) + + @property + def centroid(self) -> tuple[float, float]: + return (float(self.corners[:, 0].mean()), float(self.corners[:, 1].mean())) + + +def detect_aruco_markers( + image: np.ndarray, dictionary_name: str = "DICT_4X4_50" +) -> list[ArucoDetection]: + """Find every ArUco marker of the given dictionary in ``image``. + + Fiducial-marker detection (`cv2.aruco`) -- the recommended way to find a physical object + reliably regardless of what else is happening in the scene (background clutter, reflections, + a state-dependent feature like a gripper's notch disappearing when a pin is fully seated -- + see ``AI_docs/GRIPPER_LOCALIZATION_PLAN.md``'s fiducial-marker recommendation), rather than + trying to recognize the object itself. Each marker decodes to a specific integer id, so -- + unlike ``find_dark_holes()``/contour shape matching -- a scratch, glare spot, or other noise + essentially never false-triggers a detection: it would have to coincidentally decode to a + valid marker pattern, not just look vaguely similar in shape or area. + + Args: + image: The frame to search (any of this module's usual RGB/grayscale conventions). + dictionary_name: Name of a predefined ``cv2.aruco`` dictionary constant, e.g. + ``"DICT_4X4_50"`` (the toolkit's own default -- small tags, robust at low resolution, + plenty of ids for a handful of fiducials) or ``"DICT_5X5_100"``. Must match whatever + dictionary the physical markers were generated from -- detection against the wrong + dictionary finds nothing, not an error. + + Returns: + Every detected marker (id + corners), in no particular order -- an empty list, not an + error, if none were found (the same "not found is an expected state" convention as + :func:`find_dark_holes`/:func:`best_reference_match`). + + Raises: + ValueError: if ``dictionary_name`` isn't a real ``cv2.aruco`` dictionary constant. + """ + try: + dictionary_id = getattr(cv2.aruco, dictionary_name) + except AttributeError: + raise ValueError(f"Unknown ArUco dictionary '{dictionary_name}'.") from None + dictionary = cv2.aruco.getPredefinedDictionary(dictionary_id) + detector = cv2.aruco.ArucoDetector(dictionary, cv2.aruco.DetectorParameters()) + gray = to_grayscale(image) + corners, ids, _ = detector.detectMarkers(gray) + if ids is None: + return [] + return [ + ArucoDetection(marker_id=int(marker_id), corners=marker_corners.reshape(4, 2)) + for marker_corners, marker_id in zip(corners, ids.flatten()) + ] + + def contour_similarity_score(contour_a: np.ndarray | None, contour_b: np.ndarray | None) -> float: """Shape-distance between two contours (lower = more similar); +inf if either is missing. diff --git a/tests/tests_devices/test_vision_toolkit.py b/tests/tests_devices/test_vision_toolkit.py index 091b748f..e2843152 100644 --- a/tests/tests_devices/test_vision_toolkit.py +++ b/tests/tests_devices/test_vision_toolkit.py @@ -161,6 +161,57 @@ def test_find_dark_holes_excludes_the_unbounded_background(): assert x > 0 and y > 0 and x + w < 100 and y + h < 100 +def _aruco_marker_image(marker_id: int, side: int = 200, margin: int = 30) -> np.ndarray: + """A single ArUco marker (DICT_4X4_50) with a white quiet-zone margin, as a 3-channel RGB + image (matching this module's usual convention) -- the same shape a printed, photographed + marker would have.""" + dictionary = cv2.aruco.getPredefinedDictionary(cv2.aruco.DICT_4X4_50) + marker = cv2.aruco.generateImageMarker(dictionary, marker_id, side) + bordered = cv2.copyMakeBorder( + marker, margin, margin, margin, margin, cv2.BORDER_CONSTANT, value=255 + ) + return cv2.cvtColor(bordered, cv2.COLOR_GRAY2RGB) + + +def test_detect_aruco_markers_finds_a_generated_marker(): + image = _aruco_marker_image(marker_id=7) + detections = vt.detect_aruco_markers(image) + assert len(detections) == 1 + assert detections[0].marker_id == 7 + assert detections[0].corners.shape == (4, 2) + # Centroid should land near the marker's actual center (200x200 marker + 30px margin each + # side = 260x260 canvas, center at (130, 130)). + cx, cy = detections[0].centroid + assert abs(cx - 130) < 15 + assert abs(cy - 130) < 15 + + +def test_detect_aruco_markers_no_markers_returns_empty_list(): + image = np.zeros((100, 100, 3), dtype=np.uint8) + assert vt.detect_aruco_markers(image) == [] + + +def test_detect_aruco_markers_unknown_dictionary_raises(): + image = _aruco_marker_image(marker_id=0) + with pytest.raises(ValueError): + vt.detect_aruco_markers(image, dictionary_name="NOT_A_REAL_DICTIONARY") + + +def test_detect_aruco_markers_distinguishes_two_markers_by_id(): + left = _aruco_marker_image(marker_id=0, side=100, margin=10) # 120x120 + right = _aruco_marker_image(marker_id=1, side=100, margin=10) + canvas = np.full((140, 400, 3), 255, dtype=np.uint8) + canvas[10:130, 10:130] = left + canvas[10:130, 250:370] = right + + detections = vt.detect_aruco_markers(canvas) + found_ids = sorted(d.marker_id for d in detections) + assert found_ids == [0, 1] + + by_id = {d.marker_id: d for d in detections} + assert by_id[0].centroid[0] < by_id[1].centroid[0] # id 0 is the left one + + def test_register_to_reference_identity_for_identical_images(): image = _square_image() result = vt.register_to_reference(image, image)