diff --git a/dimos/perception/experimental/objectDB.py b/dimos/perception/experimental/objectDB.py index 1b29c171b7..99bbf5719f 100644 --- a/dimos/perception/experimental/objectDB.py +++ b/dimos/perception/experimental/objectDB.py @@ -84,25 +84,27 @@ def add_objects(self, objects: list[Object]) -> list[Object]: "matched_distance": 0, } - results: list[Object] = [] + results: dict[str, Object] = {} now = time.time() with self._lock: self._prune_stale_pending(now) for obj in objects: matched, reason = self._match(obj, now) if matched is None: - results.append(self._insert_pending(obj, now)) + inserted = self._insert_pending(obj, now) + results[inserted.object_id] = inserted stats["created"] += 1 continue - self._update_existing(matched, obj, now) - results.append(matched) - stats["updated"] += 1 + updated = self._update_existing(matched, obj, now) + results[matched.object_id] = matched + if updated: + stats["updated"] += 1 if reason == "track": stats["matched_track"] += 1 elif reason == "distance": stats["matched_distance"] += 1 - if self._check_promotion(matched): + if updated and self._check_promotion(matched): stats["promoted"] += 1 stats["pending"] = len(self._pending_objects) @@ -110,7 +112,7 @@ def add_objects(self, objects: list[Object]) -> list[Object]: self._last_add_stats = stats if stats["created"] > 0 or stats["promoted"] > 0: logger.info(f"ObjectDB: {stats}") - return results + return list(results.values()) def get_last_add_stats(self) -> dict[str, int]: with self._lock: @@ -218,17 +220,25 @@ def _match(self, obj: Object, now: float) -> tuple[Object | None, str | None]: def _insert_pending(self, obj: Object, now: float) -> Object: if not obj.ts: obj.ts = now + obj.last_seen_ts = now self._pending_objects[obj.object_id] = obj if obj.track_id >= 0: self._track_id_map[obj.track_id] = obj.object_id logger.info(f"Created new pending object {obj.object_id} ({obj.name})") return obj - def _update_existing(self, existing: Object, obj: Object, now: float) -> None: - existing.update_object(obj) - existing.ts = obj.ts or now + def _update_existing(self, existing: Object, obj: Object, now: float) -> bool: if obj.track_id >= 0: self._track_id_map[obj.track_id] = existing.object_id + # Multiple prompts or repeated scans may produce the same object from + # one camera frame. Only distinct source observations advance memory. + if existing.ts == obj.ts: + return False + + existing.update_object(obj) + existing.ts = obj.ts or now + existing.last_seen_ts = now + return True def _match_by_track_id(self, track_id: int, now: float) -> Object | None: """Find object with matching track_id from YOLOE.""" @@ -248,7 +258,7 @@ def _match_by_track_id(self, track_id: int, now: float) -> Object | None: del self._track_id_map[track_id] return None - last_seen = obj.ts if obj.ts else now + last_seen = obj.last_seen_ts if obj.last_seen_ts is not None else now if now - last_seen > self._track_id_ttl_s: del self._track_id_map[track_id] return None @@ -284,7 +294,9 @@ def _prune_stale_pending(self, now: float) -> None: return cutoff = now - self._pending_ttl_s stale_ids = [ - obj_id for obj_id, obj in self._pending_objects.items() if (obj.ts or now) < cutoff + obj_id + for obj_id, obj in self._pending_objects.items() + if (obj.last_seen_ts if obj.last_seen_ts is not None else now) < cutoff ] for obj_id in stale_ids: del self._pending_objects[obj_id] diff --git a/dimos/perception/experimental/object_scene_registration.py b/dimos/perception/experimental/object_scene_registration.py index 914301afab..d73d171dff 100644 --- a/dimos/perception/experimental/object_scene_registration.py +++ b/dimos/perception/experimental/object_scene_registration.py @@ -12,15 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import threading import time -from typing import Any +from typing import Any, Literal import numpy as np from numpy.typing import NDArray from dimos.agents.annotation import skill from dimos.core.core import rpc -from dimos.core.module import Module +from dimos.core.module import Module, ModuleConfig from dimos.core.stream import In, Out from dimos.msgs.geometry_msgs.Transform import Transform from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo @@ -46,8 +47,21 @@ logger = setup_logger() +class ObjectSceneRegistrationConfig(ModuleConfig): + target_frame: str = "base_link" + prompt_mode: YoloePromptMode = YoloePromptMode.LRPC + detector_backend: Literal["yoloe", "owlv2", "moondream"] = "yoloe" + segmentation_backend: Literal["yolo", "edgetam"] = "yolo" + detect_on_request: bool = False + distance_threshold: float = 0.2 + min_detections_for_permanent: int = 6 + max_distance: float = 0.0 + use_aabb: bool = False + max_obstacle_width: float = 0.0 + + class ObjectSceneRegistrationModule(Module): - """Module for detecting objects in camera images using YOLO-E with 2D and 3D detection.""" + """Register prompted camera detections as stable 3D scene objects.""" color_image: In[Image] depth_image: In[Image] @@ -59,50 +73,71 @@ class ObjectSceneRegistrationModule(Module): objects: Out[list[DetObject]] pointcloud: Out[PointCloud2] - _detector: Yoloe2DDetector | None = None + _detector: Any | None = None + _segmenter: Any | None = None _camera_info: CameraInfo | None = None _object_db: ObjectDB + _text_prompts: list[str] + _latest_aligned_frames: tuple[Image, Image] | None = None + _processing_lock: threading.RLock # A tuple assignment/read is atomic, so depth and its transform cannot be # observed from different frames by get_full_scene_pointcloud(). _latest_scene_snapshot: tuple[Image, Transform | None] | None = None - def __init__( - self, - target_frame: str = "map", - prompt_mode: YoloePromptMode = YoloePromptMode.LRPC, - # ObjectDB tuning - distance_threshold: float = 0.2, - min_detections_for_permanent: int = 6, - # Object 3D reconstruction tuning - max_distance: float = 0.0, - use_aabb: bool = False, - max_obstacle_width: float = 0.0, - **kwargs: Any, - ) -> None: + config: ObjectSceneRegistrationConfig + + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - self._target_frame = target_frame - self._prompt_mode = prompt_mode + self._target_frame = self.config.target_frame + self._prompt_mode = self.config.prompt_mode + self._detector_backend = self.config.detector_backend + self._segmentation_backend = self.config.segmentation_backend + self._detector_confidence = 0.6 + self._detect_on_request = self.config.detect_on_request self._object_db = ObjectDB( - distance_threshold=distance_threshold, - min_detections_for_permanent=min_detections_for_permanent, + distance_threshold=self.config.distance_threshold, + min_detections_for_permanent=self.config.min_detections_for_permanent, ) - self._max_distance = max_distance - self._use_aabb = use_aabb - self._max_obstacle_width = max_obstacle_width + self._processing_lock = threading.RLock() + self._text_prompts = [] + self._max_distance = self.config.max_distance + self._use_aabb = self.config.use_aabb + self._max_obstacle_width = self.config.max_obstacle_width @rpc def start(self) -> None: super().start() - if self._prompt_mode == YoloePromptMode.LRPC: - model_name = "yoloe-11l-seg-pf.pt" + if self._detector_backend == "owlv2": + from dimos.perception.detection.detectors.owlv2 import Owlv2Detector + + self._detector = Owlv2Detector() + self._detector.start() + elif self._detector_backend == "moondream": + from dimos.models.vl.moondream import MoondreamVlModel + + self._detector = MoondreamVlModel() + self._detector.start() else: - model_name = "yoloe-11l-seg.pt" + if self._prompt_mode == YoloePromptMode.LRPC: + model_name = "yoloe-11l-seg-pf.pt" + else: + model_name = "yoloe-11l-seg.pt" + self._detector = Yoloe2DDetector( + model_name=model_name, + prompt_mode=self._prompt_mode, + conf=self._detector_confidence, + ) - self._detector = Yoloe2DDetector( - model_name=model_name, - prompt_mode=self._prompt_mode, - ) + if self._segmentation_backend == "edgetam": + try: + from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter + + self._segmenter = EdgeTAMImageSegmenter() + except ModuleNotFoundError as e: + raise ModuleNotFoundError( + "EdgeTAM requires the optional dependencies from dimos[misc]" + ) from e self.camera_info.subscribe(lambda msg: setattr(self, "_camera_info", msg)) @@ -118,11 +153,14 @@ def start(self) -> None: def stop(self) -> None: """Stop the module and clean up resources.""" - if self._detector: - self._detector.stop() - self._detector = None + with self._processing_lock: + if self._detector: + self._detector.stop() + self._detector = None + self._segmenter = None - self._object_db.clear() + self._object_db.clear() + self._latest_aligned_frames = None logger.info("ObjectSceneRegistrationModule stopped") super().stop() @@ -134,9 +172,44 @@ def set_prompts( bboxes: NDArray[np.float64] | None = None, ) -> None: """Set prompts for detection. Provide either text or bboxes, not both.""" - if self._detector is not None: + with self._processing_lock: + self._set_prompts(text=text, bboxes=bboxes) + + def _set_prompts( + self, + text: list[str] | None = None, + bboxes: NDArray[np.float64] | None = None, + ) -> None: + if self._detector_backend in {"owlv2", "moondream"}: + if bboxes is not None: + raise ValueError(f"{self._detector_backend} supports text prompts only") + self._text_prompts = text or [] + elif self._detector is not None: self._detector.set_prompts(text=text, bboxes=bboxes) + @rpc + def scan_scene(self, text: list[str] | None = None) -> Detection3DArray: + """Run one serialized detection pass over the latest aligned RGB-D frame.""" + with self._processing_lock: + if text is not None: + self._set_prompts(text=text) + frames = self._latest_aligned_frames + if frames is None: + return to_detection3d_array([], frame_id=self._target_frame) + objects = self._scan_scene_objects(frames) + return to_detection3d_array( + objects, + frame_id=self._target_frame, + ts=frames[0].ts, + ) + + def _scan_scene_objects(self, frames: tuple[Image, Image] | None = None) -> list[DetObject]: + """Process one frame and return only objects observed during this scan.""" + frames = frames or self._latest_aligned_frames + if frames is None: + return [] + return self._process_images(*frames) + @rpc def select_object(self, track_id: int) -> dict[str, Any] | None: """Get object data by track_id and promote to permanent.""" @@ -245,31 +318,35 @@ def get_full_scene_pointcloud( return pc @skill - def detect(self, *prompts: str) -> str: + def detect(self, prompts: list[str]) -> str: """Detect objects matching the given text prompts. Do NOT call this tool multiple times for one query. Pass all objects in a single call. - For example, to detect a cup and mouse, call detect("cup", "mouse") not detect("cup") then detect("mouse"). + For example, to detect a cup and mouse, pass ["cup", "mouse"] in one call. Args: - prompts (str): Text descriptions of objects to detect (e.g., "person", "car", "dog") + prompts: Text descriptions of concrete object categories to detect. Returns: str: Detected objects with their object_id (stable UUID) and name. Example: - detect("person", "car", "dog") - detect("cup") + detect(["person", "car", "dog"]) + detect(["cup"]) """ if not prompts: return "No prompts provided." - if self._detector is None: - return "Detector not initialized." - - self._detector.set_prompts(text=list(prompts)) - time.sleep(2.0) - - detected = self.get_detected_objects() + with self._processing_lock: + if self._detector is None: + return "Detector not initialized." + self._set_prompts(text=prompts) + if self._detect_on_request: + detected = [obj.agent_encode() for obj in self._scan_scene_objects()] + else: + detected = None + if detected is None: + time.sleep(2.0) + detected = self.get_detected_objects() if not detected: return "No objects detected." @@ -290,12 +367,16 @@ def select(self, track_id: int) -> str: def _on_aligned_frames(self, frames) -> None: # type: ignore[no-untyped-def] color_msg, depth_msg = frames - self._process_images(color_msg, depth_msg) + self._latest_aligned_frames = (color_msg, depth_msg) + if self._detect_on_request: + return + with self._processing_lock: + self._process_images(color_msg, depth_msg) - def _process_images(self, color_msg: Image, depth_msg: Image) -> None: + def _process_images(self, color_msg: Image, depth_msg: Image) -> list[DetObject]: """Process synchronized color and depth images (runs in background thread).""" if not self._detector or not self._camera_info: - return + return [] color_image = color_msg # Convert depth to meters (float32) @@ -308,8 +389,29 @@ def _process_images(self, color_msg: Image, depth_msg: Image) -> None: data=depth_cv, format=ImageFormat.DEPTH, frame_id=depth_msg.frame_id, ts=depth_msg.ts ) - # Run 2D detection - detections_2d: ImageDetections2D[Any] = self._detector.process_image(color_image) + if self._detector_backend == "owlv2": + if not self._text_prompts: + detections_2d = ImageDetections2D(color_image, []) + else: + detections_2d = self._detector.query_detections( + color_image, + self._text_prompts, + threshold=self._detector_confidence, + ) + elif self._detector_backend == "moondream": + detections_2d = ImageDetections2D(color_image, []) + for class_id, prompt in enumerate(self._text_prompts): + prompted = self._detector.query_detections(color_image, prompt) + for detection in prompted.detections: + # Moondream's per-query indices are not temporal track IDs. + detection.track_id = -1 + detection.class_id = class_id + detections_2d.detections.extend(prompted.detections) + else: + detections_2d = self._detector.process_image(color_image) + + if self._segmenter is not None: + detections_2d = self._segmenter.segment(detections_2d) detections_2d_msg = Detection2DArray( detections_length=len(detections_2d.detections), @@ -319,17 +421,17 @@ def _process_images(self, color_msg: Image, depth_msg: Image) -> None: self.detections_2d.publish(detections_2d_msg) # Process 3D detections - self._process_3d_detections(detections_2d, color_image, depth_image) + return self._process_3d_detections(detections_2d, color_image, depth_image) def _process_3d_detections( self, detections_2d: ImageDetections2D[Any], color_image: Image, depth_image: Image, - ) -> None: + ) -> list[DetObject]: """Convert 2D detections to 3D and publish.""" if self._camera_info is None: - return + return [] # Look up transform from camera frame to target frame (e.g., map) camera_transform = None @@ -343,7 +445,7 @@ def _process_3d_detections( ) if camera_transform is None: logger.info("Failed to lookup transform from camera frame to target frame") - return + return [] # Cache depth and transform together, only after the lookup succeeds. self._latest_scene_snapshot = (depth_image, camera_transform) @@ -358,11 +460,11 @@ def _process_3d_detections( use_aabb=self._use_aabb, max_obstacle_width=self._max_obstacle_width, ) - if not objects: - return - # Add objects to spatial memory database - self._object_db.add_objects(objects) + # Empty observations still advance pending-object expiry. + observed_objects = self._object_db.add_objects(objects) + if not objects: + return [] # Publish ALL permanent objects so downstream consumers get the full set, # not just this frame's batch (which may be a subset of what's on the table). @@ -375,4 +477,4 @@ def _process_3d_detections( objects_for_pc = all_permanent aggregated_pc = aggregate_pointclouds(objects_for_pc) self.pointcloud.publish(aggregated_pc) - return + return observed_objects diff --git a/dimos/perception/experimental/object_scene_registration_spec.py b/dimos/perception/experimental/object_scene_registration_spec.py index 59aae79cab..5f0cbbf8e1 100644 --- a/dimos/perception/experimental/object_scene_registration_spec.py +++ b/dimos/perception/experimental/object_scene_registration_spec.py @@ -15,10 +15,13 @@ from typing import Protocol from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.msgs.vision_msgs.Detection3DArray import Detection3DArray from dimos.spec.utils import Spec class ObjectSceneRegistrationSpec(Spec, Protocol): + def set_prompts(self, text: list[str] | None = None) -> None: ... + def scan_scene(self, text: list[str] | None = None) -> Detection3DArray: ... def get_object_pointcloud_by_name(self, name: str) -> PointCloud2 | None: ... def get_object_pointcloud_by_object_id(self, object_id: str) -> PointCloud2 | None: ... def get_full_scene_pointcloud( diff --git a/dimos/perception/experimental/test_object_scene_registration_temporal.py b/dimos/perception/experimental/test_object_scene_registration_temporal.py index 99e79eb876..6e63c53b54 100644 --- a/dimos/perception/experimental/test_object_scene_registration_temporal.py +++ b/dimos/perception/experimental/test_object_scene_registration_temporal.py @@ -16,15 +16,18 @@ from collections.abc import Iterator import sys +from threading import Event, Thread from typing import Any -from unittest.mock import MagicMock +from unittest.mock import ANY, MagicMock, call import numpy as np import pytest from dimos.msgs.sensor_msgs.Image import Image, ImageFormat +from dimos.msgs.vision_msgs.Detection3DArray import Detection3DArray from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D from dimos.perception.experimental.object_scene_registration import ObjectSceneRegistrationModule +from dimos.perception.experimental.objectDB import ObjectDB class _FakeTF: @@ -143,3 +146,284 @@ def voxel_down_sample(self, voxel_size: float) -> _PointCloud: module.get_full_scene_pointcloud() result.transform.assert_called_once_with(transform) + + +def test_owlv2_queries_configured_prompts(monkeypatch: Any) -> None: + module = ObjectSceneRegistrationModule(detector_backend="owlv2") + module._camera_info = MagicMock() + module._detector = MagicMock() + module._text_prompts = ["mug"] + module.detections_2d = MagicMock() + color = Image( + data=np.zeros((2, 2, 3), dtype=np.uint8), + format=ImageFormat.BGR, + frame_id="camera", + ts=4.0, + ) + detections = ImageDetections2D(color, []) + module._detector.query_detections.return_value = detections + process_3d = MagicMock() + monkeypatch.setattr(module, "_process_3d_detections", process_3d) + + module._process_images(color, _image(4.0)) + + module._detector.query_detections.assert_called_once_with(color, ["mug"], threshold=0.6) + process_3d.assert_called_once_with(detections, color, ANY) + module.stop() + + +def test_moondream_queries_each_configured_prompt(monkeypatch: Any) -> None: + module = ObjectSceneRegistrationModule(detector_backend="moondream") + module._camera_info = MagicMock() + module._detector = MagicMock() + module._text_prompts = ["cup", "bottle"] + module.detections_2d = MagicMock() + color = Image( + data=np.zeros((2, 2, 3), dtype=np.uint8), + format=ImageFormat.BGR, + frame_id="camera", + ts=4.0, + ) + cup = MagicMock(track_id=0, class_id=-1) + bottle = MagicMock(track_id=0, class_id=-1) + module._detector.query_detections.side_effect = [ + ImageDetections2D(color, [cup]), + ImageDetections2D(color, [bottle]), + ] + process_3d = MagicMock() + monkeypatch.setattr(module, "_process_3d_detections", process_3d) + + module._process_images(color, _image(4.0)) + + assert module._detector.query_detections.call_args_list == [ + call(color, "cup"), + call(color, "bottle"), + ] + combined = process_3d.call_args.args[0] + assert combined.detections == [cup, bottle] + assert (cup.track_id, cup.class_id) == (-1, 0) + assert (bottle.track_id, bottle.class_id) == (-1, 1) + module.stop() + + +def test_edgetam_refines_detector_output(monkeypatch: Any) -> None: + module = ObjectSceneRegistrationModule(segmentation_backend="edgetam") + module._camera_info = MagicMock() + color = Image( + data=np.zeros((2, 2, 3), dtype=np.uint8), + format=ImageFormat.BGR, + frame_id="camera", + ts=4.0, + ) + raw_detections = ImageDetections2D(color, []) + segmented_detections = ImageDetections2D(color, []) + module._segmenter = MagicMock() + module._segmenter.segment.return_value = segmented_detections + module._detector = MagicMock() + module._detector.process_image.return_value = raw_detections + module.detections_2d = MagicMock() + process_3d = MagicMock() + monkeypatch.setattr(module, "_process_3d_detections", process_3d) + + module._process_images(color, _image(4.0)) + + module._segmenter.segment.assert_called_once_with(raw_detections) + process_3d.assert_called_once_with(segmented_detections, color, ANY) + module.stop() + + +def test_request_driven_scan_processes_latest_aligned_frame(monkeypatch: Any) -> None: + module = ObjectSceneRegistrationModule( + target_frame="camera", detector_backend="owlv2", detect_on_request=True + ) + module._object_db = MagicMock() + color = Image( + data=np.zeros((2, 2, 3), dtype=np.uint8), + format=ImageFormat.BGR, + frame_id="camera", + ts=4.0, + ) + depth = _image(4.0) + module._on_aligned_frames((color, depth)) + output = MagicMock() + result = MagicMock(spec=Detection3DArray) + + def process_images(got_color: Image, got_depth: Image) -> list[MagicMock]: + assert (got_color, got_depth) == (color, depth) + return [output] + + monkeypatch.setattr(module, "_process_images", process_images) + monkeypatch.setattr( + "dimos.perception.experimental.object_scene_registration.to_detection3d_array", + lambda objects, **kwargs: result, + ) + + assert module.scan_scene(text=["mug"]) is result + assert module._text_prompts == ["mug"] + module.stop() + + +def test_request_driven_detect_triggers_scan(monkeypatch: Any) -> None: + module = ObjectSceneRegistrationModule(detector_backend="owlv2", detect_on_request=True) + module._detector = MagicMock() + current_object = MagicMock() + current_object.agent_encode.return_value = {"name": "mug", "object_id": "current"} + scan_objects = MagicMock(return_value=[current_object]) + monkeypatch.setattr(module, "_scan_scene_objects", scan_objects) + monkeypatch.setattr(module, "get_detected_objects", lambda: pytest.fail("read stale database")) + + assert module.detect(["mug"]) == "Detected 1 object(s):\n - mug (object_id='current')" + assert module._text_prompts == ["mug"] + scan_objects.assert_called_once_with() + module.stop() + + +def test_scan_output_includes_pending_objects(monkeypatch: Any) -> None: + module = ObjectSceneRegistrationModule(target_frame="camera") + module._camera_info = MagicMock() + module._object_db = MagicMock() + pending = MagicMock() + permanent = MagicMock() + module._object_db.get_all_objects.return_value = [pending, permanent] + module._object_db.get_objects.return_value = [permanent] + module._object_db.add_objects.return_value = [pending] + module.detections_3d = MagicMock() + module.objects = MagicMock() + module.pointcloud = MagicMock() + converted_objects = [pending] + monkeypatch.setattr( + "dimos.perception.experimental.object_scene_registration.Object.from_2d_to_list", + lambda **_: converted_objects, + ) + monkeypatch.setattr( + "dimos.perception.experimental.object_scene_registration.to_detection3d_array", + MagicMock(), + ) + monkeypatch.setattr( + "dimos.perception.experimental.object_scene_registration.aggregate_pointclouds", + MagicMock(), + ) + + observed = ObjectSceneRegistrationModule._process_3d_detections( + module, + MagicMock(spec=ImageDetections2D), + _image(4.0), + _image(4.0), + ) + + assert observed == [pending] + + converted_objects.clear() + module._object_db.add_objects.reset_mock() + observed = ObjectSceneRegistrationModule._process_3d_detections( + module, + MagicMock(spec=ImageDetections2D), + _image(5.0), + _image(5.0), + ) + + assert observed == [] + module._object_db.add_objects.assert_called_once_with([]) + module.stop() + + +def test_concurrent_request_scans_do_not_overlap(monkeypatch: Any) -> None: + module = ObjectSceneRegistrationModule( + target_frame="camera", detector_backend="owlv2", detect_on_request=True + ) + module._object_db = MagicMock() + color = Image( + data=np.zeros((2, 2, 3), dtype=np.uint8), + format=ImageFormat.BGR, + frame_id="camera", + ts=4.0, + ) + module._on_aligned_frames((color, _image(4.0))) + first_started = Event() + release_first = Event() + second_started = Event() + seen_prompts: list[list[str]] = [] + + def process_images(_: Image, __: Image) -> list[MagicMock]: + seen_prompts.append(list(module._text_prompts)) + if len(seen_prompts) == 1: + first_started.set() + assert release_first.wait(timeout=1.0) + else: + second_started.set() + return [] + + monkeypatch.setattr(module, "_process_images", process_images) + monkeypatch.setattr( + "dimos.perception.experimental.object_scene_registration.to_detection3d_array", + MagicMock(), + ) + first = Thread(target=lambda: module.scan_scene(text=["cup"])) + second = Thread(target=lambda: module.scan_scene(text=["mug"])) + first.start() + assert first_started.wait(timeout=1.0) + second.start() + assert not second_started.wait(timeout=0.05) + release_first.set() + first.join(timeout=1.0) + second.join(timeout=1.0) + + assert not first.is_alive() + assert not second.is_alive() + assert seen_prompts == [["cup"], ["mug"]] + module.stop() + + +def test_object_db_uses_wall_clock_for_pending_ttl(monkeypatch: Any) -> None: + object_db = ObjectDB(pending_ttl_s=5.0) + detected = MagicMock() + detected.object_id = "stable-id" + detected.track_id = -1 + detected.ts = 4.0 # Hardware timestamps are relative to camera boot. + detected.last_seen_ts = None + detected.center = None + now = [1000.0] + monkeypatch.setattr("dimos.perception.experimental.objectDB.time.time", lambda: now[0]) + + object_db.add_objects([detected]) + assert detected.last_seen_ts == 1000.0 + now[0] = 1004.0 + object_db.add_objects([]) + assert object_db.find_by_object_id("stable-id") is detected + now[0] = 1006.0 + object_db.add_objects([]) + assert object_db.find_by_object_id("stable-id") is None + + +def test_object_db_counts_each_source_frame_once(monkeypatch: Any) -> None: + object_db = ObjectDB(min_detections_for_permanent=10) + now = [1000.0] + monkeypatch.setattr("dimos.perception.experimental.objectDB.time.time", lambda: now[0]) + + first = MagicMock( + object_id="first-id", + track_id=-1, + ts=4.0, + last_seen_ts=None, + detections_count=1, + ) + first.center = MagicMock() + duplicate = MagicMock(object_id="duplicate-id", track_id=-1, ts=4.0) + duplicate.center = MagicMock() + duplicate.center.distance.return_value = 0.0 + + observed = object_db.add_objects([first, duplicate]) + + assert observed == [first] + first.update_object.assert_not_called() + assert first.last_seen_ts == 1000.0 + + newer = MagicMock(object_id="newer-id", track_id=-1, ts=5.0) + newer.center = MagicMock() + newer.center.distance.return_value = 0.0 + first.update_object.side_effect = lambda _: setattr(first, "detections_count", 2) + now[0] = 1001.0 + + assert object_db.add_objects([newer]) == [first] + first.update_object.assert_called_once_with(newer) + assert first.last_seen_ts == 1001.0