From 1dafbba7a9c5223b4c2c9e72a8e1abfe27920ca3 Mon Sep 17 00:00:00 2001
From: Ruthwik <ruthwikdasyam@gmail.com>
Date: Wed, 26 Aug 2026 18:03:43 -0700
Subject: [PATCH 1/6] feat(perception): add configurable OSR scan backends

---
 .../experimental/object_scene_registration.py | 128 +++++++++++++-----
 .../object_scene_registration_spec.py         |   3 +
 ...test_object_scene_registration_temporal.py |  97 ++++++++++++-
 3 files changed, 193 insertions(+), 35 deletions(-)

diff --git a/dimos/perception/experimental/object_scene_registration.py b/dimos/perception/experimental/object_scene_registration.py
index 914301afab..6fd8bc54db 100644
--- a/dimos/perception/experimental/object_scene_registration.py
+++ b/dimos/perception/experimental/object_scene_registration.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 import time
-from typing import Any
+from typing import Any, Literal
 
 import numpy as np
 from numpy.typing import NDArray
 
 from dimos.agents.annotation import skill
 from dimos.core.core import rpc
-from dimos.core.module import Module
+from dimos.core.module import Module, ModuleConfig
 from dimos.core.stream import In, Out
 from dimos.msgs.geometry_msgs.Transform import Transform
 from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo
@@ -46,6 +46,20 @@
 logger = setup_logger()
 
 
+class ObjectSceneRegistrationConfig(ModuleConfig):
+    target_frame: str = "map"
+    prompt_mode: YoloePromptMode = YoloePromptMode.LRPC
+    distance_threshold: float = 0.2
+    min_detections_for_permanent: int = 6
+    detector_backend: Literal["yoloe", "owlv2"] = "yoloe"
+    segmentation_backend: Literal["yolo", "edgetam"] = "yolo"
+    detector_confidence: float = 0.6
+    detect_on_request: bool = False
+    max_distance: float = 0.0
+    use_aabb: bool = False
+    max_obstacle_width: float = 0.0
+
+
 class ObjectSceneRegistrationModule(Module):
     """Module for detecting objects in camera images using YOLO-E with 2D and 3D detection."""
 
@@ -59,50 +73,59 @@ class ObjectSceneRegistrationModule(Module):
     objects: Out[list[DetObject]]
     pointcloud: Out[PointCloud2]
 
-    _detector: Yoloe2DDetector | None = None
+    _detector: Any | None = None
+    _segmenter: Any | None = None
     _camera_info: CameraInfo | None = None
     _object_db: ObjectDB
+    _owlv2_prompts: list[str]
+    _latest_aligned_frames: tuple[Image, Image] | None = None
+    _latest_output_objects: tuple[DetObject, ...] = ()
     # A tuple assignment/read is atomic, so depth and its transform cannot be
     # observed from different frames by get_full_scene_pointcloud().
     _latest_scene_snapshot: tuple[Image, Transform | None] | None = None
 
-    def __init__(
-        self,
-        target_frame: str = "map",
-        prompt_mode: YoloePromptMode = YoloePromptMode.LRPC,
-        # ObjectDB tuning
-        distance_threshold: float = 0.2,
-        min_detections_for_permanent: int = 6,
-        # Object 3D reconstruction tuning
-        max_distance: float = 0.0,
-        use_aabb: bool = False,
-        max_obstacle_width: float = 0.0,
-        **kwargs: Any,
-    ) -> None:
+    config: ObjectSceneRegistrationConfig
+
+    def __init__(self, **kwargs: Any) -> None:
         super().__init__(**kwargs)
-        self._target_frame = target_frame
-        self._prompt_mode = prompt_mode
+        self._target_frame = self.config.target_frame
+        self._prompt_mode = self.config.prompt_mode
+        self._detector_backend = self.config.detector_backend
+        self._segmentation_backend = self.config.segmentation_backend
+        self._detector_confidence = self.config.detector_confidence
+        self._detect_on_request = self.config.detect_on_request
         self._object_db = ObjectDB(
-            distance_threshold=distance_threshold,
-            min_detections_for_permanent=min_detections_for_permanent,
+            distance_threshold=self.config.distance_threshold,
+            min_detections_for_permanent=self.config.min_detections_for_permanent,
         )
-        self._max_distance = max_distance
-        self._use_aabb = use_aabb
-        self._max_obstacle_width = max_obstacle_width
+        self._owlv2_prompts = []
+        self._max_distance = self.config.max_distance
+        self._use_aabb = self.config.use_aabb
+        self._max_obstacle_width = self.config.max_obstacle_width
 
     @rpc
     def start(self) -> None:
         super().start()
 
-        if self._prompt_mode == YoloePromptMode.LRPC:
-            model_name = "yoloe-11l-seg-pf.pt"
+        if self._detector_backend == "owlv2":
+            from dimos.perception.detection.detectors.owlv2 import Owlv2Detector
+
+            self._detector = Owlv2Detector()
         else:
-            model_name = "yoloe-11l-seg.pt"
+            if self._prompt_mode == YoloePromptMode.LRPC:
+                model_name = "yoloe-11l-seg-pf.pt"
+            else:
+                model_name = "yoloe-11l-seg.pt"
+            self._detector = Yoloe2DDetector(
+                model_name=model_name,
+                prompt_mode=self._prompt_mode,
+                conf=self._detector_confidence,
+            )
 
-        self._detector = Yoloe2DDetector(
-            model_name=model_name,
-            prompt_mode=self._prompt_mode,
-        )
+        if self._segmentation_backend == "edgetam":
+            from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter
+
+            self._segmenter = EdgeTAMImageSegmenter()
 
         self.camera_info.subscribe(lambda msg: setattr(self, "_camera_info", msg))
 
@@ -121,8 +144,11 @@ def stop(self) -> None:
         if self._detector:
             self._detector.stop()
             self._detector = None
+        self._segmenter = None
 
         self._object_db.clear()
+        self._latest_aligned_frames = None
+        self._latest_output_objects = ()
 
         logger.info("ObjectSceneRegistrationModule stopped")
         super().stop()
@@ -134,9 +160,26 @@ def set_prompts(
         bboxes: NDArray[np.float64] | None = None,
     ) -> None:
         """Set prompts for detection. Provide either text or bboxes, not both."""
-        if self._detector is not None:
+        if self._detector_backend == "owlv2":
+            if bboxes is not None:
+                raise ValueError("OWLv2 supports text prompts only")
+            self._owlv2_prompts = text or []
+        elif self._detector is not None:
             self._detector.set_prompts(text=text, bboxes=bboxes)
 
+    @rpc
+    def scan_scene(self) -> Detection3DArray:
+        frames = self._latest_aligned_frames
+        if frames is None:
+            return to_detection3d_array([], frame_id=self._target_frame)
+        self._latest_output_objects = ()
+        self._process_images(*frames)
+        return to_detection3d_array(
+            list(self._latest_output_objects),
+            frame_id=self._target_frame,
+            ts=frames[0].ts,
+        )
+
     @rpc
     def select_object(self, track_id: int) -> dict[str, Any] | None:
         """Get object data by track_id and promote to permanent."""
@@ -266,7 +309,7 @@ def detect(self, *prompts: str) -> str:
         if self._detector is None:
             return "Detector not initialized."
 
-        self._detector.set_prompts(text=list(prompts))
+        self.set_prompts(text=list(prompts))
         time.sleep(2.0)
 
         detected = self.get_detected_objects()
@@ -290,6 +333,9 @@ def select(self, track_id: int) -> str:
 
     def _on_aligned_frames(self, frames) -> None:  # type: ignore[no-untyped-def]
         color_msg, depth_msg = frames
+        self._latest_aligned_frames = (color_msg, depth_msg)
+        if self._detect_on_request:
+            return
         self._process_images(color_msg, depth_msg)
 
     def _process_images(self, color_msg: Image, depth_msg: Image) -> None:
@@ -308,8 +354,20 @@ def _process_images(self, color_msg: Image, depth_msg: Image) -> None:
             data=depth_cv, format=ImageFormat.DEPTH, frame_id=depth_msg.frame_id, ts=depth_msg.ts
         )
 
-        # Run 2D detection
-        detections_2d: ImageDetections2D[Any] = self._detector.process_image(color_image)
+        if self._detector_backend == "owlv2":
+            if not self._owlv2_prompts:
+                detections_2d = ImageDetections2D(color_image, [])
+            else:
+                detections_2d = self._detector.query_detections(
+                    color_image,
+                    self._owlv2_prompts,
+                    threshold=self._detector_confidence,
+                )
+        else:
+            detections_2d = self._detector.process_image(color_image)
+
+        if self._segmenter is not None:
+            detections_2d = self._segmenter.segment(detections_2d)
 
         detections_2d_msg = Detection2DArray(
             detections_length=len(detections_2d.detections),
@@ -359,6 +417,7 @@ def _process_3d_detections(
             max_obstacle_width=self._max_obstacle_width,
         )
         if not objects:
+            self._latest_output_objects = ()
             return
 
         # Add objects to spatial memory database
@@ -367,6 +426,7 @@ def _process_3d_detections(
         # Publish ALL permanent objects so downstream consumers get the full set,
         # not just this frame's batch (which may be a subset of what's on the table).
         all_permanent = self._object_db.get_objects()
+        self._latest_output_objects = tuple(all_permanent)
 
         detections_3d = to_detection3d_array(all_permanent)
         self.detections_3d.publish(detections_3d)
diff --git a/dimos/perception/experimental/object_scene_registration_spec.py b/dimos/perception/experimental/object_scene_registration_spec.py
index 59aae79cab..24a63e0335 100644
--- a/dimos/perception/experimental/object_scene_registration_spec.py
+++ b/dimos/perception/experimental/object_scene_registration_spec.py
@@ -15,10 +15,13 @@
 from typing import Protocol
 
 from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2
+from dimos.msgs.vision_msgs.Detection3DArray import Detection3DArray
 from dimos.spec.utils import Spec
 
 
 class ObjectSceneRegistrationSpec(Spec, Protocol):
+    def set_prompts(self, text: list[str] | None = None) -> None: ...
+    def scan_scene(self) -> Detection3DArray: ...
     def get_object_pointcloud_by_name(self, name: str) -> PointCloud2 | None: ...
     def get_object_pointcloud_by_object_id(self, object_id: str) -> PointCloud2 | None: ...
     def get_full_scene_pointcloud(
diff --git a/dimos/perception/experimental/test_object_scene_registration_temporal.py b/dimos/perception/experimental/test_object_scene_registration_temporal.py
index 99e79eb876..048e3d2f9f 100644
--- a/dimos/perception/experimental/test_object_scene_registration_temporal.py
+++ b/dimos/perception/experimental/test_object_scene_registration_temporal.py
@@ -17,14 +17,17 @@
 from collections.abc import Iterator
 import sys
 from typing import Any
-from unittest.mock import MagicMock
+from unittest.mock import ANY, MagicMock
 
 import numpy as np
 import pytest
 
 from dimos.msgs.sensor_msgs.Image import Image, ImageFormat
+from dimos.msgs.vision_msgs.Detection3DArray import Detection3DArray
 from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D
 from dimos.perception.experimental.object_scene_registration import ObjectSceneRegistrationModule
+from dimos.perception.experimental.object_scene_registration_spec import ObjectSceneRegistrationSpec
+from dimos.spec.utils import spec_annotation_compliance
 
 
 class _FakeTF:
@@ -143,3 +146,95 @@ def voxel_down_sample(self, voxel_size: float) -> _PointCloud:
 
     module.get_full_scene_pointcloud()
     result.transform.assert_called_once_with(transform)
+
+
+def test_osr_implements_request_driven_scan_spec(module: ObjectSceneRegistrationModule) -> None:
+    assert spec_annotation_compliance(module, ObjectSceneRegistrationSpec)
+
+
+def test_owlv2_prompts_are_text_only() -> None:
+    module = ObjectSceneRegistrationModule(detector_backend="owlv2")
+    try:
+        module.set_prompts(text=["mug"])
+        assert module._owlv2_prompts == ["mug"]
+        with pytest.raises(ValueError, match="text prompts"):
+            module.set_prompts(bboxes=np.zeros((1, 4)))
+    finally:
+        module.stop()
+
+
+def test_owlv2_queries_configured_prompts(monkeypatch: Any) -> None:
+    module = ObjectSceneRegistrationModule(detector_backend="owlv2")
+    module._camera_info = MagicMock()
+    module._detector = MagicMock()
+    module._owlv2_prompts = ["mug"]
+    module.detections_2d = MagicMock()
+    color = Image(
+        data=np.zeros((2, 2, 3), dtype=np.uint8),
+        format=ImageFormat.BGR,
+        frame_id="camera",
+        ts=4.0,
+    )
+    detections = ImageDetections2D(color, [])
+    module._detector.query_detections.return_value = detections
+    process_3d = MagicMock()
+    monkeypatch.setattr(module, "_process_3d_detections", process_3d)
+
+    module._process_images(color, _image(4.0))
+
+    module._detector.query_detections.assert_called_once_with(color, ["mug"], threshold=0.6)
+    process_3d.assert_called_once_with(detections, color, ANY)
+    module.stop()
+
+
+def test_edgetam_refines_detector_output(monkeypatch: Any) -> None:
+    module = ObjectSceneRegistrationModule(segmentation_backend="edgetam")
+    module._camera_info = MagicMock()
+    color = Image(
+        data=np.zeros((2, 2, 3), dtype=np.uint8),
+        format=ImageFormat.BGR,
+        frame_id="camera",
+        ts=4.0,
+    )
+    raw_detections = ImageDetections2D(color, [])
+    segmented_detections = ImageDetections2D(color, [])
+    module._segmenter = MagicMock()
+    module._segmenter.segment.return_value = segmented_detections
+    module._detector = MagicMock()
+    module._detector.process_image.return_value = raw_detections
+    module.detections_2d = MagicMock()
+    process_3d = MagicMock()
+    monkeypatch.setattr(module, "_process_3d_detections", process_3d)
+
+    module._process_images(color, _image(4.0))
+
+    module._segmenter.segment.assert_called_once_with(raw_detections)
+    process_3d.assert_called_once_with(segmented_detections, color, ANY)
+    module.stop()
+
+
+def test_request_driven_scan_processes_latest_aligned_frame(monkeypatch: Any) -> None:
+    module = ObjectSceneRegistrationModule(target_frame="camera", detect_on_request=True)
+    color = Image(
+        data=np.zeros((2, 2, 3), dtype=np.uint8),
+        format=ImageFormat.BGR,
+        frame_id="camera",
+        ts=4.0,
+    )
+    depth = _image(4.0)
+    module._on_aligned_frames((color, depth))
+    output = MagicMock()
+    result = MagicMock(spec=Detection3DArray)
+
+    def process_images(got_color: Image, got_depth: Image) -> None:
+        assert (got_color, got_depth) == (color, depth)
+        module._latest_output_objects = (output,)
+
+    monkeypatch.setattr(module, "_process_images", process_images)
+    monkeypatch.setattr(
+        "dimos.perception.experimental.object_scene_registration.to_detection3d_array",
+        lambda objects, **kwargs: result,
+    )
+
+    assert module.scan_scene() is result
+    module.stop()

From fd3a8b74285b14be1511a9129a29d295d5b2620d Mon Sep 17 00:00:00 2001
From: Ruthwik <ruthwikdasyam@gmail.com>
Date: Wed, 26 Aug 2026 18:24:08 -0700
Subject: [PATCH 2/6] fix(perception): return pending objects from OSR scans

---
 .../experimental/object_scene_registration.py |  7 ++-
 ...test_object_scene_registration_temporal.py | 48 +++++++++++++++++++
 2 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/dimos/perception/experimental/object_scene_registration.py b/dimos/perception/experimental/object_scene_registration.py
index 6fd8bc54db..9350abd9cd 100644
--- a/dimos/perception/experimental/object_scene_registration.py
+++ b/dimos/perception/experimental/object_scene_registration.py
@@ -310,7 +310,10 @@ def detect(self, *prompts: str) -> str:
             return "Detector not initialized."
 
         self.set_prompts(text=list(prompts))
-        time.sleep(2.0)
+        if self._detect_on_request:
+            self.scan_scene()
+        else:
+            time.sleep(2.0)
 
         detected = self.get_detected_objects()
         if not detected:
@@ -426,7 +429,7 @@ def _process_3d_detections(
         # Publish ALL permanent objects so downstream consumers get the full set,
         # not just this frame's batch (which may be a subset of what's on the table).
         all_permanent = self._object_db.get_objects()
-        self._latest_output_objects = tuple(all_permanent)
+        self._latest_output_objects = tuple(self._object_db.get_all_objects())
 
         detections_3d = to_detection3d_array(all_permanent)
         self.detections_3d.publish(detections_3d)
diff --git a/dimos/perception/experimental/test_object_scene_registration_temporal.py b/dimos/perception/experimental/test_object_scene_registration_temporal.py
index 048e3d2f9f..4be45ed690 100644
--- a/dimos/perception/experimental/test_object_scene_registration_temporal.py
+++ b/dimos/perception/experimental/test_object_scene_registration_temporal.py
@@ -238,3 +238,51 @@ def process_images(got_color: Image, got_depth: Image) -> None:
 
     assert module.scan_scene() is result
     module.stop()
+
+
+def test_request_driven_detect_triggers_scan(monkeypatch: Any) -> None:
+    module = ObjectSceneRegistrationModule(detector_backend="owlv2", detect_on_request=True)
+    module._detector = MagicMock()
+    scan_scene = MagicMock()
+    monkeypatch.setattr(module, "scan_scene", scan_scene)
+    monkeypatch.setattr(module, "get_detected_objects", lambda: [])
+
+    assert module.detect("mug") == "No objects detected."
+    assert module._owlv2_prompts == ["mug"]
+    scan_scene.assert_called_once_with()
+    module.stop()
+
+
+def test_scan_output_includes_pending_objects(monkeypatch: Any) -> None:
+    module = ObjectSceneRegistrationModule(target_frame="camera")
+    module._camera_info = MagicMock()
+    module._object_db = MagicMock()
+    pending = MagicMock()
+    permanent = MagicMock()
+    module._object_db.get_all_objects.return_value = [pending, permanent]
+    module._object_db.get_objects.return_value = [permanent]
+    module.detections_3d = MagicMock()
+    module.objects = MagicMock()
+    module.pointcloud = MagicMock()
+    monkeypatch.setattr(
+        "dimos.perception.experimental.object_scene_registration.Object.from_2d_to_list",
+        lambda **_: [pending],
+    )
+    monkeypatch.setattr(
+        "dimos.perception.experimental.object_scene_registration.to_detection3d_array",
+        MagicMock(),
+    )
+    monkeypatch.setattr(
+        "dimos.perception.experimental.object_scene_registration.aggregate_pointclouds",
+        MagicMock(),
+    )
+
+    ObjectSceneRegistrationModule._process_3d_detections(
+        module,
+        MagicMock(spec=ImageDetections2D),
+        _image(4.0),
+        _image(4.0),
+    )
+
+    assert module._latest_output_objects == (pending, permanent)
+    module.stop()

From bd779d30245edaaba8303ec0aa08caf4a98cf647 Mon Sep 17 00:00:00 2001
From: Ruthwik <ruthwikdasyam@gmail.com>
Date: Thu, 27 Aug 2026 16:16:14 -0700
Subject: [PATCH 3/6] fix(perception): serialize on-demand OSR scans

---
 dimos/perception/experimental/objectDB.py     |   8 +-
 .../experimental/object_scene_registration.py | 130 ++++++++++--------
 .../object_scene_registration_spec.py         |   2 +-
 ...test_object_scene_registration_temporal.py |  99 +++++++++++--
 4 files changed, 171 insertions(+), 68 deletions(-)

diff --git a/dimos/perception/experimental/objectDB.py b/dimos/perception/experimental/objectDB.py
index 1b29c171b7..9cc8519cdd 100644
--- a/dimos/perception/experimental/objectDB.py
+++ b/dimos/perception/experimental/objectDB.py
@@ -218,6 +218,7 @@ def _match(self, obj: Object, now: float) -> tuple[Object | None, str | None]:
     def _insert_pending(self, obj: Object, now: float) -> Object:
         if not obj.ts:
             obj.ts = now
+        obj.last_seen_ts = now
         self._pending_objects[obj.object_id] = obj
         if obj.track_id >= 0:
             self._track_id_map[obj.track_id] = obj.object_id
@@ -227,6 +228,7 @@ def _insert_pending(self, obj: Object, now: float) -> Object:
     def _update_existing(self, existing: Object, obj: Object, now: float) -> None:
         existing.update_object(obj)
         existing.ts = obj.ts or now
+        existing.last_seen_ts = now
         if obj.track_id >= 0:
             self._track_id_map[obj.track_id] = existing.object_id
 
@@ -248,7 +250,7 @@ def _match_by_track_id(self, track_id: int, now: float) -> Object | None:
             del self._track_id_map[track_id]
             return None
 
-        last_seen = obj.ts if obj.ts else now
+        last_seen = obj.last_seen_ts if obj.last_seen_ts is not None else now
         if now - last_seen > self._track_id_ttl_s:
             del self._track_id_map[track_id]
             return None
@@ -284,7 +286,9 @@ def _prune_stale_pending(self, now: float) -> None:
             return
         cutoff = now - self._pending_ttl_s
         stale_ids = [
-            obj_id for obj_id, obj in self._pending_objects.items() if (obj.ts or now) < cutoff
+            obj_id
+            for obj_id, obj in self._pending_objects.items()
+            if (obj.last_seen_ts if obj.last_seen_ts is not None else now) < cutoff
         ]
         for obj_id in stale_ids:
             del self._pending_objects[obj_id]
diff --git a/dimos/perception/experimental/object_scene_registration.py b/dimos/perception/experimental/object_scene_registration.py
index 9350abd9cd..29fa6ea5da 100644
--- a/dimos/perception/experimental/object_scene_registration.py
+++ b/dimos/perception/experimental/object_scene_registration.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import threading
 import time
 from typing import Any, Literal
 
@@ -47,17 +48,10 @@
 
 
 class ObjectSceneRegistrationConfig(ModuleConfig):
-    target_frame: str = "map"
     prompt_mode: YoloePromptMode = YoloePromptMode.LRPC
-    distance_threshold: float = 0.2
-    min_detections_for_permanent: int = 6
     detector_backend: Literal["yoloe", "owlv2"] = "yoloe"
     segmentation_backend: Literal["yolo", "edgetam"] = "yolo"
-    detector_confidence: float = 0.6
     detect_on_request: bool = False
-    max_distance: float = 0.0
-    use_aabb: bool = False
-    max_obstacle_width: float = 0.0
 
 
 class ObjectSceneRegistrationModule(Module):
@@ -79,29 +73,39 @@ class ObjectSceneRegistrationModule(Module):
     _object_db: ObjectDB
     _owlv2_prompts: list[str]
     _latest_aligned_frames: tuple[Image, Image] | None = None
-    _latest_output_objects: tuple[DetObject, ...] = ()
+    _processing_lock: threading.RLock
     # A tuple assignment/read is atomic, so depth and its transform cannot be
     # observed from different frames by get_full_scene_pointcloud().
     _latest_scene_snapshot: tuple[Image, Transform | None] | None = None
 
     config: ObjectSceneRegistrationConfig
 
-    def __init__(self, **kwargs: Any) -> None:
+    def __init__(
+        self,
+        target_frame: str = "map",
+        distance_threshold: float = 0.2,
+        min_detections_for_permanent: int = 6,
+        max_distance: float = 0.0,
+        use_aabb: bool = False,
+        max_obstacle_width: float = 0.0,
+        **kwargs: Any,
+    ) -> None:
         super().__init__(**kwargs)
-        self._target_frame = self.config.target_frame
+        self._target_frame = target_frame
         self._prompt_mode = self.config.prompt_mode
         self._detector_backend = self.config.detector_backend
         self._segmentation_backend = self.config.segmentation_backend
-        self._detector_confidence = self.config.detector_confidence
+        self._detector_confidence = 0.6
         self._detect_on_request = self.config.detect_on_request
         self._object_db = ObjectDB(
-            distance_threshold=self.config.distance_threshold,
-            min_detections_for_permanent=self.config.min_detections_for_permanent,
+            distance_threshold=distance_threshold,
+            min_detections_for_permanent=min_detections_for_permanent,
         )
+        self._processing_lock = threading.RLock()
         self._owlv2_prompts = []
-        self._max_distance = self.config.max_distance
-        self._use_aabb = self.config.use_aabb
-        self._max_obstacle_width = self.config.max_obstacle_width
+        self._max_distance = max_distance
+        self._use_aabb = use_aabb
+        self._max_obstacle_width = max_obstacle_width
 
     @rpc
     def start(self) -> None:
@@ -141,14 +145,14 @@ def start(self) -> None:
     def stop(self) -> None:
         """Stop the module and clean up resources."""
 
-        if self._detector:
-            self._detector.stop()
-            self._detector = None
-        self._segmenter = None
+        with self._processing_lock:
+            if self._detector:
+                self._detector.stop()
+                self._detector = None
+            self._segmenter = None
 
-        self._object_db.clear()
-        self._latest_aligned_frames = None
-        self._latest_output_objects = ()
+            self._object_db.clear()
+            self._latest_aligned_frames = None
 
         logger.info("ObjectSceneRegistrationModule stopped")
         super().stop()
@@ -160,6 +164,14 @@ def set_prompts(
         bboxes: NDArray[np.float64] | None = None,
     ) -> None:
         """Set prompts for detection. Provide either text or bboxes, not both."""
+        with self._processing_lock:
+            self._set_prompts(text=text, bboxes=bboxes)
+
+    def _set_prompts(
+        self,
+        text: list[str] | None = None,
+        bboxes: NDArray[np.float64] | None = None,
+    ) -> None:
         if self._detector_backend == "owlv2":
             if bboxes is not None:
                 raise ValueError("OWLv2 supports text prompts only")
@@ -168,17 +180,27 @@ def set_prompts(
             self._detector.set_prompts(text=text, bboxes=bboxes)
 
     @rpc
-    def scan_scene(self) -> Detection3DArray:
-        frames = self._latest_aligned_frames
+    def scan_scene(self, text: list[str] | None = None) -> Detection3DArray:
+        """Run one serialized detection pass over the latest aligned RGB-D frame."""
+        with self._processing_lock:
+            if text is not None:
+                self._set_prompts(text=text)
+            frames = self._latest_aligned_frames
+            if frames is None:
+                return to_detection3d_array([], frame_id=self._target_frame)
+            objects = self._scan_scene_objects(frames)
+            return to_detection3d_array(
+                objects,
+                frame_id=self._target_frame,
+                ts=frames[0].ts,
+            )
+
+    def _scan_scene_objects(self, frames: tuple[Image, Image] | None = None) -> list[DetObject]:
+        """Process one frame and return only objects observed during this scan."""
+        frames = frames or self._latest_aligned_frames
         if frames is None:
-            return to_detection3d_array([], frame_id=self._target_frame)
-        self._latest_output_objects = ()
-        self._process_images(*frames)
-        return to_detection3d_array(
-            list(self._latest_output_objects),
-            frame_id=self._target_frame,
-            ts=frames[0].ts,
-        )
+            return []
+        return self._process_images(*frames)
 
     @rpc
     def select_object(self, track_id: int) -> dict[str, Any] | None:
@@ -306,16 +328,17 @@ def detect(self, *prompts: str) -> str:
         """
         if not prompts:
             return "No prompts provided."
-        if self._detector is None:
-            return "Detector not initialized."
-
-        self.set_prompts(text=list(prompts))
-        if self._detect_on_request:
-            self.scan_scene()
-        else:
+        with self._processing_lock:
+            if self._detector is None:
+                return "Detector not initialized."
+            self._set_prompts(text=list(prompts))
+            if self._detect_on_request:
+                detected = [obj.agent_encode() for obj in self._scan_scene_objects()]
+            else:
+                detected = None
+        if detected is None:
             time.sleep(2.0)
-
-        detected = self.get_detected_objects()
+            detected = self.get_detected_objects()
         if not detected:
             return "No objects detected."
 
@@ -339,12 +362,13 @@ def _on_aligned_frames(self, frames) -> None:  # type: ignore[no-untyped-def]
         self._latest_aligned_frames = (color_msg, depth_msg)
         if self._detect_on_request:
             return
-        self._process_images(color_msg, depth_msg)
+        with self._processing_lock:
+            self._process_images(color_msg, depth_msg)
 
-    def _process_images(self, color_msg: Image, depth_msg: Image) -> None:
+    def _process_images(self, color_msg: Image, depth_msg: Image) -> list[DetObject]:
         """Process synchronized color and depth images (runs in background thread)."""
         if not self._detector or not self._camera_info:
-            return
+            return []
 
         color_image = color_msg
         # Convert depth to meters (float32)
@@ -380,17 +404,17 @@ def _process_images(self, color_msg: Image, depth_msg: Image) -> None:
         self.detections_2d.publish(detections_2d_msg)
 
         # Process 3D detections
-        self._process_3d_detections(detections_2d, color_image, depth_image)
+        return self._process_3d_detections(detections_2d, color_image, depth_image)
 
     def _process_3d_detections(
         self,
         detections_2d: ImageDetections2D[Any],
         color_image: Image,
         depth_image: Image,
-    ) -> None:
+    ) -> list[DetObject]:
         """Convert 2D detections to 3D and publish."""
         if self._camera_info is None:
-            return
+            return []
 
         # Look up transform from camera frame to target frame (e.g., map)
         camera_transform = None
@@ -404,7 +428,7 @@ def _process_3d_detections(
             )
             if camera_transform is None:
                 logger.info("Failed to lookup transform from camera frame to target frame")
-                return
+                return []
 
         # Cache depth and transform together, only after the lookup succeeds.
         self._latest_scene_snapshot = (depth_image, camera_transform)
@@ -420,16 +444,14 @@ def _process_3d_detections(
             max_obstacle_width=self._max_obstacle_width,
         )
         if not objects:
-            self._latest_output_objects = ()
-            return
+            return []
 
         # Add objects to spatial memory database
-        self._object_db.add_objects(objects)
+        observed_objects = self._object_db.add_objects(objects)
 
         # Publish ALL permanent objects so downstream consumers get the full set,
         # not just this frame's batch (which may be a subset of what's on the table).
         all_permanent = self._object_db.get_objects()
-        self._latest_output_objects = tuple(self._object_db.get_all_objects())
 
         detections_3d = to_detection3d_array(all_permanent)
         self.detections_3d.publish(detections_3d)
@@ -438,4 +460,4 @@ def _process_3d_detections(
         objects_for_pc = all_permanent
         aggregated_pc = aggregate_pointclouds(objects_for_pc)
         self.pointcloud.publish(aggregated_pc)
-        return
+        return observed_objects
diff --git a/dimos/perception/experimental/object_scene_registration_spec.py b/dimos/perception/experimental/object_scene_registration_spec.py
index 24a63e0335..5f0cbbf8e1 100644
--- a/dimos/perception/experimental/object_scene_registration_spec.py
+++ b/dimos/perception/experimental/object_scene_registration_spec.py
@@ -21,7 +21,7 @@
 
 class ObjectSceneRegistrationSpec(Spec, Protocol):
     def set_prompts(self, text: list[str] | None = None) -> None: ...
-    def scan_scene(self) -> Detection3DArray: ...
+    def scan_scene(self, text: list[str] | None = None) -> Detection3DArray: ...
     def get_object_pointcloud_by_name(self, name: str) -> PointCloud2 | None: ...
     def get_object_pointcloud_by_object_id(self, object_id: str) -> PointCloud2 | None: ...
     def get_full_scene_pointcloud(
diff --git a/dimos/perception/experimental/test_object_scene_registration_temporal.py b/dimos/perception/experimental/test_object_scene_registration_temporal.py
index 4be45ed690..4ccd850ebb 100644
--- a/dimos/perception/experimental/test_object_scene_registration_temporal.py
+++ b/dimos/perception/experimental/test_object_scene_registration_temporal.py
@@ -16,6 +16,7 @@
 
 from collections.abc import Iterator
 import sys
+from threading import Event, Thread
 from typing import Any
 from unittest.mock import ANY, MagicMock
 
@@ -27,6 +28,7 @@
 from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D
 from dimos.perception.experimental.object_scene_registration import ObjectSceneRegistrationModule
 from dimos.perception.experimental.object_scene_registration_spec import ObjectSceneRegistrationSpec
+from dimos.perception.experimental.objectDB import ObjectDB
 from dimos.spec.utils import spec_annotation_compliance
 
 
@@ -214,7 +216,10 @@ def test_edgetam_refines_detector_output(monkeypatch: Any) -> None:
 
 
 def test_request_driven_scan_processes_latest_aligned_frame(monkeypatch: Any) -> None:
-    module = ObjectSceneRegistrationModule(target_frame="camera", detect_on_request=True)
+    module = ObjectSceneRegistrationModule(
+        target_frame="camera", detector_backend="owlv2", detect_on_request=True
+    )
+    module._object_db = MagicMock()
     color = Image(
         data=np.zeros((2, 2, 3), dtype=np.uint8),
         format=ImageFormat.BGR,
@@ -226,9 +231,9 @@ def test_request_driven_scan_processes_latest_aligned_frame(monkeypatch: Any) ->
     output = MagicMock()
     result = MagicMock(spec=Detection3DArray)
 
-    def process_images(got_color: Image, got_depth: Image) -> None:
+    def process_images(got_color: Image, got_depth: Image) -> list[MagicMock]:
         assert (got_color, got_depth) == (color, depth)
-        module._latest_output_objects = (output,)
+        return [output]
 
     monkeypatch.setattr(module, "_process_images", process_images)
     monkeypatch.setattr(
@@ -236,20 +241,23 @@ def process_images(got_color: Image, got_depth: Image) -> None:
         lambda objects, **kwargs: result,
     )
 
-    assert module.scan_scene() is result
+    assert module.scan_scene(text=["mug"]) is result
+    assert module._owlv2_prompts == ["mug"]
     module.stop()
 
 
 def test_request_driven_detect_triggers_scan(monkeypatch: Any) -> None:
     module = ObjectSceneRegistrationModule(detector_backend="owlv2", detect_on_request=True)
     module._detector = MagicMock()
-    scan_scene = MagicMock()
-    monkeypatch.setattr(module, "scan_scene", scan_scene)
-    monkeypatch.setattr(module, "get_detected_objects", lambda: [])
+    current_object = MagicMock()
+    current_object.agent_encode.return_value = {"name": "mug", "object_id": "current"}
+    scan_objects = MagicMock(return_value=[current_object])
+    monkeypatch.setattr(module, "_scan_scene_objects", scan_objects)
+    monkeypatch.setattr(module, "get_detected_objects", lambda: pytest.fail("read stale database"))
 
-    assert module.detect("mug") == "No objects detected."
+    assert module.detect("mug") == "Detected 1 object(s):\n  - mug (object_id='current')"
     assert module._owlv2_prompts == ["mug"]
-    scan_scene.assert_called_once_with()
+    scan_objects.assert_called_once_with()
     module.stop()
 
 
@@ -261,6 +269,7 @@ def test_scan_output_includes_pending_objects(monkeypatch: Any) -> None:
     permanent = MagicMock()
     module._object_db.get_all_objects.return_value = [pending, permanent]
     module._object_db.get_objects.return_value = [permanent]
+    module._object_db.add_objects.return_value = [pending]
     module.detections_3d = MagicMock()
     module.objects = MagicMock()
     module.pointcloud = MagicMock()
@@ -277,12 +286,80 @@ def test_scan_output_includes_pending_objects(monkeypatch: Any) -> None:
         MagicMock(),
     )
 
-    ObjectSceneRegistrationModule._process_3d_detections(
+    observed = ObjectSceneRegistrationModule._process_3d_detections(
         module,
         MagicMock(spec=ImageDetections2D),
         _image(4.0),
         _image(4.0),
     )
 
-    assert module._latest_output_objects == (pending, permanent)
+    assert observed == [pending]
     module.stop()
+
+
+def test_concurrent_request_scans_do_not_overlap(monkeypatch: Any) -> None:
+    module = ObjectSceneRegistrationModule(
+        target_frame="camera", detector_backend="owlv2", detect_on_request=True
+    )
+    module._object_db = MagicMock()
+    color = Image(
+        data=np.zeros((2, 2, 3), dtype=np.uint8),
+        format=ImageFormat.BGR,
+        frame_id="camera",
+        ts=4.0,
+    )
+    module._on_aligned_frames((color, _image(4.0)))
+    first_started = Event()
+    release_first = Event()
+    second_started = Event()
+    seen_prompts: list[list[str]] = []
+
+    def process_images(_: Image, __: Image) -> list[MagicMock]:
+        seen_prompts.append(list(module._owlv2_prompts))
+        if len(seen_prompts) == 1:
+            first_started.set()
+            assert release_first.wait(timeout=1.0)
+        else:
+            second_started.set()
+        return []
+
+    monkeypatch.setattr(module, "_process_images", process_images)
+    monkeypatch.setattr(
+        "dimos.perception.experimental.object_scene_registration.to_detection3d_array",
+        MagicMock(),
+    )
+    first = Thread(target=lambda: module.scan_scene(text=["cup"]))
+    second = Thread(target=lambda: module.scan_scene(text=["mug"]))
+    first.start()
+    assert first_started.wait(timeout=1.0)
+    second.start()
+    assert not second_started.wait(timeout=0.05)
+    release_first.set()
+    first.join(timeout=1.0)
+    second.join(timeout=1.0)
+
+    assert not first.is_alive()
+    assert not second.is_alive()
+    assert seen_prompts == [["cup"], ["mug"]]
+    module.stop()
+
+
+def test_object_db_uses_wall_clock_for_pending_ttl(monkeypatch: Any) -> None:
+    object_db = ObjectDB(pending_ttl_s=5.0)
+    detected = MagicMock()
+    detected.object_id = "stable-id"
+    detected.track_id = -1
+    detected.ts = 4.0  # Hardware timestamps are relative to camera boot.
+    detected.last_seen_ts = None
+    detected.center = None
+    now = [1000.0]
+    monkeypatch.setattr("dimos.perception.experimental.objectDB.time.time", lambda: now[0])
+
+    object_db.add_objects([detected])
+    assert detected.last_seen_ts == 1000.0
+    now[0] = 1004.0
+    object_db.add_objects([])
+    assert object_db.find_by_object_id("stable-id") is detected
+    now[0] = 1006.0
+    object_db.add_objects([])
+    assert object_db.find_by_object_id("stable-id") is None

From e09009c83d947eb7354bf1baa515b05ca726d533 Mon Sep 17 00:00:00 2001
From: Ruthwik <ruthwikdasyam@gmail.com>
Date: Thu, 27 Aug 2026 17:35:09 -0700
Subject: [PATCH 4/6] feat(perception): add Moondream OSR backend

---
 .../experimental/object_scene_registration.py | 45 ++++++++++------
 ...test_object_scene_registration_temporal.py | 53 +++++++++++++++----
 2 files changed, 74 insertions(+), 24 deletions(-)

diff --git a/dimos/perception/experimental/object_scene_registration.py b/dimos/perception/experimental/object_scene_registration.py
index 29fa6ea5da..0d11095690 100644
--- a/dimos/perception/experimental/object_scene_registration.py
+++ b/dimos/perception/experimental/object_scene_registration.py
@@ -49,13 +49,13 @@
 
 class ObjectSceneRegistrationConfig(ModuleConfig):
     prompt_mode: YoloePromptMode = YoloePromptMode.LRPC
-    detector_backend: Literal["yoloe", "owlv2"] = "yoloe"
+    detector_backend: Literal["yoloe", "owlv2", "moondream"] = "yoloe"
     segmentation_backend: Literal["yolo", "edgetam"] = "yolo"
     detect_on_request: bool = False
 
 
 class ObjectSceneRegistrationModule(Module):
-    """Module for detecting objects in camera images using YOLO-E with 2D and 3D detection."""
+    """Register prompted camera detections as stable 3D scene objects."""
 
     color_image: In[Image]
     depth_image: In[Image]
@@ -71,7 +71,7 @@ class ObjectSceneRegistrationModule(Module):
     _segmenter: Any | None = None
     _camera_info: CameraInfo | None = None
     _object_db: ObjectDB
-    _owlv2_prompts: list[str]
+    _text_prompts: list[str]
     _latest_aligned_frames: tuple[Image, Image] | None = None
     _processing_lock: threading.RLock
     # A tuple assignment/read is atomic, so depth and its transform cannot be
@@ -102,7 +102,7 @@ def __init__(
             min_detections_for_permanent=min_detections_for_permanent,
         )
         self._processing_lock = threading.RLock()
-        self._owlv2_prompts = []
+        self._text_prompts = []
         self._max_distance = max_distance
         self._use_aabb = use_aabb
         self._max_obstacle_width = max_obstacle_width
@@ -115,6 +115,12 @@ def start(self) -> None:
             from dimos.perception.detection.detectors.owlv2 import Owlv2Detector
 
             self._detector = Owlv2Detector()
+            self._detector.start()
+        elif self._detector_backend == "moondream":
+            from dimos.models.vl.moondream import MoondreamVlModel
+
+            self._detector = MoondreamVlModel()
+            self._detector.start()
         else:
             if self._prompt_mode == YoloePromptMode.LRPC:
                 model_name = "yoloe-11l-seg-pf.pt"
@@ -172,10 +178,10 @@ def _set_prompts(
         text: list[str] | None = None,
         bboxes: NDArray[np.float64] | None = None,
     ) -> None:
-        if self._detector_backend == "owlv2":
+        if self._detector_backend in {"owlv2", "moondream"}:
             if bboxes is not None:
-                raise ValueError("OWLv2 supports text prompts only")
-            self._owlv2_prompts = text or []
+                raise ValueError(f"{self._detector_backend} supports text prompts only")
+            self._text_prompts = text or []
         elif self._detector is not None:
             self._detector.set_prompts(text=text, bboxes=bboxes)
 
@@ -310,28 +316,28 @@ def get_full_scene_pointcloud(
         return pc
 
     @skill
-    def detect(self, *prompts: str) -> str:
+    def detect(self, prompts: list[str]) -> str:
         """Detect objects matching the given text prompts.
 
         Do NOT call this tool multiple times for one query. Pass all objects in a single call.
-        For example, to detect a cup and mouse, call detect("cup", "mouse") not detect("cup") then detect("mouse").
+        For example, to detect a cup and mouse, pass ["cup", "mouse"] in one call.
 
         Args:
-            prompts (str): Text descriptions of objects to detect (e.g., "person", "car", "dog")
+            prompts: Text descriptions of concrete object categories to detect.
 
         Returns:
             str: Detected objects with their object_id (stable UUID) and name.
 
         Example:
-            detect("person", "car", "dog")
-            detect("cup")
+            detect(["person", "car", "dog"])
+            detect(["cup"])
         """
         if not prompts:
             return "No prompts provided."
         with self._processing_lock:
             if self._detector is None:
                 return "Detector not initialized."
-            self._set_prompts(text=list(prompts))
+            self._set_prompts(text=prompts)
             if self._detect_on_request:
                 detected = [obj.agent_encode() for obj in self._scan_scene_objects()]
             else:
@@ -382,14 +388,23 @@ def _process_images(self, color_msg: Image, depth_msg: Image) -> list[DetObject]
         )
 
         if self._detector_backend == "owlv2":
-            if not self._owlv2_prompts:
+            if not self._text_prompts:
                 detections_2d = ImageDetections2D(color_image, [])
             else:
                 detections_2d = self._detector.query_detections(
                     color_image,
-                    self._owlv2_prompts,
+                    self._text_prompts,
                     threshold=self._detector_confidence,
                 )
+        elif self._detector_backend == "moondream":
+            detections_2d = ImageDetections2D(color_image, [])
+            for class_id, prompt in enumerate(self._text_prompts):
+                prompted = self._detector.query_detections(color_image, prompt)
+                for detection in prompted.detections:
+                    # Moondream's per-query indices are not temporal track IDs.
+                    detection.track_id = -1
+                    detection.class_id = class_id
+                detections_2d.detections.extend(prompted.detections)
         else:
             detections_2d = self._detector.process_image(color_image)
 
diff --git a/dimos/perception/experimental/test_object_scene_registration_temporal.py b/dimos/perception/experimental/test_object_scene_registration_temporal.py
index 4ccd850ebb..a02fab07a2 100644
--- a/dimos/perception/experimental/test_object_scene_registration_temporal.py
+++ b/dimos/perception/experimental/test_object_scene_registration_temporal.py
@@ -18,7 +18,7 @@
 import sys
 from threading import Event, Thread
 from typing import Any
-from unittest.mock import ANY, MagicMock
+from unittest.mock import ANY, MagicMock, call
 
 import numpy as np
 import pytest
@@ -154,11 +154,12 @@ def test_osr_implements_request_driven_scan_spec(module: ObjectSceneRegistration
     assert spec_annotation_compliance(module, ObjectSceneRegistrationSpec)
 
 
-def test_owlv2_prompts_are_text_only() -> None:
-    module = ObjectSceneRegistrationModule(detector_backend="owlv2")
+@pytest.mark.parametrize("backend", ["owlv2", "moondream"])
+def test_text_detector_prompts_reject_boxes(backend: str) -> None:
+    module = ObjectSceneRegistrationModule(detector_backend=backend)
     try:
         module.set_prompts(text=["mug"])
-        assert module._owlv2_prompts == ["mug"]
+        assert module._text_prompts == ["mug"]
         with pytest.raises(ValueError, match="text prompts"):
             module.set_prompts(bboxes=np.zeros((1, 4)))
     finally:
@@ -169,7 +170,7 @@ def test_owlv2_queries_configured_prompts(monkeypatch: Any) -> None:
     module = ObjectSceneRegistrationModule(detector_backend="owlv2")
     module._camera_info = MagicMock()
     module._detector = MagicMock()
-    module._owlv2_prompts = ["mug"]
+    module._text_prompts = ["mug"]
     module.detections_2d = MagicMock()
     color = Image(
         data=np.zeros((2, 2, 3), dtype=np.uint8),
@@ -189,6 +190,40 @@ def test_owlv2_queries_configured_prompts(monkeypatch: Any) -> None:
     module.stop()
 
 
+def test_moondream_queries_each_configured_prompt(monkeypatch: Any) -> None:
+    module = ObjectSceneRegistrationModule(detector_backend="moondream")
+    module._camera_info = MagicMock()
+    module._detector = MagicMock()
+    module._text_prompts = ["cup", "bottle"]
+    module.detections_2d = MagicMock()
+    color = Image(
+        data=np.zeros((2, 2, 3), dtype=np.uint8),
+        format=ImageFormat.BGR,
+        frame_id="camera",
+        ts=4.0,
+    )
+    cup = MagicMock(track_id=0, class_id=-1)
+    bottle = MagicMock(track_id=0, class_id=-1)
+    module._detector.query_detections.side_effect = [
+        ImageDetections2D(color, [cup]),
+        ImageDetections2D(color, [bottle]),
+    ]
+    process_3d = MagicMock()
+    monkeypatch.setattr(module, "_process_3d_detections", process_3d)
+
+    module._process_images(color, _image(4.0))
+
+    assert module._detector.query_detections.call_args_list == [
+        call(color, "cup"),
+        call(color, "bottle"),
+    ]
+    combined = process_3d.call_args.args[0]
+    assert combined.detections == [cup, bottle]
+    assert (cup.track_id, cup.class_id) == (-1, 0)
+    assert (bottle.track_id, bottle.class_id) == (-1, 1)
+    module.stop()
+
+
 def test_edgetam_refines_detector_output(monkeypatch: Any) -> None:
     module = ObjectSceneRegistrationModule(segmentation_backend="edgetam")
     module._camera_info = MagicMock()
@@ -242,7 +277,7 @@ def process_images(got_color: Image, got_depth: Image) -> list[MagicMock]:
     )
 
     assert module.scan_scene(text=["mug"]) is result
-    assert module._owlv2_prompts == ["mug"]
+    assert module._text_prompts == ["mug"]
     module.stop()
 
 
@@ -255,8 +290,8 @@ def test_request_driven_detect_triggers_scan(monkeypatch: Any) -> None:
     monkeypatch.setattr(module, "_scan_scene_objects", scan_objects)
     monkeypatch.setattr(module, "get_detected_objects", lambda: pytest.fail("read stale database"))
 
-    assert module.detect("mug") == "Detected 1 object(s):\n  - mug (object_id='current')"
-    assert module._owlv2_prompts == ["mug"]
+    assert module.detect(["mug"]) == "Detected 1 object(s):\n  - mug (object_id='current')"
+    assert module._text_prompts == ["mug"]
     scan_objects.assert_called_once_with()
     module.stop()
 
@@ -315,7 +350,7 @@ def test_concurrent_request_scans_do_not_overlap(monkeypatch: Any) -> None:
     seen_prompts: list[list[str]] = []
 
     def process_images(_: Image, __: Image) -> list[MagicMock]:
-        seen_prompts.append(list(module._owlv2_prompts))
+        seen_prompts.append(list(module._text_prompts))
         if len(seen_prompts) == 1:
             first_started.set()
             assert release_first.wait(timeout=1.0)

From f7a3efc97da6302f8b289c0bbab991da03e22bc8 Mon Sep 17 00:00:00 2001
From: Ruthwik <ruthwikdasyam@gmail.com>
Date: Thu, 27 Aug 2026 19:17:33 -0700
Subject: [PATCH 5/6] fix(perception): make OSR scans frame-idempotent

---
 dimos/perception/experimental/objectDB.py     | 28 +++++---
 .../experimental/object_scene_registration.py | 44 ++++++------
 ...test_object_scene_registration_temporal.py | 67 +++++++++++++------
 3 files changed, 89 insertions(+), 50 deletions(-)

diff --git a/dimos/perception/experimental/objectDB.py b/dimos/perception/experimental/objectDB.py
index 9cc8519cdd..99bbf5719f 100644
--- a/dimos/perception/experimental/objectDB.py
+++ b/dimos/perception/experimental/objectDB.py
@@ -84,25 +84,27 @@ def add_objects(self, objects: list[Object]) -> list[Object]:
             "matched_distance": 0,
         }
 
-        results: list[Object] = []
+        results: dict[str, Object] = {}
         now = time.time()
         with self._lock:
             self._prune_stale_pending(now)
             for obj in objects:
                 matched, reason = self._match(obj, now)
                 if matched is None:
-                    results.append(self._insert_pending(obj, now))
+                    inserted = self._insert_pending(obj, now)
+                    results[inserted.object_id] = inserted
                     stats["created"] += 1
                     continue
 
-                self._update_existing(matched, obj, now)
-                results.append(matched)
-                stats["updated"] += 1
+                updated = self._update_existing(matched, obj, now)
+                results[matched.object_id] = matched
+                if updated:
+                    stats["updated"] += 1
                 if reason == "track":
                     stats["matched_track"] += 1
                 elif reason == "distance":
                     stats["matched_distance"] += 1
-                if self._check_promotion(matched):
+                if updated and self._check_promotion(matched):
                     stats["promoted"] += 1
 
         stats["pending"] = len(self._pending_objects)
@@ -110,7 +112,7 @@ def add_objects(self, objects: list[Object]) -> list[Object]:
         self._last_add_stats = stats
         if stats["created"] > 0 or stats["promoted"] > 0:
             logger.info(f"ObjectDB: {stats}")
-        return results
+        return list(results.values())
 
     def get_last_add_stats(self) -> dict[str, int]:
         with self._lock:
@@ -225,12 +227,18 @@ def _insert_pending(self, obj: Object, now: float) -> Object:
         logger.info(f"Created new pending object {obj.object_id} ({obj.name})")
         return obj
 
-    def _update_existing(self, existing: Object, obj: Object, now: float) -> None:
+    def _update_existing(self, existing: Object, obj: Object, now: float) -> bool:
+        if obj.track_id >= 0:
+            self._track_id_map[obj.track_id] = existing.object_id
+        # Multiple prompts or repeated scans may produce the same object from
+        # one camera frame. Only distinct source observations advance memory.
+        if existing.ts == obj.ts:
+            return False
+
         existing.update_object(obj)
         existing.ts = obj.ts or now
         existing.last_seen_ts = now
-        if obj.track_id >= 0:
-            self._track_id_map[obj.track_id] = existing.object_id
+        return True
 
     def _match_by_track_id(self, track_id: int, now: float) -> Object | None:
         """Find object with matching track_id from YOLOE."""
diff --git a/dimos/perception/experimental/object_scene_registration.py b/dimos/perception/experimental/object_scene_registration.py
index 0d11095690..c4ceedf9c1 100644
--- a/dimos/perception/experimental/object_scene_registration.py
+++ b/dimos/perception/experimental/object_scene_registration.py
@@ -48,10 +48,16 @@
 
 
 class ObjectSceneRegistrationConfig(ModuleConfig):
+    target_frame: str = "map"
     prompt_mode: YoloePromptMode = YoloePromptMode.LRPC
     detector_backend: Literal["yoloe", "owlv2", "moondream"] = "yoloe"
     segmentation_backend: Literal["yolo", "edgetam"] = "yolo"
     detect_on_request: bool = False
+    distance_threshold: float = 0.2
+    min_detections_for_permanent: int = 6
+    max_distance: float = 0.0
+    use_aabb: bool = False
+    max_obstacle_width: float = 0.0
 
 
 class ObjectSceneRegistrationModule(Module):
@@ -80,32 +86,23 @@ class ObjectSceneRegistrationModule(Module):
 
     config: ObjectSceneRegistrationConfig
 
-    def __init__(
-        self,
-        target_frame: str = "map",
-        distance_threshold: float = 0.2,
-        min_detections_for_permanent: int = 6,
-        max_distance: float = 0.0,
-        use_aabb: bool = False,
-        max_obstacle_width: float = 0.0,
-        **kwargs: Any,
-    ) -> None:
+    def __init__(self, **kwargs: Any) -> None:
         super().__init__(**kwargs)
-        self._target_frame = target_frame
+        self._target_frame = self.config.target_frame
         self._prompt_mode = self.config.prompt_mode
         self._detector_backend = self.config.detector_backend
         self._segmentation_backend = self.config.segmentation_backend
         self._detector_confidence = 0.6
         self._detect_on_request = self.config.detect_on_request
         self._object_db = ObjectDB(
-            distance_threshold=distance_threshold,
-            min_detections_for_permanent=min_detections_for_permanent,
+            distance_threshold=self.config.distance_threshold,
+            min_detections_for_permanent=self.config.min_detections_for_permanent,
         )
         self._processing_lock = threading.RLock()
         self._text_prompts = []
-        self._max_distance = max_distance
-        self._use_aabb = use_aabb
-        self._max_obstacle_width = max_obstacle_width
+        self._max_distance = self.config.max_distance
+        self._use_aabb = self.config.use_aabb
+        self._max_obstacle_width = self.config.max_obstacle_width
 
     @rpc
     def start(self) -> None:
@@ -133,9 +130,14 @@ def start(self) -> None:
             )
 
         if self._segmentation_backend == "edgetam":
-            from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter
+            try:
+                from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter
 
-            self._segmenter = EdgeTAMImageSegmenter()
+                self._segmenter = EdgeTAMImageSegmenter()
+            except ModuleNotFoundError as e:
+                raise ModuleNotFoundError(
+                    "EdgeTAM requires the optional dependencies from dimos[misc]"
+                ) from e
 
         self.camera_info.subscribe(lambda msg: setattr(self, "_camera_info", msg))
 
@@ -458,11 +460,11 @@ def _process_3d_detections(
             use_aabb=self._use_aabb,
             max_obstacle_width=self._max_obstacle_width,
         )
-        if not objects:
-            return []
 
-        # Add objects to spatial memory database
+        # Empty observations still advance pending-object expiry.
         observed_objects = self._object_db.add_objects(objects)
+        if not objects:
+            return []
 
         # Publish ALL permanent objects so downstream consumers get the full set,
         # not just this frame's batch (which may be a subset of what's on the table).
diff --git a/dimos/perception/experimental/test_object_scene_registration_temporal.py b/dimos/perception/experimental/test_object_scene_registration_temporal.py
index a02fab07a2..6e63c53b54 100644
--- a/dimos/perception/experimental/test_object_scene_registration_temporal.py
+++ b/dimos/perception/experimental/test_object_scene_registration_temporal.py
@@ -27,9 +27,7 @@
 from dimos.msgs.vision_msgs.Detection3DArray import Detection3DArray
 from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D
 from dimos.perception.experimental.object_scene_registration import ObjectSceneRegistrationModule
-from dimos.perception.experimental.object_scene_registration_spec import ObjectSceneRegistrationSpec
 from dimos.perception.experimental.objectDB import ObjectDB
-from dimos.spec.utils import spec_annotation_compliance
 
 
 class _FakeTF:
@@ -150,22 +148,6 @@ def voxel_down_sample(self, voxel_size: float) -> _PointCloud:
     result.transform.assert_called_once_with(transform)
 
 
-def test_osr_implements_request_driven_scan_spec(module: ObjectSceneRegistrationModule) -> None:
-    assert spec_annotation_compliance(module, ObjectSceneRegistrationSpec)
-
-
-@pytest.mark.parametrize("backend", ["owlv2", "moondream"])
-def test_text_detector_prompts_reject_boxes(backend: str) -> None:
-    module = ObjectSceneRegistrationModule(detector_backend=backend)
-    try:
-        module.set_prompts(text=["mug"])
-        assert module._text_prompts == ["mug"]
-        with pytest.raises(ValueError, match="text prompts"):
-            module.set_prompts(bboxes=np.zeros((1, 4)))
-    finally:
-        module.stop()
-
-
 def test_owlv2_queries_configured_prompts(monkeypatch: Any) -> None:
     module = ObjectSceneRegistrationModule(detector_backend="owlv2")
     module._camera_info = MagicMock()
@@ -308,9 +290,10 @@ def test_scan_output_includes_pending_objects(monkeypatch: Any) -> None:
     module.detections_3d = MagicMock()
     module.objects = MagicMock()
     module.pointcloud = MagicMock()
+    converted_objects = [pending]
     monkeypatch.setattr(
         "dimos.perception.experimental.object_scene_registration.Object.from_2d_to_list",
-        lambda **_: [pending],
+        lambda **_: converted_objects,
     )
     monkeypatch.setattr(
         "dimos.perception.experimental.object_scene_registration.to_detection3d_array",
@@ -329,6 +312,18 @@ def test_scan_output_includes_pending_objects(monkeypatch: Any) -> None:
     )
 
     assert observed == [pending]
+
+    converted_objects.clear()
+    module._object_db.add_objects.reset_mock()
+    observed = ObjectSceneRegistrationModule._process_3d_detections(
+        module,
+        MagicMock(spec=ImageDetections2D),
+        _image(5.0),
+        _image(5.0),
+    )
+
+    assert observed == []
+    module._object_db.add_objects.assert_called_once_with([])
     module.stop()
 
 
@@ -398,3 +393,37 @@ def test_object_db_uses_wall_clock_for_pending_ttl(monkeypatch: Any) -> None:
     now[0] = 1006.0
     object_db.add_objects([])
     assert object_db.find_by_object_id("stable-id") is None
+
+
+def test_object_db_counts_each_source_frame_once(monkeypatch: Any) -> None:
+    object_db = ObjectDB(min_detections_for_permanent=10)
+    now = [1000.0]
+    monkeypatch.setattr("dimos.perception.experimental.objectDB.time.time", lambda: now[0])
+
+    first = MagicMock(
+        object_id="first-id",
+        track_id=-1,
+        ts=4.0,
+        last_seen_ts=None,
+        detections_count=1,
+    )
+    first.center = MagicMock()
+    duplicate = MagicMock(object_id="duplicate-id", track_id=-1, ts=4.0)
+    duplicate.center = MagicMock()
+    duplicate.center.distance.return_value = 0.0
+
+    observed = object_db.add_objects([first, duplicate])
+
+    assert observed == [first]
+    first.update_object.assert_not_called()
+    assert first.last_seen_ts == 1000.0
+
+    newer = MagicMock(object_id="newer-id", track_id=-1, ts=5.0)
+    newer.center = MagicMock()
+    newer.center.distance.return_value = 0.0
+    first.update_object.side_effect = lambda _: setattr(first, "detections_count", 2)
+    now[0] = 1001.0
+
+    assert object_db.add_objects([newer]) == [first]
+    first.update_object.assert_called_once_with(newer)
+    assert first.last_seen_ts == 1001.0

From 7a8666a383e761dc1b5790d13ad920f74afd78f8 Mon Sep 17 00:00:00 2001
From: Ruthwik <ruthwikdasyam@gmail.com>
Date: Fri, 28 Aug 2026 13:42:59 -0700
Subject: [PATCH 6/6] fix(perception): default OSR to base frame

---
 dimos/perception/experimental/object_scene_registration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dimos/perception/experimental/object_scene_registration.py b/dimos/perception/experimental/object_scene_registration.py
index c4ceedf9c1..d73d171dff 100644
--- a/dimos/perception/experimental/object_scene_registration.py
+++ b/dimos/perception/experimental/object_scene_registration.py
@@ -48,7 +48,7 @@
 
 
 class ObjectSceneRegistrationConfig(ModuleConfig):
-    target_frame: str = "map"
+    target_frame: str = "base_link"
     prompt_mode: YoloePromptMode = YoloePromptMode.LRPC
     detector_backend: Literal["yoloe", "owlv2", "moondream"] = "yoloe"
     segmentation_backend: Literal["yolo", "edgetam"] = "yolo"