use teleoprtc

.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-24.04, ubuntu-24.04-arm, macos-14, macos-15]
+        os: [ubuntu-22.04, ubuntu-24.04, ubuntu-24.04-arm, macos-15]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v7

README.md

@@ -46,7 +46,7 @@ python examples/02_vision.py zidane.jpg --model segment --output masks.jpg
 
 Run YOLO26 detection or segmentation on a webcam.
 Use `--frames 100` for more. `--source` also accepts a video path or stream URL.
-Add `--preview` for a live window that runs until you press Esc.
+Add `--preview` for a live preview window
 
 ```sh
 python examples/03_camera.py --source 0
@@ -87,20 +87,15 @@ DEV=USB+AMD:LLVM python examples/03_camera.py --source 0
 
 ## Comma camera
 
-On a comma device, run setup and activate the environment as above. Requires openpilot at `/data/openpilot`.
 Select the road, driver, or wide road camera with `--source comma:road`, `comma:driver`, or `comma:wide`.
 Add `--model segment` to any camera command for segmentation.
 
 ```sh
-# Comma CPU
-python examples/03_camera.py --source comma:road
-python examples/03_camera.py --source comma:driver
-python examples/03_camera.py --source comma:wide
-
-# chestnut GPU connected to comma
-DEV=USB+AMD:LLVM python examples/03_camera.py --source comma:road
-DEV=USB+AMD:LLVM python examples/03_camera.py --source comma:driver
-DEV=USB+AMD:LLVM python examples/03_camera.py --source comma:wide
+python examples/03_camera.py --host <comma-ip> --source comma:road
+python examples/03_camera.py --host <comma-ip> --source comma:driver
+python examples/03_camera.py --host <comma-ip> --source comma:wide
+
+DEV=USB+AMD:LLVM python examples/03_camera.py --host <comma-ip> --source comma:road
 ```
 
 ## Performance

examples/03_camera.py

@@ -1,36 +1,10 @@
 import argparse
+import asyncio
 from pathlib import Path
-import subprocess
-import sys
 import cv2
-import numpy as np
 from vision import Vision
 
 
-def comma_frames(stream):
-  checkout = Path('/data/openpilot')
-  sys.path.append(str(checkout))
-  from msgq.visionipc import VisionIpcClient
-  process = None
-  if subprocess.run(['pgrep', '-x', 'camerad'], stdout=subprocess.DEVNULL).returncode:
-    process = subprocess.Popen([str(checkout / 'openpilot/system/camerad/camerad')],
-                               cwd=checkout, stdout=subprocess.DEVNULL)
-  try:
-    client = VisionIpcClient('camerad', stream, True)
-    client.connect(True)
-    while True:
-      buf = client.recv()
-      if buf is None: continue
-      y = np.ndarray((buf.height, buf.width), np.uint8, buf.data, strides=(buf.stride, 1))
-      uv = np.ndarray((buf.height//2, buf.width//2, 2), np.uint8, buf.data,
-                      offset=buf.uv_offset, strides=(buf.stride, 2, 1))
-      yield cv2.cvtColorTwoPlane(y, uv, cv2.COLOR_YUV2BGR_NV12)
-  finally:
-    if process is not None:
-      process.terminate()
-      process.wait()
-
-
 def video_frames(source):
   capture = cv2.VideoCapture(int(source) if source.isdecimal() else source)
   try:
@@ -43,28 +17,42 @@ def video_frames(source):
     capture.release()
 
 
-if __name__ == '__main__':
+async def main():
   parser = argparse.ArgumentParser(description='YOLO26 on a webcam, video, or comma camera.')
   parser.add_argument('--source', default='0', help='Webcam, video/URL, or comma:road, comma:driver, comma:wide')
+  parser.add_argument('--host', help='comma IP address')
   parser.add_argument('--model', choices=['yolo', 'segment'], default='yolo')
   parser.add_argument('--frames', type=int, default=10, help='Frames to save without preview')
   parser.add_argument('--preview', action='store_true', help='Show a live preview window')
   args = parser.parse_args()
+  if args.source.startswith('comma') and not args.host: parser.error('--host is required for a comma camera')
 
   model = Vision(args.model)
   Path('frames').mkdir(exist_ok=True)
-  comma_streams = {'comma': 0, 'comma:road': 0, 'comma:wide': 1, 'comma:driver': 2}
-  stream = comma_frames(comma_streams[args.source]) if args.source in comma_streams else video_frames(args.source)
+  comma_streams = {'comma': 'road', 'comma:road': 'road', 'comma:wide': 'wideRoad', 'comma:driver': 'driver'}
+  if args.source in comma_streams:
+    from teleop import frames
+    stream = frames(args.host, comma_streams[args.source])
+  else:
+    stream = video_frames(args.source)
   print('Saving to frames/.', flush=True)
   try:
-    for i, frame in enumerate(stream):
+    i = 0
+    while True:
+      try: frame = await anext(stream) if args.source in comma_streams else next(stream)
+      except (StopAsyncIteration, StopIteration): break
       result = model(frame)
       result.save(f'frames/{i:05d}.jpg')
       if args.preview:
         cv2.imshow('chestnut', result.plot())
         if cv2.waitKey(1) == 27: break
       print(f'Frame {i + 1}', flush=True)
       if not args.preview and i + 1 >= args.frames: break
+      i += 1
   finally:
-    stream.close()
+    await stream.aclose() if args.source in comma_streams else stream.close()
     if args.preview: cv2.destroyAllWindows()
+
+
+if __name__ == '__main__':
+  asyncio.run(main())

examples/teleop.py

@@ -0,0 +1,77 @@
+import asyncio
+import json
+import urllib.request
+
+import av
+from libdatachannel import H264RtpDepacketizer, NalUnit, RtcpReceivingSession
+from teleoprtc import StreamingOffer, WebRTCOfferBuilder
+from teleoprtc.stream import RTCSessionDescription
+
+class Connection:
+  def __init__(self, host):
+    self.url = f'http://{host}:5001/stream'
+
+  async def __call__(self, offer: StreamingOffer):
+    return await asyncio.to_thread(self.connect, offer)
+
+  def connect(self, offer):
+    body = json.dumps({'sdp': offer.sdp, 'cameras': offer.video, 'enabled': True,
+                       'bridge_services_in': [], 'bridge_services_out': []}).encode()
+    request = urllib.request.Request(self.url, body, {'Content-Type': 'application/json'})
+    with urllib.request.urlopen(request, timeout=10) as response:
+      answer = json.load(response)
+    return RTCSessionDescription(answer['sdp'], answer['type'])
+
+
+class Receiver:
+  def __init__(self, track):
+    self.loop = asyncio.get_running_loop()
+    self.queue = asyncio.Queue(2)
+    self.decoder = av.CodecContext.create('h264', 'r')
+    self.depacketizer = H264RtpDepacketizer(NalUnit.Separator.StartSequence)
+    self.rtcp = RtcpReceivingSession()
+    track.set_media_handler(self.depacketizer)
+    track.chain_media_handler(self.rtcp)
+    track.on_frame(lambda data, _: self.loop.call_soon_threadsafe(self.enqueue, bytes(data)))
+    self.track = track
+    track.request_keyframe()
+
+  def enqueue(self, data):
+    if self.queue.full(): self.queue.get_nowait()
+    self.queue.put_nowait(data)
+
+  async def recv(self):
+    while True:
+      try:
+        for packet in self.decoder.parse(await self.queue.get()):
+          frames = self.decoder.decode(packet)
+          if frames: return frames[-1].to_ndarray(format='bgr24')
+      except av.FFmpegError:
+        self.track.request_keyframe()
+
+  def close(self):
+    self.track.reset_callbacks()
+    self.track.close()
+    self.track = self.depacketizer = self.rtcp = None
+
+
+async def frames(host, camera):
+  tunnel = await asyncio.create_subprocess_exec('ssh', '-N', '-L', '5001:localhost:5001', f'comma@{host}')
+  await asyncio.sleep(0.5)
+  builder = WebRTCOfferBuilder(Connection('localhost'))
+  builder.offer_to_receive_video_stream(camera)
+  stream = builder.stream()
+  await stream.start()
+  await stream.wait_for_connection()
+  receiver = Receiver(stream.get_incoming_video_track(camera))
+  try:
+    while stream.is_connected_and_ready:
+      try:
+        yield await asyncio.wait_for(receiver.recv(), 0.5)
+      except TimeoutError:
+        receiver.track.request_keyframe()
+  finally:
+    receiver.close()
+    await stream.stop()
+    tunnel.terminate()
+    await tunnel.wait()

pyproject.toml

@@ -1,13 +1,15 @@
 [project]
 name = "chestnut"
 version = "0.1.0"
-requires-python = ">=3.11,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
   "tinygrad",
   "jinja2==3.1.6",
   "numpy==2.3.5",
-  "opencv-python-headless>=4.11,<5",
-  "ultralytics-opencv-headless==8.4.153",
+  "opencv-python>=4.11,<5",
+  "ultralytics==8.4.153",
+  "av>=15,<17",
+  "teleoprtc @ https://github.com/commaai/teleoprtc/archive/1aa8fc433bef1519a95c0700c96258c3be6dfb34.tar.gz",
   "onnx>=1.17,<2",
   "torch>=2.5",
   "torchvision>=0.20",

setup.sh

@@ -15,18 +15,20 @@ case "$(uname -s):$(uname -m)" in
     getconf GNU_LIBC_VERSION >/dev/null || { echo 'glibc Linux is required.' >&2; exit 1; }
     ;;
   Darwin:arm64)
-    [ "$(sw_vers -productVersion | cut -d. -f1)" -ge 14 ] || { echo 'macOS 14+ is required.' >&2; exit 1; }
+    [ "$(sw_vers -productVersion | cut -d. -f1)" -ge 15 ] || { echo 'macOS 15+ is required.' >&2; exit 1; }
     command -v brew >/dev/null || { echo 'Install Homebrew and rerun setup.' >&2; exit 1; }
     brew list --versions llvm@21 libusb >/dev/null 2>&1 || brew install llvm@21 libusb
     export PATH="/opt/homebrew/opt/llvm@21/bin:$PATH"
     ;;
-  *) echo 'Use x86_64/aarch64 Linux or Apple Silicon macOS 14+.' >&2; exit 1 ;;
+  *) echo 'Use x86_64/aarch64 Linux or Apple Silicon macOS 15+.' >&2; exit 1 ;;
 esac
 
 if [ "$(uname -s)" = Linux ] && { ! command -v clang >/dev/null || ! command -v curl >/dev/null ||
-   ! command -v awk >/dev/null || ! command -v tar >/dev/null || ! command -v gzip >/dev/null ||
+   ! command -v awk >/dev/null || ! command -v find >/dev/null || ! command -v tar >/dev/null || ! command -v gzip >/dev/null ||
    ! ldconfig -p 2>/dev/null | grep -E 'libLLVM(-|\.so\.)(19|20|21)' >/dev/null ||
-   ! ldconfig -p 2>/dev/null | grep -F 'libusb-1.0.so' >/dev/null; }; then
+   ! ldconfig -p 2>/dev/null | grep -F 'libusb-1.0.so' >/dev/null ||
+   ! ldconfig -p 2>/dev/null | grep -F 'libGL.so.1' >/dev/null ||
+   ! ldconfig -p 2>/dev/null | grep -F 'libgthread-2.0.so.0' >/dev/null; }; then
   if command -v apt-get >/dev/null; then
     "${as_root[@]}" apt-get update
     llvm_package=
@@ -49,14 +51,14 @@ if [ "$(uname -s)" = Linux ] && { ! command -v clang >/dev/null || ! command -v
       "${as_root[@]}" apt-get update
       llvm_package=libllvm20
     fi
-    "${as_root[@]}" apt-get install -y --no-install-recommends ca-certificates curl clang "$llvm_package" libusb-1.0-0 gawk tar gzip
+    "${as_root[@]}" apt-get install -y --no-install-recommends ca-certificates curl clang "$llvm_package" libusb-1.0-0 libgl1 libglib2.0-0 libxcb1 findutils gawk tar gzip
   elif command -v dnf >/dev/null; then
-    "${as_root[@]}" dnf install -y ca-certificates clang llvm-libs libusb1 curl gawk tar gzip
+    "${as_root[@]}" dnf install -y ca-certificates clang llvm-libs libusb1 mesa-libGL glib2 libxcb findutils curl gawk tar gzip
   elif command -v pacman >/dev/null; then
-    "${as_root[@]}" pacman -Syu --needed --noconfirm ca-certificates clang llvm20-libs libusb curl gawk tar gzip
+    "${as_root[@]}" pacman -Syu --needed --noconfirm ca-certificates clang llvm20-libs libusb libglvnd glib2 libxcb findutils curl gawk tar gzip
   elif command -v zypper >/dev/null; then
     "${as_root[@]}" zypper --non-interactive refresh
-    "${as_root[@]}" zypper --non-interactive install ca-certificates clang libLLVM20 libusb-1_0-0 curl gawk tar gzip
+    "${as_root[@]}" zypper --non-interactive install ca-certificates clang libLLVM20 libusb-1_0-0 libglvnd libglib-2_0-0 libgthread-2_0-0 libxcb1 findutils curl gawk tar gzip
   else
     echo 'Install clang, curl, LLVM 19–21, and libusb with your package manager.' >&2
     exit 1