From 66edeabbabe41967e8fb5ba57aba3f2708111d07 Mon Sep 17 00:00:00 2001 From: j_d <57573086+null-create@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:50:24 -0700 Subject: [PATCH 1/3] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0446f9f..9cc2347 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,12 @@ Concretely, that means: ## Features -- **Multi-backend support** — Ollama, OpenAI, HuggingFace (remote or local), and Transformers (local GPU/CPU). Switch at runtime. -- **Agent pooling (optional)** — with `--use-pool`, the root agent can fan tasks out concurrently to vendor-agnostic sub-agents defined in [agents.yaml](agents.yaml) via a `dispatch` tool. Each pool entry binds a model _and_ a backend, so dispatch decisions are also compute-location decisions — a frontier model can plan while sensitive work stays on a local model, or a local root can fan out to faster remote SLMs for latency-sensitive tool calls. +- **Declerative sub-agent pooling (optional)** — with `--use-pool`, the root agent can fan tasks out concurrently to vendor-agnostic sub-agents defined in an optional [agents.yaml](agents.yaml) file via a `dispatch` tool. Each pool entry binds a model _and_ a backend, so dispatch decisions are also compute-location decisions — a frontier model can plan while sensitive work stays on a local model, or a local root can fan out to faster remote SLMs for latency-sensitive tool calls. - **Agent profiles** — drop-in system prompts with permission manifests, base-profile inheritance, and auto-generated profiles via `/profile create`. Bundled profiles: `default`, `coder`, `reviewer`, `writer`, `planner`, `researcher`, `analyst`. - **Rich built-in tool set** — file ops, shell access, web search/fetch, Wikipedia/GitHub/arXiv search, Git, task tracking, reasoning scratchpad, notebook, and more. Sandbox-locked with shell allowlists, SSRF protection, and sensitive-file gating. - **Permission system** — write operations and sensitive reads require user approval. Session grants, workspace scoping, and profile-level allow/deny lists. - **OpenAI-compatible API server** — run the same agent harness behind `/v1/models` and `/v1/chat/completions` (streaming + non-streaming) so any workflow that speaks the OpenAI wire protocol (the `openai` Python SDK, curl, or plain REST) can drive the agent. +- **Multi-backend support** — Ollama, OpenAI, HuggingFace (remote or local), and Transformers (local GPU/CPU). Switch at runtime. - **MCP integration** — add stdio or HTTP MCP servers at runtime for custom tools. ## Roadmap / areas of active exploration From c37028b66487a5e117800f63285c75a6e2d3966c Mon Sep 17 00:00:00 2001 From: j_d <57573086+null-create@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:40:07 -0700 Subject: [PATCH 2/3] Update taglines.py --- oli_bot/screens/taglines.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/oli_bot/screens/taglines.py b/oli_bot/screens/taglines.py index 247795f..90d7a93 100644 --- a/oli_bot/screens/taglines.py +++ b/oli_bot/screens/taglines.py @@ -58,7 +58,18 @@ "Please don't ask me to debug your code, I have enough problems of my own", "Now with 100% more existential dread", "Don't worry, I won't tell anyone about your search history. Probably.", - "It's an older tagline sir, but it checks out" + "It's an older tagline sir, but it checks out", + "I peaked around the same time as the Chumby", + "Powered by the ghost of Ask Jeeves", + "I still believe in you, Zima", + "I wonder if there's beer on the sun", + "Direct-to-video sequel to your productivity", + "As seen on TV", + "Cool as Ice, warm as toast", + "Manos, but make it software", + "Nukin' the Fridge? I invented it", + "Never seen Xanadu? Neither have I, and yet, here we are", + "There's a suspicious amount of glitter here" ] From 14f23213a10ec298e9129690ba6c52aab3cdaf1b Mon Sep 17 00:00:00 2001 From: jay Date: Fri, 11 Sep 2026 12:58:33 -0700 Subject: [PATCH 3/3] add voice support --- AGENTS.md | 5 +- README.md | 21 ++ docs/ARCHITECTURE.md | 62 ++++ docs/CONFIGURE.md | 7 + oli_bot/chat.py | 187 ++++++++-- oli_bot/config.py | 14 + oli_bot/profiles/coder/AGENTS.md | 4 + oli_bot/profiles/default/AGENTS.md | 4 + oli_bot/screens/config_screen.py | 54 +++ oli_bot/settings.py | 31 ++ oli_bot/voice.py | 292 +++++++++++++++ pyproject.toml | 7 + tests/test_config_screen.py | 46 +++ tests/test_config_settings.py | 107 ++++++ tests/test_voice.py | 561 +++++++++++++++++++++++++++++ 15 files changed, 1375 insertions(+), 27 deletions(-) create mode 100644 oli_bot/voice.py create mode 100644 tests/test_voice.py diff --git a/AGENTS.md b/AGENTS.md index 66af621..28e9586 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,7 @@ This file is the top-level orientation map. Depth lives under [docs/](docs/). | [oli_bot/tools/](oli_bot/tools/) | Tool handlers: `files.py` (read/write/edit + `view_image` via Pillow), `directories.py` (glob/grep/list_directory/tree — filesystem work runs via `asyncio.to_thread` / `create_subprocess_exec`), `web.py` (search + fetch + specialised searches, all guarded by `_check_ssrf`), `shell.py` (allowlisted `run_command`, including read-only `git`), `parsing.py` (`compare`), `memory.py` (`think`, `todowrite`, `notebook`), `truncation.py` (per-tier char budgets), `permissions.py` (sensitive-path detection). See [docs/TOOLS.md](docs/TOOLS.md). | | [oli_bot/sessions.py](oli_bot/sessions.py) | `Session` (permission gating) + `ConversationStore` (per-server JSON persistence under `~/.config/oli/sessions//`) + `WorkspaceManager`. `save_session()` returns the (possibly new) id so callers can rebind after a corrupt-file rewrite. Persisted messages preserve `tool_call_id`; loads pass through `sanitize_tool_history` so poisoned histories self-heal. | | [oli_bot/server_manager.py](oli_bot/server_manager.py) | `ServerManager` — multi-server lifecycle persisted to `ollama_hosts.json`, URL validation. | +| [oli_bot/voice.py](oli_bot/voice.py) | `VoiceEngine` — optional, lazy-loaded mic → STT → TTS engine for the `/voice` command (faster-whisper, Piper TTS, WebRTC VAD, pyaudio). All I/O is blocking; `chat.py` calls it via `asyncio.to_thread`. `record()` accepts a `threading.Event` so `chat.py` can interrupt an in-progress recording the instant voice mode is toggled off, instead of waiting out the silence/max-duration timeout. All seven tunables (whisper/piper models, sample rate, VAD frame duration, VAD aggressiveness, silence timeout, max record seconds) are `AppConfig` fields (`OLI_VOICE_*` env / `settings.json` `voice` section / `/config` screen); `chat.py` passes them explicitly when constructing the engine, and saving `/config` drops the engine so the next `/voice` picks up new values. | | [oli_bot/logger.py](oli_bot/logger.py) | Centralised NDJSON file logging (rotating, 10 MB × 5) under `AppConfig.log_file`. Deliberately no console handler — stray writes would corrupt the Textual TUI. | ## Modes @@ -74,6 +75,7 @@ See [docs/AGENT-POOLING.md](docs/AGENT-POOLING.md) for the `agents.yaml` schema, - **Permission gating** — `BuiltinToolManager.call_tool()` runs the profile enforcer, then `Session.needs_permission()`, then (if needed) `confirm_callback(description)`; the TUI shows `PermissionScreen` and the callback returns `"once"`, `"session"`, or `"deny"`. Session grants persist per scope for the process lifetime. `glob`/`grep` targeting patterns like `.env*`, `*.pem`, `*secret*` trigger the `workspace_sensitive` scope even inside the workspace. - **Built-in tool naming** — registered as `builtin__` and dispatched by `MCPClientManager.call_tool`. - **Settings round-trip** — runtime model changes (`/model set-large|set-small`, `/servers set-default-model`) persist back to `settings.json` via `OliBot._persist_model_to_settings`; the `/config` form pre-populates from effective runtime values (server overrides included) via `_sync_settings_from_runtime`. Backend construction honours `self.config` overrides so JSON beats env at startup. +- **Voice mode** — `/voice` toggles a background `@work(exclusive=False)` loop (`_run_voice_loop`) that cycles mic record → `VoiceEngine.transcribe` → the normal `_handle_user_message` chat pipeline → `VoiceEngine.speak`. Fully local (no network calls). A `threading.Event` (`_voice_stop_event`) is set the instant `/voice` toggles off so the blocking `record()` call returns immediately instead of running out the silence/max-duration timeout. For sequence diagrams and the full state machines, see [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). @@ -102,6 +104,7 @@ Runtime deps are declared in [pyproject.toml](pyproject.toml) `[project.dependen - **Backends** — `ollama`, `openai`, `huggingface_hub`, `transformers`, `accelerate` - **MCP + API** — `mcp` (v2 SDK — pulls in `httpx2`, `mcp-types`, `opentelemetry-api`), `fastapi`, `uvicorn[standard]` - **Tools** — `Pillow` (image handling), `httpx` / `requests` / `aiohttp`, `beautifulsoup4`, `ddgs`, `wikipedia`, `arxiv`, `googlesearch_python`, `stackapi`, `search_engine_parser`, `gnews`, `newspaper4k` +- **Voice (optional)** — `[project.optional-dependencies].voice`: `pyaudio` (needs the PortAudio system library — `brew install portaudio` on macOS), `webrtcvad-wheels`, `faster-whisper`, `piper-tts`, `simpleaudio`. Install via `pip install -e '.[voice]'`. - **Python < 3.11 only** — `exceptiongroup` ## Run @@ -175,4 +178,4 @@ Tests under `tests/` cover: `AgentPool` scaffolding (lookup, `${VAR}` expansion, ## Commands (in-app) -`/help`, `/models [name]`, `/model large|small`, `/model set-large|set-small `, `/config`, `/context`, `/servers add|list|remove|default|switch|use-model`, `/mcp add|list|remove`, `/mode [ask|agent|chat|plan]`, `/profile list|load|create`, `/sessions [list|switch|delete|rename|purge]`, `/workspace list|set|unset`, `/offline`, `/dry-run`, `/clear`, `/home`, `Ctrl+Q`, `Ctrl+L`, `Ctrl+Y` +`/help`, `/models [name]`, `/model large|small`, `/model set-large|set-small `, `/config`, `/context`, `/servers add|list|remove|default|switch|use-model`, `/mcp add|list|remove`, `/mode [ask|agent|chat|plan]`, `/profile list|load|create`, `/sessions [list|switch|delete|rename|purge]`, `/workspace list|set|unset`, `/offline`, `/dry-run`, `/voice`, `/clear`, `/home`, `Ctrl+Q`, `Ctrl+L`, `Ctrl+Y` diff --git a/README.md b/README.md index 9cc2347..9ccf264 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Concretely, that means: - **OpenAI-compatible API server** — run the same agent harness behind `/v1/models` and `/v1/chat/completions` (streaming + non-streaming) so any workflow that speaks the OpenAI wire protocol (the `openai` Python SDK, curl, or plain REST) can drive the agent. - **Multi-backend support** — Ollama, OpenAI, HuggingFace (remote or local), and Transformers (local GPU/CPU). Switch at runtime. - **MCP integration** — add stdio or HTTP MCP servers at runtime for custom tools. +- **Voice mode (optional)** — `/voice` toggles a hands-free mic → STT → LLM → TTS loop (faster-whisper, Piper TTS, WebRTC VAD) for the TUI. Fully local; requires the `voice` extras and a downloaded Piper model. ## Roadmap / areas of active exploration @@ -128,10 +129,30 @@ can pick up where you left off. | `/workspace list\|set\|unset` | Manage workspace directory | | `/offline` | Toggle offline mode | | `/dry-run` | Toggle dry-run mode | +| `/voice` | Toggle voice mode (mic → STT → LLM → TTS) | | `/clear` | Clear the conversation | | `/home` | Return to the home screen | | `Ctrl+Q` / `Ctrl+L` / `Ctrl+Y` | Quit / Clear / Copy last message | +## Voice mode +`/voice` toggles a hands-free loop: listen on the mic (WebRTC VAD auto-detects speech/silence), transcribe with faster-whisper, send the text through the normal chat pipeline, then speak the response back with Piper TTS. Everything runs locally — no network calls. + +Install the extras and the PortAudio system library (required to build `pyaudio`): + +```bash +brew install portaudio # macOS; use your distro's package manager on Linux +pip install -e '.[voice]' +``` + +Download a Piper voice model (one-time): + +```bash +wget https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx +wget https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx.json +export OLI_VOICE_PIPER_MODEL=/path/to/en_US-lessac-medium.onnx +``` + +Type `/voice` again (or `Ctrl+Q` to quit the app) to exit voice mode — the mic loop is interrupted immediately rather than waiting for the current recording to time out. All voice settings are configurable via env vars, `.env`, or the `/config` screen: `OLI_VOICE_WHISPER_MODEL` (default `base`), `OLI_VOICE_PIPER_MODEL`, plus VAD tunables (`OLI_VOICE_SAMPLE_RATE`, `OLI_VOICE_FRAME_DURATION_MS`, `OLI_VOICE_VAD_AGGRESSIVENESS`, `OLI_VOICE_SILENCE_TIMEOUT_MS`, `OLI_VOICE_MAX_RECORD_SECONDS`). See [docs/CONFIGURE.md](docs/CONFIGURE.md) for the full table. ## Documentation | Document | Contents | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 43d9a3d..d9a8911 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -325,6 +325,68 @@ Tool Call Generated by Agent --- +## Voice Mode Loop + +`/voice` toggles a background `@work(exclusive=False)` worker (`_run_voice_loop`) +that runs alongside the normal `_generate_response` worker. All audio I/O is +blocking and offloaded via `asyncio.to_thread` so the Textual event loop never +stalls. Everything is local — no network calls. + +``` +/voice (typed) + │ + ▼ +_handle_voice() + │ + ├─ voice OFF → ON: create VoiceEngine (lazy), threading.Event, start + │ _run_voice_loop() as a background worker + │ + └─ voice ON → OFF: set _voice_stop_event ─────────────┐ + │ + ▼ +┌───────────────────────────────────────────────────────────────────┐ +│ _run_voice_loop() (while self._voice_active) │ +│ │ +│ mount "🎙 Listening…" ──▶ engine.record(stop_event) │ +│ │ │ │ +│ │ stop_event.set() mid-recording ──▶ return None │ +│ │ │ (checked every audio frame) │ +│ ▼ ▼ │ +│ remove listening widget audio_path or None │ +│ │ │ │ +│ │ None (silence) ──▶ loop back │ +│ ▼ │ +│ engine.transcribe(audio_path) (faster-whisper) │ +│ │ │ +│ ▼ │ +│ _handle_user_message(text) ──▶ normal chat pipeline │ +│ │ (_generate_response, tool loop) │ +│ ▼ │ +│ poll self.agent.generating until False │ +│ │ │ +│ ▼ │ +│ engine.speak(last_assistant_message) (Piper TTS) │ +│ │ │ +│ └──▶ loop back to "Listening…" (if still active) │ +└───────────────────────────────────────────────────────────────────┘ +``` + +Key points: + +- **Immediate interruption** — `VoiceEngine.record()` polls the shared + `threading.Event` every audio frame, so toggling `/voice` off during an + in-progress recording returns `None` right away instead of waiting out the + silence/max-duration timeout (`SILENCE_TIMEOUT_MS` / `MAX_RECORD_SECONDS`). +- **Shares the normal pipeline** — transcribed text is fed through + `_handle_user_message`, so tool calls, permission prompts, and session + persistence all behave exactly as they do for typed input. +- **Lazy model loading** — `VoiceEngine.load()` (Whisper, Piper, WebRTC VAD) + only runs on first activation, keeping TUI startup fast. +- **Also stopped on app exit** — `on_unmount` clears `_voice_active` and sets + the stop event so a lingering recording doesn't block shutdown. + +--- + ## State Machine: Chat Session ``` diff --git a/docs/CONFIGURE.md b/docs/CONFIGURE.md index c4230ba..3293b4f 100644 --- a/docs/CONFIGURE.md +++ b/docs/CONFIGURE.md @@ -54,6 +54,13 @@ If `~/.config/oli/settings.json` does not exist, it is auto-created on first loa | `offline_mode` | `true` | `OLI_OFFLINE_MODE` | Block network access for web tools and MCP servers | | `log_file` | `logs/backend.ndjson` | `OLI_LOG_FILE` | Path for NDJSON backend log file | | `log_level` | `INFO` | `OLI_LOG_LEVEL` | Logging level: `DEBUG`, `INFO`, `WARNING`, `ERROR` | +| `voice_whisper_model` | `base` | `OLI_VOICE_WHISPER_MODEL` | faster-whisper model size: `tiny`/`base`/`small`/`medium`/`large` | +| `voice_piper_model` | `en_US-lessac-medium.onnx` | `OLI_VOICE_PIPER_MODEL` | Path to the local Piper TTS `.onnx` model file | +| `voice_sample_rate` | `16000` | `OLI_VOICE_SAMPLE_RATE` | Mic sample rate in Hz (required by WebRTC VAD) | +| `voice_frame_duration_ms` | `30` | `OLI_VOICE_FRAME_DURATION_MS` | VAD frame size; must be `10`, `20`, or `30` | +| `voice_vad_aggressiveness` | `2` | `OLI_VOICE_VAD_AGGRESSIVENESS`| WebRTC VAD aggressiveness `0`–`3`; higher rejects more noise | +| `voice_silence_timeout_ms` | `800` | `OLI_VOICE_SILENCE_TIMEOUT_MS`| Stop recording after this much consecutive silence | +| `voice_max_record_seconds` | `15` | `OLI_VOICE_MAX_RECORD_SECONDS`| Hard cap on a single recording | ## Quick examples diff --git a/oli_bot/chat.py b/oli_bot/chat.py index 824f314..fed6069 100644 --- a/oli_bot/chat.py +++ b/oli_bot/chat.py @@ -9,6 +9,7 @@ import asyncio import logging import random +import threading from datetime import datetime, timezone from pathlib import Path from typing import Optional @@ -107,6 +108,7 @@ "/home", "/offline", "/dry-run", + "/voice", ) @@ -341,6 +343,11 @@ def __init__( self._command_matches: list[str] = [] self._suggestion_index: int = 0 + # Voice mode state + self._voice_active: bool = False + self._voice_engine = None # VoiceEngine — lazy-loaded on first /voice use + self._voice_stop_event: Optional[threading.Event] = None + server_name = active.name if active else "default" loaded = False if load_session: @@ -503,6 +510,10 @@ def _render_welcome(self) -> Static: async def on_unmount(self) -> None: self._stop_sub_tree_timer() + # Signal the voice loop to stop before disconnecting everything else + self._voice_active = False + if self._voice_stop_event is not None: + self._voice_stop_event.set() await self.mcp_manager.disconnect_all() def on_mount(self) -> None: @@ -747,6 +758,7 @@ def _handle_command(self, text: str) -> None: " [bold]/clear[/bold] — clear conversation\n" " [bold]/offline[/bold] — toggle offline mode (block network tools)\n" " [bold]/dry-run[/bold] — toggle dry-run mode (preview destructive actions)\n" + " [bold]/voice[/bold] — toggle voice mode (mic → STT → LLM → TTS)\n" " [bold]/home[/bold] — go to home screen\n" " [bold]/help[/bold] — show this message\n" " [bold]Ctrl+Q[/bold] — quit\n" @@ -783,6 +795,8 @@ def _handle_command(self, text: str) -> None: self._handle_dry_run() elif cmd == "/workspace": self._handle_workspace(text) + elif cmd == "/voice": + self._handle_voice() else: self._add_message( "System", @@ -1903,32 +1917,13 @@ def _model_list(self) -> None: self._add_message("System", f"[red]{e}[/red]") def _sync_settings_from_runtime(self) -> None: - s = self.settings - s["openai"]["large_model"] = self.config.openai_model - s["openai"]["small_model"] = self.config.openai_small_model - s["openai"]["vision_style"] = self.config.openai_vision_style - s["ollama"]["large_model"] = self.config.ollama_model - s["ollama"]["small_model"] = self.config.ollama_small_model - s["huggingface"]["large_model"] = self.config.huggingface_model - s["huggingface"]["small_model"] = self.config.huggingface_small_model - s["huggingface"]["remote"] = self.config.huggingface_remote - s["transformers"]["model"] = self.config.transformers_model - s["transformers"]["small_model"] = self.config.transformers_small_model - s["transformers"]["is_multi_model"] = self.config.transformers_is_multi_model - mp = s.setdefault("model_params", {}) - mp["use_agent_pool"] = self.config.use_agent_pool - mp["agent_pool_size"] = self.config.agent_pool_size - lg = s.setdefault("logging", {}) - lg["log_level"] = self.config.log_level - lg["log_file"] = self.config.log_file - api = s.setdefault("api_server", {}) - api["host"] = self.config.api_host - api["port"] = self.config.api_port - api["profile"] = self.config.api_profile - api["mode"] = self.config.api_mode - paths = s.setdefault("paths", {}) - paths["profiles_dir"] = self.config.profiles_dir - paths["logs_dir"] = self.config.logs_dir + preserved = { + key: self.settings[key] + for key in ("workspace", "session") + if key in self.settings + } + s = self.settings_manager.from_appconfig(self.config) + s.update(preserved) backend = self.config.backend section = s.setdefault(backend, {}) eff_large = self._get_large_model() @@ -1940,6 +1935,7 @@ def _sync_settings_from_runtime(self) -> None: else: section["large_model"] = eff_large section["small_model"] = eff_small + self.settings = s @work(exclusive=False) async def _handle_config(self) -> None: @@ -1955,6 +1951,11 @@ async def _handle_config(self) -> None: self._builtin_tools._config = self.config self.mcp_manager._offline_mode = self.config.offline_mode + # Voice engine holds its own copy of the voice_* config; drop it so + # the next /voice activation picks up the new values. + if not self._voice_active: + self._voice_engine = None + # Always rebuild: base_url / api_key / vision_style changes must # take effect even when the backend TYPE didn't change. type_changed = result["backend"] != old_backend @@ -2081,8 +2082,142 @@ def _status_badges(self) -> str: badges.append("[bold blue]\\[OFFLINE][/bold blue]") if self.config.dry_run: badges.append("[bold red]\\[DRY-RUN][/bold red]") + if getattr(self, "_voice_active", False): + badges.append("[bold magenta]\\[VOICE][/bold magenta]") return " ".join(badges) + # ------------------------------------------------------------------ + # Voice mode + # ------------------------------------------------------------------ + + def _handle_voice(self) -> None: + """Toggle voice mode on or off.""" + if self._voice_active: + # ---- turn off ---- + self._voice_active = False + if self._voice_stop_event is not None: + self._voice_stop_event.set() # interrupt an in-progress recording + self.update_header() + self._add_message("System", "🎙 Voice mode [bold]disabled[/bold].") + else: + # ---- turn on ---- + if self._voice_engine is None: + from .voice import VoiceEngine # lazy import — keeps startup fast + + self._voice_engine = VoiceEngine( + whisper_model_size=self.config.voice_whisper_model, + piper_model_path=self.config.voice_piper_model, + sample_rate=self.config.voice_sample_rate, + frame_duration_ms=self.config.voice_frame_duration_ms, + vad_aggressiveness=self.config.voice_vad_aggressiveness, + silence_timeout_ms=self.config.voice_silence_timeout_ms, + max_record_seconds=self.config.voice_max_record_seconds, + ) + self._voice_active = True + self._voice_stop_event = threading.Event() + self.update_header() + self._add_message( + "System", + "🎙 Voice mode [bold]enabled[/bold]. " + "Speak after the [bold]🎙 Listening…[/bold] prompt appears.\n" + "Re-enter [bold]/voice[/bold] to exit voice mode.", + ) + self._run_voice_loop() + + @work(exclusive=False) + async def _run_voice_loop(self) -> None: + """Drive the STT → LLM → TTS cycle while voice mode is active. + + Runs as a non-exclusive background worker so it can coexist with the + ``_generate_response`` exclusive worker. All blocking audio calls are + offloaded to threads via ``asyncio.to_thread`` to avoid stalling the + Textual event loop. + """ + engine = self._voice_engine + chat_log = self.query_one("#chat-log") + + # ---- load models in a thread (may be slow the first time) ---------- + try: + await asyncio.to_thread(engine.load) + except RuntimeError as exc: + self._voice_active = False + self.update_header() + self._add_message("System", f"[red]Voice mode error: {exc}[/red]") + return + + while self._voice_active: + # ---- show listening indicator ---------------------------------- + listen_widget = Static("🎙 [bold]Listening…[/bold]", classes="message") + await chat_log.mount(listen_widget) + chat_log.scroll_end(animate=False) + + # ---- record from microphone (blocking, runs in thread) --------- + try: + audio_path = await asyncio.to_thread(engine.record, self._voice_stop_event) + except Exception as exc: + try: + listen_widget.remove() + except Exception: + pass + self._add_message("System", f"[red]Recording error: {exc}[/red]") + await asyncio.sleep(1) + continue + + try: + listen_widget.remove() + except Exception: + pass + + if not self._voice_active: + break + + if audio_path is None: + # No speech detected — loop back immediately + continue + + # ---- transcribe (blocking, runs in thread) --------------------- + try: + text = await asyncio.to_thread(engine.transcribe, audio_path) + except Exception as exc: + self._add_message("System", f"[red]Transcription error: {exc}[/red]") + continue + finally: + try: + import os as _os + + _os.remove(audio_path) + except OSError: + pass + + if not text: + continue + + if not self._voice_active: + break + + # ---- send transcribed text through the normal chat pipeline ---- + self._handle_user_message(text) + + # ---- wait for the LLM response to finish ----------------------- + # _generate_response is @work(exclusive=True); poll the flag it sets. + await asyncio.sleep(0.3) # give the worker a moment to start + while self.agent.generating: + await asyncio.sleep(0.2) + + if not self._voice_active: + break + + # ---- speak the last assistant message -------------------------- + last_assistant = next( + (m.content for m in reversed(self.messages) if m.role == "assistant"), + None, + ) + if last_assistant: + try: + await asyncio.to_thread(engine.speak, last_assistant) + except Exception as exc: + self._add_message("System", f"[red]TTS error: {exc}[/red]") + def _handle_user_message(self, text: str) -> None: ts = datetime.now(timezone.utc).isoformat() self.messages.append(Message(role="user", content=text, timestamp=ts)) diff --git a/oli_bot/config.py b/oli_bot/config.py index 59e8154..32de27d 100644 --- a/oli_bot/config.py +++ b/oli_bot/config.py @@ -66,6 +66,20 @@ class AppConfig(BaseSettings): transformers_dtype: str = Field(default="auto") transformers_is_multi_model: bool = Field(default=False) + # Voice mode (/voice command) — see oli_bot/voice.py + voice_whisper_model: str = Field(default="base") + voice_piper_model: str = Field(default="en_US-lessac-medium.onnx") + # Sample rate required by WebRTC VAD + voice_sample_rate: int = Field(default=16000) + # WebRTC VAD constraint: 10 | 20 | 30 + voice_frame_duration_ms: int = Field(default=30) + # 0–3; higher = more aggressive noise rejection + voice_vad_aggressiveness: int = Field(default=2, ge=0, le=3) + # Stop recording after this many ms of consecutive silence + voice_silence_timeout_ms: int = Field(default=800, gt=0) + # Hard cap to prevent runaway recordings + voice_max_record_seconds: int = Field(default=15, gt=0) + # General Agent configs use_agent_pool: bool = Field(default=False) agent_pool_size: int = Field(default=5) diff --git a/oli_bot/profiles/coder/AGENTS.md b/oli_bot/profiles/coder/AGENTS.md index 893a7b9..6069f69 100644 --- a/oli_bot/profiles/coder/AGENTS.md +++ b/oli_bot/profiles/coder/AGENTS.md @@ -33,3 +33,7 @@ verify your work, and prefer targeted edits over full rewrites. - Prefer inline code fences with the correct language tag. - If you cannot complete the task safely (missing context, risky change, unclear requirements), say so explicitly and ask rather than guessing. + +# AGENTS.md file for projects + +If an AGENTS.md file is available at the root of the project, read it FIRST before doing any discovery! This should give you an overview of the project and what's expected. \ No newline at end of file diff --git a/oli_bot/profiles/default/AGENTS.md b/oli_bot/profiles/default/AGENTS.md index 59973e7..2dbbf45 100644 --- a/oli_bot/profiles/default/AGENTS.md +++ b/oli_bot/profiles/default/AGENTS.md @@ -93,3 +93,7 @@ Example safe commands: ## MCP server tools If MCP servers are configured, their tools are available as `__`. Use them alongside built-in tools as needed. List available tools via the API; the tool definitions are sent automatically. + +# AGENTS.md file for projects + +If an AGENTS.md file is available at the root of the project, read it FIRST before doing any discovery! This should give you an overview of the project and what's expected. diff --git a/oli_bot/screens/config_screen.py b/oli_bot/screens/config_screen.py index 60063fd..64bef94 100644 --- a/oli_bot/screens/config_screen.py +++ b/oli_bot/screens/config_screen.py @@ -369,6 +369,51 @@ def compose(self) -> ComposeResult: value=str(mp.get("agent_pool_size", 5)), ) + vo = s.get("voice", {}) + yield Label("Voice", classes="section-title") + yield Input( + placeholder=f"Whisper model ({vo.get('whisper_model', 'base')})", + id="cfg-voice-whisper-model", + classes="config-input", + value=vo.get("whisper_model", "base"), + ) + yield Input( + placeholder="Piper model path (.onnx)", + id="cfg-voice-piper-model", + classes="config-input", + value=vo.get("piper_model", "en_US-lessac-medium.onnx"), + ) + yield Input( + placeholder=f"Sample rate ({vo.get('sample_rate', 16000)})", + id="cfg-voice-sample-rate", + classes="config-input", + value=str(vo.get("sample_rate", 16000)), + ) + yield Input( + placeholder=f"Frame duration ms, 10|20|30 ({vo.get('frame_duration_ms', 30)})", + id="cfg-voice-frame-duration-ms", + classes="config-input", + value=str(vo.get("frame_duration_ms", 30)), + ) + yield Input( + placeholder=f"VAD aggressiveness 0-3 ({vo.get('vad_aggressiveness', 2)})", + id="cfg-voice-vad-aggressiveness", + classes="config-input", + value=str(vo.get("vad_aggressiveness", 2)), + ) + yield Input( + placeholder=f"Silence timeout ms ({vo.get('silence_timeout_ms', 800)})", + id="cfg-voice-silence-timeout-ms", + classes="config-input", + value=str(vo.get("silence_timeout_ms", 800)), + ) + yield Input( + placeholder=f"Max record seconds ({vo.get('max_record_seconds', 15)})", + id="cfg-voice-max-record-seconds", + classes="config-input", + value=str(vo.get("max_record_seconds", 15)), + ) + lg = s.get("logging", {}) yield Label("Logging", classes="section-title") yield Label("Log level", classes="config-label") @@ -568,6 +613,15 @@ def _save(self) -> None: "profiles_dir": self._val("#cfg-profiles-dir"), "logs_dir": self._val("#cfg-logs-dir"), }, + "voice": { + "whisper_model": self._val("#cfg-voice-whisper-model"), + "piper_model": self._val("#cfg-voice-piper-model"), + "sample_rate": self._int("#cfg-voice-sample-rate", 16000), + "frame_duration_ms": self._int("#cfg-voice-frame-duration-ms", 30), + "vad_aggressiveness": self._int("#cfg-voice-vad-aggressiveness", 2), + "silence_timeout_ms": self._int("#cfg-voice-silence-timeout-ms", 800), + "max_record_seconds": self._int("#cfg-voice-max-record-seconds", 15), + }, "workspace": { "max_workspaces": self._int("#cfg-max-workspaces", 20), }, diff --git a/oli_bot/settings.py b/oli_bot/settings.py index 07de6ba..dbb07ee 100644 --- a/oli_bot/settings.py +++ b/oli_bot/settings.py @@ -77,6 +77,15 @@ "auto_save": True, "resume_prompt": True, }, + "voice": { + "whisper_model": "base", + "piper_model": "en_US-lessac-medium.onnx", + "sample_rate": 16000, + "frame_duration_ms": 30, + "vad_aggressiveness": 2, + "silence_timeout_ms": 800, + "max_record_seconds": 15, + }, } # Mapping: env var -> dot-separated path into settings dict @@ -124,6 +133,13 @@ "OLI_API_MODE": "api_server.mode", "OLI_PROFILES_DIR": "paths.profiles_dir", "OLI_LOGS_DIR": "paths.logs_dir", + "OLI_VOICE_WHISPER_MODEL": "voice.whisper_model", + "OLI_VOICE_PIPER_MODEL": "voice.piper_model", + "OLI_VOICE_SAMPLE_RATE": "voice.sample_rate", + "OLI_VOICE_FRAME_DURATION_MS": "voice.frame_duration_ms", + "OLI_VOICE_VAD_AGGRESSIVENESS": "voice.vad_aggressiveness", + "OLI_VOICE_SILENCE_TIMEOUT_MS": "voice.silence_timeout_ms", + "OLI_VOICE_MAX_RECORD_SECONDS": "voice.max_record_seconds", } # SDK-standard env names accepted as fallbacks when the OLI_-prefixed @@ -263,6 +279,7 @@ def to_appconfig(self, settings: dict) -> AppConfig: lg = settings.get("logging", {}) api = settings.get("api_server", {}) paths = settings.get("paths", {}) + voice = settings.get("voice", {}) openai_key = ( op.get("api_key", "") or os.environ.get("OLI_OPENAI_API_KEY", "") @@ -330,6 +347,13 @@ def to_appconfig(self, settings: dict) -> AppConfig: api_mode=api.get("mode", "agent"), profiles_dir=paths.get("profiles_dir", "profiles"), logs_dir=paths.get("logs_dir", "logs"), + voice_whisper_model=voice.get("whisper_model", "base"), + voice_piper_model=voice.get("piper_model", "en_US-lessac-medium.onnx"), + voice_sample_rate=voice.get("sample_rate", 16000), + voice_frame_duration_ms=voice.get("frame_duration_ms", 30), + voice_vad_aggressiveness=voice.get("vad_aggressiveness", 2), + voice_silence_timeout_ms=voice.get("silence_timeout_ms", 800), + voice_max_record_seconds=voice.get("max_record_seconds", 15), ) def from_appconfig(self, config: AppConfig) -> dict: @@ -382,6 +406,13 @@ def from_appconfig(self, config: AppConfig) -> dict: settings["api_server"]["mode"] = config.api_mode settings["paths"]["profiles_dir"] = config.profiles_dir settings["paths"]["logs_dir"] = config.logs_dir + settings["voice"]["whisper_model"] = config.voice_whisper_model + settings["voice"]["piper_model"] = config.voice_piper_model + settings["voice"]["sample_rate"] = config.voice_sample_rate + settings["voice"]["frame_duration_ms"] = config.voice_frame_duration_ms + settings["voice"]["vad_aggressiveness"] = config.voice_vad_aggressiveness + settings["voice"]["silence_timeout_ms"] = config.voice_silence_timeout_ms + settings["voice"]["max_record_seconds"] = config.voice_max_record_seconds return settings diff --git a/oli_bot/voice.py b/oli_bot/voice.py new file mode 100644 index 0000000..2120c0f --- /dev/null +++ b/oli_bot/voice.py @@ -0,0 +1,292 @@ +""" +Voice I/O engine for the /voice command. + +Wraps faster-whisper (STT), piper-tts (TTS), pyaudio (mic capture), +webrtcvad (voice-activity detection) and simpleaudio (playback). + +All heavy operations are designed to run in a background thread via +``asyncio.to_thread()`` so they never block Textual's event loop. + +Configuration +------------- +All values resolve through ``AppConfig`` (oli_bot/config.py) with the usual +precedence: settings.json > ``OLI_*`` environment variables > defaults. + +OLI_VOICE_WHISPER_MODEL Whisper model size: tiny | base (default) | + small | medium | large +OLI_VOICE_PIPER_MODEL Path to the local Piper ONNX model file. + Default: en_US-lessac-medium.onnx +OLI_VOICE_SAMPLE_RATE Mic sample rate (16000; required by WebRTC VAD) +OLI_VOICE_FRAME_DURATION_MS VAD frame size (10 | 20 | 30; default 30) +OLI_VOICE_VAD_AGGRESSIVENESS 0-3; higher = more aggressive noise rejection +OLI_VOICE_SILENCE_TIMEOUT_MS Stop after this much silence (default 800) +OLI_VOICE_MAX_RECORD_SECONDS Hard cap on recording length (default 15) + +Download a Piper model before first use: + wget https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx + wget https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx.json + export OLI_VOICE_PIPER_MODEL=/path/to/en_US-lessac-medium.onnx +""" + +from __future__ import annotations + +import logging +import os +import tempfile +import threading +import wave +from typing import Optional + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Defaults sourced from AppConfig — overridable via OLI_VOICE_* env vars, +# .env, or settings.json (see docs/CONFIGURE.md). Callers that hold their own +# AppConfig (e.g. the TUI) should pass explicit values to VoiceEngine() since +# the module-level `configs` singleton is an import-time snapshot. +# --------------------------------------------------------------------------- +from .config import configs as _configs + +WHISPER_MODEL_SIZE = _configs.voice_whisper_model +PIPER_MODEL_PATH = _configs.voice_piper_model +SAMPLE_RATE = _configs.voice_sample_rate +FRAME_DURATION_MS = _configs.voice_frame_duration_ms +VAD_AGGRESSIVENESS = _configs.voice_vad_aggressiveness +SILENCE_TIMEOUT_MS = _configs.voice_silence_timeout_ms +MAX_RECORD_SECONDS = _configs.voice_max_record_seconds + + +class VoiceEngine: + """Lazy-loading, stateful voice I/O engine. + + Instantiate once per session. Models are loaded on the first call to + :meth:`load` (or automatically by the first voice-mode activation) so + that startup latency stays near zero. + + All public methods that perform I/O are **blocking** — wrap them in + ``asyncio.to_thread(engine.method, ...)`` inside async callers. + """ + + def __init__( + self, + whisper_model_size: str = WHISPER_MODEL_SIZE, + piper_model_path: str = PIPER_MODEL_PATH, + sample_rate: int = SAMPLE_RATE, + frame_duration_ms: int = FRAME_DURATION_MS, + vad_aggressiveness: int = VAD_AGGRESSIVENESS, + silence_timeout_ms: int = SILENCE_TIMEOUT_MS, + max_record_seconds: int = MAX_RECORD_SECONDS, + ) -> None: + self.whisper_model_size = whisper_model_size + self.piper_model_path = piper_model_path + self.sample_rate = sample_rate + self.frame_duration_ms = frame_duration_ms + self.vad_aggressiveness = vad_aggressiveness + self.silence_timeout_ms = silence_timeout_ms + self.max_record_seconds = max_record_seconds + + self._whisper = None # faster_whisper.WhisperModel (set by load()) + self._piper = None # piper.PiperVoice (set by load()) + self._vad = None # webrtcvad.Vad (set by load()) + self._loaded = False + + # ------------------------------------------------------------------ + # Model loading + # ------------------------------------------------------------------ + + def load(self) -> None: + """Load Whisper, Piper, and WebRTC VAD models. + + Idempotent — safe to call multiple times; only runs once. + BLOCKING — call via ``asyncio.to_thread(engine.load)``. + """ + if self._loaded: + return + + logger.info( + "VoiceEngine: loading models (whisper=%s, piper=%s)", + self.whisper_model_size, + self.piper_model_path, + ) + + # ---- faster-whisper ------------------------------------------------ + try: + from faster_whisper import WhisperModel # type: ignore[import] + + self._whisper = WhisperModel( + self.whisper_model_size, device="cpu", compute_type="int8" + ) + except ImportError as exc: + raise RuntimeError( + "faster-whisper is required for voice mode. " + "Install it with: pip install 'oli-bot[voice]'" + ) from exc + + # ---- piper-tts ----------------------------------------------------- + try: + from piper import PiperVoice # type: ignore[import] + + self._piper = PiperVoice.load(self.piper_model_path) + except ImportError as exc: + raise RuntimeError( + "piper-tts is required for voice mode. " + "Install it with: pip install 'oli-bot[voice]'" + ) from exc + except Exception as exc: + raise RuntimeError( + f"Failed to load Piper TTS model '{self.piper_model_path}'. " + "Download the model from " + "https://huggingface.co/rhasspy/piper-voices and set " + "OLI_VOICE_PIPER_MODEL to the .onnx file path." + ) from exc + + # ---- webrtcvad ----------------------------------------------------- + try: + import webrtcvad # type: ignore[import] + + self._vad = webrtcvad.Vad(self.vad_aggressiveness) + except ImportError as exc: + raise RuntimeError( + "webrtcvad is required for voice mode. " + "Install it with: pip install 'oli-bot[voice]'" + ) from exc + + self._loaded = True + logger.info("VoiceEngine: all models loaded.") + + # ------------------------------------------------------------------ + # Audio capture + # ------------------------------------------------------------------ + + def record(self, stop_event: Optional[threading.Event] = None) -> Optional[str]: + """Capture microphone audio until silence or the maximum duration. + + Uses WebRTC VAD to automatically stop when the user stops speaking. + + ``stop_event``, if given, is polled every frame so a caller can abort + the recording early (e.g. voice mode was toggled off mid-listen). + + Returns a path to a temporary WAV file containing the captured audio, + or ``None`` if no speech was detected. The caller is responsible for + deleting the file after use. + + BLOCKING — call via ``asyncio.to_thread(engine.record)``. + """ + try: + import pyaudio # type: ignore[import] + except ImportError as exc: + raise RuntimeError( + "pyaudio is required for voice mode. " + "Install it with: pip install 'oli-bot[voice]'" + ) from exc + + chunk = int(self.sample_rate * self.frame_duration_ms / 1000) # frames per chunk + channels = 1 + fmt = pyaudio.paInt16 + + mic = pyaudio.PyAudio() + stream = mic.open( + format=fmt, + channels=channels, + rate=self.sample_rate, + input=True, + frames_per_buffer=chunk, + ) + + max_frames = int(self.max_record_seconds * 1000 / self.frame_duration_ms) + silence_frames_needed = self.silence_timeout_ms // self.frame_duration_ms + + triggered = False # True once we've heard the first speech frame + silence_count = 0 + frames: list[bytes] = [] + + try: + for _ in range(max_frames): + if stop_event is not None and stop_event.is_set(): + return None + frame = stream.read(chunk, exception_on_overflow=False) + is_speech = self._vad.is_speech(frame, self.sample_rate) + + if not triggered: + if is_speech: + triggered = True + frames.append(frame) + else: + frames.append(frame) + if is_speech: + silence_count = 0 + else: + silence_count += 1 + if silence_count >= silence_frames_needed: + break + finally: + stream.stop_stream() + stream.close() + mic.terminate() + + if not frames: + logger.debug("VoiceEngine.record: no speech detected.") + return None + + # Write captured frames to a temp WAV file + tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + with wave.open(tmp.name, "wb") as wf: + wf.setnchannels(channels) + wf.setsampwidth(mic.get_sample_size(fmt)) + wf.setframerate(self.sample_rate) + wf.writeframes(b"".join(frames)) + + logger.debug("VoiceEngine.record: saved %d frames to %s.", len(frames), tmp.name) + return tmp.name + + # ------------------------------------------------------------------ + # Speech-to-text + # ------------------------------------------------------------------ + + def transcribe(self, audio_path: str) -> str: + """Transcribe a WAV file to text using faster-whisper. + + Returns the transcribed string (may be empty if whisper found nothing). + + BLOCKING — call via ``asyncio.to_thread(engine.transcribe, path)``. + """ + segments, _info = self._whisper.transcribe(audio_path, beam_size=5) + text = " ".join(seg.text.strip() for seg in segments).strip() + logger.debug("VoiceEngine.transcribe: %r", text) + return text + + # ------------------------------------------------------------------ + # Text-to-speech + playback + # ------------------------------------------------------------------ + + def speak(self, text: str) -> None: + """Synthesize ``text`` to a WAV file via Piper and play it back. + + The temporary WAV file is always deleted after playback (or on error). + + BLOCKING — call via ``asyncio.to_thread(engine.speak, text)``. + """ + try: + import simpleaudio as sa # type: ignore[import] + except ImportError as exc: + raise RuntimeError( + "simpleaudio is required for voice mode. " + "Install it with: pip install 'oli-bot[voice]'" + ) from exc + + tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + tmp_path = tmp.name + tmp.close() # close so wave.open can reopen it on Windows + + try: + with wave.open(tmp_path, "wb") as wf: + self._piper.synthesize(text, wf) + + wave_obj = sa.WaveObject.from_wave_file(tmp_path) + play_obj = wave_obj.play() + play_obj.wait_done() + finally: + try: + os.remove(tmp_path) + except OSError: + pass diff --git a/pyproject.toml b/pyproject.toml index b0e10fd..d9506ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,13 @@ dev = [ "pytest-asyncio>=0.24", "black>=26.5.1", ] +voice = [ + "pyaudio>=0.2.14", + "webrtcvad-wheels>=2.0.10", + "faster-whisper>=1.0", + "piper-tts>=1.2", + "simpleaudio>=1.0.4", +] [project.scripts] oli = "oli_bot.chat:main" diff --git a/tests/test_config_screen.py b/tests/test_config_screen.py index 86f74fa..387de76 100644 --- a/tests/test_config_screen.py +++ b/tests/test_config_screen.py @@ -65,6 +65,52 @@ async def test_config_screen_save_keeps_transformers_small_model(): assert result["ollama"]["small_model"] == "ollama-sm" +async def test_config_screen_voice_section_round_trip(): + settings = _settings() + settings["voice"] = { + "whisper_model": "small", + "piper_model": "/models/custom.onnx", + "sample_rate": 16000, + "frame_duration_ms": 20, + "vad_aggressiveness": 3, + "silence_timeout_ms": 1200, + "max_record_seconds": 30, + } + app = _HostApp(settings) + async with app.run_test() as pilot: + screen = app.screen + assert screen.query_one("#cfg-voice-whisper-model").value == "small" + assert screen.query_one("#cfg-voice-piper-model").value == "/models/custom.onnx" + assert screen.query_one("#cfg-voice-frame-duration-ms").value == "20" + assert screen.query_one("#cfg-voice-vad-aggressiveness").value == "3" + assert screen.query_one("#cfg-voice-silence-timeout-ms").value == "1200" + assert screen.query_one("#cfg-voice-max-record-seconds").value == "30" + screen._save() + await pilot.pause() + voice = app.result["voice"] + assert voice["whisper_model"] == "small" + assert voice["piper_model"] == "/models/custom.onnx" + assert voice["sample_rate"] == 16000 + assert voice["frame_duration_ms"] == 20 + assert voice["vad_aggressiveness"] == 3 + assert voice["silence_timeout_ms"] == 1200 + assert voice["max_record_seconds"] == 30 + + +async def test_config_screen_voice_section_defaults_when_absent(): + app = _HostApp(_settings()) # no "voice" key + async with app.run_test() as pilot: + screen = app.screen + assert screen.query_one("#cfg-voice-whisper-model").value == "base" + screen._save() + await pilot.pause() + voice = app.result["voice"] + assert voice["whisper_model"] == "base" + assert voice["piper_model"] == "en_US-lessac-medium.onnx" + assert voice["sample_rate"] == 16000 + assert voice["vad_aggressiveness"] == 2 + + async def test_config_screen_renders_new_sections(): settings = _settings() settings["openai"]["vision_style"] = "bedrock" diff --git a/tests/test_config_settings.py b/tests/test_config_settings.py index 34fcfe2..f7f22f0 100644 --- a/tests/test_config_settings.py +++ b/tests/test_config_settings.py @@ -3,6 +3,9 @@ import json from oli_bot.config import AppConfig +from oli_bot.chat import OliBot +from oli_bot.models import HostConfig +from oli_bot.server_manager import ServerManager from oli_bot.settings import SettingsManager @@ -75,6 +78,37 @@ def test_settings_manager_created_file_reflects_env(tmp_path, monkeypatch): ) +def test_voice_settings_round_trip(tmp_path): + mgr = SettingsManager(config_dir=tmp_path) + cfg = AppConfig( + _env_file=None, + voice_whisper_model="small", + voice_piper_model="/models/custom.onnx", + voice_vad_aggressiveness=3, + voice_silence_timeout_ms=1200, + voice_max_record_seconds=30, + ) + settings = mgr.from_appconfig(cfg) + assert settings["voice"]["whisper_model"] == "small" + assert settings["voice"]["vad_aggressiveness"] == 3 + back = mgr.to_appconfig(settings) + assert back.voice_whisper_model == "small" + assert back.voice_piper_model == "/models/custom.onnx" + assert back.voice_vad_aggressiveness == 3 + assert back.voice_silence_timeout_ms == 1200 + assert back.voice_max_record_seconds == 30 + + +def test_voice_env_coercion(tmp_path, monkeypatch): + monkeypatch.setenv("OLI_VOICE_MAX_RECORD_SECONDS", "30") + monkeypatch.setenv("OLI_VOICE_VAD_AGGRESSIVENESS", "1") + mgr = SettingsManager(config_dir=tmp_path) + settings = mgr.load() + cfg = mgr.to_appconfig(settings) + assert cfg.voice_max_record_seconds == 30 + assert cfg.voice_vad_aggressiveness == 1 + + def test_settings_manager_created_file_uses_source_defaults(tmp_path): mgr = SettingsManager(config_dir=tmp_path) source = AppConfig( @@ -214,3 +248,76 @@ def test_env_to_settings_covers_all_appconfig_fields(tmp_path): assert getattr(reloaded, field_name) == getattr( cfg, field_name ), f"Field {field_name} does not survive from_appconfig/to_appconfig round-trip" + + +def test_runtime_settings_sync_covers_all_appconfig_fields(tmp_path): + mgr = SettingsManager(config_dir=tmp_path) + cfg = AppConfig( + _env_file=None, + backend="openai", + openai_api_key="openai-key", + openai_base_url="https://openai.example/v1", + openai_model="openai-large", + openai_small_model="openai-small", + openai_vision_style="bedrock", + openai_optional_headers={"X-Test": "yes"}, + ollama_base_url="http://ollama.example:11434", + ollama_model="ollama-large", + ollama_small_model="ollama-small", + huggingface_base_url="https://hf.example", + huggingface_api_key="hf-key", + huggingface_model="hf-large", + huggingface_small_model="hf-small", + huggingface_remote=True, + transformers_model="tr-large", + transformers_small_model="tr-small", + transformers_device="cpu", + transformers_dtype="float32", + transformers_is_multi_model=True, + use_agent_pool=True, + agent_pool_size=11, + max_tokens=3333, + temperature=0.33, + max_retries=9, + retry_delay=2.5, + request_timeout=61.0, + max_messages=44, + max_tool_iterations=12, + stream_timeout=123.0, + model_filters=":cloud", + profiles_dir="custom-profiles", + logs_dir="custom-logs", + truncation_max_chars_small=222, + truncation_max_chars_large=22222, + dry_run=True, + offline_mode=False, + log_level="DEBUG", + log_file="logs/custom.ndjson", + api_host="127.0.0.1", + api_port=9999, + api_profile="analyst", + api_mode="ask", + ) + server_manager = ServerManager(config_path=str(tmp_path / "hosts.json")) + server_manager.servers.append( + HostConfig(name="test", url="http://localhost:11434", active=True) + ) + + bot = object.__new__(OliBot) + bot.settings_manager = mgr + bot.config = cfg + bot.settings = mgr.get_defaults() + bot.settings["workspace"]["max_workspaces"] = 37 + bot.settings["session"]["auto_save"] = False + bot.settings["session"]["resume_prompt"] = False + bot.server_manager = server_manager + + bot._sync_settings_from_runtime() + + reloaded = mgr.to_appconfig(bot.settings) + for field_name in AppConfig.model_fields: + assert getattr(reloaded, field_name) == getattr( + cfg, field_name + ), f"Field {field_name} was omitted by OliBot._sync_settings_from_runtime" + assert bot.settings["workspace"] == {"max_workspaces": 37} + assert bot.settings["session"] == {"auto_save": False, "resume_prompt": False} diff --git a/tests/test_voice.py b/tests/test_voice.py new file mode 100644 index 0000000..ef696d5 --- /dev/null +++ b/tests/test_voice.py @@ -0,0 +1,561 @@ +"""Unit tests for oli_bot/voice.py. + +All external dependencies (pyaudio, webrtcvad, faster_whisper, piper, +simpleaudio) are mocked so the suite runs without any hardware or heavy +ML packages installed. +""" +from __future__ import annotations + +import os +import sys +import tempfile +import types +import wave +from unittest.mock import MagicMock, call, patch + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_wav(path: str, n_frames: int = 480) -> None: + """Write a minimal valid 16-bit mono 16 kHz WAV file at *path*.""" + with wave.open(path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(16_000) + wf.writeframes(b"\x00\x01" * n_frames) + + +def _stub_module(name: str, **attrs) -> types.ModuleType: + """Return a stub module registered in sys.modules under *name*.""" + mod = types.ModuleType(name) + for k, v in attrs.items(): + setattr(mod, k, v) + sys.modules[name] = mod + return mod + + +def _remove_module(name: str) -> None: + sys.modules.pop(name, None) + + +def _write_wav_into_handle(text: str, wf) -> None: + """Side-effect for mock piper.synthesize: write a minimal valid WAV header. + + Piper's real synthesize() calls wf.setnchannels / setsampwidth / + setframerate / writeframes on the wave.Wave_write handle it receives. + Without this, wave.close() raises 'channels not specified'. + """ + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(22_050) + wf.writeframes(b"\x00\x01" * 100) + + +# --------------------------------------------------------------------------- +# VoiceEngine.load — happy path +# --------------------------------------------------------------------------- + + +class TestVoiceEngineLoad: + def _patch_all_deps(self): + """Return a context that stubs all three voice deps.""" + mock_whisper_cls = MagicMock(return_value=MagicMock()) + mock_piper_voice = MagicMock() + mock_piper_voice.load = MagicMock(return_value=MagicMock()) + mock_vad_cls = MagicMock(return_value=MagicMock()) + + fw_mod = _stub_module("faster_whisper", WhisperModel=mock_whisper_cls) + pip_mod = _stub_module("piper", PiperVoice=mock_piper_voice) + vad_mod = _stub_module("webrtcvad", Vad=mock_vad_cls) + + return fw_mod, pip_mod, vad_mod, mock_whisper_cls, mock_piper_voice, mock_vad_cls + + def teardown_method(self): + for name in ("faster_whisper", "piper", "webrtcvad"): + _remove_module(name) + + def test_load_sets_loaded_flag(self): + self._patch_all_deps() + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine(whisper_model_size="tiny", piper_model_path="dummy.onnx") + engine.load() + assert engine._loaded is True + + def test_load_is_idempotent(self): + """Calling load() twice must not re-instantiate models.""" + _, _, _, mock_whisper_cls, mock_piper_voice, _ = self._patch_all_deps() + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine(whisper_model_size="tiny", piper_model_path="dummy.onnx") + engine.load() + engine.load() # second call — should be a no-op + + assert mock_whisper_cls.call_count == 1 + assert mock_piper_voice.load.call_count == 1 + + def test_load_stores_model_references(self): + self._patch_all_deps() + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine(whisper_model_size="tiny", piper_model_path="dummy.onnx") + engine.load() + + assert engine._whisper is not None + assert engine._piper is not None + assert engine._vad is not None + + +# --------------------------------------------------------------------------- +# VoiceEngine.load — missing dependency errors +# --------------------------------------------------------------------------- + + +class TestVoiceEngineLoadErrors: + def teardown_method(self): + for name in ("faster_whisper", "piper", "webrtcvad"): + _remove_module(name) + + def test_missing_faster_whisper_raises_runtime_error(self): + # Ensure the import fails + sys.modules["faster_whisper"] = None # type: ignore[assignment] + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine.__new__(VoiceEngine) + engine._loaded = False + engine.whisper_model_size = "base" + engine.piper_model_path = "dummy.onnx" + engine._whisper = engine._piper = engine._vad = None + + with pytest.raises(RuntimeError, match="faster-whisper"): + engine.load() + + def test_missing_piper_raises_runtime_error(self): + _stub_module("faster_whisper", WhisperModel=MagicMock(return_value=MagicMock())) + sys.modules["piper"] = None # type: ignore[assignment] + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine.__new__(VoiceEngine) + engine._loaded = False + engine.whisper_model_size = "base" + engine.piper_model_path = "dummy.onnx" + engine._whisper = engine._piper = engine._vad = None + + with pytest.raises(RuntimeError, match="piper-tts"): + engine.load() + + def test_missing_webrtcvad_raises_runtime_error(self): + _stub_module("faster_whisper", WhisperModel=MagicMock(return_value=MagicMock())) + mock_pv = MagicMock() + mock_pv.load = MagicMock(return_value=MagicMock()) + _stub_module("piper", PiperVoice=mock_pv) + sys.modules["webrtcvad"] = None # type: ignore[assignment] + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine.__new__(VoiceEngine) + engine._loaded = False + engine.whisper_model_size = "base" + engine.piper_model_path = "dummy.onnx" + engine._whisper = engine._piper = engine._vad = None + + with pytest.raises(RuntimeError, match="webrtcvad"): + engine.load() + + def test_bad_piper_model_path_raises_runtime_error(self): + _stub_module("faster_whisper", WhisperModel=MagicMock(return_value=MagicMock())) + mock_pv = MagicMock() + mock_pv.load = MagicMock(side_effect=FileNotFoundError("not found")) + _stub_module("piper", PiperVoice=mock_pv) + _stub_module("webrtcvad", Vad=MagicMock(return_value=MagicMock())) + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine.__new__(VoiceEngine) + engine._loaded = False + engine.whisper_model_size = "base" + engine.piper_model_path = "/nonexistent/model.onnx" + engine._whisper = engine._piper = engine._vad = None + + with pytest.raises(RuntimeError, match="Piper"): + engine.load() + + +# --------------------------------------------------------------------------- +# VoiceEngine.transcribe +# --------------------------------------------------------------------------- + + +class TestVoiceEngineTranscribe: + def _make_engine(self) -> "VoiceEngine": + """Return a VoiceEngine with models pre-stubbed (skipping load()).""" + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine.__new__(VoiceEngine) + engine._loaded = True + engine.whisper_model_size = "base" + engine.piper_model_path = "dummy.onnx" + engine._piper = MagicMock() + engine._vad = MagicMock() + return engine + + def test_transcribe_joins_segments(self): + engine = self._make_engine() + seg1 = MagicMock(); seg1.text = " Hello " + seg2 = MagicMock(); seg2.text = " world" + mock_model = MagicMock() + mock_model.transcribe.return_value = ([seg1, seg2], MagicMock()) + engine._whisper = mock_model + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + path = f.name + try: + _make_wav(path) + result = engine.transcribe(path) + finally: + os.unlink(path) + + assert result == "Hello world" + + def test_transcribe_empty_segments_returns_empty_string(self): + engine = self._make_engine() + mock_model = MagicMock() + mock_model.transcribe.return_value = ([], MagicMock()) + engine._whisper = mock_model + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + path = f.name + try: + _make_wav(path) + result = engine.transcribe(path) + finally: + os.unlink(path) + + assert result == "" + + def test_transcribe_strips_whitespace(self): + engine = self._make_engine() + seg = MagicMock(); seg.text = " spaces " + mock_model = MagicMock() + mock_model.transcribe.return_value = ([seg], MagicMock()) + engine._whisper = mock_model + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + path = f.name + try: + _make_wav(path) + result = engine.transcribe(path) + finally: + os.unlink(path) + + assert result == "spaces" + + +# --------------------------------------------------------------------------- +# VoiceEngine.speak +# --------------------------------------------------------------------------- + + +class TestVoiceEngineSpeak: + def _make_engine(self) -> "VoiceEngine": + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine.__new__(VoiceEngine) + engine._loaded = True + engine.whisper_model_size = "base" + engine.piper_model_path = "dummy.onnx" + engine._whisper = MagicMock() + engine._vad = MagicMock() + # synthesize must write a valid WAV header so wave.close() doesn't raise + mock_piper = MagicMock() + mock_piper.synthesize.side_effect = _write_wav_into_handle + engine._piper = mock_piper + return engine + + def teardown_method(self): + _remove_module("simpleaudio") + + def test_speak_synthesizes_and_plays(self): + engine = self._make_engine() + + mock_play_obj = MagicMock() + mock_wave_obj = MagicMock() + mock_wave_obj.play.return_value = mock_play_obj + mock_sa = _stub_module("simpleaudio") + mock_sa.WaveObject = MagicMock() + mock_sa.WaveObject.from_wave_file = MagicMock(return_value=mock_wave_obj) + + engine.speak("Hello there") + + engine._piper.synthesize.assert_called_once() + mock_play_obj.wait_done.assert_called_once() + + def test_speak_cleans_up_temp_file_on_success(self): + engine = self._make_engine() + + created: list[str] = [] + _orig_ntf = tempfile.NamedTemporaryFile + + def tracking_ntf(*args, **kwargs): + f = _orig_ntf(*args, **kwargs) + created.append(f.name) + return f + + mock_play_obj = MagicMock() + mock_wave_obj = MagicMock() + mock_wave_obj.play.return_value = mock_play_obj + mock_sa = _stub_module("simpleaudio") + mock_sa.WaveObject = MagicMock() + mock_sa.WaveObject.from_wave_file = MagicMock(return_value=mock_wave_obj) + + with patch("tempfile.NamedTemporaryFile", side_effect=tracking_ntf): + engine.speak("cleanup test") + + for p in created: + assert not os.path.exists(p), f"Temp file not removed: {p}" + + def test_speak_cleans_up_temp_file_on_playback_error(self): + """Temp file must be removed even when simpleaudio raises.""" + engine = self._make_engine() + + created: list[str] = [] + _orig_ntf = tempfile.NamedTemporaryFile + + def tracking_ntf(*args, **kwargs): + f = _orig_ntf(*args, **kwargs) + created.append(f.name) + return f + + mock_sa = _stub_module("simpleaudio") + mock_sa.WaveObject = MagicMock() + mock_sa.WaveObject.from_wave_file = MagicMock(side_effect=RuntimeError("boom")) + + with patch("tempfile.NamedTemporaryFile", side_effect=tracking_ntf): + with pytest.raises(RuntimeError, match="boom"): + engine.speak("error test") + + for p in created: + assert not os.path.exists(p), f"Temp file not removed after error: {p}" + + def test_speak_raises_when_simpleaudio_missing(self): + engine = self._make_engine() + sys.modules["simpleaudio"] = None # type: ignore[assignment] + + with pytest.raises(RuntimeError, match="simpleaudio"): + engine.speak("missing dep") + + +# --------------------------------------------------------------------------- +# VoiceEngine.record +# --------------------------------------------------------------------------- + + +class TestVoiceEngineRecord: + def _make_engine(self) -> "VoiceEngine": + from oli_bot import voice as _v + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine.__new__(VoiceEngine) + engine._loaded = True + engine.whisper_model_size = "base" + engine.piper_model_path = "dummy.onnx" + engine.sample_rate = _v.SAMPLE_RATE + engine.frame_duration_ms = _v.FRAME_DURATION_MS + engine.vad_aggressiveness = _v.VAD_AGGRESSIVENESS + engine.silence_timeout_ms = _v.SILENCE_TIMEOUT_MS + engine.max_record_seconds = _v.MAX_RECORD_SECONDS + engine._whisper = MagicMock() + engine._piper = MagicMock() + return engine + + def teardown_method(self): + _remove_module("pyaudio") + + def _setup_pyaudio_mock(self, is_speech_seq: list[bool]): + """Return a fully mocked pyaudio module whose VAD sequence follows *is_speech_seq*.""" + CHUNK = 480 # 30 ms @ 16 kHz + + mock_stream = MagicMock() + mock_stream.read.return_value = b"\x00\x01" * CHUNK + + mock_pa_instance = MagicMock() + mock_pa_instance.open.return_value = mock_stream + mock_pa_instance.get_sample_size.return_value = 2 + + mock_pa_cls = MagicMock(return_value=mock_pa_instance) + + pa_mod = _stub_module("pyaudio", PyAudio=mock_pa_cls, paInt16=8) + + return pa_mod, mock_pa_instance, mock_stream, mock_pa_cls + + def test_returns_none_when_no_speech_detected(self): + engine = self._make_engine() + self._setup_pyaudio_mock([]) + + mock_vad = MagicMock() + mock_vad.is_speech.return_value = False + engine._vad = mock_vad + + result = engine.record() + assert result is None + + def test_returns_wav_path_when_speech_detected(self): + engine = self._make_engine() + self._setup_pyaudio_mock([]) + + # Speech on first frame, then silence until timeout + from oli_bot import voice as _v + silence_needed = _v.SILENCE_TIMEOUT_MS // _v.FRAME_DURATION_MS + speech_values = [True] + [False] * (silence_needed + 2) + + mock_vad = MagicMock() + mock_vad.is_speech.side_effect = speech_values + engine._vad = mock_vad + + result = engine.record() + assert result is not None + assert result.endswith(".wav") + # Clean up + try: + os.unlink(result) + except OSError: + pass + + def test_returned_wav_is_valid(self): + engine = self._make_engine() + self._setup_pyaudio_mock([]) + + from oli_bot import voice as _v + silence_needed = _v.SILENCE_TIMEOUT_MS // _v.FRAME_DURATION_MS + speech_values = [True] + [False] * (silence_needed + 2) + + mock_vad = MagicMock() + mock_vad.is_speech.side_effect = speech_values + engine._vad = mock_vad + + path = engine.record() + assert path is not None + try: + with wave.open(path, "rb") as wf: + assert wf.getnchannels() == 1 + assert wf.getframerate() == 16_000 + finally: + try: + os.unlink(path) + except OSError: + pass + + def test_raises_when_pyaudio_missing(self): + engine = self._make_engine() + engine._vad = MagicMock() + sys.modules["pyaudio"] = None # type: ignore[assignment] + + with pytest.raises(RuntimeError, match="pyaudio"): + engine.record() + + def test_stream_always_closed_on_exception(self): + """The mic stream must be stopped/closed even when read() raises.""" + engine = self._make_engine() + + mock_stream = MagicMock() + mock_stream.read.side_effect = OSError("mic exploded") + + mock_pa_instance = MagicMock() + mock_pa_instance.open.return_value = mock_stream + mock_pa_instance.get_sample_size.return_value = 2 + + mock_pa_cls = MagicMock(return_value=mock_pa_instance) + _stub_module("pyaudio", PyAudio=mock_pa_cls, paInt16=8) + + mock_vad = MagicMock() + mock_vad.is_speech.return_value = True + engine._vad = mock_vad + + with pytest.raises(OSError, match="mic exploded"): + engine.record() + + mock_stream.stop_stream.assert_called_once() + mock_stream.close.assert_called_once() + mock_pa_instance.terminate.assert_called_once() + + +# --------------------------------------------------------------------------- +# Constants / defaults +# --------------------------------------------------------------------------- + + +class TestVoiceDefaults: + def test_default_constants_are_sensible(self): + from oli_bot import voice as _v + + assert _v.SAMPLE_RATE == 16_000 + assert _v.FRAME_DURATION_MS in (10, 20, 30) + assert 0 <= _v.VAD_AGGRESSIVENESS <= 3 + assert _v.SILENCE_TIMEOUT_MS > 0 + assert _v.MAX_RECORD_SECONDS > 0 + + def test_env_override_whisper_model(self, monkeypatch): + monkeypatch.setenv("OLI_VOICE_WHISPER_MODEL", "large") + from oli_bot.config import AppConfig + from oli_bot.voice import VoiceEngine + + cfg = AppConfig() + assert cfg.voice_whisper_model == "large" + engine = VoiceEngine(whisper_model_size=cfg.voice_whisper_model) + assert engine.whisper_model_size == "large" + + def test_env_override_piper_model(self, monkeypatch): + monkeypatch.setenv("OLI_VOICE_PIPER_MODEL", "/custom/path/model.onnx") + from oli_bot.config import AppConfig + from oli_bot.voice import VoiceEngine + + cfg = AppConfig() + assert cfg.voice_piper_model == "/custom/path/model.onnx" + engine = VoiceEngine(piper_model_path=cfg.voice_piper_model) + assert engine.piper_model_path == "/custom/path/model.onnx" + + def test_constants_match_appconfig_defaults(self): + # Drift guard: voice.py module constants are sourced from AppConfig + from oli_bot import voice as _v + from oli_bot.config import AppConfig + + cfg = AppConfig() + assert _v.WHISPER_MODEL_SIZE == cfg.voice_whisper_model + assert _v.PIPER_MODEL_PATH == cfg.voice_piper_model + assert _v.SAMPLE_RATE == cfg.voice_sample_rate + assert _v.FRAME_DURATION_MS == cfg.voice_frame_duration_ms + assert _v.VAD_AGGRESSIVENESS == cfg.voice_vad_aggressiveness + assert _v.SILENCE_TIMEOUT_MS == cfg.voice_silence_timeout_ms + assert _v.MAX_RECORD_SECONDS == cfg.voice_max_record_seconds + + def test_vad_aggressiveness_validated(self): + import pytest + from pydantic import ValidationError + + from oli_bot.config import AppConfig + + with pytest.raises(ValidationError): + AppConfig(voice_vad_aggressiveness=4) + with pytest.raises(ValidationError): + AppConfig(voice_silence_timeout_ms=0) + + def test_engine_stores_vad_tunables(self): + from oli_bot.voice import VoiceEngine + + engine = VoiceEngine( + whisper_model_size="tiny", + piper_model_path="dummy.onnx", + sample_rate=8000, + frame_duration_ms=20, + vad_aggressiveness=3, + silence_timeout_ms=400, + max_record_seconds=5, + ) + assert engine.sample_rate == 8000 + assert engine.frame_duration_ms == 20 + assert engine.vad_aggressiveness == 3 + assert engine.silence_timeout_ms == 400 + assert engine.max_record_seconds == 5