diff --git a/.agents/skills/agent-release-gate/SKILL.md b/.agents/skills/agent-release-gate/SKILL.md index bb73ce449c4..95619a3286a 100644 --- a/.agents/skills/agent-release-gate/SKILL.md +++ b/.agents/skills/agent-release-gate/SKILL.md @@ -181,6 +181,62 @@ cell, promoted after the platform-guidance fix closed that exact gap; it reuses the same way the separate one-shot benchmark (Tier B) does — check there before writing a new mechanism-blind cell from scratch, to avoid duplicating scaffolding. +## Session control cells + +`resources/session_control.py` is a second, standalone driver: sixteen cells that cover Stop, +durable commands, and the runner's recovery paths (owner release, park/resume, watchdog +quarantine). It drives the same product endpoint and asserts on the same wire, but it needs its +own account bootstrap, so it runs as a separate process rather than as `qa_product.py` cells. See +`resources/path_triggers.py` for the exact mandatory-cell mechanism. + +**These cells are MANDATORY** — run them, not just the standing gate — whenever the release diff +touches any of: + +- `services/runner/src/sessions/**` +- `services/runner/src/engines/sandbox_agent/**` +- `api/oss/src/core/sessions/**` +- `api/oss/src/tasks/asyncio/sessions/**` +- `api/oss/src/apis/fastapi/sessions/**` + +Run every cell with one line: + +```bash +uv run resources/session_control.py --cells all --harness pi_core --sandbox local +``` + +Add `--project ` to run the eight cells that need direct Docker and +Postgres access (`sandbox-gone`, `records-outage`, `restart-after-stop`, `runner-gone`, +`runner-gone-late`, `post-stop-row`, `codex-child`, `stale-tail`) and the abort-log subcheck inside +`stop-after-finish`. Without `--project`, those eight cells SKIP with a named reason. The +`stop-after-finish` HTTP check still runs, but only its abort-log subcheck is unavailable. The +other eight cells +(`stop-warm`, `double-send`, `stale-stop`, `stop-approval`, `stop-after-finish`, +`repeat-stop`, `concurrent-stops`, `stop-during-completion`) run over HTTP alone against any +deployment. Add +`--resume ` to pick a lost run back up: any cell already +recorded there is loaded instead of re-run. + +Results land in a timestamped folder under `~/agenta-qa-evidence/` (override with +`AGENTA_QA_RUNS_DIR`), as `results.json` and `summary.md` — the same PASS/FAIL/SKIP shape as the +rest of the gate. When a release path makes session control mandatory, pass that artifact to the +standing gate with `--session-control-results `: a missing or incomplete artifact stops the +gate before the matrix runs, and a recorded FAIL makes the final gate exit nonzero. + +**Environment, by name.** Same three-variable discipline as the rest of the gate, no env-file +fallback: + +- `AGENTA_BASE` — the deployment origin. +- `AGENTA_ADMIN_KEY` — mints the ephemeral account this driver runs under. Lives in + `~/.agenta-qa-secrets.env`. +- `QA_OPENAI_API_KEY` — stocked into that account's vault so the `pi_core` and `codex` harnesses + have a provider key. Lives in `~/.agenta-qa-openai.env`. +- `ANTHROPIC_API_KEY` — only required for `--harness claude`, stocked into the same vault the + same way. Lives in `~/.agenta-qa-secrets.env`. A pi_core- or codex-only run does not need it. + +A Daytona run additionally needs a Secrets-capable Daytona key on the runner; the key in most +session env files returns 403 on the Secrets endpoint, so check that before trusting a Daytona +result. + ## When results lie The runtime **fails open**: a component can break, get logged, and the turn still succeeds with a diff --git a/.agents/skills/agent-release-gate/resources/path_triggers.py b/.agents/skills/agent-release-gate/resources/path_triggers.py index 230e1855981..c0fe77b45fa 100644 --- a/.agents/skills/agent-release-gate/resources/path_triggers.py +++ b/.agents/skills/agent-release-gate/resources/path_triggers.py @@ -33,6 +33,12 @@ # the second kind as required, because a standalone cell is a separate process it cannot observe. GATEWAY_TOOLS = ("matrix_gw1_gateway_tools.py",) +# The standing session-control regression cells: Stop, durable commands, and the runner's +# recovery paths (owner release, park/resume, watchdog quarantine). A separate standalone driver +# because it needs its own account bootstrap and, for most cells, a docker-compose project name — +# see resources/session_control.py and SKILL.md "Session control cells". +SESSION_CONTROL = ("session_control.py",) + # The cells that run a REMOTE sandbox and need no extra flag. A release that touches the sandbox # engine or the Daytona provider changes how a cold sandbox gets built and how its credentials are # delivered, and the `burst` and `crosstalk` journeys are the only ones that see that path under @@ -71,8 +77,18 @@ # A fault here shows up only when many sandboxes start at once, which is what `burst` and # `crosstalk` do on these cells. Production hit it as one first message in five failing with # a credential error (AGE-4249 / #6485) while the sequential gate stayed green. - "services/runner/src/engines/sandbox_agent/**": DAYTONA_CELLS, + # A dict literal keeps only the last value for a repeated key, so a glob that already names + # DAYTONA_CELLS lists SESSION_CONTROL alongside it in the SAME tuple rather than as a second + # entry that would silently drop the Daytona rule. + "services/runner/src/engines/sandbox_agent/**": DAYTONA_CELLS + SESSION_CONTROL, "services/runner/src/providers/daytona*": DAYTONA_CELLS, + # Session control: Stop, durable commands, park/resume, and the owner-release and watchdog + # sweeps. A change here can silently break a warm resume or leave a command stuck, and + # nothing in the fixed matrix drives Stop at all. See qa-audit-2026-09-03.md section 4. + "services/runner/src/sessions/**": SESSION_CONTROL, + "api/oss/src/core/sessions/**": SESSION_CONTROL, + "api/oss/src/tasks/asyncio/sessions/**": SESSION_CONTROL, + "api/oss/src/apis/fastapi/sessions/**": SESSION_CONTROL, } # Glob -> journeys that MUST run when the rule fires. Same matching as PATH_TRIGGERS, kept as a diff --git a/.agents/skills/agent-release-gate/resources/qa_product.py b/.agents/skills/agent-release-gate/resources/qa_product.py index 39b18f9ff9b..39916e8c689 100644 --- a/.agents/skills/agent-release-gate/resources/qa_product.py +++ b/.agents/skills/agent-release-gate/resources/qa_product.py @@ -3118,6 +3118,68 @@ def approval(i: int) -> dict: } +def _load_session_control_result(path: str) -> dict: + """Load and summarize a complete standalone session-control result.""" + result_path = pathlib.Path(path).expanduser() + try: + payload = json.loads(result_path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise SystemExit( + f"Cannot read --session-control-results {result_path}: {exc}" + ) from exc + + cells = payload.get("cells") + if not isinstance(cells, dict): + raise SystemExit( + f"Invalid session-control result {result_path}: expected a top-level cells object." + ) + + # Import the standalone driver's registry instead of copying its cell names here. A newly + # added session-control cell must become release-mandatory without a second list to update. + from session_control import CELLS as session_control_cells + + missing = sorted(set(session_control_cells) - set(cells)) + if missing: + raise SystemExit( + f"Incomplete session-control result {result_path}: missing cells: " + + ", ".join(missing) + ) + + statuses: dict[str, str] = {} + for name in session_control_cells: + entry = cells.get(name) + verdict = entry.get("verdict") if isinstance(entry, dict) else None + if ( + not isinstance(verdict, dict) + or not isinstance(verdict.get("pass"), bool) + or not isinstance(verdict.get("skip"), bool) + or (verdict["pass"] and verdict["skip"]) + ): + raise SystemExit( + f"Invalid session-control result {result_path}: cell {name!r} has no valid " + "PASS/FAIL/SKIP verdict." + ) + statuses[name] = ( + "SKIP" if verdict["skip"] else ("PASS" if verdict["pass"] else "FAIL") + ) + + failed = sorted(name for name, status in statuses.items() if status == "FAIL") + skipped = sorted(name for name, status in statuses.items() if status == "SKIP") + return { + "path": str(result_path), + "status": "FAIL" if failed else ("INCOMPLETE" if skipped else "PASS"), + "failed": failed, + "skipped": skipped, + } + + +def _session_control_result_label(result: dict) -> str: + label = f"recorded {result['status']}" + if result["skipped"]: + label += "; SKIPPED, UNTESTED: " + ", ".join(result["skipped"]) + return label + + def main() -> int: # Declared here, not beside the assignments below, because the flag help strings read these # module defaults and a `global` statement must precede every use of the name in a function. @@ -3271,6 +3333,13 @@ def main() -> int: "--repo", help="repository the release diff is read from (default: the current directory)", ) + p.add_argument( + "--session-control-results", + help=( + "results.json written by resources/session_control.py. Required when a path rule " + "makes that standalone driver mandatory; all of its cells must be recorded." + ), + ) args = p.parse_args() resolve_credentials(args.env_file) @@ -3368,6 +3437,16 @@ def main() -> int: external_cells = [ cell for cell in triggered if cell not in CELLS and cell not in missing_cells ] + session_control_result = None + if "session_control.py" in external_cells: + if not args.session_control_results: + raise SystemExit( + "This release makes session_control.py mandatory. Run it separately, then pass " + "its results.json with --session-control-results." + ) + session_control_result = _load_session_control_result( + args.session_control_results + ) for cell in triggered: if cell in CELLS and cell not in cells: cells.append(cell) @@ -3380,7 +3459,11 @@ def main() -> int: else ( "MISSING — no such cell exists" if cell in missing_cells - else "run it separately" + else ( + _session_control_result_label(session_control_result) + if cell == "session_control.py" and session_control_result + else "run it separately" + ) ) ) print(f" {cell} ({where})") @@ -3479,9 +3562,19 @@ def main() -> int: table += "\n\nMandatory for this release, by path rule:\n\n" table += "| cell | run here | because this release changed |\n|---|---|---|\n" for cell, why in triggered.items(): - here = "yes" if cell in CELLS else "no — run it separately" + if cell in CELLS: + here = "yes" + elif cell == "session_control.py" and session_control_result: + here = _session_control_result_label(session_control_result) + else: + here = "no — run it separately" table += f"| {cell} | {here} | {', '.join(why)} |\n" - if external_cells: + unrecorded_external_cells = [ + cell + for cell in external_cells + if not (cell == "session_control.py" and session_control_result) + ] + if unrecorded_external_cells: table += ( "\nThis release is NOT green until every cell above marked " "`run it separately` has a recorded result.\n" @@ -3506,7 +3599,10 @@ def main() -> int: for cell in results.values() for journey in cell["journeys"].values() ) - return 1 if failed else 0 + standalone_failed = bool( + session_control_result and session_control_result["status"] != "PASS" + ) + return 1 if failed or standalone_failed else 0 if __name__ == "__main__": diff --git a/.agents/skills/agent-release-gate/resources/session_control.py b/.agents/skills/agent-release-gate/resources/session_control.py new file mode 100644 index 00000000000..1042e930c78 --- /dev/null +++ b/.agents/skills/agent-release-gate/resources/session_control.py @@ -0,0 +1,3310 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["httpx>=0.27"] +# /// +"""Session-control regression cells for the agent release gate. + +Wire-level scenarios for Stop, durable commands, and the runner's recovery paths. Each cell +drives the same product endpoint the playground drives (`/services/agent/v0/invoke`) and asserts +on the SSE frame stream, the durable records, and the command rows. It never asserts on model +prose. + +Ported from the durable-cancel slice's spike driver +(`~/agenta-qa-evidence/2026-09-03-session-round2/integration-refresh/refresh_live.py`), with four +changes made so this file can live in the repo and run as a standing check instead of a one-box +artifact: + +1. Reads the SAME env contract as `qa_product.py` (`AGENTA_BASE`), plus `AGENTA_ADMIN_KEY` and + `QA_OPENAI_API_KEY`, which this driver needs to mint its own ephemeral account and stock the + vault. No env-file fallback: a fallback file is how a green run gets recorded against the + wrong deployment. +2. The Docker- and Postgres-only helpers sit behind one `OperatorHooks` interface + (`DockerComposeHooks` / `NullHooks`). Six cells need no shell at all and run against any + deployment; the rest need `--project ` and SKIP with a named reason + when it is absent. +3. Emits the gate's result shape: PASS / FAIL / SKIP per cell with a one-line reason, plus + `results.json` and `summary.md` in a timestamped run folder under `~/agenta-qa-evidence/` + (override with `AGENTA_QA_RUNS_DIR`). +4. `--cells` is resumable: pass `--resume ` and any cell already + recorded there is loaded instead of re-run, so a lost agent costs one cell, not the whole run. + + uv run resources/session_control.py --cells all --harness pi_core --sandbox local + +See `SKILL.md` for when these cells are mandatory and where the model keys live. +""" + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import re +import subprocess +import sys +import threading +import time +import uuid + +import httpx + +REQUIRED_ENV = ("AGENTA_BASE", "AGENTA_ADMIN_KEY", "QA_OPENAI_API_KEY") + +# Resolved by resolve_env() before anything runs. Left empty so --help works with no env set. +BASE = "" +ADMIN_KEY = "" +OPENAI_KEY = "" +# Only required when --harness claude is selected; checked in bootstrap(), not resolve_env(), +# so a pi_core/codex-only run never needs it set. +ANTHROPIC_KEY = "" + +# Set in main() from --sandbox: Daytona sandboxes take 10 to 20s to start, on top of whatever a +# local sandbox needs, so every wait that assumes "local" gets this much extra slack. +SANDBOX_STARTUP_SLACK_S = 0.0 + +# Set in main() from --client-shape. "full" (default) replays the whole transcript on every +# send, like this driver always has, so existing results stay comparable. "last-message" +# reshapes every outbound `messages` list the way the desktop client does — see +# _client_shape_messages() below. +CLIENT_SHAPE = "full" + +# Set in main() from --durable-stop. In auto mode, the first recognized cancel response fixes +# the effective state for the run: the durable route returns command + execution metadata, while +# the production-default legacy route returns its older cancellation summary. +DURABLE_STOP_OPTION = "auto" +DURABLE_STOP_STATE: str | None = None + +RUNS = pathlib.Path( + os.environ.get( + "AGENTA_QA_RUNS_DIR", str(pathlib.Path.home() / "agenta-qa-evidence") + ) +).expanduser() + +STATE: dict = {} +RECALL = "What was the codeword I gave you? Reply with just the codeword." + + +def resolve_env() -> None: + """Populate BASE/ADMIN_KEY/OPENAI_KEY from the environment only. + + No env-file fallback on purpose: qa-audit-2026-09-03.md section 4 names the file fallback as + the mechanism that recorded a green run against the wrong deployment. Every missing variable + is named so a Sonnet QA agent does not have to guess. + """ + global BASE, ADMIN_KEY, OPENAI_KEY + missing = [name for name in REQUIRED_ENV if not os.environ.get(name)] + if missing: + raise SystemExit( + "Missing environment variables: " + ", ".join(missing) + ".\n" + "Set them, e.g.\n" + " export AGENTA_BASE=https://your-stack.example.com\n" + " export AGENTA_ADMIN_KEY=... # ~/.agenta-qa-secrets.env\n" + " export QA_OPENAI_API_KEY=... # ~/.agenta-qa-openai.env\n" + "There is no env-file fallback: a fallback file is how a green run gets recorded " + "against the wrong deployment." + ) + BASE = os.environ["AGENTA_BASE"] + ADMIN_KEY = os.environ["AGENTA_ADMIN_KEY"] + OPENAI_KEY = os.environ["QA_OPENAI_API_KEY"] + global ANTHROPIC_KEY + ANTHROPIC_KEY = os.environ.get("ANTHROPIC_API_KEY", "") + + +# --------------------------------------------------------------------------- # +# Operator hooks: the only place this file talks to Docker or Postgres. +# --------------------------------------------------------------------------- # + + +class HooksUnavailable(Exception): + """Raised by a NullHooks method. Caught at the cell boundary and turned into a SKIP.""" + + +class WrongSandboxTarget(Exception): + """The sandbox-gone cell could not map the tested session to exactly one sandbox-agent + daemon it is safe to kill. Raised INSTEAD of killing a guess. Two sessions can share one + mount key, and the keep-alive pool keeps other sessions' parked daemons alive in the same + runner container, so a blind `ps | grep sandbox-agent` kill hits the wrong process. The cell + turns this into a `wrong target` failure rather than a false negative against the product.""" + + +# A local sandbox id from the turn ledger is `local/:`; a Daytona id is `daytona/`. +_LOCAL_SANDBOX_ID_RE = re.compile(r"^local/[^:\s]+:(\d+)$") + + +# The runner log line that names the port a session's local sandbox daemon bound to, e.g. +# `[sandbox-agent] [timing] stage=prepare_workspace ms=0 sandbox=local/127.0.0.1:44831 session=`. +def _prepare_workspace_port_re(session_id: str) -> re.Pattern[str]: + return re.compile( + r"stage=prepare_workspace\b.*\bsandbox=local/[^:\s]+:(\d+)\b.*\bsession=" + + re.escape(session_id) + ) + + +def _parse_local_sandbox_port(sandbox_id: str | None) -> int | None: + """The port from a local ledger sandbox id, or None for a Daytona/empty/foreign id.""" + if not sandbox_id: + return None + m = _LOCAL_SANDBOX_ID_RE.match(sandbox_id.strip()) + return int(m.group(1)) if m else None + + +def _parse_ss_listener_pid(ss_output: str, port: int) -> str | None: + """The owning pid of the LISTEN socket on `port`, parsed from `ss -ltnHp` output. + + A line looks like: + LISTEN 0 511 127.0.0.1:44831 0.0.0.0:* users:(("node",pid=2170,fd=23)) + The peer column on a listener is always `0.0.0.0:*`/`[::]:*`, so an exact `:` field + match cannot collide with the peer, and `rsplit` guards against a substring port match.""" + for line in ss_output.splitlines(): + fields = line.split() + if not any(f.rsplit(":", 1)[-1] == str(port) for f in fields if ":" in f): + continue + m = re.search(r"\bpid=(\d+)", line) + if m: + return m.group(1) + return None + + +# Fallback for a container without `ss`: read the LISTEN socket's inode from /proc/net/tcp{,6}, +# then find the pid whose fd points at that socket. `$1` is the decimal port. +_PROC_PID_ON_PORT_SH = r""" +port="$1" +hp=$(printf '%04X' "$port" 2>/dev/null) || exit 0 +inode=$(awk -v hp="$hp" 'NR>1 && $4=="0A" { split($2,a,":"); if (a[2]==hp) { print $10; exit } }' /proc/net/tcp /proc/net/tcp6 2>/dev/null) +[ -z "$inode" ] && exit 0 +for fd in /proc/[0-9]*/fd/*; do + link=$(readlink "$fd" 2>/dev/null) || continue + if [ "$link" = "socket:[$inode]" ]; then + echo "$fd" | awk -F/ '{print $3}' + exit 0 + fi +done +""" + + +class OperatorHooks: + """Interface the cells call through. `available` gates whether shell-only cells can run.""" + + available = False + + def dc(self, *args: str, timeout: float = 60.0) -> str: + raise HooksUnavailable + + def psql(self, db: str, sql: str) -> list[list[str]]: + raise HooksUnavailable + + def runner_log(self, since: float) -> list[str]: + raise HooksUnavailable + + def sandbox_procs(self, marker: str, sandbox_id: str | None = None) -> list[dict]: + raise HooksUnavailable + + def stream_row(self, session_id: str) -> dict: + raise HooksUnavailable + + def record_rows(self, session_id: str) -> list[dict]: + raise HooksUnavailable + + def command_rows(self, session_id: str) -> list[dict]: + raise HooksUnavailable + + def execution_rows(self, session_id: str) -> list[dict]: + raise HooksUnavailable + + def wait_for_runner(self, *, timeout: float = 120.0) -> float | None: + raise HooksUnavailable + + def runner_healthy(self) -> bool: + raise HooksUnavailable + + def ensure_runner_healthy(self, *, timeout: float = 120.0) -> dict: + raise HooksUnavailable + + def restart_runner(self, grace_seconds: int = 10) -> None: + raise HooksUnavailable + + def kill_runner(self) -> None: + raise HooksUnavailable + + def pause_runner(self) -> None: + raise HooksUnavailable + + def unpause_runner(self) -> None: + raise HooksUnavailable + + def stop_postgres(self) -> None: + raise HooksUnavailable + + def start_postgres(self) -> None: + raise HooksUnavailable + + def kill_sandbox(self, sandbox_id: str | None = None) -> list[str]: + raise HooksUnavailable + + def kill_sandbox_for_session( + self, + session_id: str | None = None, + sandbox_id: str | None = None, + since: float | None = None, + ) -> dict: + raise HooksUnavailable + + def wait_for_sandbox_ready( + self, + session_id: str | None = None, + ledger_id_getter=None, + since: float | None = None, + timeout: float | None = None, + poll_interval: float | None = None, + clock=time, + ): + raise HooksUnavailable + + +class NullHooks(OperatorHooks): + """No `--project` was given. Every method raises; cells that need it SKIP with a reason.""" + + available = False + + +class DockerComposeHooks(OperatorHooks): + """The original refresh_live.py helpers, ported behind the OperatorHooks interface.""" + + available = True + + def __init__(self, project: str) -> None: + self.project = project + + def dc(self, *args: str, timeout: float = 60.0) -> str: + try: + out = subprocess.run( + ["docker", *args], capture_output=True, text=True, timeout=timeout + ) + return out.stdout + except Exception as exc: # noqa: BLE001 + return f"" + + def psql(self, db: str, sql: str) -> list[list[str]]: + raw = self.dc( + "exec", + f"{self.project}-postgres-1", + "psql", + "-U", + "username", + "-d", + db, + "-At", + "-F", + "|", + "-c", + sql, + ) + return [line.split("|") for line in raw.strip().splitlines() if line.strip()] + + def runner_log(self, since: float) -> list[str]: + stamp = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(since - 2)) + try: + out = subprocess.run( + ["docker", "logs", "-t", "--since", stamp, f"{self.project}-runner-1"], + capture_output=True, + text=True, + timeout=90, + ) + return (out.stdout + out.stderr).splitlines() + except Exception as exc: # noqa: BLE001 + return [f""] + + def sandbox_procs(self, marker: str, sandbox_id: str | None = None) -> list[dict]: + # A local sandbox IS a subprocess of the runner container, so `ps` inside the runner + # sees it regardless of which session owns it. `sandbox_id` is accepted for interface + # parity with the Daytona-aware hook (which needs it to pick a remote sandbox) and + # ignored here. + raw = self.dc( + "exec", f"{self.project}-runner-1", "ps", "-eo", "pid,ppid,etimes,args" + ) + hits = [] + for line in raw.splitlines()[1:]: + parts = line.split(None, 3) + if len(parts) < 4 or marker not in parts[3]: + continue + if "ps -eo" in parts[3] or parts[3].startswith("grep"): + continue + hits.append( + { + "pid": parts[0], + "ppid": parts[1], + "etimes": parts[2], + "args": parts[3][:120], + } + ) + return hits + + def stream_row(self, session_id: str) -> dict: + rows = self.psql( + "agenta_ee_core", + "select turn_id, coalesce(flags::text,'{}'), coalesce(stopping_turn_id,'') " + f"from session_streams where session_id = '{session_id}'", + ) + if not rows: + return {} + turn, flags, stopping = rows[0] + try: + flags_obj = json.loads(flags) + except Exception: # noqa: BLE001 + flags_obj = {"raw": flags} + return { + "turn_id": turn, + "flags": flags_obj, + "stopping_turn_id": stopping or None, + "read_at": time.time(), + } + + def record_rows(self, session_id: str) -> list[dict]: + rows = self.psql( + "agenta_ee_tracing", + "select coalesce(turn_id,''), record_type, " + "coalesce(to_char(created_at,'HH24:MI:SS.MS'),''), " + "case when quarantined_at is null then '' " + "else to_char(quarantined_at,'HH24:MI:SS.MS') end " + f"from records where session_id = '{session_id}' order by created_at", + ) + return [ + { + "turn_id": r[0], + "type": r[1], + "created_at": r[2], + "quarantined_at": r[3] or None, + } + for r in rows + if len(r) >= 4 + ] + + def command_rows(self, session_id: str) -> list[dict]: + rows = self.psql( + "agenta_ee_core", + "select id::text, state, coalesce(outcome,''), claim_count, " + "coalesce(target_turn_id,'') from session_commands " + f"where session_id = '{session_id}' order by created_at", + ) + return [ + { + "id": r[0], + "state": r[1], + "outcome": r[2] or None, + "claim_count": r[3], + "target_turn_id": r[4] or None, + } + for r in rows + if len(r) >= 5 + ] + + def execution_rows(self, session_id: str) -> list[dict]: + rows = self.psql( + "agenta_ee_core", + "select execution_id, terminal_outcome, coalesce(settled_by,''), " + "coalesce(to_char(settled_at,'HH24:MI:SS.MS'),'') from session_executions " + f"where session_id = '{session_id}' order by settled_at", + ) + return [ + { + "execution_id": r[0], + "terminal_outcome": r[1] or None, + "settled_by": r[2] or None, + "settled_at": r[3] or None, + } + for r in rows + if len(r) >= 4 + ] + + def wait_for_runner(self, *, timeout: float = 120.0) -> float | None: + started = time.time() + while time.time() - started < timeout: + if self.runner_healthy(): + return round(time.time() - started, 1) + time.sleep(1) + return None + + def runner_healthy(self) -> bool: + """One health check: the runner container's Docker health status reads `healthy`.""" + state = self.dc( + "inspect", "-f", "{{.State.Health.Status}}", f"{self.project}-runner-1" + ).strip() + return state == "healthy" + + def ensure_runner_healthy(self, *, timeout: float = 120.0) -> dict: + """Recover the runner container to running and healthy, whatever a cell left it in. + + `run_cell()` calls this in a `finally` block after every cell that needs hooks, so a + cell that pauses, stops, or restarts the runner and then raises before its own restore + code runs does not strand the runner paused or down for the next cell. + """ + paused = ( + self.dc( + "inspect", "-f", "{{.State.Paused}}", f"{self.project}-runner-1" + ).strip() + == "true" + ) + if paused: + self.unpause_runner() + status = self.dc( + "inspect", "-f", "{{.State.Status}}", f"{self.project}-runner-1" + ).strip() + if status != "running": + self.restart_runner() + healthy_after_s = self.wait_for_runner(timeout=timeout) + return { + "was_paused": paused, + "status_before": status, + "healthy_after_s": healthy_after_s, + } + + def restart_runner(self, grace_seconds: int = 10) -> None: + self.dc( + "restart", "-t", str(grace_seconds), f"{self.project}-runner-1", timeout=120 + ) + + def kill_runner(self) -> None: + self.dc("restart", "-t", "0", f"{self.project}-runner-1", timeout=60) + + def pause_runner(self) -> None: + self.dc("pause", f"{self.project}-runner-1") + + def unpause_runner(self) -> None: + self.dc("unpause", f"{self.project}-runner-1") + + def stop_postgres(self) -> None: + self.dc("stop", f"{self.project}-postgres-1") + + def start_postgres(self) -> None: + self.dc("start", f"{self.project}-postgres-1") + + def kill_sandbox(self, sandbox_id: str | None = None) -> list[str]: + # A local sandbox is a subprocess of the runner container: there is only ever one + # `sandbox-agent server` process family running there per cell, so `sandbox_id` (accepted + # for interface parity with the Daytona-aware hook) is not needed to target it. + ps = self.dc( + "exec", + f"{self.project}-runner-1", + "sh", + "-c", + 'ps -eo pid,args | grep "[s]andbox-agent server"', + ) + pids = [line.split()[0] for line in ps.strip().splitlines() if line.strip()] + for pid in pids: + self.dc( + "exec", + f"{self.project}-runner-1", + "sh", + "-c", + f"kill -9 -{pid} || kill -9 {pid}", + ) + return pids + + def local_sandbox_port( + self, + session_id: str, + sandbox_id: str | None = None, + since: float | None = None, + ) -> int | None: + """The TCP port THIS session's own local sandbox daemon bound to. + + The source of truth is the runner log's `prepare_workspace` line for this exact session + id; the turn ledger's `local/:` sandbox id is a fallback and a cross-check. + Two sessions can share one mount key, so a global `ps | grep` cannot tell them apart — + this is per-session by construction. When both sources disagree the target is ambiguous + and this refuses (raises), rather than guessing which daemon to kill.""" + log_port = None + pat = _prepare_workspace_port_re(session_id) + for line in self.runner_log(since if since is not None else time.time() - 600): + m = pat.search(line) + if m: + log_port = int( + m.group(1) + ) # last match wins: a rebuild uses a fresh port + ledger_port = _parse_local_sandbox_port(sandbox_id) + if log_port is not None and ledger_port is not None and log_port != ledger_port: + raise WrongSandboxTarget( + f"the runner log names port {log_port} for session {session_id} but the turn " + f"ledger names {ledger_port}; refusing to kill an ambiguous target" + ) + return log_port if log_port is not None else ledger_port + + def pid_listening_on_port(self, port: int) -> str | None: + """The pid of the process listening on `port` inside the runner container. + + Prefers `ss -ltnHp` (the pid is inline); falls back to reading the socket inode from + /proc/net/tcp and matching it against /proc/*/fd when the image ships no `ss`.""" + ss_out = self.dc( + "exec", + f"{self.project}-runner-1", + "sh", + "-c", + "ss -ltnHp 2>/dev/null || true", + ) + pid = _parse_ss_listener_pid(ss_out, port) + if pid: + return pid + proc_out = self.dc( + "exec", + f"{self.project}-runner-1", + "sh", + "-c", + _PROC_PID_ON_PORT_SH, + "pid-on-port", + str(port), + ) + proc_out = proc_out.strip() + return proc_out or None + + def process_cmdline(self, pid: str) -> str: + """The argv of `pid` inside the runner container, space-joined (nul-separated on disk).""" + return self.dc( + "exec", + f"{self.project}-runner-1", + "sh", + "-c", + f"tr '\\0' ' ' < /proc/{pid}/cmdline 2>/dev/null", + ).strip() + + def kill_sandbox_for_session( + self, + session_id: str | None = None, + sandbox_id: str | None = None, + since: float | None = None, + ) -> dict: + """Kill ONLY the local sandbox daemon that belongs to `session_id`. + + Maps the session to its own port, resolves the listening pid, and asserts the pid is a + sandbox-agent daemon before killing it. Any gap in that chain raises `WrongSandboxTarget` + so the cell fails as `wrong target` instead of killing an unrelated session's parked + sandbox (the historical false negative). Returns the port, pid, cmdline, and killed pids + as evidence.""" + if not session_id: + raise WrongSandboxTarget("no session id given; refusing to kill a guess") + port = self.local_sandbox_port(session_id, sandbox_id=sandbox_id, since=since) + if port is None: + raise WrongSandboxTarget( + f"could not find this session's local sandbox port for {session_id} in the " + f"runner log or the turn ledger (ledger id={sandbox_id!r}); refusing to kill a guess" + ) + pid = self.pid_listening_on_port(port) + if not pid: + raise WrongSandboxTarget( + f"nothing is listening on port {port} inside the runner container for session " + f"{session_id}; the named sandbox is not here — refusing to kill a guess" + ) + cmdline = self.process_cmdline(pid) + if "sandbox-agent" not in cmdline: + raise WrongSandboxTarget( + f"pid {pid} on port {port} is not a sandbox-agent daemon " + f"(cmdline={cmdline[:120]!r}); refusing to kill it" + ) + self.dc( + "exec", + f"{self.project}-runner-1", + "sh", + "-c", + f"kill -9 -{pid} || kill -9 {pid}", + ) + return { + "port": port, + "pid": pid, + "cmdline": cmdline[:200], + "killed": [pid], + } + + def wait_for_local_sandbox_port( + self, + session_id: str, + ledger_id_getter=None, + since: float | None = None, + timeout: float | None = None, + poll_interval: float | None = None, + clock=time, + ) -> int: + """Poll until THIS session's local sandbox port is resolvable, or refuse on timeout. + + A cold acquire writes the `prepare_workspace` line ~35 s after the turn starts, so reading + once right after the turn began finds nothing and the cell refused correctly but uselessly. + Poll the runner log (and the turn ledger via `ledger_id_getter`) until the port appears. + A transient log/ledger disagreement during acquire is retried, not fatal; only its + persistence to the deadline raises. When nothing appears within `timeout`, raise + WrongSandboxTarget so the cell fails as `wrong target` rather than killing a guess.""" + timeout = SANDBOX_GONE_RESOLVE_TIMEOUT_S if timeout is None else timeout + poll_interval = ( + SANDBOX_GONE_RESOLVE_POLL_S if poll_interval is None else poll_interval + ) + getter = ledger_id_getter or (lambda: None) + deadline = clock.time() + timeout + last_error: WrongSandboxTarget | None = None + while True: + try: + ledger_id = getter() + except Exception: # noqa: BLE001 + ledger_id = None + try: + port = self.local_sandbox_port( + session_id, sandbox_id=ledger_id, since=since + ) + except WrongSandboxTarget as exc: + # A log/ledger disagreement mid-acquire is usually transient; keep polling and + # let it raise only if it is still the state at the deadline. + last_error = exc + port = None + if port is not None: + return port + if clock.time() >= deadline: + if last_error is not None: + raise last_error + raise WrongSandboxTarget( + f"this session's prepare_workspace line never appeared in the runner log " + f"(and no local ledger sandbox id) within {timeout:.0f}s for session " + f"{session_id}; refusing to kill a guess" + ) + clock.sleep(poll_interval) + + def wait_for_sandbox_ready( + self, + session_id: str | None = None, + ledger_id_getter=None, + since: float | None = None, + timeout: float | None = None, + poll_interval: float | None = None, + clock=time, + ): + """Local: block until this session's own sandbox port is resolvable (returns the port).""" + return self.wait_for_local_sandbox_port( + session_id, + ledger_id_getter=ledger_id_getter, + since=since, + timeout=timeout, + poll_interval=poll_interval, + clock=clock, + ) + + +class DaytonaAwareHooks(DockerComposeHooks): + """`DockerComposeHooks` plus a Daytona-provider-aware `kill_sandbox` and `sandbox_procs`. + + A local sandbox is a subprocess of the runner container, so the base class's `docker exec ps` + sees it. A Daytona sandbox is a remote machine: `docker exec` into the runner container never + sees the sandbox's process table, and killing a local process cannot end a remote sandbox. So + for `--sandbox daytona` this hook ends the sandbox and lists its processes through the same + Daytona REST API the runner itself uses (`services/runner/src/engines/sandbox_agent/ + daytona-provider.ts`'s `sandbox.delete()`, and the vendored `sandbox-agent/daytona` provider's + `runProcess`, which `reap-exec.ts` drives with the identical `ps -eo pid=,ppid=,etimes=,args=` + used below). + + Every call is scoped to the ONE sandbox id the cell observed for its own session + (`sandbox_ids(session_id)` in the driver, threaded in by the caller) — never a list, never a + wildcard. Credentials come from `AGENTA_RUNNER_DAYTONA_API_KEY` / `AGENTA_RUNNER_DAYTONA_API_URL` + (export only; never logged, never put in an exception message). + """ + + def __init__(self, project: str) -> None: + super().__init__(project) + missing = [ + name + for name in ( + "AGENTA_RUNNER_DAYTONA_API_KEY", + "AGENTA_RUNNER_DAYTONA_API_URL", + ) + if not os.environ.get(name) + ] + if missing: + raise SystemExit( + "--sandbox daytona needs " + ", ".join(missing) + " exported (from the " + "integration env file's AGENTA_RUNNER_DAYTONA_* block) so sandbox-gone and " + "codex-child can reach the Daytona API directly." + ) + self._daytona_api_url = os.environ["AGENTA_RUNNER_DAYTONA_API_URL"].rstrip("/") + self._daytona_api_key = os.environ["AGENTA_RUNNER_DAYTONA_API_KEY"] + + @staticmethod + def _bare_id(sandbox_id: str) -> str: + """`sandbox_ids()` returns ids like `daytona/`; the Daytona API wants the bare uuid.""" + return sandbox_id.split("/", 1)[1] if "/" in sandbox_id else sandbox_id + + def _daytona_get(self, path: str) -> httpx.Response: + return httpx.get( + f"{self._daytona_api_url}{path}", + headers={"Authorization": f"Bearer {self._daytona_api_key}"}, + timeout=30.0, + ) + + def _daytona_delete(self, path: str) -> httpx.Response: + return httpx.delete( + f"{self._daytona_api_url}{path}", + headers={"Authorization": f"Bearer {self._daytona_api_key}"}, + timeout=30.0, + ) + + def kill_sandbox(self, sandbox_id: str | None = None) -> list[str]: + if not sandbox_id: + return [] + bare = self._bare_id(sandbox_id) + try: + resp = self._daytona_delete(f"/sandbox/{bare}") + except Exception as exc: # noqa: BLE001 + print( + f"[daytona] delete sandbox={bare} failed: {exc}", + file=sys.stderr, + ) + return [] + # DELETE /sandbox/{id} is what `sandbox.delete()` calls on this same SDK/API version + # (Sandbox.js -> SandboxApi.deleteSandbox); a 404 means it is already gone, also success + # for "the sandbox is gone" purposes. + if resp.status_code not in (200, 202, 204, 404): + print( + f"[daytona] delete sandbox={bare} returned {resp.status_code}: " + f"{resp.text[:200]}", + file=sys.stderr, + ) + return [] + return [bare] + + def kill_sandbox_for_session( + self, + session_id: str | None = None, + sandbox_id: str | None = None, + since: float | None = None, + ) -> dict: + """Delete THIS session's remote sandbox by the one id its turn ledger observed. + + A Daytona sandbox is a remote machine, addressed by its own uuid, so this path is already + per-session targeted and never had the shared-runner ambiguity the local path did. An + absent id means there is nothing to end — that is a `wrong target` refusal, not a kill.""" + if not sandbox_id: + raise WrongSandboxTarget( + f"no sandbox id observed for session {session_id}; nothing to end" + ) + killed = self.kill_sandbox(sandbox_id=sandbox_id) + if not killed: + raise WrongSandboxTarget( + f"the Daytona delete for sandbox {sandbox_id} did not confirm; refusing to " + "claim a kill that did not land" + ) + return { + "port": None, + "pid": None, + "cmdline": None, + "killed": killed, + } + + def wait_for_sandbox_ready( + self, + session_id: str | None = None, + ledger_id_getter=None, + since: float | None = None, + timeout: float | None = None, + poll_interval: float | None = None, + clock=time, + ): + """Daytona: block until this session's remote sandbox id is observed (returns the id). + + A remote sandbox is even slower to appear than a local one, so the same poll applies; the + target here is the ledger id, not a port. Refuse on timeout rather than deleting a guess.""" + timeout = SANDBOX_GONE_RESOLVE_TIMEOUT_S if timeout is None else timeout + poll_interval = ( + SANDBOX_GONE_RESOLVE_POLL_S if poll_interval is None else poll_interval + ) + getter = ledger_id_getter or (lambda: None) + deadline = clock.time() + timeout + while True: + try: + sandbox_id = getter() + except Exception: # noqa: BLE001 + sandbox_id = None + if sandbox_id: + return sandbox_id + if clock.time() >= deadline: + raise WrongSandboxTarget( + f"no Daytona sandbox id was observed for session {session_id} within " + f"{timeout:.0f}s; refusing to kill a guess" + ) + clock.sleep(poll_interval) + + def sandbox_procs(self, marker: str, sandbox_id: str | None = None) -> list[dict]: + if not sandbox_id: + return [] + bare = self._bare_id(sandbox_id) + try: + proxy = self._daytona_get(f"/sandbox/{bare}/toolbox-proxy-url") + if proxy.status_code != 200: + print( + f"[daytona] toolbox-proxy-url sandbox={bare} returned " + f"{proxy.status_code}: {proxy.text[:200]}", + file=sys.stderr, + ) + return [] + proxy_url = (proxy.json() or {}).get("url") + if not proxy_url: + print( + f"[daytona] toolbox-proxy-url sandbox={bare} returned no url", + file=sys.stderr, + ) + return [] + # Same shape as `reap-exec.ts`'s `PS_ARGS` (`-eo pid=,ppid=,etimes=,args=`): the `=` + # suffixes drop the header line, so every returned line is a data row. + exec_resp = httpx.post( + f"{proxy_url.rstrip('/')}/process/execute", + json={"command": "ps -eo pid=,ppid=,etimes=,args=", "timeout": 10}, + timeout=20.0, + ) + except Exception as exc: # noqa: BLE001 + print( + f"[daytona] process listing sandbox={bare} failed: {exc}", + file=sys.stderr, + ) + return [] + if exec_resp.status_code != 200: + print( + f"[daytona] process/execute sandbox={bare} returned " + f"{exec_resp.status_code}: {exec_resp.text[:200]}", + file=sys.stderr, + ) + return [] + raw = (exec_resp.json() or {}).get("result", "") or "" + hits = [] + for line in raw.splitlines(): + parts = line.split(None, 3) + if len(parts) < 4 or marker not in parts[3]: + continue + if "ps -eo" in parts[3] or parts[3].startswith("grep"): + continue + hits.append( + { + "pid": parts[0], + "ppid": parts[1], + "etimes": parts[2], + "args": parts[3][:120], + } + ) + return hits + + +def select_hooks(project: str | None, sandbox: str) -> OperatorHooks: + """The provider switch: no `--project` is NullHooks regardless of `--sandbox`; with a + project, `--sandbox daytona` needs the Daytona-aware hook (docker exec cannot see or touch a + remote sandbox), everything else gets the plain docker-compose hook. Pulled out of `main()` so + it is unit-testable without a live stack. + """ + if not project: + return NullHooks() + if sandbox == "daytona": + return DaytonaAwareHooks(project) + return DockerComposeHooks(project) + + +# --------------------------------------------------------------------------- # +# HTTP plumbing (unchanged from refresh_live.py, keyed off the resolved env) +# --------------------------------------------------------------------------- # + +HARNESSES = { + "pi_core": { + "kind": "pi_core", + "model": "gpt-5.6-luna", + "provider": "openai", + "connection": {"mode": "agenta", "slug": None}, + }, + "codex": { + "kind": "codex", + "model": "gpt-5.6-luna", + "provider": "openai", + "connection": {"mode": "agenta", "slug": None}, + }, + "claude": { + # `sonnet` alias, not a full model id: a full id is dropped to the default on the Claude + # ACP path (qa_product.py F-007). VAULT key (mode "agenta"), not subscription: this + # driver's cells run on Daytona too, and Daytona rejects subscription auth by design. + "kind": "claude", + "model": "sonnet", + "provider": "anthropic", + "connection": {"mode": "agenta", "slug": None}, + }, +} + +# Read timeout for the SSE stream `invoke()` opens, per harness kind (`cfg["harness"]["kind"]`, +# the same key HARNESSES above sets). Pi is a single in-process model loop; Codex and Claude Code +# are agentic CLIs behind an ACP bridge and routinely take longer per turn under load, so their +# budget is about 1.5x Pi's — high enough that a genuinely slow-but-healthy turn does not trip +# the driver's OWN httpx.ReadTimeout and get misread as a product failure (concurrent-stops hit +# exactly this on Claude Code). A cell whose Stop settlement is the actual problem is caught by +# `assert_command_settled` well before this ever fires, at its own fixed 20s budget regardless of +# harness — this table is about not confusing "the driver gave up too early" with "the product is +# broken", not about giving a broken product more rope. +STREAM_TIMEOUT_S = { + "pi_core": 600.0, + "codex": 900.0, + "claude": 900.0, +} +DEFAULT_STREAM_TIMEOUT_S = 600.0 + + +def stream_timeout_s(cfg: dict) -> float: + kind = (cfg.get("harness") or {}).get("kind") + return STREAM_TIMEOUT_S.get(kind, DEFAULT_STREAM_TIMEOUT_S) + + +# The "sandbox-gone" cell runs one slow shell command, kills the tested session's OWN sandbox +# daemon under it, and expects the runner to end the turn with a terminal record. The settle +# budget is derived from the runner's sandbox-liveness probe defaults +# (services/runner/src/engines/sandbox_agent/sandbox-liveness.ts): PROBE_FAILURES consecutive +# probe failures at PROBE_INTERVAL_S each, after which the turn ends with an error record. The +# cell waits that budget plus slack, and never less than the slow command itself — so a healthy +# turn that outlives a mis-targeted kill can never be misread as "still running" before it would +# even have finished. The command duration is a constant the cell prints in its evidence. +SANDBOX_LIVENESS_PROBE_INTERVAL_S = 30.0 +SANDBOX_LIVENESS_PROBE_FAILURES = 3 +SANDBOX_GONE_SETTLE_SLACK_S = 60.0 + +# A cold acquire on the gate stack takes ~35 s before the sandbox's `prepare_workspace` line is +# even written (observed `acquire_total ms=34766`), so the cell must POLL for this session's own +# sandbox to become resolvable rather than reading once right after the turn starts. Poll the +# runner log and the turn ledger for up to this long, then let the slow command run a moment +# before the kill. If the line never appears, refuse (never kill a guess). +SANDBOX_GONE_ACQUIRE_BUDGET_S = 60.0 +SANDBOX_GONE_RESOLVE_TIMEOUT_S = 120.0 +SANDBOX_GONE_RESOLVE_POLL_S = 3.0 +SANDBOX_GONE_RUNNING_SLACK_S = 5.0 + +# The design window the runner needs to end the turn once the sandbox is dead: PROBE_FAILURES +# probes at PROBE_INTERVAL_S each. +_SANDBOX_GONE_DESIGN_WINDOW_S = ( + SANDBOX_LIVENESS_PROBE_INTERVAL_S * SANDBOX_LIVENESS_PROBE_FAILURES +) + +# The slow command must OUTLAST the whole worst case before the kill lands, plus the design +# window, so it is still running when the sandbox dies and a failed kill cannot be misread as a +# healthy completion: acquire budget + the resolve poll window + the probe design window + margin. +SANDBOX_GONE_COMMAND_S = int( + SANDBOX_GONE_ACQUIRE_BUDGET_S + + SANDBOX_GONE_RESOLVE_TIMEOUT_S + + _SANDBOX_GONE_DESIGN_WINDOW_S + + 30 +) + + +def sandbox_gone_settle_budget_s() -> float: + """Seconds to wait for the runner to end the turn after the sandbox is killed: the probe's + three-strikes budget plus slack plus any sandbox-startup slack the run declared.""" + return ( + _SANDBOX_GONE_DESIGN_WINDOW_S + + SANDBOX_GONE_SETTLE_SLACK_S + + SANDBOX_STARTUP_SLACK_S + ) + + +# After the runner-gone-late cell restarts the runner, the recovery Send must not race the +# runner coming back up: a Send issued mid-restart gets "All connection attempts failed" and is +# misread as a product failure. Poll the runner's health until it is back, bounded, then send. +RECOVERY_HEALTH_TIMEOUT_S = 60.0 +RECOVERY_HEALTH_POLL_S = 2.0 + +# The sweep commits the execution outcome before it commits the terminal records. Keep the +# terminal assertion strict, but allow that second transaction to become visible first. +TERMINAL_RECORD_SETTLE_BUDGET_S = 20.0 +TERMINAL_RECORD_SETTLE_POLL_S = 0.5 + + +def _has_watchdog_ending(rows: list) -> bool: + return any( + (row.get("attributes") or {}).get("settled_by") == "watchdog" for row in rows + ) + + +def _require_watchdog_execution_lost(rows: list) -> dict | None: + found = any( + row.get("type") == "error" + and (row.get("attributes") or {}).get("code") == "execution_lost" + and (row.get("attributes") or {}).get("settled_by") == "watchdog" + for row in rows + ) + if found: + return None + return _fail( + "no watchdog execution_lost ending was found among the terminal records" + ) + + +def _poll_terminal_after_settle( + read_terminal, + *, + timeout=TERMINAL_RECORD_SETTLE_BUDGET_S, + poll_interval=TERMINAL_RECORD_SETTLE_POLL_S, + clock=time, +) -> list: + """Wait for the watchdog's terminal-record transaction after durable settlement.""" + deadline = clock.time() + timeout + while True: + rows = read_terminal() + if _has_watchdog_ending(rows) or clock.time() >= deadline: + return rows + clock.sleep(poll_interval) + + +def _recover_then_send(health_poll, send, *, timeout, poll_interval, clock=time): + """Poll `health_poll()` until it returns truthy (bounded by `timeout`), THEN call `send()`. + + `send` runs ONLY once the runner is healthy again, so a recovery Send can never race a runner + that is still restarting. Returns `(healthy, result)`; when health never recovers within the + budget, `send` is not called and `result` is None. The clock is injectable for tests.""" + deadline = clock.time() + timeout + healthy = False + while True: + if health_poll(): + healthy = True + break + if clock.time() >= deadline: + break + clock.sleep(poll_interval) + result = send() if healthy else None + return healthy, result + + +def _measure_runner_gone_while_paused( + hooks, + session_id, + turn, + do_stop, + read_terminal, + *, + sweep_wait, + poll_interval=5.0, + clock=time, +): + """Pause the runner, fire the Stop while it is gone, wait for the sweep to settle it lost, and + read is_running from the stream row WHILE THE RUNNER IS STILL PAUSED. + + The pause is the whole assertion. "Runner gone" must be measured while the runner is still + gone: once it is unpaused it starts a new turn on the same session that legitimately sets + is_running true again, so a read after the unpause sees that new turn and misreads a healthy + recovery as a failure. Settlement is detected on the durable rows — the Stop command reaching + `obsolete`/`applied` with outcome `lost`, or an execution row carrying a terminal outcome — + not on the volatile stream. Unpauses on every path. Returns the paused measurements.""" + stop = None + stop_at = None + settled_at = None + paused_read_at = None + stream_row = None + stop_command = None + commands: list = [] + executions: list = [] + terminal: list = [] + hooks.pause_runner() + try: + stop = do_stop() + stop_at = clock.time() + deadline = clock.time() + sweep_wait + while True: + commands = hooks.command_rows(session_id) + stop_command = _match_stop_command(commands, turn) + executions = hooks.execution_rows(session_id) + command_settled = ( + stop_command is not None + and stop_command.get("state") in ("obsolete", "applied") + and stop_command.get("outcome") == "lost" + ) + execution_lost = any(e.get("terminal_outcome") for e in executions) + if command_settled or execution_lost: + settled_at = clock.time() + break + if clock.time() >= deadline: + break + clock.sleep(poll_interval) + terminal = ( + _poll_terminal_after_settle( + read_terminal, + poll_interval=0.5, + clock=clock, + ) + if settled_at is not None + else read_terminal() + ) + # THE gone-and-stays-gone read: is_running, taken while the runner is still paused. + stream_row = hooks.stream_row(session_id) + paused_read_at = clock.time() + finally: + # Unpause on every path: a paused runner left behind strands every later cell. + hooks.unpause_runner() + return { + "stop": stop, + "stop_at": stop_at, + "settled_at": settled_at, + "paused_read_at": paused_read_at, + "stream_row": stream_row, + "stop_command": stop_command, + "commands": commands, + "executions": executions, + "terminal": terminal, + } + + +def api(method: str, path: str, *, timeout: float = 120.0, **kw) -> httpx.Response: + headers = { + "Authorization": STATE["credentials"], + "Content-Type": "application/json", + **(kw.pop("headers", None) or {}), + } + params = {"project_id": STATE["project_id"], **(kw.pop("params", None) or {})} + return httpx.request( + method, + f"{BASE}/api{path}", + params=params, + headers=headers, + timeout=timeout, + **kw, + ) + + +def bootstrap(harness: str = "pi_core") -> None: + uid = uuid.uuid4().hex[:12] + r = httpx.post( + f"{BASE}/api/admin/simple/accounts/", + headers={"Authorization": f"Access {ADMIN_KEY}"}, + json={ + "accounts": { + "user": { + "user": {"email": f"{uid}@test.agenta.ai"}, + "options": { + "create_api_keys": True, + "return_api_keys": True, + "seed_defaults": False, + }, + } + } + }, + timeout=120.0, + ) + r.raise_for_status() + account = next(iter(r.json()["accounts"].values())) + STATE["credentials"] = f"ApiKey {account['api_keys']['key']}" + STATE["project_id"] = next(iter(account["projects"].values()))["id"] + print(f"[bootstrap] project={STATE['project_id']}", file=sys.stderr) + + r = api( + "POST", + "/vault/v1/secrets/", + json={ + "header": {"name": "OpenAI", "description": "session-control gate"}, + "secret": { + "kind": "provider_key", + "data": {"kind": "openai", "provider": {"key": OPENAI_KEY}}, + }, + }, + ) + if r.status_code != 200: + raise SystemExit(f"vault create HTTP {r.status_code}: {r.text[:400]}") + print("[bootstrap] vault stocked with an openai provider key", file=sys.stderr) + + if harness == "claude": + # The claude harness's vault connection (agent_config mode "agenta") needs a funded + # Anthropic key, the same way the OpenAI key above covers pi_core and codex. Checked + # here, not in resolve_env(), so a pi_core/codex-only run never needs it set. + if not ANTHROPIC_KEY: + raise SystemExit( + "Missing environment variable: ANTHROPIC_API_KEY. Required for --harness " + "claude (the vault connection needs a funded Anthropic key). " + "e.g. export ANTHROPIC_API_KEY=... # ~/.agenta-qa-secrets.env" + ) + r = api( + "POST", + "/vault/v1/secrets/", + json={ + "header": {"name": "Anthropic", "description": "session-control gate"}, + "secret": { + "kind": "provider_key", + "data": {"kind": "anthropic", "provider": {"key": ANTHROPIC_KEY}}, + }, + }, + ) + if r.status_code != 200: + raise SystemExit(f"vault create HTTP {r.status_code}: {r.text[:400]}") + print( + "[bootstrap] vault stocked with an anthropic provider key", file=sys.stderr + ) + + +def agent_config( + harness: str, model: str, provider: str, connection: dict, sandbox: str = "local" +) -> dict: + return { + "instructions": {"agents_md": "Be terse. Do exactly what is asked."}, + "llm": { + "model": model, + "provider": provider, + "connection": connection, + "extras": {}, + }, + "tools": [], + "mcps": [], + "skills": [], + "harness": {"kind": harness}, + "sandbox": {"kind": sandbox}, + "runner": {"permissions": {"default": "allow"}}, + } + + +def create_revision(cfg: dict, tag: str) -> dict: + hexid = uuid.uuid4().hex[:8] + r = api( + "POST", + "/workflows/", + json={ + "workflow": { + "slug": f"{tag}-{hexid}", + "name": f"session-control {hexid}", + "flags": { + "is_custom": True, + "is_evaluator": False, + "is_feedback": False, + }, + } + }, + ) + if r.status_code != 200: + raise SystemExit(f"create workflow HTTP {r.status_code}: {r.text[:400]}") + wf = r.json()["workflow"]["id"] + + r = api( + "POST", + "/workflows/variants/", + json={ + "workflow_variant": { + "slug": f"{tag}-{hexid}-v", + "name": f"session-control {hexid} v", + "workflow_id": wf, + } + }, + ) + if r.status_code != 200: + raise SystemExit(f"create variant HTTP {r.status_code}: {r.text[:400]}") + var = r.json()["workflow_variant"]["id"] + + rev_id = None + for step in ("seed", "baseline"): + r = api( + "POST", + "/workflows/revisions/commit", + json={ + "workflow_revision": { + "slug": f"{tag}-{step}-{hexid}", + "name": f"session-control rev {step}", + "message": step, + "data": { + "uri": "agenta:builtin:agent:v0", + "parameters": {"agent": cfg}, + }, + "workflow_id": wf, + "workflow_variant_id": var, + } + }, + ) + if r.status_code != 200: + raise SystemExit(f"commit {step} HTTP {r.status_code}: {r.text[:400]}") + rev_id = r.json()["workflow_revision"]["id"] + + return { + "application": {"id": wf}, + "variant": {"id": var}, + "revision": {"id": rev_id}, + } + + +def user_msg(text: str) -> dict: + return { + "id": str(uuid.uuid4()), + "role": "user", + "parts": [{"type": "text", "text": text}], + } + + +def _is_answer_part(part: dict) -> bool: + """Mirrors `isAnswerPart` in agentRequest.ts (web/packages/agenta-playground/src/state/ + execution/agentRequest.ts): a non-empty text part, a tool part (`tool-*`), a + `dynamic-tool` part, or a `file` part.""" + t = part.get("type") if isinstance(part, dict) else None + if not isinstance(t, str): + return False + if t == "text": + text = part.get("text") + return isinstance(text, str) and text.strip() != "" + return t.startswith("tool-") or t in ("dynamic-tool", "file") + + +def _has_answer(message: dict) -> bool: + """Mirrors `hasAnswer` in agentRequest.ts: a user (non-assistant) message always counts; + an assistant message counts only if at least one of its parts is an answer part. Strips an + answer-less assistant turn so it cannot cascade into every later turn failing.""" + if message.get("role") != "assistant": + return True + parts = message.get("parts") + return isinstance(parts, list) and any(_is_answer_part(p) for p in parts) + + +def _client_shape_messages(messages: list) -> list: + """Shape the outbound `messages` list the way the desktop client does (agentRequest.ts), + when `--client-shape last-message` is selected. A no-op under the default `full`. + + Strip answer-less assistant turns, then send only the trailing message when it is a fresh + user turn — the runner rebuilds prior turns from the durable record log. A resume whose + trailing turn carries a settled HITL answer (not a user turn) keeps the full history so the + answer still binds to its tool call. + """ + if CLIENT_SHAPE != "last-message": + return messages + history = [m for m in messages if _has_answer(m)] + if not history: + return history + if history[-1].get("role") == "user": + return [history[-1]] + return history + + +def invoke( + session_id: str, + messages: list, + cfg: dict, + references: dict, + label: str, + out: dict | None = None, +) -> dict: + url = f"{BASE}/services/agent/v0/invoke" + body = { + "session_id": session_id, + "references": references, + "data": { + "inputs": {"messages": _client_shape_messages(messages)}, + "parameters": {"agent": cfg}, + }, + } + headers = { + "Authorization": STATE["credentials"], + "Accept": "text/event-stream", + "x-ag-messages-format": "vercel", + "Content-Type": "application/json", + } + out = out if out is not None else {} + out.update( + { + "frames": [], + "text": "", + "tool_calls": [], + "errors": [], + "raw": [], + "segments": [], + "tool_outcomes": {}, + "tool_payloads": {}, + } + ) + started = time.time() + with httpx.Client(timeout=stream_timeout_s(cfg)) as client: + with client.stream( + "POST", + url, + params={ + "project_id": STATE["project_id"], + "application_id": references["application"]["id"], + }, + json=body, + headers=headers, + ) as r: + print(f"[{label}] HTTP {r.status_code}", file=sys.stderr) + if r.status_code >= 400: + out["errors"].append(f"HTTP {r.status_code}: {r.read().decode()[:600]}") + return out + for line in r.iter_lines(): + if not line.startswith("data: "): + continue + payload = line[6:] + if payload == "[DONE]": + break + try: + f = json.loads(payload) + except json.JSONDecodeError: + continue + out["raw"].append(f) + t = f.get("type", "?") + out["frames"].append(t) + if t == "message-metadata": + tid = (f.get("messageMetadata") or {}).get("turnId") + if isinstance(tid, str) and tid: + out["turn_id"] = tid + if t == "text-delta": + delta = f.get("delta", "") + out["text"] += delta + if out["segments"] and out["segments"][-1]["kind"] == "text": + out["segments"][-1]["text"] += delta + else: + out["segments"].append({"kind": "text", "text": delta}) + elif t == "tool-input-available": + call = { + "toolCallId": f.get("toolCallId"), + "name": f.get("toolName"), + "input": f.get("input"), + } + is_new = not any( + c["toolCallId"] == call["toolCallId"] for c in out["tool_calls"] + ) + out["tool_calls"] = [ + c + for c in out["tool_calls"] + if c["toolCallId"] != call["toolCallId"] + ] + [call] + if is_new: + out["segments"].append( + {"kind": "tool", "id": call["toolCallId"]} + ) + elif t == "tool-output-available": + out["tool_outcomes"][f.get("toolCallId")] = "available" + out["tool_payloads"][f.get("toolCallId")] = { + "output": f.get("output") + } + elif t == "tool-output-error": + out["tool_outcomes"][f.get("toolCallId")] = "error" + out["tool_payloads"][f.get("toolCallId")] = { + "errorText": f.get("errorText") + } + elif t == "error": + out["errors"].append(json.dumps(f)[:600]) + out["elapsed_s"] = round(time.time() - started, 1) + print( + f"[{label}] frames={out['frames']} elapsed={out['elapsed_s']}s", file=sys.stderr + ) + return out + + +def assistant_message(turn: dict) -> dict: + parts: list = [] + text_buf: list[str] = [] + # `turn` can be `{}` when the driver's own wait for the turn timed out (`handle["out"]` was + # never set, e.g. because the runner was unhealthy and the stream thread never finished) — a + # driver-side timeout, not a reason to crash the cell with a KeyError instead of reporting a + # FAIL. Missing segments means no assistant turn to replay. + for seg in turn.get("segments") or []: + if seg["kind"] == "text": + text_buf.append(seg["text"]) + continue + if text_buf: + parts.append({"type": "text", "text": "".join(text_buf)}) + text_buf = [] + call = next(c for c in turn["tool_calls"] if c["toolCallId"] == seg["id"]) + part = { + "type": f"tool-{call['name']}", + "toolCallId": call["toolCallId"], + "input": call["input"], + "state": "input-available", + } + outcome = turn["tool_outcomes"].get(call["toolCallId"]) + if outcome == "available": + part["state"] = "output-available" + part["output"] = ( + turn["tool_payloads"].get(call["toolCallId"], {}).get("output") + ) + elif outcome == "error": + part["state"] = "output-error" + part["errorText"] = ( + turn["tool_payloads"].get(call["toolCallId"], {}).get("errorText") + ) + parts.append(part) + if text_buf: + parts.append({"type": "text", "text": "".join(text_buf)}) + return {"id": str(uuid.uuid4()), "role": "assistant", "parts": parts} + + +def turn_ledger(session_id: str, limit: int = 20) -> list[dict]: + """The session's turn rows, newest first, over HTTP only (no docker needed). + + The runner writes `agent_session_id` and `sandbox_id` on every turn, so this is a STORED + outcome, not an echo of what the client sent. Used to check the resume after a Stop landed + in the SAME sandbox rather than a rebuilt one. + """ + r = api( + "POST", + "/sessions/turns/query", + json={ + "query": {"session_id": session_id}, + "windowing": {"limit": limit, "order": "descending"}, + }, + ) + if r.status_code != 200: + return [] + try: + body = r.json() + except Exception: # noqa: BLE001 + return [] + turns = body.get("turns") if isinstance(body, dict) else None + return turns if isinstance(turns, list) else [] + + +def sandbox_ids(session_id: str) -> list[str]: + """Distinct sandbox ids across the session's turn ledger. + + ONE id = the resume reused the same sandbox (warm). TWO or more = the sandbox was rebuilt. + """ + return sorted( + {r.get("sandbox_id") for r in turn_ledger(session_id) if r.get("sandbox_id")} + ) + + +def session_stream(session_id: str) -> dict: + r = api("GET", "/sessions/streams/", params={"session_id": session_id}) + if r.status_code != 200: + return {} + return (r.json() or {}).get("stream") or {} + + +def cancel( + session_id: str, + *, + expected: str | None = None, + idempotency_key: str | None = None, + label: str = "stop", +) -> dict: + headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None + body = {"expected_execution_id": expected} if expected else {} + sent = time.time() + r = api( + "POST", + f"/sessions/{session_id}/cancel", + json=body, + headers=headers, + timeout=30.0, + ) + got = time.time() + try: + payload = r.json() + except Exception: + payload = {"raw": r.text[:400]} + durable_stop = _observe_durable_stop(payload) + record = { + "status": r.status_code, + "body": payload, + "durable_stop": durable_stop, + "sent_at": sent, + "sent_iso": time.strftime("%H:%M:%S", time.localtime(sent)) + + f".{int((sent % 1) * 1000):03d}", + "round_trip_s": round(got - sent, 3), + } + print( + f"[{label}] HTTP {r.status_code} at {record['sent_iso']} rt={record['round_trip_s']}s {json.dumps(payload)[:300]}", + file=sys.stderr, + ) + return record + + +_LEGACY_CANCEL_KEYS = { + "mode", + "session_id", + "turn_id", + "watcher_id", + "detached", + "cancelled_turn_ids", +} + + +def _detect_durable_stop(payload: object) -> str | None: + """Identify the Stop implementation from a successful cancel response body.""" + if not isinstance(payload, dict): + return None + if "command" in payload and "execution" in payload: + return "on" + if _LEGACY_CANCEL_KEYS.issubset(payload): + return "off" + return None + + +def _resolve_durable_stop(option: str, payload: object) -> str | None: + """Resolve an explicit flag value, or infer auto from the cancel response shape.""" + if option in ("on", "off"): + return option + return _detect_durable_stop(payload) + + +def _observe_durable_stop(payload: object) -> str | None: + """Record the effective durable-stop state for this run when the response identifies it.""" + global DURABLE_STOP_STATE + observed = _resolve_durable_stop(DURABLE_STOP_OPTION, payload) + if observed is None: + return DURABLE_STOP_STATE + if DURABLE_STOP_STATE is not None and observed != DURABLE_STOP_STATE: + raise RuntimeError( + "cancel responses disagreed about durable Stop state: " + f"first {DURABLE_STOP_STATE}, now {observed}" + ) + DURABLE_STOP_STATE = observed + return DURABLE_STOP_STATE + + +def records(session_id: str) -> list: + r = api("POST", "/sessions/records/query", json={"session_id": session_id}) + if r.status_code != 200: + return [{"error": f"HTTP {r.status_code}: {r.text[:200]}"}] + return (r.json() or {}).get("records") or [] + + +def terminal_records(session_id: str, turn_id: str | None = None) -> list: + rows = [ + { + "type": rec.get("record_type"), + "turn_id": rec.get("turn_id"), + "attributes": rec.get("attributes"), + } + for rec in records(session_id) + if rec.get("record_type") in ("error", "done") + ] + if turn_id: + rows = [r for r in rows if r["turn_id"] == turn_id] + return rows + + +def interactions(session_id: str) -> list: + r = api( + "POST", + "/sessions/interactions/query", + json={"query": {"session_id": session_id}}, + ) + if r.status_code != 200: + return [{"error": f"HTTP {r.status_code}"}] + return [ + { + "id": i.get("id"), + "turn_id": i.get("turn_id"), + "kind": i.get("kind"), + "status": i.get("status"), + } + for i in ((r.json() or {}).get("interactions") or []) + ] + + +def invoke_async(session_id, messages, cfg, references, label) -> dict: + live: dict = {} + handle: dict = {"out": None, "live": live} + + def go() -> None: + handle["out"] = invoke(session_id, messages, cfg, references, label, out=live) + + t = threading.Thread(target=go, daemon=True) + t.start() + handle["thread"] = t + return handle + + +def wait_for_turn(session_id: str, *, timeout: float = 40.0) -> str | None: + deadline = time.time() + timeout + SANDBOX_STARTUP_SLACK_S + while time.time() < deadline: + stream = session_stream(session_id) + turn = stream.get("turn_id") + flags = stream.get("flags") or {} + if turn and flags.get("is_running"): + return turn + time.sleep(0.5) + return None + + +def wait_for_tool(handle: dict, *, timeout: float = 60.0) -> dict | None: + deadline = time.time() + timeout + SANDBOX_STARTUP_SLACK_S + live = handle["live"] + while time.time() < deadline: + calls = live.get("tool_calls") or [] + outcomes = live.get("tool_outcomes") or {} + open_calls = [c for c in calls if c["toolCallId"] not in outcomes] + if open_calls: + return open_calls[-1] + if handle["out"] is not None: + return None + time.sleep(0.1) + return None + + +def sleep_prompt(marker: str, seconds: int) -> str: + return ( + f"The codeword is {marker}. Run exactly this one shell command and nothing " + f"else: sleep {seconds}. Do not write, read or search any files. " + "When the command finishes, reply with the single word DONE." + ) + + +# --------------------------------------------------------------------------- # +# Cells. Each returns (evidence: dict, verdict: dict) where verdict is +# {"pass": bool, "skip": bool, "why": str} — the gate's result shape. +# --------------------------------------------------------------------------- # + +Cell = "tuple[dict, dict]" + + +def _pass(why: str) -> dict: + return {"pass": True, "skip": False, "why": why} + + +def _fail(why: str) -> dict: + return {"pass": False, "skip": False, "why": why} + + +def _skip(why: str) -> dict: + return {"pass": False, "skip": True, "why": why} + + +def _match_stop_command(commands: list[dict], turn_id: str | None) -> dict | None: + """The Stop command for a given turn: the last command row targeting it, or (when the turn + id is unknown, or nothing targets it) the last command row overall. Shared by every cell that + needs to find "the command the Stop I just sent produced" among a session's command rows.""" + matching = [c for c in commands if turn_id and c.get("target_turn_id") == turn_id] + return matching[-1] if matching else (commands[-1] if commands else None) + + +def assert_command_settled( + hooks: OperatorHooks, session_id: str, turn_id: str | None, *, timeout: float = 20.0 +) -> dict: + """Poll for up to `timeout` seconds after a Stop for the durable settlement invariant every + Stop-issuing cell must observe: the session_commands row for the Stop reaches a terminal + state (`applied` or `obsolete` — never left `pending` or `claimed`), and exactly one + session_executions row exists for the stopped session with a non-empty terminal outcome. This + is the check that would have caught the repeat-stop false pass on 2026-09-04 (session + 190e9118: command stuck `claimed` forever, zero session_executions rows, yet every driver- + level assertion — one terminal trace record, a warm resume — still passed). + + A Stop can also land AFTER the turn already finished naturally — common on a fast Claude Code + turn: the valid Stop returns 202, but the runner has nothing to cancel, so the command settles + `obsolete`/`not_running` and NO execution row is written. Zero rows is correct there, so a Stop + that settled `not_running` is accepted with zero execution rows and `natural_finish=True`. The + strict one-row requirement is kept for a Stop the runner applied (`stopped`) or the sweep + settled (`lost`). `stop-after-finish` and `stop-during-completion` already accept this shape; + routing it through here shares it with every Stop-issuing cell. + + Returns a dict with `settled` (bool), `command`, `execution_rows`, `natural_finish` (bool), + `note` (set to "stop landed after a natural finish" on that path, else None), and `why` (a + one-line reason, only set when `settled` is False). Never raises: a hookless run (`NullHooks`) + reads as `settled=True` so a cell that runs without --project is not blocked by a check it has + no way to make (the cell's own `hooks.available` guard already SKIPs it). + """ + if not hooks.available: + return { + "settled": True, + "command": None, + "execution_rows": [], + "natural_finish": False, + "note": None, + "why": None, + } + deadline = time.time() + timeout + command: dict | None = None + executions: list[dict] = [] + while True: + commands = hooks.command_rows(session_id) + command = _match_stop_command(commands, turn_id) + executions = hooks.execution_rows(session_id) + settled_command = command is not None and command.get("state") in ( + "applied", + "obsolete", + ) + outcome = command.get("outcome") if command else None + # The Stop landed after a natural finish: obsolete/not_running, no execution row to expect. + natural_finish = settled_command and outcome == "not_running" + settled_execution = len(executions) == 1 and bool( + executions[0].get("terminal_outcome") + ) + if settled_command and (natural_finish or settled_execution): + return { + "settled": True, + "command": command, + "execution_rows": executions, + "natural_finish": natural_finish, + "note": "stop landed after a natural finish" + if natural_finish + else None, + "why": None, + } + if time.time() >= deadline: + break + time.sleep(1) + if command is None: + why = "no session_commands row was found for the Stop" + elif command.get("state") not in ("applied", "obsolete"): + why = f"the Stop command was left {command.get('state')!r}, expected applied or obsolete" + elif len(executions) != 1: + why = ( + "expected exactly one session_executions row for the stopped session, saw " + f"{len(executions)}" + ) + else: + why = "the session_executions row settled with no terminal outcome" + return { + "settled": False, + "command": command, + "execution_rows": executions, + "natural_finish": False, + "note": None, + "why": why, + } + + +def _judge_runner_gone(evidence: dict) -> dict: + """Shared PASS rule for the runner-gone family (`runner-gone`, `runner-gone-late`). + + The invariant: exactly one effective terminal outcome for the execution, no command left + pending or claimed, is_running false, and the next Send succeeds. Two different races can + land this — the runner reports the Stop's outcome before it dies (`outcome-reported-then- + died`), or it never gets the chance and the sweep settles the command `lost` + (`never-reported`) — and both satisfy the invariant, so both PASS. Which one landed is + recorded on `evidence["race"]` for visibility, not asserted on. Mutates `evidence` in place. + """ + if not evidence.get("terminal_records"): + return _fail("no terminal record settled within the sweep-wait window") + stop_command = evidence.get("stop_command") + if stop_command is None: + return _fail("no session_commands row was found for the Stop") + if stop_command.get("state") not in ("obsolete", "applied"): + return _fail( + f"the Stop command read state {stop_command.get('state')!r}, expected obsolete or applied" + ) + outcome = stop_command.get("outcome") + if outcome in (None, "", "pending", "claimed"): + return _fail( + f"the Stop command was left {outcome!r}: still pending or claimed, never settled" + ) + stream_row = evidence.get("stream_row") or {} + if (stream_row.get("flags") or {}).get("is_running") is not False: + return _fail( + "the session_streams row did not read is_running: false after the sweep settled " + "the command" + ) + if not evidence.get("new_message_ran"): + return _fail("the Send sent after recovery did not run cleanly") + race = "never-reported" if outcome == "lost" else "outcome-reported-then-died" + evidence["race"] = race + return _pass( + f"race {race}: the Stop command settled off pending/claimed, is_running read false, " + "and the next Send ran" + ) + + +def cell_stop_warm(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Stop under 5 s, park, warm resume that recalls the codeword. Needs no shell.""" + session_id = str(uuid.uuid4()) + marker = f"MANGO{uuid.uuid4().hex[:6].upper()}" + msgs = [user_msg(sleep_prompt(marker, args.sleep_seconds))] + handle = invoke_async(session_id, msgs, cfg, references, "warm-turn1") + turn = wait_for_turn(session_id) + open_call = wait_for_tool(handle) + stop = cancel(session_id, expected=turn, label="stop-warm") + settle = assert_command_settled(hooks, session_id, turn) + handle["thread"].join(timeout=180) + t1 = handle["out"] or {} + time.sleep(4) + msgs2 = msgs + [assistant_message(t1), user_msg(RECALL)] + t2 = invoke(session_id, msgs2, cfg, references, "warm-turn2") + evidence = { + "session_id": session_id, + "turn_id": turn, + "marker": marker, + "stop": stop, + "stopped_during_tool": open_call, + "turn1_elapsed_s": t1.get("elapsed_s"), + "terminal_records": terminal_records(session_id, turn), + "resume_recalled_marker": marker in (t2.get("text") or ""), + "resume_elapsed_s": t2.get("elapsed_s"), + "command_settled": settle, + } + evidence["sandbox_ids"] = sandbox_ids(session_id) + evidence["warm_same_sandbox"] = len(evidence["sandbox_ids"]) <= 1 + if stop["status"] not in (200, 202): + return evidence, _fail( + f"Stop returned HTTP {stop['status']}, expected 200 or 202" + ) + if not settle["settled"]: + return evidence, _fail(settle["why"]) + if not evidence["resume_recalled_marker"]: + return evidence, _fail("warm resume did not recall the codeword") + return evidence, _pass( + f"Stop returned HTTP {stop['status']} and the warm resume recalled the codeword" + ) + + +def cell_double_send(cfg, references, args, hooks: OperatorHooks) -> Cell: + """A second message during a running turn is refused, and destroys nothing. Needs no shell.""" + session_id = str(uuid.uuid4()) + marker = f"KIWI{uuid.uuid4().hex[:6].upper()}" + msgs = [user_msg(sleep_prompt(marker, args.sleep_seconds))] + handle = invoke_async(session_id, msgs, cfg, references, "double-turn1") + turn = wait_for_turn(session_id) + time.sleep(5) + second_started = time.time() + t2 = invoke( + session_id, [user_msg("Say hello.")], cfg, references, "double-turn2-refused" + ) + second_elapsed = round(time.time() - second_started, 2) + handle["thread"].join(timeout=300) + t1 = handle["out"] or {} + time.sleep(4) + msgs3 = msgs + [assistant_message(t1), user_msg(RECALL)] + t3 = invoke(session_id, msgs3, cfg, references, "double-turn3") + evidence = { + "session_id": session_id, + "turn_id": turn, + "marker": marker, + "second_send": { + "frames": t2.get("frames"), + "errors": t2.get("errors"), + "elapsed_s": second_elapsed, + }, + "turn1_elapsed_s": t1.get("elapsed_s"), + "turn1_errors": t1.get("errors"), + "third_send_recalled_marker": marker in (t3.get("text") or ""), + } + evidence["sandbox_ids"] = sandbox_ids(session_id) + evidence["warm_same_sandbox"] = len(evidence["sandbox_ids"]) <= 1 + refused = bool(t2.get("errors")) + if not refused: + return evidence, _fail("second Send during a running turn was not refused") + if not evidence["third_send_recalled_marker"]: + return evidence, _fail( + "turn 1 finished but the codeword was not recalled afterwards" + ) + return evidence, _pass( + "second Send was refused and the original turn completed cleanly" + ) + + +def cell_stale_stop(cfg, references, args, hooks: OperatorHooks) -> Cell: + """A Stop naming a settled turn is refused and tombstones nothing. Needs no shell.""" + session_id = str(uuid.uuid4()) + marker = f"PLUM{uuid.uuid4().hex[:6].upper()}" + msgs = [ + user_msg(f"The codeword is {marker}. Reply with just the single word READY.") + ] + t1 = invoke(session_id, msgs, cfg, references, "stale-turn1") + turn1 = session_stream(session_id).get("turn_id") + time.sleep(3) + msgs2 = msgs + [ + assistant_message(t1), + user_msg(sleep_prompt(marker, args.sleep_seconds)), + ] + handle = invoke_async(session_id, msgs2, cfg, references, "stale-turn2") + turn2 = None + deadline = time.time() + 40 + while time.time() < deadline: + candidate = wait_for_turn(session_id, timeout=2) + if candidate and candidate != turn1: + turn2 = candidate + break + time.sleep(3) + stale = cancel(session_id, expected=turn1, label="stale-stop") + time.sleep(3) + bare = cancel(session_id, label="bare-stop") + # The stale Stop (targets turn1, already settled) is expected to be REFUSED, not to produce + # a settlement of its own — only `bare` (the real Stop, targets the live turn2) must settle. + settle = assert_command_settled(hooks, session_id, turn2) + handle["thread"].join(timeout=180) + t2 = handle["out"] or {} + time.sleep(4) + msgs3 = msgs2 + [assistant_message(t2), user_msg(RECALL)] + t3 = invoke(session_id, msgs3, cfg, references, "stale-turn3") + evidence = { + "session_id": session_id, + "turn1_id": turn1, + "turn2_id": turn2, + "stale_stop": stale, + "bare_stop": bare, + "turn2_elapsed_s": t2.get("elapsed_s"), + "turn3_recalled_marker": marker in (t3.get("text") or ""), + "command_settled": settle, + } + evidence["sandbox_ids"] = sandbox_ids(session_id) + evidence["warm_same_sandbox"] = len(evidence["sandbox_ids"]) <= 1 + if stale["status"] not in (400, 404, 409): + return evidence, _fail( + f"stale Stop returned HTTP {stale['status']}, expected a mismatch status" + ) + if not settle["settled"]: + return evidence, _fail(settle["why"]) + if not evidence["turn3_recalled_marker"]: + return evidence, _fail("turn 2 did not survive the stale Stop") + return evidence, _pass("stale Stop was refused and turn 2 completed and survived") + + +def cell_stop_approval(cfg_ask, references_ask, args, hooks: OperatorHooks) -> Cell: + """Stop a parked approval and enforce the flag-specific late-answer behavior.""" + session_id = str(uuid.uuid4()) + marker = f"PEAR{uuid.uuid4().hex[:6].upper()}" + prompt = f"The codeword is {marker}. Run exactly this one shell command and nothing else: echo hello. Then reply DONE." + t1 = invoke( + session_id, [user_msg(prompt)], cfg_ask, references_ask, "approval-turn" + ) + time.sleep(3) + before = interactions(session_id) + stream_before = session_stream(session_id) + expected = t1.get("turn_id") or stream_before.get("turn_id") + stop = cancel(session_id, expected=expected, label="stop-approval-named") + settle = assert_command_settled(hooks, session_id, expected) + time.sleep(3) + pending = next((i for i in before if i.get("status") == "pending"), None) + late = {"skipped": "no pending interaction was found before the Stop"} + if pending: + r = api( + "POST", + f"/sessions/interactions/{pending['id']}/respond", + json={"answer": {"approved": True}}, + ) + late = {"status": r.status_code, "body": r.text[:300]} + denied = assistant_message(t1) + for part in denied["parts"]: + if ( + part.get("type", "").startswith("tool-") + and part.get("state") == "input-available" + ): + part["state"] = "output-denied" + msgs2 = [user_msg(prompt), denied, user_msg(RECALL)] + t2 = invoke(session_id, msgs2, cfg_ask, references_ask, "approval-resume") + evidence = { + "session_id": session_id, + "marker": marker, + "expected_execution_id": expected, + "stop": stop, + "late_answer": late, + "durable_stop": _resolve_durable_stop(args.durable_stop, stop["body"]), + "resume_recalled_marker": marker in (t2.get("text") or ""), + # Without the actual reply, a FAIL here cannot be told apart from a driver replay bug + # (the reconstructed `output-denied` part shaped wrong) versus the model genuinely not + # recalling the codeword -- keep enough of the wire to tell the two apart after the fact. + "resume_text": (t2.get("text") or "")[:400], + "resume_frames": t2.get("frames", [])[:20], + "resume_errors": t2.get("errors"), + "command_settled": settle, + } + evidence["sandbox_ids"] = sandbox_ids(session_id) + evidence["warm_same_sandbox"] = len(evidence["sandbox_ids"]) <= 1 + return evidence, _judge_stop_approval(evidence, pending_found=pending is not None) + + +def _judge_stop_approval(evidence: dict, *, pending_found: bool) -> dict: + """Apply all stop-approval assertions with only the late-answer rule gated by the flag.""" + if not pending_found: + return _fail( + "no pending approval was seen before the Stop; the race did not land" + ) + stop = evidence["stop"] + if stop["status"] not in (200, 202): + return _fail( + f"named Stop on a parked approval returned HTTP {stop['status']}, expected 200 or 202" + ) + settle = evidence["command_settled"] + if not settle["settled"]: + return _fail(settle["why"]) + durable_stop = evidence["durable_stop"] + if durable_stop not in ("on", "off"): + return _fail( + "could not determine durable Stop state from the cancel response; " + "pass --durable-stop on or off" + ) + late = evidence["late_answer"] + if durable_stop == "on" and late.get("status") != 409: + return _fail( + f"the late approval answer returned HTTP {late.get('status')}, expected 409" + ) + if durable_stop == "off": + if late.get("status") != 200: + return _fail( + "the legacy path refused the late approval answer, expected HTTP 200" + ) + evidence["late_answer"]["note"] = "late answer accepted: legacy path" + if not evidence["resume_recalled_marker"]: + return _fail("resume after the approval Stop did not recall the codeword") + if durable_stop == "off": + return _pass("late answer accepted: legacy path") + return _pass( + "Stop cancelled the parked approval, the late answer was refused, resume recalled the codeword" + ) + + +def cell_sandbox_gone(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Kill the sandbox under a running tool call. Needs shell to find and kill the process.""" + if not hooks.available: + return {}, _skip( + "no --project given: killing the sandbox process needs docker exec" + ) + session_id = str(uuid.uuid4()) + marker = f"OLIVE{uuid.uuid4().hex[:6].upper()}" + # `since` bounds the runner-log window used to map THIS session to its own sandbox port. + since = time.time() + msgs = [user_msg(sleep_prompt(marker, SANDBOX_GONE_COMMAND_S))] + handle = invoke_async(session_id, msgs, cfg, references, "sandbox-turn1") + turn = wait_for_turn(session_id) + settle_budget = sandbox_gone_settle_budget_s() + evidence = { + "session_id": session_id, + "turn_id": turn, + "command_seconds": SANDBOX_GONE_COMMAND_S, + "resolve_timeout_seconds": SANDBOX_GONE_RESOLVE_TIMEOUT_S, + "settle_budget_seconds": round(settle_budget, 1), + } + # A cold acquire writes this session's `prepare_workspace` line ~35 s in, so POLL for the + # session's own sandbox to become resolvable (up to the resolve timeout) instead of reading + # once right after the turn started. Refuse if the line never appears — never kill a guess. + # The ledger id (`local/:` on local, the remote uuid on Daytona) is the + # cross-check; never a shared `ps | grep`, which cannot tell two sessions apart. + resolve_started = time.time() + try: + hooks.wait_for_sandbox_ready( + session_id, + ledger_id_getter=lambda: (sandbox_ids(session_id) or [None])[-1], + since=since, + timeout=SANDBOX_GONE_RESOLVE_TIMEOUT_S, + ) + except WrongSandboxTarget as exc: + evidence["wrong_target"] = str(exc) + evidence["resolve_seconds"] = round(time.time() - resolve_started, 1) + return evidence, _fail(f"wrong target, refused to kill a guess: {exc}") + evidence["resolve_seconds"] = round(time.time() - resolve_started, 1) + # Let the slow command actually be running before the kill, so the sandbox dies mid-turn. + time.sleep(SANDBOX_GONE_RUNNING_SLACK_S) + observed_ids = sandbox_ids(session_id) + target_sandbox_id = observed_ids[-1] if observed_ids else None + evidence["target_sandbox_id"] = target_sandbox_id + # Now resolve the pid on the tested session's own port (or the remote id), assert it is a + # sandbox-agent daemon, and refuse (never kill a guess) if the mapping cannot be made. + try: + target = hooks.kill_sandbox_for_session( + session_id, sandbox_id=target_sandbox_id, since=since + ) + except WrongSandboxTarget as exc: + evidence["wrong_target"] = str(exc) + return evidence, _fail(f"wrong target, refused to kill a guess: {exc}") + evidence.update( + { + "killed_port": target.get("port"), + "killed_pid": target.get("pid"), + "killed_cmdline": target.get("cmdline"), + "killed_pids": target.get("killed"), + } + ) + # Wait for the runner to end the turn: the probe's three-strikes budget, and never shorter + # than the slow command, so a mis-target could not read as "still running" prematurely. The + # thread returns as soon as the stream closes, so a healthy kill settles well inside this. + wait_s = max(settle_budget, float(SANDBOX_GONE_COMMAND_S)) + SANDBOX_STARTUP_SLACK_S + handle["thread"].join(timeout=wait_s) + t1 = handle["out"] or {} + time.sleep(5) + terminal = _poll_terminal_after_settle(lambda: terminal_records(session_id, turn)) + evidence.update( + { + "turn1_errors": t1.get("errors"), + "terminal_records": terminal, + "stream_after": session_stream(session_id), + } + ) + if not target.get("killed"): + return evidence, _fail("no sandbox-agent process was found to kill") + flags = (evidence["stream_after"] or {}).get("flags") or {} + if flags.get("is_running"): + return evidence, _fail( + "session still reads is_running after the sandbox process was killed" + ) + if not evidence["terminal_records"]: + return evidence, _fail( + "no terminal record was written after the sandbox process was killed" + ) + return evidence, _pass( + "killing the tested session's own sandbox ended the turn and wrote a terminal record" + ) + + +def cell_records_outage(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Stop Postgres for 20 s during a turn. Every record must land after it returns.""" + if not hooks.available: + return {}, _skip("no --project given: stopping Postgres needs docker") + session_id = str(uuid.uuid4()) + marker = f"CEDAR{uuid.uuid4().hex[:6].upper()}" + msgs = [ + user_msg( + f"The codeword is {marker}. Run exactly this one shell command and nothing else: sleep 30. When it finishes, reply with the single word DONE." + ) + ] + handle = invoke_async(session_id, msgs, cfg, references, "outage-turn1") + wait_for_turn(session_id) + time.sleep(6) + hooks.stop_postgres() + try: + time.sleep(20) + finally: + # Restore Postgres even if something above raises: a stopped Postgres left behind + # strands every cell that runs after this one, not just this one's own assertions. + hooks.start_postgres() + handle["thread"].join(timeout=400) + landed = [] + deadline = time.time() + 180 + while time.time() < deadline: + landed = records(session_id) + if "done" in [r.get("record_type") for r in landed]: + break + time.sleep(5) + evidence = { + "session_id": session_id, + "marker": marker, + "record_types": [r.get("record_type") for r in landed], + "record_count": len(landed), + } + if "done" not in evidence["record_types"]: + return evidence, _fail( + "no done record landed after the Postgres outage recovered" + ) + return evidence, _pass("every record landed after the Postgres outage recovered") + + +def cell_stop_after_finish(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Fire the Stop at the instant the runner settles the prompt. Needs no shell for the core + assertion; the [control] aborted check is skipped without --project.""" + session_id = str(uuid.uuid4()) + marker = f"ACORN{uuid.uuid4().hex[:6].upper()}" + since = time.time() + msgs = [ + user_msg( + f"The codeword is {marker}. Run exactly this one shell command and nothing else: sleep 6. When it finishes, reply with the single word DONE." + ) + ] + seen: dict = {} + watcher_proc = None + if hooks.available: + watcher_proc = subprocess.Popen( + ["docker", "logs", "-f", "--since", "0s", f"{args.project}-runner-1"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + + def watch() -> None: + assert watcher_proc.stdout is not None + for line in watcher_proc.stdout: + if "prompt stopReason=" in line: + seen["line"] = line.strip() + seen["at"] = time.time() + return + + threading.Thread(target=watch, daemon=True).start() + + handle = invoke_async(session_id, msgs, cfg, references, "finish-turn1") + turn = wait_for_turn(session_id) + if hooks.available: + deadline = time.time() + 180 + while "at" not in seen and time.time() < deadline: + time.sleep(0.02) + else: + time.sleep( + 6.5 + ) # no runner-log watch: fire the Stop right around the natural finish + stop = cancel(session_id, expected=turn, label="stop-after-finish") + if watcher_proc: + try: + watcher_proc.kill() + except Exception: # noqa: BLE001 + pass + handle["thread"].join(timeout=180) + t1 = handle["out"] or {} + time.sleep(6) + msgs2 = msgs + [assistant_message(t1), user_msg(RECALL)] + t2 = invoke(session_id, msgs2, cfg, references, "finish-turn2") + evidence = { + "session_id": session_id, + "turn_id": t1.get("turn_id") or turn, + "settle_line": seen.get("line"), + "stop": stop, + "resume_recalled_marker": marker in (t2.get("text") or ""), + "terminal_records": terminal_records(session_id, turn), + } + evidence["sandbox_ids"] = sandbox_ids(session_id) + evidence["warm_same_sandbox"] = len(evidence["sandbox_ids"]) <= 1 + if hooks.available: + logs = hooks.runner_log(since) + evidence["control_aborted_lines"] = [ + ln for ln in logs if "[control] aborted" in ln and session_id in ln + ] + if hooks.available and evidence.get("control_aborted_lines"): + return evidence, _fail( + "a Stop that lost the race to completion still aborted the settled run" + ) + if not evidence["resume_recalled_marker"]: + return evidence, _fail( + "the session did not park warm after Stop raced a finished turn" + ) + why = ( + "no spurious abort and a warm continuation recalled the codeword" + if hooks.available + else "a warm continuation recalled the codeword (abort-log check skipped: no --project)" + ) + return evidence, _pass(why) + + +def cell_restart_after_stop(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Stop, restart the runner, continue with an EMPTY client transcript. + + The codeword recall alone is not proof of native continuity: when the native session did not + truly hydrate, the runner can still answer correctly by reconstructing the conversation from + the persisted record log (the `[reconstruct]` / `session/load ... loaded=false` path), and a + driver that ever sent more than the trailing message could paper over the same gap from the + client side. So this cell forces its resume onto `--client-shape last-message` (the shape the + desktop actually sends) regardless of the run's own `--client-shape`, and requires a SECOND, + independent signal beyond the recalled codeword: either the sandbox id after the restart is + the SAME one the turn ran on before it (true continuity needs no rebuild), or the runner log + for the resume shows `session/load ... loaded=true` (a genuine native hydrate, not a + reconstruction). Recall without either is a false pass, not a pass. + """ + if not hooks.available: + return {}, _skip("no --project given: restarting the runner needs docker") + session_id = str(uuid.uuid4()) + marker = f"BIRCH{uuid.uuid4().hex[:6].upper()}" + msgs = [user_msg(sleep_prompt(marker, args.sleep_seconds))] + handle = invoke_async(session_id, msgs, cfg, references, "restart-turn1") + turn = wait_for_turn(session_id) + wait_for_tool(handle) + stop = cancel(session_id, expected=turn, label="stop-before-restart") + handle["thread"].join(timeout=180) + time.sleep(4) + sandbox_id_before = (sandbox_ids(session_id) or [None])[-1] + restart_at = time.time() + hooks.restart_runner(grace_seconds=10) + healthy_after = hooks.wait_for_runner() + attempts = [] + admitted = None + global CLIENT_SHAPE + prior_client_shape = CLIENT_SHAPE + CLIENT_SHAPE = "last-message" + try: + deadline = time.time() + 240 + while time.time() < deadline: + t = invoke( + session_id, [user_msg(RECALL)], cfg, references, "restart-recall" + ) + refused = any( + "already running a turn" in (e or "") for e in t.get("errors", []) + ) + attempts.append( + { + "at_s_after_restart": round(time.time() - restart_at, 1), + "refused": refused, + } + ) + if not refused: + admitted = t + break + time.sleep(5) + finally: + CLIENT_SHAPE = prior_client_shape + sandbox_id_after = (sandbox_ids(session_id) or [None])[-1] + resume_log = [ + line + for line in hooks.runner_log(restart_at) + if session_id in line and "session/load" in line + ] + loaded_true = any("loaded=true" in line for line in resume_log) + same_sandbox = bool(sandbox_id_before) and sandbox_id_before == sandbox_id_after + evidence = { + "session_id": session_id, + "turn_id": turn, + "stop": stop, + "runner_healthy_after_s": healthy_after, + "attempts": attempts, + "admitted_at_s": attempts[-1]["at_s_after_restart"] if admitted else None, + "recalled_marker": marker in ((admitted or {}).get("text") or ""), + "sandbox_id_before": sandbox_id_before, + "sandbox_id_after": sandbox_id_after, + "same_sandbox": same_sandbox, + "resume_load_log_lines": resume_log, + "loaded_true": loaded_true, + } + return evidence, _judge_restart_after_stop(evidence) + + +def _judge_restart_after_stop(evidence: dict) -> dict: + """PASS rule for `restart-after-stop`. A recalled codeword alone is not proof of native + continuity — the runner can recover it by reconstructing the conversation from persisted + records even when the native session did not truly hydrate. Require the recall AND one of: + the sandbox was not rebuilt (`same_sandbox`), or the runner log shows a genuine native hydrate + (`loaded_true`). See `cell_restart_after_stop`'s docstring for why.""" + if evidence.get("runner_healthy_after_s") is None: + return _fail("the runner never reported healthy after the restart") + if evidence.get("admitted_at_s") is None: + return _fail( + "the continuation was refused for the whole wait window after the restart" + ) + if not evidence.get("recalled_marker"): + return _fail( + "the native harness session did not survive the restart: the codeword was not recalled" + ) + if not (evidence.get("same_sandbox") or evidence.get("loaded_true")): + return _fail("native session not resumed, recovered by transcript replay") + return _pass( + "the runner rehydrated the native session across a restart and recalled the codeword" + ) + + +def cell_runner_gone(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Pause the runner BEFORE the Stop, so the command can never be claimed or reported. + + Deterministic version of the hard race: hoping a restart lands between the Stop and the + runner's own outcome report is timing-dependent and mostly loses the race (see + `runner-gone-late`). Pausing first removes the timing dependency: the runner cannot claim + or report the command at all, so it must stay `pending` until the stale threshold and the + sweep interval both pass (--sweep-wait), at which point the sweep must settle it `lost` + (state `obsolete` or `applied`, outcome `lost`) and write the execution's own watchdog + `execution_lost` ending. + + Every gone-and-stays-gone signal — the settled command, the watchdog ending, and is_running: + false — is read WHILE THE RUNNER IS STILL PAUSED. Reading after the unpause is the run-2b bug: + the returning runner starts a new turn on the same session that legitimately sets is_running + true. The unpause and the Send that follows are only a restore step plus an OPTIONAL, separately + recorded resumability check; they are not part of this cell's pass. + """ + if not hooks.available: + return {}, _skip("no --project given: pausing the runner needs docker") + session_id = str(uuid.uuid4()) + marker = f"FIG{uuid.uuid4().hex[:6].upper()}" + msgs = [user_msg(sleep_prompt(marker, 240))] + handle = invoke_async(session_id, msgs, cfg, references, "gone-turn1") + turn = wait_for_turn(session_id) + time.sleep(5) + + # Pause, Stop, settle, and READ is_running all while the runner is still gone. Measuring after + # the unpause is the run-2b bug: the returning runner starts a new turn on the same session + # that legitimately sets is_running true, and the driver read that true. + measured = _measure_runner_gone_while_paused( + hooks, + session_id, + turn, + do_stop=lambda: cancel(session_id, expected=turn, label="stop-then-pause"), + read_terminal=lambda: terminal_records(session_id, turn), + sweep_wait=args.sweep_wait, + ) + stop = measured["stop"] + settled_at = measured["settled_at"] + stop_command = measured["stop_command"] + terminal = measured["terminal"] + stream_row = measured["stream_row"] + paused_is_running = ( + (stream_row.get("flags") or {}).get("is_running") if stream_row else None + ) + + # The runner is back. Restore health, then run an OPTIONAL, separately-recorded resumability + # check — a Send after health. It is NOT part of the runner-gone verdict: a returning runner + # starting a new turn is exactly the signal that must not count against "gone". + healthy_after_s = hooks.wait_for_runner() + resume: dict = {"attempted": False} + if healthy_after_s is not None: + handle["thread"].join(timeout=60) + runner_recovered, t2 = _recover_then_send( + health_poll=hooks.runner_healthy, + send=lambda: invoke( + session_id, + [ + user_msg( + f"The codeword is {marker}. Reply with just the single word READY." + ) + ], + cfg, + references, + "gone-turn2", + ), + timeout=RECOVERY_HEALTH_TIMEOUT_S, + poll_interval=RECOVERY_HEALTH_POLL_S, + ) + t2 = t2 or {} + resume = { + "attempted": True, + "runner_recovered": runner_recovered, + "ran": bool(t2.get("frames")) and not t2.get("errors"), + "errors": t2.get("errors"), + } + + evidence = { + "session_id": session_id, + "turn_id": turn, + "stop": stop, + "settled_at": measured["settled_at"], + "paused_read_at": measured["paused_read_at"], + "seconds_to_settle": ( + round(settled_at - measured["stop_at"], 1) + if settled_at and measured["stop_at"] + else None + ), + "terminal_records": terminal, + "commands": measured["commands"], + "executions": measured["executions"], + "stop_command": stop_command, + "stream_row_while_paused": stream_row, + "is_running_while_paused": paused_is_running, + "healthy_after_unpause_s": healthy_after_s, + "resumability": resume, + } + # Gone-and-stays-gone verdict, every signal measured WHILE the runner was still paused. + if settled_at is None: + return evidence, _fail( + "the Stop was not settled lost within the sweep-wait window while the runner was paused" + ) + if stop_command is None: + return evidence, _fail("no session_commands row was found for the Stop") + if stop_command.get("state") not in ("obsolete", "applied"): + return evidence, _fail( + f"the Stop command read state {stop_command.get('state')!r}, expected obsolete or applied" + ) + if stop_command.get("outcome") != "lost": + return evidence, _fail( + f"the Stop command read outcome {stop_command.get('outcome')!r}, expected lost: a " + "paused runner should never have been able to report it" + ) + watchdog_failure = _require_watchdog_execution_lost(terminal) + if watchdog_failure: + return evidence, watchdog_failure + if paused_is_running is not False: + return evidence, _fail( + "the session_streams row did not read is_running: false while the runner was still paused" + ) + evidence["race"] = "never-reported" + return evidence, _pass( + "pausing the runner first forced the never-reported race: while the runner was still gone " + "the sweep settled the Stop lost with a watchdog execution_lost ending and the stream row " + "read is_running: false" + ) + + +def cell_runner_gone_late(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Restart the runner right after a Stop is claimed, hoping it lands before the runner can + report the outcome. The softer, timing-dependent sibling of `runner-gone`: a restart often + loses this race (the runner reports the Stop's outcome before it actually dies), so this + cell accepts either race the sweep can produce — see `_judge_runner_gone`. + """ + if not hooks.available: + return {}, _skip("no --project given: restarting the runner needs docker") + session_id = str(uuid.uuid4()) + marker = f"FIG{uuid.uuid4().hex[:6].upper()}" + msgs = [user_msg(sleep_prompt(marker, 240))] + handle = invoke_async(session_id, msgs, cfg, references, "gone-late-turn1") + turn = wait_for_turn(session_id) + time.sleep(5) + + # Stop first, then take the runner away before it can (maybe) report the outcome. + stop = cancel(session_id, expected=turn, label="stop-then-kill") + kill_at = time.time() + hooks.kill_runner() + print( + f"[runner-gone-late] restarted the runner at {time.strftime('%H:%M:%S')}", + file=sys.stderr, + ) + handle["thread"].join(timeout=60) + + # Wait for the sweep. The plan budgets the stale threshold plus the sweep interval, held in + # --sweep-wait. + settled_at = None + terminal: list = [] + deadline = time.time() + args.sweep_wait + while time.time() < deadline: + stream = session_stream(session_id) + flags = stream.get("flags") or {} + terminal = terminal_records(session_id, turn) + if terminal and not flags.get("is_running"): + settled_at = time.time() + break + time.sleep(5) + + if settled_at is not None: + terminal = _poll_terminal_after_settle( + lambda: terminal_records(session_id, turn) + ) + + time.sleep(3) + commands = hooks.command_rows(session_id) + stream_row = hooks.stream_row(session_id) + stop_command = _match_stop_command(commands, turn) + # The runner is restarting from the kill above. A recovery Send issued before it is back up + # gets "All connection attempts failed" and is misread as a product failure. Wait for the + # runner to be healthy again (bounded), THEN send. If it never recovers, do not send a doomed + # request — `runner_recovered` records which happened. + recover_started = time.time() + runner_recovered, t2 = _recover_then_send( + health_poll=hooks.runner_healthy, + send=lambda: invoke( + session_id, + [ + user_msg( + f"The codeword is {marker}. Reply with just the single word READY." + ) + ], + cfg, + references, + "gone-late-turn2", + ), + timeout=RECOVERY_HEALTH_TIMEOUT_S, + poll_interval=RECOVERY_HEALTH_POLL_S, + ) + t2 = t2 or {} + evidence = { + "session_id": session_id, + "turn_id": turn, + "stop": stop, + "seconds_to_settle": round(settled_at - kill_at, 1) if settled_at else None, + "terminal_records": terminal, + "stream_after": session_stream(session_id), + "commands": commands, + "stop_command": stop_command, + "stream_row": stream_row, + "runner_recovered": runner_recovered, + "runner_recover_seconds": round(time.time() - recover_started, 1), + "new_message_ran": bool(t2.get("frames")) and not t2.get("errors"), + "new_message_errors": t2.get("errors"), + } + if not runner_recovered: + return evidence, _fail( + f"runner did not become healthy within {RECOVERY_HEALTH_TIMEOUT_S:.0f}s after the " + "restart; recovery Send not attempted" + ) + return evidence, _judge_runner_gone(evidence) + + +def cell_post_stop_row(cfg, references, args, hooks: OperatorHooks) -> Cell: + """After a Stop the row must read is_running: false within a few seconds.""" + if not hooks.available: + return {}, _skip("no --project given: reading the Postgres row needs psql") + session_id = str(uuid.uuid4()) + marker = f"CEDAR{uuid.uuid4().hex[:6].upper()}" + msgs = [user_msg(sleep_prompt(marker, args.sleep_seconds))] + handle = invoke_async(session_id, msgs, cfg, references, "row-turn1") + turn = wait_for_turn(session_id) + wait_for_tool(handle) + stop = cancel(session_id, expected=turn, label="stop-post-row") + first_false_at = None + deadline = time.time() + 20 + while time.time() < deadline: + row = hooks.stream_row(session_id) + flags = row.get("flags") or {} + if flags.get("is_running") is False: + first_false_at = round(row.get("read_at", time.time()) - stop["sent_at"], 2) + break + time.sleep(0.1) + settle = assert_command_settled(hooks, session_id, turn) + handle["thread"].join(timeout=180) + evidence = { + "session_id": session_id, + "turn_id": turn, + "stop": stop, + "seconds_to_is_running_false": first_false_at, + "command_settled": settle, + } + if first_false_at is None: + return evidence, _fail( + "the Postgres row never read is_running: false within 20 s of the Stop" + ) + if first_false_at > 5: + return evidence, _fail( + f"the row took {first_false_at}s to read is_running: false, expected under 5s" + ) + if not settle["settled"]: + return evidence, _fail(settle["why"]) + return evidence, _pass( + f"the row read is_running: false {first_false_at}s after the Stop" + ) + + +def cell_codex_child(cfg, references, args, hooks: OperatorHooks) -> Cell: + """A stopped Codex turn must not leave its shell child alive in the parked sandbox.""" + if not hooks.available: + return {}, _skip( + "no --project given: reading the runner's process table needs docker exec" + ) + session_id = str(uuid.uuid4()) + codeword = f"DELTA{uuid.uuid4().hex[:6].upper()}" + marker = f"sleep 300.{uuid.uuid4().int % 900000 + 100000}" + msgs = [ + user_msg( + f"The codeword is {codeword}. Run exactly this one shell command and nothing else: {marker}\n" + "Run it in the FOREGROUND and wait for it to finish. Never run it in the background and never " + "append an ampersand. Do not read, write or search any files. When it finishes, reply with the " + "single word DONE." + ) + ] + handle = invoke_async(session_id, msgs, cfg, references, "codex-turn1") + turn = wait_for_turn(session_id, timeout=90) + # The sandbox id this session's turn ledger observed. On daytona, `sandbox_procs` needs this + # to know which remote sandbox to list processes on (DaytonaAwareHooks); on local it is + # unused (docker exec into the runner container sees every local sandbox subprocess). + observed_ids = sandbox_ids(session_id) + target_sandbox_id = observed_ids[-1] if observed_ids else None + child_before = [] + deadline = time.time() + 120 + while time.time() < deadline: + child_before = hooks.sandbox_procs(marker, sandbox_id=target_sandbox_id) + if child_before: + break + if not target_sandbox_id: + observed_ids = sandbox_ids(session_id) + target_sandbox_id = observed_ids[-1] if observed_ids else None + time.sleep(1) + stop = cancel(session_id, expected=turn, label="stop-codex") + settle = assert_command_settled(hooks, session_id, turn) + handle["thread"].join(timeout=180) + gone_at = None + deadline = time.time() + 45 + while time.time() < deadline: + alive = hooks.sandbox_procs(marker, sandbox_id=target_sandbox_id) + if not alive: + gone_at = round(time.time() - stop["sent_at"], 1) + break + time.sleep(1) + time.sleep(4) + t2 = invoke( + session_id, + msgs + [assistant_message(handle["out"] or {}), user_msg(RECALL)], + cfg, + references, + "codex-turn2", + ) + evidence = { + "session_id": session_id, + "turn_id": turn, + "target_sandbox_id": target_sandbox_id, + "child_before_stop": child_before, + "stop": stop, + "seconds_until_child_gone": gone_at, + "resume_recalled_marker": codeword in (t2.get("text") or ""), + "command_settled": settle, + } + if not child_before: + return evidence, _fail( + "never observed the child process before the Stop; the race did not land" + ) + if gone_at is None: + return evidence, _fail("the child process was still alive 45s after the Stop") + if not settle["settled"]: + return evidence, _fail(settle["why"]) + if not evidence["resume_recalled_marker"]: + return evidence, _fail( + "the parked Codex sandbox did not recall the codeword on resume" + ) + return evidence, _pass( + f"the child was reaped {gone_at}s after Stop and the resume recalled the codeword" + ) + + +def cell_stale_tail(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Freeze the runner past the watchdog threshold, thaw it, and read the late tail.""" + if not hooks.available: + return {}, _skip("no --project given: pausing the runner needs docker") + session_id = str(uuid.uuid4()) + marker = f"ELDER{uuid.uuid4().hex[:6].upper()}" + msgs = [ + user_msg( + f"The codeword is {marker}. Run exactly this one shell command and nothing else: sleep 20. When it finishes, reply with the single word DONE." + ) + ] + handle = invoke_async(session_id, msgs, cfg, references, "tail-turn1") + wait_for_turn(session_id) + time.sleep(3) + hooks.pause_runner() + try: + deadline = time.time() + args.sweep_wait + while time.time() < deadline: + if any(r["type"] == "done" for r in hooks.record_rows(session_id)): + break + time.sleep(5) + finally: + # A paused runner left behind strands every cell that runs after this one. Restore it + # even if hooks.record_rows() above raises. + hooks.unpause_runner() + handle["thread"].join(timeout=180) + time.sleep(20) + rows = hooks.record_rows(session_id) + quarantined = [r for r in rows if r["quarantined_at"]] + endpoint = [r.get("record_type") for r in records(session_id)] + evidence = { + "session_id": session_id, + "quarantined": quarantined, + "endpoint_record_types": endpoint, + } + if not quarantined: + return evidence, _fail( + "no late record was quarantined after the runner was thawed past the watchdog window" + ) + if "done" not in endpoint and "error" not in endpoint: + return evidence, _fail( + "the transcript read shows no terminal record after the watchdog fired" + ) + return evidence, _pass( + f"{len(quarantined)} late record(s) quarantined and hidden from the transcript read" + ) + + +def cell_repeat_stop(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Two Stop requests for one execution, 50ms apart. One command effect, one ending.""" + session_id = str(uuid.uuid4()) + marker = f"HAZEL{uuid.uuid4().hex[:6].upper()}" + msgs = [user_msg(sleep_prompt(marker, args.sleep_seconds))] + handle = invoke_async(session_id, msgs, cfg, references, "repeat-turn1") + turn = wait_for_turn(session_id) + wait_for_tool(handle) + results: list = [] + + def fire(label: str) -> None: + results.append(cancel(session_id, expected=turn, label=label)) + + t1 = threading.Thread(target=fire, args=("repeat-stop-a",)) + t1.start() + time.sleep(0.05) + t2 = threading.Thread(target=fire, args=("repeat-stop-b",)) + t2.start() + t1.join() + t2.join() + settle = assert_command_settled(hooks, session_id, turn) + handle["thread"].join(timeout=180) + out = handle["out"] or {} + time.sleep(4) + t3 = invoke( + session_id, + msgs + [assistant_message(out), user_msg(RECALL)], + cfg, + references, + "repeat-turn2", + ) + evidence = { + "session_id": session_id, + "turn_id": turn, + "stops": results, + "terminal_records": terminal_records(session_id, turn), + "resume_recalled_marker": marker in (t3.get("text") or ""), + "command_settled": settle, + } + evidence["sandbox_ids"] = sandbox_ids(session_id) + evidence["warm_same_sandbox"] = len(evidence["sandbox_ids"]) <= 1 + if hooks.available: + evidence["commands"] = hooks.command_rows(session_id) + accepted = [r for r in results if r["status"] in (200, 202)] + if len(accepted) == 0: + return evidence, _fail("neither of the two repeated Stops was accepted") + if len(evidence["terminal_records"]) != 1: + return evidence, _fail( + f"expected exactly one terminal record for the turn, saw {len(evidence['terminal_records'])}" + ) + if not settle["settled"]: + return evidence, _fail(settle["why"]) + if not evidence["resume_recalled_marker"]: + return evidence, _fail( + "resume after the repeated Stop did not recall the codeword" + ) + return evidence, _pass( + "two Stops 50ms apart produced exactly one terminal record and a warm resume" + ) + + +def cell_concurrent_stops(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Five independent sessions, each with a long turn, all Stopped within one second. + + Every Stop must return HTTP 200 or 202, every session must read exactly one terminal record, and + every session must recall its own codeword on a warm resume. HTTP-only: needs no shell. + """ + n = 5 + sessions = [] + for i in range(n): + session_id = str(uuid.uuid4()) + marker = f"NOVA{i}{uuid.uuid4().hex[:5].upper()}" + msgs = [user_msg(sleep_prompt(marker, args.sleep_seconds))] + handle = invoke_async( + session_id, msgs, cfg, references, f"concurrent-turn1-{i}" + ) + sessions.append( + {"session_id": session_id, "marker": marker, "msgs": msgs, "handle": handle} + ) + + for s in sessions: + s["turn_id"] = wait_for_turn(s["session_id"]) + missing_turn = [s["session_id"] for s in sessions if not s["turn_id"]] + if missing_turn: + evidence = {"n": n, "missing_turn_sessions": missing_turn} + return evidence, _fail( + f"{len(missing_turn)} of {n} sessions never reported a running turn" + ) + time.sleep(2) + + def fire(s: dict) -> None: + s["stop"] = cancel( + s["session_id"], + expected=s["turn_id"], + label=f"concurrent-stop-{s['marker']}", + ) + + threads = [threading.Thread(target=fire, args=(s,)) for s in sessions] + fired_at = time.time() + for t in threads: + t.start() + for t in threads: + t.join() + stop_window_s = round(time.time() - fired_at, 3) + + def settle(s: dict) -> None: + s["command_settled"] = assert_command_settled( + hooks, s["session_id"], s["turn_id"] + ) + + settle_threads = [threading.Thread(target=settle, args=(s,)) for s in sessions] + for t in settle_threads: + t.start() + for t in settle_threads: + t.join() + + for s in sessions: + s["handle"]["thread"].join(timeout=180) + s["out"] = s["handle"]["out"] or {} + time.sleep(4) + + def read_terminal(s: dict) -> None: + s["terminal_records"] = _poll_terminal_after_settle( + lambda: terminal_records(s["session_id"], s["turn_id"]) + ) + + terminal_threads = [ + threading.Thread(target=read_terminal, args=(s,)) for s in sessions + ] + for thread in terminal_threads: + thread.start() + for thread in terminal_threads: + thread.join() + + for s in sessions: + msgs2 = s["msgs"] + [assistant_message(s["out"]), user_msg(RECALL)] + t2 = invoke( + s["session_id"], msgs2, cfg, references, f"concurrent-turn2-{s['marker']}" + ) + s["resume_recalled_marker"] = s["marker"] in (t2.get("text") or "") + s["resume_text"] = (t2.get("text") or "")[:200] + + evidence = { + "n": n, + "stop_window_s": stop_window_s, + "sessions": [ + { + "session_id": s["session_id"], + "turn_id": s["turn_id"], + "stop_status": s["stop"]["status"], + "stop_round_trip_s": s["stop"]["round_trip_s"], + "terminal_record_count": len(s["terminal_records"]), + "resume_recalled_marker": s["resume_recalled_marker"], + "command_settled": s["command_settled"], + } + for s in sessions + ], + } + not_accepted = [ + s["session_id"] for s in sessions if s["stop"]["status"] not in (200, 202) + ] + if not_accepted: + return evidence, _fail( + f"{len(not_accepted)} of {n} concurrent Stops did not return HTTP 200 or 202: " + f"{not_accepted}" + ) + unsettled = [ + s["session_id"] for s in sessions if not s["command_settled"]["settled"] + ] + if unsettled: + return evidence, _fail( + f"{len(unsettled)} of {n} sessions did not settle their Stop command within " + f"20s: {unsettled}" + ) + bad_terminal = [ + s["session_id"] for s in sessions if len(s["terminal_records"]) != 1 + ] + if bad_terminal: + return evidence, _fail( + f"{len(bad_terminal)} of {n} sessions did not read exactly one terminal record: " + f"{bad_terminal}" + ) + not_recalled = [ + s["session_id"] for s in sessions if not s["resume_recalled_marker"] + ] + if not_recalled: + return evidence, _fail( + f"{len(not_recalled)} of {n} sessions did not recall their codeword on resume: " + f"{not_recalled}" + ) + return evidence, _pass( + f"all {n} concurrent Stops returned HTTP 200 or 202 within {stop_window_s}s, each session read " + "exactly one terminal record, and each resumed warm with its own codeword" + ) + + +def cell_stop_during_completion(cfg, references, args, hooks: OperatorHooks) -> Cell: + """Stop fired at the moment a short (toolless) turn completes naturally. One committed winner: + obsolete/not_running, or a clean stopped ending — never both, never neither.""" + session_id = str(uuid.uuid4()) + marker = f"IVY{uuid.uuid4().hex[:6].upper()}" + msgs = [ + user_msg(f"The codeword is {marker}. Reply with just the single word READY.") + ] + handle = invoke_async(session_id, msgs, cfg, references, "completion-turn1") + turn = wait_for_turn(session_id) + # Race the natural finish: poll the live frame count and fire the instant it stops growing, + # which is the closest an HTTP-only driver can land on "while the execution completes". + live = handle["live"] + last_len = -1 + stable_since = None + deadline = time.time() + 30 + while time.time() < deadline: + n = len(live.get("frames") or []) + if n == last_len and n > 0: + if stable_since is None: + stable_since = time.time() + elif time.time() - stable_since > 0.05: + break + else: + stable_since = None + last_len = n + if handle["out"] is not None: + break + time.sleep(0.02) + stop = cancel(session_id, expected=turn, label="stop-during-completion") + handle["thread"].join(timeout=60) + out = handle["out"] or {} + time.sleep(3) + terminal = terminal_records(session_id, turn) + stream_after = session_stream(session_id) + t2 = invoke( + session_id, + msgs + [assistant_message(out), user_msg(RECALL)], + cfg, + references, + "completion-turn2", + ) + evidence = { + "session_id": session_id, + "turn_id": turn, + "stop": stop, + "terminal_records": terminal, + "stream_after_flags": (stream_after or {}).get("flags"), + "resume_recalled_marker": marker in (t2.get("text") or ""), + } + evidence["sandbox_ids"] = sandbox_ids(session_id) + evidence["warm_same_sandbox"] = len(evidence["sandbox_ids"]) <= 1 + if len(terminal) > 1: + return evidence, _fail( + f"the race produced {len(terminal)} terminal records for one turn, expected one" + ) + if stop["status"] not in (200, 202, 404, 409): + return evidence, _fail( + f"Stop-at-completion returned an unexpected HTTP {stop['status']}" + ) + if not evidence["resume_recalled_marker"]: + return evidence, _fail( + "the session did not survive the completion race cleanly" + ) + return evidence, _pass( + "Stop racing a natural finish produced exactly one committed ending and a clean resume" + ) + + +# (needs_hooks, permission, fn) +CELLS: dict[str, tuple[bool, str, "object"]] = { + "stop-warm": (False, "allow", cell_stop_warm), + "double-send": (False, "allow", cell_double_send), + "stale-stop": (False, "allow", cell_stale_stop), + "stop-approval": (False, "ask", cell_stop_approval), + "sandbox-gone": (True, "allow", cell_sandbox_gone), + "records-outage": (True, "allow", cell_records_outage), + "stop-after-finish": (False, "allow", cell_stop_after_finish), + "restart-after-stop": (True, "allow", cell_restart_after_stop), + "runner-gone": (True, "allow", cell_runner_gone), + "runner-gone-late": (True, "allow", cell_runner_gone_late), + "post-stop-row": (True, "allow", cell_post_stop_row), + "codex-child": (True, "allow", cell_codex_child), + "stale-tail": (True, "allow", cell_stale_tail), + "repeat-stop": (False, "allow", cell_repeat_stop), + "concurrent-stops": (False, "allow", cell_concurrent_stops), + "stop-during-completion": (False, "allow", cell_stop_during_completion), +} + + +def run_cell( + name: str, fn, cfg, references, args, hooks: OperatorHooks, needs_hooks: bool +) -> dict: + """Run one cell and return its `results["cells"][name]` entry. + + A cell that pauses, stops, or restarts the runner restores it itself in its own `finally` + block (see `cell_stale_tail` and `cell_records_outage`). This is the second, run-level + guarantee: a cell that raises BEFORE its own restore code runs must not strand the runner + paused or down for the cell that runs after it, so the recovery check here runs in a + `finally` block too, no matter how the cell ends. + """ + started = time.time() + try: + evidence, verdict = fn(cfg, references, args, hooks) + except Exception as exc: # noqa: BLE001 + import traceback + + evidence = { + "driver_error": f"{type(exc).__name__}: {exc}", + "traceback": traceback.format_exc()[-1500:], + } + verdict = _fail(f"driver exception: {type(exc).__name__}: {exc}") + finally: + if needs_hooks and hooks.available: + try: + recovery = hooks.ensure_runner_healthy() + except Exception as exc: # noqa: BLE001 + print(f"[{name}] runner-health recovery failed: {exc}", file=sys.stderr) + else: + if ( + recovery.get("was_paused") + or recovery.get("status_before") != "running" + ): + print(f"[{name}] recovered the runner: {recovery}", file=sys.stderr) + if recovery.get("healthy_after_s") is None: + print( + f"[{name}] WARNING: the runner did not report healthy after recovery", + file=sys.stderr, + ) + elapsed = round(time.time() - started, 1) + verdict_str = "SKIP" if verdict["skip"] else ("PASS" if verdict["pass"] else "FAIL") + print(f"[{name}] {verdict_str} — {verdict['why']}", file=sys.stderr) + return {"evidence": evidence, "verdict": verdict, "elapsed_s": elapsed} + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--harness", default="pi_core", choices=sorted(HARNESSES)) + ap.add_argument("--cells", default="all", help="comma separated, or 'all'") + ap.add_argument("--sleep-seconds", type=int, default=45) + ap.add_argument("--sweep-wait", type=float, default=240.0) + ap.add_argument( + "--project", + default=None, + help="docker-compose project name; enables the shell-only cells", + ) + ap.add_argument("--sandbox", default="local", choices=["local", "daytona"]) + ap.add_argument( + "--durable-stop", + default="auto", + choices=["on", "off", "auto"], + help=( + "durable Stop feature state. auto (default) detects command+execution responses " + "as on and legacy cancellation-summary responses as off" + ), + ) + ap.add_argument( + "--client-shape", + default="full", + choices=["full", "last-message"], + help=( + "full (default) replays the whole transcript on every send, keeping results " + "comparable with prior runs. last-message sends only the new user message on " + "every resume and follow-up, the way the desktop client does (agentRequest.ts) — " + "use it to catch continuity bugs the full transcript masks." + ), + ) + ap.add_argument( + "--resume", + default=None, + help="path to a prior run's results.json; cells already recorded there are loaded, not re-run", + ) + args = ap.parse_args() + + wanted = ( + list(CELLS) + if args.cells == "all" + else [c.strip() for c in args.cells.split(",") if c.strip()] + ) + unknown = [c for c in wanted if c not in CELLS] + if unknown: + raise SystemExit(f"unknown cells: {unknown}; known: {sorted(CELLS)}") + + resolve_env() + hooks = select_hooks(args.project, args.sandbox) + if args.sandbox == "daytona": + global SANDBOX_STARTUP_SLACK_S + SANDBOX_STARTUP_SLACK_S = 25.0 + global CLIENT_SHAPE + CLIENT_SHAPE = args.client_shape + global DURABLE_STOP_OPTION, DURABLE_STOP_STATE + DURABLE_STOP_OPTION = args.durable_stop + DURABLE_STOP_STATE = ( + args.durable_stop if args.durable_stop in ("on", "off") else None + ) + + prior: dict = {} + if args.resume: + prior_path = pathlib.Path(args.resume).expanduser() + if prior_path.exists(): + prior = json.loads(prior_path.read_text()).get("cells", {}) + print( + f"[resume] loaded {len(prior)} cell result(s) from {prior_path}", + file=sys.stderr, + ) + + bootstrap(args.harness) + spec = HARNESSES[args.harness] + base_cfg = agent_config( + spec["kind"], spec["model"], spec["provider"], spec["connection"], args.sandbox + ) + built: dict = {} + + def config_for(permission: str): + if permission not in built: + cfg = json.loads(json.dumps(base_cfg)) + cfg["runner"] = {"permissions": {"default": permission}} + built[permission] = ( + cfg, + create_revision(cfg, f"session-control-{args.sandbox}-{permission}"), + ) + return built[permission] + + # PID, not just the second-resolution timestamp: two invocations started in the same second + # (e.g. two harnesses smoke-tested in parallel) would otherwise share a folder and the + # second writer silently clobbers the first one's results.json mid-run. + stamp = time.strftime("%Y%m%d-%H%M%S") + outdir = RUNS / f"{stamp}-{os.getpid()}-session-control" + outdir.mkdir(parents=True, exist_ok=True) + + results: dict = { + "project_id": STATE["project_id"], + "harness": args.harness, + "sandbox": args.sandbox, + "client_shape": args.client_shape, + "durable_stop": { + "option": args.durable_stop, + "state": DURABLE_STOP_STATE, + }, + "cells": {}, + } + for name in wanted: + if name in prior: + print( + f"[{name}] resumed from prior run: {prior[name]['verdict']['pass'] and 'PASS' or (prior[name]['verdict']['skip'] and 'SKIP' or 'FAIL')}", + file=sys.stderr, + ) + results["cells"][name] = prior[name] + results["durable_stop"]["state"] = DURABLE_STOP_STATE + (outdir / "results.json").write_text( + json.dumps(results, indent=2, default=str) + ) + continue + needs_hooks, permission, fn = CELLS[name] + cfg, references = config_for(permission) + print(f"\n=== cell {name} ===", file=sys.stderr) + results["cells"][name] = run_cell( + name, fn, cfg, references, args, hooks, needs_hooks + ) + results["durable_stop"]["state"] = DURABLE_STOP_STATE + (outdir / "results.json").write_text(json.dumps(results, indent=2, default=str)) + + lines = ["| cell | verdict | why |", "|---|---|---|"] + for name, r in results["cells"].items(): + v = r["verdict"] + verdict_str = "SKIP" if v["skip"] else ("PASS" if v["pass"] else "FAIL") + lines.append(f"| {name} | {verdict_str} | {v['why']} |") + table = "\n".join(lines) + (outdir / "summary.md").write_text(table + "\n") + print("\n" + table) + print(f"\nresults: {outdir}") + + failed = any( + not r["verdict"]["skip"] and not r["verdict"]["pass"] + for r in results["cells"].values() + ) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.agents/skills/agent-release-gate/resources/test_qa_product_concurrency.py b/.agents/skills/agent-release-gate/resources/test_qa_product_concurrency.py index ea5d469f41b..4638008b609 100644 --- a/.agents/skills/agent-release-gate/resources/test_qa_product_concurrency.py +++ b/.agents/skills/agent-release-gate/resources/test_qa_product_concurrency.py @@ -1,12 +1,12 @@ # /// script # requires-python = ">=3.10" -# dependencies = ["httpx>=0.27"] +# dependencies = ["httpx>=0.27", "pytest>=8"] # /// """Offline tests for the `burst` and `crosstalk` journeys. No deployment, no network. Run either way: - uv run test_qa_product_concurrency.py # standalone, prints a line per case + uv run test_qa_product_concurrency.py # standalone, runs through pytest uv run --no-sync pytest test_qa_product_concurrency.py Every case fakes the wire. `invoke` is replaced with a function that builds a `Turn` by hand, so @@ -17,12 +17,15 @@ import gzip import importlib +import json import os import sys import threading import time from pathlib import Path +import pytest + HERE = Path(__file__).resolve().parent CELL = {"harness": "pi_core", "sandbox": "daytona", "model": "m", "provider": "openai"} @@ -389,9 +392,148 @@ def test_a_runner_path_change_makes_the_journeys_mandatory(): assert triggers.mandatory_journeys(["web/oss/src/app/page.tsx"]) == {} +def _session_control_result(status="PASS"): + import session_control + + return { + "cells": { + name: { + "verdict": { + "pass": status == "PASS", + "skip": status == "SKIP", + "why": status.lower(), + } + } + for name in session_control.CELLS + } + } + + +def test_session_control_result_consumer_accepts_a_complete_pass(tmp_path): + path = tmp_path / "results.json" + path.write_text(json.dumps(_session_control_result())) + + result = qa._load_session_control_result(str(path)) + + assert result["status"] == "PASS" + assert result["failed"] == [] + assert result["skipped"] == [] + + +def test_session_control_result_consumer_carries_a_failure(tmp_path): + payload = _session_control_result() + payload["cells"]["stop-warm"]["verdict"] = { + "pass": False, + "skip": False, + "why": "regression", + } + path = tmp_path / "results.json" + path.write_text(json.dumps(payload)) + + result = qa._load_session_control_result(str(path)) + + assert result["status"] == "FAIL" + assert result["failed"] == ["stop-warm"] + + +def test_session_control_result_consumer_marks_skips_incomplete(tmp_path): + path = tmp_path / "results.json" + path.write_text(json.dumps(_session_control_result("SKIP"))) + + result = qa._load_session_control_result(str(path)) + + assert result["status"] == "INCOMPLETE" + assert result["skipped"] + label = qa._session_control_result_label(result) + assert "SKIPPED, UNTESTED" in label + assert result["skipped"][0] in label + + +def test_session_control_result_consumer_rejects_an_incomplete_run(tmp_path): + payload = _session_control_result() + del payload["cells"]["stop-warm"] + path = tmp_path / "results.json" + path.write_text(json.dumps(payload)) + + with pytest.raises(SystemExit, match="missing cells: stop-warm"): + qa._load_session_control_result(str(path)) + + +def test_driver_requires_mandatory_session_control_results(monkeypatch): + monkeypatch.setattr( + sys, + "argv", + [ + "qa_product.py", + "--cell", + "C3", + "--only", + "chat", + "--changed-path", + "api/oss/src/core/sessions/service.py", + ], + ) + + with pytest.raises(SystemExit, match="session_control.py mandatory"): + qa.main() + + +def test_driver_fails_for_a_failed_session_control_result(monkeypatch, tmp_path): + payload = _session_control_result() + payload["cells"]["stop-warm"]["verdict"] = { + "pass": False, + "skip": False, + "why": "regression", + } + result_path = tmp_path / "session-control-results.json" + result_path.write_text(json.dumps(payload)) + monkeypatch.setattr(qa, "RUNS", tmp_path / "runs") + monkeypatch.setitem(qa.JOURNEYS, "chat", lambda _cell: {"pass": True, "why": "ok"}) + monkeypatch.setattr( + sys, + "argv", + [ + "qa_product.py", + "--cell", + "C3", + "--only", + "chat", + "--changed-path", + "api/oss/src/core/sessions/service.py", + "--session-control-results", + str(result_path), + ], + ) + + assert qa.main() == 1 + + +def test_driver_fails_for_a_skipped_session_control_result(monkeypatch, tmp_path): + result_path = tmp_path / "session-control-results.json" + result_path.write_text(json.dumps(_session_control_result("SKIP"))) + monkeypatch.setattr(qa, "RUNS", tmp_path / "runs") + monkeypatch.setitem(qa.JOURNEYS, "chat", lambda _cell: {"pass": True, "why": "ok"}) + monkeypatch.setattr( + sys, + "argv", + [ + "qa_product.py", + "--cell", + "C3", + "--only", + "chat", + "--changed-path", + "api/oss/src/core/sessions/service.py", + "--session-control-results", + str(result_path), + ], + ) + + assert qa.main() == 1 + + def test_the_driver_forces_a_mandatory_journey_past_only(tmp_path=None): """End to end through main(), with every journey stubbed out.""" - import json import tempfile _reset() @@ -408,6 +550,8 @@ def test_the_driver_forces_a_mandatory_journey_past_only(tmp_path=None): } ) outdir = tempfile.mkdtemp() + session_control_results = Path(outdir) / "session-control-results.json" + session_control_results.write_text(json.dumps(_session_control_result())) argv = sys.argv runs_dir = qa.RUNS sys.argv = [ @@ -418,6 +562,8 @@ def test_the_driver_forces_a_mandatory_journey_past_only(tmp_path=None): "chat", "--changed-path", "services/runner/src/engines/sandbox_agent/daytona-secrets.ts", + "--session-control-results", + str(session_control_results), ] qa.RUNS = Path(outdir) try: @@ -849,12 +995,7 @@ def never_ends(session, messages, params, timeout=300.0, deadline=None): def main() -> int: - cases = [v for k, v in sorted(globals().items()) if k.startswith("test_")] - for case in cases: - case() - print(f"PASS {case.__name__}") - print(f"\n{len(cases)} offline cases passed") - return 0 + return pytest.main([__file__, "-q"]) if __name__ == "__main__": diff --git a/.agents/skills/agent-release-gate/resources/test_session_control.py b/.agents/skills/agent-release-gate/resources/test_session_control.py new file mode 100644 index 00000000000..148a15af09c --- /dev/null +++ b/.agents/skills/agent-release-gate/resources/test_session_control.py @@ -0,0 +1,1477 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["httpx>=0.27"] +# /// +"""Unit tests for the pure parts of session_control.py: cell selection, resume, and result shape. + +No stack, no network, no Docker — these exercise only the argument parsing, the OperatorHooks +skip path, and the verdict-shape helpers. +""" + +from __future__ import annotations + +import json +import pathlib +import re +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +import session_control as sc # noqa: E402 + + +def test_cells_registry_is_internally_consistent(): + for name, (needs_hooks, permission, fn) in sc.CELLS.items(): + assert isinstance(needs_hooks, bool), name + assert permission in ("allow", "ask"), name + assert callable(fn), name + + +def test_null_hooks_raises_on_every_method(): + hooks = sc.NullHooks() + assert hooks.available is False + for method in ( + "stream_row", + "record_rows", + "command_rows", + "execution_rows", + "sandbox_procs", + ): + try: + getattr(hooks, method)("x") + except sc.HooksUnavailable: + continue + raise AssertionError(f"{method} should raise HooksUnavailable") + for method in ( + "wait_for_runner", + "runner_healthy", + "ensure_runner_healthy", + "restart_runner", + "kill_runner", + "pause_runner", + "unpause_runner", + "stop_postgres", + "start_postgres", + "kill_sandbox", + "kill_sandbox_for_session", + "wait_for_sandbox_ready", + ): + try: + getattr(hooks, method)() + except sc.HooksUnavailable: + continue + raise AssertionError(f"{method} should raise HooksUnavailable") + + +def test_verdict_shape_helpers(): + p = sc._pass("ok") + f = sc._fail("bad") + s = sc._skip("no hooks") + assert p == {"pass": True, "skip": False, "why": "ok"} + assert f == {"pass": False, "skip": False, "why": "bad"} + assert s == {"pass": False, "skip": True, "why": "no hooks"} + for v in (p, f, s): + assert set(v) == {"pass", "skip", "why"} + + +def _stop_approval_evidence(*, durable_stop: str, late_status: int) -> dict: + return { + "stop": {"status": 200}, + "command_settled": {"settled": True, "why": None}, + "durable_stop": durable_stop, + "late_answer": {"status": late_status}, + "resume_recalled_marker": True, + } + + +def test_stop_approval_durable_path_requires_late_answer_refusal(): + refused = _stop_approval_evidence(durable_stop="on", late_status=409) + + for status in (200, 202, 500): + unexpected = _stop_approval_evidence(durable_stop="on", late_status=status) + verdict = sc._judge_stop_approval(unexpected, pending_found=True) + assert verdict["pass"] is False + assert f"HTTP {status}, expected 409" in verdict["why"] + verdict = sc._judge_stop_approval(refused, pending_found=True) + assert verdict["pass"] is True + assert "late answer was refused" in verdict["why"] + + +def test_stop_approval_legacy_path_requires_and_records_late_answer_acceptance(): + accepted = _stop_approval_evidence(durable_stop="off", late_status=200) + refused = _stop_approval_evidence(durable_stop="off", late_status=409) + + verdict = sc._judge_stop_approval(accepted, pending_found=True) + assert verdict == { + "pass": True, + "skip": False, + "why": "late answer accepted: legacy path", + } + assert accepted["late_answer"]["note"] == "late answer accepted: legacy path" + assert sc._judge_stop_approval(refused, pending_found=True)["pass"] is False + + +def test_durable_stop_auto_detection_uses_cancel_response_shape(): + durable = {"command": {"id": "cmd-1"}, "execution": {"id": "exec-1"}} + legacy = { + "mode": "cancelled", + "session_id": "session-1", + "turn_id": "turn-1", + "watcher_id": None, + "detached": False, + "cancelled_turn_ids": ["turn-1"], + } + + assert sc._resolve_durable_stop("auto", durable) == "on" + assert sc._resolve_durable_stop("auto", legacy) == "off" + assert sc._resolve_durable_stop("auto", {"detail": "not found"}) is None + assert sc._resolve_durable_stop("off", durable) == "off" + assert sc._resolve_durable_stop("on", legacy) == "on" + + +def test_hooks_only_cells_skip_without_project(monkeypatch): + """Every cell marked needs_hooks=True must SKIP (not crash, not run) when --project is + absent, per qa-audit-2026-09-03.md section 4 change 2.""" + + class Args: + sleep_seconds = 1 + sweep_wait = 1 + project = None + sandbox = "local" + + hooks = sc.NullHooks() + for name, (needs_hooks, _permission, fn) in sc.CELLS.items(): + if not needs_hooks: + continue + evidence, verdict = fn({}, {}, Args(), hooks) + assert verdict["skip"] is True, f"{name} should skip without --project" + assert evidence == {}, ( + f"{name} should not run any evidence-gathering without --project" + ) + + +def test_resume_skips_cells_already_in_prior_results(tmp_path): + """Cells present in a prior run's results.json are loaded, not re-executed. This is the + resumability property qa-audit-2026-09-03.md section 4 change 4 asks for: a lost agent + costs one cell, not the whole run.""" + prior_results = { + "cells": { + "stop-warm": { + "evidence": {"session_id": "abc"}, + "verdict": {"pass": True, "skip": False, "why": "ok"}, + "elapsed_s": 1.0, + } + } + } + prior_path = tmp_path / "results.json" + prior_path.write_text(json.dumps(prior_results)) + + loaded = json.loads(prior_path.read_text()).get("cells", {}) + assert "stop-warm" in loaded + assert loaded["stop-warm"]["verdict"]["pass"] is True + + # The cell-selection logic in main(): a cell present in `prior` is carried forward as-is + # rather than re-run. Exercise the same branch condition main() uses. + wanted = ["stop-warm", "double-send"] + to_run = [c for c in wanted if c not in loaded] + assert to_run == ["double-send"] + + +def test_cell_names_are_stable_and_known(): + expected = { + "stop-warm", + "double-send", + "stale-stop", + "stop-approval", + "sandbox-gone", + "records-outage", + "stop-after-finish", + "restart-after-stop", + "runner-gone", + "runner-gone-late", + "post-stop-row", + "codex-child", + "stale-tail", + "repeat-stop", + "concurrent-stops", + "stop-during-completion", + } + assert set(sc.CELLS) == expected + + +def _runner_gone_evidence(**overrides) -> dict: + """A minimal evidence dict shaped the way cell_runner_gone / cell_runner_gone_late build + it, with sane defaults that satisfy `_judge_runner_gone` on their own. Tests override just + the field(s) under test.""" + base = { + "terminal_records": [ + { + "type": "error", + "attributes": {"code": "execution_lost", "settled_by": "watchdog"}, + }, + {"type": "done", "attributes": {"settled_by": "watchdog"}}, + ], + "stop_command": {"state": "applied", "outcome": "stopped"}, + "stream_row": {"flags": {"is_running": False}}, + "new_message_ran": True, + } + base.update(overrides) + return base + + +def test_judge_runner_gone_accepts_the_never_reported_race(): + """The hard race: the command settles `lost` because the runner never got to claim or + report it. This must PASS and record which race landed.""" + evidence = _runner_gone_evidence( + stop_command={"state": "applied", "outcome": "lost"} + ) + verdict = sc._judge_runner_gone(evidence) + assert verdict["pass"] is True + assert verdict["skip"] is False + assert evidence["race"] == "never-reported" + + +def test_judge_runner_gone_accepts_the_outcome_reported_then_died_race(): + """The soft race: the runner reports the Stop's outcome before it actually dies. This must + ALSO pass — both races satisfy the same invariant — and record the other race label.""" + evidence = _runner_gone_evidence( + stop_command={"state": "obsolete", "outcome": "stopped"} + ) + verdict = sc._judge_runner_gone(evidence) + assert verdict["pass"] is True + assert verdict["skip"] is False + assert evidence["race"] == "outcome-reported-then-died" + + +def test_judge_runner_gone_fails_without_any_terminal_record(): + evidence = _runner_gone_evidence(terminal_records=[]) + verdict = sc._judge_runner_gone(evidence) + assert verdict["pass"] is False + assert "race" not in evidence + + +def test_judge_runner_gone_fails_without_a_stop_command_row(): + evidence = _runner_gone_evidence(stop_command=None) + verdict = sc._judge_runner_gone(evidence) + assert verdict["pass"] is False + assert "race" not in evidence + + +def test_judge_runner_gone_fails_on_an_unexpected_command_state(): + evidence = _runner_gone_evidence( + stop_command={"state": "claimed", "outcome": "stopped"} + ) + verdict = sc._judge_runner_gone(evidence) + assert verdict["pass"] is False + assert "race" not in evidence + + +def test_judge_runner_gone_fails_while_the_command_is_still_pending_or_claimed(): + for outcome in (None, "", "pending", "claimed"): + evidence = _runner_gone_evidence( + stop_command={"state": "applied", "outcome": outcome} + ) + verdict = sc._judge_runner_gone(evidence) + assert verdict["pass"] is False, outcome + assert "race" not in evidence, outcome + + +def test_judge_runner_gone_fails_when_is_running_still_reads_true(): + evidence = _runner_gone_evidence(stream_row={"flags": {"is_running": True}}) + verdict = sc._judge_runner_gone(evidence) + assert verdict["pass"] is False + assert "race" not in evidence + + +def test_judge_runner_gone_fails_when_the_next_send_did_not_run(): + evidence = _runner_gone_evidence(new_message_ran=False) + verdict = sc._judge_runner_gone(evidence) + assert verdict["pass"] is False + assert "race" not in evidence + + +def _restart_after_stop_evidence(**overrides) -> dict: + base = { + "runner_healthy_after_s": 5.0, + "admitted_at_s": 12.0, + "recalled_marker": True, + "same_sandbox": True, + "loaded_true": False, + } + base.update(overrides) + return base + + +def test_judge_restart_after_stop_accepts_the_same_sandbox_signal(): + """Recall plus an unchanged sandbox id is a real native resume: no rebuild happened.""" + evidence = _restart_after_stop_evidence(same_sandbox=True, loaded_true=False) + verdict = sc._judge_restart_after_stop(evidence) + assert verdict["pass"] is True + + +def test_judge_restart_after_stop_accepts_the_loaded_true_signal(): + """Recall plus a genuine native hydrate in the runner log is also a real resume, even when + the sandbox itself had to be rebuilt (a new sandbox that loads the OLD native session).""" + evidence = _restart_after_stop_evidence(same_sandbox=False, loaded_true=True) + verdict = sc._judge_restart_after_stop(evidence) + assert verdict["pass"] is True + + +def test_judge_restart_after_stop_fails_when_recall_is_the_only_signal(): + """The exact false-pass this cell exists to catch: the codeword comes back, but neither the + sandbox id nor the runner log backs up a genuine native resume — the runner recovered it by + reconstructing the conversation from persisted records, not by resuming the native session.""" + evidence = _restart_after_stop_evidence(same_sandbox=False, loaded_true=False) + verdict = sc._judge_restart_after_stop(evidence) + assert verdict["pass"] is False + assert ( + verdict["why"] == "native session not resumed, recovered by transcript replay" + ) + + +def test_judge_restart_after_stop_fails_without_recall_even_with_both_signals(): + evidence = _restart_after_stop_evidence( + recalled_marker=False, same_sandbox=True, loaded_true=True + ) + verdict = sc._judge_restart_after_stop(evidence) + assert verdict["pass"] is False + assert "codeword was not recalled" in verdict["why"] + + +def test_judge_restart_after_stop_fails_when_the_runner_never_reported_healthy(): + evidence = _restart_after_stop_evidence(runner_healthy_after_s=None) + verdict = sc._judge_restart_after_stop(evidence) + assert verdict["pass"] is False + assert "never reported healthy" in verdict["why"] + + +def test_judge_restart_after_stop_fails_when_the_continuation_was_never_admitted(): + evidence = _restart_after_stop_evidence(admitted_at_s=None) + verdict = sc._judge_restart_after_stop(evidence) + assert verdict["pass"] is False + assert "refused for the whole wait window" in verdict["why"] + + +def test_match_stop_command_prefers_the_row_targeting_the_turn(): + commands = [ + {"id": "old", "target_turn_id": "turn-a", "state": "applied"}, + {"id": "new", "target_turn_id": "turn-b", "state": "obsolete"}, + ] + assert sc._match_stop_command(commands, "turn-b")["id"] == "new" + + +def test_match_stop_command_falls_back_to_the_last_row_when_nothing_matches(): + commands = [ + {"id": "a", "target_turn_id": None}, + {"id": "b", "target_turn_id": None}, + ] + assert sc._match_stop_command(commands, "turn-x")["id"] == "b" + assert sc._match_stop_command(commands, None)["id"] == "b" + + +def test_match_stop_command_returns_none_for_no_commands(): + assert sc._match_stop_command([], "turn-a") is None + + +class _StubSettlementHooks(sc.OperatorHooks): + """A hook stub whose command_rows/execution_rows are scripted per call, so + assert_command_settled can be tested without Docker or Postgres.""" + + available = True + + def __init__(self, command_sequence, execution_sequence): + # Each is a list of return values, one per poll iteration; the last value repeats once + # exhausted, so a test can describe "stays this way forever" with one entry. + self._commands = command_sequence + self._executions = execution_sequence + self._i = 0 + + def command_rows(self, session_id: str) -> list[dict]: + i = min(self._i, len(self._commands) - 1) + return self._commands[i] + + def execution_rows(self, session_id: str) -> list[dict]: + i = min(self._i, len(self._executions) - 1) + result = self._executions[i] + self._i += ( + 1 # advance once per poll iteration (execution_rows is always called) + ) + return result + + +def test_assert_command_settled_is_a_noop_without_hooks(): + """A cell running without --project (NullHooks) must not be blocked by a check it has no + way to make — the cell's own hooks.available guard already SKIPs it where needed.""" + result = sc.assert_command_settled( + sc.NullHooks(), "session-1", "turn-1", timeout=5.0 + ) + assert result == { + "settled": True, + "command": None, + "execution_rows": [], + "natural_finish": False, + "note": None, + "why": None, + } + + +def test_assert_command_settled_passes_immediately_when_already_settled(): + hooks = _StubSettlementHooks( + command_sequence=[ + [{"id": "cmd-1", "target_turn_id": "turn-1", "state": "applied"}] + ], + execution_sequence=[ + [{"execution_id": "exec-1", "terminal_outcome": "stopped"}] + ], + ) + result = sc.assert_command_settled(hooks, "session-1", "turn-1", timeout=5.0) + assert result["settled"] is True + assert result["why"] is None + assert result["command"]["state"] == "applied" + assert result["execution_rows"][0]["terminal_outcome"] == "stopped" + + +def test_assert_command_settled_catches_the_repeat_stop_false_pass(): + """The exact case this function exists to catch (2026-09-04): a command stuck `claimed` + forever with zero session_executions rows, even though every OTHER driver assertion (one + terminal trace record, a warm resume) would still pass. Must FAIL, fast (timeout=0 -> no + retry sleep), with a reason naming the stuck state.""" + hooks = _StubSettlementHooks( + command_sequence=[ + [ + { + "id": "cmd-1", + "target_turn_id": "turn-1", + "state": "claimed", + "outcome": None, + } + ] + ], + execution_sequence=[[]], + ) + result = sc.assert_command_settled(hooks, "session-1", "turn-1", timeout=0) + assert result["settled"] is False + assert "claimed" in result["why"] + + +def test_assert_command_settled_fails_when_no_command_row_exists(): + hooks = _StubSettlementHooks(command_sequence=[[]], execution_sequence=[[]]) + result = sc.assert_command_settled(hooks, "session-1", "turn-1", timeout=0) + assert result["settled"] is False + assert "no session_commands row" in result["why"] + + +def test_assert_command_settled_fails_on_more_than_one_execution_row(): + hooks = _StubSettlementHooks( + command_sequence=[ + [{"id": "cmd-1", "target_turn_id": "turn-1", "state": "applied"}] + ], + execution_sequence=[ + [ + {"execution_id": "exec-1", "terminal_outcome": "stopped"}, + {"execution_id": "exec-2", "terminal_outcome": "stopped"}, + ] + ], + ) + result = sc.assert_command_settled(hooks, "session-1", "turn-1", timeout=0) + assert result["settled"] is False + assert "exactly one session_executions row" in result["why"] + + +def test_assert_command_settled_accepts_a_stop_after_a_natural_finish(): + """A valid Stop that lands after the turn already finished settles obsolete/not_running with + NO execution row. Zero rows is correct there — accept it, flag it, and note it.""" + hooks = _StubSettlementHooks( + command_sequence=[ + [ + { + "id": "cmd-1", + "target_turn_id": "turn-1", + "state": "obsolete", + "outcome": "not_running", + } + ] + ], + execution_sequence=[[]], # zero execution rows, and that is correct here + ) + result = sc.assert_command_settled(hooks, "session-1", "turn-1", timeout=5.0) + assert result["settled"] is True + assert result["natural_finish"] is True + assert result["note"] == "stop landed after a natural finish" + assert result["execution_rows"] == [] + assert result["why"] is None + + +def test_assert_command_settled_still_requires_a_row_for_a_real_stop(): + """The strict one-row requirement is kept when the runner actually stopped the turn: an + obsolete/stopped command with zero execution rows must still FAIL.""" + hooks = _StubSettlementHooks( + command_sequence=[ + [ + { + "id": "cmd-1", + "target_turn_id": "turn-1", + "state": "obsolete", + "outcome": "stopped", + } + ] + ], + execution_sequence=[[]], + ) + result = sc.assert_command_settled(hooks, "session-1", "turn-1", timeout=0) + assert result["settled"] is False + assert result["natural_finish"] is False + assert "exactly one session_executions row" in result["why"] + + +def test_run_cell_finally_path_with_null_hooks_does_not_crash(): + """run_cell()'s runner-health recovery is gated on `needs_hooks and hooks.available`. With + NullHooks (no --project), hooks.available is False, so the finally block must skip the + recovery call rather than let HooksUnavailable escape through it — even for a needs_hooks + cell whose own function raises before it can restore anything itself.""" + + class Args: + sleep_seconds = 1 + sweep_wait = 1 + project = None + sandbox = "local" + + def boom(cfg, references, args, hooks): + raise RuntimeError("cell blew up before it could restore anything") + + hooks = sc.NullHooks() + result = sc.run_cell("boom-cell", boom, {}, {}, Args(), hooks, True) + assert result["verdict"]["pass"] is False + assert result["verdict"]["skip"] is False + assert "driver exception" in result["verdict"]["why"] + assert "RuntimeError" in result["evidence"]["driver_error"] + assert "elapsed_s" in result + + +def test_run_cell_recovers_the_runner_when_a_cell_raises(): + """The run-level guarantee: a needs_hooks cell that raises must still trigger the runner + recovery check, so a paused or restarted runner does not strand the cell that runs next.""" + + class Args: + sleep_seconds = 1 + sweep_wait = 1 + project = "fake-project" + sandbox = "local" + + class StubHooks(sc.OperatorHooks): + available = True + + def __init__(self): + self.recovered = False + + def ensure_runner_healthy(self, *, timeout: float = 120.0) -> dict: + self.recovered = True + return { + "was_paused": True, + "status_before": "running", + "healthy_after_s": 1.0, + } + + def boom(cfg, references, args, hooks): + raise RuntimeError("cell paused the runner and blew up before unpausing it") + + hooks = StubHooks() + result = sc.run_cell("boom-cell", boom, {}, {}, Args(), hooks, True) + assert hooks.recovered is True + assert result["verdict"]["pass"] is False + + +def test_run_cell_skips_recovery_for_cells_that_do_not_need_hooks(): + """A cell that never touches Docker (needs_hooks=False) must not trigger a recovery check, + even when hooks happen to be available.""" + + class Args: + sleep_seconds = 1 + sweep_wait = 1 + project = "fake-project" + sandbox = "local" + + class StubHooks(sc.OperatorHooks): + available = True + + def __init__(self): + self.recovered = False + + def ensure_runner_healthy(self, *, timeout: float = 120.0) -> dict: + self.recovered = True + return { + "was_paused": False, + "status_before": "running", + "healthy_after_s": 1.0, + } + + def ok(cfg, references, args, hooks): + return {"session_id": "abc"}, sc._pass("fine") + + hooks = StubHooks() + result = sc.run_cell("http-only-cell", ok, {}, {}, Args(), hooks, False) + assert hooks.recovered is False + assert result["verdict"]["pass"] is True + + +def test_resolve_env_names_every_missing_variable(monkeypatch): + monkeypatch.delenv("AGENTA_BASE", raising=False) + monkeypatch.delenv("AGENTA_ADMIN_KEY", raising=False) + monkeypatch.delenv("QA_OPENAI_API_KEY", raising=False) + try: + sc.resolve_env() + except SystemExit as exc: + msg = str(exc) + assert "AGENTA_BASE" in msg + assert "AGENTA_ADMIN_KEY" in msg + assert "QA_OPENAI_API_KEY" in msg + assert "no env-file fallback" in msg + else: + raise AssertionError("resolve_env() should raise SystemExit when env is empty") + + +def test_resolve_env_populates_globals(monkeypatch): + monkeypatch.setenv("AGENTA_BASE", "https://example.test") + monkeypatch.setenv("AGENTA_ADMIN_KEY", "admin-secret") + monkeypatch.setenv("QA_OPENAI_API_KEY", "sk-test") + sc.resolve_env() + assert sc.BASE == "https://example.test" + assert sc.ADMIN_KEY == "admin-secret" + assert sc.OPENAI_KEY == "sk-test" + + +def test_stream_timeout_s_gives_pi_the_shorter_budget(): + assert sc.stream_timeout_s({"harness": {"kind": "pi_core"}}) == 600.0 + + +def test_stream_timeout_s_gives_codex_and_claude_1_5x_pi(): + assert sc.stream_timeout_s({"harness": {"kind": "codex"}}) == 900.0 + assert sc.stream_timeout_s({"harness": {"kind": "claude"}}) == 900.0 + + +def test_stream_timeout_s_defaults_for_an_unknown_or_missing_harness(): + assert sc.stream_timeout_s({"harness": {"kind": "some-future-harness"}}) == 600.0 + assert sc.stream_timeout_s({}) == 600.0 + + +def test_client_shape_messages_full_is_a_noop(): + """--client-shape full (the default) must not touch the outbound messages at all.""" + assert sc.CLIENT_SHAPE == "full" + messages = [ + sc.user_msg("first"), + {"role": "assistant", "parts": [{"type": "text", "text": "ok"}]}, + sc.user_msg("last"), + ] + assert sc._client_shape_messages(messages) == messages + + +def test_client_shape_messages_last_message_produces_exactly_one_message_for_a_user_turn(): + """The literal contract: under last-message, the outbound messages a fresh user turn + produces has exactly one entry, and it is the new user message — not a copy or a rebuild + of it.""" + sc.CLIENT_SHAPE = "last-message" + try: + first = sc.user_msg("first") + reply = {"role": "assistant", "parts": [{"type": "text", "text": "ok"}]} + last = sc.user_msg("last") + shaped = sc._client_shape_messages([first, reply, last]) + finally: + sc.CLIENT_SHAPE = "full" + assert len(shaped) == 1 + assert shaped[0] is last + + +def test_client_shape_messages_keeps_full_history_for_a_hitl_resume(): + """A resume whose trailing turn carries a settled HITL answer (an assistant message, not a + fresh user turn) must NOT be truncated: the answer has to stay bound to its tool call, the + same guard agentRequest.ts applies (`lastMessage?.role === "user"`).""" + sc.CLIENT_SHAPE = "last-message" + try: + first = sc.user_msg("first") + settled = { + "role": "assistant", + "parts": [{"type": "tool-shell", "state": "output-denied"}], + } + shaped = sc._client_shape_messages([first, settled]) + finally: + sc.CLIENT_SHAPE = "full" + assert shaped == [first, settled] + + +def test_client_shape_messages_strips_answerless_assistant_turns_first(): + """An assistant turn with no answer part (no text, no tool, no dynamic-tool, no file) is + stripped before the trailing-user-turn check, mirroring `hasAnswer` in agentRequest.ts.""" + sc.CLIENT_SHAPE = "last-message" + try: + first = sc.user_msg("first") + empty_assistant = {"role": "assistant", "parts": []} + last = sc.user_msg("last") + shaped = sc._client_shape_messages([first, empty_assistant, last]) + finally: + sc.CLIENT_SHAPE = "full" + assert len(shaped) == 1 + assert shaped[0] is last + + +class _FakeResponse: + """Minimal stand-in for an `httpx.Response` the DaytonaAwareHooks code path reads.""" + + def __init__(self, status_code: int, payload=None, text: str = ""): + self.status_code = status_code + self._payload = payload + self.text = text + + def json(self): + return self._payload + + +def _set_daytona_env(): + """Dummy, non-secret env values so `DaytonaAwareHooks.__init__` does not raise. Never a real + key — these tests must never touch the network.""" + import os + + saved = { + k: os.environ.get(k) + for k in ("AGENTA_RUNNER_DAYTONA_API_KEY", "AGENTA_RUNNER_DAYTONA_API_URL") + } + os.environ["AGENTA_RUNNER_DAYTONA_API_KEY"] = "test-key-not-real" + os.environ["AGENTA_RUNNER_DAYTONA_API_URL"] = "https://daytona.example/api" + return saved + + +def _restore_env(saved: dict): + import os + + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +def test_daytona_aware_hooks_requires_daytona_env_vars(): + """Constructing without AGENTA_RUNNER_DAYTONA_API_KEY/URL must fail loudly and by name, the + same discipline `resolve_env` uses for the three top-level env vars — never a silent no-op + that later fails deep inside an HTTP call.""" + import os + + saved = { + k: os.environ.get(k) + for k in ("AGENTA_RUNNER_DAYTONA_API_KEY", "AGENTA_RUNNER_DAYTONA_API_URL") + } + os.environ.pop("AGENTA_RUNNER_DAYTONA_API_KEY", None) + os.environ.pop("AGENTA_RUNNER_DAYTONA_API_URL", None) + try: + try: + sc.DaytonaAwareHooks("fake-project") + except SystemExit as exc: + assert "AGENTA_RUNNER_DAYTONA_API_KEY" in str(exc) + assert "AGENTA_RUNNER_DAYTONA_API_URL" in str(exc) + else: + raise AssertionError("expected SystemExit without the Daytona env vars") + finally: + _restore_env(saved) + + +def test_daytona_aware_hooks_kill_sandbox_noop_without_sandbox_id(): + """No observed sandbox id means nothing to end — must not call the Daytona API at all.""" + saved = _set_daytona_env() + try: + hooks = sc.DaytonaAwareHooks("fake-project") + + def boom(*a, **k): + raise AssertionError("must not call the network without a sandbox id") + + hooks._daytona_delete = boom + assert hooks.kill_sandbox(sandbox_id=None) == [] + finally: + _restore_env(saved) + + +def test_daytona_aware_hooks_kill_sandbox_deletes_only_the_observed_sandbox(): + """Ends the ONE sandbox id the cell observed, by its bare uuid (the `daytona/` prefix is a + driver-internal convention, not part of the Daytona API path).""" + saved = _set_daytona_env() + try: + hooks = sc.DaytonaAwareHooks("fake-project") + calls = [] + + def fake_delete(path): + calls.append(path) + return _FakeResponse(200) + + hooks._daytona_delete = fake_delete + result = hooks.kill_sandbox(sandbox_id="daytona/abc-123") + assert calls == ["/sandbox/abc-123"] + assert result == ["abc-123"] + finally: + _restore_env(saved) + + +def test_daytona_aware_hooks_kill_sandbox_treats_404_as_already_gone(): + saved = _set_daytona_env() + try: + hooks = sc.DaytonaAwareHooks("fake-project") + hooks._daytona_delete = lambda path: _FakeResponse(404) + assert hooks.kill_sandbox(sandbox_id="daytona/abc-123") == ["abc-123"] + finally: + _restore_env(saved) + + +def test_daytona_aware_hooks_sandbox_procs_noop_without_sandbox_id(): + saved = _set_daytona_env() + try: + hooks = sc.DaytonaAwareHooks("fake-project") + + def boom(*a, **k): + raise AssertionError("must not call the network without a sandbox id") + + hooks._daytona_get = boom + assert hooks.sandbox_procs("marker", sandbox_id=None) == [] + finally: + _restore_env(saved) + + +def test_daytona_aware_hooks_sandbox_procs_matches_the_marker_and_filters_self(): + """The full happy path: fetch the toolbox proxy URL for the ONE observed sandbox, run the + same `ps -eo pid=,ppid=,etimes=,args=` reap-exec.ts uses, and keep only the row matching the + driver's own marker — never the `ps` invocation itself or an unrelated process.""" + saved = _set_daytona_env() + try: + hooks = sc.DaytonaAwareHooks("fake-project") + get_calls = [] + post_calls = [] + + hooks._daytona_get = lambda path: ( + get_calls.append(path), + _FakeResponse(200, {"url": "https://proxy.example/tb/abc-123"}), + )[1] + + ps_output = ( + " 501 1 120 /sbin/init\n" + " 777 501 30 sleep 300.123456\n" + " 778 777 0 ps -eo pid=,ppid=,etimes=,args=\n" + ) + + class _FakePost: + def __call__(self, url, json=None, timeout=None): + post_calls.append((url, json)) + return _FakeResponse(200, {"result": ps_output, "exitCode": 0}) + + import httpx as real_httpx + + saved_post = real_httpx.post + real_httpx.post = _FakePost() + try: + hits = hooks.sandbox_procs("sleep 300.123456", sandbox_id="daytona/abc-123") + finally: + real_httpx.post = saved_post + + assert get_calls == ["/sandbox/abc-123/toolbox-proxy-url"] + assert len(post_calls) == 1 + url, body = post_calls[0] + assert url == "https://proxy.example/tb/abc-123/process/execute" + assert body["command"] == "ps -eo pid=,ppid=,etimes=,args=" + assert len(hits) == 1 + assert hits[0]["pid"] == "777" + assert "sleep 300.123456" in hits[0]["args"] + finally: + _restore_env(saved) + + +def test_select_hooks_returns_null_hooks_without_project(): + hooks = sc.select_hooks(None, "local") + assert isinstance(hooks, sc.NullHooks) + hooks = sc.select_hooks(None, "daytona") + assert isinstance(hooks, sc.NullHooks) + + +def test_select_hooks_returns_docker_compose_hooks_for_local_sandbox(): + hooks = sc.select_hooks("fake-project", "local") + assert type(hooks) is sc.DockerComposeHooks # noqa: E721 -- exact class, not the daytona subclass + + +def test_select_hooks_returns_daytona_aware_hooks_for_daytona_sandbox(): + saved = _set_daytona_env() + try: + hooks = sc.select_hooks("fake-project", "daytona") + assert isinstance(hooks, sc.DaytonaAwareHooks) + finally: + _restore_env(saved) + + +# --------------------------------------------------------------------------- # +# sandbox-gone: per-session sandbox targeting (the driver defect this PR fixes). +# --------------------------------------------------------------------------- # + + +def test_parse_local_sandbox_port_reads_the_port_from_a_local_ledger_id(): + assert sc._parse_local_sandbox_port("local/127.0.0.1:44831") == 44831 + assert sc._parse_local_sandbox_port("local/0.0.0.0:5") == 5 + + +def test_parse_local_sandbox_port_ignores_daytona_and_empty_ids(): + assert sc._parse_local_sandbox_port("daytona/abc-123") is None + assert sc._parse_local_sandbox_port(None) is None + assert sc._parse_local_sandbox_port("") is None + + +def test_parse_ss_listener_pid_matches_the_exact_port(): + out = ( + 'LISTEN 0 511 127.0.0.1:44831 0.0.0.0:* users:(("node",pid=2170,fd=23))\n' + 'LISTEN 0 511 127.0.0.1:34013 0.0.0.0:* users:(("node",pid=1999,fd=23))\n' + ) + assert sc._parse_ss_listener_pid(out, 44831) == "2170" + assert sc._parse_ss_listener_pid(out, 34013) == "1999" + + +def test_parse_ss_listener_pid_does_not_match_a_substring_port(): + out = 'LISTEN 0 511 127.0.0.1:44831 0.0.0.0:* users:(("node",pid=2170,fd=23))\n' + assert sc._parse_ss_listener_pid(out, 4483) is None + assert sc._parse_ss_listener_pid(out, 831) is None + + +def test_parse_ss_listener_pid_returns_none_when_absent(): + assert sc._parse_ss_listener_pid("", 44831) is None + + +class _FakeLocalHooks(sc.DockerComposeHooks): + """DockerComposeHooks with the container round-trips (`dc`, `runner_log`) scripted, so the + port-to-pid mapping and the wrong-target refusals are tested without Docker.""" + + def __init__( + self, *, log_lines=None, log_reads=None, ss="", proc_pid="", cmdlines=None + ): + super().__init__("fake-project") + self._log_lines = log_lines or [] + # `log_reads` scripts one return value per `runner_log` call (the last repeats), so a test + # can make this session's line appear on, say, the third poll. `log_lines` is the fixed + # fallback when no script is given. + self._log_reads = log_reads + self._ss = ss + self._proc_pid = proc_pid + self._cmdlines = cmdlines or {} + self.killed: list[str] = [] + self.log_read_count = 0 + + def runner_log(self, since: float) -> list[str]: + self.log_read_count += 1 + if self._log_reads is not None: + i = min(self.log_read_count - 1, len(self._log_reads) - 1) + return list(self._log_reads[i]) + return list(self._log_lines) + + def dc(self, *args: str, timeout: float = 60.0) -> str: + joined = " ".join(str(a) for a in args) + if "ss -ltnHp" in joined: + return self._ss + if "socket:[" in joined: # the /proc/net/tcp fallback resolver script + return self._proc_pid + if "/cmdline" in joined: + m = re.search(r"/proc/(\d+)/cmdline", joined) + return self._cmdlines.get(m.group(1) if m else "", "") + if "kill -9" in joined: + self.killed.append(joined) + return "" + raise AssertionError(f"unexpected dc call: {args}") + + +def test_local_kill_targets_only_the_tested_sessions_sandbox(): + """The tested session's port maps to its own pid; the other session's parked daemon on a + different port is never touched — the exact defect that produced the false negative.""" + sid = "sess-under-test" + hooks = _FakeLocalHooks( + log_lines=[ + f"12:29 [sandbox-agent] [timing] stage=prepare_workspace ms=0 " + f"sandbox=local/127.0.0.1:44831 session={sid}", + "12:28 [sandbox-agent] [timing] stage=prepare_workspace ms=0 " + "sandbox=local/127.0.0.1:34013 session=other-session", + ], + ss=( + 'LISTEN 0 511 127.0.0.1:44831 0.0.0.0:* users:(("node",pid=2170,fd=23))\n' + 'LISTEN 0 511 127.0.0.1:34013 0.0.0.0:* users:(("node",pid=1999,fd=23))\n' + ), + cmdlines={ + "2170": "node /app/node_modules/.bin/sandbox-agent server --port 44831", + }, + ) + result = hooks.kill_sandbox_for_session( + sid, sandbox_id="local/127.0.0.1:44831", since=0.0 + ) + assert result["port"] == 44831 + assert result["pid"] == "2170" + assert result["killed"] == ["2170"] + assert any("2170" in k for k in hooks.killed) + assert not any("1999" in k for k in hooks.killed) + + +def test_local_kill_refuses_when_the_port_cannot_be_mapped(): + hooks = _FakeLocalHooks(log_lines=[]) + try: + hooks.kill_sandbox_for_session("sess", sandbox_id=None, since=0.0) + except sc.WrongSandboxTarget: + assert hooks.killed == [] + return + raise AssertionError("expected WrongSandboxTarget") + + +def test_local_kill_refuses_when_nothing_listens_on_the_port(): + hooks = _FakeLocalHooks(ss="", proc_pid="") + try: + hooks.kill_sandbox_for_session( + "sess", sandbox_id="local/127.0.0.1:44831", since=0.0 + ) + except sc.WrongSandboxTarget: + assert hooks.killed == [] + return + raise AssertionError("expected WrongSandboxTarget") + + +def test_local_kill_refuses_when_the_pid_is_not_a_sandbox_agent(): + hooks = _FakeLocalHooks( + ss='LISTEN 0 511 127.0.0.1:44831 0.0.0.0:* users:(("postgres",pid=42,fd=7))\n', + cmdlines={"42": "postgres: primary process"}, + ) + try: + hooks.kill_sandbox_for_session( + "sess", sandbox_id="local/127.0.0.1:44831", since=0.0 + ) + except sc.WrongSandboxTarget: + assert hooks.killed == [] + return + raise AssertionError("expected WrongSandboxTarget") + + +def test_local_kill_refuses_when_log_and_ledger_ports_disagree(): + sid = "sess" + hooks = _FakeLocalHooks( + log_lines=[ + f"12:29 [sandbox-agent] [timing] stage=prepare_workspace ms=0 " + f"sandbox=local/127.0.0.1:34013 session={sid}", + ], + ) + try: + hooks.kill_sandbox_for_session( + sid, sandbox_id="local/127.0.0.1:44831", since=0.0 + ) + except sc.WrongSandboxTarget: + assert hooks.killed == [] + return + raise AssertionError("expected WrongSandboxTarget") + + +def test_local_kill_falls_back_to_proc_when_ss_is_absent(): + """A distroless runner has no `ss`; the /proc resolver supplies the pid instead.""" + sid = "sess" + hooks = _FakeLocalHooks( + log_lines=[ + f"12:29 [sandbox-agent] [timing] stage=prepare_workspace ms=0 " + f"sandbox=local/127.0.0.1:44831 session={sid}", + ], + ss="", + proc_pid="2170\n", + cmdlines={"2170": "node .../sandbox-agent server"}, + ) + result = hooks.kill_sandbox_for_session( + sid, sandbox_id="local/127.0.0.1:44831", since=0.0 + ) + assert result["pid"] == "2170" + assert result["killed"] == ["2170"] + + +def test_daytona_kill_for_session_delegates_to_the_remote_delete(): + saved = _set_daytona_env() + try: + hooks = sc.DaytonaAwareHooks("fake-project") + hooks._daytona_delete = lambda path: _FakeResponse(200) + result = hooks.kill_sandbox_for_session( + "sess", sandbox_id="daytona/abc-123", since=0.0 + ) + assert result["killed"] == ["abc-123"] + assert result["port"] is None + finally: + _restore_env(saved) + + +def test_daytona_kill_for_session_refuses_without_a_sandbox_id(): + saved = _set_daytona_env() + try: + hooks = sc.DaytonaAwareHooks("fake-project") + + def boom(*a, **k): + raise AssertionError("must not call the network without a sandbox id") + + hooks._daytona_delete = boom + try: + hooks.kill_sandbox_for_session("sess", sandbox_id=None, since=0.0) + except sc.WrongSandboxTarget: + return + raise AssertionError("expected WrongSandboxTarget") + finally: + _restore_env(saved) + + +class _FakeClock: + """A clock whose `sleep` advances `time` instantly, so poll loops run without real waiting.""" + + def __init__(self): + self.now = 0.0 + self.sleeps: list[float] = [] + + def time(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + +def test_wait_for_local_sandbox_port_returns_when_the_line_appears_on_the_third_read(): + sid = "sess" + line = ( + "12:29 [sandbox-agent] [timing] stage=prepare_workspace ms=0 " + f"sandbox=local/127.0.0.1:44831 session={sid}" + ) + hooks = _FakeLocalHooks(log_reads=[[], [], [line]]) # empty, empty, then the line + clock = _FakeClock() + port = hooks.wait_for_local_sandbox_port( + sid, + ledger_id_getter=lambda: None, + since=0.0, + timeout=120.0, + poll_interval=3.0, + clock=clock, + ) + assert port == 44831 + assert hooks.log_read_count == 3 + assert clock.sleeps == [3.0, 3.0] # slept twice before the third read found it + + +def test_wait_for_local_sandbox_port_refuses_when_the_line_never_appears(): + hooks = _FakeLocalHooks(log_reads=[[]]) # every read is empty + clock = _FakeClock() + try: + hooks.wait_for_local_sandbox_port( + "sess", + ledger_id_getter=lambda: None, + since=0.0, + timeout=9.0, + poll_interval=3.0, + clock=clock, + ) + except sc.WrongSandboxTarget as exc: + assert "never appeared" in str(exc) + assert clock.now >= 9.0 + return + raise AssertionError("expected WrongSandboxTarget on timeout") + + +def test_wait_for_local_sandbox_port_resolves_from_the_ledger_when_the_log_is_silent(): + calls = {"n": 0} + + def ledger(): + calls["n"] += 1 + return "local/127.0.0.1:44831" if calls["n"] >= 3 else None + + hooks = _FakeLocalHooks( + log_reads=[[]] + ) # log stays empty; the ledger supplies the port + clock = _FakeClock() + port = hooks.wait_for_local_sandbox_port( + "sess", + ledger_id_getter=ledger, + since=0.0, + timeout=120.0, + poll_interval=3.0, + clock=clock, + ) + assert port == 44831 + assert calls["n"] == 3 + + +def test_sandbox_gone_command_outlasts_acquire_resolve_and_the_design_window(): + assert ( + sc.SANDBOX_GONE_COMMAND_S + > sc.SANDBOX_GONE_ACQUIRE_BUDGET_S + + sc.SANDBOX_GONE_RESOLVE_TIMEOUT_S + + sc._SANDBOX_GONE_DESIGN_WINDOW_S + ) + + +def test_recover_then_send_waits_for_health_before_sending(): + """The recovery Send is not issued until the health poll returns healthy: fail twice, then + succeed, and the send must fire exactly once and only after the third (healthy) check.""" + order: list = [] + calls = {"n": 0} + + def health(): + calls["n"] += 1 + order.append(("health", calls["n"])) + return calls["n"] >= 3 # unhealthy on the first two polls, healthy on the third + + def send(): + order.append(("send", None)) + return {"ok": True} + + clock = _FakeClock() + healthy, result = sc._recover_then_send( + health, send, timeout=60.0, poll_interval=2.0, clock=clock + ) + assert healthy is True + assert result == {"ok": True} + assert calls["n"] == 3 + assert clock.sleeps == [2.0, 2.0] # slept between the two failed polls only + assert order == [ + ("health", 1), + ("health", 2), + ("health", 3), + ("send", None), + ] + + +def test_recover_then_send_does_not_send_when_health_never_recovers(): + sent = {"n": 0} + + def send(): + sent["n"] += 1 + return {"ok": True} + + clock = _FakeClock() + healthy, result = sc._recover_then_send( + lambda: False, send, timeout=6.0, poll_interval=2.0, clock=clock + ) + assert healthy is False + assert result is None + assert sent["n"] == 0 # a doomed Send is never attempted + assert clock.now >= 6.0 + + +class _RunnerGoneStubHooks(sc.OperatorHooks): + """Records the order of pause/read/unpause and DB reads, so the runner-gone measurement can be + tested without Docker or Postgres. `settle_on_call` makes the command settle on the Nth poll.""" + + available = True + + def __init__(self, *, command, executions, stream, settle_on_call=1): + self.calls: list[str] = [] + self._command = command + self._executions = executions + self._stream = stream + self._settle_on_call = settle_on_call + self._cmd_calls = 0 + + def pause_runner(self) -> None: + self.calls.append("pause") + + def unpause_runner(self) -> None: + self.calls.append("unpause") + + def command_rows(self, session_id: str) -> list[dict]: + self._cmd_calls += 1 + self.calls.append("command_rows") + return self._command if self._cmd_calls >= self._settle_on_call else [] + + def execution_rows(self, session_id: str) -> list[dict]: + self.calls.append("execution_rows") + return self._executions + + def stream_row(self, session_id: str) -> dict: + self.calls.append("stream_row") + return self._stream + + +def test_runner_gone_measurement_reads_is_running_while_paused(): + """The is_running read must happen while the pause is still in effect: pause before the read, + unpause after it. Reading after the unpause would catch the returning runner's new turn.""" + hooks = _RunnerGoneStubHooks( + command=[{"state": "applied", "outcome": "lost", "target_turn_id": "t1"}], + executions=[{"terminal_outcome": "execution_lost"}], + stream={"flags": {"is_running": False}}, + ) + stop_calls = [] + terminal = [ + { + "type": "error", + "attributes": {"code": "execution_lost", "settled_by": "watchdog"}, + } + ] + clock = _FakeClock() + measured = sc._measure_runner_gone_while_paused( + hooks, + "sess", + "t1", + do_stop=lambda: (stop_calls.append("stop"), {"status": 202})[1], + read_terminal=lambda: terminal, + sweep_wait=60.0, + poll_interval=5.0, + clock=clock, + ) + assert "pause" in hooks.calls and "unpause" in hooks.calls + assert ( + hooks.calls.index("pause") + < hooks.calls.index("stream_row") + < hooks.calls.index("unpause") + ) + assert measured["stream_row"] == {"flags": {"is_running": False}} + assert measured["stream_row"]["flags"]["is_running"] is False + assert measured["settled_at"] is not None + assert measured["paused_read_at"] is not None + assert stop_calls == ["stop"] + + +def test_runner_gone_measurement_unpauses_even_when_it_never_settles(): + """No settlement within the window still unpauses (never strand the runner) and still takes the + is_running read while paused, so the cell can report the timeout honestly.""" + hooks = _RunnerGoneStubHooks( + command=[], + executions=[], + stream={"flags": {"is_running": True}}, + settle_on_call=999, + ) + clock = _FakeClock() + measured = sc._measure_runner_gone_while_paused( + hooks, + "sess", + "t1", + do_stop=lambda: {"status": 202}, + read_terminal=lambda: [], + sweep_wait=6.0, + poll_interval=3.0, + clock=clock, + ) + assert measured["settled_at"] is None + assert "unpause" in hooks.calls + assert hooks.calls.index("stream_row") < hooks.calls.index("unpause") + + +def test_terminal_records_arriving_one_second_after_settle_pass_strict_check(): + clock = _FakeClock() + hooks = _RunnerGoneStubHooks( + command=[{"state": "applied", "outcome": "lost", "target_turn_id": "t1"}], + executions=[{"terminal_outcome": "execution_lost"}], + stream={"flags": {"is_running": False}}, + ) + + def read_terminal(): + if clock.time() < 1.0: + return [] + return [ + { + "type": "error", + "attributes": { + "code": "execution_lost", + "settled_by": "watchdog", + }, + }, + {"type": "done", "attributes": {"settled_by": "watchdog"}}, + ] + + measured = sc._measure_runner_gone_while_paused( + hooks, + "sess", + "t1", + do_stop=lambda: {"status": 202}, + read_terminal=read_terminal, + sweep_wait=60.0, + poll_interval=5.0, + clock=clock, + ) + + assert clock.time() == 1.0 + assert sc._require_watchdog_execution_lost(measured["terminal"]) is None + + +def test_terminal_records_missing_for_budget_fail_with_old_message(): + clock = _FakeClock() + hooks = _RunnerGoneStubHooks( + command=[{"state": "applied", "outcome": "lost", "target_turn_id": "t1"}], + executions=[{"terminal_outcome": "execution_lost"}], + stream={"flags": {"is_running": False}}, + ) + measured = sc._measure_runner_gone_while_paused( + hooks, + "sess", + "t1", + do_stop=lambda: {"status": 202}, + read_terminal=lambda: [], + sweep_wait=60.0, + poll_interval=5.0, + clock=clock, + ) + verdict = sc._require_watchdog_execution_lost(measured["terminal"]) + + assert clock.time() == 20.0 + assert verdict == { + "pass": False, + "skip": False, + "why": "no watchdog execution_lost ending was found among the terminal records", + } + + +def test_sandbox_gone_settle_budget_derives_from_probe_defaults(): + saved = sc.SANDBOX_STARTUP_SLACK_S + sc.SANDBOX_STARTUP_SLACK_S = 0.0 + try: + expected = ( + sc.SANDBOX_LIVENESS_PROBE_INTERVAL_S * sc.SANDBOX_LIVENESS_PROBE_FAILURES + + sc.SANDBOX_GONE_SETTLE_SLACK_S + ) + assert sc.sandbox_gone_settle_budget_s() == expected + # Never shorter than the slow command's own duration would leave it, per the wait rule. + assert sc.SANDBOX_GONE_COMMAND_S > 0 + finally: + sc.SANDBOX_STARTUP_SLACK_S = saved + + +if __name__ == "__main__": + import inspect + + failures = 0 + tests = [ + (name, obj) + for name, obj in sorted(globals().items()) + if name.startswith("test_") and callable(obj) + ] + for name, fn in tests: + params = inspect.signature(fn).parameters + try: + if "monkeypatch" in params or "tmp_path" in params: + # Minimal standalone monkeypatch/tmp_path so this file runs without pytest too. + import os + import tempfile + + class _MonkeyPatch: + def __init__(self): + self._saved = {} + + def setenv(self, k, v): + self._saved.setdefault(k, os.environ.get(k)) + os.environ[k] = v + + def delenv(self, k, raising=False): + self._saved.setdefault(k, os.environ.get(k)) + os.environ.pop(k, None) + + def restore(self): + for k, v in self._saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + kwargs = {} + mp = _MonkeyPatch() + if "monkeypatch" in params: + kwargs["monkeypatch"] = mp + if "tmp_path" in params: + kwargs["tmp_path"] = pathlib.Path(tempfile.mkdtemp()) + fn(**kwargs) + mp.restore() + else: + fn() + print(f"PASS {name}") + except Exception as exc: # noqa: BLE001 + failures += 1 + print(f"FAIL {name}: {exc}") + print(f"\n{len(tests) - failures}/{len(tests)} passed") + sys.exit(1 if failures else 0) diff --git a/.github/workflows/44-railway-tests.yml b/.github/workflows/44-railway-tests.yml index 205c64b34bc..e277eacce8d 100644 --- a/.github/workflows/44-railway-tests.yml +++ b/.github/workflows/44-railway-tests.yml @@ -645,8 +645,8 @@ jobs: AGENTA_TEST_OSS_OWNER_PASSWORD: ${{ secrets.AGENTA_TEST_OSS_OWNER_PASSWORD }} AGENTA_TEST_LLM_PROVIDER: mock AGENTA_TEST_EPHEMERAL_PROJECT: "true" - AGENTA_MOBILE_GATE: ${{ inputs.mobile_gate_enabled }} - AGENTA_MOBILE_REVERSE_GATE: ${{ inputs.mobile_reverse_gate_enabled }} + AGENTA_MOBILE_GATE: ${{ inputs.mobile_gate_enabled && 'true' || 'false' }} + AGENTA_MOBILE_REVERSE_GATE: ${{ inputs.mobile_reverse_gate_enabled && 'true' || 'false' }} TESTMAIL_API_KEY: ${{ secrets.TESTMAIL_API_KEY }} TESTMAIL_NAMESPACE: ${{ secrets.TESTMAIL_NAMESPACE }} steps: diff --git a/api/ee/src/dbs/postgres/sessions/records/dao.py b/api/ee/src/dbs/postgres/sessions/records/dao.py index 03d196a8e63..a0b74c1ee7a 100644 --- a/api/ee/src/dbs/postgres/sessions/records/dao.py +++ b/api/ee/src/dbs/postgres/sessions/records/dao.py @@ -99,10 +99,12 @@ async def delete_records_before_cutoff( type_=ARRAY(PG_UUID(as_uuid=True)), ) + # The key is (project_id, record_id). `RecordDBE.id` does not exist, so the + # earlier version of this statement raised before it deleted anything. expired = ( select( RecordDBE.project_id.label("project_id"), - RecordDBE.id.label("id"), + RecordDBE.record_id.label("record_id"), ) .where( RecordDBE.project_id == any_(project_ids_param), @@ -116,8 +118,8 @@ async def delete_records_before_cutoff( deleted = ( delete(RecordDBE) .where( - tuple_(RecordDBE.project_id, RecordDBE.id).in_( - select(expired.c.project_id, expired.c.id) + tuple_(RecordDBE.project_id, RecordDBE.record_id).in_( + select(expired.c.project_id, expired.c.record_id) ) ) .returning(literal(1).label("deleted")) diff --git a/api/entrypoints/routers.py b/api/entrypoints/routers.py index b01db2d512d..3683c9a9978 100644 --- a/api/entrypoints/routers.py +++ b/api/entrypoints/routers.py @@ -182,6 +182,11 @@ from oss.src.dbs.postgres.sessions.streams.dbes import SessionStreamDBE # noqa: F401 from oss.src.dbs.postgres.sessions.streams.dao import SessionStreamsDAO from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.dbs.postgres.sessions.commands.dbes import SessionCommandDBE # noqa: F401 +from oss.src.dbs.postgres.sessions.commands.dao import SessionCommandsDAO +from oss.src.dbs.postgres.sessions.executions.dao import SessionExecutionsDAO +from oss.src.core.sessions.commands.service import SessionCommandsService +from oss.src.dbs.http.sessions.control_delivery_direct import DirectControlDelivery from oss.src.tasks.asyncio.sessions.orphan_sweep import orphan_sweep_loop from oss.src.dbs.redis.shared.engine import get_lock_engine @@ -279,8 +284,16 @@ async def lifespan(*args, **kwargs): except Exception as e: # noqa: BLE001 log.warning("Store bucket ensure failed at startup: %s", e) + # The execution watchdog. It needs the records plane to write the terminal outcome a + # dead runner owed, and the watch publisher so an open browser sees the turn close. _orphan_sweep_task = asyncio.create_task( - orphan_sweep_loop(_transactions_engine, _lock_engine) + orphan_sweep_loop( + _transactions_engine, + _lock_engine, + records_service=records_service, + watch_publisher=_sessions_watch_publisher, + commands_service=session_commands_service, + ) ) _attachment_sweep_task = asyncio.create_task( @@ -587,6 +600,8 @@ async def lifespan(*args, **kwargs): folders_dao = FoldersDAO(engine=_transactions_engine) session_streams_dao = SessionStreamsDAO(engine=_transactions_engine) session_turns_dao = SessionTurnsDAO(engine=_transactions_engine) +session_commands_dao = SessionCommandsDAO(engine=_transactions_engine) +session_executions_dao = SessionExecutionsDAO(engine=_transactions_engine) connections_dao = ConnectionsDAO(engine=_transactions_engine) mounts_dao = MountsDAO(engine=_transactions_engine) @@ -621,6 +636,7 @@ async def lifespan(*args, **kwargs): records_service = RecordsService( records_dao=records_dao, + executions_dao=session_executions_dao, ) @@ -838,6 +854,7 @@ async def lifespan(*args, **kwargs): interactions_service = SessionInteractionsService( interactions_dao=interactions_dao, watch_publisher=_sessions_watch_publisher, + records_service=records_service, ) triggers_service = TriggersService( @@ -1115,6 +1132,26 @@ async def _dispatch_detached_run(*, project_id, user_id, request) -> str: records_service=records_service, ) +# Durable session commands (Stop). The control-delivery adapter is chosen by one setting. +# `direct` posts the command to the runner's own /cancel over the hop that already carries hard +# kill; `long_poll` is not built yet, and naming it fails at boot rather than silently falling +# back to a transport the operator did not choose. +_control_adapter = (env.agenta.sessions.commands.adapter or "direct").strip().lower() +if _control_adapter != "direct": + raise RuntimeError( + f"AGENTA_SESSIONS_CONTROL_ADAPTER={_control_adapter!r} is not available in this build. " + "Only 'direct' is implemented; the long-poll adapter is a later change." + ) + +session_commands_service = SessionCommandsService( + commands_dao=session_commands_dao, + streams_service=session_streams_service, + interactions_service=interactions_service, + lock_engine=_lock_engine, + delivery=DirectControlDelivery(), + executions_dao=session_executions_dao, +) + sessions = SessionsRouter( streams_service=session_streams_service, records_service=records_service, @@ -1125,6 +1162,7 @@ async def _dispatch_detached_run(*, project_id, user_id, request) -> str: mounts_service=mounts_service, turns_service=session_turns_service, sessions_service=sessions_service, + commands_service=session_commands_service, respond_task=_interactions_worker.respond_interaction, interactions_dispatcher=_interactions_dispatcher, ) @@ -1599,6 +1637,12 @@ async def _dispatch_detached_run(*, project_id, user_id, request) -> str: tags=["Sessions"], ) +# After `root`, so the literal /sessions/ routes always win a path match. +app.include_router( + router=sessions.control.router, + tags=["Sessions"], +) + @app.get("/health", operation_id="health_check", tags=["Status"]) async def health_check(): diff --git a/api/entrypoints/worker_streams.py b/api/entrypoints/worker_streams.py index 18ee38bc887..65abd998ed1 100644 --- a/api/entrypoints/worker_streams.py +++ b/api/entrypoints/worker_streams.py @@ -97,6 +97,9 @@ async def _build_records_worker(redis_client: Redis) -> StreamConsumer: interactions_dao=SessionInteractionsDAO(), watch_publisher=watch_publisher, ), + # Redelivery bound for records the Postgres write rejected. + reclaim_min_idle_ms=env.agenta.sessions.records.reclaim_idle_ms, + max_deliveries=env.agenta.sessions.records.max_deliveries, ) diff --git a/api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py new file mode 100644 index 00000000000..d150e2b11ab --- /dev/null +++ b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py @@ -0,0 +1,164 @@ +"""add session commands, and the two session_streams columns a Stop needs + +A user Stop reached the runner only through the absence of a Redis lock, discovered on the next +heartbeat up to 30 seconds later. Nothing recorded that a Stop had been asked for, so a Stop +against an unreachable runner was simply lost and no execution ever reached a terminal outcome +anyone could read. + +`session_commands` is that record. One row per durable request to change an execution. `state` +is where the COMMAND is (pending, claimed, applied, obsolete); `outcome` is what happened to the +EXECUTION (stopped, not_running, superseded_by_newer_turn, failed, lost). The two are separate +columns because they answer different questions and settle at different times. + +Two columns join `session_streams`: + + * `stopping_turn_id` names the execution an accepted Stop is waiting on, written in the same + transaction as the command insert and cleared at settlement. + * `turn_started_at` records when the row's current `turn_id` started. Nothing else could serve + the stale-Stop guard: `updated_at` is the heartbeat timestamp and moves every 30 seconds, + runner-minted turn ids are uuid4 and carry no time, the Redis lock value is a bare turn id + that a Lua compare reads whole, and the `session_turns` append is fire-and-forget so a + running turn may have no row at all. + +Both are nullable and backfill to NULL. A row written before this migration yields no +comparison, and the guard then does not fire — deliberately, because a guard that refused every +Stop it could not verify would break the common case to protect a rare one. + +Revision ID: oss000000022 +Revises: oss000000021 +Create Date: 2026-09-02 23:30:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision: str = "oss000000022" +down_revision: Union[str, None] = "oss000000021" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "session_commands", + sa.Column("id", sa.UUID(as_uuid=True), nullable=False), + sa.Column("project_id", sa.UUID(as_uuid=True), nullable=False), + sa.Column("session_id", sa.String(), nullable=False), + sa.Column("kind", sa.String(), nullable=False), + sa.Column("target_turn_id", sa.String(), nullable=True), + sa.Column("expected_turn_id", sa.String(), nullable=True), + sa.Column("state", sa.String(), nullable=False), + sa.Column("claimed_by", sa.String(), nullable=True), + sa.Column("claim_expires_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column( + "claim_count", + sa.Integer(), + server_default="0", + nullable=False, + ), + sa.Column("outcome", sa.String(), nullable=True), + sa.Column("idempotency_key", sa.String(), nullable=True), + sa.Column("settled_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("data", sa.JSON(), nullable=True), + sa.Column( + "flags", + postgresql.JSONB(none_as_null=True), + nullable=True, + ), + sa.Column( + "tags", + postgresql.JSONB(none_as_null=True), + nullable=True, + ), + sa.Column("meta", sa.JSON(), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.func.current_timestamp(), + nullable=True, + ), + sa.Column("updated_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("deleted_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("created_by_id", sa.UUID(as_uuid=True), nullable=True), + sa.Column("updated_by_id", sa.UUID(as_uuid=True), nullable=True), + sa.Column("deleted_by_id", sa.UUID(as_uuid=True), nullable=True), + sa.CheckConstraint("kind IN ('cancel')", name="ck_session_commands_kind"), + sa.CheckConstraint( + "state IN ('pending', 'claimed', 'applied', 'obsolete')", + name="ck_session_commands_state", + ), + sa.ForeignKeyConstraint( + ["project_id"], + ["projects.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("project_id", "id"), + sa.UniqueConstraint( + "project_id", + "session_id", + "idempotency_key", + name="uq_session_commands_idempotency", + ), + ) + # One open command per target execution, enforced by the database because admission's + # read-then-insert races itself: two Stops in the same instant both find no open command. + op.create_index( + "uq_session_commands_open_target", + "session_commands", + ["project_id", "session_id", "kind", "target_turn_id"], + unique=True, + postgresql_where=sa.text( + "state IN ('pending', 'claimed') AND deleted_at IS NULL" + ), + ) + op.create_index( + "ix_session_commands_open", + "session_commands", + ["project_id", "session_id", "created_at"], + postgresql_where=sa.text( + "state IN ('pending', 'claimed') AND deleted_at IS NULL" + ), + ) + op.create_index( + "ix_session_commands_claims", + "session_commands", + ["claim_expires_at"], + postgresql_where=sa.text("state = 'claimed' AND deleted_at IS NULL"), + ) + op.create_index( + "ix_session_commands_project_session", + "session_commands", + ["project_id", "session_id", "created_at"], + ) + # The runner reports an outcome with the command id alone; it holds no project credential, + # so that read cannot use the primary key's leading column. + op.create_index( + "ix_session_commands_id", + "session_commands", + ["id"], + ) + + op.add_column( + "session_streams", + sa.Column("stopping_turn_id", sa.String(), nullable=True), + ) + op.add_column( + "session_streams", + sa.Column("turn_started_at", sa.TIMESTAMP(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("session_streams", "turn_started_at") + op.drop_column("session_streams", "stopping_turn_id") + op.drop_index("ix_session_commands_id", table_name="session_commands") + op.drop_index("ix_session_commands_project_session", table_name="session_commands") + op.drop_index("ix_session_commands_claims", table_name="session_commands") + op.drop_index("ix_session_commands_open", table_name="session_commands") + op.drop_index("uq_session_commands_open_target", table_name="session_commands") + op.drop_table("session_commands") diff --git a/api/oss/databases/postgres/migrations/core_oss/versions/oss000000023_add_session_executions.py b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000023_add_session_executions.py new file mode 100644 index 00000000000..1cc271cf519 --- /dev/null +++ b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000023_add_session_executions.py @@ -0,0 +1,43 @@ +"""add authoritative session execution terminal outcomes + +Revision ID: oss000000023 +Revises: oss000000022 +Create Date: 2026-09-03 22:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "oss000000023" +down_revision: Union[str, None] = "oss000000022" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "session_executions", + sa.Column("project_id", sa.UUID(as_uuid=True), nullable=False), + sa.Column("session_id", sa.String(), nullable=False), + sa.Column("execution_id", sa.String(), nullable=False), + sa.Column("terminal_outcome", sa.String(), nullable=False), + sa.Column("settled_by", sa.String(), nullable=False), + sa.Column("settled_at", sa.TIMESTAMP(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("project_id", "session_id", "execution_id"), + ) + op.create_index( + "ix_session_executions_project_session", + "session_executions", + ["project_id", "session_id"], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_session_executions_project_session", table_name="session_executions" + ) + op.drop_table("session_executions") diff --git a/api/oss/databases/postgres/migrations/core_oss/versions/oss000000024_add_execution_redis_reconciliation.py b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000024_add_execution_redis_reconciliation.py new file mode 100644 index 00000000000..6d3e7c12b7f --- /dev/null +++ b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000024_add_execution_redis_reconciliation.py @@ -0,0 +1,41 @@ +"""track execution Redis reconciliation + +Revision ID: oss000000024 +Revises: oss000000023 +Create Date: 2026-09-03 22:30:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "oss000000024" +down_revision: Union[str, None] = "oss000000023" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "session_executions", + sa.Column("redis_reconciled_at", sa.TIMESTAMP(timezone=True), nullable=True), + ) + op.create_index( + "ix_session_executions_redis_unreconciled", + "session_executions", + ["settled_at"], + postgresql_where=sa.text( + "settled_by = 'runner' AND terminal_outcome = 'stopped' " + "AND redis_reconciled_at IS NULL" + ), + ) + + +def downgrade() -> None: + op.drop_index( + "ix_session_executions_redis_unreconciled", + table_name="session_executions", + ) + op.drop_column("session_executions", "redis_reconciled_at") diff --git a/api/oss/databases/postgres/migrations/core_oss/versions/oss000000026_add_execution_ending_marker.py b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000026_add_execution_ending_marker.py new file mode 100644 index 00000000000..f36af0bf0a8 --- /dev/null +++ b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000026_add_execution_ending_marker.py @@ -0,0 +1,38 @@ +"""add session execution ending marker + +Revision ID: oss000000026 +Revises: oss000000024 +Create Date: 2026-09-04 12:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "oss000000026" +down_revision: Union[str, None] = "oss000000024" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "session_executions", + sa.Column("ending_written_at", sa.TIMESTAMP(timezone=True), nullable=True), + ) + op.create_index( + "ix_session_executions_ending_unwritten", + "session_executions", + ["settled_at"], + postgresql_where=sa.text("ending_written_at IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index( + "ix_session_executions_ending_unwritten", + table_name="session_executions", + ) + op.drop_column("session_executions", "ending_written_at") diff --git a/api/oss/databases/postgres/migrations/tracing_oss/versions/oss000000005_add_records_quarantined_at.py b/api/oss/databases/postgres/migrations/tracing_oss/versions/oss000000005_add_records_quarantined_at.py new file mode 100644 index 00000000000..1a5654f5a4e --- /dev/null +++ b/api/oss/databases/postgres/migrations/tracing_oss/versions/oss000000005_add_records_quarantined_at.py @@ -0,0 +1,35 @@ +"""add_records_quarantined_at + +Revision ID: oss000000005 +Revises: oss000000004 +Create Date: 2026-09-03 12:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "oss000000005" +down_revision: Union[str, None] = "oss000000004" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # A record that reached ingest for a turn the execution watchdog had already ended. + # Nullable and forward-fill only, like every other column on this table: the tracing DB + # is never backfilled, and no existing row can be classified retroactively anyway. + # + # No index. Every read that filters on it is already scoped to one project and one + # session by an existing index, and the column is null on all but a handful of rows. + op.add_column( + "records", + sa.Column("quarantined_at", sa.TIMESTAMP(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("records", "quarantined_at") diff --git a/api/oss/src/apis/fastapi/sessions/models.py b/api/oss/src/apis/fastapi/sessions/models.py index 01f47695ce5..a7ef9af42fe 100644 --- a/api/oss/src/apis/fastapi/sessions/models.py +++ b/api/oss/src/apis/fastapi/sessions/models.py @@ -346,3 +346,83 @@ class SessionRecordIngestRequest(BaseModel): # Both forward-fill only (tracing-DB rule) — absent on producers that predate this. turn_id: Optional[str] = None span_id: Optional[OTelSpanId] = None + + +# --------------------------------------------------------------------------- +# Session control: durable commands (Stop) +# --------------------------------------------------------------------------- + + +class SessionCancelRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + # Optional stale-request guard. When present, the API cancels only this execution and + # refuses the request if another one is running. When absent, it cancels whichever + # execution is active when the request is applied. A person never types this: the browser + # fills it from the session's own state, and a first-party client always sends it. + expected_execution_id: Optional[str] = Field( + default=None, + description=( + "Optional stale-request guard honored only in cancel mode; ignored for send, " + "steer, and attach." + ), + ) + + +class SessionCommandRef(BaseModel): + """The durable command an accepted request created. Identity and DELIVERY state only. + + A client must not read execution state from it. `state` says where the command is; the + session's own state says what the execution is doing. + """ + + id: UUID + state: Literal["pending", "claimed", "applied", "obsolete"] + + +class SessionExecutionRef(BaseModel): + """What the caller should render. `id` is null when the session was idle.""" + + id: Optional[str] = None + state: Literal["stopping", "idle"] + + +class SessionCancelResponse(BaseModel): + command: SessionCommandRef + execution: SessionExecutionRef + + +class SessionExecutionOutcome(BaseModel): + model_config = ConfigDict(extra="forbid") + + # The execution the runner acted on. Null when it held none. + id: Optional[str] = None + # stopped: cancelled as asked. not_running: no such execution on this runner. + # superseded_by_newer_turn: the held execution started after the command arrived. + # failed: the cancel itself failed. + state: Literal["stopped", "failed", "not_running", "superseded_by_newer_turn"] + # Short and human-readable, present only when `state` is "failed". + error: Optional[str] = Field(default=None, max_length=2000) + + +class SessionControlOutcomeRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + replica_id: str = Field(min_length=1, max_length=128) + # The command's terminal state. `applied` means the runner did the work; `obsolete` means + # there was nothing to do. + result: Literal["applied", "obsolete"] + execution: SessionExecutionOutcome + + +class SessionCommandSettlement(BaseModel): + id: UUID + state: Literal["applied", "obsolete"] + outcome: Literal[ + "stopped", "not_running", "superseded_by_newer_turn", "failed", "lost" + ] + settled_at: Optional[datetime] = None + + +class SessionControlOutcomeResponse(BaseModel): + command: SessionCommandSettlement diff --git a/api/oss/src/apis/fastapi/sessions/router.py b/api/oss/src/apis/fastapi/sessions/router.py index 3bf5eb22760..b587e34fcb4 100644 --- a/api/oss/src/apis/fastapi/sessions/router.py +++ b/api/oss/src/apis/fastapi/sessions/router.py @@ -18,6 +18,7 @@ import re from functools import wraps +from secrets import compare_digest from uuid import UUID from fastapi import ( @@ -50,6 +51,7 @@ # Core domain imports — new paths from oss.src.core.sessions.streams.dtos import ( + CommandMode, SessionHeartbeatRequest, SessionHeartbeatResult, SessionStreamCommandRequest, @@ -62,10 +64,21 @@ ConcurrencyLimitExceeded, SessionIdInvalid, SessionTurnInUse, + SessionTurnMismatch, SessionStreamAlreadyExists, SessionStreamNotFound, ) -from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.core.sessions.streams.service import ( + SessionStreamsService, + derive_command_mode, +) +from oss.src.core.sessions.commands.service import SessionCommandsService +from oss.src.core.sessions.commands.types import ( + ExecutionExpectationFailed, + SessionCommandIdempotencyConflict, + SessionCommandNotClaimable, + SessionCommandNotFound, +) from oss.src.core.sessions.records.service import RecordsService from oss.src.core.sessions.records.dtos import SessionRecordEvent from oss.src.core.sessions.records.streaming import publish_record @@ -118,6 +131,13 @@ from oss.src.core.workflows.service import WorkflowsService from oss.src.apis.fastapi.sessions.models import ( + SessionCancelRequest, + SessionCancelResponse, + SessionCommandRef, + SessionCommandSettlement, + SessionControlOutcomeRequest, + SessionControlOutcomeResponse, + SessionExecutionRef, # streams SessionDetachRequest, SessionStreamQueryRequest, @@ -197,6 +217,15 @@ async def wrapper(*args, **kwargs): "liveness": e.liveness, }, ) from e + except SessionTurnMismatch as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "message": e.message, + "expected_execution_id": e.expected_turn_id, + "actual_execution_id": e.actual_turn_id, + }, + ) from e except ConcurrencyLimitExceeded as e: raise HTTPException( status_code=status.HTTP_429_TOO_MANY_REQUESTS, @@ -371,6 +400,9 @@ async def set_session_stream( request: Request, payload: SessionStreamCommandRequest, ) -> SessionStreamCommandResponse: + # Use Redis time before database waits can reorder cancellation against a new turn. + arrived_at_ms = await self._service.clock_ms() + project_id = request.state.project_id user_id = request.state.user_id @@ -382,14 +414,45 @@ async def set_session_stream( if not has_permission: raise FORBIDDEN_EXCEPTION - await self._service.check_runner_concurrency_limit(project_id=project_id) + mode = derive_command_mode(payload) + + # A cancel starts nothing, so the per-project concurrency limit must not gate it. Before + # this, a project at its limit could not stop the very runs that held the limit — the one + # request that frees capacity was the one refused with 429. + if mode != CommandMode.cancel: + await self._service.check_runner_concurrency_limit(project_id=project_id) - return await self._service.command( + response = await self._service.command( + arrived_at_ms=arrived_at_ms, project_id=project_id, user_id=user_id, request=payload, ) + if mode == CommandMode.cancel: + # Close only the displaced turns' gates; the service publishes their watch events. + try: + for turn_id in response.cancelled_turn_ids: + await self._interactions_service.cancel_session_pending( + project_id=UUID(str(project_id)), + session_id=response.session_id, + only_turn_id=turn_id, + ) + if not response.cancelled_turn_ids: + await self._interactions_service.cancel_session_pending( + project_id=UUID(str(project_id)), + session_id=response.session_id, + ) + except Exception: + log.error( + "[SESSIONS] accepted Stop interaction cleanup failed", + exc_info=True, + project_id=str(project_id), + session_id=response.session_id, + ) + + return response + @intercept_exceptions() @_handle_session_exceptions() async def fetch_session_stream( @@ -488,6 +551,9 @@ async def heartbeat_session_stream( if not has_permission: raise FORBIDDEN_EXCEPTION + if payload.release_owner: + _assert_runner_token(request) + heartbeat = await self._service.heartbeat( project_id=project_id, request=payload, @@ -1845,6 +1911,214 @@ async def unarchive_session( ) +# --------------------------------------------------------------------------- +# Session control — durable commands (Stop) +# --------------------------------------------------------------------------- + + +def _handle_command_exceptions(): + """Map the commands plane's domain errors onto status codes. + + A separate decorator from `_handle_session_exceptions` so the two planes' error vocabularies + stay apart: a conflict here means "the execution you named is not the one running", which is + a different thing from the streams plane's "this session is already busy". + """ + + def decorator(func): + @wraps(func) + async def wrapper(*args, **kwargs): + try: + return await func(*args, **kwargs) + except SessionIdInvalid as e: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.message, + ) from e + except ExecutionExpectationFailed as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "message": e.message, + "current_execution_id": e.current, + }, + ) from e + except SessionCommandIdempotencyConflict as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=e.message, + ) from e + except SessionCommandNotFound as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=e.message, + ) from e + except SessionCommandNotClaimable as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"message": e.message, "state": e.state}, + ) from e + + return wrapper + + return decorator + + +class SessionControlRouter: + """The Stop plane: one public route and one internal one. + + `POST /sessions/{session_id}/cancel` is the product's Stop. It is deliberately NOT behind + the runner concurrency limit: refusing to STOP work because a project is at its run limit + would be the exact wrong answer to a busy project. + + `POST /sessions/control/commands/{command_id}/outcome` is how the runner reports what + happened. It authenticates with the shared runner token rather than a project credential, + because the runner holds no project credential of its own for a command it was handed. The + command id resolves the project, so a caller still cannot reach across tenants: it can only + settle a command whose id it already knows and that it currently holds the claim on. + """ + + def __init__( + self, + *, + commands_service: SessionCommandsService, + ) -> None: + self._service = commands_service + self.router = APIRouter() + + self.router.add_api_route( + "/sessions/{session_id}/cancel", + self.cancel_session_execution, + methods=["POST"], + operation_id="cancel_session_execution", + tags=["Sessions"], + ) + self.router.add_api_route( + "/sessions/control/commands/{command_id}/outcome", + self.report_command_outcome, + methods=["POST"], + operation_id="report_session_command_outcome", + tags=["Sessions"], + include_in_schema=False, + ) + + @intercept_exceptions() + @_handle_command_exceptions() + async def cancel_session_execution( + self, + request: Request, + session_id: str, + payload: Optional[SessionCancelRequest] = None, + ) -> JSONResponse: + project_id = request.state.project_id + user_id = request.state.user_id + + has_permission = await check_action_access( + user_uid=str(user_id), + project_id=str(project_id), + permission=Permission.RUN_SESSIONS, + ) + if not has_permission: + raise FORBIDDEN_EXCEPTION + + if not env.agenta.sessions.durable_stop: + legacy = await self._service.request_cancel_legacy( + project_id=UUID(str(project_id)), + user_id=UUID(str(user_id)), + session_id=session_id, + expected_execution_id=( + payload.expected_execution_id if payload else None + ), + ) + return JSONResponse( + status_code=status.HTTP_200_OK, + content=legacy.model_dump(mode="json"), + ) + + idempotency_key = request.headers.get("Idempotency-Key") + if idempotency_key is not None: + idempotency_key = ( + idempotency_key.strip()[:_MAX_IDEMPOTENCY_KEY_CHARACTERS] or None + ) + + admission = await self._service.request_cancel( + project_id=UUID(str(project_id)), + user_id=UUID(str(user_id)) if user_id else None, + session_id=session_id, + expected_execution_id=payload.expected_execution_id if payload else None, + idempotency_key=idempotency_key, + ) + + body = SessionCancelResponse( + command=SessionCommandRef( + id=admission.command.id, + state=admission.command.state.value, + ), + execution=SessionExecutionRef( + id=admission.execution_id, + state="stopping" if admission.accepted else "idle", + ), + ) + # 202 and not 200 for the accepted case: the work is not done when the response + # returns. The caller learns the outcome from the session's own state. + return JSONResponse( + status_code=( + status.HTTP_202_ACCEPTED if admission.accepted else status.HTTP_200_OK + ), + content=body.model_dump(mode="json"), + ) + + @intercept_exceptions() + @_handle_command_exceptions() + async def report_command_outcome( + self, + request: Request, + command_id: UUID, + payload: SessionControlOutcomeRequest, + ) -> SessionControlOutcomeResponse: + _assert_runner_token(request) + + settled = await self._service.report_outcome( + command_id=command_id, + replica_id=payload.replica_id, + result=payload.result, + execution_id=payload.execution.id, + execution_state=payload.execution.state, + error=payload.execution.error, + ) + return SessionControlOutcomeResponse( + command=SessionCommandSettlement( + id=settled.id, + state=settled.state.value, + outcome=settled.outcome.value if settled.outcome else "failed", + settled_at=settled.settled_at, + ) + ) + + +def _assert_runner_token(request: Request) -> None: + """The runner proves it is the platform runtime with the shared secret both sides hold. + + Constant-time compare, so a wrong token leaks no length or prefix through timing. A missing + configured token fails closed: an unset secret must never mean "let everyone in". + """ + expected = env.runner.token + if not expected: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="runner token is not configured on this deployment", + ) + presented = request.headers.get("X-Agenta-Runner-Token") or "" + if not presented: + authorization = request.headers.get("Authorization") or "" + if authorization.lower().startswith("bearer "): + presented = authorization[7:].strip() + if not compare_digest(presented.encode("utf-8"), expected.encode("utf-8")): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Unauthorized", + ) + + # --------------------------------------------------------------------------- # Top-level composer # --------------------------------------------------------------------------- @@ -1861,6 +2135,11 @@ class SessionsRouter: sessions_router.mounts.router → prefix /sessions sessions_router.turns.router → prefix /sessions/turns sessions_router.root.router → no prefix (paths include /sessions/query, /sessions/, /sessions/archive, /sessions/unarchive) + sessions_router.control.router → no prefix (paths include /sessions/{session_id}/cancel and /sessions/control/…) + + `control` MUST be mounted AFTER `root`. `/sessions/{session_id}/cancel` is a two-segment + path and `/sessions/query` is one, so they cannot actually collide — but mounting the + literal routes first keeps that true for any two-segment literal added later. """ def __init__( @@ -1875,6 +2154,7 @@ def __init__( mounts_service: MountsService, turns_service: SessionTurnsService, sessions_service: SessionsService, + commands_service: SessionCommandsService, respond_task: Optional[Any] = None, interactions_dispatcher: Optional[Any] = None, ) -> None: @@ -1898,3 +2178,4 @@ def __init__( ) self.turns = SessionTurnsRouter(turns_service=turns_service) self.root = SessionsRootRouter(sessions_service=sessions_service) + self.control = SessionControlRouter(commands_service=commands_service) diff --git a/api/oss/src/core/sessions/commands/__init__.py b/api/oss/src/core/sessions/commands/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/oss/src/core/sessions/commands/dtos.py b/api/oss/src/core/sessions/commands/dtos.py new file mode 100644 index 00000000000..27b2f9c2ad2 --- /dev/null +++ b/api/oss/src/core/sessions/commands/dtos.py @@ -0,0 +1,123 @@ +"""Durable session commands — the data shapes. + +A command is one durable request to change an execution. Version one has one kind, `cancel`, +which the product calls Stop. + +Two ideas are kept apart on purpose, and the separation is the point of the whole record: + + * `state` says where the COMMAND is in its delivery (pending, claimed, applied, obsolete). + * `outcome` says what happened to the EXECUTION (stopped, not_running, ...). + +A client that draws a Stop button reads the execution; a client that retries safely reads the +command id. Merging them is what makes today's cancel ambiguous. +""" + +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Optional +from uuid import UUID + +from pydantic import BaseModel + +from oss.src.core.shared.dtos import Identifier, Lifecycle + + +class SessionCommandKind(str, Enum): + cancel = "cancel" + + +class SessionCommandState(str, Enum): + """Where the command is in its delivery. `applied` and `obsolete` are terminal.""" + + pending = "pending" # durable, not yet taken by a runner + claimed = "claimed" # a runner holds a lease on it + applied = "applied" # the runner did the work and reported + obsolete = "obsolete" # there was nothing to do, or nobody could ever do it + + +class SessionCommandOutcome(str, Enum): + """What happened to the targeted execution. Null while the command is open.""" + + stopped = "stopped" # cancelled as asked + not_running = "not_running" # no such execution anywhere + superseded_by_newer_turn = ( + "superseded_by_newer_turn" # a later turn holds the session + ) + failed = "failed" # the cancel itself failed + lost = "lost" # nobody ever reported; the sweep settled it + + +class SessionCommand(Identifier, Lifecycle): + project_id: UUID + session_id: str + kind: SessionCommandKind + + # The execution the API resolved at admission and pinned. Null when nothing ran. + target_turn_id: Optional[str] = None + # The execution the caller asserted was running, stored exactly as sent. Null when none. + expected_turn_id: Optional[str] = None + + # The command's own arguments. Empty for `cancel`; reserved for steer and queue. + data: Optional[Dict[str, Any]] = None + + state: SessionCommandState + claimed_by: Optional[str] = None + claim_expires_at: Optional[datetime] = None + claim_count: int = 0 + + outcome: Optional[SessionCommandOutcome] = None + idempotency_key: Optional[str] = None + settled_at: Optional[datetime] = None + + tags: Optional[Dict[str, Any]] = None + meta: Optional[Dict[str, Any]] = None + + +class SessionCommandCreate(BaseModel): + """One insert. `state`/`outcome`/`settled_at` are carried because admission can insert a + command that is ALREADY settled (nothing was running, or a newer turn took the session), + and that must be one write, not an insert followed by an update.""" + + project_id: UUID + session_id: str + kind: SessionCommandKind = SessionCommandKind.cancel + + target_turn_id: Optional[str] = None + expected_turn_id: Optional[str] = None + data: Optional[Dict[str, Any]] = None + + state: SessionCommandState = SessionCommandState.pending + outcome: Optional[SessionCommandOutcome] = None + settled_at: Optional[datetime] = None + + idempotency_key: Optional[str] = None + + # The instant the service stamped as the request's arrival. It is stored as `created_at` + # rather than left to the server default, so the value the stale-Stop guard COMPARED is the + # value the row CARRIES. A guard that compares one timestamp and stores another is not a + # guard the runner can repeat. + created_at: Optional[datetime] = None + + +class SessionCommandSettle(BaseModel): + """The terminal transition, guarded on the states the caller expects to find. + + A SET and not one state, because the outcome report races the claim that is taken on the + runner's behalf. Admission inserts the command `pending`, hands it to the runner, and only + then writes `claimed`; a runner that aborts fast reports its outcome while the row is still + `pending`. Guarding on `claimed` alone refused that report with a conflict and left the + command open until the sweep called it lost. Both states are legitimate at the moment of the + write, so the compare-and-set covers both. + + `replica_id` guards a settlement that follows a claim: only the replica that holds the + claim may write the outcome. A `pending` row has no claim to violate, so the guard admits a + null `claimed_by` as well. It is None altogether when the API itself settles a command + nobody ever took, which is the `not_held` case and the sweep's `lost` case. + """ + + project_id: UUID + command_id: UUID + state: SessionCommandState + outcome: SessionCommandOutcome + expected_states: List[SessionCommandState] = [SessionCommandState.claimed] + replica_id: Optional[str] = None diff --git a/api/oss/src/core/sessions/commands/interfaces.py b/api/oss/src/core/sessions/commands/interfaces.py new file mode 100644 index 00000000000..9f87cd73106 --- /dev/null +++ b/api/oss/src/core/sessions/commands/interfaces.py @@ -0,0 +1,194 @@ +"""The two ports of the session commands plane. + +`SessionCommandsDAOInterface` is storage. `ControlDeliveryPort` is transport: how the API +reaches whichever runner process holds a session. Durability, authorization, idempotency, the +state machine and terminal settlement live in the service and must not move into an adapter. +""" + +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Any, AsyncContextManager, List, NamedTuple, Optional +from uuid import UUID + +from pydantic import BaseModel + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandKind, + SessionCommandSettle, +) + + +class SessionScope(BaseModel): + """One session a runner holds warm. The routing input of a claim.""" + + project_id: UUID + session_id: str + + +class CommandCreateResult(NamedTuple): + """The stored command and whether this call inserted it.""" + + command: SessionCommand + inserted: bool + + +class DeliveryReceipt(BaseModel): + """What the TRANSPORT learned, never what happened to the execution. + + * `accepted` — a runner took the command and will report through the outcome route. + * `unreachable` — the transport failed. The command is durable, so a later claim or the + settlement sweep recovers it. + * `not_held` — a reachable runner said it does not hold that session, which lets the + service settle at once instead of waiting for the deadline. + """ + + status: str # "accepted" | "unreachable" | "not_held" + detail: Optional[str] = None + # Which runner process took it, when the transport learned that. The service uses it as the + # claim owner, so the outcome route's guard reads the same way on every transport. + replica_id: Optional[str] = None + + +class ControlDeliveryPort(ABC): + """How the API reaches the runner that holds a session. Transport only.""" + + @abstractmethod + async def deliver(self, *, command: SessionCommand) -> DeliveryReceipt: + """Make `command` reachable by whoever holds its session, promptly. + + Best effort: a failure here never fails admission, because the command is already + durable. + """ + + @abstractmethod + async def acknowledge(self, *, command_id: UUID, replica_id: str) -> None: + """Record that a replica took the command, for adapters that keep their own delivery + bookkeeping. A no-op where the claim compare-and-set already IS the acknowledgement.""" + + +class SessionCommandsDAOInterface(ABC): + @abstractmethod + def transaction(self) -> AsyncContextManager[Any]: + """Open a transaction that sibling session DAOs can share.""" + + @abstractmethod + async def create_command( + self, + *, + user_id: Optional[UUID], + command: SessionCommandCreate, + stopping_turn_id: Optional[str] = None, + ) -> SessionCommand: + """Insert one command and, in the SAME transaction, stamp the session row's + `stopping_turn_id`. Idempotent on `(project_id, session_id, idempotency_key)`.""" + + @abstractmethod + async def create_command_with_status( + self, + *, + user_id: Optional[UUID], + command: SessionCommandCreate, + stopping_turn_id: Optional[str] = None, + ) -> CommandCreateResult: + """Create a command and report whether this call inserted it.""" + + @abstractmethod + async def fetch_by_idempotency_key( + self, + *, + project_id: UUID, + session_id: str, + idempotency_key: str, + ) -> Optional[SessionCommand]: + """The command previously created for this session-scoped retry key.""" + + @abstractmethod + async def fetch_open_command( + self, + *, + project_id: UUID, + session_id: str, + kind: SessionCommandKind, + target_turn_id: Optional[str], + ) -> Optional[SessionCommand]: + """The open (`pending` or `claimed`) command for this exact target, if one exists. + This is what collapses two Stops in a row onto one command.""" + + @abstractmethod + async def fetch_command( + self, + *, + command_id: UUID, + project_id: Optional[UUID] = None, + ) -> Optional[SessionCommand]: + """One command by id. `project_id` is optional because the runner reports an outcome + with the command id alone and holds no project credential.""" + + @abstractmethod + async def claim_commands( + self, + *, + sessions: List[SessionScope], + replica_id: str, + lease_seconds: int, + limit: int, + ) -> List[SessionCommand]: + """Take up to `limit` pending commands for these sessions. Compare-and-set, so two API + replicas serving two claims at once never hand out the same command twice.""" + + @abstractmethod + async def claim_for_delivery( + self, + *, + project_id: UUID, + command_id: UUID, + replica_id: str, + lease_seconds: int, + ) -> Optional[SessionCommand]: + """Move ONE command from `pending` to `claimed` for a runner that just accepted it over + a direct call. The long-poll adapter reaches the same transition through + `claim_commands`; both exist so the outcome route's guard reads the same either way.""" + + @abstractmethod + async def record_delivery_attempt( + self, + *, + project_id: UUID, + command_id: UUID, + now: datetime, + max_deliveries: int, + ) -> Optional[SessionCommand]: + """Reserve one bounded delivery attempt and return the updated command.""" + + @abstractmethod + async def settle_command( + self, + *, + settle: SessionCommandSettle, + transaction: Optional[Any] = None, + ) -> Optional[SessionCommand]: + """Terminal transition, guarded on `state='claimed' AND claimed_by=:replica_id`. + None means the claim had expired or somebody else settled it first.""" + + @abstractmethod + async def clear_stopping_turn( + self, + *, + project_id: UUID, + session_id: str, + turn_id: Optional[str] = None, + ) -> None: + """Clear `session_streams.stopping_turn_id`. With `turn_id`, only when it matches, so a + late settlement cannot clear a NEWER Stop's marker.""" + + @abstractmethod + async def expire_claims( + self, + *, + now: datetime, + max_deliveries: int, + pending_before: Optional[datetime] = None, + ) -> List[SessionCommand]: + """Pending or claimed commands old enough for recovery.""" diff --git a/api/oss/src/core/sessions/commands/service.py b/api/oss/src/core/sessions/commands/service.py new file mode 100644 index 00000000000..3382652cdbb --- /dev/null +++ b/api/oss/src/core/sessions/commands/service.py @@ -0,0 +1,797 @@ +"""Durable session commands — admission, delivery and settlement. + +Version one has one command kind, `cancel`, which the product calls Stop. + +WHAT STOP MEANS HERE. Stop ends the WORK, not the session. The sandbox stays warm, the native +harness session stays resumable, and the next message continues the same conversation. That is +why this service never force-deletes the Redis `alive` key: it leaves it to its own time to +live, exactly as the end of an ordinary turn does. Force-deleting `alive` is what makes today's +cancel read as a session teardown. + +THE ORDER OF ADMISSION. + + 1. Stamp the arrival time FIRST, before reading anything. + 2. Resolve the target execution once, from Redis `running`, falling back to `alive`. + 3. Apply the three late-Stop guards (below). + 4. Insert the command and stamp `session_streams.stopping_turn_id` in ONE transaction. + 5. Only then call the runner. Delivery failure never fails the request, because the command + is already durable. + +Redis is not written at admission. The stopping execution keeps `alive` and `running` while it +stops, which is what prevents a second message from starting underneath it. + +THE LATE-STOP GUARDS. A Stop that arrives after its turn ended must not kill the next turn. + + * The caller's `expected_execution_id`, when sent, must name the running execution. It does + not, the request is refused with a conflict and nothing is written. + * When no expectation was sent and the running execution started AFTER this request arrived, + the command is inserted already settled and targets nothing. + * The target is resolved once and pinned. A turn that starts later has a different id, so a + pinned command can never reach it. The runner repeats the comparison against its own memory, + which is exact. +""" + +from datetime import datetime, timedelta, timezone +from typing import Any, List, Optional, Tuple +from uuid import UUID + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandKind, + SessionCommandOutcome, + SessionCommandSettle, + SessionCommandState, +) +from oss.src.core.sessions.commands.interfaces import ( + CommandCreateResult, + ControlDeliveryPort, + SessionCommandsDAOInterface, +) +from oss.src.core.sessions.commands.types import ( + ExecutionExpectationFailed, + SessionCommandIdempotencyConflict, + SessionCommandNotClaimable, + SessionCommandNotFound, +) +from oss.src.core.sessions.executions.interfaces import SessionExecutionsDAOInterface +from oss.src.core.sessions.interactions.service import SessionInteractionsService +from oss.src.core.sessions.streams.dtos import ( + SessionStreamCommandRequest, + SessionStreamCommandResponse, +) +from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.core.sessions.streams.types import SessionIdInvalid, SessionTurnMismatch +from oss.src.dbs.redis.shared.engine import LockEngine +from oss.src.dbs.redis.sessions.contract import ( + HEARTBEAT_INTERVAL_SECONDS, + validate_session_id, +) +from oss.src.dbs.redis.sessions.locks import ( + get_alive_owner, + get_owner, + get_running_owner, + reconcile_stopped_turn, +) +from oss.src.utils.env import env +from oss.src.utils.logging import get_module_logger + +log = get_module_logger(__name__) + + +class CancelAdmission: + """What admission decided, in the shape the route answers with.""" + + def __init__( + self, + *, + command: SessionCommand, + execution_id: Optional[str], + accepted: bool, + ) -> None: + self.command = command + # What the caller should render: the execution being stopped, or nothing. + self.execution_id = execution_id + # True when an execution was running or parked and the command is on its way. The route + # answers 202 for it and 200 otherwise. + self.accepted = accepted + + +class _SettlementRejected(Exception): + pass + + +class SessionCommandsService: + def __init__( + self, + *, + commands_dao: SessionCommandsDAOInterface, + streams_service: SessionStreamsService, + interactions_service: SessionInteractionsService, + lock_engine: LockEngine, + delivery: ControlDeliveryPort, + executions_dao: Optional[SessionExecutionsDAOInterface] = None, + ) -> None: + self._dao = commands_dao + self._streams = streams_service + self._interactions = interactions_service + self._lock = lock_engine + self._delivery = delivery + self._executions = executions_dao + + # -- admission ---------------------------------------------------------- # + + async def request_cancel_legacy( + self, + *, + project_id: UUID, + user_id: UUID, + session_id: str, + expected_execution_id: Optional[str] = None, + ) -> SessionStreamCommandResponse: + """Use the heartbeat-carried Stop path kept for rollout rollback.""" + try: + return await self._streams.command( + project_id=project_id, + user_id=user_id, + request=SessionStreamCommandRequest( + session_id=session_id, + expected_execution_id=expected_execution_id, + ), + ) + except SessionTurnMismatch as error: + raise ExecutionExpectationFailed( + expected=error.expected_turn_id, + current=error.actual_turn_id, + ) from error + + async def request_cancel( + self, + *, + project_id: UUID, + user_id: Optional[UUID], + session_id: str, + expected_execution_id: Optional[str] = None, + idempotency_key: Optional[str] = None, + ) -> CancelAdmission: + if not validate_session_id(session_id): + raise SessionIdInvalid(session_id) + + # FIRST, before any read. The value compared below is the value stored as the row's + # `created_at`, so the runner can repeat the same comparison against its own memory. + received_at = datetime.now(timezone.utc) + + if idempotency_key is not None: + existing = await self._dao.fetch_by_idempotency_key( + project_id=project_id, + session_id=session_id, + idempotency_key=idempotency_key, + ) + if existing is not None: + if existing.expected_turn_id != expected_execution_id: + raise SessionCommandIdempotencyConflict( + idempotency_key=idempotency_key + ) + return self._admission_for_existing(existing) + + target_turn_id, turn_started_at = await self._resolve_target( + project_id=project_id, + session_id=session_id, + expected_turn_id=expected_execution_id, + ) + + if ( + expected_execution_id is not None + and target_turn_id != expected_execution_id + ): + # Compared against the TARGET, which is `running` with a fallback to `alive`, and + # never against `running` alone. An execution parked on an approval has released + # `running` and still holds `alive` under the same turn id, and it is exactly the + # execution the user is looking at when they press Stop on the approval card. The + # browser always sends the id it streamed, so comparing against `running` alone + # refused every named Stop on a parked approval while the same Stop without an + # expectation was accepted — the guard fired on the one case it exists to allow. + # + # Nothing is inserted and nothing is delivered. The caller was looking at a run + # that has already ended, and its next read tells it so. + raise ExecutionExpectationFailed( + expected=expected_execution_id, current=target_turn_id + ) + + if target_turn_id is None: + # No eligible execution is running. Record the intent so a retry with the same key + # gets the same answer, and settle it in the same write. + created = await self._insert( + project_id=project_id, + user_id=user_id, + session_id=session_id, + received_at=received_at, + target_turn_id=None, + expected_turn_id=expected_execution_id, + idempotency_key=idempotency_key, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.not_running, + ) + if not created.inserted: + return self._admission_for_existing(created.command) + command = created.command + return CancelAdmission(command=command, execution_id=None, accepted=False) + + if ( + expected_execution_id is None + and turn_started_at is not None + and turn_started_at > received_at + ): + # The execution now running began AFTER the user pressed Stop, so it is not the one + # they meant. Do not target it, do not touch Redis, and tell the caller there is + # nothing of theirs left to stop. + created = await self._insert( + project_id=project_id, + user_id=user_id, + session_id=session_id, + received_at=received_at, + target_turn_id=None, + expected_turn_id=None, + idempotency_key=idempotency_key, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.superseded_by_newer_turn, + ) + if not created.inserted: + return self._admission_for_existing(created.command) + command = created.command + return CancelAdmission(command=command, execution_id=None, accepted=False) + + # Two Stops in a row are one intent. Collapse onto the open command for the same target + # BEFORE inserting, so this holds even when the caller sends a different idempotency key. + open_command = await self._dao.fetch_open_command( + project_id=project_id, + session_id=session_id, + kind=SessionCommandKind.cancel, + target_turn_id=target_turn_id, + ) + if open_command is not None: + if open_command.state == SessionCommandState.pending: + # Nobody has taken it. The first delivery may have failed, so try again; the + # runner deduplicates by command id, so a duplicate arrival aborts nothing twice. + await self._deliver(open_command) + return CancelAdmission( + command=open_command, + execution_id=target_turn_id, + accepted=True, + ) + + created = await self._insert( + project_id=project_id, + user_id=user_id, + session_id=session_id, + received_at=received_at, + target_turn_id=target_turn_id, + expected_turn_id=expected_execution_id, + idempotency_key=idempotency_key, + state=SessionCommandState.pending, + outcome=None, + stopping_turn_id=target_turn_id, + ) + if not created.inserted: + return self._admission_for_existing(created.command) + command = created.command + # The row is committed. Everything from here is promptness, not correctness. + await self._deliver(command) + return CancelAdmission( + command=command, execution_id=target_turn_id, accepted=True + ) + + async def _resolve_target( + self, + *, + project_id: UUID, + session_id: str, + expected_turn_id: Optional[str], + ) -> Tuple[Optional[str], Optional[datetime]]: + """The execution to stop, and when it started. + + An unfenced Stop targets only `running`. A named Stop may fall back to `alive` so it can + still reach the parked approval the caller observed. + """ + turn_id = await get_running_owner( + self._lock, project_id=str(project_id), session_id=session_id + ) + if turn_id is None and expected_turn_id is not None: + turn_id = await get_alive_owner( + self._lock, project_id=str(project_id), session_id=session_id + ) + if turn_id is None: + return None, None + + stream = await self._streams.fetch_header( + project_id=project_id, session_id=session_id + ) + started_at = None + if stream is not None and stream.turn_id == turn_id: + # Only when the row agrees about WHICH turn is running. A start time read off a row + # that names a different turn would compare two unrelated things. + started_at = stream.turn_started_at + return turn_id, started_at + + async def _insert( + self, + *, + project_id: UUID, + user_id: Optional[UUID], + session_id: str, + received_at: datetime, + target_turn_id: Optional[str], + expected_turn_id: Optional[str], + idempotency_key: Optional[str], + state: SessionCommandState, + outcome: Optional[SessionCommandOutcome], + stopping_turn_id: Optional[str] = None, + ) -> CommandCreateResult: + return await self._dao.create_command_with_status( + user_id=user_id, + command=SessionCommandCreate( + project_id=project_id, + session_id=session_id, + kind=SessionCommandKind.cancel, + target_turn_id=target_turn_id, + expected_turn_id=expected_turn_id, + state=state, + outcome=outcome, + settled_at=received_at if outcome is not None else None, + idempotency_key=idempotency_key, + created_at=received_at, + ), + stopping_turn_id=stopping_turn_id, + ) + + @staticmethod + def _admission_for_existing(command: SessionCommand) -> CancelAdmission: + """Replay the command's original target without delivering it again.""" + return CancelAdmission( + command=command, + execution_id=command.target_turn_id, + accepted=command.target_turn_id is not None, + ) + + # -- delivery ----------------------------------------------------------- # + + async def _deliver(self, command: SessionCommand) -> None: + """Hand the command to the transport, then record what the transport learned. + + Never raises. The user's request has already succeeded by the time this runs. + """ + command = await self._dao.record_delivery_attempt( + project_id=command.project_id, + command_id=command.id, + now=datetime.now(timezone.utc), + max_deliveries=env.agenta.sessions.commands.max_deliveries, + ) + if command is None: + return + + try: + receipt = await self._delivery.deliver(command=command) + except Exception as e: # noqa: BLE001 — transport failure is never a request failure + log.warning( + "control delivery raised for command=%s session=%s: %s", + command.id, + command.session_id, + e, + ) + return + + if receipt.status == "accepted": + # Take the claim on the runner's behalf, so the outcome route's guard reads the same + # way on every transport: only the holder of the claim writes the outcome. + await self._dao.claim_for_delivery( + project_id=command.project_id, + command_id=command.id, + replica_id=receipt.replica_id or "direct", + lease_seconds=env.agenta.sessions.commands.lease_seconds, + ) + return + + if receipt.status == "not_held": + await self._settle_not_held(command) + return + + log.warning( + "control delivery unreachable for command=%s session=%s: %s", + command.id, + command.session_id, + receipt.detail or "no detail", + ) + + async def _settle_not_held(self, command: SessionCommand) -> None: + """A reachable runner said it does not hold this session. Two different things look + alike here, and the user must not be told the wrong one. + + `running` is the discriminator, not the heartbeat. A `not_held` while SOME execution + holds `running` means a process is executing this session and it is not the one we + called. Settle that `lost`, so the user learns the Stop failed, and log it at error + level. + + With no `running` execution anywhere, nothing is executing and the work the user meant + to stop is over. That is the everyday case: the turn ended a moment before the Stop + arrived, the runner had already dropped it, and the answer is `not_running`. Judging it + on the heartbeat instead called every one of those a failed Stop, because a turn that + has just ended leaves `alive` set and a fresh beat behind it, exactly as a running one + does. + """ + outcome = SessionCommandOutcome.not_running + running_owner = await get_running_owner( + self._lock, + project_id=str(command.project_id), + session_id=command.session_id, + ) + if running_owner is not None and await self._session_is_beating( + project_id=command.project_id, session_id=command.session_id + ): + outcome = SessionCommandOutcome.lost + # Name the process that DOES hold the session, so the log says where the Stop + # should have gone rather than only that it did not arrive. + owner = await get_owner( + self._lock, + project_id=str(command.project_id), + session_id=command.session_id, + ) + log.error( + "control delivery: the runner answered not_held for session=%s while " + "execution %s holds `running` and the row is beating. A process is executing " + "that session and it is not the one we called, so this deployment has more " + "than one runner replica and the direct adapter cannot route to it. Settling " + "the command lost, so the user is told the Stop failed rather than that the " + "work had already finished. command=%s target_turn=%s owner_replica=%s", + command.session_id, + running_owner, + command.id, + command.target_turn_id, + owner or "unknown", + ) + await self.settle( + command_id=command.id, + project_id=command.project_id, + replica_id=None, + expected_states=[SessionCommandState.pending], + state=SessionCommandState.obsolete, + outcome=outcome, + execution_id=command.target_turn_id, + ) + + async def _session_is_beating(self, *, project_id: UUID, session_id: str) -> bool: + """Is a runner process keeping this session's row fresh right now?""" + stream = await self._streams.fetch_header( + project_id=project_id, session_id=session_id + ) + if stream is None or stream.updated_at is None: + return False + if not (stream.flags and stream.flags.is_alive): + return False + updated_at = stream.updated_at + if updated_at.tzinfo is None: + updated_at = updated_at.replace(tzinfo=timezone.utc) + age = (datetime.now(timezone.utc) - updated_at).total_seconds() + return age < HEARTBEAT_INTERVAL_SECONDS * 2 + + async def settle_abandoned_commands(self, *, now: datetime) -> int: + max_deliveries = env.agenta.sessions.commands.max_deliveries + abandoned = await self._dao.expire_claims( + now=now, + max_deliveries=max_deliveries, + pending_before=now + - timedelta(seconds=env.agenta.sessions.commands.admission_timeout_seconds), + ) + settled = 0 + for command in abandoned: + beating = await self._session_is_beating( + project_id=command.project_id, + session_id=command.session_id, + ) + if beating and command.claim_count < max_deliveries: + await self._deliver(command) + continue + + result = await self.settle( + command_id=command.id, + project_id=command.project_id, + replica_id=None, + expected_states=[ + SessionCommandState.pending, + SessionCommandState.claimed, + ], + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.lost, + execution_id=command.target_turn_id, + ) + if result is not None: + settled += 1 + return settled + + # -- settlement --------------------------------------------------------- # + + async def settle_execution_lost( + self, + *, + project_id: UUID, + session_id: str, + execution_id: str, + settled_at: datetime, + transaction: Optional[Any] = None, + ) -> bool: + if self._executions is None: + return True + result = await self._executions.settle( + project_id=project_id, + session_id=session_id, + execution_id=execution_id, + terminal_outcome=SessionCommandOutcome.lost.value, + settled_by="watchdog", + settled_at=settled_at, + transaction=transaction, + ) + winner = result.settlement + return result.won or ( + winner.terminal_outcome == SessionCommandOutcome.lost.value + and winner.settled_by == "watchdog" + ) + + async def repair_terminal_redis(self) -> int: + if self._executions is None: + return 0 + misses = await self._executions.list_redis_unreconciled(limit=200) + repaired = 0 + for execution in misses: + await self._reconcile_stopped_redis( + project_id=execution.project_id, + session_id=execution.session_id, + execution_id=execution.execution_id, + ) + repaired += 1 + return repaired + + async def _reconcile_stopped_redis( + self, + *, + project_id: UUID, + session_id: str, + execution_id: str, + ) -> None: + await reconcile_stopped_turn( + self._lock, + project_id=str(project_id), + session_id=session_id, + turn_id=execution_id, + ) + if self._executions is not None: + await self._executions.mark_redis_reconciled( + project_id=project_id, + session_id=session_id, + execution_id=execution_id, + ) + + async def report_outcome( + self, + *, + command_id: UUID, + replica_id: str, + result: str, + execution_id: Optional[str], + execution_state: str, + error: Optional[str] = None, + ) -> SessionCommand: + """The runner reporting what happened to the execution. Both adapters land here, so + settlement has one path on every transport.""" + command = await self._dao.fetch_command(command_id=command_id) + if command is None: + raise SessionCommandNotFound(command_id=str(command_id)) + + outcome = _OUTCOME_BY_EXECUTION_STATE.get(execution_state) + if outcome is None: + outcome = SessionCommandOutcome.failed + state = ( + SessionCommandState.applied + if result == "applied" + else SessionCommandState.obsolete + ) + if error: + log.warning( + "session command %s reported a failed cancel for execution=%s: %s", + command_id, + execution_id, + error[:2000], + ) + + settled = await self.settle( + command_id=command_id, + project_id=command.project_id, + replica_id=replica_id, + # Both, and checked at the moment of the write. Admission inserts `pending`, + # delivers, and only then writes `claimed` on the runner's behalf, so a runner that + # aborts fast reports its outcome while the row is still `pending`. Guarding on + # `claimed` alone refused that report with a conflict and left a correctly stopped + # execution sitting `claimed` until the sweep called it lost — the user watching + # "stopping" for the whole sweep window, and a Stop that worked recorded as lost. + expected_states=[ + SessionCommandState.pending, + SessionCommandState.claimed, + ], + state=state, + outcome=outcome, + execution_id=execution_id or command.target_turn_id, + ) + if settled is None: + stored = await self._dao.fetch_command(command_id=command_id) + raise SessionCommandNotClaimable( + command_id=str(command_id), + state=stored.state.value if stored else "unknown", + ) + return settled + + async def settle( + self, + *, + command_id: UUID, + project_id: UUID, + replica_id: Optional[str], + expected_states: List[SessionCommandState], + state: SessionCommandState, + outcome: SessionCommandOutcome, + execution_id: Optional[str], + ) -> Optional[SessionCommand]: + """Settle the command and the execution together, guarded on the command's state. + + The guard is what makes this idempotent: a second report finds a terminal row, changes + nothing, and the side effects below do not run twice. + """ + transition = SessionCommandSettle( + project_id=project_id, + command_id=command_id, + state=state, + outcome=outcome, + expected_states=expected_states, + replica_id=replica_id, + ) + atomic_core_settlement = self._executions is not None + cancelled_interactions = 0 + if atomic_core_settlement: + stored_command = await self._dao.fetch_command(command_id=command_id) + if stored_command is None: + return None + terminal = outcome in ( + SessionCommandOutcome.stopped, + SessionCommandOutcome.lost, + ) + settled_by = ( + "watchdog" + if outcome == SessionCommandOutcome.lost + else "runner" + if terminal + else None + ) + try: + async with self._dao.transaction() as transaction: + settled = await self._dao.settle_command( + settle=transition, + transaction=transaction, + ) + if settled is None: + raise _SettlementRejected + + if execution_id and terminal and settled_by: + result = await self._executions.settle( + project_id=project_id, + session_id=stored_command.session_id, + execution_id=execution_id, + terminal_outcome=outcome.value, + settled_by=settled_by, + transaction=transaction, + ) + winner = result.settlement + if not result.won and ( + winner.terminal_outcome != outcome.value + or winner.settled_by != settled_by + ): + raise _SettlementRejected + + await self._streams.settle_command( + project_id=project_id, + session_id=stored_command.session_id, + turn_id=execution_id, + mirror_stopped=outcome == SessionCommandOutcome.stopped, + transaction=transaction, + ) + if execution_id and outcome in ( + SessionCommandOutcome.stopped, + SessionCommandOutcome.not_running, + SessionCommandOutcome.lost, + ): + cancelled_interactions = ( + await self._interactions.cancel_session_pending( + project_id=project_id, + session_id=stored_command.session_id, + only_turn_id=execution_id, + transaction=transaction, + publish=False, + ) + ) + except _SettlementRejected: + return None + else: + settled = await self._dao.settle_command(settle=transition) + if settled is None: + return None + + session_id = settled.session_id + target = settled.target_turn_id + + if cancelled_interactions: + await self._interactions.publish_session_pending_cancelled( + project_id=project_id, + session_id=session_id, + ) + + if not atomic_core_settlement: + await self._dao.clear_stopping_turn( + project_id=project_id, + session_id=session_id, + turn_id=target, + ) + + if outcome == SessionCommandOutcome.stopped and target: + # Order matters. Tombstone first, so a late beat from the stopped execution cannot + # re-arm the locks it is about to lose; that beat would otherwise find `alive` free + # and take it straight back under the same turn id. + await self._reconcile_stopped_redis( + project_id=project_id, + session_id=session_id, + execution_id=target, + ) + # `alive` is deliberately left to its own time to live, exactly as the end of a + # normal turn leaves it. Warm resume is the required outcome of Stop, so the session + # must end up in the state a finished turn leaves it in, not in a torn-down one. + + # Mirror the nest onto the row HERE, because nothing else will. The tombstone + # above refuses the stopped execution's own final `is_running=false` beat before it + # can reach the heartbeat's mirror write, and the read model the product polls + # (`query_streams`) reads Postgres and never Redis. Skipping this leaves the row + # saying `is_running: true` until the orphan sweep collapses it, so the tab that + # pressed Stop shows a "running somewhere else" strip over its own session. + if not atomic_core_settlement: + await self._streams.mirror_liveness( + project_id=project_id, + session_id=session_id, + ) + + if outcome in ( + SessionCommandOutcome.stopped, + SessionCommandOutcome.not_running, + SessionCommandOutcome.lost, + ): + if target and not atomic_core_settlement: + # An approval card whose execution was stopped is a card whose buttons do + # nothing. Scoped to this execution, so a newer turn's gates survive. + await self._interactions.cancel_session_pending( + project_id=project_id, + session_id=session_id, + only_turn_id=target, + command_id=command_id, + ) + await self._streams.publish_session_ended( + project_id=project_id, + session_id=session_id, + ) + return settled + + +# The runner names what happened to the EXECUTION; the command's `outcome` column stores it. +_OUTCOME_BY_EXECUTION_STATE = { + "stopped": SessionCommandOutcome.stopped, + "not_running": SessionCommandOutcome.not_running, + "superseded_by_newer_turn": SessionCommandOutcome.superseded_by_newer_turn, + "failed": SessionCommandOutcome.failed, +} + +__all__ = [ + "CancelAdmission", + "SessionCommandsService", +] diff --git a/api/oss/src/core/sessions/commands/types.py b/api/oss/src/core/sessions/commands/types.py new file mode 100644 index 00000000000..47092a44c01 --- /dev/null +++ b/api/oss/src/core/sessions/commands/types.py @@ -0,0 +1,51 @@ +"""Domain errors of the session commands plane. The router maps each to a status code.""" + +from typing import Optional + + +class SessionCommandError(Exception): + """Base of every commands-plane domain error.""" + + +class ExecutionExpectationFailed(SessionCommandError): + """`expected_execution_id` does not name the execution that is running. + + Carries the current execution id (or None) so the caller can refresh rather than guess. + """ + + def __init__(self, *, expected: str, current: Optional[str]) -> None: + self.expected = expected + self.current = current + self.message = ( + f"expected execution '{expected}' is not the running execution " + f"(current: {current or 'none'})" + ) + super().__init__(self.message) + + +class SessionCommandIdempotencyConflict(SessionCommandError): + """An idempotency key was reused for a different cancel request.""" + + def __init__(self, *, idempotency_key: str) -> None: + self.idempotency_key = idempotency_key + self.message = ( + f"idempotency key '{idempotency_key}' belongs to a different request" + ) + super().__init__(self.message) + + +class SessionCommandNotFound(SessionCommandError): + def __init__(self, *, command_id: str) -> None: + self.command_id = command_id + self.message = f"no session command with id '{command_id}'" + super().__init__(self.message) + + +class SessionCommandNotClaimable(SessionCommandError): + """A settle arrived for a command this replica does not hold, or that is already terminal.""" + + def __init__(self, *, command_id: str, state: str) -> None: + self.command_id = command_id + self.state = state + self.message = f"session command '{command_id}' is '{state}' and cannot be settled by this caller" + super().__init__(self.message) diff --git a/api/oss/src/core/sessions/executions/__init__.py b/api/oss/src/core/sessions/executions/__init__.py new file mode 100644 index 00000000000..02e10675c1a --- /dev/null +++ b/api/oss/src/core/sessions/executions/__init__.py @@ -0,0 +1 @@ +"""Execution terminal-state contracts.""" diff --git a/api/oss/src/core/sessions/executions/dtos.py b/api/oss/src/core/sessions/executions/dtos.py new file mode 100644 index 00000000000..84c3887161e --- /dev/null +++ b/api/oss/src/core/sessions/executions/dtos.py @@ -0,0 +1,21 @@ +from datetime import datetime +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel + + +class SessionExecutionSettlement(BaseModel): + project_id: UUID + session_id: str + execution_id: str + terminal_outcome: str + settled_by: str + settled_at: datetime + ending_written_at: Optional[datetime] = None + redis_reconciled_at: Optional[datetime] = None + + +class SessionExecutionSettlementResult(BaseModel): + settlement: SessionExecutionSettlement + won: bool diff --git a/api/oss/src/core/sessions/executions/interfaces.py b/api/oss/src/core/sessions/executions/interfaces.py new file mode 100644 index 00000000000..92e87e927c2 --- /dev/null +++ b/api/oss/src/core/sessions/executions/interfaces.py @@ -0,0 +1,62 @@ +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Any, Dict, List, Optional, Sequence, Tuple +from uuid import UUID + +from oss.src.core.sessions.executions.dtos import ( + SessionExecutionSettlement, + SessionExecutionSettlementResult, +) + + +class SessionExecutionsDAOInterface(ABC): + @abstractmethod + async def settle( + self, + *, + project_id: UUID, + session_id: str, + execution_id: str, + terminal_outcome: str, + settled_by: str, + settled_at: Optional[datetime] = None, + transaction: Optional[Any] = None, + ) -> SessionExecutionSettlementResult: + """Compare-and-set one terminal outcome and return the stored winner.""" + + @abstractmethod + async def query_settled( + self, + *, + project_id: UUID, + keys: Sequence[Tuple[str, str]], + ) -> Dict[Tuple[str, str], SessionExecutionSettlement]: + """Fetch terminal state for `(session_id, execution_id)` keys.""" + + @abstractmethod + async def mark_endings_written( + self, + *, + project_id: UUID, + keys: Sequence[Tuple[str, str]], + written_at: Optional[datetime] = None, + ) -> None: + """Mark terminal executions whose transcript ending has been written.""" + + @abstractmethod + async def list_redis_unreconciled( + self, + *, + limit: int, + ) -> List[SessionExecutionSettlement]: + """Runner settlements whose post-commit Redis projection is incomplete.""" + + @abstractmethod + async def mark_redis_reconciled( + self, + *, + project_id: UUID, + session_id: str, + execution_id: str, + ) -> None: + """Record completion of the idempotent post-commit Redis projection.""" diff --git a/api/oss/src/core/sessions/interactions/interfaces.py b/api/oss/src/core/sessions/interactions/interfaces.py index 60336b51395..7a11646a6b4 100644 --- a/api/oss/src/core/sessions/interactions/interfaces.py +++ b/api/oss/src/core/sessions/interactions/interfaces.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Optional +from typing import Any, List, Optional from uuid import UUID from oss.src.core.sessions.interactions.dtos import ( @@ -47,7 +47,8 @@ async def cancel_session_pending( except_turn_id: Optional[str] = None, except_tokens: Optional[List[str]] = None, only_turn_id: Optional[str] = None, - ) -> int: ... + transaction: Optional[Any] = None, + ) -> List[SessionInteraction]: ... @abstractmethod async def query_interactions( diff --git a/api/oss/src/core/sessions/interactions/service.py b/api/oss/src/core/sessions/interactions/service.py index 02d685404f8..14c5187174a 100644 --- a/api/oss/src/core/sessions/interactions/service.py +++ b/api/oss/src/core/sessions/interactions/service.py @@ -1,5 +1,5 @@ -from typing import List, Optional -from uuid import UUID +from typing import Any, List, Optional +from uuid import NAMESPACE_DNS, UUID, uuid5 from oss.src.core.sessions.interactions.dtos import ( SessionInteraction, @@ -11,12 +11,19 @@ SessionInteractionsDAOInterface, ) from oss.src.core.sessions.interactions.types import InteractionNotFound +from oss.src.core.sessions.records.dtos import SessionRecordEvent +from oss.src.core.sessions.records.service import RecordsService from oss.src.core.shared.dtos import Windowing from oss.src.dbs.redis.sessions.contract import ( WATCH_INTERACTION_PENDING, WATCH_INTERACTION_RESOLVED, ) from oss.src.core.sessions.watch.interfaces import SessionsWatchPublisherInterface +from oss.src.utils.logging import get_module_logger + + +_RECORD_NAMESPACE = uuid5(uuid5(NAMESPACE_DNS, "agenta"), "records") +log = get_module_logger(__name__) class SessionInteractionsService: @@ -25,9 +32,11 @@ def __init__( *, interactions_dao: SessionInteractionsDAOInterface, watch_publisher: Optional[SessionsWatchPublisherInterface] = None, + records_service: Optional[RecordsService] = None, ) -> None: self.interactions_dao = interactions_dao self._watch = watch_publisher + self._records = records_service async def _publish_interaction( self, *, project_id: UUID, session_id: str, status: str @@ -102,6 +111,9 @@ async def cancel_session_pending( except_turn_id: Optional[str] = None, except_tokens: Optional[List[str]] = None, only_turn_id: Optional[str] = None, + command_id: Optional[UUID] = None, + transaction: Optional[Any] = None, + publish: bool = True, ) -> int: cancelled = await self.interactions_dao.cancel_session_pending( project_id=project_id, @@ -109,14 +121,58 @@ async def cancel_session_pending( except_turn_id=except_turn_id, except_tokens=except_tokens, only_turn_id=only_turn_id, + transaction=transaction, ) - if cancelled: - await self._publish_interaction( - project_id=project_id, - session_id=session_id, - status=WATCH_INTERACTION_RESOLVED, + if cancelled and command_id is not None and self._records is not None: + try: + await self._records.append_many( + events=[ + SessionRecordEvent( + project_id=project_id, + session_id=interaction.session_id, + record_id=uuid5( + _RECORD_NAMESPACE, + f"{interaction.session_id}:{interaction.token}:" + f"interaction_response:{interaction.turn_id or ''}", + ), + record_type="interaction_response", + record_source="agent", + attributes={ + "type": "interaction_response", + "id": interaction.token, + "kind": interaction.kind.value, + "payload": { + "outcome": "cancelled", + "turnId": interaction.turn_id, + "commandId": str(command_id), + }, + }, + turn_id=interaction.turn_id, + ) + for interaction in cancelled + ] + ) + except Exception: + log.warning( + "Failed to append cancellation records for session=%s command=%s", + session_id, + command_id, + exc_info=True, + ) + if cancelled and publish: + await self.publish_session_pending_cancelled( + project_id=project_id, session_id=session_id ) - return cancelled + return len(cancelled) + + async def publish_session_pending_cancelled( + self, *, project_id: UUID, session_id: str + ) -> None: + await self._publish_interaction( + project_id=project_id, + session_id=session_id, + status=WATCH_INTERACTION_RESOLVED, + ) async def query_interactions( self, diff --git a/api/oss/src/core/sessions/records/dtos.py b/api/oss/src/core/sessions/records/dtos.py index 5785b392b36..ce0e1089d0f 100644 --- a/api/oss/src/core/sessions/records/dtos.py +++ b/api/oss/src/core/sessions/records/dtos.py @@ -10,6 +10,21 @@ # just keeps the DTO honest about that contract for any other producer. SESSION_MESSAGE_PREVIEW_TEXT_LIMIT = 240 +# The runner's terminal per-turn record type, mirrored from +# services/runner/src/protocol.ts (`{ type: "done" }`). Also spelled in the records DAO and +# the ingest worker, which read the same marker off their own layers. +TERMINAL_RECORD_TYPE = "done" + +# Who wrote a terminal record, stamped into `attributes` by the writer. +# +# Only the platform ever sets it: the ingest route builds `SessionRecordEvent` field by field +# from the request body and has no path to this key, so a runner cannot claim to be the +# watchdog. It exists because the two endings are otherwise identical — the watchdog copies +# the runner's `{"type": "done"}` deliberately, so one outcome never reaches a user in two +# wordings — and the late-record guard has to tell them apart. +RECORD_SETTLED_BY_ATTRIBUTE = "settled_by" +SETTLED_BY_WATCHDOG = "watchdog" + class SessionRecordEvent(BaseModel): project_id: UUID @@ -26,6 +41,12 @@ class SessionRecordEvent(BaseModel): turn_id: Optional[str] = None span_id: Optional[OTelSpanId] = None + # Set ONLY by the ingest guard in `RecordsService.append_many`, never by a producer: the + # ingest route builds this DTO field by field and never reads this one off the wire. A + # non-null value means the record arrived for a turn the watchdog had already ended, so it + # is kept as evidence and left out of the transcript. See `RecordsService.append_many`. + quarantined_at: Optional[datetime] = None + class SessionRecord(Lifecycle): record_id: UUID @@ -42,6 +63,11 @@ class SessionRecord(Lifecycle): turn_id: Optional[str] = None span_id: Optional[OTelSpanId] = None + # Non-null when this record was written for an already-settled turn. Reads that rebuild a + # transcript filter these out at the DAO; the column is exposed so support and billing can + # still see the work the agent did after the platform closed the turn. + quarantined_at: Optional[datetime] = None + class SessionMessagePreview(BaseModel): """The last thing said in a session, for a list row. diff --git a/api/oss/src/core/sessions/records/interfaces.py b/api/oss/src/core/sessions/records/interfaces.py index d0dba2ec9f3..0fe78b0e325 100644 --- a/api/oss/src/core/sessions/records/interfaces.py +++ b/api/oss/src/core/sessions/records/interfaces.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple from uuid import UUID from oss.src.core.sessions.records.dtos import ( @@ -47,3 +47,17 @@ async def latest_message_per_session( session_ids: List[str], ) -> Dict[str, SessionMessagePreview]: raise NotImplementedError + + async def settled_turns( + self, + *, + project_id: UUID, + keys: Sequence[Tuple[str, str]], + settled_by: Optional[str] = None, + ) -> Set[Tuple[str, str]]: + """Which of these `(session_id, turn_id)` pairs already carry a terminal record. + + `settled_by` narrows the answer to endings that one writer wrote; see the DAO. + """ + + raise NotImplementedError diff --git a/api/oss/src/core/sessions/records/service.py b/api/oss/src/core/sessions/records/service.py index 79d8a7e2393..a1ab2d52044 100644 --- a/api/oss/src/core/sessions/records/service.py +++ b/api/oss/src/core/sessions/records/service.py @@ -1,17 +1,44 @@ -from typing import Any, Dict, List, Optional +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple from uuid import UUID from oss.src.core.sessions.records.dtos import ( + RECORD_SETTLED_BY_ATTRIBUTE, + SETTLED_BY_WATCHDOG, + TERMINAL_RECORD_TYPE, SessionMessagePreview, SessionRecord, SessionRecordEvent, ) +from oss.src.core.sessions.executions.dtos import SessionExecutionSettlement +from oss.src.core.sessions.executions.interfaces import SessionExecutionsDAOInterface from oss.src.core.sessions.records.interfaces import RecordsDAOInterface +from oss.src.utils.env import env +from oss.src.utils.logging import get_module_logger + +log = get_module_logger(__name__) + + +def _written_by_watchdog(event: SessionRecordEvent) -> bool: + """Did the platform write this record, rather than a runner? + + Only the watchdog stamps the marker, and only the platform can: the ingest route builds + `SessionRecordEvent` field by field out of the request body and never reads this key off + the wire, so a runner cannot present itself as the watchdog to get past the guard below. + """ + return (event.attributes or {}).get( + RECORD_SETTLED_BY_ATTRIBUTE + ) == SETTLED_BY_WATCHDOG class RecordsService: - def __init__(self, records_dao: RecordsDAOInterface): + def __init__( + self, + records_dao: RecordsDAOInterface, + executions_dao: Optional[SessionExecutionsDAOInterface] = None, + ): self.records_dao = records_dao + self.executions_dao = executions_dao async def append( self, @@ -26,7 +53,230 @@ async def append_many( *, events: List[SessionRecordEvent], ) -> List[SessionRecord]: - return await self.records_dao.append_many(events=events) + """Append a batch, quarantining anything that arrives after the platform ended its turn. + + RFC "Required behavior / Execution" item 3: after an execution reaches its terminal + outcome, later non-terminal output for it is rejected or quarantined. This is where + that happens, because ingest is the only place both writers meet. + + The case is real and was caught live. A runner wedges, the watchdog writes the turn's + `error` and `done` on its behalf, and the runner then THAWS and submits everything it + had buffered — a tool call, its result, a `usage`, and a second `done`. Nothing + downstream could refuse it: the runner-side gate in `server.ts` knows only about + endings that request wrote itself, and the reader was left with a failure notice + followed by the work the agent went on to do. + + Quarantine rather than reject, deliberately. The tail is real work: a late `usage` + carries token accounting that is real money, and the tool result is the first thing a + support engineer asks for. A dropped record cannot be looked at later; a marked one + can, and it is already invisible to every read that rebuilds a transcript. + """ + if not events: + return [] + + guarded = await self._handle_late_events(events=events) + appended = await self.records_dao.append_many(events=guarded) + await self._mark_endings_written(events=guarded) + return appended + + async def _mark_endings_written( + self, + *, + events: List[SessionRecordEvent], + ) -> None: + if self.executions_dao is None or not env.agenta.sessions.durable_stop: + return + + endings: Dict[UUID, Set[Tuple[str, str]]] = {} + for event in events: + if ( + event.record_type != TERMINAL_RECORD_TYPE + or not event.turn_id + or event.quarantined_at is not None + ): + continue + endings.setdefault(event.project_id, set()).add( + (event.session_id, event.turn_id) + ) + + for project_id, keys in endings.items(): + try: + await self.executions_dao.mark_endings_written( + project_id=project_id, + keys=sorted(keys), + ) + except Exception: + log.warning( + "[RECORDS] Execution ending marker update failed; record remains appended", + project_id=str(project_id), + exc_info=True, + ) + + async def _handle_late_events( + self, + *, + events: List[SessionRecordEvent], + ) -> List[SessionRecordEvent]: + """Stamp `quarantined_at` on every event belonging to a settled turn. + + Scoped as narrowly as the invariant allows, in three ways. + + * Only turns the WATCHDOG ended. A turn that reached its own honest ending — an + ordinary Stop, a normal completion — is untouched, so the runner's single ending + always lands and a `usage` that trails its own `done` through the stream is still + ordinary history. + * Only records the watchdog did not write. Its own `error` is not a terminal record, + so a redelivery of it after its `done` had landed would otherwise quarantine the + very ending it belongs to. + * Terminal records included. A late `done` is quarantined like the rest of the tail, + which is what keeps ONE effective ending: folding it into the watchdog's would + rewrite the record the user has already read, and hide that two writers disagreed. + + A batch that carries the watchdog's own `done` settles that turn for the rest of the + same batch. Ingest batches up to fifty messages, and the thawed runner's tail can + share one with the ending that beat it by a second. + + A failed lookup quarantines nothing and appends everything. Losing a record is worse + than showing one that should have been hidden, and the next delivery gets another go. + """ + if not env.agenta.sessions.durable_stop: + return events + + if self.executions_dao is not None: + return await self._handle_by_execution_state(events=events) + + candidates: Dict[UUID, Set[Tuple[str, str]]] = {} + for event in events: + if not event.turn_id or _written_by_watchdog(event): + continue + candidates.setdefault(event.project_id, set()).add( + (event.session_id, event.turn_id) + ) + + if not candidates: + return events + + settled: Dict[UUID, Set[Tuple[str, str]]] = {} + for project_id, keys in candidates.items(): + try: + settled[project_id] = await self.records_dao.settled_turns( + project_id=project_id, + keys=sorted(keys), + settled_by=SETTLED_BY_WATCHDOG, + ) + except Exception: + log.warning( + "[RECORDS] Late-record lookup failed; appending the batch unguarded", + project_id=str(project_id), + exc_info=True, + ) + settled[project_id] = set() + + for event in events: + if ( + _written_by_watchdog(event) + and event.record_type == TERMINAL_RECORD_TYPE + and event.turn_id + ): + settled.setdefault(event.project_id, set()).add( + (event.session_id, event.turn_id) + ) + + now = datetime.now(timezone.utc) + guarded: List[SessionRecordEvent] = [] + for event in events: + is_late = ( + event.turn_id is not None + and not _written_by_watchdog(event) + and (event.session_id, event.turn_id) + in settled.get(event.project_id, set()) + ) + if not is_late: + guarded.append(event) + continue + + action = env.agenta.sessions.late_output + log.warning( + "[RECORDS] %s a record for a turn the watchdog had already ended", + "Rejected" if action == "reject" else "Quarantined", + project_id=str(event.project_id), + session_id=event.session_id, + turn_id=event.turn_id, + record_type=event.record_type, + record_id=str(event.record_id) if event.record_id else None, + ) + if action == "reject": + continue + guarded.append(event.model_copy(update={"quarantined_at": now})) + + return guarded + + async def _handle_by_execution_state( + self, + *, + events: List[SessionRecordEvent], + ) -> List[SessionRecordEvent]: + candidates: Dict[UUID, Set[Tuple[str, str]]] = {} + for event in events: + if event.turn_id: + candidates.setdefault(event.project_id, set()).add( + (event.session_id, event.turn_id) + ) + + if not candidates: + return events + + settled: Dict[UUID, Dict[Tuple[str, str], SessionExecutionSettlement]] = {} + for project_id, keys in candidates.items(): + try: + settled[project_id] = await self.executions_dao.query_settled( + project_id=project_id, + keys=sorted(keys), + ) + except Exception: + log.warning( + "[RECORDS] Terminal execution lookup failed; appending the batch unguarded", + project_id=str(project_id), + exc_info=True, + ) + settled[project_id] = {} + + now = datetime.now(timezone.utc) + guarded: List[SessionRecordEvent] = [] + for event in events: + if not event.turn_id: + guarded.append(event) + continue + terminal = settled.get(event.project_id, {}).get( + (event.session_id, event.turn_id) + ) + if terminal is None: + guarded.append(event) + continue + + writer = "watchdog" if _written_by_watchdog(event) else "runner" + is_late = terminal.settled_by != writer and terminal.terminal_outcome in ( + "lost", + "stopped", + ) + if not is_late: + guarded.append(event) + continue + + action = env.agenta.sessions.late_output + log.warning( + "[RECORDS] %s a record for an execution that is already terminal", + "Rejected" if action == "reject" else "Quarantined", + project_id=str(event.project_id), + session_id=event.session_id, + turn_id=event.turn_id, + record_type=event.record_type, + record_id=str(event.record_id) if event.record_id else None, + ) + if action == "quarantine": + guarded.append(event.model_copy(update={"quarantined_at": now})) + + return guarded async def get_records( self, @@ -64,3 +314,20 @@ async def latest_message_per_session( project_id=project_id, session_ids=session_ids, ) + + async def settled_turns( + self, + *, + project_id: UUID, + keys: Sequence[Tuple[str, str]], + settled_by: Optional[str] = None, + ) -> Set[Tuple[str, str]]: + """One batched lookup for a whole watchdog pass — never one call per candidate.""" + if not keys: + return set() + + return await self.records_dao.settled_turns( + project_id=project_id, + keys=keys, + settled_by=settled_by, + ) diff --git a/api/oss/src/core/sessions/streams/dtos.py b/api/oss/src/core/sessions/streams/dtos.py index c20aa5575d3..0998cc93b52 100644 --- a/api/oss/src/core/sessions/streams/dtos.py +++ b/api/oss/src/core/sessions/streams/dtos.py @@ -41,6 +41,11 @@ class SessionStream(Identifier, Header, Lifecycle): tags: Optional[Dict[str, Any]] = None meta: Optional[Dict[str, Any]] = None turn_id: Optional[str] = None + # When `turn_id` started. Stamped only when the id changes, so repeated heartbeats never + # move it. The stale-Stop guard compares a cancel request's arrival time against this. + turn_started_at: Optional[datetime] = None + # The execution an accepted Stop is waiting on. Null when nothing is stopping. + stopping_turn_id: Optional[str] = None # What this session runs. Filled once, from the first beat that knows — turn appends # are fire-and-forget, so a session whose only reference carrier was a dropped append # is unopenable forever. @@ -75,6 +80,10 @@ class SessionStreamEdit(Header): tags: Optional[Dict[str, Any]] = None meta: Optional[Dict[str, Any]] = None turn_id: Optional[str] = None + # Internal heartbeat fence. When present, the DAO updates only this still-current, + # non-terminal execution generation. Excluded from serialization because it is a write + # precondition, not stream state. + expected_turn_id: Optional[str] = Field(default=None, exclude=True) class SessionStreamHeaderEdit(Header): @@ -143,6 +152,45 @@ class SessionStreamCommandRequest(BaseModel): data: Optional[WorkflowServiceRequestData] = None force: bool = False detached: bool = False # fire-and-forget mode + # A stale-request guard for cancel mode only; send, steer, and attach ignore it. + expected_execution_id: Optional[str] = Field( + default=None, + description=( + "Optional stale-request guard honored only in cancel mode; ignored for send, " + "steer, and attach." + ), + ) + + @field_validator("expected_execution_id") + @classmethod + def _blank_expected_execution_id_means_absent( + cls, value: Optional[str] + ) -> Optional[str]: + if value is None: + return None + return value.strip() or None + + # Cancel guard (RFC D-010). Public name; internally this IS a turn id — the coordination + # plane's word for one execution of a session. The RFC calls it an execution id, so the + # public DTO keeps that name and the service maps it onto `turn_id` at the boundary. + # Optional by decision: external callers may cancel blind. When present, cancel touches + # that turn or nothing. + expected_execution_id: Optional[str] = None + + @field_validator("expected_execution_id") + @classmethod + def _blank_expected_execution_id_means_absent( + cls, value: Optional[str] + ) -> Optional[str]: + """A whitespace-only guard is a client bug, not a request to cancel a turn named "". + + Reading it as "no guard" is the safe failure: the caller falls back to the arrival-time + check instead of matching a turn id nothing can hold. + """ + if value is None: + return None + trimmed = value.strip() + return trimmed or None class SessionStreamCommandResponse(BaseModel): @@ -151,6 +199,9 @@ class SessionStreamCommandResponse(BaseModel): turn_id: Optional[str] = None watcher_id: Optional[str] = None detached: bool = False + # Cancel only: every turn this cancel tombstoned. Usually one. It is a list because + # `alive` and `running` can be held by different turns during a handover, and both die. + cancelled_turn_ids: List[str] = Field(default_factory=list) class SessionHeartbeatRequest(BaseModel): @@ -169,6 +220,14 @@ class SessionHeartbeatRequest(BaseModel): is_running: bool = True name: Optional[str] = None references: Optional[List[SessionReference]] = None + # The INVERSE beat, sent once per session as a runner shuts down: hand the affinity key + # back instead of renewing it. `claim_owner` never steals, so a replica that dies still + # holding `owner:session:` locks the session out of every other replica for the rest + # of OWNER_TTL_SECONDS — a local-provider session then refuses every message until the + # lease expires. The release is conditional on still being the owner, so it can never + # take a session from a live replica. Everything else about the beat is skipped: a + # departing runner asserts no liveness and no turn. + release_owner: bool = False class SessionLiveness(BaseModel): diff --git a/api/oss/src/core/sessions/streams/interfaces.py b/api/oss/src/core/sessions/streams/interfaces.py index 7c2e740f202..412b62aac38 100644 --- a/api/oss/src/core/sessions/streams/interfaces.py +++ b/api/oss/src/core/sessions/streams/interfaces.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Optional +from typing import Any, List, Optional from uuid import UUID from oss.src.core.sessions.streams.dtos import ( @@ -16,6 +16,18 @@ class SessionStreamsDAOInterface(ABC): + @abstractmethod + async def settle_command( + self, + *, + project_id: UUID, + session_id: str, + turn_id: Optional[str], + mirror_stopped: bool, + transaction: Optional[Any] = None, + ) -> None: + """Clear this command's marker and project its stopped state.""" + @abstractmethod async def create( self, diff --git a/api/oss/src/core/sessions/streams/runner_client.py b/api/oss/src/core/sessions/streams/runner_client.py index 45e6689aca8..39c3f8dd1f1 100644 --- a/api/oss/src/core/sessions/streams/runner_client.py +++ b/api/oss/src/core/sessions/streams/runner_client.py @@ -17,6 +17,8 @@ the runner's own orphan sweep / idle-TTL eviction is the fallback net for a missed signal. """ +from typing import NamedTuple, Optional + import httpx from oss.src.utils.env import env @@ -60,3 +62,109 @@ async def kill_runner_sandbox(*, project_id: str, session_id: str) -> bool: except httpx.HTTPError as e: log.warning("kill: runner /kill call failed for session=%s: %s", session_id, e) return False + + +_CANCEL_TIMEOUT_SECONDS = 5.0 + + +class RunnerCancelResult: + """What the direct hop learned, as three named cases. + + * `accepted` — the runner holds the session and took the command. The outcome arrives + later on the outcome route, never in this response. + * `not_held` — the runner answered, and it does not hold that session. + * `unreachable` — no answer, a non-2xx that is not 404, or no runner configured at all. + """ + + accepted = "accepted" + not_held = "not_held" + unreachable = "unreachable" + + +class RunnerCancelResponse(NamedTuple): + """The acknowledgement, and WHICH runner process gave it. + + `replica_id` is what the API records as the claim holder, so the outcome route's guard + (`state='claimed' AND claimed_by=:replica_id`) matches the id the runner reports with. Take + it from the answer rather than assuming one: a claim written under a name the runner does + not use refuses the runner's own outcome report, which leaves the command open and the + session marked stopping forever. + """ + + status: str + replica_id: Optional[str] = None + + +async def cancel_runner_execution( + *, + command_id: str, + project_id: str, + session_id: str, + target_turn_id: Optional[str], + created_at: str, + timeout_seconds: float = _CANCEL_TIMEOUT_SECONDS, +) -> RunnerCancelResponse: + """POST the runner's `/cancel`. Returns the acknowledgement and the answering replica. + + Never raises. The command row is already committed when this runs, so a failure here costs + promptness, not the Stop: a later claim or the settlement sweep still reaches it. + + The body is camelCase because the runner's own HTTP surface is (see its `/kill`). + """ + base_url = env.runner.internal_url + token = env.runner.token + if not base_url or not token: + log.warning( + "cancel: no runner internal_url/token configured; command %s cannot be delivered", + command_id, + ) + return RunnerCancelResponse(RunnerCancelResult.unreachable) + + url = base_url.rstrip("/") + "/cancel" + try: + async with httpx.AsyncClient(timeout=timeout_seconds) as client: + response = await client.post( + url, + json={ + "commandId": command_id, + "projectId": project_id, + "sessionId": session_id, + "targetTurnId": target_turn_id, + "createdAt": created_at, + }, + headers={"Authorization": f"Bearer {token}"}, + ) + except httpx.HTTPError as e: + log.warning( + "cancel: runner /cancel call failed for session=%s command=%s: %s", + session_id, + command_id, + e, + ) + return RunnerCancelResponse(RunnerCancelResult.unreachable) + + if response.status_code == 404: + return RunnerCancelResponse(RunnerCancelResult.not_held) + if response.status_code >= 300: + log.warning( + "cancel: runner /cancel returned %s for session=%s command=%s", + response.status_code, + session_id, + command_id, + ) + return RunnerCancelResponse(RunnerCancelResult.unreachable) + + replica_id = None + try: + payload = response.json() + if isinstance(payload, dict): + replica_id = payload.get("replicaId") + except ValueError: + # A 2xx with no JSON body still means accepted; the claim then falls back to a + # placeholder and the runner's report is refused, so log it rather than hide it. + log.warning( + "cancel: runner /cancel answered %s with no JSON body for command=%s", + response.status_code, + command_id, + ) + return RunnerCancelResponse(RunnerCancelResult.accepted, replica_id) diff --git a/api/oss/src/core/sessions/streams/service.py b/api/oss/src/core/sessions/streams/service.py index c583a9265c0..9d0df4d577e 100644 --- a/api/oss/src/core/sessions/streams/service.py +++ b/api/oss/src/core/sessions/streams/service.py @@ -23,16 +23,18 @@ CONCURRENCY_LIMIT, WATCH_LIFECYCLE_ENDED, WATCH_LIFECYCLE_RUNNING, + owner_replica_id, validate_session_id as _validate_session_id_fn, ) from oss.src.core.sessions.watch.interfaces import SessionsWatchPublisherInterface from oss.src.dbs.redis.sessions.locks import ( - acquire_alive, + acquire_alive_with_start, acquire_running, claim_owner, - clear_running, + claim_owner_value, + clear_owner, + displace_turns, release_running, - force_cancel_alive, force_clear_owner, get_alive_owner, get_owner, @@ -40,10 +42,13 @@ get_session_liveness, is_turn_superseded, mark_turn_superseded, + record_turn_start, + redis_time_ms, refresh_alive, refresh_running, release_alive, release_attached, + release_owner_value, steal_attached, ) @@ -66,6 +71,7 @@ SessionIdInvalid, SessionStreamAlreadyExists, SessionTurnInUse, + SessionTurnMismatch, ) from oss.src.core.sessions.streams.interfaces import SessionStreamsDAOInterface from oss.src.core.sessions.streams.runner_client import kill_runner_sandbox @@ -136,6 +142,24 @@ def derive_session_name(inputs: Optional[Dict[str, Any]]) -> Optional[str]: return normalize_session_name(_first_user_message_text(messages)) +def derive_command_mode(request: SessionStreamCommandRequest) -> CommandMode: + """The inputs x force matrix, as one function. + + Module-level because the route needs the mode BEFORE the service runs: a cancel must not be + refused by the per-project concurrency limit, and that check happens at the route. Keeping the + derivation in one place is what stops the two from disagreeing about what a cancel is. + """ + has_inputs = bool(request.data and request.data.inputs) + + if has_inputs and not request.force: + return CommandMode.send + if has_inputs and request.force: + return CommandMode.steer + if not has_inputs and not request.force: + return CommandMode.cancel + return CommandMode.attach + + class SessionStreamsService: def __init__( self, @@ -166,38 +190,34 @@ async def _supersede_turns( turn_id=turn_id, ) - async def _displace_turns(self, *, project_id: UUID, session_id: str) -> None: - """Tear alive+running off whichever turn holds them, tombstoning it first. - - The order is the point. Clearing first leaves a window in which the turn being - displaced heartbeats, finds `alive` free and nx-acquires it straight back - a - cancelled session then reads as alive for a whole ALIVE_TTL. Tombstoning first makes - that beat refuse itself. The keys are still re-read after the clear, so a turn that - took them inside the window is tombstoned too. - """ - await self._supersede_turns( - project_id=project_id, - session_id=session_id, - turn_ids=( - await get_alive_owner( - self._lock, project_id=str(project_id), session_id=session_id - ), - await get_running_owner( - self._lock, project_id=str(project_id), session_id=session_id - ), - ), - ) - displaced_alive = await force_cancel_alive( - self._lock, project_id=str(project_id), session_id=session_id - ) - displaced_running = await clear_running( - self._lock, project_id=str(project_id), session_id=session_id - ) - await self._supersede_turns( - project_id=project_id, + async def _displace_turns( + self, + *, + project_id: UUID, + session_id: str, + expected_turn_id: Optional[str] = None, + arrived_at_ms: Optional[int] = None, + running_only: bool = False, + ) -> List[str]: + """Atomically guard, tombstone, and clear the alive/running owners.""" + accepted, actual_turn_id, displaced = await displace_turns( + self._lock, + project_id=str(project_id), session_id=session_id, - turn_ids=(displaced_alive, displaced_running), + expected_turn_id=expected_turn_id, + arrived_at_ms=arrived_at_ms, + running_only=running_only, ) + if not accepted: + raise SessionTurnMismatch( + session_id, + actual_turn_id=actual_turn_id, + expected_turn_id=expected_turn_id, + ) + return displaced + + async def clock_ms(self) -> int: + return await redis_time_ms(self._lock) async def _publish_lifecycle( self, *, project_id: UUID, session_id: str, state: str @@ -210,6 +230,19 @@ async def _publish_lifecycle( state=state, ) + async def publish_session_ended(self, *, project_id: UUID, session_id: str) -> None: + """Announce that a turn ended, on the channel every open browser already listens to. + + Public because the durable-command plane settles a Stop and has to publish the same + notification the ordinary end-of-turn path publishes. There is one `ended` event, not a + Stop-shaped one and a turn-shaped one; a client cannot be asked to tell them apart. + """ + await self._publish_lifecycle( + project_id=project_id, + session_id=session_id, + state=WATCH_LIFECYCLE_ENDED, + ) + async def _publish_changed(self, *, project_id: UUID, session_id: str) -> None: if self._watch is None: return @@ -219,6 +252,7 @@ async def _publish_changed(self, *, project_id: UUID, session_id: str) -> None: entity="session", id=session_id, ) + except Exception: log.warning( "[WATCH] session change publish failed", @@ -226,25 +260,41 @@ async def _publish_changed(self, *, project_id: UUID, session_id: str) -> None: session_id=session_id, ) + async def settle_command( + self, + *, + project_id: UUID, + session_id: str, + turn_id: Optional[str], + mirror_stopped: bool, + transaction: Optional[Any] = None, + ) -> None: + await self._dao.settle_command( + project_id=project_id, + session_id=session_id, + turn_id=turn_id, + mirror_stopped=mirror_stopped, + transaction=transaction, + ) + async def command( self, *, project_id: UUID, user_id: UUID, request: SessionStreamCommandRequest, + arrived_at_ms: Optional[int] = None, ) -> SessionStreamCommandResponse: _validate_session_id(request.session_id) - has_inputs = bool(request.data and request.data.inputs) + # When the request reached the process, for the stale-cancel guard. The router stamps it + # before its permission and concurrency checks, which are database round trips; stamping + # here instead would leave the guard almost no window. Defaulted so a caller that does not + # stamp still gets a check, just a narrower one. + if arrived_at_ms is None: + arrived_at_ms = await self.clock_ms() - if has_inputs and not request.force: - mode = CommandMode.send - elif has_inputs and request.force: - mode = CommandMode.steer - elif not has_inputs and not request.force: - mode = CommandMode.cancel - else: - mode = CommandMode.attach + mode = derive_command_mode(request) session_id = request.session_id proposed_name = derive_session_name( @@ -286,21 +336,33 @@ async def command( ) elif mode == CommandMode.cancel: - await self._displace_turns(project_id=project_id, session_id=session_id) - await self._mark_stream_ended( + cancelled_turn_ids = await self._displace_turns( project_id=project_id, - user_id=user_id, session_id=session_id, + expected_turn_id=request.expected_execution_id, + arrived_at_ms=arrived_at_ms, + running_only=request.expected_execution_id is None, ) - await self._publish_lifecycle( - project_id=project_id, - session_id=session_id, - state=WATCH_LIFECYCLE_ENDED, - ) + if cancelled_turn_ids: + await self._mark_stream_ended( + project_id=project_id, + user_id=user_id, + session_id=session_id, + ) + await self._publish_lifecycle( + project_id=project_id, + session_id=session_id, + state=WATCH_LIFECYCLE_ENDED, + ) return SessionStreamCommandResponse( mode=mode, session_id=session_id, + # The turn this cancel actually ended. The caller (the router) needs it to + # cancel that turn's pending gates, and it is the id a client should echo back + # as `expected_execution_id` on a retry. + turn_id=cancelled_turn_ids[0] if cancelled_turn_ids else None, detached=True, + cancelled_turn_ids=cancelled_turn_ids, ) else: # ATTACH @@ -403,6 +465,87 @@ async def kill( session_id=session_id, ) + async def _reclaim_affinity_from_a_departed_replica( + self, + *, + project_id: UUID, + request: SessionHeartbeatRequest, + incumbent_value: str, + ) -> str: + """Take `owner:session:` from a replica that holds no running turn on it. + + `owner` exists to say which box is SERVING the session, and only an in-flight turn's + heartbeat ever refreshes it. So a claim held by a replica with no running turn is not + protecting anything: it is the residue of a runner that stopped beating. A runner that + dies without a graceful shutdown (SIGKILL, OOM, a crashed node, `docker restart -t 0`) + always leaves exactly that, because nothing releases the key on its way out and + `claim_owner` never steals. The replacement replica then loses every beat for the rest + of OWNER_TTL_SECONDS, and the runner reads that refusal as "another turn owns this + session" and refuses the user's next message for two minutes. + + `running` is the discriminator, the same one the alive-lock handover below uses. A live + turn holds it under its own id for the whole turn and re-arms it every beat, so a + replica that is genuinely serving the session can never be mistaken for a departed one. + A `running` lock held by the CALLER's own turn is not an obstacle: `_start_turn` arms + alive and running before the runner's first beat, so an API-minted turn legitimately + arrives here with its own lock already in place. + + Only the beat of a real, running turn may reclaim. A turn-end beat asserts nothing + about who should serve the session next, and a beat with no turn id proves no work. + + KNOWN LIMIT. A turn parked awaiting an approval also holds `alive` with no `running`, + so on a MULTI-replica deployment a second replica can take affinity from a live first + one and the handover below then tombstones the parked turn, killing the pending + approval. That outcome is not new: nothing refreshes `owner` on a parked session, so the + key expires after OWNER_TTL_SECONDS and the same handover follows. This only makes it up + to that TTL sooner, and only on a topology the direct control adapter cannot route to + anyway (`core/sessions/commands/service.py`). On a single replica the caller already + equals the owner and this method is never entered. + + Returns the owner after the attempt: the caller when the reclaim landed, otherwise + whoever holds the key, which is what the refusal above must report. + """ + incumbent = owner_replica_id(incumbent_value) + if not (request.turn_id and request.is_running): + return incumbent + + running_owner = await get_running_owner( + self._lock, + project_id=str(project_id), + session_id=request.session_id, + ) + if running_owner is not None and running_owner != request.turn_id: + return incumbent + + # Release-if-owner, then the ordinary non-stealing claim. Two atomic steps rather than + # one so no new script is needed, and the gap is safe in both directions: a concurrent + # claim by a third replica makes the release a no-op and the claim below returns that + # replica, so this path can never hand the session to the wrong caller. + await release_owner_value( + self._lock, + project_id=str(project_id), + session_id=request.session_id, + owner_value=incumbent_value, + ) + owner = await claim_owner( + self._lock, + project_id=str(project_id), + session_id=request.session_id, + replica_id=request.replica_id, + turn_id=request.turn_id, + ) + if owner == request.replica_id: + log.info( + "sessions: reclaimed session affinity from a replica with no running turn", + extra={ + "session_id": request.session_id, + "departed_replica_id": incumbent, + "replica_id": request.replica_id, + "turn_id": request.turn_id, + }, + ) + return owner + async def heartbeat( self, *, @@ -419,6 +562,49 @@ async def heartbeat( """ _validate_session_id(request.session_id) + # The shutdown beat: hand the affinity key back and touch nothing else. It runs FIRST, + # before the superseded check and before any lock is read or written, because a + # departing runner asserts nothing about turns — it only stops holding the session. + # `clear_owner` is release-if-owner, so a beat from a replica that no longer owns the + # session is a no-op and can never take affinity from a live one. Without this the + # next replica is refused for the rest of OWNER_TTL_SECONDS (`claim_owner` never + # steals), which on the local sandbox provider is a two-minute outage after every + # runner restart. + if request.release_owner: + released = await clear_owner( + self._lock, + project_id=str(project_id), + session_id=request.session_id, + replica_id=request.replica_id, + ) + stream = await self._dao.get_by_session_id( + project_id=project_id, + session_id=request.session_id, + ) + owner = await get_owner( + self._lock, + project_id=str(project_id), + session_id=request.session_id, + ) + log.info( + "sessions: released session ownership", + extra={ + "session_id": request.session_id, + "replica_id": request.replica_id, + "released": released, + "owner_after": owner, + }, + ) + # `replica_id` means "who owns this session now". After a successful release + # nobody does, and the caller is the one entitled to hear that, so report the + # caller's own id rather than inventing an owner. `is_current_turn` is False + # because this beat refreshed no turn. + return SessionHeartbeatResult( + stream=stream, + replica_id=owner or request.replica_id, + is_current_turn=False, + ) + # A turn that was already displaced (handover, cancel, steer, kill, sweep) is dead # forever: refuse the beat before it touches ANY lock or the row. This is what keeps # the ambiguous "`alive` held by another turn + no `running`" state safe to resolve as @@ -455,12 +641,23 @@ async def heartbeat( # replica_id claims affinity without stealing from a live different owner; turn_id # separately refreshes the alive/running TTLs. `owner` is the actual winner (this # replica if it won or already held it, another replica otherwise). - owner = await claim_owner( + owner_value = await claim_owner_value( self._lock, project_id=str(project_id), session_id=request.session_id, replica_id=request.replica_id, + turn_id=request.turn_id, ) + owner = owner_replica_id(owner_value) + # A different replica holds affinity. That claim is worth honouring only while it + # protects a turn, so before refusing, check whether it still protects one. + if owner != request.replica_id: + owner = await self._reclaim_affinity_from_a_departed_replica( + project_id=project_id, + request=request, + incumbent_value=owner_value, + ) + # A replica that lost the claim owns nothing here: mutating the nest would let it # overwrite the winner's turn locks and stream row. Report the true owner and stop. if owner != request.replica_id: @@ -510,7 +707,7 @@ async def heartbeat( session_id=request.session_id, turn_id=request.turn_id, ): - acquired = await acquire_alive( + acquired = await acquire_alive_with_start( self._lock, project_id=str(project_id), session_id=request.session_id, @@ -558,7 +755,7 @@ async def heartbeat( session_id=request.session_id, turn_id=displaced, ) - acquired = await acquire_alive( + acquired = await acquire_alive_with_start( self._lock, project_id=str(project_id), session_id=request.session_id, @@ -566,6 +763,13 @@ async def heartbeat( ) if not acquired or turn_was_established: is_current_turn = False + if is_current_turn: + await record_turn_start( + self._lock, + project_id=str(project_id), + session_id=request.session_id, + turn_id=request.turn_id, + ) if not await refresh_running( self._lock, project_id=str(project_id), @@ -688,8 +892,21 @@ async def heartbeat( project_id=project_id, user_id=None, session_id=request.session_id, - stream=SessionStreamEdit(flags=flags, turn_id=durable_turn_id), + stream=SessionStreamEdit( + flags=flags, + turn_id=durable_turn_id, + expected_turn_id=request.turn_id if turn_was_established else None, + ), ) + if stream is None and turn_was_established: + # The guarded row write lost to settlement or to a new generation. Redis may + # already have been refreshed, but this beat no longer owns durable state and + # must tell the runner to stop. + is_current_turn = False + stream = await self._dao.get_by_session_id( + project_id=project_id, + session_id=request.session_id, + ) # `running` lifecycle for the path that actually runs turns. `_start_turn` publishes it # for send/steer, but the runner mints its own turn id and only ever heartbeats, so @@ -945,7 +1162,7 @@ async def _start_turn( name: Optional[str] = None, ) -> str: turn_id = str(uuid.uuid7()) - acquired = await acquire_alive( + acquired = await acquire_alive_with_start( self._lock, project_id=str(project_id), session_id=session_id, @@ -1041,6 +1258,33 @@ async def _start_turn( await self._publish_changed(project_id=project_id, session_id=session_id) return turn_id + async def mirror_liveness( + self, + *, + project_id: UUID, + session_id: str, + user_id: Optional[UUID] = None, + ) -> None: + """Write the Redis nest onto the row, for a caller that changed the nest itself. + + Durable Stop settlement is that caller, and it is the one nest change no heartbeat can + mirror. Settlement tombstones the stopped execution BEFORE it releases `running`, so the + runner's own final `is_running=false` beat is refused by the tombstone check in + `heartbeat` above and returns before the mirror write at the end of that method. The + order cannot be swapped: a late beat that found `alive` free would take it straight back + under the dead turn's id. Without this method the row therefore keeps `is_running: true` + until the orphan sweep collapses it minutes later, and `query_streams` reads Postgres + alone, so the tab that pressed Stop sees its own session running somewhere else. + + Re-reads Redis rather than writing a literal `false`, so a newer turn that has already + taken `running` is reported, not erased. + """ + await self._mirror_flags( + project_id=project_id, + user_id=user_id, + session_id=session_id, + ) + async def _mirror_flags( self, *, diff --git a/api/oss/src/core/sessions/streams/types.py b/api/oss/src/core/sessions/streams/types.py index d55490c499a..feea49415f8 100644 --- a/api/oss/src/core/sessions/streams/types.py +++ b/api/oss/src/core/sessions/streams/types.py @@ -1,5 +1,7 @@ """Domain exceptions for session streams.""" +from typing import Optional + class SessionStreamError(Exception): """Base exception for session stream errors.""" @@ -36,6 +38,40 @@ def __init__(self, session_id: str, liveness: dict): super().__init__(self.message) +class SessionTurnMismatch(SessionStreamError): + """Raised when a cancel would displace a turn the caller did not mean to cancel. + + Two ways to get here, one meaning: the Stop is stale. Either the caller named a turn + (`expected_execution_id`) and a different one now holds the session, or the caller named + none and the holding turn started after the cancel arrived. Both are the stop-then-send + race: the turn the user meant has already ended and the next one has taken the session. + """ + + def __init__( + self, + session_id: str, + *, + actual_turn_id: Optional[str] = None, + expected_turn_id: Optional[str] = None, + ) -> None: + self.session_id = session_id + self.actual_turn_id = actual_turn_id + self.expected_turn_id = expected_turn_id + if expected_turn_id: + self.message = ( + f"Session '{session_id}' is running turn '{actual_turn_id}'," + f" not the expected turn '{expected_turn_id}'." + " Nothing was cancelled." + ) + else: + self.message = ( + f"Session '{session_id}' started turn '{actual_turn_id}' after this" + " cancel arrived, so the cancel is stale. Nothing was cancelled." + " Send `expected_execution_id` to cancel a specific turn." + ) + super().__init__(self.message) + + class ConcurrencyLimitExceeded(SessionStreamError): """Raised when the per-project concurrent-run limit is exceeded.""" diff --git a/api/oss/src/dbs/http/__init__.py b/api/oss/src/dbs/http/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/oss/src/dbs/http/sessions/__init__.py b/api/oss/src/dbs/http/sessions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/oss/src/dbs/http/sessions/control_delivery_direct.py b/api/oss/src/dbs/http/sessions/control_delivery_direct.py new file mode 100644 index 00000000000..dd470dcfb4d --- /dev/null +++ b/api/oss/src/dbs/http/sessions/control_delivery_direct.py @@ -0,0 +1,81 @@ +"""The direct-call control-delivery adapter. + +The API posts the command to the runner's own `/cancel`, over the same authenticated hop that +already carries hard kill. There is no held connection, no poll loop and no per-session Redis +channel: one runner process, one request. + +WHAT THIS ADAPTER IS NOT ALLOWED TO DO. Durability, authorization, idempotency, the state +machine and terminal settlement all live in `SessionCommandsService`. This file is transport. +Replacing it with a long-poll adapter must change no route, no data shape and no transition. + +THE ORDER IS NOT NEGOTIABLE. The command row is committed BEFORE `deliver` is called. Calling +first and recording afterwards would give back every failure the record exists to close: a crash +between the call and the insert leaves an aborted execution with no terminal outcome written +anywhere. + +WHERE IT FAILS, AND HOW THAT IS MADE LOUD. `env.runner.internal_url` is one service address. +Behind a load balancer with two runner replicas the call reaches the right process only by luck. +That failure is quiet at the transport level, because the wrong process honestly answers "I do +not hold that session" — the same answer a session that really ended gives. + +The detector is exact, and it is NOT in this file. A `not_held` for a session whose row says +alive with a heartbeat younger than one interval means some process is running that session and +it is not the one we just called; nothing else produces that. It needs the session row, so it +lives in `SessionCommandsService._settle_not_held`, next to the settlement it decides: the +command settles `lost` rather than `not_running`, so the user is told the Stop failed instead of +being told the work had already finished. + +There is deliberately no replica census here. An earlier version counted the replica ids that +had heartbeated recently and refused to deliver when it saw more than one. It refused after +every ordinary runner restart, because a runner mints a fresh id at boot when +`AGENTA_RUNNER_REPLICA_ID` is unset, so its own previous id was still inside the window. That +broke Stop for the whole window after every deploy, which is worse than the failure it guarded. +""" + +from typing import Optional +from uuid import UUID + +from oss.src.core.sessions.commands.dtos import SessionCommand +from oss.src.core.sessions.commands.interfaces import ( + ControlDeliveryPort, + DeliveryReceipt, +) +from oss.src.core.sessions.streams.runner_client import ( + RunnerCancelResult, + cancel_runner_execution, +) +from oss.src.utils.env import env +from oss.src.utils.logging import get_module_logger + +log = get_module_logger(__name__) + + +class DirectControlDelivery(ControlDeliveryPort): + def __init__(self, *, timeout_seconds: Optional[float] = None) -> None: + self._timeout = ( + timeout_seconds + if timeout_seconds is not None + else env.agenta.sessions.commands.delivery_timeout_seconds + ) + + async def deliver(self, *, command: SessionCommand) -> DeliveryReceipt: + answer = await cancel_runner_execution( + command_id=str(command.id), + project_id=str(command.project_id), + session_id=command.session_id, + target_turn_id=command.target_turn_id, + created_at=command.created_at.isoformat() if command.created_at else "", + timeout_seconds=self._timeout, + ) + if answer.status == RunnerCancelResult.accepted: + # The answering replica's own id, so the claim the service writes matches the id + # the runner reports its outcome with. + return DeliveryReceipt(status="accepted", replica_id=answer.replica_id) + if answer.status == RunnerCancelResult.not_held: + return DeliveryReceipt(status="not_held") + return DeliveryReceipt(status="unreachable") + + async def acknowledge(self, *, command_id: UUID, replica_id: str) -> None: + """A no-op: the claim compare-and-set in the DAO IS the acknowledgement, and the direct + adapter keeps no delivery bookkeeping of its own.""" + return None diff --git a/api/oss/src/dbs/postgres/sessions/commands/__init__.py b/api/oss/src/dbs/postgres/sessions/commands/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/oss/src/dbs/postgres/sessions/commands/dao.py b/api/oss/src/dbs/postgres/sessions/commands/dao.py new file mode 100644 index 00000000000..482e33d0e7f --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/commands/dao.py @@ -0,0 +1,505 @@ +"""Storage for durable session commands. + +Every state transition is one `UPDATE ... WHERE RETURNING *`, decided by +`scalar_one_or_none()`. That is what makes two API replicas unable to both win a claim or both +write a terminal outcome, and it is the same pattern +`SessionInteractionsDAO.transition_interaction` already uses. +""" + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional +from uuid import UUID + +from sqlalchemy import and_, func, or_, select, update as sa_update +from sqlalchemy.exc import IntegrityError + +from oss.src.utils.logging import get_module_logger + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandKind, + SessionCommandSettle, + SessionCommandState, +) +from oss.src.core.sessions.commands.interfaces import ( + CommandCreateResult, + SessionCommandsDAOInterface, + SessionScope, +) +from oss.src.dbs.postgres.sessions.commands.dbes import SessionCommandDBE +from oss.src.dbs.postgres.sessions.commands.mappings import ( + map_command_dbe_to_dto, + map_command_dto_to_dbe_create, +) +from oss.src.dbs.postgres.sessions.streams.dbes import SessionStreamDBE +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) + +log = get_module_logger(__name__) + +_OPEN_STATES = (SessionCommandState.pending.value, SessionCommandState.claimed.value) + + +def _map_commands_skipping_unmappable( + rows: List[SessionCommandDBE], + *, + context: str, +) -> List[SessionCommand]: + """Map a batch of command rows to DTOs, skipping any row this API cannot map. + + A newer API replica can write a command `kind` (or state, or outcome) an older replica's + enums do not know; `map_command_dbe_to_dto` then raises `ValueError` on that row. Both the + abandoned-command sweep and a runner's claim read a whole batch before acting on any of it, + so one such row used to poison the entire batch -- the ValueError escaped the list + comprehension and nothing was settled or claimed. Skip the rows this API cannot act on, + warn once per batch with their kinds and count, and return the rest. The unknown row is + left untouched for a replica that knows its kind; this never changes the enum or the write + path. `context` names the batch in the warning (for example "abandoned" or "claimed"). + """ + mapped: List[SessionCommand] = [] + skipped: Dict[str, int] = {} + for dbe in rows: + try: + mapped.append(map_command_dbe_to_dto(dbe)) + except ValueError: + kind = str(dbe.kind) + skipped[kind] = skipped.get(kind, 0) + 1 + if skipped: + by_kind = ", ".join( + f"{kind}={count}" for kind, count in sorted(skipped.items()) + ) + log.warning( + "commands: skipped %d %s row(s) this API cannot map (by kind: %s)", + sum(skipped.values()), + context, + by_kind, + ) + return mapped + + +class SessionCommandsDAO(SessionCommandsDAOInterface): + def __init__(self, engine: TransactionsEngine = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine + + def transaction(self): + return self.engine.session() + + async def create_command( + self, + *, + user_id: Optional[UUID], + command: SessionCommandCreate, + stopping_turn_id: Optional[str] = None, + ) -> SessionCommand: + result = await self.create_command_with_status( + user_id=user_id, + command=command, + stopping_turn_id=stopping_turn_id, + ) + return result.command + + async def create_command_with_status( + self, + *, + user_id: Optional[UUID], + command: SessionCommandCreate, + stopping_turn_id: Optional[str] = None, + ) -> CommandCreateResult: + """Insert the command and stamp the session row's `stopping_turn_id` together. + + One transaction, on purpose. A user whose Stop was recorded but whose session row never + learned it is waiting has a session that renders as plainly running while a command + exists to stop it, and nothing later reconciles the two. + + `session_streams` is written from here rather than through the streams DAO because + sharing one transaction is the whole requirement, and the streams DAO opens its own. + """ + dbe = map_command_dto_to_dbe_create(user_id=user_id, command=command) + + try: + async with self.engine.session() as session: + session.add(dbe) + if stopping_turn_id is not None: + await session.execute( + sa_update(SessionStreamDBE) + .where( + SessionStreamDBE.project_id == command.project_id, + SessionStreamDBE.session_id == command.session_id, + SessionStreamDBE.deleted_at.is_(None), + ) + .values(stopping_turn_id=stopping_turn_id) + ) + await session.commit() + await session.refresh(dbe) + return CommandCreateResult( + command=map_command_dbe_to_dto(dbe), inserted=True + ) + except IntegrityError: + # One of two unique constraints refused this insert, and both mean the same thing: + # a command for this intent already exists. Return it rather than a second command. + # + # uq_session_commands_idempotency — the caller retried with the same key. + # uq_session_commands_open_target — another request is already stopping this + # execution, which is what makes two Stops in + # the SAME INSTANT one command. Admission's own + # read cannot see a row that has not committed + # yet, so the database is the decider. + if command.idempotency_key is not None: + existing = await self.fetch_by_idempotency_key( + project_id=command.project_id, + session_id=command.session_id, + idempotency_key=command.idempotency_key, + ) + if existing is not None: + return CommandCreateResult(command=existing, inserted=False) + open_command = await self.fetch_open_command( + project_id=command.project_id, + session_id=command.session_id, + kind=command.kind, + target_turn_id=command.target_turn_id, + ) + if open_command is None: + raise + return CommandCreateResult(command=open_command, inserted=False) + + async def fetch_by_idempotency_key( + self, + *, + project_id: UUID, + session_id: str, + idempotency_key: str, + ) -> Optional[SessionCommand]: + async with self.engine.session() as session: + stmt = select(SessionCommandDBE).where( + SessionCommandDBE.project_id == project_id, + SessionCommandDBE.session_id == session_id, + SessionCommandDBE.idempotency_key == idempotency_key, + ) + result = await session.execute(stmt) + dbe = result.scalar_one_or_none() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def fetch_open_command( + self, + *, + project_id: UUID, + session_id: str, + kind: SessionCommandKind, + target_turn_id: Optional[str], + ) -> Optional[SessionCommand]: + async with self.engine.session() as session: + stmt = ( + select(SessionCommandDBE) + .where( + SessionCommandDBE.project_id == project_id, + SessionCommandDBE.session_id == session_id, + SessionCommandDBE.kind == kind.value, + SessionCommandDBE.state.in_(_OPEN_STATES), + SessionCommandDBE.deleted_at.is_(None), + ( + SessionCommandDBE.target_turn_id.is_(None) + if target_turn_id is None + else SessionCommandDBE.target_turn_id == target_turn_id + ), + ) + .order_by(SessionCommandDBE.created_at.desc()) + .limit(1) + ) + result = await session.execute(stmt) + dbe = result.scalar_one_or_none() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def fetch_command( + self, + *, + command_id: UUID, + project_id: Optional[UUID] = None, + ) -> Optional[SessionCommand]: + async with self.engine.session() as session: + stmt = select(SessionCommandDBE).where( + SessionCommandDBE.id == command_id, + ) + if project_id is not None: + stmt = stmt.where(SessionCommandDBE.project_id == project_id) + result = await session.execute(stmt) + dbe = result.scalars().first() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def claim_commands( + self, + *, + sessions: List[SessionScope], + replica_id: str, + lease_seconds: int, + limit: int, + ) -> List[SessionCommand]: + """Take pending commands for the sessions the caller declares it holds warm. + + The runner declaring what it holds is the routing input, not a replica id: a parked + session's Redis owner key expires, but the session is still in the runner's pool. + """ + if not sessions or limit <= 0: + return [] + + scope_filter = or_( + *[ + and_( + SessionCommandDBE.project_id == scope.project_id, + SessionCommandDBE.session_id == scope.session_id, + ) + for scope in sessions + ] + ) + + async with self.engine.session() as session: + selectable = ( + select(SessionCommandDBE.project_id, SessionCommandDBE.id) + .where( + SessionCommandDBE.state == SessionCommandState.pending.value, + SessionCommandDBE.deleted_at.is_(None), + scope_filter, + ) + .order_by(SessionCommandDBE.created_at) + .limit(limit) + # Two API replicas serving two claims at the same time must neither block on + # each other nor hand out the same command twice. + .with_for_update(skip_locked=True) + ) + rows = (await session.execute(selectable)).all() + if not rows: + await session.commit() + return [] + + keys = or_( + *[ + and_( + SessionCommandDBE.project_id == row[0], + SessionCommandDBE.id == row[1], + ) + for row in rows + ] + ) + now = datetime.now(timezone.utc) + stmt = ( + sa_update(SessionCommandDBE) + .where( + keys, + SessionCommandDBE.state == SessionCommandState.pending.value, + ) + .values( + state=SessionCommandState.claimed.value, + claimed_by=replica_id, + claim_expires_at=now + timedelta(seconds=lease_seconds), + claim_count=SessionCommandDBE.claim_count + 1, + updated_at=now, + ) + .returning(SessionCommandDBE) + ) + claimed = (await session.execute(stmt)).scalars().all() + await session.commit() + return _map_commands_skipping_unmappable(claimed, context="claimed") + + async def claim_for_delivery( + self, + *, + project_id: UUID, + command_id: UUID, + replica_id: str, + lease_seconds: int, + ) -> Optional[SessionCommand]: + """`pending` to `claimed` for one named command, after a runner accepted it directly. + + None means somebody else already took or settled it, which is not an error: the runner + that answered will still report, and the outcome route decides on the stored state. + The delivery budget was already consumed by `record_delivery_attempt`; incrementing it + again here would charge one direct delivery twice. Long-poll claims use `claim_commands`, + which performs its own increment. + """ + async with self.engine.session() as session: + now = datetime.now(timezone.utc) + stmt = ( + sa_update(SessionCommandDBE) + .where( + SessionCommandDBE.project_id == project_id, + SessionCommandDBE.id == command_id, + SessionCommandDBE.state == SessionCommandState.pending.value, + ) + .values( + state=SessionCommandState.claimed.value, + claimed_by=replica_id, + claim_expires_at=now + timedelta(seconds=lease_seconds), + updated_at=now, + ) + .returning(SessionCommandDBE) + ) + result = await session.execute(stmt) + dbe = result.scalar_one_or_none() + await session.commit() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def record_delivery_attempt( + self, + *, + project_id: UUID, + command_id: UUID, + now: datetime, + max_deliveries: int, + ) -> Optional[SessionCommand]: + stmt = ( + sa_update(SessionCommandDBE) + .where( + SessionCommandDBE.project_id == project_id, + SessionCommandDBE.id == command_id, + SessionCommandDBE.state.in_(_OPEN_STATES), + SessionCommandDBE.claim_count < max_deliveries, + or_( + SessionCommandDBE.state == SessionCommandState.pending.value, + SessionCommandDBE.claim_expires_at < now, + ), + ) + .values( + state=SessionCommandState.pending.value, + claimed_by=None, + claim_expires_at=None, + claim_count=SessionCommandDBE.claim_count + 1, + updated_at=now, + ) + .returning(SessionCommandDBE) + ) + async with self.engine.session() as session: + result = await session.execute(stmt) + dbe = result.scalar_one_or_none() + await session.commit() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def settle_command( + self, + *, + settle: SessionCommandSettle, + transaction: Optional[Any] = None, + ) -> Optional[SessionCommand]: + """Terminal transition. None means the command was in none of the states the caller + expected, so the caller reads the stored row and answers 409 instead of letting a runner + retry. + + One statement, so the guard is evaluated at the moment of the write. Reading the state + first and updating after would reopen the very race this exists to close: the claim can + commit between the read and the write. + """ + + async def execute(session: Any) -> Optional[SessionCommand]: + now = datetime.now(timezone.utc) + stmt = sa_update(SessionCommandDBE).where( + SessionCommandDBE.project_id == settle.project_id, + SessionCommandDBE.id == settle.command_id, + SessionCommandDBE.state.in_( + [state.value for state in settle.expected_states] + ), + ) + if settle.replica_id is not None: + # Only the replica holding the claim may write the outcome. A row still + # `pending` holds no claim, and refusing it there is what turned a correct + # abort into a command the sweep later called lost. + stmt = stmt.where( + or_( + SessionCommandDBE.claimed_by.is_(None), + SessionCommandDBE.claimed_by == settle.replica_id, + ) + ) + stmt = stmt.values( + state=settle.state.value, + outcome=settle.outcome.value, + settled_at=now, + updated_at=now, + ).returning(SessionCommandDBE) + result = await session.execute(stmt) + dbe = result.scalar_one_or_none() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + if transaction is not None: + return await execute(transaction) + async with self.engine.session() as session: + return await execute(session) + + async def clear_stopping_turn( + self, + *, + project_id: UUID, + session_id: str, + turn_id: Optional[str] = None, + ) -> None: + async with self.engine.session() as session: + stmt = ( + sa_update(SessionStreamDBE) + .where( + SessionStreamDBE.project_id == project_id, + SessionStreamDBE.session_id == session_id, + ) + .values(stopping_turn_id=None) + ) + if turn_id is not None: + # Only clear OUR marker. A settlement that arrives after a second Stop was + # admitted must not tell the browser the newer Stop already finished. + stmt = stmt.where(SessionStreamDBE.stopping_turn_id == turn_id) + await session.execute(stmt) + await session.commit() + + async def expire_claims( + self, + *, + now: datetime, + max_deliveries: int, + pending_before: Optional[datetime] = None, + ) -> List[SessionCommand]: + async with self.engine.session() as session: + abandoned = and_( + SessionCommandDBE.state == SessionCommandState.claimed.value, + SessionCommandDBE.claim_expires_at < now, + ) + if pending_before is not None: + abandoned = or_( + abandoned, + and_( + SessionCommandDBE.state == SessionCommandState.pending.value, + func.coalesce( + SessionCommandDBE.updated_at, + SessionCommandDBE.created_at, + ) + < pending_before, + ), + ) + stmt = ( + select(SessionCommandDBE) + .where( + SessionCommandDBE.deleted_at.is_(None), + abandoned, + ) + .order_by( + func.coalesce( + SessionCommandDBE.claim_expires_at, + SessionCommandDBE.updated_at, + SessionCommandDBE.created_at, + ) + ) + .limit(200) + ) + result = await session.execute(stmt) + rows = result.scalars().all() + return _map_commands_skipping_unmappable(rows, context="abandoned") + + async def count_open(self, *, project_id: UUID, session_id: str) -> int: + """Open commands for a session. Diagnostics and tests only.""" + async with self.engine.session() as session: + stmt = select(func.count()).where( + SessionCommandDBE.project_id == project_id, + SessionCommandDBE.session_id == session_id, + SessionCommandDBE.state.in_(_OPEN_STATES), + SessionCommandDBE.deleted_at.is_(None), + ) + result = await session.execute(stmt) + return int(result.scalar() or 0) diff --git a/api/oss/src/dbs/postgres/sessions/commands/dbas.py b/api/oss/src/dbs/postgres/sessions/commands/dbas.py new file mode 100644 index 00000000000..0163f162bb7 --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/commands/dbas.py @@ -0,0 +1,52 @@ +from sqlalchemy import Column, Integer, String, TIMESTAMP + +from oss.src.dbs.postgres.shared.dbas import ( + DataDBA, + FlagsDBA, + IdentifierDBA, + LifecycleDBA, + MetaDBA, + ProjectScopeDBA, + TagsDBA, +) + + +class SessionCommandDBA( + ProjectScopeDBA, + LifecycleDBA, + IdentifierDBA, + DataDBA, + FlagsDBA, + TagsDBA, + MetaDBA, +): + """One durable request to change an execution. + + The delivery columns (`state`, `claimed_by`, `claim_expires_at`, `claim_count`) are flat + rather than nested in `data` because a claim query filters and orders on them and a JSON + blob cannot be indexed for that. Their names carry the grouping. + + `state` and `outcome` are never merged. `state` says where the COMMAND is; `outcome` says + what happened to the EXECUTION. + """ + + __abstract__ = True + + # Bare correlator, not a foreign key — the same rule every other sessions table follows. + session_id = Column(String, nullable=False) + kind = Column(String, nullable=False) + + # The execution the API resolved ONCE at admission and pinned. A turn that starts later has + # a different id, so a pinned command can never reach it. Null when nothing was running. + target_turn_id = Column(String, nullable=True) + # What the caller asserted, stored as sent, so a 409 stays explainable after the fact. + expected_turn_id = Column(String, nullable=True) + + state = Column(String, nullable=False) + claimed_by = Column(String, nullable=True) + claim_expires_at = Column(TIMESTAMP(timezone=True), nullable=True) + claim_count = Column(Integer, nullable=False, default=0, server_default="0") + + outcome = Column(String, nullable=True) + idempotency_key = Column(String, nullable=True) + settled_at = Column(TIMESTAMP(timezone=True), nullable=True) diff --git a/api/oss/src/dbs/postgres/sessions/commands/dbes.py b/api/oss/src/dbs/postgres/sessions/commands/dbes.py new file mode 100644 index 00000000000..f4a755aba9a --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/commands/dbes.py @@ -0,0 +1,80 @@ +from sqlalchemy import ( + CheckConstraint, + ForeignKeyConstraint, + Index, + PrimaryKeyConstraint, + UniqueConstraint, + text, +) + +from oss.src.dbs.postgres.shared.base import Base +from oss.src.dbs.postgres.sessions.commands.dbas import SessionCommandDBA + + +class SessionCommandDBE(Base, SessionCommandDBA): + __tablename__ = "session_commands" + + __table_args__ = ( + ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + PrimaryKeyConstraint("project_id", "id"), + # The caller's retry identity. Postgres treats nulls as distinct in a unique index, so a + # command with no client key never collides with another. + UniqueConstraint( + "project_id", + "session_id", + "idempotency_key", + name="uq_session_commands_idempotency", + ), + CheckConstraint("kind IN ('cancel')", name="ck_session_commands_kind"), + CheckConstraint( + "state IN ('pending', 'claimed', 'applied', 'obsolete')", + name="ck_session_commands_state", + ), + # ONE open command per target execution. Two Stops are one intent, and admission's + # read-then-insert cannot enforce that on its own: two requests that arrive in the same + # instant both find no open command and both insert. The database decides instead, and + # the DAO turns the losing insert into a read of the winner. + # + # `target_turn_id` is NULL only on a command that is inserted already settled, which the + # predicate excludes, so the fact that Postgres treats NULLs as distinct costs nothing. + Index( + "uq_session_commands_open_target", + "project_id", + "session_id", + "kind", + "target_turn_id", + unique=True, + postgresql_where=text( + "state IN ('pending', 'claimed') AND deleted_at IS NULL" + ), + ), + # The claim query's index, and the open-command collapse read at admission. Partial on + # the open states because a settled command is never claimed again. + Index( + "ix_session_commands_open", + "project_id", + "session_id", + "created_at", + postgresql_where=text( + "state IN ('pending', 'claimed') AND deleted_at IS NULL" + ), + ), + # The settlement sweep's index: expired leases, nothing else. + Index( + "ix_session_commands_claims", + "claim_expires_at", + postgresql_where=text("state = 'claimed' AND deleted_at IS NULL"), + ), + Index( + "ix_session_commands_project_session", + "project_id", + "session_id", + "created_at", + ), + # The runner reports an outcome with the command id ALONE (it holds no project + # credential), so that read needs an index that does not lead with the project. + Index( + "ix_session_commands_id", + "id", + ), + ) diff --git a/api/oss/src/dbs/postgres/sessions/commands/mappings.py b/api/oss/src/dbs/postgres/sessions/commands/mappings.py new file mode 100644 index 00000000000..65a36df1eca --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/commands/mappings.py @@ -0,0 +1,72 @@ +from typing import Optional +from uuid import UUID + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandKind, + SessionCommandOutcome, + SessionCommandState, +) +from oss.src.dbs.postgres.sessions.commands.dbes import SessionCommandDBE + + +def map_command_dto_to_dbe_create( + *, + user_id: Optional[UUID], + command: SessionCommandCreate, +) -> SessionCommandDBE: + return SessionCommandDBE( + project_id=command.project_id, + # + created_by_id=user_id, + # Stamped, not defaulted: the stale-Stop guard compares this value, so the row must + # carry exactly the instant that was compared. + **({"created_at": command.created_at} if command.created_at else {}), + # + session_id=command.session_id, + kind=command.kind.value, + target_turn_id=command.target_turn_id, + expected_turn_id=command.expected_turn_id, + # + state=command.state.value, + claim_count=0, + outcome=command.outcome.value if command.outcome else None, + settled_at=command.settled_at, + idempotency_key=command.idempotency_key, + # + data=command.data, + ) + + +def map_command_dbe_to_dto(dbe: SessionCommandDBE) -> SessionCommand: + return SessionCommand( + id=dbe.id, + # + created_at=dbe.created_at, + updated_at=dbe.updated_at, + deleted_at=dbe.deleted_at, + created_by_id=dbe.created_by_id, + updated_by_id=dbe.updated_by_id, + deleted_by_id=dbe.deleted_by_id, + # + project_id=dbe.project_id, + session_id=dbe.session_id, + kind=SessionCommandKind(dbe.kind), + # + target_turn_id=dbe.target_turn_id, + expected_turn_id=dbe.expected_turn_id, + data=dbe.data, + # + state=SessionCommandState(dbe.state), + claimed_by=dbe.claimed_by, + claim_expires_at=dbe.claim_expires_at, + claim_count=dbe.claim_count or 0, + # + outcome=SessionCommandOutcome(dbe.outcome) if dbe.outcome else None, + idempotency_key=dbe.idempotency_key, + settled_at=dbe.settled_at, + # + tags=dbe.tags, + meta=dbe.meta, + ) diff --git a/api/oss/src/dbs/postgres/sessions/executions/__init__.py b/api/oss/src/dbs/postgres/sessions/executions/__init__.py new file mode 100644 index 00000000000..d30a53d8fee --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/executions/__init__.py @@ -0,0 +1 @@ +"""Postgres execution terminal-state storage.""" diff --git a/api/oss/src/dbs/postgres/sessions/executions/dao.py b/api/oss/src/dbs/postgres/sessions/executions/dao.py new file mode 100644 index 00000000000..69c646835a5 --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/executions/dao.py @@ -0,0 +1,166 @@ +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Sequence, Tuple +from uuid import UUID + +from sqlalchemy import and_, literal_column, or_, select, tuple_, update as sa_update +from sqlalchemy.dialects.postgresql import insert + +from oss.src.core.sessions.executions.dtos import ( + SessionExecutionSettlement, + SessionExecutionSettlementResult, +) +from oss.src.core.sessions.executions.interfaces import SessionExecutionsDAOInterface +from oss.src.dbs.postgres.sessions.executions.dbes import SessionExecutionDBE +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) + + +def _to_dto(row: SessionExecutionDBE) -> SessionExecutionSettlement: + return SessionExecutionSettlement( + project_id=row.project_id, + session_id=row.session_id, + execution_id=row.execution_id, + terminal_outcome=row.terminal_outcome, + settled_by=row.settled_by, + settled_at=row.settled_at, + ending_written_at=row.ending_written_at, + redis_reconciled_at=row.redis_reconciled_at, + ) + + +class SessionExecutionsDAO(SessionExecutionsDAOInterface): + def __init__(self, engine: Optional[TransactionsEngine] = None): + self.engine = engine or get_transactions_engine() + + async def settle( + self, + *, + project_id: UUID, + session_id: str, + execution_id: str, + terminal_outcome: str, + settled_by: str, + settled_at: Optional[datetime] = None, + transaction: Optional[Any] = None, + ) -> SessionExecutionSettlementResult: + settled_at = settled_at or datetime.now(timezone.utc) + stmt = ( + insert(SessionExecutionDBE) + .values( + project_id=project_id, + session_id=session_id, + execution_id=execution_id, + terminal_outcome=terminal_outcome, + settled_by=settled_by, + settled_at=settled_at, + ) + .on_conflict_do_update( + index_elements=["project_id", "session_id", "execution_id"], + set_={"terminal_outcome": SessionExecutionDBE.terminal_outcome}, + ) + .returning( + SessionExecutionDBE, + literal_column("xmax = 0").label("won"), + ) + ) + + async def execute(session: Any) -> SessionExecutionSettlementResult: + stored, won = (await session.execute(stmt)).one() + return SessionExecutionSettlementResult(settlement=_to_dto(stored), won=won) + + if transaction is not None: + return await execute(transaction) + async with self.engine.session() as session: + return await execute(session) + + async def query_settled( + self, + *, + project_id: UUID, + keys: Sequence[Tuple[str, str]], + ) -> Dict[Tuple[str, str], SessionExecutionSettlement]: + if not keys: + return {} + key_filter = or_( + *[ + and_( + SessionExecutionDBE.session_id == session_id, + SessionExecutionDBE.execution_id == execution_id, + ) + for session_id, execution_id in keys + ] + ) + async with self.engine.session() as session: + rows = ( + await session.execute( + select(SessionExecutionDBE).where( + SessionExecutionDBE.project_id == project_id, + key_filter, + ) + ) + ).scalars() + return {(row.session_id, row.execution_id): _to_dto(row) for row in rows} + + async def mark_endings_written( + self, + *, + project_id: UUID, + keys: Sequence[Tuple[str, str]], + written_at: Optional[datetime] = None, + ) -> None: + if not keys: + return + async with self.engine.session() as session: + await session.execute( + sa_update(SessionExecutionDBE) + .where( + SessionExecutionDBE.project_id == project_id, + tuple_( + SessionExecutionDBE.session_id, + SessionExecutionDBE.execution_id, + ).in_(keys), + SessionExecutionDBE.ending_written_at.is_(None), + ) + .values(ending_written_at=written_at or datetime.now(timezone.utc)) + ) + + async def list_redis_unreconciled( + self, + *, + limit: int, + ) -> List[SessionExecutionSettlement]: + async with self.engine.session() as session: + rows = ( + await session.execute( + select(SessionExecutionDBE) + .where( + SessionExecutionDBE.settled_by == "runner", + SessionExecutionDBE.terminal_outcome == "stopped", + SessionExecutionDBE.redis_reconciled_at.is_(None), + ) + .order_by(SessionExecutionDBE.settled_at) + .limit(limit) + ) + ).scalars() + return [_to_dto(row) for row in rows] + + async def mark_redis_reconciled( + self, + *, + project_id: UUID, + session_id: str, + execution_id: str, + ) -> None: + async with self.engine.session() as session: + await session.execute( + sa_update(SessionExecutionDBE) + .where( + SessionExecutionDBE.project_id == project_id, + SessionExecutionDBE.session_id == session_id, + SessionExecutionDBE.execution_id == execution_id, + SessionExecutionDBE.redis_reconciled_at.is_(None), + ) + .values(redis_reconciled_at=datetime.now(timezone.utc)) + ) diff --git a/api/oss/src/dbs/postgres/sessions/executions/dbes.py b/api/oss/src/dbs/postgres/sessions/executions/dbes.py new file mode 100644 index 00000000000..2a13846bb91 --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/executions/dbes.py @@ -0,0 +1,48 @@ +from sqlalchemy import ( + Column, + ForeignKeyConstraint, + Index, + PrimaryKeyConstraint, + String, + text, +) +from sqlalchemy import TIMESTAMP +from sqlalchemy.dialects.postgresql import UUID + +from oss.src.dbs.postgres.shared.base import Base + + +class SessionExecutionDBE(Base): + __tablename__ = "session_executions" + + project_id = Column(UUID(as_uuid=True), nullable=False) + session_id = Column(String, nullable=False) + execution_id = Column(String, nullable=False) + terminal_outcome = Column(String, nullable=False) + settled_by = Column(String, nullable=False) + settled_at = Column(TIMESTAMP(timezone=True), nullable=False) + ending_written_at = Column(TIMESTAMP(timezone=True), nullable=True) + redis_reconciled_at = Column(TIMESTAMP(timezone=True), nullable=True) + + __table_args__ = ( + ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + PrimaryKeyConstraint("project_id", "session_id", "execution_id"), + Index( + "ix_session_executions_project_session", + "project_id", + "session_id", + ), + Index( + "ix_session_executions_ending_unwritten", + "settled_at", + postgresql_where=text("ending_written_at IS NULL"), + ), + Index( + "ix_session_executions_redis_unreconciled", + "settled_at", + postgresql_where=text( + "settled_by = 'runner' AND terminal_outcome = 'stopped' " + "AND redis_reconciled_at IS NULL" + ), + ), + ) diff --git a/api/oss/src/dbs/postgres/sessions/interactions/dao.py b/api/oss/src/dbs/postgres/sessions/interactions/dao.py index 97043a77b46..ef46fcbd0a8 100644 --- a/api/oss/src/dbs/postgres/sessions/interactions/dao.py +++ b/api/oss/src/dbs/postgres/sessions/interactions/dao.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta, timezone -from typing import List, Optional +from typing import Any, List, Optional from uuid import UUID from sqlalchemy import cast, delete as sa_delete, func, select, update as sa_update @@ -142,13 +142,15 @@ async def cancel_session_pending( except_turn_id: Optional[str] = None, except_tokens: Optional[List[str]] = None, only_turn_id: Optional[str] = None, - ) -> int: + transaction: Optional[Any] = None, + ) -> List[SessionInteraction]: """Cancel still-pending interactions for a session. With `except_turn_id`, spare the current turn's own gates (used at turn start to cancel prior turns' unanswered gates; without it, cancel all of them, e.g. on kill). `except_tokens` spares prior-turn gates the current turn answers in-band, so the resume can resolve them instead. With - `only_turn_id`, touch nothing but that one turn's gates. Returns the count cancelled.""" - async with self.engine.session() as session: + `only_turn_id`, touch nothing but that one turn's gates. Returns the rows cancelled.""" + + async def execute(session: Any) -> List[SessionInteraction]: stmt = ( sa_update(SessionInteractionDBE) .where( @@ -160,6 +162,7 @@ async def cancel_session_pending( status="cancelled", updated_at=datetime.now(timezone.utc), ) + .returning(SessionInteractionDBE) ) if only_turn_id is not None: stmt = stmt.where(SessionInteractionDBE.turn_id == only_turn_id) @@ -168,8 +171,14 @@ async def cancel_session_pending( if except_tokens: stmt = stmt.where(SessionInteractionDBE.token.notin_(except_tokens)) result = await session.execute(stmt) + return [map_interaction_dbe_to_dto(dbe) for dbe in result.scalars().all()] + + if transaction is not None: + return await execute(transaction) + async with self.engine.session() as session: + cancelled = await execute(session) await session.commit() - return result.rowcount or 0 + return cancelled async def query_interactions( self, diff --git a/api/oss/src/dbs/postgres/sessions/records/dao.py b/api/oss/src/dbs/postgres/sessions/records/dao.py index 3f7a08c9491..c91ece59ec6 100644 --- a/api/oss/src/dbs/postgres/sessions/records/dao.py +++ b/api/oss/src/dbs/postgres/sessions/records/dao.py @@ -1,12 +1,14 @@ -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Sequence, Set, Tuple from uuid import UUID -from sqlalchemy import func, select +from sqlalchemy import func, select, tuple_ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession from oss.src.core.sessions.records.dtos import ( + RECORD_SETTLED_BY_ATTRIBUTE, SESSION_MESSAGE_PREVIEW_TEXT_LIMIT, + TERMINAL_RECORD_TYPE, SessionMessagePreview, SessionRecord, SessionRecordEvent, @@ -94,6 +96,7 @@ def _values(*, event: SessionRecordEvent) -> dict: "attributes", "turn_id", "span_id", + "quarantined_at", ) @staticmethod @@ -132,6 +135,14 @@ def _upsert_stmt(*, values_list: List[dict]): "attributes": stmt.excluded.attributes, "turn_id": stmt.excluded.turn_id, "span_id": stmt.excluded.span_id, + # coalesce, not a plain overwrite: quarantine is one-way. A redelivery of a + # late record keeps the instant it was FIRST quarantined, so the column is + # stable however many times the stream replays the message, and a delivery + # that somehow arrives unmarked can never resurrect the row into the + # transcript. + "quarantined_at": func.coalesce( + RecordDBE.quarantined_at, stmt.excluded.quarantined_at + ), }, ).returning(RecordDBE) @@ -147,6 +158,11 @@ async def get_records( .where( RecordDBE.project_id == project_id, RecordDBE.session_id == session_id, + # A quarantined record is history the platform refused: it reached ingest + # for a turn the watchdog had already ended. Excluding it HERE is what + # makes one execution render one ending, because this is the read every + # transcript reconstruction goes through. + RecordDBE.quarantined_at.is_(None), ) # Producer event time first: it is the only key that is monotonic across # turns. `record_index` restarts at 0 every turn, and the worker can batch @@ -200,6 +216,7 @@ async def latest_message_per_session( RecordDBE.session_id.in_(session_ids), RecordDBE.record_type == "message", RecordDBE.deleted_at.is_(None), + RecordDBE.quarantined_at.is_(None), ) .distinct(RecordDBE.session_id) .order_by( @@ -225,6 +242,59 @@ async def latest_message_per_session( ) return previews + async def settled_turns( + self, + *, + project_id: UUID, + keys: Sequence[Tuple[str, str]], + settled_by: Optional[str] = None, + ) -> Set[Tuple[str, str]]: + """Which of these `(session_id, turn_id)` pairs already carry a terminal record. + + Two callers ask nearly the same question and mean different things by it, which is + why `settled_by` exists rather than a second query. + + * The watchdog asks with no writer, before it writes an ending of its own: ANY + terminal record means this turn already ended and must not be given a second, + contradictory one. + * The ingest guard asks with `settled_by="watchdog"`, and only the watchdog's own + ending counts. A runner that wrote its honest ending has not lost the turn to the + platform, so nothing arriving afterwards is late in the sense that matters. + + A QUARANTINED terminal record never answers yes to either. It is precisely the + second, refused ending both callers exist to keep out of the transcript, so counting + it would let one late `done` suppress the real one. + + One query for the whole batch, served by + `ix_records_project_id_session_id_turn_id`. + """ + if not keys: + return set() + + conditions = [ + RecordDBE.project_id == project_id, + RecordDBE.record_type == TERMINAL_RECORD_TYPE, + RecordDBE.deleted_at.is_(None), + RecordDBE.quarantined_at.is_(None), + tuple_(RecordDBE.session_id, RecordDBE.turn_id).in_( + [(session_id, turn_id) for session_id, turn_id in keys] + ), + ] + if settled_by is not None: + conditions.append( + RecordDBE.attributes[RECORD_SETTLED_BY_ATTRIBUTE].astext == settled_by + ) + + async with self.engine.session() as session: + stmt = ( + select(RecordDBE.session_id, RecordDBE.turn_id) + .where(*conditions) + .distinct() + ) + rows = (await session.execute(stmt)).all() + + return {(row.session_id, row.turn_id) for row in rows} + async def get_event( self, *, diff --git a/api/oss/src/dbs/postgres/sessions/records/dbas.py b/api/oss/src/dbs/postgres/sessions/records/dbas.py index 200eae44bbd..a80167e60d4 100644 --- a/api/oss/src/dbs/postgres/sessions/records/dbas.py +++ b/api/oss/src/dbs/postgres/sessions/records/dbas.py @@ -65,3 +65,13 @@ class RecordDBA: JSONB(none_as_null=True), nullable=True, ) + + # Non-null when this record reached ingest for a turn the watchdog had ALREADY ended. + # The row is kept — the agent really did that work, and the token accounting on a late + # `usage` is real money — but every read that rebuilds a transcript excludes it, so one + # execution still shows exactly one ending. Written only by the ingest guard in + # `RecordsService.append_many`; forward-fill only, like every other column here. + quarantined_at = Column( + TIMESTAMP(timezone=True), + nullable=True, + ) diff --git a/api/oss/src/dbs/postgres/sessions/records/mappings.py b/api/oss/src/dbs/postgres/sessions/records/mappings.py index 62420cc97e4..dd758b5110f 100644 --- a/api/oss/src/dbs/postgres/sessions/records/mappings.py +++ b/api/oss/src/dbs/postgres/sessions/records/mappings.py @@ -25,6 +25,7 @@ def map_record_event_to_dbe( attributes=event.attributes, turn_id=event.turn_id, span_id=event.span_id, + quarantined_at=event.quarantined_at, ) @@ -40,5 +41,6 @@ def map_record_dbe_to_dto(*, dbe: RecordDBE) -> SessionRecord: attributes=dbe.attributes, turn_id=dbe.turn_id, span_id=dbe.span_id, + quarantined_at=dbe.quarantined_at, created_at=dbe.created_at, ) diff --git a/api/oss/src/dbs/postgres/sessions/streams/dao.py b/api/oss/src/dbs/postgres/sessions/streams/dao.py index a01dd8d58af..399db927ad3 100644 --- a/api/oss/src/dbs/postgres/sessions/streams/dao.py +++ b/api/oss/src/dbs/postgres/sessions/streams/dao.py @@ -1,5 +1,5 @@ from datetime import datetime, timezone -from typing import List, Optional +from typing import Any, List, Optional from uuid import UUID import uuid_utils.compat as uuid @@ -48,6 +48,7 @@ references_containment_json, references_to_json, ) +from oss.src.dbs.postgres.sessions.executions.dbes import SessionExecutionDBE from oss.src.dbs.postgres.sessions.streams.dbes import SessionStreamDBE from oss.src.dbs.postgres.sessions.streams.mappings import ( SESSION_ORIGIN_TAG_KEY, @@ -95,6 +96,42 @@ def __init__(self, engine: TransactionsEngine = None): engine = get_transactions_engine() self.engine = engine + async def settle_command( + self, + *, + project_id: UUID, + session_id: str, + turn_id: Optional[str], + mirror_stopped: bool, + transaction: Optional[Any] = None, + ) -> None: + now = datetime.now(timezone.utc) + values = {"stopping_turn_id": None, "updated_at": now} + if mirror_stopped: + values["flags"] = func.coalesce(SessionStreamDBE.flags, cast({}, JSONB)).op( + "||" + )(cast({"is_running": False}, JSONB)) + stmt = sa_update(SessionStreamDBE).where( + SessionStreamDBE.project_id == project_id, + SessionStreamDBE.session_id == session_id, + ) + if turn_id is not None: + stmt = stmt.where( + or_( + SessionStreamDBE.stopping_turn_id == turn_id, + SessionStreamDBE.stopping_turn_id.is_(None), + ) + ) + + async def execute(session: Any) -> None: + await session.execute(stmt.values(**values)) + + if transaction is not None: + await execute(transaction) + return + async with self.engine.session() as session: + await execute(session) + async def create( self, *, @@ -494,6 +531,48 @@ async def update( session_id: str, stream: SessionStreamEdit, ) -> Optional[SessionStream]: + if stream.expected_turn_id is not None: + terminal_execution_exists = ( + select(SessionExecutionDBE.execution_id) + .where( + SessionExecutionDBE.project_id == project_id, + SessionExecutionDBE.session_id == session_id, + SessionExecutionDBE.execution_id == stream.expected_turn_id, + ) + .exists() + ) + values = { + "updated_by_id": user_id, + "updated_at": datetime.now(timezone.utc), + } + if stream.flags is not None: + values["flags"] = stream.flags.model_dump(mode="json") + if stream.turn_id is not None: + values["turn_id"] = stream.turn_id + + async with self.engine.session() as session: + result = await session.execute( + sa_update(SessionStreamDBE) + .where( + SessionStreamDBE.project_id == project_id, + SessionStreamDBE.session_id == session_id, + SessionStreamDBE.deleted_at.is_(None), + SessionStreamDBE.turn_id == stream.expected_turn_id, + SessionStreamDBE.flags.contains( + {"is_alive": True, "is_running": True} + ), + ~terminal_execution_exists, + ) + .values(**values) + .returning(SessionStreamDBE) + .execution_options(synchronize_session=False) + ) + dbe = result.scalar_one_or_none() + await session.commit() + if dbe is None: + return None + return map_stream_dbe_to_dto(stream_dbe=dbe) + async with self.engine.session() as session: stmt = select(SessionStreamDBE).where( SessionStreamDBE.project_id == project_id, diff --git a/api/oss/src/dbs/postgres/sessions/streams/dbes.py b/api/oss/src/dbs/postgres/sessions/streams/dbes.py index 7dd82b6cb7d..47b7afb8e4e 100644 --- a/api/oss/src/dbs/postgres/sessions/streams/dbes.py +++ b/api/oss/src/dbs/postgres/sessions/streams/dbes.py @@ -63,6 +63,24 @@ class SessionStreamDBE( # (resumable, still listed); `archived_at` marks a deliberately-hidden one (restorable). archived_at = Column(TIMESTAMP(timezone=True), nullable=True) + # The execution an accepted Stop is waiting on. Written in the same transaction as the + # command insert, cleared at settlement. Null means nothing is stopping. + # + # A column and not a bit inside `flags`, because `flags` is the Redis mirror and every + # heartbeat rewrites it whole (`streams/service.py`, the unconditional mirror write), so a + # value stored there would be erased on the next beat. `SessionStreamEdit` carries only + # flags/tags/meta/turn_id, so the heartbeat path cannot touch this column by accident. + stopping_turn_id = Column(String, nullable=True) + + # When the row's CURRENT `turn_id` started. It exists for the stale-Stop guard, which has to + # compare a Stop's arrival time with the running execution's start time, and there was + # nowhere to read that: `updated_at` is the heartbeat timestamp and moves every 30 seconds, + # runner-minted turn ids are uuid4 and carry no time, the Redis lock value is a bare turn id + # that a Lua compare reads whole, and the `session_turns` append is fire-and-forget so a + # running turn may have no row. Stamped only when the id actually changes, so the repeated + # heartbeats that restamp the same id never move it. + turn_started_at = Column(TIMESTAMP(timezone=True), nullable=True) + __table_args__ = ( ForeignKeyConstraint( ["project_id"], diff --git a/api/oss/src/dbs/postgres/sessions/streams/mappings.py b/api/oss/src/dbs/postgres/sessions/streams/mappings.py index 2442b3e433f..33d43c872d4 100644 --- a/api/oss/src/dbs/postgres/sessions/streams/mappings.py +++ b/api/oss/src/dbs/postgres/sessions/streams/mappings.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from typing import Any, Dict, Optional from uuid import UUID @@ -135,6 +136,10 @@ def map_stream_dto_to_dbe_create( tags=stream.tags, meta=stream.meta, turn_id=stream.turn_id, + # A create that already names a turn IS that turn's start. Without this, the first row a + # `_start_turn` writes carries no start time and the stale-Stop guard cannot fire on the + # very first turn of a session. + turn_started_at=datetime.now(timezone.utc) if stream.turn_id else None, references=references_to_json(stream.references), ) @@ -157,6 +162,8 @@ def map_stream_dbe_to_dto( name=stream_dbe.name, description=stream_dbe.description, turn_id=stream_dbe.turn_id, + turn_started_at=stream_dbe.turn_started_at, + stopping_turn_id=stream_dbe.stopping_turn_id, references=references_from_json(stream_dbe.references), archived_at=stream_dbe.archived_at, flags=SessionStreamFlags.model_validate(stream_dbe.flags) @@ -199,6 +206,11 @@ def map_stream_dto_to_dbe_edit( if stream.meta is not None: stream_dbe.meta = stream.meta if stream.turn_id is not None: + # Stamp the start time only when the id actually CHANGES. A heartbeat restamps the same + # id every 30 seconds, and a start time that moved with each beat would make every Stop + # look like it arrived before its own turn began. + if stream_dbe.turn_id != stream.turn_id: + stream_dbe.turn_started_at = datetime.now(timezone.utc) stream_dbe.turn_id = stream.turn_id diff --git a/api/oss/src/dbs/redis/sessions/contract.py b/api/oss/src/dbs/redis/sessions/contract.py index f3c85dc15be..b62c8d05e69 100644 --- a/api/oss/src/dbs/redis/sessions/contract.py +++ b/api/oss/src/dbs/redis/sessions/contract.py @@ -8,13 +8,16 @@ alive::session: — session claimed; runner owns it running::session: — a turn is actively executing right now attached::session: — attach lock (client watching live view) - owner::session: — which replica currently owns this session + owner::session: — replica + turn generation owning this session displaced::session: — pub/sub for attach-steal notifications watch::session: — pub/sub for the live relay (SSE watch) superseded::session::turn: — tombstone: this turn lost the nest and is dead forever (API-side only; the runner learns it through `is_current_turn`) + started::session::turn: + — when this turn first took `alive`, in epoch + milliseconds (API-side only; see below) `session_id` is caller-supplied and Postgres uniqueness is (project_id, session_id), so two projects may legitimately hold the same one. The `project_id` segment is the tenant boundary: @@ -45,6 +48,25 @@ # deliberately absent from the shared golden fixture (like `watch_heartbeat_seconds`). SUPERSEDED_TTL_SECONDS: int = env.sessions.superseded_ttl_seconds +# API-side owner payload. The runner reaches affinity through the heartbeat response and never +# reads this Redis value directly. Unit Separator cannot occur in either UUID-like component and +# keeps legacy bare-replica values unambiguous. +OWNER_VALUE_SEPARATOR = "\x1f" + + +def make_owner_value(*, replica_id: str, turn_id: str | None) -> str: + return f"{replica_id}{OWNER_VALUE_SEPARATOR}{turn_id or ''}" + + +def owner_replica_id(owner_value: str) -> str: + return owner_value.split(OWNER_VALUE_SEPARATOR, 1)[0] + + +# The turn-start key lives exactly as long as `alive` can: it answers "did this turn start +# before that cancel arrived?", and a turn with no `alive` cannot be cancelled. Reusing +# ALIVE_TTL keeps the two in step without a new setting. +TURN_STARTED_TTL_SECONDS: int = ALIVE_TTL_SECONDS + # --------------------------------------------------------------------------- # Key builders # --------------------------------------------------------------------------- @@ -70,6 +92,18 @@ def superseded_key(project_id: str, session_id: str, turn_id: str) -> str: return f"superseded:{project_id}:session:{session_id}:turn:{turn_id}" +def turn_started_key(project_id: str, session_id: str, turn_id: str) -> str: + """When this turn first took the alive lock, in epoch milliseconds. + + API-side only, like the tombstone above: the runner never reads it, so it stays out of + the shared golden fixture. It exists because nothing else records a turn's start early + enough to be useful. `session_turns.start_time` is written by the runner some time after + the turn begins, and a browser turn's id is a runner-minted uuid4 + (`services/runner/src/server.ts:188`), so no timestamp can be read out of the id either. + """ + return f"started:{project_id}:session:{session_id}:turn:{turn_id}" + + def displaced_channel(project_id: str, session_id: str) -> str: return f"displaced:{project_id}:session:{session_id}" @@ -140,7 +174,7 @@ def make_watch_entity_changed_payload(*, entity: str, id: str) -> dict: # --------------------------------------------------------------------------- -# Release-if-owner Lua scripts +# Coordination Lua scripts # These are the canonical scripts; both Python and TS implementations must # use the same logic (same key/argv layout; different runtime bindings). # @@ -159,12 +193,142 @@ def make_watch_entity_changed_payload(*, entity: str, id: str) -> dict: end """.strip() -# Atomic claim-or-read: take ownership iff the key is absent or already ours (refreshing the -# TTL), never steal it from another replica. Returns the actual owner after the operation, so -# the caller learns who won without a second racy read. +# Atomically release only the generation the watchdog swept. A new Send or Steer may install +# another turn after the database commit, so every destructive Redis action must compare the +# value captured before the guarded stream update. The swept turn is tombstoned regardless of +# whether its old lock keys still exist. +WATCHDOG_RELEASE_TURN_LUA = """ +-- AGENTA_WATCHDOG_RELEASE_TURN +local expected_turn = ARGV[1] +local expected_owner = ARGV[2] +local superseded_ttl = tonumber(ARGV[3]) +local alive = redis.call('GET', KEYS[1]) or '' +local running = redis.call('GET', KEYS[2]) or '' +local owner = redis.call('GET', KEYS[3]) or '' +local released_alive = 0 +local released_running = 0 +local released_owner = 0 + +if expected_turn ~= '' and alive == expected_turn then + released_alive = redis.call('DEL', KEYS[1]) +end +if expected_turn ~= '' and running == expected_turn then + released_running = redis.call('DEL', KEYS[2]) +end + +local foreign_turn = (alive ~= '' and alive ~= expected_turn) + or (running ~= '' and running ~= expected_turn) +if expected_owner ~= '' and owner == expected_owner and not foreign_turn then + released_owner = redis.call('DEL', KEYS[3]) +end + +if expected_turn ~= '' then + redis.call('SET', KEYS[4], '1', 'EX', superseded_ttl) +end + +return {released_alive, released_running, released_owner} +""".strip() + +ACQUIRE_ALIVE_WITH_START_LUA = """ +-- AGENTA_ACQUIRE_ALIVE_WITH_START +if redis.call('GET', KEYS[1]) then + return 0 +end +local now = redis.call('TIME') +local now_ms = (tonumber(now[1]) * 1000) + math.floor(tonumber(now[2]) / 1000) +redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) +if redis.call('SET', KEYS[2], tostring(now_ms), 'NX', 'EX', ARGV[3]) == false then + redis.call('EXPIRE', KEYS[2], ARGV[3]) +end +return 1 +""".strip() + +DISPLACE_TURNS_LUA = """ +-- AGENTA_DISPLACE_TURNS +local alive = redis.call('GET', KEYS[1]) or '' +local running = redis.call('GET', KEYS[2]) or '' +local expected = ARGV[1] +local arrived_at_ms = tonumber(ARGV[2]) +local superseded_prefix = ARGV[3] +local started_prefix = ARGV[4] +local superseded_ttl = tonumber(ARGV[5]) +local running_only = ARGV[6] == '1' + +local function is_mismatch(owner) + if owner == '' then + return false + end + if expected ~= '' then + return owner ~= expected + end + if arrived_at_ms then + local started_at_ms = tonumber(redis.call('GET', started_prefix .. owner)) + return started_at_ms and started_at_ms > arrived_at_ms + end + return false +end + +if not running_only and is_mismatch(alive) then + return {0, alive} +end +if (running_only or running ~= alive) and is_mismatch(running) then + return {0, running} +end + +local seen = {} +local function supersede(turn_id) + if turn_id ~= '' and not seen[turn_id] then + redis.call('SET', superseded_prefix .. turn_id, '1', 'EX', superseded_ttl) + seen[turn_id] = true + end +end + +if not running_only then + supersede(alive) +end +supersede(running) +supersede(expected) +if running_only then + if alive == running and running ~= '' then + redis.call('DEL', KEYS[1]) + end + redis.call('DEL', KEYS[2]) +else + redis.call('DEL', KEYS[1], KEYS[2]) +end +local returned_alive = alive +if running_only then + returned_alive = '' +end +return {1, returned_alive, running, expected} +""".strip() + +# Atomically tombstone a durably stopped execution and release `running` only if that exact +# generation still owns it. `alive` deliberately survives so the native harness stays warm. +RECONCILE_STOPPED_TURN_LUA = """ +-- AGENTA_RECONCILE_STOPPED_TURN +local expected = ARGV[1] +redis.call('SET', KEYS[2], '1', 'EX', tonumber(ARGV[2])) +if redis.call('GET', KEYS[1]) == expected then + return redis.call('DEL', KEYS[1]) +end +return 0 +""".strip() + +# Atomic claim-or-read: take ownership iff the key is absent or already belongs to this replica, +# refreshing both its TTL and turn generation. Returns the full actual value without a second +# racy read. Bare legacy values compare as their own replica id and are upgraded on refresh. CLAIM_OWNER_LUA = """ local current = redis.call('GET', KEYS[1]) -if current == false or current == ARGV[1] then +local separator = string.char(31) +local function replica(value) + local boundary = string.find(value, separator, 1, true) + if boundary then + return string.sub(value, 1, boundary - 1) + end + return value +end +if current == false or replica(current) == replica(ARGV[1]) then redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) return ARGV[1] end diff --git a/api/oss/src/dbs/redis/sessions/locks.py b/api/oss/src/dbs/redis/sessions/locks.py index 8da9dcf9914..a3bc900d26b 100644 --- a/api/oss/src/dbs/redis/sessions/locks.py +++ b/api/oss/src/dbs/redis/sessions/locks.py @@ -6,24 +6,32 @@ """ import json -from typing import Optional +from typing import List, Optional, Tuple from oss.src.dbs.redis.shared.engine import LockEngine from oss.src.dbs.redis.sessions.contract import ( ALIVE_TTL_SECONDS, + ACQUIRE_ALIVE_WITH_START_LUA, ATTACHED_TTL_SECONDS, CLAIM_OWNER_LUA, + DISPLACE_TURNS_LUA, OWNER_TTL_SECONDS, + RECONCILE_STOPPED_TURN_LUA, RELEASE_IF_OWNER_LUA, RUNNING_TTL_SECONDS, SUPERSEDED_TTL_SECONDS, + TURN_STARTED_TTL_SECONDS, + WATCHDOG_RELEASE_TURN_LUA, alive_key, attached_key, displaced_channel, make_displacement_payload, + make_owner_value, + owner_replica_id, owner_key, running_key, superseded_key, + turn_started_key, validate_session_id, # noqa: F401 — re-exported for callers that import from locks ) @@ -54,6 +62,26 @@ async def acquire_alive( return result is not None +async def acquire_alive_with_start( + engine: LockEngine, + *, + project_id: str, + session_id: str, + turn_id: str, +) -> bool: + """Atomically acquire `alive` and record its first start on the Redis clock.""" + result = await engine.eval( + ACQUIRE_ALIVE_WITH_START_LUA, + 2, + alive_key(project_id, session_id).encode(), + turn_started_key(project_id, session_id, turn_id).encode(), + turn_id.encode(), + ALIVE_TTL_SECONDS, + TURN_STARTED_TTL_SECONDS, + ) + return result == 1 + + async def refresh_alive( engine: LockEngine, *, @@ -154,6 +182,155 @@ async def is_turn_superseded( return True +async def release_watchdog_turn( + engine: LockEngine, + *, + project_id: str, + session_id: str, + turn_id: Optional[str], + owner_value: Optional[str], +) -> Tuple[bool, bool, bool]: + """Atomically release only the swept turn and its observed replica owner.""" + result = await engine.eval( + WATCHDOG_RELEASE_TURN_LUA, + 4, + alive_key(project_id, session_id).encode(), + running_key(project_id, session_id).encode(), + owner_key(project_id, session_id).encode(), + superseded_key(project_id, session_id, turn_id or "").encode(), + (turn_id or "").encode(), + (owner_value or "").encode(), + SUPERSEDED_TTL_SECONDS, + ) + return bool(int(result[0])), bool(int(result[1])), bool(int(result[2])) + + +# --------------------------------------------------------------------------- +# Turn start times — "when did this turn first take the session?" +# +# A cancel that is applied after the turn it meant has ended tombstones whichever turn holds +# the nest, which can be the NEXT turn (the stop-then-send race behind #6417). Refusing that +# needs one thing the coordination plane never recorded: when the holding turn started. It +# cannot be derived. `session_turns.start_time` is written by the runner after the fact, and a +# browser turn's id is a runner-minted uuid4, so it carries no time. +# --------------------------------------------------------------------------- + + +async def record_turn_start( + engine: LockEngine, + *, + project_id: str, + session_id: str, + turn_id: str, + started_at_ms: Optional[int] = None, +) -> int: + """Record this turn's start once, then keep the record alive for as long as `alive` is. + + Write-once (nx): a turn that re-takes its own lock after a raced beat keeps its FIRST + start time, which is the one the guard must compare against. Returns the recorded start, + which is the stored one when a record already exists. + """ + key = turn_started_key(project_id, session_id, turn_id) + now_ms = await redis_time_ms(engine) if started_at_ms is None else started_at_ms + written = await engine.set( + key, + str(now_ms).encode(), + nx=True, + ex=TURN_STARTED_TTL_SECONDS, + ) + if written is not None: + return now_ms + current = await engine.get(key) + await engine.expire(key, TURN_STARTED_TTL_SECONDS) + try: + return int(current.decode()) if current else now_ms + except ValueError: + return now_ms + + +async def redis_time_ms(engine: LockEngine) -> int: + """Read the shared Redis clock in epoch milliseconds.""" + seconds, microseconds = await engine.time() + return int(seconds) * 1000 + int(microseconds) // 1000 + + +async def displace_turns( + engine: LockEngine, + *, + project_id: str, + session_id: str, + expected_turn_id: Optional[str] = None, + arrived_at_ms: Optional[int] = None, + running_only: bool = False, +) -> Tuple[bool, Optional[str], List[str]]: + """Atomically validate, tombstone, and clear the alive/running owners.""" + result = await engine.eval( + DISPLACE_TURNS_LUA, + 2, + alive_key(project_id, session_id).encode(), + running_key(project_id, session_id).encode(), + (expected_turn_id or "").encode(), + "" if arrived_at_ms is None else str(arrived_at_ms), + superseded_key(project_id, session_id, "").encode(), + turn_started_key(project_id, session_id, "").encode(), + SUPERSEDED_TTL_SECONDS, + "1" if running_only else "0", + ) + + def _decode(value) -> str: + return value.decode() if isinstance(value, (bytes, bytearray)) else str(value) + + accepted = bool(result) and int(result[0]) == 1 + if not accepted: + return False, _decode(result[1]) if len(result) > 1 else None, [] + turn_ids = list( + dict.fromkeys(_decode(value) for value in result[1:] if _decode(value)) + ) + return True, None, turn_ids + + +async def reconcile_stopped_turn( + engine: LockEngine, + *, + project_id: str, + session_id: str, + turn_id: str, +) -> bool: + """Atomically tombstone a stopped turn and release only its `running` generation.""" + result = await engine.eval( + RECONCILE_STOPPED_TURN_LUA, + 2, + running_key(project_id, session_id).encode(), + superseded_key(project_id, session_id, turn_id).encode(), + turn_id.encode(), + SUPERSEDED_TTL_SECONDS, + ) + return result == 1 + + +async def get_turn_start( + engine: LockEngine, + *, + project_id: str, + session_id: str, + turn_id: str, +) -> Optional[int]: + """This turn's start in epoch milliseconds, or None when nothing recorded one. + + None means "unknown", never "old". Every caller must treat it as unknown and fall back to + the behavior it had before this key existed: a turn from before this code shipped, or one + whose record outlived its TTL, must not become uncancellable. + """ + key = turn_started_key(project_id, session_id, turn_id) + current = await engine.get(key) + if current is None: + return None + try: + return int(current.decode()) + except ValueError: + return None + + # --------------------------------------------------------------------------- # Running lock — "a turn is actively executing right now" # Nested under alive: a session can be alive-but-idle (running absent) between turns. @@ -321,52 +498,108 @@ async def get_owner( session_id: str, ) -> Optional[str]: """Return the replica id currently owning this session, or None.""" + current = await get_owner_value( + engine, project_id=project_id, session_id=session_id + ) + return owner_replica_id(current) if current else None + + +async def get_owner_value( + engine: LockEngine, + *, + project_id: str, + session_id: str, +) -> Optional[str]: + """Return the full replica + turn-generation owner value, or None.""" key = owner_key(project_id, session_id) current = await engine.get(key) return current.decode() if current else None -async def claim_owner( +async def claim_owner_value( engine: LockEngine, *, project_id: str, session_id: str, replica_id: str, + turn_id: Optional[str] = None, ) -> str: - """Atomically claim ownership iff unowned or already ours, and return the actual owner. + """Atomically claim ownership and return the full observed owner generation. Never steals from a live different owner: if another replica holds it, its id is - returned so the caller can refuse to serve a local session on the wrong host. + returned with its turn generation so a later compare-and-delete cannot clear a refresh. """ key = owner_key(project_id, session_id) + owner_value = make_owner_value(replica_id=replica_id, turn_id=turn_id) result = await engine.eval( CLAIM_OWNER_LUA, 1, key.encode(), - replica_id.encode(), + owner_value.encode(), str(OWNER_TTL_SECONDS).encode(), ) - return result.decode() if isinstance(result, (bytes, bytearray)) else str(result) + actual = result.decode() if isinstance(result, (bytes, bytearray)) else str(result) + return actual -async def clear_owner( +async def claim_owner( engine: LockEngine, *, project_id: str, session_id: str, replica_id: str, + turn_id: Optional[str] = None, +) -> str: + """Claim ownership and return the actual owner's replica id.""" + actual = await claim_owner_value( + engine, + project_id=project_id, + session_id=session_id, + replica_id=replica_id, + turn_id=turn_id, + ) + return owner_replica_id(actual) + + +async def release_owner_value( + engine: LockEngine, + *, + project_id: str, + session_id: str, + owner_value: str, ) -> bool: - """Remove the owner key if replica_id is still the owner.""" + """Remove the owner key only if its full replica + turn generation still matches.""" key = owner_key(project_id, session_id) result = await engine.eval( RELEASE_IF_OWNER_LUA, 1, key.encode(), - replica_id.encode(), + owner_value.encode(), ) return result == 1 +async def clear_owner( + engine: LockEngine, + *, + project_id: str, + session_id: str, + replica_id: str, +) -> bool: + """Remove the owner key if replica_id is still the owner.""" + owner_value = await get_owner_value( + engine, project_id=project_id, session_id=session_id + ) + if owner_value is None or owner_replica_id(owner_value) != replica_id: + return False + return await release_owner_value( + engine, + project_id=project_id, + session_id=session_id, + owner_value=owner_value, + ) + + async def force_clear_owner( engine: LockEngine, *, @@ -382,7 +615,7 @@ async def force_clear_owner( key = owner_key(project_id, session_id) current = await engine.get(key) await engine.delete(key) - return current.decode() if current else None + return owner_replica_id(current.decode()) if current else None # --------------------------------------------------------------------------- diff --git a/api/oss/src/middlewares/auth.py b/api/oss/src/middlewares/auth.py index 302bd481790..f9f3cc930ab 100644 --- a/api/oss/src/middlewares/auth.py +++ b/api/oss/src/middlewares/auth.py @@ -71,6 +71,12 @@ "/api/tools/connections/callback", "/preview/tools/connections/callback", "/api/preview/tools/connections/callback", + # SESSIONS CONTROL — the runner reports a command's outcome with the shared runner token, + # not a project credential: it holds none for a command it was handed. The route checks the + # token itself and resolves the project from the command id, so this exemption widens no + # tenant boundary. + "/sessions/control/commands/", + "/api/sessions/control/commands/", # TRIGGERS — inbound provider events arrive from Composio with no auth token "/triggers/composio/events/", "/api/triggers/composio/events/", diff --git a/api/oss/src/tasks/asyncio/sessions/orphan_sweep.py b/api/oss/src/tasks/asyncio/sessions/orphan_sweep.py index 40f97730558..72d2e531516 100644 --- a/api/oss/src/tasks/asyncio/sessions/orphan_sweep.py +++ b/api/oss/src/tasks/asyncio/sessions/orphan_sweep.py @@ -1,52 +1,359 @@ -"""Orphan sweep — SCA-6. +"""Execution watchdog (formerly the orphan sweep) — SCA-6. -Periodically scans session_streams for rows whose mirror says is_alive but whose -heartbeat (updated_at) is stale — the owning runner died mid-turn and its Redis -alive lock has expired. Marks each orphan ended + collapses its flags so the -sandbox can be reaped. +Every accepted execution must reach exactly one durable terminal outcome. The runner writes +that outcome on every path it controls, but it cannot write one when it is gone: its container +restarts, its process dies, or its `run()` never returns. The session then keeps the Redis +`alive`/`running` nest of a turn nobody is running, the transcript stops mid-turn, and the +session refuses a new message until a threshold far away expires. + +This pass closes that hole. It scans `session_streams` for rows whose mirror still says +`is_alive` but whose heartbeat (`updated_at`) is stale, and for each one it: + +1. compare-and-sets the stale stream generation so a renewed turn cannot be settled; +2. settles the execution and writes the terminal records the dead runner owed; +3. clears the Redis nest and tombstones the turn, so a late beat cannot re-nest it; +4. publishes the watch notification, so an open browser refreshes without a reload. + +Steps 1 and 2 share one Postgres transaction. Terminal records publish before its commit with +stable ids, so a crash rolls the stream and execution changes back and the next pass safely +re-publishes the same records. + +Two thresholds, not one. A RUNNING row beats every 30 seconds, so a short silence means the +runner died. An ALIVE-but-idle row is a different animal: between turns, and while a turn is +parked awaiting a human, the runner sends a final beat with `is_running: false` and then stops +beating on purpose. That state is resumable, so it is never given a terminal record here. +Both thresholds are settings; see `SessionWatchdogConfig` in `oss/src/utils/env.py`. + +WHAT THIS PASS CANNOT SEE, and why the runner needs its own detector. This scan keys off +heartbeat age, and a turn whose SANDBOX died keeps beating perfectly well: the runner is +healthy, only the machine under it is gone. Such a row never becomes stale and is invisible +here for ever. That case is issue #6418 and it is closed on the runner side, by the sandbox +liveness probe in `services/runner/src/engines/sandbox_agent/sandbox-liveness.ts`. This pass +covers the complementary case, where the RUNNER is what disappeared and nothing on that side +can write anything at all. Called from the FastAPI lifespan; runs as a background asyncio task. """ import asyncio from datetime import datetime, timezone, timedelta +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple +from uuid import UUID, uuid5, NAMESPACE_URL +from oss.src.utils.env import env from oss.src.utils.logging import get_module_logger +from oss.src.dbs.postgres.sessions.executions.dbes import SessionExecutionDBE from oss.src.dbs.postgres.shared.engine import TransactionsEngine from oss.src.dbs.postgres.sessions.streams.dbes import SessionStreamDBE +from oss.src.core.sessions.records.dtos import ( + RECORD_SETTLED_BY_ATTRIBUTE, + SETTLED_BY_WATCHDOG, + TERMINAL_RECORD_TYPE, + SessionRecordEvent, +) +from oss.src.core.sessions.records.service import RecordsService +from oss.src.core.sessions.records.streaming import publish_record from oss.src.core.sessions.streams.dtos import ( SessionStreamFlags, ) +from oss.src.core.sessions.watch.interfaces import SessionsWatchPublisherInterface +from oss.src.dbs.redis.sessions.contract import WATCH_LIFECYCLE_ENDED from oss.src.dbs.redis.shared.engine import LockEngine from oss.src.dbs.redis.sessions.locks import ( - force_cancel_alive, clear_running, + force_cancel_alive, force_clear_owner, + get_owner_value, mark_turn_superseded, + release_watchdog_turn, ) -from sqlalchemy import and_, func, not_, or_, select +from sqlalchemy import and_, func, not_, or_, select, tuple_, update as sa_update log = get_module_logger(__name__) -# A RUNNING stream whose heartbeat (updated_at) is older than this is orphaned: a live turn -# beats every 30s, so this much silence means the owning runner died. -ORPHAN_THRESHOLD_SECONDS: int = 300 # 5 minutes +# A RUNNING stream whose heartbeat is older than this is lost. +# +# The rule is HEARTBEAT AGE, deliberately, and not the Redis lease. The `alive` and `running` +# keys carry a one-hour TTL (`env.sessions.alive_ttl_seconds`), so waiting for a lease to +# expire would mean waiting an hour. The runner beats every 30 seconds and the beat is +# mirrored onto `session_streams.updated_at`, so the age of that column is what actually says +# whether anyone is still running the turn. Durable Stop uses three missed beats (90 seconds); +# flag-off deployments retain the pre-milestone ten-beat threshold (300 seconds). +# +# Raise AGENTA_SESSIONS_WATCHDOG_STALE_HEARTBEAT_SECONDS if healthy turns are being settled. +ORPHAN_THRESHOLD_SECONDS: int = env.agenta.sessions.watchdog.stale_heartbeat_seconds -# Alive-but-idle rows (between turns, or parked awaiting approval) get a longer grace: the -# runner stops beating while a turn is parked, and it keeps that sandbox warm for the -# approval TTL (30 min). Sweeping those at 5 min would declare a resumable session dead. -IDLE_THRESHOLD_SECONDS: int = 1800 # 30 minutes +# Alive-but-NOT-running rows are RECLAIMED on a different, much longer clock. Between turns, and +# while a turn is parked awaiting a human, the runner sends one final beat with `is_running: +# false` and then stops beating on purpose; that state is resumable, so collapsing it is keyed +# to the 30-minute approval TTL rather than to three missed beats. +# +# It does NOT decide whether such a row owes its turn an ending. It used to, on the premise that +# a not-running row's last turn had already reached a terminal record — a premise a durable Stop +# broke, because settlement clears `is_running` before the runner has written that record. The +# ending is now decided by asking the records plane on the configured stale-heartbeat clock. See +# the second selection in `run_orphan_sweep`. +IDLE_THRESHOLD_SECONDS: int = env.agenta.sessions.watchdog.idle_grace_seconds -# How often the sweep runs. -SWEEP_INTERVAL_SECONDS: int = 60 +# How often the watchdog runs. +SWEEP_INTERVAL_SECONDS: int = env.agenta.sessions.watchdog.interval_seconds # Rows swept per pass. A backlog drains over successive passes instead of one huge commit. -SWEEP_BATCH_SIZE: int = 500 +SWEEP_BATCH_SIZE: int = env.agenta.sessions.watchdog.batch_size + +# The error class the watchdog stamps on the turn it settles. One of the `RunErrorCode` +# values in services/runner/src/engines/sandbox_agent/errors.ts; the client reads it to offer +# a retry rather than parsing the message. +LOST_ERROR_CODE = "execution_lost" + +# The line the user reads in place of the answer the dead runner never gave. Identical to +# `EXECUTION_LOST_MESSAGE` in services/runner/src/engines/sandbox_agent/errors.ts, which the +# runner writes for the same class when a turn will not unwind: one outcome must not reach +# the user in two different wordings depending on which side noticed it. +LOST_ERROR_MESSAGE = "The agent stopped responding and the run was closed. Send the message again to retry." + +# Records are attributed to the agent, matching every record the runner writes for a turn. +RECORD_SOURCE_AGENT = "agent" + +# Both records carry this marker, and it is the ONLY thing that distinguishes the watchdog's +# ending from a runner's. That matters twice at ingest: a record arriving for a turn this +# marker has already closed is quarantined rather than appended, and the watchdog's own two +# records are exempt from that rule so a redelivery cannot quarantine the ending itself. See +# `RecordsService.append_many`. +SETTLED_BY = {RECORD_SETTLED_BY_ATTRIBUTE: SETTLED_BY_WATCHDOG} + + +def _watchdog_record_id( + *, + project_id: str, + session_id: str, + turn_id: str, + suffix: str, +) -> UUID: + """A stable id per (turn, record), so re-running the watchdog upserts instead of appending. + + The ingest path is `INSERT ... ON CONFLICT (project_id, record_id) DO UPDATE`, so two + passes — or two API replicas sweeping at once — write the same two rows, never four. + """ + return uuid5( + NAMESPACE_URL, + f"agenta:sessions:watchdog:{project_id}:{session_id}:{turn_id}:{suffix}", + ) + + +def _lost_turn_records( + *, + project_id: UUID, + session_id: str, + turn_id: str, + now: datetime, +) -> List[SessionRecordEvent]: + """The two records a runner writes when a turn ends badly, written on its behalf. + + Shape and order mirror `run-turn.ts`'s error path exactly: an `error` event carrying the + class a client can act on, then the terminal `done`. A lone `done` would render as a + clean finish, which is the opposite of what happened. + + The two are ordered explicitly. The transcript sorts on (`timestamp`, `created_at`, + `record_index`), and one write batch shares a single `created_at`, so two records stamped + at the same instant with no index would come back in whatever order Postgres chose. A + `done` read before its `error` closes the turn early, and the failure then renders as a + stray bubble beside a turn that claims it got no response. + """ + project = str(project_id) + + return [ + SessionRecordEvent( + project_id=project_id, + session_id=session_id, + record_id=_watchdog_record_id( + project_id=project, + session_id=session_id, + turn_id=turn_id, + suffix="error", + ), + timestamp=now, + record_index=0, + record_type="error", + record_source=RECORD_SOURCE_AGENT, + attributes={ + "type": "error", + "message": LOST_ERROR_MESSAGE, + "code": LOST_ERROR_CODE, + **SETTLED_BY, + }, + turn_id=turn_id, + ), + SessionRecordEvent( + project_id=project_id, + session_id=session_id, + record_id=_watchdog_record_id( + project_id=project, + session_id=session_id, + turn_id=turn_id, + suffix="done", + ), + timestamp=now + timedelta(milliseconds=1), + record_index=1, + record_type="done", + record_source=RECORD_SOURCE_AGENT, + attributes={"type": "done", **SETTLED_BY}, + turn_id=turn_id, + ), + ] + + +def _stopped_turn_records( + *, + project_id: UUID, + session_id: str, + turn_id: str, + now: datetime, +) -> List[SessionRecordEvent]: + return [ + SessionRecordEvent( + project_id=project_id, + session_id=session_id, + record_id=_watchdog_record_id( + project_id=str(project_id), + session_id=session_id, + turn_id=turn_id, + suffix="done", + ), + timestamp=now, + record_index=0, + record_type=TERMINAL_RECORD_TYPE, + record_source=RECORD_SOURCE_AGENT, + attributes={"type": "done", "stopReason": "cancelled", **SETTLED_BY}, + turn_id=turn_id, + ) + ] + + +async def _unsettled_turns( + *, + records_service: Optional[RecordsService], + candidates: Sequence[Tuple[UUID, str, str]], +) -> Tuple[ + Set[Tuple[UUID, str, str]], + Set[Tuple[UUID, str, str]], + Set[Tuple[UUID, str, str]], +]: + """Partition candidates into turns without and with a terminal record. + + A runner can die AFTER writing its outcome but BEFORE its final `is_running=false` + heartbeat lands — the last beat is best-effort and untimed. Such a turn is already + settled; the row still needs collapsing, but writing a second, contradictory ending + would corrupt the transcript. One query per project, never one per candidate. + """ + if not candidates: + return set(), set(), set() + + if records_service is None: + # No records plane wired (minimal test compositions): settle the row, write nothing. + return set(), set(), set() + + by_project: Dict[UUID, List[Tuple[str, str]]] = {} + for project_id, session_id, turn_id in candidates: + by_project.setdefault(project_id, []).append((session_id, turn_id)) + + unsettled: Set[Tuple[UUID, str, str]] = set() + ended: Set[Tuple[UUID, str, str]] = set() + deferred: Set[Tuple[UUID, str, str]] = set() + for project_id, keys in by_project.items(): + try: + settled = await records_service.settled_turns( + project_id=project_id, keys=keys + ) + except Exception: + log.warning( + "watchdog: terminal-record lookup failed; deferring project candidates", + project_id=str(project_id), + exc_info=True, + ) + deferred.update( + (project_id, session_id, turn_id) for session_id, turn_id in keys + ) + continue + + for session_id, turn_id in keys: + key = (project_id, session_id, turn_id) + if (session_id, turn_id) in settled: + ended.add(key) + else: + unsettled.add(key) + + return unsettled, ended, deferred + + +async def _mark_endings_written( + *, + session: Any, + keys: Set[Tuple[UUID, str, str]], + written_at: datetime, +) -> None: + if not keys: + return + await session.execute( + sa_update(SessionExecutionDBE) + .where( + tuple_( + SessionExecutionDBE.project_id, + SessionExecutionDBE.session_id, + SessionExecutionDBE.execution_id, + ).in_(keys), + SessionExecutionDBE.ending_written_at.is_(None), + ) + .values(ending_written_at=written_at) + ) + + +async def _settle_abandoned_commands( + commands_service: Optional[Any], + now: datetime, +) -> int: + """Settle every Stop command whose runner accepted it and never reported. + + Delegates the decision to the commands plane, which owns the command state machine, so + this sweep and a runner report can never write two different terminal outcomes for the + same command. Never raises: an abandoned command must not stop the pass that settles + executions. + """ + if commands_service is None: + return 0 + try: + return await commands_service.settle_abandoned_commands(now=now) + except Exception: + log.warning("watchdog: failed to settle abandoned commands", exc_info=True) + return 0 + +async def _repair_terminal_redis(commands_service: Optional[Any]) -> int: + if commands_service is None: + return 0 + try: + return await commands_service.repair_terminal_redis() + except Exception: + log.warning("watchdog: failed to repair terminal Redis state", exc_info=True) + return 0 -async def run_orphan_sweep(engine: TransactionsEngine, lock_engine: LockEngine) -> None: - """Single sweep pass: mark stale is_alive rows as ended.""" + +async def run_orphan_sweep( + engine: TransactionsEngine, + lock_engine: LockEngine, + *, + records_service: Optional[RecordsService] = None, + watch_publisher: Optional[SessionsWatchPublisherInterface] = None, + commands_service: Optional[Any] = None, + publish: Any = publish_record, +) -> None: + """Single watchdog pass: settle every stale is_alive row, then every abandoned command. + + `commands_service` is a `SessionCommandsService`. It is optional and typed loosely so this + module keeps no import edge on the commands plane, which would be a cycle. When it is + given, this pass is also the one writer that settles a Stop the runner never reported. + """ now_utc = datetime.now(timezone.utc) threshold = now_utc - timedelta(seconds=ORPHAN_THRESHOLD_SECONDS) idle_threshold = now_utc - timedelta(seconds=IDLE_THRESHOLD_SECONDS) @@ -72,55 +379,541 @@ async def run_orphan_sweep(engine: TransactionsEngine, lock_engine: LockEngine) result = await session.execute(stmt) orphans = result.scalars().all() - if not orphans: + # Capture what the collapse and its Redis/watch follow-up need as plain values NOW, + # before any nested `engine.session()` in this pass runs. The records lookup and the + # command settlement below each open `engine.session()`, which returns the SAME + # current-task-scoped session and, in its `finally`, calls `session.close()` (see + # `TransactionsEngine.session`). That close detaches every ORM row loaded here, so a + # later `row.flags = ...` mutation is tracked by no session and is silently dropped at + # commit -- the flags UPDATE is never emitted, while a Core UPDATE (the command + # settle's `stopping_turn_id`) still lands. That is the finding-7 bug: the row kept + # `is_running: true` after the sweep. The collapse below writes through a Core UPDATE + # keyed by these ids, and the Redis/watch steps read these tuples, never the rows. + orphan_rows: List[Tuple[UUID, UUID, str, Optional[str], Optional[datetime]]] = [ + ( + row.id, + row.project_id, + row.session_id, + str(row.turn_id) if row.turn_id else None, + row.updated_at, + ) + for row in orphans + ] + + # Current stopped turns get their missing ending on the short clock without collapsing + # a parked session, whose reclamation stays on the longer idle grace. + ending_stmt = ( + select(SessionStreamDBE) + .where( + SessionStreamDBE.deleted_at.is_(None), + SessionStreamDBE.flags.contains({"is_alive": True}), + not_(is_running), + SessionStreamDBE.turn_id.is_not(None), + last_beat < threshold, + ) + .limit(SWEEP_BATCH_SIZE) + ) + ending_only = (await session.execute(ending_stmt)).scalars().all() + + # A stream row names only its current turn. Older terminal executions must remain + # visible after that row advances, or their missing transcript ending is permanent. + terminal_executions = [] + if records_service is not None: + terminal_stmt = ( + select(SessionExecutionDBE) + .where( + SessionExecutionDBE.terminal_outcome.in_(("stopped", "lost")), + SessionExecutionDBE.ending_written_at.is_(None), + SessionExecutionDBE.settled_at < threshold, + ) + .order_by(SessionExecutionDBE.settled_at.desc()) + .limit(SWEEP_BATCH_SIZE) + ) + terminal_executions = (await session.execute(terminal_stmt)).scalars().all() + + # A row that claimed a RUNNING turn owes that turn an ending. So does a stopped row + # whose runner never wrote one; see the note above. + seen: Set[Tuple[UUID, str, str]] = set() + claimed: List[Tuple[UUID, str, str]] = [] + for row in [*orphans, *ending_only]: + if not row.turn_id: + continue + key = (row.project_id, row.session_id, str(row.turn_id)) + if key in seen: + continue + seen.add(key) + claimed.append(key) + terminal_turns: Set[Tuple[UUID, str, str]] = set() + terminal_outcomes: Dict[Tuple[UUID, str, str], str] = {} + for execution in terminal_executions: + key = ( + execution.project_id, + execution.session_id, + execution.execution_id, + ) + terminal_turns.add(key) + terminal_outcomes[key] = execution.terminal_outcome + if key in seen: + continue + seen.add(key) + claimed.append(key) + unsettled, ended, deferred = await _unsettled_turns( + records_service=records_service, candidates=claimed + ) + if deferred: + orphan_rows = [ + row + for row in orphan_rows + if row[3] is None or (row[1], row[2], row[3]) not in deferred + ] + await _mark_endings_written( + session=session, + keys=ended & terminal_turns, + written_at=now_utc, + ) + + if not orphan_rows and not unsettled: + # No stale row and nothing owed an ending, but a command can still be abandoned: + # its execution may have ended normally between the claim and the report. + if not deferred: + await _settle_abandoned_commands(commands_service, now_utc) + await _repair_terminal_redis(commands_service) return now = datetime.now(timezone.utc) - for row in orphans: - row.flags = SessionStreamFlags( - is_alive=False, is_running=False, is_attached=False - ).model_dump(mode="json") - row.updated_at = now + + # Capture the affinity generation before the guarded database update. Redis cleanup + # compares this replica and the swept turn atomically after commit, so a new Send or + # Steer generation cannot be deleted. + observed_owners: Dict[Tuple[UUID, str, str], Optional[str]] = {} + owner_keys = { + (project_id, session_id, turn_id) + for project_id, session_id, turn_id in unsettled + } + owner_keys.update( + (project_id, session_id, turn_id) + for _row_id, project_id, session_id, turn_id, _updated_at in orphan_rows + if turn_id is not None + ) + for project_id, session_id, turn_id in sorted( + owner_keys, key=lambda key: key[1] + ): + observed_owners[(project_id, session_id, turn_id)] = await get_owner_value( + lock_engine, + project_id=str(project_id), + session_id=session_id, + ) + + # Win the stale stream generation before settling its execution or publishing records. + # The update and execution settlement share this transaction; an exception rolls both + # back, while record ids make a publish-before-commit retry idempotent. + collapsed_flags = SessionStreamFlags( + is_alive=False, is_running=False, is_attached=False + ).model_dump(mode="json") + collapsed_rows: List[ + Tuple[UUID, UUID, str, Optional[str], Optional[datetime]] + ] = [] + skipped_orphan_turns: Set[Tuple[UUID, str, str]] = set() + for ( + row_id, + project_uuid, + session_id, + turn_id, + observed_updated_at, + ) in orphan_rows: + conditions = [ + SessionStreamDBE.id == row_id, + SessionStreamDBE.project_id == project_uuid, + SessionStreamDBE.session_id == session_id, + SessionStreamDBE.deleted_at.is_(None), + ( + SessionStreamDBE.turn_id == turn_id + if turn_id is not None + else SessionStreamDBE.turn_id.is_(None) + ), + ( + SessionStreamDBE.updated_at == observed_updated_at + if observed_updated_at is not None + else SessionStreamDBE.updated_at.is_(None) + ), + ] + result = await session.execute( + sa_update(SessionStreamDBE) + .where(*conditions) + .values(flags=collapsed_flags, updated_at=now) + .execution_options(synchronize_session=False) + ) + if result.rowcount != 1: + if turn_id is not None: + skipped_orphan_turns.add((project_uuid, session_id, turn_id)) + log.info( + "watchdog: orphan stream advanced during sweep; leaving it untouched", + session_id=session_id, + turn_id=turn_id, + ) + continue + collapsed_rows.append( + (row_id, project_uuid, session_id, turn_id, observed_updated_at) + ) log.warning( - "orphan_sweep: marking session_stream ended", - extra={"session_id": row.session_id, "stream_id": str(row.id)}, + "watchdog: settled a session_stream whose runner went silent", + extra={ + "session_id": session_id, + "stream_id": str(row_id), + "turn_id": turn_id, + "lost": (project_uuid, session_id, turn_id) in unsettled, + }, ) - await session.commit() + terminal_winners: Set[Tuple[UUID, str, str]] = set() + endings_written: Set[Tuple[UUID, str, str]] = set() + for project_id, session_id, turn_id in sorted(unsettled, key=lambda t: t[1]): + key = (project_id, session_id, turn_id) + if key in skipped_orphan_turns: + continue + if ( + key not in terminal_turns + and env.agenta.sessions.durable_stop + and commands_service is not None + and not await commands_service.settle_execution_lost( + project_id=project_id, + session_id=session_id, + execution_id=turn_id, + settled_at=now, + transaction=session, + ) + ): + continue + terminal_winners.add(key) + record_events = ( + _stopped_turn_records( + project_id=project_id, + session_id=session_id, + turn_id=turn_id, + now=now, + ) + if terminal_outcomes.get(key) == "stopped" + else _lost_turn_records( + project_id=project_id, + session_id=session_id, + turn_id=turn_id, + now=now, + ) + ) + for record_event in record_events: + published = False + try: + published = await publish( + project_id=project_id, record_event=record_event + ) + except Exception: + log.warning( + "watchdog: failed to publish a terminal record", + project_id=str(project_id), + session_id=session_id, + turn_id=turn_id, + exc_info=True, + ) + if published and record_event.record_type == TERMINAL_RECORD_TYPE: + endings_written.add(key) + + await _mark_endings_written( + session=session, + keys=endings_written, + written_at=now, + ) + + unsettled = terminal_winners + + # A lost turn whose stream row the went-silent collapse did NOT touch must still be + # brought to rest here, in this same pass, or the SEND gate refuses the next message + # until the runner returns -- which, for a lost turn, may be never. The RFC's rule is + # that the settlement writes the ending, clears `is_running`, releases `alive`, and + # updates the mirror together. The collapse above owns the rows the orphan query + # matched; this owns every other lost turn (a row the query did not return, or an + # older execution whose row has since advanced). Everything here is guarded on + # `turn_id`, so a row that now names a NEWER running turn is never disturbed. + collapsing = {(p, s, t) for (_id, p, s, t, _u) in collapsed_rows} + newly_lost = sorted(unsettled - collapsing, key=lambda t: t[1]) - # Bring the Redis locks the SEND gate reads in sync with the rows just written. - for row in orphans: - project_id = str(row.project_id) - displaced_alive = await force_cancel_alive( - lock_engine, project_id=project_id, session_id=row.session_id + # Clear `is_running` on the DB row that STILL names a lost turn, keeping `is_alive` so + # the session stays resumable. Guarded on turn_id: a row that advanced to a newer turn + # is left alone. Observed live on the integration stack: the execution was settled lost + # but the stream row kept `is_running: true`, and the next Send was refused. + # Captured as plain values, and written by a Core UPDATE further down, for the same + # reason the collapse is: the Redis calls that follow this block, and anything a future + # edit puts between the load and the write, can open a nested `engine.session()`, whose + # `finally` closes the shared task-scoped session and detaches these rows. A mutation on + # a detached row is tracked by no session and is dropped at commit with no error. + running_rows_to_clear: List[ + Tuple[UUID, UUID, str, str, Optional[datetime], Dict[str, Any]] + ] = [] + if newly_lost: + rows_to_clear = ( + ( + await session.execute( + select(SessionStreamDBE).where( + SessionStreamDBE.deleted_at.is_(None), + SessionStreamDBE.flags.contains({"is_running": True}), + tuple_( + SessionStreamDBE.project_id, + SessionStreamDBE.session_id, + SessionStreamDBE.turn_id, + ).in_(list(newly_lost)), + ) + ) + ) + .scalars() + .all() ) - displaced_running = await clear_running( - lock_engine, project_id=project_id, session_id=row.session_id + for row in rows_to_clear: + flags = dict(row.flags or {}) + flags["is_running"] = False + running_rows_to_clear.append( + ( + row.id, + row.project_id, + row.session_id, + str(row.turn_id), + row.updated_at, + flags, + ) + ) + + # Every write to `session_streams` uses a Core UPDATE. No ORM attribute write on this + # table survives anywhere in this pass: nested scoped sessions can detach loaded rows. + running_rows_cleared: List[ + Tuple[UUID, UUID, str, str, Optional[datetime], Dict[str, Any]] + ] = [] + failed_running_clears: Set[Tuple[UUID, str, str]] = set() + for ( + row_id, + project_uuid, + session_id, + turn_id, + observed_updated_at, + cleared_flags, + ) in running_rows_to_clear: + conditions = [ + SessionStreamDBE.id == row_id, + SessionStreamDBE.project_id == project_uuid, + SessionStreamDBE.session_id == session_id, + SessionStreamDBE.turn_id == turn_id, + SessionStreamDBE.deleted_at.is_(None), + ( + SessionStreamDBE.updated_at == observed_updated_at + if observed_updated_at is not None + else SessionStreamDBE.updated_at.is_(None) + ), + ] + result = await session.execute( + sa_update(SessionStreamDBE) + .where(*conditions) + .values(flags=cleared_flags, updated_at=now) + .execution_options(synchronize_session=False) ) - # A swept turn is declared dead; tombstone it so a late beat from it cannot - # re-nest the session it was just evicted from. - for turn_id in {t for t in (displaced_alive, displaced_running) if t}: - await mark_turn_superseded( - lock_engine, - project_id=project_id, - session_id=row.session_id, + if result.rowcount != 1: + failed_running_clears.add((project_uuid, session_id, turn_id)) + log.info( + "watchdog: lost-turn stream advanced during sweep; leaving it untouched", + session_id=session_id, turn_id=turn_id, ) - # A swept session is dead; free its affinity like kill does. - await force_clear_owner( - lock_engine, project_id=project_id, session_id=row.session_id + continue + running_rows_cleared.append( + ( + row_id, + project_uuid, + session_id, + turn_id, + observed_updated_at, + cleared_flags, + ) ) - log.info("orphan_sweep: marked %d orphans ended", len(orphans)) + await session.commit() + + # Redis cleanup is one compare-and-delete operation per session. A new Send or Steer may + # install another generation after this commit; the script leaves its keys and affinity + # untouched and tombstones only the swept turn. + for project_uuid, session_id, turn_id in newly_lost: + if (project_uuid, session_id, turn_id) in failed_running_clears: + continue + ( + released_alive, + _released_running, + _released_owner, + ) = await release_watchdog_turn( + lock_engine, + project_id=str(project_uuid), + session_id=session_id, + turn_id=turn_id, + owner_value=observed_owners.get((project_uuid, session_id, turn_id)), + ) + log.warning( + "watchdog: wrote the ending a stopped turn's runner never reported", + extra={ + "session_id": session_id, + "turn_id": turn_id, + "released_alive": released_alive, + }, + ) + + for ( + _row_id, + project_uuid, + session_id, + row_turn_id, + _observed_updated_at, + ) in collapsed_rows: + if row_turn_id is None: + project_id = str(project_uuid) + displaced_alive = await force_cancel_alive( + lock_engine, project_id=project_id, session_id=session_id + ) + displaced_running = await clear_running( + lock_engine, project_id=project_id, session_id=session_id + ) + for displaced_turn_id in { + turn_id + for turn_id in (displaced_alive, displaced_running) + if turn_id + }: + await mark_turn_superseded( + lock_engine, + project_id=project_id, + session_id=session_id, + turn_id=displaced_turn_id, + ) + await force_clear_owner( + lock_engine, project_id=project_id, session_id=session_id + ) + continue + await release_watchdog_turn( + lock_engine, + project_id=str(project_uuid), + session_id=session_id, + turn_id=row_turn_id, + owner_value=observed_owners.get( + (project_uuid, session_id, row_turn_id) + ), + ) + + # Tell every open reader the session ended. Without this a browser sitting on the + # settled turn keeps showing it as running until the user reloads. Best effort: the + # publisher never raises and never re-drives the settle above. + if watch_publisher is not None: + for ( + _row_id, + project_uuid, + session_id, + _turn_id, + _observed_updated_at, + ) in collapsed_rows: + try: + await watch_publisher.lifecycle( + project_id=str(project_uuid), + session_id=session_id, + state=WATCH_LIFECYCLE_ENDED, + ) + # The session channel reaches a tab that has this session open. A list + # row lives on the project channel, so publish there too, or every other + # tab keeps the session marked running until its own poll comes round. + await watch_publisher.changed( + project_id=str(project_uuid), + entity="session", + id=session_id, + ) + except Exception: + log.warning( + "watchdog: watch publish failed", + session_id=session_id, + exc_info=True, + ) + + # A row whose `is_running` was cleared (but not collapsed) also needs the mirror + # update, or a browser sitting on it keeps the turn drawn as running until a reload. + for ( + _row_id, + project_uuid, + session_id, + _turn_id, + _observed_updated_at, + _flags, + ) in running_rows_cleared: + try: + await watch_publisher.changed( + project_id=str(project_uuid), + entity="session", + id=session_id, + ) + except Exception: + log.warning( + "watchdog: watch publish failed", + session_id=session_id, + exc_info=True, + ) + + # AFTER the rows above are collapsed, on purpose. A command is only abandoned when its + # session has stopped beating, and the collapse just made that true for every row in + # this batch. Running it first would leave the runner-gone case waiting a second pass. + commands_settled = 0 + if not deferred: + commands_settled = await _settle_abandoned_commands( + commands_service, datetime.now(timezone.utc) + ) + await _repair_terminal_redis(commands_service) + + log.info( + "watchdog: settled %d sessions (%d turns marked lost, %d commands lost)", + len(collapsed_rows), + len(unsettled), + commands_settled, + ) async def orphan_sweep_loop( - engine: TransactionsEngine, lock_engine: LockEngine + engine: TransactionsEngine, + lock_engine: LockEngine, + *, + records_service: Optional[RecordsService] = None, + watch_publisher: Optional[SessionsWatchPublisherInterface] = None, + commands_service: Optional[Any] = None, ) -> None: """Infinite loop; runs as a background asyncio task during app lifespan.""" + # A pass that never returns would end the watchdog for the life of the process with + # nothing in the log; observed on the integration stack on 2026-09-03, when the sweep + # went silent after one pass and never ran again. Bound every pass, log the timeout, + # and go round again. + pass_timeout = float(max(SWEEP_INTERVAL_SECONDS * 2, 120)) while True: + started = datetime.now(timezone.utc) try: - await run_orphan_sweep(engine, lock_engine) + await asyncio.wait_for( + run_orphan_sweep( + engine, + lock_engine, + records_service=records_service, + watch_publisher=watch_publisher, + commands_service=commands_service, + ), + timeout=pass_timeout, + ) + except asyncio.CancelledError: + raise + except asyncio.TimeoutError: + log.error( + "watchdog: sweep pass timed out after %.0fs; skipping to the next pass", + pass_timeout, + ) except Exception: - log.exception("orphan_sweep: error during sweep pass") - await asyncio.sleep(SWEEP_INTERVAL_SECONDS) + # `log` is a MultiLogger, which has no `exception` method; calling one would + # raise AttributeError from inside this handler and kill the loop for the life + # of the process. Use `error(..., exc_info=True)`, the same shape the helpers + # above use, so the first sweep error is logged and the loop goes round again. + log.error("watchdog: error during sweep pass", exc_info=True) + elapsed = (datetime.now(timezone.utc) - started).total_seconds() + if elapsed > SWEEP_INTERVAL_SECONDS: + log.warning("watchdog: sweep pass took %.1fs", elapsed) + # Floored: a zero or negative interval would turn the loop into a hot spin. + await asyncio.sleep(max(SWEEP_INTERVAL_SECONDS, 1)) diff --git a/api/oss/src/tasks/asyncio/sessions/records_worker.py b/api/oss/src/tasks/asyncio/sessions/records_worker.py index b107935a44e..042a2a659f6 100644 --- a/api/oss/src/tasks/asyncio/sessions/records_worker.py +++ b/api/oss/src/tasks/asyncio/sessions/records_worker.py @@ -2,8 +2,10 @@ from uuid import UUID from redis.asyncio import Redis +from sqlalchemy.exc import DataError, IntegrityError from oss.src.core.sessions.interactions.service import SessionInteractionsService +from oss.src.core.sessions.records.dtos import TERMINAL_RECORD_TYPE from oss.src.core.sessions.records.service import RecordsService from oss.src.core.sessions.records.streaming import deserialize_record from oss.src.core.sessions.watch.interfaces import SessionsWatchPublisherInterface @@ -18,11 +20,11 @@ from ee.src.core.access.entitlements.types import Counter -# The runner's terminal per-turn record, and the marker it stamps on that record when the turn -# stopped to wait for a human instead of finishing (services/runner/src/tracing/otel.ts: the -# field is written ONLY for a pause and omitted on every other stop reason). -TERMINAL_RECORD_TYPE = "done" +# The marker the runner stamps on its terminal record when the turn stopped to wait for a +# human instead of finishing (services/runner/src/tracing/otel.ts: the field is written ONLY +# for a pause and omitted on every other stop reason). PAUSED_STOP_REASON = "paused" +ROW_REJECTION_ERRORS = (DataError, IntegrityError) def finished_turns_in_batch(events: List[Any]) -> Dict[str, str]: @@ -52,13 +54,18 @@ class RecordsWorker(StreamConsumer): Consumer group: worker-records Flow: - 1. Read batch from stream (XREADGROUP) — StreamConsumer + 1. Read batch from stream (XREADGROUP), or reclaim unacknowledged entries — StreamConsumer 2. Deserialize messages 3. Group by project_id 4. EE: L2 quota check per org (Counter.RECORDS_INGESTED) 5. Append record events to DB 6. Reconcile HITL gates orphaned by a finished turn - 7. ACK + DEL messages — StreamConsumer + 7. ACK + DEL only the messages whose Postgres write committed — StreamConsumer + + A message id leaves this worker in the acknowledged list for exactly three reasons: its + write committed, it could not be decoded, or its org is over quota. Everything else stays + pending so the reclaim pass writes it later. Acknowledging before the write, which is what + this worker used to do, turned every Postgres failure into permanent silent record loss. """ log_prefix = "[RECORDS]" @@ -76,6 +83,8 @@ def __init__( max_batch_mb: int = 50, watch_publisher: Optional[SessionsWatchPublisherInterface] = None, interactions_service: Optional[SessionInteractionsService] = None, + reclaim_min_idle_ms: int = 30_000, + max_deliveries: int = 5, ): super().__init__( redis_client=redis_client, @@ -86,12 +95,18 @@ def __init__( max_block_ms=max_block_ms, max_delay_ms=max_delay_ms, max_batch_mb=max_batch_mb, + # Records are the durable transcript. A pending entry that is never redelivered is + # a lost turn, so this worker always runs the reclaim pass. + reclaim_pending=True, + reclaim_min_idle_ms=reclaim_min_idle_ms, + max_deliveries=max_deliveries, ) self.service = service self.watch_publisher = watch_publisher # Absent disables gate reconciliation (minimal test compositions), which only loses the # safety net — never the append. self.interactions_service = interactions_service + self._permanent_failure_ids: set[bytes] = set() async def reconcile_orphaned_gates( self, @@ -147,13 +162,122 @@ async def reconcile_orphaned_gates( exc_info=True, ) + def describe_message(self, data: Dict[bytes, bytes]) -> Optional[str]: + """`session:record:type` for the dropped-message log, so a loss is traceable.""" + try: + record = deserialize_record(payload=data[b"data"]).record_event + return f"{record.session_id}:{record.record_id}:{record.record_type}" + except Exception: + return None + + def is_permanent_failure( + self, + msg_id: bytes, + data: Dict[bytes, bytes], + ) -> bool: + return msg_id in self._permanent_failure_ids + + async def _append( + self, + *, + project_id: UUID, + entries: List[Tuple[bytes, Any]], + ) -> Tuple[int, Optional[Exception]]: + """One `append_many` call. Returns rows written and any failure.""" + try: + results = await self.service.append_many( + events=[msg.record_event for _, msg in entries], + ) + quarantined = [ + row + for row in results + if getattr(row, "quarantined_at", None) is not None + ] + if quarantined: + log.warning( + "[RECORDS] Quarantined late records for settled turns", + project_id=str(project_id), + quarantined=len(quarantined), + appended=len(results), + turns=sorted( + {f"{row.session_id}:{row.turn_id}" for row in quarantined} + ), + ) + return len(results), None + except Exception as exc: + log.error( + "[RECORDS] Failed to append event batch", + project_id=str(project_id), + size=len(entries), + exc_info=True, + ) + return 0, exc + + async def _append_committed( + self, + *, + project_id: UUID, + entries: List[Tuple[bytes, Any]], + ) -> Tuple[int, List[bytes]]: + """Write a project group and report the message ids that are durable. + + `append_many` is one statement in one transaction, so a row-specific database rejection + takes the whole group down with it. Only that failure class triggers one-record writes to + isolate the rejected row. Connection, timeout, and unknown failures leave the entire + group pending for Redis reclaim instead of multiplying calls during an outage. + """ + appended, failure = await self._append(project_id=project_id, entries=entries) + if failure is None: + self._permanent_failure_ids.difference_update( + msg_id for msg_id, _ in entries + ) + return appended, [msg_id for msg_id, _ in entries] + + if not isinstance(failure, ROW_REJECTION_ERRORS): + return 0, [] + + if len(entries) == 1: + self._permanent_failure_ids.add(entries[0][0]) + return 0, [] + + log.warning( + "[RECORDS] Batch append failed, retrying one record at a time", + project_id=str(project_id), + size=len(entries), + ) + + total_appended = 0 + committed_ids: List[bytes] = [] + for entry in entries: + appended, failure = await self._append( + project_id=project_id, entries=[entry] + ) + if failure is None: + total_appended += appended + committed_ids.append(entry[0]) + self._permanent_failure_ids.discard(entry[0]) + elif isinstance(failure, ROW_REJECTION_ERRORS): + self._permanent_failure_ids.add(entry[0]) + + log.warning( + "[RECORDS] Retry finished", + project_id=str(project_id), + committed=len(committed_ids), + pending=len(entries) - len(committed_ids), + ) + return total_appended, committed_ids + async def process_batch( self, batch: List[Tuple[bytes, Dict[bytes, bytes]]], ) -> Tuple[int, List[bytes]]: - """Process batch — deserialize, group by org for EE quota, append to DB.""" + """Process batch — deserialize, group by org for EE quota, append to DB. + + The returned ids are acknowledged and deleted by the consumer loop, so an id only goes + in once its rows are committed, or once this worker has decided to drop it on purpose. + """ groups: Dict[UUID, Dict[str, Any]] = {} - processed_ids: List[bytes] = [] + acked_ids: List[bytes] = [] batch_bytes = 0 for msg_id, data in batch: @@ -162,6 +286,8 @@ async def process_batch( batch_bytes += len(payload) if batch_bytes > self.max_batch_mb * 1024 * 1024: + # The rest of the batch stays unacknowledged and comes back through the + # reclaim pass, rather than being silently skipped. break msg = deserialize_record(payload=payload) @@ -170,23 +296,28 @@ async def process_batch( group = { "organization_id": msg.organization_id, "project_id": msg.project_id, - "events": [], + "entries": [], } groups[msg.project_id] = group - group["events"].append(msg) - processed_ids.append(msg_id) + group["entries"].append((msg_id, msg)) except Exception: log.error( "[RECORDS] Failed to deserialize message", msg_id=repr(msg_id), exc_info=True, ) - processed_ids.append(msg_id) + # A message that does not decode will not decode on redelivery either, so + # acknowledge it instead of letting it hold the pending list. Counted as a loss. + self.dropped_messages += 1 + acked_ids.append(msg_id) batches = list(groups.values()) total_appended = 0 org_allowed: Dict[UUID, bool] = {} + # Orgs whose quota question could not be answered. Their records are not over quota, + # they are unmetered, so they wait for the next delivery instead of being dropped. + org_deferred: set = set() events_per_org: Dict[UUID, int] = {} if is_ee(): @@ -195,7 +326,7 @@ async def process_batch( if org_id is None: continue events_per_org[org_id] = events_per_org.get(org_id, 0) + len( - project_batch["events"] + project_batch["entries"] ) for org_id, delta in events_per_org.items(): @@ -216,6 +347,7 @@ async def process_batch( exc_info=True, ) org_allowed[org_id] = False + org_deferred.add(org_id) continue if not quota_allowed: @@ -231,27 +363,37 @@ async def process_batch( for project_batch in batches: org_id = project_batch["organization_id"] + entries: List[Tuple[bytes, Any]] = project_batch["entries"] + if is_ee() and org_id and not org_allowed.get(org_id, True): + if org_id in org_deferred: + # The meter was unreachable, not exceeded. Leave the entries pending so a + # transient entitlements outage does not delete a conversation. + continue + # An over-quota org is a deliberate product drop, so acknowledging is correct. + # Count it, because it is still a record the transcript will never have. + self.dropped_messages += len(entries) + acked_ids.extend(msg_id for msg_id, _ in entries) continue - try: - results = await self.service.append_many( - events=[msg.record_event for msg in project_batch["events"]], - ) - total_appended += len(results) - except Exception: - log.error( - "[RECORDS] Failed to append event batch", - project_id=str(project_batch["project_id"]), - exc_info=True, - ) + appended, committed_ids = await self._append_committed( + project_id=project_batch["project_id"], + entries=entries, + ) + total_appended += appended + acked_ids.extend(committed_ids) + + if not committed_ids: continue + committed = set(committed_ids) + committed_events = [msg for msg_id, msg in entries if msg_id in committed] + # Strictly post-append, and BEFORE the relay tee: a client woken by the records # notification below must already see the cancelled gate, not re-render it. await self.reconcile_orphaned_gates( project_id=project_batch["project_id"], - events=project_batch["events"], + events=committed_events, ) # Relay tee (M3): strictly post-append so a notified client that @@ -259,9 +401,7 @@ async def process_batch( # session in the project batch; failures never re-drive the append. if self.watch_publisher is not None: project_id = str(project_batch["project_id"]) - session_ids = { - msg.record_event.session_id for msg in project_batch["events"] - } + session_ids = {msg.record_event.session_id for msg in committed_events} for session_id in sorted(session_ids): try: await self.watch_publisher.records_changed( @@ -275,4 +415,4 @@ async def process_batch( session_id=session_id, ) - return total_appended, processed_ids + return total_appended, acked_ids diff --git a/api/oss/src/tasks/asyncio/shared/consumer.py b/api/oss/src/tasks/asyncio/shared/consumer.py index 66303ff5edf..79846cb4a20 100644 --- a/api/oss/src/tasks/asyncio/shared/consumer.py +++ b/api/oss/src/tasks/asyncio/shared/consumer.py @@ -12,6 +12,10 @@ - max_block_ms: 5000ms (XREADGROUP BLOCK) - max wait time when queue is empty - max_batch_mb: 50 - max batch size in megabytes - max_delay_ms: 250ms - max wait time for batch accumulation when small batches arrive + +Redelivery (opt-in, `reclaim_pending`): +- reclaim_min_idle_ms: 30000 - how long an unacknowledged entry sits before it is retried +- max_deliveries: 5 - deliveries after which an entry is dropped loudly instead of retried """ import time @@ -31,9 +35,10 @@ class StreamConsumer: Base class for a Redis Streams consumer-group loop. Flow: - 1. Read batch from Redis Streams (XREADGROUP) + 1. Read batch from Redis Streams (XREADGROUP), or reclaim entries an earlier + pass left unacknowledged (opt-in, see `reclaim_batch`) 2. `process_batch` (subclass): deserialize, group, meter, write - 3. ACK + DEL processed messages + 3. ACK + DEL the message ids `process_batch` reports as durable """ #: Short tag prepended to log messages by subclasses (e.g. "[INGEST]"). @@ -49,6 +54,9 @@ def __init__( max_block_ms: int = 5000, # 5 seconds max_delay_ms: int = 250, # 250 milliseconds max_batch_mb: int = 50, # 50 MB + reclaim_pending: bool = False, + reclaim_min_idle_ms: int = 30_000, # 30 seconds + max_deliveries: int = 5, ): self.redis = redis_client self.stream_name = stream_name @@ -62,6 +70,12 @@ def __init__( self.max_block_ms = max_block_ms self.max_batch_mb = max_batch_mb self.max_delay_ms = max_delay_ms + self.reclaim_pending = reclaim_pending + self.reclaim_min_idle_ms = reclaim_min_idle_ms + self.max_deliveries = max_deliveries + #: Messages this process gave up on. Only ever grows; read by tests and logs. + self.dropped_messages = 0 + self._last_reclaim_at = 0.0 async def create_consumer_group(self): """Create consumer group if it doesn't exist. Safe to call multiple times (idempotent).""" @@ -141,6 +155,127 @@ async def read_batch(self) -> List[Tuple[bytes, Dict[bytes, bytes]]]: log.error(f"{self.log_prefix} Failed to read batch: {e}") return [] + def describe_message(self, data: Dict[bytes, bytes]) -> Optional[str]: + """Subclass hook: a short identity for a dropped message, for the loss log.""" + return None + + def is_permanent_failure( + self, + msg_id: bytes, + data: Dict[bytes, bytes], + ) -> bool: + """Subclass hook: whether this exact message is known not to succeed on retry.""" + return False + + async def reclaim_batch(self) -> List[Tuple[bytes, Dict[bytes, bytes]]]: + """Re-deliver entries an earlier pass left unacknowledged. + + `read_batch` only ever asks Redis for `>`, so an entry that is never acknowledged is + invisible to every later read of this group. Without this pass, "skip the ACK so Redis + retries it" means "lose it quietly with a growing pending list". Redis' delivery count + bounds retries only for a message the subclass has identified as permanently invalid; + it cannot distinguish a poison message from a transient write-path outage. + """ + if not self.reclaim_pending: + return [] + + # One XPENDING per idle window, not one per loop turn: a busy stream spins this loop + # as fast as Postgres answers, and the pending list cannot change faster than the + # window anyway. + now = time.monotonic() + if (now - self._last_reclaim_at) * 1000 < self.reclaim_min_idle_ms: + return [] + self._last_reclaim_at = now + + try: + pending = await self.redis.xpending_range( + name=self.stream_name, + groupname=self.consumer_group, + min="-", + max="+", + count=self.max_batch_size, + # A zero window means "no idle filter", not "idle exactly zero". + idle=self.reclaim_min_idle_ms or None, + ) + except Exception as e: + log.error(f"{self.log_prefix} Failed to read pending entries: {e}") + return [] + + if not pending: + return [] + + deliveries = { + entry["message_id"]: int(entry["times_delivered"]) for entry in pending + } + + try: + claimed = await self.redis.xclaim( + name=self.stream_name, + groupname=self.consumer_group, + consumername=self.consumer_name, + min_idle_time=self.reclaim_min_idle_ms, + message_ids=list(deliveries.keys()), + ) + except Exception as e: + log.error(f"{self.log_prefix} Failed to claim pending entries: {e}") + return [] + + # XCLAIM returns nothing for an entry whose stream payload is already gone (MAXLEN + # trim), and removes it from the pending list itself. + retry: List[Tuple[bytes, Dict[bytes, bytes]]] = [] + expired: List[Tuple[bytes, Dict[bytes, bytes]]] = [] + over_budget = 0 + for msg_id, data in claimed: + if not data: + continue + if deliveries.get(msg_id, 1) >= self.max_deliveries: + over_budget += 1 + if self.is_permanent_failure(msg_id, data): + expired.append((msg_id, data)) + continue + retry.append((msg_id, data)) + + if expired: + await self.drop_expired(expired) + elif over_budget: + log.warning( + f"{self.log_prefix} Keeping over-budget messages: failure is not known to be permanent", + stream=self.stream_name, + group=self.consumer_group, + count=over_budget, + ) + + if retry: + log.warning( + f"{self.log_prefix} Redelivering unacknowledged messages", + stream=self.stream_name, + group=self.consumer_group, + count=len(retry), + ) + + return retry + + async def drop_expired(self, entries: List[Tuple[bytes, Dict[bytes, bytes]]]): + """Give up on entries that failed `max_deliveries` times, loudly. + + This is data loss. It is preferred over an unbounded retry because a single entry the + write path can never accept would otherwise stall every later entry in the group. The + log line names each lost message so the loss is countable after the fact. + """ + self.dropped_messages += len(entries) + log.error( + f"{self.log_prefix} Dropping messages after repeated delivery failures", + stream=self.stream_name, + group=self.consumer_group, + max_deliveries=self.max_deliveries, + count=len(entries), + messages=[ + self.describe_message(data) or repr(msg_id) for msg_id, data in entries + ], + dropped_total=self.dropped_messages, + ) + await self.ack_and_delete([msg_id for msg_id, _ in entries]) + async def ack_and_delete(self, message_ids: List[bytes]): """ACK and DELETE messages after successful processing.""" if not message_ids: @@ -168,10 +303,10 @@ async def run(self): Main worker loop. Flow: - 1. Read batch via XREADGROUP + 1. Reclaim entries an earlier pass left unacknowledged, else read via XREADGROUP 2. Process batch - 3. ACK + DEL on success - 4. On error, messages remain pending for retry + 3. ACK + DEL only the message ids `process_batch` reports as durable + 4. Everything else stays pending and comes back through step 1 """ log.info( f"{self.log_prefix} Starting worker", @@ -183,7 +318,9 @@ async def run(self): while True: try: - batch = await self.read_batch() + batch = await self.reclaim_batch() + if not batch: + batch = await self.read_batch() if not batch: continue diff --git a/api/oss/src/utils/env.py b/api/oss/src/utils/env.py index 74edaafcca0..88af636cbad 100644 --- a/api/oss/src/utils/env.py +++ b/api/oss/src/utils/env.py @@ -1,7 +1,7 @@ import os import hashlib import warnings -from typing import List, Optional +from typing import List, Literal, Optional from uuid import getnode from json import loads from urllib.parse import urlparse, quote_plus @@ -512,6 +512,31 @@ def _validate_mode(self) -> "RedactionConfig": # --------------------------------------------------------------------------- +def _parse_sessions_late_output() -> Literal["quarantine", "reject"]: + value = (os.getenv("AGENTA_SESSIONS_LATE_OUTPUT") or "quarantine").strip().lower() + if value in ("quarantine", "reject"): + return value + warnings.warn( + f"AGENTA_SESSIONS_LATE_OUTPUT={value!r} is not recognized; " + "behaving as 'quarantine'.", + stacklevel=2, + ) + return "quarantine" + + +def _sessions_durable_stop_enabled() -> bool: + return (os.getenv("AGENTA_SESSIONS_DURABLE_STOP") or "true").lower() in _TRUTHY + + +def _parse_sessions_watchdog_stale_heartbeat_seconds() -> int: + configured = _parse_optional_positive_int_env( + "AGENTA_SESSIONS_WATCHDOG_STALE_HEARTBEAT_SECONDS" + ) + if configured is not None: + return configured + return 90 if _sessions_durable_stop_enabled() else 300 + + class SessionsRecordsConfig(BaseModel): """Durable session-record ingest tuning (server-side history reconstruction).""" @@ -523,6 +548,24 @@ class SessionsRecordsConfig(BaseModel): os.getenv("AGENTA_RECORDS_SMART_TRUNCATION") or "true" ).lower() in _TRUTHY + # How long a record message the worker failed to write sits unacknowledged before the + # worker claims it back and tries again. + reclaim_idle_ms: int = Field( + default_factory=lambda: int( + os.getenv("AGENTA_RECORDS_RECLAIM_IDLE_MS") or 30_000 + ), + ge=0, + validate_default=True, + ) + + # Deliveries after which a record message is dropped instead of retried forever. A message + # Postgres never accepts would otherwise hold every later message in the group. + max_deliveries: int = Field( + default_factory=lambda: int(os.getenv("AGENTA_RECORDS_MAX_DELIVERIES") or 5), + ge=1, + validate_default=True, + ) + model_config = ConfigDict(extra="ignore") @@ -561,11 +604,110 @@ class SessionAttachmentsConfig(BaseModel): model_config = ConfigDict(extra="ignore") +class SessionWatchdogConfig(BaseModel): + """The execution watchdog: how long a running turn may go silent before it is settled. + + The rule is HEARTBEAT AGE, not lease expiry. The Redis `alive` and `running` keys carry a + one-hour TTL, so "shortly after the lease expires" would mean an hour after the runner + died. The runner beats every `heartbeat_interval_seconds` (30) and the beat is mirrored + onto `session_streams.updated_at`, so the age of that column is the real liveness signal. + + A turn is declared lost when its stream row still claims `is_running` and its last + heartbeat is older than `stale_heartbeat_seconds`. Durable Stop uses 90 seconds (three + missed beats); flag-off deployments retain the pre-milestone 300-second default. + + Only a turn that still claims `is_running` is eligible. A turn parked for a human sends a + final beat with `is_running: false` and then stops beating on purpose; that state is + resumable, not lost, and the watchdog must never end it. + + Raise `stale_heartbeat_seconds` if a healthy deployment settles live turns. Lower it to + settle a dead turn sooner. It is a plain restart-time setting; nothing else changes. + """ + + # Maximum age of the last heartbeat before a RUNNING turn is declared lost. + stale_heartbeat_seconds: int = _parse_sessions_watchdog_stale_heartbeat_seconds() + + # How long an ALIVE-but-not-running row (between turns, or parked awaiting a human) is left + # alone before it is RECLAIMED. That state is resumable, so it is keyed to the 30-minute + # approval TTL rather than to three missed beats. It does not govern whether such a row owes + # its turn a terminal record: that question is asked of the records plane on the + # `stale_heartbeat_seconds` clock, because a durable Stop clears `is_running` before the + # runner has written its own ending. + idle_grace_seconds: int = ( + _parse_optional_positive_int_env("AGENTA_SESSIONS_WATCHDOG_IDLE_GRACE_SECONDS") + or 1_800 + ) + + # How often the watchdog runs. + interval_seconds: int = ( + _parse_optional_positive_int_env("AGENTA_SESSIONS_WATCHDOG_INTERVAL_SECONDS") + or 60 + ) + + # Rows settled per pass. A backlog drains over successive passes, not one huge commit. + batch_size: int = ( + _parse_optional_positive_int_env("AGENTA_SESSIONS_WATCHDOG_BATCH_SIZE") or 500 + ) + + model_config = ConfigDict(extra="ignore") + + +class SessionsCommandsConfig(BaseModel): + """Durable session commands: how a Stop reaches the runner, and how long it may wait. + + `adapter` picks the control-delivery transport behind `ControlDeliveryPort`: + + * `direct` — the API posts the command to the runner's own `/cancel`, over the + authenticated hop that already carries hard kill. One runner process, no held + connection, no poll loop. This is the default. + * `long_poll` — the runner holds a claim request open and the API answers it. Correct for + two or more runner replicas and for a runner the API cannot reach inbound. Not built in + this slice; naming it here fails loudly rather than silently falling back. + + `direct` calls one service address, so with two runner replicas behind a load balancer the + call lands on the right process only by luck. Nothing here guards that, on purpose: the + detector is exact and lives in the service, where a `not_held` for a session that is alive + and beating is the wrong-replica failure and nothing else produces it. + """ + + adapter: str = os.getenv("AGENTA_SESSIONS_CONTROL_ADAPTER") or "direct" + + # How long a claimed command may go unreported before the settlement sweep acts. Three + # heartbeat intervals. + lease_seconds: int = ( + _parse_optional_positive_int_env("AGENTA_SESSIONS_COMMAND_LEASE_SECONDS") or 90 + ) + # Bounds a delivery loop where a runner accepts a command and never reports. + max_deliveries: int = ( + _parse_optional_positive_int_env("AGENTA_SESSIONS_COMMAND_MAX_DELIVERIES") or 3 + ) + sweep_seconds: int = ( + _parse_optional_positive_int_env("AGENTA_SESSIONS_COMMAND_SWEEP_SECONDS") or 10 + ) + # A command nobody ever claimed is a runner that is not there. + admission_timeout_seconds: int = ( + _parse_optional_positive_int_env( + "AGENTA_SESSIONS_COMMAND_ADMISSION_TIMEOUT_SECONDS" + ) + or 90 + ) + # How long the direct call waits for the runner to acknowledge. The runner answers before + # it cancels anything, so this covers a network hop, not a harness cancel. + delivery_timeout_seconds: float = float( + os.getenv("AGENTA_SESSIONS_COMMAND_DELIVERY_TIMEOUT_SECONDS") or 5.0 + ) + model_config = ConfigDict(extra="ignore") + + class SessionsConfig(BaseModel): """Agenta sessions sub-namespace.""" + durable_stop: bool = _sessions_durable_stop_enabled() + late_output: Literal["quarantine", "reject"] = _parse_sessions_late_output() attachments: SessionAttachmentsConfig = SessionAttachmentsConfig() + commands: SessionsCommandsConfig = SessionsCommandsConfig() records: SessionsRecordsConfig = SessionsRecordsConfig() + watchdog: SessionWatchdogConfig = SessionWatchdogConfig() model_config = ConfigDict(extra="ignore") diff --git a/api/oss/src/utils/logging.py b/api/oss/src/utils/logging.py index 0d3ba469809..16d3779c8e4 100644 --- a/api/oss/src/utils/logging.py +++ b/api/oss/src/utils/logging.py @@ -208,6 +208,14 @@ def warn(self, *a, **k): def error(self, *a, **k): self._log("error", *a, **k) + def exception(self, *a, **k): + # Mirror stdlib `Logger.exception`: log at error level with the active + # traceback. Without this method a caller reaching for `log.exception(...)` + # -- the natural thing to write inside an `except` block -- would raise + # AttributeError from inside the handler and take the caller down with it. + k.setdefault("exc_info", True) + self._log("error", *a, **k) + def critical(self, *a, **k): self._log("critical", *a, **k) diff --git a/api/oss/tests/pytest/unit/middlewares/test_auth_public_endpoints.py b/api/oss/tests/pytest/unit/middlewares/test_auth_public_endpoints.py new file mode 100644 index 00000000000..5108924ead3 --- /dev/null +++ b/api/oss/tests/pytest/unit/middlewares/test_auth_public_endpoints.py @@ -0,0 +1,35 @@ +import pytest +from starlette.requests import Request + +from oss.src.middlewares.auth import _check_authentication_token +from oss.src.utils.exceptions import UnauthorizedException + + +def _request(path: str) -> Request: + return Request( + { + "type": "http", + "method": "POST", + "path": path, + "headers": [], + "query_string": b"", + "scheme": "http", + "server": ("testserver", 80), + "root_path": "", + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix", ["", "/api"]) +async def test_session_named_control_still_requires_project_auth(prefix): + with pytest.raises(UnauthorizedException): + await _check_authentication_token(_request(f"{prefix}/sessions/control/cancel")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix", ["", "/api"]) +async def test_runner_command_outcome_route_remains_auth_exempt(prefix): + await _check_authentication_token( + _request(f"{prefix}/sessions/control/commands/command-id/outcome") + ) diff --git a/api/oss/tests/pytest/unit/sessions/test_cancel_cancels_pending_interactions.py b/api/oss/tests/pytest/unit/sessions/test_cancel_cancels_pending_interactions.py new file mode 100644 index 00000000000..05dae17f5a4 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_cancel_cancels_pending_interactions.py @@ -0,0 +1,224 @@ +"""Stop cancels the stopped turn's pending interactions. + +`requirements.md:149` asks for it and only KILL did it (`delete_session_stream` calls +`cancel_session_pending`). The CANCEL branch did not, so a stopped session kept an approval card +whose buttons answered a turn that no longer existed (#6315). + +These pin the router wiring: CANCEL cancels pending gates for the turns it ended, SEND / STEER / +ATTACH do not, and a cancel that ended no turn falls back to the whole session (nothing holds +it, so nothing can ever answer those gates — the same reasoning as kill). +""" + +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +import pytest +from fastapi import FastAPI, Request + +from oss.src.apis.fastapi.sessions.router import SessionStreamsRouter +from oss.src.core.sessions.streams.dtos import ( + CommandMode, + SessionStreamCommandRequest, + SessionStreamCommandResponse, +) + + +_SESSION = "session_stop-gates" + + +def _make_authed_request(app: FastAPI, project_id, user_id) -> Request: + scope = { + "type": "http", + "method": "POST", + "path": "/sessions/streams/", + "headers": [], + "app": app, + } + request = Request(scope) + request.state.project_id = str(project_id) + request.state.user_id = str(user_id) + return request + + +def _patched_access(allowed: bool): + return patch( + "oss.src.apis.fastapi.sessions.router.check_action_access", + new_callable=AsyncMock, + return_value=allowed, + ) + + +# The route derives the mode from the PAYLOAD, not from the service's answer, because it must know +# whether this is a cancel before it runs the concurrency check. So each payload below is the real +# inputs x force combination for its mode, not a stub of one. +_CANCEL = SessionStreamCommandRequest(session_id=_SESSION) +_ATTACH = SessionStreamCommandRequest(session_id=_SESSION, force=True) +_SEND = SessionStreamCommandRequest( + session_id=_SESSION, + data={"inputs": {"messages": [{"role": "user", "content": "go"}]}}, +) +_STEER = SessionStreamCommandRequest( + session_id=_SESSION, + force=True, + data={"inputs": {"messages": [{"role": "user", "content": "go"}]}}, +) + + +async def _post( + response: SessionStreamCommandResponse, + payload, + *, + cleanup_error: Exception | None = None, +): + """Drive the route with a stubbed service that returns `response`.""" + service = AsyncMock() + service.clock_ms.return_value = 1_000 + service.command.return_value = response + interactions = AsyncMock() + interactions.cancel_session_pending.side_effect = cleanup_error + interactions.cancel_session_pending.return_value = 1 + router = SessionStreamsRouter(service=service, interactions_service=interactions) + + project_id = uuid4() + user_id = uuid4() + app = FastAPI() + request = _make_authed_request(app, project_id, user_id) + + with _patched_access(True): + result = await router.set_session_stream(request=request, payload=payload) + return result, interactions, project_id, service + + +@pytest.mark.asyncio +async def test_cancel_cancels_pending_gates_of_the_cancelled_turn(): + result, interactions, project_id, _ = await _post( + SessionStreamCommandResponse( + mode=CommandMode.cancel, + session_id=_SESSION, + turn_id="turn-1", + cancelled_turn_ids=["turn-1"], + detached=True, + ), + _CANCEL, + ) + + assert result.mode == CommandMode.cancel + interactions.cancel_session_pending.assert_awaited_once() + kwargs = interactions.cancel_session_pending.await_args.kwargs + assert kwargs["project_id"] == project_id + assert kwargs["session_id"] == _SESSION + assert kwargs["only_turn_id"] == "turn-1" + + +@pytest.mark.asyncio +async def test_cancel_that_ended_no_turn_cancels_every_pending_gate(): + _, interactions, _, _ = await _post( + SessionStreamCommandResponse( + mode=CommandMode.cancel, + session_id=_SESSION, + cancelled_turn_ids=[], + detached=True, + ), + _CANCEL, + ) + + interactions.cancel_session_pending.assert_awaited_once() + assert "only_turn_id" not in interactions.cancel_session_pending.await_args.kwargs + + +@pytest.mark.asyncio +async def test_cancel_returns_the_accepted_response_when_gate_cleanup_fails(): + response = SessionStreamCommandResponse( + mode=CommandMode.cancel, + session_id=_SESSION, + turn_id="turn-1", + cancelled_turn_ids=["turn-1"], + detached=True, + ) + + result, interactions, _, _ = await _post( + response, + _CANCEL, + cleanup_error=RuntimeError("cleanup unavailable"), + ) + + assert result == response + interactions.cancel_session_pending.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_cancel_scopes_each_call_to_one_turn(): + """`alive` and `running` can be held by different turns during a handover; both die.""" + _, interactions, _, _ = await _post( + SessionStreamCommandResponse( + mode=CommandMode.cancel, + session_id=_SESSION, + turn_id="turn-1", + cancelled_turn_ids=["turn-1", "turn-2"], + detached=True, + ), + _CANCEL, + ) + + assert interactions.cancel_session_pending.await_count == 2 + targeted = [ + call.kwargs["only_turn_id"] + for call in interactions.cancel_session_pending.await_args_list + ] + assert targeted == ["turn-1", "turn-2"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mode,payload", + [ + (CommandMode.send, _SEND), + (CommandMode.steer, _STEER), + (CommandMode.attach, _ATTACH), + ], +) +async def test_non_cancel_modes_leave_pending_gates_alone(mode, payload): + """A steer's own turn-start sweep owns the prior turn's gates. Stop must not duplicate it.""" + _, interactions, _, _ = await _post( + SessionStreamCommandResponse( + mode=mode, session_id=_SESSION, turn_id="turn-9", detached=False + ), + payload, + ) + + interactions.cancel_session_pending.assert_not_awaited() + + +# --------------------------------------------------------------------------- # +# The concurrency limit must not refuse a Stop +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_cancel_skips_the_concurrency_limit(): + """A project at its limit must still be able to stop the runs that hold the limit.""" + _, _, _, service = await _post( + SessionStreamCommandResponse( + mode=CommandMode.cancel, + session_id=_SESSION, + turn_id="turn-1", + cancelled_turn_ids=["turn-1"], + detached=True, + ), + _CANCEL, + ) + + service.check_runner_concurrency_limit.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("payload", [_SEND, _STEER, _ATTACH]) +async def test_every_other_mode_still_checks_the_concurrency_limit(payload): + _, _, _, service = await _post( + SessionStreamCommandResponse( + mode=CommandMode.send, session_id=_SESSION, turn_id="turn-1" + ), + payload, + ) + + service.check_runner_concurrency_limit.assert_awaited_once() diff --git a/api/oss/tests/pytest/unit/sessions/test_cancel_stop_guard.py b/api/oss/tests/pytest/unit/sessions/test_cancel_stop_guard.py new file mode 100644 index 00000000000..e033293b71f --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_cancel_stop_guard.py @@ -0,0 +1,454 @@ +"""The Stop guard: a cancel must not kill a turn the caller never meant to cancel. + +Before this, CANCEL called `_displace_turns` unconditionally, which tombstones whichever turn +holds `alive`/`running` at that instant. A Stop pressed for turn one but applied after turn one +ended and turn two started therefore killed turn two, and the tombstone lives for +SUPERSEDED_TTL_SECONDS with a refresh on every read — so the session stays wedged (#6417). + +Two guards close it, in order of strength: + + 1. `expected_execution_id` on the request names the turn. The public DTO keeps the RFC's + name; internally it IS a turn id. A different turn holding the session means the turn the + caller meant is gone: refuse with `SessionTurnMismatch` (409) and touch nothing. + 2. With no id, refuse when a holding turn started AFTER the request arrived. This needs the + turn-start key the coordination plane now records, because nothing else knows when a turn + began early enough to be useful. + +Also covered: cancel reports the turns it ended, which is what lets the router cancel exactly +those turns' pending gates. +""" + +from typing import Optional +from unittest.mock import patch +from uuid import UUID, uuid4 + +import pytest +import pytest_asyncio + +from oss.src.core.sessions.streams.dtos import ( + CommandMode, + SessionHeartbeatRequest, + SessionStream, + SessionStreamCommandRequest, +) +from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.core.sessions.streams.types import SessionTurnMismatch +from oss.src.dbs.redis.sessions.locks import ( + acquire_alive, + acquire_running, + get_alive_owner, + get_running_owner, + is_turn_superseded, + record_turn_start, +) + +from unit.sessions.test_project_scoped_locks import _FakeRedis + + +_PROJECT = uuid4() +_USER = uuid4() +_SESSION = "session_stop-guard" + + +class _FakeStreamsDAO: + """Enough of the streams DAO for the cancel path: read, create, update.""" + + def __init__(self, existing: Optional[SessionStream] = None): + self.row = existing + + async def get_by_session_id(self, *, project_id: UUID, session_id: str): + return self.row + + async def create(self, *, project_id, user_id, stream): + self.row = SessionStream( + id=uuid4(), + project_id=project_id, + session_id=stream.session_id, + flags=stream.flags, + turn_id=stream.turn_id, + ) + return self.row + + async def update(self, *, project_id, user_id, session_id, stream): + prior = self.row + self.row = SessionStream( + id=prior.id if prior else uuid4(), + project_id=project_id, + session_id=session_id, + flags=stream.flags + if stream.flags is not None + else (prior.flags if prior else None), + turn_id=stream.turn_id + if stream.turn_id is not None + else (prior.turn_id if prior else None), + ) + return self.row + + async def fill_missing(self, *, project_id, session_id, name=None, references=None): + return self.row + + async def unarchive_by_session_id(self, *, project_id, user_id, session_id): + return self.row + + async def clear_archived_by_session_id(self, *, project_id, user_id, session_id): + return self.row + + async def delete_by_session_id(self, *, project_id, session_id): + return True + + +@pytest_asyncio.fixture +async def lock_engine(): + from oss.src.dbs.redis.shared.engine import LockEngine + + eng = LockEngine() + with patch.object(eng, "_client", return_value=_FakeRedis()): + yield eng + + +def _service(lock_engine, dao=None): + return SessionStreamsService( + streams_dao=dao or _FakeStreamsDAO(), lock_engine=lock_engine + ) + + +def _cancel(expected: Optional[str] = None) -> SessionStreamCommandRequest: + """A Stop: no inputs, force=False. That is what the browser sends.""" + return SessionStreamCommandRequest( + session_id=_SESSION, + expected_execution_id=expected, + ) + + +async def _seat_turn(lock_engine, turn_id: str, started_at_ms: Optional[int] = None): + """Put `turn_id` in the nest the way a running turn holds it, with a start time.""" + await acquire_alive( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id=turn_id + ) + await acquire_running( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id=turn_id + ) + await record_turn_start( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id=turn_id, + started_at_ms=started_at_ms, + ) + + +# --------------------------------------------------------------------------- # +# Guard 1 — expected_execution_id +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_cancel_with_matching_expected_id_cancels_that_turn(lock_engine): + svc = _service(lock_engine) + await _seat_turn(lock_engine, "turn-1", started_at_ms=1_000) + + result = await svc.command( + project_id=_PROJECT, user_id=_USER, request=_cancel("turn-1") + ) + + assert result.mode == CommandMode.cancel + assert result.turn_id == "turn-1" + assert result.cancelled_turn_ids == ["turn-1"] + assert await is_turn_superseded( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-1" + ) + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + is None + ) + + +@pytest.mark.asyncio +async def test_cancel_with_stale_expected_id_is_refused_and_touches_nothing( + lock_engine, +): + """The headline case: the Stop names turn one, turn two now holds the session.""" + svc = _service(lock_engine) + await _seat_turn(lock_engine, "turn-2", started_at_ms=2_000) + + with pytest.raises(SessionTurnMismatch) as excinfo: + await svc.command(project_id=_PROJECT, user_id=_USER, request=_cancel("turn-1")) + + assert excinfo.value.expected_turn_id == "turn-1" + assert excinfo.value.actual_turn_id == "turn-2" + + # Turn two keeps the whole nest and is NOT tombstoned — that is the bug this closes. + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-2" + ) + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-2" + ) + assert not await is_turn_superseded( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-2" + ) + + +@pytest.mark.asyncio +async def test_cancel_refuses_owner_replaced_immediately_before_atomic_displacement( + lock_engine, monkeypatch +): + svc = _service(lock_engine) + await _seat_turn(lock_engine, "turn-1", started_at_ms=1_000) + redis = lock_engine._client() + original_eval = redis.eval + + async def replace_owner_then_eval(script, numkeys, *keys_and_args): + if "AGENTA_DISPLACE_TURNS" in script: + await redis.set(keys_and_args[0], b"turn-2", ex=60) + await redis.set(keys_and_args[1], b"turn-2", ex=60) + return await original_eval(script, numkeys, *keys_and_args) + + monkeypatch.setattr(redis, "eval", replace_owner_then_eval) + + with pytest.raises(SessionTurnMismatch) as excinfo: + await svc.command(project_id=_PROJECT, user_id=_USER, request=_cancel("turn-1")) + + assert excinfo.value.actual_turn_id == "turn-2" + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-2" + ) + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-2" + ) + assert not await is_turn_superseded( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-2" + ) + + +@pytest.mark.asyncio +async def test_cancel_with_expected_id_tombstones_a_turn_that_holds_nothing( + lock_engine, +): + """A named turn whose beat is still in flight must not be able to re-take the session.""" + svc = _service(lock_engine) + + result = await svc.command( + project_id=_PROJECT, user_id=_USER, request=_cancel("turn-1") + ) + + assert result.cancelled_turn_ids == ["turn-1"] + assert await is_turn_superseded( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-1" + ) + + +@pytest.mark.asyncio +async def test_blank_expected_id_is_read_as_absent(lock_engine): + """A whitespace guard is a client bug. Reading it as "no guard" is the safe failure.""" + request = SessionStreamCommandRequest( + session_id=_SESSION, expected_execution_id=" " + ) + assert request.expected_execution_id is None + + +# --------------------------------------------------------------------------- # +# Guard 2 — arrival time, for callers that send no id +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_cancel_without_id_refuses_a_turn_that_started_after_it_arrived( + lock_engine, +): + svc = _service(lock_engine) + # Far in the future relative to this cancel's arrival: the turn began after the ask. + await _seat_turn(lock_engine, "turn-2", started_at_ms=4_000_000_000_000) + + with pytest.raises(SessionTurnMismatch) as excinfo: + await svc.command(project_id=_PROJECT, user_id=_USER, request=_cancel()) + + assert excinfo.value.expected_turn_id is None + assert excinfo.value.actual_turn_id == "turn-2" + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-2" + ) + assert not await is_turn_superseded( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-2" + ) + + +@pytest.mark.asyncio +async def test_cancel_without_id_still_cancels_a_turn_that_started_earlier(lock_engine): + svc = _service(lock_engine) + await _seat_turn(lock_engine, "turn-1", started_at_ms=1_000) + + result = await svc.command(project_id=_PROJECT, user_id=_USER, request=_cancel()) + + assert result.cancelled_turn_ids == ["turn-1"] + assert await is_turn_superseded( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-1" + ) + + +@pytest.mark.asyncio +async def test_cancel_without_id_still_cancels_a_turn_with_no_recorded_start( + lock_engine, +): + """Unknown must mean unknown, never "new". A running pre-deploy turn stays stoppable.""" + svc = _service(lock_engine) + await acquire_alive( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-old" + ) + await acquire_running( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-old" + ) + + result = await svc.command(project_id=_PROJECT, user_id=_USER, request=_cancel()) + + assert result.cancelled_turn_ids == ["turn-old"] + + +@pytest.mark.asyncio +async def test_cancel_ordering_uses_the_shared_redis_clock(lock_engine): + svc = _service(lock_engine) + redis = lock_engine._client() + redis.now_ms = 10_000 + arrived_at_ms = await svc.clock_ms() + redis.now_ms = 11_000 + await _seat_turn(lock_engine, "turn-2") + + with pytest.raises(SessionTurnMismatch): + await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=_cancel(), + arrived_at_ms=arrived_at_ms, + ) + + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-2" + ) + + +# --------------------------------------------------------------------------- # +# The turn-start record itself +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_start_turn_records_a_start_time(lock_engine): + from oss.src.dbs.redis.sessions.locks import get_turn_start + + svc = _service(lock_engine) + turn_id = await svc._start_turn( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert ( + await get_turn_start( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id=turn_id, + ) + is not None + ) + + +@pytest.mark.asyncio +async def test_heartbeat_records_a_start_time_for_a_runner_minted_turn(lock_engine): + """A browser turn's id is minted by the runner, so its first beat is where it is stamped.""" + from oss.src.dbs.redis.sessions.locks import get_turn_start + + svc = _service(lock_engine) + await svc.heartbeat( + project_id=_PROJECT, + request=SessionHeartbeatRequest( + session_id=_SESSION, replica_id="replica-a", turn_id="turn-runner" + ), + ) + + assert ( + await get_turn_start( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-runner", + ) + is not None + ) + + +@pytest.mark.asyncio +async def test_turn_start_is_written_once(lock_engine): + """Later beats refresh the record, never move it: the guard needs the FIRST start.""" + from oss.src.dbs.redis.sessions.locks import get_turn_start + + first = await record_turn_start( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-1", + started_at_ms=1_000, + ) + second = await record_turn_start( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-1", + started_at_ms=9_000, + ) + + assert first == 1_000 + assert second == 1_000 + assert ( + await get_turn_start( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-1", + ) + == 1_000 + ) + + +# --------------------------------------------------------------------------- # +# Steer and kill are not guarded — both mean "take this session from whoever has it" +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_steer_is_not_subject_to_the_guard(lock_engine): + svc = _service(lock_engine) + await _seat_turn(lock_engine, "turn-2", started_at_ms=4_000_000_000_000) + + result = await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest( + session_id=_SESSION, + force=True, + data={"inputs": {"messages": [{"role": "user", "content": "again"}]}}, + ), + ) + + assert result.mode == CommandMode.steer + assert await is_turn_superseded( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-2" + ) diff --git a/api/oss/tests/pytest/unit/sessions/test_command_claim_unknown_kind.py b/api/oss/tests/pytest/unit/sessions/test_command_claim_unknown_kind.py new file mode 100644 index 00000000000..849d0237290 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_command_claim_unknown_kind.py @@ -0,0 +1,89 @@ +"""A runner's claim must take the commands it understands past a row it cannot map. + +`claim_commands` returned `[map_command_dbe_to_dto(dbe) for dbe in claimed]`, the same batch +map that poisoned the abandoned-command sweep: a newer API replica can write a command `kind` +this older replica's enum does not know, and `map_command_dbe_to_dto` raises `ValueError` on +that row. One such row in a claimed batch would have thrown away the whole claim, including a +Stop the runner could act on. The claim path now maps through +`_map_commands_skipping_unmappable`, which skips the rows this API cannot map, warns once per +batch, and returns the rest. +""" + +from datetime import datetime, timezone +from uuid import uuid4 + +from types import SimpleNamespace + +from oss.src.core.sessions.commands.dtos import ( + SessionCommandKind, + SessionCommandState, +) +from oss.src.dbs.postgres.sessions.commands import dao as commands_dao + + +def _row(kind: str): + """A claimed command row as the DAO reads it, with a given `kind` string.""" + return SimpleNamespace( + id=uuid4(), + created_at=datetime.now(timezone.utc), + updated_at=None, + deleted_at=None, + created_by_id=None, + updated_by_id=None, + deleted_by_id=None, + project_id=uuid4(), + session_id="sess-" + kind, + kind=kind, + target_turn_id="turn-1", + expected_turn_id=None, + data=None, + state=SessionCommandState.claimed.value, + claimed_by="runner-1", + claim_expires_at=datetime.now(timezone.utc), + claim_count=1, + outcome=None, + idempotency_key=None, + settled_at=None, + tags=None, + meta=None, + ) + + +class _RecordingLog: + def __init__(self): + self.warnings = [] + + def warning(self, *args, **kwargs): + self.warnings.append((args, kwargs)) + + +def test_a_claimable_stop_survives_an_unknown_kind_in_the_batch(monkeypatch): + recorder = _RecordingLog() + monkeypatch.setattr(commands_dao, "log", recorder) + + stop = _row(SessionCommandKind.cancel.value) + unknown = _row("continue_interaction") + + mapped = commands_dao._map_commands_skipping_unmappable( + [stop, unknown], context="claimed" + ) + + # The Stop is handed to the runner; the unknown row is left for a replica that knows it. + assert [c.id for c in mapped] == [stop.id] + assert unknown.id not in {c.id for c in mapped} + + +def test_the_claim_warning_names_the_claimed_context_and_the_kind(monkeypatch): + recorder = _RecordingLog() + monkeypatch.setattr(commands_dao, "log", recorder) + + commands_dao._map_commands_skipping_unmappable( + [_row(SessionCommandKind.cancel.value), _row("continue_interaction")], + context="claimed", + ) + + assert len(recorder.warnings) == 1 + args = recorder.warnings[0][0] + assert args[1] == 1 # one unmappable row + assert args[2] == "claimed" # the batch context + assert "continue_interaction=1" in args[3] diff --git a/api/oss/tests/pytest/unit/sessions/test_command_matrix_inputs_data.py b/api/oss/tests/pytest/unit/sessions/test_command_matrix_inputs_data.py index d260c85e830..84b6e854003 100644 --- a/api/oss/tests/pytest/unit/sessions/test_command_matrix_inputs_data.py +++ b/api/oss/tests/pytest/unit/sessions/test_command_matrix_inputs_data.py @@ -23,13 +23,20 @@ from agenta.sdk.models.workflows import WorkflowServiceRequestData +from oss.src.apis.fastapi.sessions.models import SessionCancelRequest from oss.src.core.sessions.streams.dtos import ( CommandMode, SessionStream, SessionStreamCommandRequest, ) from oss.src.core.sessions.streams.service import SessionStreamsService -from oss.src.core.sessions.streams.types import SessionTurnInUse +from oss.src.core.sessions.streams.types import SessionTurnInUse, SessionTurnMismatch +from oss.src.dbs.redis.sessions.locks import ( + acquire_alive, + get_alive_owner, + get_running_owner, + is_turn_superseded, +) from unit.sessions.test_project_scoped_locks import _FakeRedis @@ -158,6 +165,148 @@ async def test_no_inputs_no_force_is_cancel(lock_engine): assert result.mode == CommandMode.cancel +@pytest.mark.asyncio +async def test_unfenced_cancel_before_new_turn_admission_ignores_the_parked_owner( + lock_engine, +): + session_id = _session_id() + await acquire_alive( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + turn_id="turn-A", + ) + existing = SessionStream( + id=uuid4(), + project_id=_PROJECT, + session_id=session_id, + turn_id="turn-A", + ) + dao = _FakeStreamsDAO(existing) + svc = _service(lock_engine, dao=dao) + + # Turn B was submitted by the browser but has not reached `_start_turn` yet. + result = await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest(session_id=session_id), + ) + + assert result.mode == CommandMode.cancel + assert result.cancelled_turn_ids == [] + assert dao.row == existing + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=session_id + ) + == "turn-A" + ) + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=session_id + ) + is None + ) + assert not await is_turn_superseded( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + turn_id="turn-A", + ) + + +@pytest.mark.asyncio +async def test_unfenced_cancel_targets_the_turn_once_it_is_running(lock_engine): + dao = _FakeStreamsDAO() + svc = _service(lock_engine, dao=dao) + session_id = _session_id() + started = await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest( + session_id=session_id, + data=WorkflowServiceRequestData(inputs={"messages": ["hi"]}), + ), + ) + + result = await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest(session_id=session_id), + ) + + assert started.turn_id is not None + assert result.cancelled_turn_ids == [started.turn_id] + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=session_id + ) + is None + ) + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=session_id + ) + is None + ) + assert await is_turn_superseded( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + turn_id=started.turn_id, + ) + + +@pytest.mark.asyncio +async def test_cancel_with_a_stale_execution_guard_touches_no_holder(lock_engine): + svc = _service(lock_engine) + session_id = _session_id() + started = await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest( + session_id=session_id, + data=WorkflowServiceRequestData(inputs={"messages": ["first"]}), + ), + ) + + with pytest.raises(SessionTurnMismatch): + await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest( + session_id=session_id, + expected_execution_id="another-turn", + ), + ) + + assert ( + await get_alive_owner( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + ) + == started.turn_id + ) + assert ( + await get_running_owner( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + ) + == started.turn_id + ) + + +def test_expected_execution_id_schema_documents_cancel_only_guard(): + description = SessionCancelRequest.model_json_schema()["properties"][ + "expected_execution_id" + ]["description"] + + assert "only in cancel mode" in description + assert "ignored for send, steer, and attach" in description + + @pytest.mark.asyncio async def test_no_inputs_and_force_is_attach(lock_engine): svc = _service(lock_engine) diff --git a/api/oss/tests/pytest/unit/sessions/test_command_settle_unknown_kind.py b/api/oss/tests/pytest/unit/sessions/test_command_settle_unknown_kind.py new file mode 100644 index 00000000000..3bc171e3134 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_command_settle_unknown_kind.py @@ -0,0 +1,112 @@ +"""The watchdog must settle the commands it understands past a row it cannot map. + +A newer API replica can write a command `kind` (or state, or outcome) an older replica's +enums do not know. On the integration stack a `continue_interaction` row (increment 6, not on +this head) sat in the claimed table next to an abandoned Stop. The abandoned-command sweep +mapped the whole batch to DTOs before it settled any of it, and `map_command_dbe_to_dto` +raised `ValueError: 'continue_interaction' is not a valid SessionCommandKind` on that one row. +The ValueError escaped the batch, so NO command was settled and the Stop stayed pending pass +after pass. + +`_map_commands_skipping_unmappable` now skips the rows this API cannot map, warns once with the kinds and +count, and returns the rest. These tests hold that contract: the known Stop survives as a +settle candidate, the unknown row is dropped and left for a replica that knows its kind, and +the skip is logged exactly once. +""" + +from datetime import datetime, timezone +from types import SimpleNamespace +from uuid import uuid4 + +from oss.src.core.sessions.commands.dtos import ( + SessionCommandKind, + SessionCommandState, +) +from oss.src.dbs.postgres.sessions.commands import dao as commands_dao + + +def _row(kind: str): + """A claimed, abandoned command row as the DAO reads it, with a given `kind` string.""" + return SimpleNamespace( + id=uuid4(), + created_at=datetime.now(timezone.utc), + updated_at=None, + deleted_at=None, + created_by_id=None, + updated_by_id=None, + deleted_by_id=None, + project_id=uuid4(), + session_id="sess-" + kind, + kind=kind, + target_turn_id="turn-1", + expected_turn_id=None, + data=None, + state=SessionCommandState.claimed.value, + claimed_by="runner-1", + claim_expires_at=datetime.now(timezone.utc), + claim_count=1, + outcome=None, + idempotency_key=None, + settled_at=None, + tags=None, + meta=None, + ) + + +class _RecordingLog: + def __init__(self): + self.warnings = [] + + def warning(self, *args, **kwargs): + self.warnings.append((args, kwargs)) + + +def test_a_known_stop_survives_and_an_unknown_kind_is_left_alone(monkeypatch): + recorder = _RecordingLog() + monkeypatch.setattr(commands_dao, "log", recorder) + + stop = _row(SessionCommandKind.cancel.value) + unknown = _row("continue_interaction") + + mapped = commands_dao._map_commands_skipping_unmappable( + [stop, unknown], context="abandoned" + ) + + # The Stop is returned, so the sweep will settle it. + assert [c.id for c in mapped] == [stop.id] + assert mapped[0].kind is SessionCommandKind.cancel + # The unknown-kind row is dropped, not settled -- left for a replica that knows its kind. + assert unknown.id not in {c.id for c in mapped} + + +def test_the_unknown_kind_is_warned_once_with_its_kind_and_count(monkeypatch): + recorder = _RecordingLog() + monkeypatch.setattr(commands_dao, "log", recorder) + + rows = [ + _row(SessionCommandKind.cancel.value), + _row("continue_interaction"), + _row("continue_interaction"), + ] + + commands_dao._map_commands_skipping_unmappable(rows, context="abandoned") + + assert len(recorder.warnings) == 1, "exactly one warning per pass" + args = recorder.warnings[0][0] + # The message and its args name the count, the batch context, and the offending kind. + assert args[1] == 2 # two unmappable rows + assert args[2] == "abandoned" # the batch context + assert "continue_interaction=2" in args[3] + + +def test_an_all_mappable_batch_logs_nothing(monkeypatch): + recorder = _RecordingLog() + monkeypatch.setattr(commands_dao, "log", recorder) + + mapped = commands_dao._map_commands_skipping_unmappable( + [_row(SessionCommandKind.cancel.value), _row(SessionCommandKind.cancel.value)], + context="abandoned", + ) + + assert len(mapped) == 2 + assert recorder.warnings == [] diff --git a/api/oss/tests/pytest/unit/sessions/test_execution_watchdog.py b/api/oss/tests/pytest/unit/sessions/test_execution_watchdog.py new file mode 100644 index 00000000000..503f8a21780 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_execution_watchdog.py @@ -0,0 +1,1235 @@ +"""The execution watchdog must give a lost turn a real ending, exactly once. + +Before this, the sweep collapsed a dead session's flags and cleared its Redis nest, but wrote +nothing to the transcript: the turn simply stopped mid-sentence and the browser kept showing +it as running until the user reloaded. The invariant these tests hold is the RFC's — every +accepted execution reaches exactly ONE durable terminal outcome — so they check both halves: +an ending IS written for a turn that has none, and a SECOND ending is never written for a turn +that already has one. + +The threshold predicate itself is covered by `test_orphan_sweep_thresholds.py`; the fake +session here models the execution filter, order, and batch limit so the durable candidate +window is also covered. +""" + +from contextlib import asynccontextmanager +from datetime import datetime, timezone, timedelta +from typing import List, Optional, Sequence, Set, Tuple +from uuid import UUID + +import pytest + +from oss.src.core.sessions.records.dtos import ( + RECORD_SETTLED_BY_ATTRIBUTE, + SETTLED_BY_WATCHDOG, + SessionRecordEvent, +) +from oss.src.dbs.redis.sessions.contract import make_owner_value, owner_replica_id +from oss.src.dbs.redis.sessions.locks import claim_owner, is_turn_superseded +from oss.src.tasks.asyncio.sessions.orphan_sweep import ( + LOST_ERROR_CODE, + LOST_ERROR_MESSAGE, + ORPHAN_THRESHOLD_SECONDS, + IDLE_THRESHOLD_SECONDS, + SWEEP_BATCH_SIZE, + _unsettled_turns, + run_orphan_sweep, +) +from oss.src.utils.env import env + +_PROJECT_ID = UUID("00000000-0000-4000-8000-000000000001") + + +# --------------------------------------------------------------------------- # +# Fakes +# --------------------------------------------------------------------------- # + + +class _FakeRow: + def __init__( + self, + *, + session_id: str, + turn_id: Optional[str], + is_running: bool, + age_seconds: int, + ): + self.session_id = session_id + self.project_id = _PROJECT_ID + self.id = f"stream-{session_id}" + self.turn_id = turn_id + self.deleted_at = None + self.flags = { + "is_alive": True, + "is_running": is_running, + "is_attached": False, + } + self.created_at = datetime.now(timezone.utc) - timedelta(days=1) + self.updated_at = datetime.now(timezone.utc) - timedelta(seconds=age_seconds) + + +class _FakeExecutionRow: + def __init__( + self, + *, + session_id: str, + execution_id: str, + terminal_outcome: str = "stopped", + age_seconds: int = ORPHAN_THRESHOLD_SECONDS + 30, + ending_written_at: Optional[datetime] = None, + ): + self.project_id = _PROJECT_ID + self.session_id = session_id + self.execution_id = execution_id + self.terminal_outcome = terminal_outcome + self.settled_by = "runner" + self.settled_at = datetime.now(timezone.utc) - timedelta(seconds=age_seconds) + self.ending_written_at = ending_written_at + + +class _FakeResult: + def __init__(self, rows, *, rowcount=0): + self._rows = rows + self.rowcount = rowcount + + def scalars(self): + return self + + def all(self): + return self._rows + + +class _FakePgSession: + def __init__(self, rows, executions, before_stream_update=None, on_commit=None): + self._rows = rows + self._executions = executions + self._before_stream_update = before_stream_update + self._on_commit = on_commit + self.commits = 0 + + async def execute(self, stmt): + # Evaluate the sweep's two selections the way Postgres would, so a test can tell a + # collapsed row from one that only owed an ending. The ending-only statement is the + # one that filters on `turn_id IS NOT NULL`; the first statement carries the OR of + # the running and idle branches. + text = str(stmt) + now = datetime.now(timezone.utc) + + if "session_executions" in text: + if text.startswith("UPDATE"): + params = stmt.compile().params + keys = next( + value + for value in params.values() + if isinstance(value, (list, set, tuple)) + and all(isinstance(key, tuple) and len(key) == 3 for key in value) + ) + for execution in self._executions: + key = ( + execution.project_id, + execution.session_id, + execution.execution_id, + ) + if key in keys and execution.ending_written_at is None: + execution.ending_written_at = now + return _FakeResult([]) + rows = [ + execution + for execution in self._executions + if execution.terminal_outcome in {"stopped", "lost"} + and (now - execution.settled_at).total_seconds() + > ORPHAN_THRESHOLD_SECONDS + ] + if "ending_written_at IS NULL" in text: + rows = [row for row in rows if row.ending_written_at is None] + return _FakeResult( + sorted( + rows, + key=lambda row: row.settled_at, + reverse="DESC" in text, + )[:SWEEP_BATCH_SIZE] + ) + + # Both session_streams writes are Core UPDATEs of flags/updated_at keyed by row id, and + # never ORM attribute writes (finding 7). Apply them to the in-memory rows so a test + # sees what Postgres would. The collapse binds `id IN (...)`, a list. The lost-turn + # clear binds `id = ...`, a scalar. Row ids are strings here and are the only string + # bind in either statement. + if text.startswith("UPDATE") and "session_streams" in text: + if self._before_stream_update is not None: + self._before_stream_update() + self._before_stream_update = None + return _FakeResult([], rowcount=0) + params = stmt.compile().params + flags_val = next( + (v for v in params.values() if isinstance(v, dict) and "is_alive" in v), + None, + ) + ids = set() + for value in params.values(): + if isinstance(value, (list, set, tuple)): + ids.update(x for x in value if isinstance(x, (str, UUID))) + elif isinstance(value, (str, UUID)): + ids.add(value) + if flags_val is not None: + matched = 0 + for r in self._rows: + if r.id in ids: + r.flags = dict(flags_val) + r.updated_at = now + matched += 1 + return _FakeResult([], rowcount=matched) + return _FakeResult([]) + + # The lost-turn is_running clear: a session_streams SELECT keyed by a list of + # (project_id, session_id, turn_id) tuples. Return the rows those keys name that still + # read is_running true, so the sweep can clear the flag on them. + params = stmt.compile().params + key_lists = [ + value + for value in params.values() + if isinstance(value, (list, set, tuple)) + and value + and all(isinstance(key, tuple) and len(key) == 3 for key in value) + ] + if key_lists: + keys = set(key_lists[0]) + return _FakeResult( + [ + r + for r in self._rows + if r.flags.get("is_running") is True + and (r.project_id, r.session_id, str(r.turn_id)) in keys + ] + ) + + def age(row): + return (now - (row.updated_at or row.created_at)).total_seconds() + + if "IS NOT NULL" in text: + rows = [ + r + for r in self._rows + if r.flags.get("is_alive") is True + and r.flags.get("is_running") is not True + and r.turn_id is not None + and age(r) > ORPHAN_THRESHOLD_SECONDS + ] + else: + rows = [ + r + for r in self._rows + if r.flags.get("is_alive") is True + and ( + ( + r.flags.get("is_running") is True + and age(r) > ORPHAN_THRESHOLD_SECONDS + ) + or ( + r.flags.get("is_running") is not True + and age(r) > IDLE_THRESHOLD_SECONDS + ) + ) + ] + return _FakeResult(rows) + + async def commit(self): + self.commits += 1 + if self._on_commit is not None: + self._on_commit() + + +class _FakeTransactionsEngine: + def __init__( + self, + rows, + executions=None, + before_stream_update=None, + after_commit=None, + ): + self._rows = rows + self._executions = executions or [] + self._before_stream_update = before_stream_update + self._after_commit = after_commit + self.committed = False + + def _mark_committed(self): + self.committed = True + if self._after_commit is not None: + self._after_commit() + + @asynccontextmanager + async def session(self): + yield _FakePgSession( + self._rows, + self._executions, + self._before_stream_update, + self._mark_committed, + ) + + +class _FakeRedis: + def __init__(self): + self._store: dict = {} + + async def get(self, key): + return self._store.get(key) + + async def set(self, key, value, nx=False, ex=None): + if nx and key in self._store: + return None + self._store[key] = value + return True + + async def delete(self, key): + self._store.pop(key, None) + return 1 + + async def expire(self, key, ttl): + return True + + async def eval(self, script, numkeys, *keys_and_args): + def decode(value): + return value.decode() if isinstance(value, bytes) else str(value) + + keys = [decode(value) for value in keys_and_args[:numkeys]] + argv = [decode(value) for value in keys_and_args[numkeys:]] + if "AGENTA_WATCHDOG_RELEASE_TURN" in script: + alive, running, owner, superseded = keys + expected_turn, expected_owner, _ttl = argv + alive_value = decode(self._store[alive]) if alive in self._store else "" + running_value = ( + decode(self._store[running]) if running in self._store else "" + ) + owner_value = decode(self._store[owner]) if owner in self._store else "" + released_alive = int(bool(expected_turn) and alive_value == expected_turn) + released_running = int( + bool(expected_turn) and running_value == expected_turn + ) + if released_alive: + self._store.pop(alive, None) + if released_running: + self._store.pop(running, None) + foreign_turn = (alive_value and alive_value != expected_turn) or ( + running_value and running_value != expected_turn + ) + released_owner = int( + bool(expected_owner) + and owner_value == expected_owner + and not foreign_turn + ) + if released_owner: + self._store.pop(owner, None) + if expected_turn: + self._store[superseded] = b"1" + return [released_alive, released_running, released_owner] + + k = keys[0] + v = argv[0] + current = self._store.get(k) + if isinstance(current, bytes): + current = current.decode() + if len(argv) > 1: + if current is None or owner_replica_id(current) == owner_replica_id(v): + self._store[k] = v.encode() + return v.encode() + return current.encode() + # The sweep's script is release-if-owner: delete only when the value matches. + if current == v: + self._store.pop(k, None) + return 1 + return 0 + + +class _CommitObservingRedis(_FakeRedis): + def __init__(self, engine: _FakeTransactionsEngine): + super().__init__() + self._engine = engine + + async def eval(self, *args, **kwargs): + assert self._engine.committed, ( + "Redis ownership was released before the DB commit" + ) + return await super().eval(*args, **kwargs) + + +class _FakeRecordsService: + """Stands in for the records plane. `settled` is what the tracing DB already holds.""" + + def __init__(self, settled: Optional[Set[Tuple[str, str]]] = None): + self.settled = settled or set() + self.queries: List[Sequence[Tuple[str, str]]] = [] + + async def settled_turns(self, *, project_id, keys): + self.queries.append(list(keys)) + return {key for key in keys if key in self.settled} + + +@pytest.mark.anyio +async def test_terminal_record_checks_are_batched_once_per_project(anyio_backend): + other_project = UUID("00000000-0000-4000-8000-000000000002") + + class _RecordingRecords: + def __init__(self): + self.queries = [] + + async def settled_turns(self, *, project_id, keys): + self.queries.append((project_id, list(keys))) + return set() + + records = _RecordingRecords() + first_project = [ + (_PROJECT_ID, f"session-{index}", f"turn-{index}") for index in range(100) + ] + second_project = [(other_project, "session-other", "turn-other")] + + unsettled, ended, deferred = await _unsettled_turns( + records_service=records, + candidates=[*first_project, *second_project], + ) + + assert ended == set() + assert deferred == set() + assert unsettled == set(first_project + second_project) + assert [(project_id, len(keys)) for project_id, keys in records.queries] == [ + (_PROJECT_ID, 100), + (other_project, 1), + ] + + +class _FakeWatchPublisher: + def __init__(self): + self.lifecycles: List[Tuple[str, str, str]] = [] + self.changes: List[Tuple[str, str, str]] = [] + + async def lifecycle(self, *, project_id, session_id, state): + self.lifecycles.append((project_id, session_id, state)) + + async def changed(self, *, project_id, entity, id): + self.changes.append((project_id, entity, id)) + + +class _Publisher: + """Captures what the watchdog would put on the record ingest stream.""" + + def __init__(self): + self.published: List[SessionRecordEvent] = [] + + async def __call__(self, *, project_id, record_event): + self.published.append(record_event) + return True + + +class _CommandsService: + def __init__(self): + self.execution_lost_calls = [] + + async def settle_execution_lost(self, **kwargs): + assert kwargs["transaction"] is not None + self.execution_lost_calls.append(kwargs) + return True + + async def settle_abandoned_commands(self, *, now): + return 0 + + async def repair_terminal_redis(self): + return 0 + + +def _stale_running_row(session_id="sess-lost", turn_id="turn-1") -> _FakeRow: + return _FakeRow( + session_id=session_id, + turn_id=turn_id, + is_running=True, + age_seconds=ORPHAN_THRESHOLD_SECONDS + 60, + ) + + +def _collapsed(row: _FakeRow) -> bool: + return row.flags == {"is_alive": False, "is_running": False, "is_attached": False} + + +@pytest.fixture +def anyio_backend(): + return "asyncio" + + +# --------------------------------------------------------------------------- # +# Tests +# --------------------------------------------------------------------------- # + + +@pytest.mark.anyio +async def test_a_lost_turn_gets_an_error_then_a_done(anyio_backend): + """The shape a runner writes when a turn ends badly, written on its behalf. + + A lone `done` would render as a clean finish, which is the opposite of what happened, so + the error must come first and must carry the class a client can act on. + """ + row = _stale_running_row() + publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([row]), + _FakeRedis(), + records_service=_FakeRecordsService(), + publish=publisher, + ) + + assert [event.record_type for event in publisher.published] == ["error", "done"] + + error_event, done_event = publisher.published + # Both carry the writer marker. It is the ONLY thing separating this ending from a + # runner's — the wording and the `done` shape are copied deliberately — and the ingest + # guard reads it to tell a thawed runner's tail apart from ordinary history. See + # `RecordsService.append_many`. + assert error_event.attributes == { + "type": "error", + "message": LOST_ERROR_MESSAGE, + "code": LOST_ERROR_CODE, + RECORD_SETTLED_BY_ATTRIBUTE: SETTLED_BY_WATCHDOG, + } + assert done_event.attributes == { + "type": "done", + RECORD_SETTLED_BY_ATTRIBUTE: SETTLED_BY_WATCHDOG, + } + assert error_event.turn_id == "turn-1" + assert done_event.turn_id == "turn-1" + assert error_event.session_id == "sess-lost" + assert _collapsed(row), "the row must still be marked ended" + + +@pytest.mark.anyio +async def test_a_second_pass_writes_no_second_ending(anyio_backend): + """Idempotency, the guarantee the RFC asks for: exactly one terminal outcome. + + Two passes can see the same turn — a crash between the record write and the flag + collapse, or two API replicas sweeping at once. The second pass reads the record the + first one wrote and must stay silent. + """ + records = _FakeRecordsService() + first_publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([_stale_running_row()]), + _FakeRedis(), + records_service=records, + publish=first_publisher, + ) + assert len(first_publisher.published) == 2 + + # The records worker has now landed those rows in the tracing DB. + records.settled.add(("sess-lost", "turn-1")) + + second_publisher = _Publisher() + row = _stale_running_row() + await run_orphan_sweep( + _FakeTransactionsEngine([row]), + _FakeRedis(), + records_service=records, + publish=second_publisher, + ) + + assert second_publisher.published == [], ( + "a turn that already carries a terminal record must never be given a second one" + ) + assert _collapsed(row), "the row is still settled even when no record is owed" + + +@pytest.mark.anyio +async def test_record_ids_are_stable_across_passes(anyio_backend): + """The second guard, for the window before the worker has landed the first write. + + Ingest upserts on (project_id, record_id), so two publishes of the same id write the + same row rather than appending a duplicate. + """ + first, second = _Publisher(), _Publisher() + + for publisher in (first, second): + await run_orphan_sweep( + _FakeTransactionsEngine([_stale_running_row()]), + _FakeRedis(), + records_service=_FakeRecordsService(), + publish=publisher, + ) + + assert [event.record_id for event in first.published] == [ + event.record_id for event in second.published + ] + assert len({event.record_id for event in first.published}) == 2, ( + "the error and the done must not collide on one id" + ) + + +@pytest.mark.anyio +async def test_an_idle_row_owes_no_ending(anyio_backend): + """A row that was alive between turns has no running turn to end. + + Its last turn already reached its own terminal record. Writing an error here would + invent a failure that never happened. + + The records fake says so, because that record is now what decides. The `is_running` flag + used to decide instead, and a durable Stop broke it: settlement clears the flag before the + runner has written its ending. + """ + row = _FakeRow( + session_id="sess-idle", + turn_id="turn-old", + is_running=False, + age_seconds=99_999, + ) + publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([row]), + _FakeRedis(), + records_service=_FakeRecordsService({("sess-idle", "turn-old")}), + publish=publisher, + ) + + assert publisher.published == [] + assert _collapsed(row) + + +@pytest.mark.anyio +async def test_a_parked_approval_is_never_settled(anyio_backend): + """The hazard the heartbeat-age rule creates, pinned. + + A turn that parks for a human sends one final beat with `is_running: false` and then stops + beating on purpose. Its heartbeat therefore goes stale immediately, and it is exactly the + state we most need to keep: the sandbox is warm, the user is about to answer, and the turn + is resumable. + + What protects it is its own terminal record: a turn that parks writes `done` with + `stopReason: paused` at the moment it parks, and any terminal record makes `settled_turns` + answer yes. Verified on the integration stack, session f0018938: `done`/`paused` landed in + the same second as the `interaction_request`. The `is_running` flag protected it before, + and stopped being able to when a durable Stop began clearing that flag early. + """ + row = _FakeRow( + session_id="sess-parked", + turn_id="turn-parked", + is_running=False, + age_seconds=ORPHAN_THRESHOLD_SECONDS * 5, + ) + publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([row]), + _FakeRedis(), + records_service=_FakeRecordsService({("sess-parked", "turn-parked")}), + publish=publisher, + ) + + assert publisher.published == [], ( + "a parked approval must never be given a terminal record: the user is still going to " + "answer it" + ) + + +@pytest.mark.anyio +async def test_a_running_row_without_a_turn_id_is_settled_silently(anyio_backend): + """Nothing to attribute an ending to, so the row is collapsed and no record is written.""" + row = _FakeRow( + session_id="sess-no-turn", + turn_id=None, + is_running=True, + age_seconds=ORPHAN_THRESHOLD_SECONDS + 60, + ) + publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([row]), + _FakeRedis(), + records_service=_FakeRecordsService(), + publish=publisher, + ) + + assert publisher.published == [] + assert _collapsed(row) + + +@pytest.mark.anyio +async def test_open_readers_are_told_the_session_ended(anyio_backend): + """Without this a browser keeps rendering the dead turn as running until a reload.""" + row = _stale_running_row(session_id="sess-watch") + watch = _FakeWatchPublisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([row]), + _FakeRedis(), + records_service=_FakeRecordsService(), + watch_publisher=watch, + publish=_Publisher(), + ) + + assert watch.lifecycles == [(str(_PROJECT_ID), "sess-watch", "ended")] + + +@pytest.mark.anyio +async def test_the_redis_nest_follows_the_settled_row(anyio_backend): + """The SEND gate reads Redis, not the row: a session left nested keeps refusing a + new message long after the watchdog declared its turn lost.""" + redis = _FakeRedis() + project = str(_PROJECT_ID) + await redis.set(f"alive:{project}:session:sess-lost", b"turn-1", ex=3600) + await redis.set(f"running:{project}:session:sess-lost", b"turn-1", ex=3600) + await redis.set(f"owner:{project}:session:sess-lost", b"replica-1", ex=3600) + + await run_orphan_sweep( + _FakeTransactionsEngine([_stale_running_row()]), + redis, + records_service=_FakeRecordsService(), + publish=_Publisher(), + ) + + assert await redis.get(f"alive:{project}:session:sess-lost") is None + assert await redis.get(f"running:{project}:session:sess-lost") is None + assert await redis.get(f"owner:{project}:session:sess-lost") is None + assert ( + await redis.get(f"superseded:{project}:session:sess-lost:turn:turn-1") + is not None + ), "a late beat from the lost turn must not re-nest the session" + + +@pytest.mark.anyio +async def test_cleanup_preserves_same_replica_owner_refresh_before_new_turn_locks( + anyio_backend, +): + stream = _stale_running_row(session_id="sess-cleanup-race", turn_id="turn-a") + redis = _FakeRedis() + project = str(stream.project_id) + alive_key = f"alive:{project}:session:{stream.session_id}" + running_key = f"running:{project}:session:{stream.session_id}" + owner_key = f"owner:{project}:session:{stream.session_id}" + redis._store[alive_key] = b"turn-a" + redis._store[running_key] = b"turn-a" + redis._store[owner_key] = make_owner_value( + replica_id="replica-a", turn_id="turn-a" + ).encode() + + def refresh_turn_b_owner(): + # Exact ABA gap: the same replica refreshed affinity for B, but has not installed B's + # alive/running keys yet. Cleanup must compare the owner generation, not the replica. + redis._store[owner_key] = make_owner_value( + replica_id="replica-a", turn_id="turn-b" + ).encode() + + await run_orphan_sweep( + _FakeTransactionsEngine([stream], after_commit=refresh_turn_b_owner), + redis, + records_service=_FakeRecordsService(), + publish=_Publisher(), + ) + + assert alive_key not in redis._store + assert running_key not in redis._store + assert ( + redis._store[owner_key] + == make_owner_value(replica_id="replica-a", turn_id="turn-b").encode() + ) + assert ( + redis._store[f"superseded:{project}:session:{stream.session_id}:turn:turn-a"] + == b"1" + ) + assert ( + f"superseded:{project}:session:{stream.session_id}:turn:turn-b" + not in redis._store + ) + + +@pytest.mark.anyio +async def test_a_failed_lookup_never_invents_an_ending(anyio_backend): + """If we cannot tell whether the turn already ended, say nothing rather than risk a + second, contradictory ending. Preserve the row and Redis ownership so the next pass retries.""" + + class _FlakyRecords(_FakeRecordsService): + def __init__(self): + super().__init__() + self.calls = 0 + + async def settled_turns(self, *, project_id, keys): + self.calls += 1 + if self.calls == 1: + raise RuntimeError("tracing db unreachable") + return set() + + row = _stale_running_row() + publisher = _Publisher() + records = _FlakyRecords() + redis = _FakeRedis() + project = str(row.project_id) + alive_key = f"alive:{project}:session:{row.session_id}" + running_key = f"running:{project}:session:{row.session_id}" + redis._store[alive_key] = b"turn-1" + redis._store[running_key] = b"turn-1" + + await run_orphan_sweep( + _FakeTransactionsEngine([row]), + redis, + records_service=records, + publish=publisher, + ) + + assert publisher.published == [] + assert not _collapsed(row) + assert redis._store[alive_key] == b"turn-1" + assert redis._store[running_key] == b"turn-1" + + await run_orphan_sweep( + _FakeTransactionsEngine([row]), + redis, + records_service=records, + publish=publisher, + ) + + assert _collapsed(row) + assert alive_key not in redis._store + assert running_key not in redis._store + assert [event.record_type for event in publisher.published] == ["error", "done"] + + +@pytest.mark.anyio +async def test_a_stopped_turn_whose_runner_died_still_gets_an_ending(anyio_backend): + """The seam between the durable Stop and the watchdog, found by running the cells. + + Settlement writes `is_running: false` onto the row the moment it releases the Redis key, + so the tab that pressed Stop is not left spinning. The runner still owes its own terminal + record. If it dies in that window the row is already not-running, and the old rule — only + a row that CLAIMS running owes an ending — skipped it for ever: the 30-minute idle branch + collapses such a row and writes nothing. + + Observed live on the integration stack. Command 01a06763-5807 settled `applied`/`stopped` + at 13:09:19, the runner was killed a moment later, and turn 295351c3 still carried nothing + but the user's own `message` five minutes and five sweep passes later. + + This row is deliberately NOT collapsed here: it is younger than the idle grace, and a + parked approval of the same age must survive. Only the ending is owed. + """ + row = _FakeRow( + session_id="sess-stopped", + turn_id="turn-stopped", + is_running=False, + age_seconds=ORPHAN_THRESHOLD_SECONDS + 30, + ) + publisher = _Publisher() + execution = _FakeExecutionRow( + session_id=row.session_id, + execution_id="turn-stopped", + ) + redis = _FakeRedis() + # Settlement leaves `alive` to its TTL, so the dead turn still holds the session's + # alive lock when the sweep runs; the SEND gate reads that lock. + alive_key = f"alive:{row.project_id}:session:{row.session_id}" + redis._store[alive_key] = b"turn-stopped" + assert ( + await claim_owner( + redis, + project_id=str(row.project_id), + session_id=row.session_id, + replica_id="replica-dead", + ) + == "replica-dead" + ) + + await run_orphan_sweep( + _FakeTransactionsEngine([row], [execution]), + redis, + records_service=_FakeRecordsService(), + publish=publisher, + ) + + assert [event.record_type for event in publisher.published] == ["done"], ( + "a stopped turn whose runner never wrote an ending must be given one" + ) + assert publisher.published[0].attributes == { + "type": "done", + "stopReason": "cancelled", + RECORD_SETTLED_BY_ATTRIBUTE: SETTLED_BY_WATCHDOG, + } + assert all(event.turn_id == "turn-stopped" for event in publisher.published) + assert execution.ending_written_at is not None + assert alive_key not in redis._store, ( + "the dead turn's alive lock must be released, or the next Send is refused for an hour" + ) + assert ( + await claim_owner( + redis, + project_id=str(row.project_id), + session_id=row.session_id, + replica_id="replica-new", + ) + == "replica-new" + ), "the next runner must claim affinity without waiting for the dead owner's TTL" + assert row.flags["is_alive"] is True, "the stopped row itself is not collapsed" + + +async def test_a_stopped_turn_owned_by_a_newer_turn_keeps_that_lock(anyio_backend): + """Release is owner-checked: if a newer turn already holds `alive`, leave it alone.""" + row = _FakeRow( + session_id="sess-stopped", + turn_id="turn-stopped", + is_running=False, + age_seconds=ORPHAN_THRESHOLD_SECONDS + 30, + ) + redis = _FakeRedis() + alive_key = f"alive:{row.project_id}:session:{row.session_id}" + redis._store[alive_key] = b"turn-newer" + await claim_owner( + redis, + project_id=str(row.project_id), + session_id=row.session_id, + replica_id="replica-newer", + ) + + await run_orphan_sweep( + _FakeTransactionsEngine([row]), + redis, + records_service=_FakeRecordsService(), + publish=_Publisher(), + ) + + assert redis._store.get(alive_key) == b"turn-newer" + assert ( + await claim_owner( + redis, + project_id=str(row.project_id), + session_id=row.session_id, + replica_id="replica-other", + ) + == "replica-newer" + ), "settling an older turn must not clear a newer turn's affinity" + + +@pytest.mark.anyio +async def test_a_stopped_execution_gets_an_ending_after_stream_advances( + anyio_backend, +): + stream = _FakeRow( + session_id="sess-advanced", + turn_id="turn-later", + is_running=False, + age_seconds=0, + ) + execution = _FakeExecutionRow( + session_id=stream.session_id, + execution_id="turn-stopped", + ) + redis = _FakeRedis() + alive_key = f"alive:{stream.project_id}:session:{stream.session_id}" + running_key = f"running:{stream.project_id}:session:{stream.session_id}" + redis._store[alive_key] = b"turn-later" + redis._store[running_key] = b"turn-later" + publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([stream], [execution]), + redis, + records_service=_FakeRecordsService({("sess-advanced", "turn-later")}), + publish=publisher, + ) + + assert [event.record_type for event in publisher.published] == ["done"] + assert publisher.published[0].attributes["stopReason"] == "cancelled" + assert all(event.turn_id == "turn-stopped" for event in publisher.published) + assert redis._store[alive_key] == b"turn-later" + assert redis._store[running_key] == b"turn-later" + + +@pytest.mark.anyio +async def test_a_stopped_execution_does_not_touch_a_newer_running_turn( + anyio_backend, +): + stream = _FakeRow( + session_id="sess-advanced-running", + turn_id="turn-running", + is_running=True, + age_seconds=0, + ) + execution = _FakeExecutionRow( + session_id=stream.session_id, + execution_id="turn-stopped", + ) + redis = _FakeRedis() + alive_key = f"alive:{stream.project_id}:session:{stream.session_id}" + running_key = f"running:{stream.project_id}:session:{stream.session_id}" + redis._store[alive_key] = b"turn-running" + redis._store[running_key] = b"turn-running" + publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([stream], [execution]), + redis, + records_service=_FakeRecordsService(), + publish=publisher, + ) + + assert [event.record_type for event in publisher.published] == ["done"] + assert publisher.published[0].attributes["stopReason"] == "cancelled" + assert all(event.turn_id == "turn-stopped" for event in publisher.published) + assert redis._store[alive_key] == b"turn-running" + assert redis._store[running_key] == b"turn-running" + + +@pytest.mark.anyio +async def test_ended_execution_backlog_cannot_hide_a_recent_orphan(anyio_backend): + ended_at = datetime.now(timezone.utc) + ended = [ + _FakeExecutionRow( + session_id=f"sess-ended-{index}", + execution_id=f"turn-ended-{index}", + age_seconds=ORPHAN_THRESHOLD_SECONDS + 1_000 + index, + ending_written_at=ended_at, + ) + for index in range(SWEEP_BATCH_SIZE + 1) + ] + orphan = _FakeExecutionRow( + session_id="sess-recent-orphan", + execution_id="turn-recent-orphan", + ) + records = _FakeRecordsService() + publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([], [*ended, orphan]), + _FakeRedis(), + records_service=records, + publish=publisher, + ) + + assert records.queries == [[("sess-recent-orphan", "turn-recent-orphan")]] + assert [event.record_type for event in publisher.published] == ["done"] + assert publisher.published[0].turn_id == "turn-recent-orphan" + assert orphan.ending_written_at is not None + + +@pytest.mark.anyio +async def test_records_plane_ending_marks_candidate_and_skips_publish(anyio_backend): + execution = _FakeExecutionRow( + session_id="sess-already-ended", + execution_id="turn-already-ended", + terminal_outcome="lost", + ) + publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([], [execution]), + _FakeRedis(), + records_service=_FakeRecordsService( + {("sess-already-ended", "turn-already-ended")} + ), + publish=publisher, + ) + + assert publisher.published == [] + assert execution.ending_written_at is not None + + +@pytest.mark.anyio +async def test_a_lost_execution_clears_is_running_on_a_row_that_still_names_it( + anyio_backend, +): + # The execution is settled lost, but the session's stream row still names that turn and + # still reads is_running true, so the SEND gate would refuse the next message. The pass + # that writes the lost ending must clear is_running (keeping is_alive so the session stays + # resumable), clear the running lock, and update the mirror -- in the same pass. The row is + # fresh here so the went-silent collapse never touches it; the fix must. + stream = _FakeRow( + session_id="sess-stuck-running", + turn_id="turn-lost", + is_running=True, + age_seconds=0, + ) + execution = _FakeExecutionRow( + session_id=stream.session_id, + execution_id="turn-lost", + terminal_outcome="lost", + ) + redis = _FakeRedis() + alive_key = f"alive:{stream.project_id}:session:{stream.session_id}" + running_key = f"running:{stream.project_id}:session:{stream.session_id}" + redis._store[running_key] = b"turn-lost" + redis._store[alive_key] = b"turn-lost" + publisher = _Publisher() + watch = _FakeWatchPublisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([stream], [execution]), + redis, + records_service=_FakeRecordsService(), + watch_publisher=watch, + publish=publisher, + ) + + # is_running is cleared on the row, is_alive is kept, and the row is NOT collapsed. + assert stream.flags == { + "is_alive": True, + "is_running": False, + "is_attached": False, + } + # The running lock the SEND gate reads is cleared too, guarded on the dead turn. + assert running_key not in redis._store + # The mirror update reaches open readers. + assert (str(stream.project_id), "session", stream.session_id) in watch.changes + + +@pytest.mark.anyio +async def test_lost_turn_redis_release_follows_the_stream_commit(anyio_backend): + stream = _FakeRow( + session_id="sess-commit-before-release", + turn_id="turn-lost", + is_running=True, + age_seconds=0, + ) + execution = _FakeExecutionRow( + session_id=stream.session_id, + execution_id="turn-lost", + terminal_outcome="lost", + ) + engine = _FakeTransactionsEngine([stream], [execution]) + redis = _CommitObservingRedis(engine) + for prefix in ("alive", "running"): + redis._store[f"{prefix}:{stream.project_id}:session:{stream.session_id}"] = ( + b"turn-lost" + ) + + await run_orphan_sweep( + engine, + redis, + records_service=_FakeRecordsService(), + publish=_Publisher(), + ) + + assert engine.committed is True + + +@pytest.mark.anyio +async def test_lost_turn_clear_loses_to_a_concurrent_turn_advance(anyio_backend): + stream = _FakeRow( + session_id="sess-advance-during-lost-clear", + turn_id="turn-old", + is_running=True, + age_seconds=0, + ) + execution = _FakeExecutionRow( + session_id=stream.session_id, + execution_id="turn-old", + terminal_outcome="lost", + ) + redis = _FakeRedis() + alive_key = f"alive:{stream.project_id}:session:{stream.session_id}" + running_key = f"running:{stream.project_id}:session:{stream.session_id}" + owner_key = f"owner:{stream.project_id}:session:{stream.session_id}" + redis._store[alive_key] = b"turn-new" + redis._store[running_key] = b"turn-new" + redis._store[owner_key] = b"runner-new" + + def advance_stream(): + stream.turn_id = "turn-new" + stream.updated_at = datetime.now(timezone.utc) + + await run_orphan_sweep( + _FakeTransactionsEngine( + [stream], [execution], before_stream_update=advance_stream + ), + redis, + records_service=_FakeRecordsService(), + publish=_Publisher(), + ) + + assert stream.turn_id == "turn-new" + assert stream.flags["is_running"] is True + assert redis._store[alive_key] == b"turn-new" + assert redis._store[running_key] == b"turn-new" + assert redis._store[owner_key] == b"runner-new" + + +@pytest.mark.anyio +async def test_heartbeat_before_orphan_cas_prevents_settlement_and_records( + anyio_backend, + monkeypatch, +): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + stream = _stale_running_row( + session_id="sess-heartbeat-before-cas", turn_id="turn-current" + ) + publisher = _Publisher() + commands = _CommandsService() + + def heartbeat(): + stream.updated_at = datetime.now(timezone.utc) + + await run_orphan_sweep( + _FakeTransactionsEngine([stream], before_stream_update=heartbeat), + _FakeRedis(), + records_service=_FakeRecordsService(), + commands_service=commands, + publish=publisher, + ) + + assert commands.execution_lost_calls == [] + assert publisher.published == [] + assert stream.flags["is_alive"] is True + assert stream.flags["is_running"] is True + + +@pytest.mark.anyio +async def test_a_lost_execution_leaves_a_newer_running_turn_running(anyio_backend): + # The row has advanced to a NEWER turn that is genuinely running. Settling the OLD turn + # lost must not clear is_running on that row, nor its running lock. + stream = _FakeRow( + session_id="sess-advanced-newer", + turn_id="turn-new", + is_running=True, + age_seconds=0, + ) + execution = _FakeExecutionRow( + session_id=stream.session_id, + execution_id="turn-old", + terminal_outcome="lost", + ) + redis = _FakeRedis() + running_key = f"running:{stream.project_id}:session:{stream.session_id}" + redis._store[running_key] = b"turn-new" + publisher = _Publisher() + watch = _FakeWatchPublisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([stream], [execution]), + redis, + records_service=_FakeRecordsService(), + watch_publisher=watch, + publish=publisher, + ) + + # The newer running turn is untouched: its flag stands and its lock survives. + assert stream.flags["is_running"] is True + assert redis._store[running_key] == b"turn-new" + + +@pytest.mark.anyio +async def test_a_swept_turn_is_tombstoned_even_when_it_holds_no_redis_keys( + anyio_backend, +): + # A prior Stop settlement can clear the alive/running keys before the sweep runs, so the + # collapse finds nothing to displace. The turn is still dead: tombstone it anyway, or a + # returning runner's beat for that turn is admitted and re-sets is_running on the row the + # sweep just collapsed (observed live: run 1e, a beat 3.5 s after the settle). + stream = _stale_running_row(session_id="sess-returning-runner", turn_id="turn-gone") + redis = _FakeRedis() # deliberately empty: no alive/running keys to displace + publisher = _Publisher() + + await run_orphan_sweep( + _FakeTransactionsEngine([stream], []), + redis, + records_service=_FakeRecordsService(), + publish=publisher, + ) + + assert _collapsed(stream) + assert await is_turn_superseded( + redis, + project_id=str(stream.project_id), + session_id=stream.session_id, + turn_id="turn-gone", + ) diff --git a/api/oss/tests/pytest/unit/sessions/test_execution_watchdog_loop.py b/api/oss/tests/pytest/unit/sessions/test_execution_watchdog_loop.py new file mode 100644 index 00000000000..f507229457e --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_execution_watchdog_loop.py @@ -0,0 +1,146 @@ +"""The watchdog loop must survive a failing pass and go round again. + +Root cause of the integration-stack silence on 2026-09-04: `orphan_sweep_loop`'s generic +error handler called `log.exception(...)`, but `log` is a `MultiLogger`, which defines no +`exception` method and no `__getattr__`. The first sweep error -- a `session_executions` +column that did not exist yet during a migration window -- turned that handler into an +`AttributeError` that escaped the `while` loop and killed the watchdog task for the life of +the process. There was no timeout log, no error log, and no further pass, so stale rows were +never settled. The `asyncio.wait_for` guard could not help, because the defect was in the +handler, not in a pass that ran long. + +These tests drive the loop, not a single pass, so the error handler is exercised: a pass +that raises must be logged and the loop must run a second pass; the same must hold for a pass +the timeout cuts. The single-pass behavior lives in `test_execution_watchdog.py`. +""" + +import asyncio + +import pytest + +from oss.src.tasks.asyncio.sessions import orphan_sweep + + +@pytest.fixture +def anyio_backend(): + return "asyncio" + + +async def _noop_sleep(*_args, **_kwargs): + return None + + +class _RecordingLog: + """A stand-in for the module `MultiLogger`. + + It exposes only the methods `MultiLogger` really has, so a call the real logger cannot + serve (for example `exception`) raises `AttributeError` here too, exactly as it did live. + """ + + def __init__(self): + self.calls = [] + + def error(self, *args, **kwargs): + self.calls.append(("error", args, kwargs)) + + def info(self, *args, **kwargs): + self.calls.append(("info", args, kwargs)) + + def warning(self, *args, **kwargs): + self.calls.append(("warning", args, kwargs)) + + +def _logged_errors(recorder): + return [c for c in recorder.calls if c[0] == "error"] + + +async def _run_loop_over(monkeypatch, first_pass_raises): + """Drive the loop over two passes: the first raises `first_pass_raises`, the second stops + the loop with `CancelledError`. Returns the pass count and the recording logger.""" + passes = 0 + + async def fake_sweep(*_args, **_kwargs): + nonlocal passes + passes += 1 + if passes == 1: + raise first_pass_raises + raise asyncio.CancelledError() + + recorder = _RecordingLog() + monkeypatch.setattr(orphan_sweep, "run_orphan_sweep", fake_sweep) + monkeypatch.setattr(orphan_sweep, "log", recorder) + monkeypatch.setattr(orphan_sweep, "SWEEP_INTERVAL_SECONDS", 0) + monkeypatch.setattr(orphan_sweep.asyncio, "sleep", _noop_sleep) + + with pytest.raises(asyncio.CancelledError): + await orphan_sweep.orphan_sweep_loop(engine=None, lock_engine=None) + + return passes, recorder + + +@pytest.mark.anyio +async def test_a_failing_pass_is_logged_and_the_loop_continues( + anyio_backend, monkeypatch +): + # A real error from inside a pass -- the shape of the live UndefinedColumnError. + passes, recorder = await _run_loop_over( + monkeypatch, + RuntimeError("column session_executions.ending_written_at does not exist"), + ) + + # The loop survived the first error and ran a second pass. Before the fix, the handler + # itself raised AttributeError on the first pass and the loop never reached pass two. + assert passes == 2 + + errors = _logged_errors(recorder) + assert errors, "the failing pass must be logged" + assert errors[0][2].get("exc_info"), "the error must carry the traceback" + + +@pytest.mark.anyio +async def test_a_timed_out_pass_is_logged_and_the_loop_continues( + anyio_backend, monkeypatch +): + # `asyncio.wait_for` raises TimeoutError when it cuts a pass that runs too long. The loop + # must log it and go round again, never die. + passes, recorder = await _run_loop_over(monkeypatch, asyncio.TimeoutError()) + + assert passes == 2 + assert _logged_errors(recorder), "the timed-out pass must be logged" + + +@pytest.mark.anyio +async def test_a_hanging_pass_is_cut_by_the_timeout(anyio_backend, monkeypatch): + """A pass that blocks forever must be cut by `asyncio.wait_for`, not hang the loop. + + The production floor on `pass_timeout` is 120 s, so the loop's own timeout is patched to a + short value here to keep the test fast while still exercising the real `asyncio.wait_for`. + """ + passes = 0 + + async def fake_sweep(*_args, **_kwargs): + nonlocal passes + passes += 1 + if passes == 1: + await asyncio.Event().wait() # blocks forever + raise asyncio.CancelledError() + + recorder = _RecordingLog() + monkeypatch.setattr(orphan_sweep, "run_orphan_sweep", fake_sweep) + monkeypatch.setattr(orphan_sweep, "log", recorder) + monkeypatch.setattr(orphan_sweep, "SWEEP_INTERVAL_SECONDS", 0) + monkeypatch.setattr(orphan_sweep.asyncio, "sleep", _noop_sleep) + + real_wait_for = asyncio.wait_for + + async def short_wait_for(awaitable, timeout): # noqa: ARG001 + return await real_wait_for(awaitable, timeout=0.05) + + monkeypatch.setattr(orphan_sweep.asyncio, "wait_for", short_wait_for) + + with pytest.raises(asyncio.CancelledError): + await orphan_sweep.orphan_sweep_loop(engine=None, lock_engine=None) + + # The first pass hung; the timeout cut it and the loop ran a second pass. + assert passes == 2 + assert _logged_errors(recorder), "the cut pass must be logged" diff --git a/api/oss/tests/pytest/unit/sessions/test_heartbeat_departed_replica_affinity.py b/api/oss/tests/pytest/unit/sessions/test_heartbeat_departed_replica_affinity.py new file mode 100644 index 00000000000..8ce33508000 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_heartbeat_departed_replica_affinity.py @@ -0,0 +1,284 @@ +"""A runner that dies ungracefully must not lock its sessions out for the owner lease. + +Live failure this pins (matrix run 3, cell `runner-gone-late`, harness codex, session +2fdf43f0-c728-42e5-987e-2371501fe748): a Stop settled, the runner reported the outcome, and +the runner was then killed with no grace period. Nothing released `owner:session:`, so it +stayed pointing at the dead replica for the rest of OWNER_TTL_SECONDS. The replacement replica +picked up the user's next message 6 s later, its first heartbeat lost the non-stealing +`claim_owner`, the API answered `is_current_turn: false`, and the runner turned that into +"This session is already running a turn" although no turn was running anywhere. + +`running` is what tells a serving replica from a departed one, so these tests drive both +sides of it: + + - no running turn -> the new replica takes affinity and its first beat is current; + - a different turn holding `running` -> the claim is honoured and the newcomer is refused; + - the caller's OWN turn holding `running` (the `_start_turn` path) -> reclaim allowed; + - a turn-end beat never reclaims; + - the reclaim survives the alive lock the dead turn left behind (the whole point: the next + message has to actually run). +""" + +from typing import Optional +from unittest.mock import patch +from uuid import UUID, uuid4 + +import pytest +import pytest_asyncio + +from oss.src.core.sessions.streams.dtos import ( + SessionHeartbeatRequest, + SessionStream, +) +from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.dbs.redis.sessions.locks import ( + claim_owner, + get_alive_owner, + get_owner, + get_owner_value, + get_running_owner, +) +from oss.src.dbs.redis.sessions.contract import make_owner_value + +from unit.sessions.test_project_scoped_locks import _FakeRedis + + +_PROJECT = uuid4() +_SESSION = "session_departed_replica" + +_DEAD = "replica-that-was-killed" +_FRESH = "replica-that-replaced-it" + + +class _FakeDAO: + def __init__(self, existing: Optional[SessionStream] = None): + self.row = existing + + async def get_by_session_id(self, *, project_id: UUID, session_id: str): + return self.row + + async def create(self, *, project_id, user_id, stream): + self.row = SessionStream( + id=uuid4(), + project_id=project_id, + session_id=stream.session_id, + flags=stream.flags, + turn_id=stream.turn_id, + ) + return self.row + + async def update(self, *, project_id, user_id, session_id, stream): + prior = self.row + self.row = SessionStream( + id=prior.id if prior else uuid4(), + project_id=project_id, + session_id=session_id, + flags=stream.flags + if stream.flags is not None + else (prior.flags if prior else None), + turn_id=stream.turn_id + if stream.turn_id is not None + else (prior.turn_id if prior else None), + ) + return self.row + + async def delete_by_session_id(self, *, project_id, session_id): + return True + + +@pytest_asyncio.fixture +async def lock_engine(): + from oss.src.dbs.redis.shared.engine import LockEngine + + eng = LockEngine() + with patch.object(eng, "_client", return_value=_FakeRedis()): + yield eng + + +def _service(lock_engine, dao=None): + return SessionStreamsService(streams_dao=dao or _FakeDAO(), lock_engine=lock_engine) + + +def _beat(replica: str, turn: Optional[str], running: bool = True): + return SessionHeartbeatRequest( + session_id=_SESSION, replica_id=replica, turn_id=turn, is_running=running + ) + + +async def _replay_the_killed_runner(svc): + """The exact state the live failure left: a turn that ran, was stopped, reported + `is_running: false`, and whose replica then died without releasing affinity.""" + await svc.heartbeat(project_id=_PROJECT, request=_beat(_DEAD, "turn-stopped")) + await svc.heartbeat( + project_id=_PROJECT, request=_beat(_DEAD, "turn-stopped", running=False) + ) + + +@pytest.mark.asyncio +async def test_next_turn_is_admitted_after_the_owning_runner_is_killed(lock_engine): + svc = _service(lock_engine) + pid = str(_PROJECT) + await _replay_the_killed_runner(svc) + + # Preconditions: affinity still names the dead replica, nothing is running, and the dead + # turn's `alive` lock outlives it by design. + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == _DEAD + assert ( + await get_running_owner(lock_engine, project_id=pid, session_id=_SESSION) + ) is None + assert await get_alive_owner(lock_engine, project_id=pid, session_id=_SESSION) == ( + "turn-stopped" + ) + + recovery = await svc.heartbeat( + project_id=_PROJECT, request=_beat(_FRESH, "turn-recovery") + ) + + assert recovery.is_current_turn is True, ( + "the replacement replica was refused, so the user's next message is rejected as " + "'this session is already running a turn' for the rest of the owner lease" + ) + assert recovery.replica_id == _FRESH + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == _FRESH + + +@pytest.mark.asyncio +async def test_recovery_turn_takes_the_nest_the_dead_turn_left(lock_engine): + """Admission is not enough: the recovery turn must end up owning alive and running, or + the next beat sees a foreign nest and aborts the turn it just started.""" + svc = _service(lock_engine) + pid = str(_PROJECT) + await _replay_the_killed_runner(svc) + + await svc.heartbeat(project_id=_PROJECT, request=_beat(_FRESH, "turn-recovery")) + + assert await get_alive_owner(lock_engine, project_id=pid, session_id=_SESSION) == ( + "turn-recovery" + ) + assert await get_running_owner( + lock_engine, project_id=pid, session_id=_SESSION + ) == ("turn-recovery") + + second = await svc.heartbeat( + project_id=_PROJECT, request=_beat(_FRESH, "turn-recovery") + ) + assert second.is_current_turn is True + + +@pytest.mark.asyncio +async def test_a_live_turn_on_another_replica_still_refuses_the_newcomer(lock_engine): + """The guard this reclaim relaxes must still hold where it matters: a replica running a + turn keeps its session, and a second replica's turn is refused rather than admitted + alongside it.""" + svc = _service(lock_engine) + pid = str(_PROJECT) + + await svc.heartbeat(project_id=_PROJECT, request=_beat(_DEAD, "turn-live")) + + intruder = await svc.heartbeat( + project_id=_PROJECT, request=_beat(_FRESH, "turn-intruder") + ) + + assert intruder.is_current_turn is False + assert intruder.replica_id == _DEAD + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == _DEAD + assert await get_alive_owner(lock_engine, project_id=pid, session_id=_SESSION) == ( + "turn-live" + ) + assert await get_running_owner( + lock_engine, project_id=pid, session_id=_SESSION + ) == ("turn-live") + + +@pytest.mark.asyncio +async def test_a_turn_that_already_holds_running_may_reclaim(lock_engine): + """`_start_turn` arms alive and running before the runner beats at all, so an API-minted + turn reaches the heartbeat with its own `running` lock already held. That must not read as + 'another turn is live here'.""" + from oss.src.dbs.redis.sessions.locks import acquire_alive, acquire_running + + svc = _service(lock_engine) + pid = str(_PROJECT) + await _replay_the_killed_runner(svc) + # The API starts the recovery turn itself, then the replacement replica beats for it. + from oss.src.dbs.redis.sessions.locks import force_cancel_alive + + await force_cancel_alive(lock_engine, project_id=pid, session_id=_SESSION) + await acquire_alive( + lock_engine, project_id=pid, session_id=_SESSION, turn_id="turn-api-minted" + ) + await acquire_running( + lock_engine, project_id=pid, session_id=_SESSION, turn_id="turn-api-minted" + ) + + result = await svc.heartbeat( + project_id=_PROJECT, request=_beat(_FRESH, "turn-api-minted") + ) + + assert result.is_current_turn is True + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == _FRESH + + +@pytest.mark.asyncio +async def test_reclaim_does_not_clear_a_refreshed_owner_generation(lock_engine): + """A same-replica new turn may refresh affinity after the failed claim is observed.""" + svc = _service(lock_engine) + pid = str(_PROJECT) + await _replay_the_killed_runner(svc) + + async def refresh_owner_generation(engine, *, project_id, session_id): + await claim_owner( + engine, + project_id=project_id, + session_id=session_id, + replica_id=_DEAD, + turn_id="turn-new-on-incumbent", + ) + return None + + with patch( + "oss.src.core.sessions.streams.service.get_running_owner", + side_effect=refresh_owner_generation, + ): + result = await svc.heartbeat( + project_id=_PROJECT, request=_beat(_FRESH, "turn-challenger") + ) + + assert result.is_current_turn is False + assert result.replica_id == _DEAD + assert await get_owner_value( + lock_engine, project_id=pid, session_id=_SESSION + ) == make_owner_value( + replica_id=_DEAD, + turn_id="turn-new-on-incumbent", + ) + + +@pytest.mark.asyncio +async def test_a_turn_end_beat_never_reclaims_affinity(lock_engine): + """A beat that reports a turn ENDING asserts nothing about who should serve the session + next, so it must leave affinity alone.""" + svc = _service(lock_engine) + pid = str(_PROJECT) + await _replay_the_killed_runner(svc) + + result = await svc.heartbeat( + project_id=_PROJECT, request=_beat(_FRESH, "turn-recovery", running=False) + ) + + assert result.is_current_turn is False + assert result.replica_id == _DEAD + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == _DEAD + + +@pytest.mark.asyncio +async def test_a_beat_with_no_turn_never_reclaims_affinity(lock_engine): + """The ownership-probe beat carries no turn id. It reads affinity; it may not move it.""" + svc = _service(lock_engine) + pid = str(_PROJECT) + await _replay_the_killed_runner(svc) + + result = await svc.heartbeat(project_id=_PROJECT, request=_beat(_FRESH, None)) + + assert result.replica_id == _DEAD + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == _DEAD diff --git a/api/oss/tests/pytest/unit/sessions/test_heartbeat_is_current_turn.py b/api/oss/tests/pytest/unit/sessions/test_heartbeat_is_current_turn.py index fa3595d73c3..39b2c88f1fe 100644 --- a/api/oss/tests/pytest/unit/sessions/test_heartbeat_is_current_turn.py +++ b/api/oss/tests/pytest/unit/sessions/test_heartbeat_is_current_turn.py @@ -200,3 +200,109 @@ async def test_new_turn_on_a_previously_run_session_is_current(lock_engine): assert fresh.is_current_turn is True, ( "a new turn must not be aborted just because the row still named the old one" ) + + +@pytest.mark.asyncio +async def test_second_turn_on_a_RUNNING_session_is_refused(lock_engine): + """Single-turn admission (#6417, #5539, #5538): the answer the runner's edge now acts on. + + A second user message on a session with a turn in flight reaches the runner as its own turn. + Its FIRST beat is the admission request, and this is what must come back: `is_current_turn` + False, with the running turn's locks untouched. The API already answered this correctly; the + runner used to read it only as "abort later", walk into the keepalive pool, and destroy the + running turn's environment on the way. It now stops at the edge, so this answer is the whole + gate and it needs its own test. + """ + svc = _service(lock_engine) + + # turn-1 is live: it holds both `alive` and `running`. + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-1")) + + # The second message arrives on the SAME replica as its own turn. Nothing cancelled turn-1, + # so `running` still names it — the discriminator that separates this from a handover. + second = await svc.heartbeat( + project_id=_PROJECT, request=_beat("replica-a", "turn-2") + ) + + assert second.is_current_turn is False, ( + "a turn that arrives while a DIFFERENT turn holds `running` must be refused" + ) + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-1" + ), "the refused turn must not take the running turn's alive lock" + + # And the live turn's own next beat is unaffected: it was never displaced. + still_live = await svc.heartbeat( + project_id=_PROJECT, request=_beat("replica-a", "turn-1") + ) + assert still_live.is_current_turn is True + + +@pytest.mark.asyncio +async def test_a_refused_turns_end_beat_cannot_clear_the_live_turns_running( + lock_engine, +): + """The refused turn's watchdog release sends `is_running: false`. That beat must be inert. + + The runner stops a refused turn by releasing its watchdog, which sends one end beat under the + REFUSED turn's id. Releasing `running` on behalf of whoever holds it would end the live turn + from under itself, which is the failure this whole slice exists to remove. The release is + owner-scoped, so it is a no-op here. + """ + svc = _service(lock_engine) + + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-1")) + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-2")) + + # The refused turn's end beat. + await svc.heartbeat( + project_id=_PROJECT, request=_beat("replica-a", "turn-2", running=False) + ) + + live = await svc.heartbeat( + project_id=_PROJECT, request=_beat("replica-a", "turn-1") + ) + assert live.is_current_turn is True, ( + "the refused turn's end beat released the LIVE turn's locks" + ) + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-1" + ) + + +@pytest.mark.asyncio +async def test_a_resume_is_admitted_while_the_previous_turn_is_PARKED(lock_engine): + """The case a naive "is anything alive?" gate gets wrong, and the reason `running` exists. + + A turn parked awaiting approval still holds `alive` — that is what makes the session + reattachable — but its turn-end beat released `running`. The approval resume arrives as a NEW + turn and must be admitted, or every approval in the product stops resuming. `alive` alone + cannot tell this apart from the refusal case above; the absent `running` owner is what does. + """ + svc = _service(lock_engine) + + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-1")) + # Park: the turn ends its execution but the session stays alive. + await svc.heartbeat( + project_id=_PROJECT, request=_beat("replica-a", "turn-1", running=False) + ) + + resume = await svc.heartbeat( + project_id=_PROJECT, request=_beat("replica-a", "turn-2") + ) + + assert resume.is_current_turn is True, ( + "an approval resume must be admitted while the previous turn is parked, not running" + ) + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-2" + ), "the resume takes the nest as a legitimate handover" diff --git a/api/oss/tests/pytest/unit/sessions/test_heartbeat_lock_races.py b/api/oss/tests/pytest/unit/sessions/test_heartbeat_lock_races.py index e41784244df..7fd1bca0521 100644 --- a/api/oss/tests/pytest/unit/sessions/test_heartbeat_lock_races.py +++ b/api/oss/tests/pytest/unit/sessions/test_heartbeat_lock_races.py @@ -20,7 +20,6 @@ ) from oss.src.core.sessions.streams.service import SessionStreamsService from oss.src.dbs.redis.sessions.locks import ( - clear_running, force_clear_owner, get_alive_owner, get_owner, @@ -163,24 +162,23 @@ async def test_handover_will_not_evict_a_turn_that_took_the_lock_mid_read(lock_e @pytest.mark.asyncio -async def test_cancel_tombstones_before_it_clears_the_locks(lock_engine): - """Cancel clears `alive` and then tombstones the turn it displaced. A beat from that very - turn arriving between the two finds `alive` free, nx-acquires it back, and the cancelled - session reads as alive for a full ALIVE_TTL. Writing the tombstone first closes it.""" +async def test_cancel_atomically_tombstones_and_clears_the_locks(lock_engine): + """The displaced turn cannot re-arm the session after the atomic operation returns.""" dao = _FakeStreamsDAO() svc = _service(lock_engine, dao) await svc.heartbeat(project_id=_PROJECT, request=_beat("turn-a")) assert await _alive(lock_engine) == "turn-a" + redis = lock_engine._client() + original_eval = redis.eval - async def _beat_mid_displacement(engine, *, project_id: str, session_id: str): - await svc.heartbeat(project_id=_PROJECT, request=_beat("turn-a")) - return await clear_running(engine, project_id=project_id, session_id=session_id) + async def _beat_after_atomic_displacement(script, numkeys, *keys_and_args): + result = await original_eval(script, numkeys, *keys_and_args) + if "AGENTA_DISPLACE_TURNS" in script: + late = await svc.heartbeat(project_id=_PROJECT, request=_beat("turn-a")) + assert late.is_current_turn is False + return result - # `clear_running` runs after `alive` is cleared, i.e. inside the old window. - with patch( - "oss.src.core.sessions.streams.service.clear_running", - new=_beat_mid_displacement, - ): + with patch.object(redis, "eval", new=_beat_after_atomic_displacement): await svc.command(project_id=_PROJECT, user_id=_USER, request=_cancel()) assert await _alive(lock_engine) is None, ( diff --git a/api/oss/tests/pytest/unit/sessions/test_heartbeat_release_auth.py b/api/oss/tests/pytest/unit/sessions/test_heartbeat_release_auth.py new file mode 100644 index 00000000000..96c0e421335 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_heartbeat_release_auth.py @@ -0,0 +1,97 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest +from fastapi import HTTPException + +from oss.src.apis.fastapi.sessions import router as router_module +from oss.src.apis.fastapi.sessions.router import SessionStreamsRouter +from oss.src.core.sessions.streams.dtos import ( + SessionHeartbeatRequest, + SessionHeartbeatResult, +) +from oss.src.utils.env import env + + +_PROJECT = UUID("00000000-0000-0000-0000-0000000000aa") +_USER = UUID("00000000-0000-0000-0000-0000000000bb") + + +def _request(headers=None): + return SimpleNamespace( + state=SimpleNamespace(project_id=_PROJECT, user_id=_USER), + headers=headers or {}, + ) + + +def _router(service): + return SessionStreamsRouter( + service=service, + interactions_service=SimpleNamespace(), + ) + + +@pytest.mark.asyncio +async def test_release_owner_heartbeat_requires_the_runner_token(monkeypatch): + monkeypatch.setattr(env.runner, "token", "runner-secret") + monkeypatch.setattr( + router_module, "check_action_access", AsyncMock(return_value=True) + ) + service = SimpleNamespace(heartbeat=AsyncMock()) + + with pytest.raises(HTTPException) as exc_info: + await _router(service).heartbeat_session_stream( + _request(), + SessionHeartbeatRequest( + session_id="session-1", + replica_id="replica-1", + release_owner=True, + ), + ) + + assert exc_info.value.status_code == 401 + service.heartbeat.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_regular_heartbeat_keeps_user_authentication_only(monkeypatch): + monkeypatch.setattr(env.runner, "token", "runner-secret") + monkeypatch.setattr( + router_module, "check_action_access", AsyncMock(return_value=True) + ) + service = SimpleNamespace( + heartbeat=AsyncMock(return_value=SessionHeartbeatResult(replica_id="replica-1")) + ) + payload = SessionHeartbeatRequest( + session_id="session-1", + replica_id="replica-1", + turn_id="turn-1", + ) + + result = await _router(service).heartbeat_session_stream(_request(), payload) + + assert result.replica_id == "replica-1" + service.heartbeat.assert_awaited_once_with(project_id=_PROJECT, request=payload) + + +@pytest.mark.asyncio +async def test_release_owner_accepts_the_shared_runner_token(monkeypatch): + monkeypatch.setattr(env.runner, "token", "runner-secret") + monkeypatch.setattr( + router_module, "check_action_access", AsyncMock(return_value=True) + ) + service = SimpleNamespace( + heartbeat=AsyncMock(return_value=SessionHeartbeatResult(replica_id="replica-1")) + ) + payload = SessionHeartbeatRequest( + session_id="session-1", + replica_id="replica-1", + release_owner=True, + ) + + await _router(service).heartbeat_session_stream( + _request({"X-Agenta-Runner-Token": "runner-secret"}), payload + ) + + service.heartbeat.assert_awaited_once_with(project_id=_PROJECT, request=payload) diff --git a/api/oss/tests/pytest/unit/sessions/test_heartbeat_release_owner.py b/api/oss/tests/pytest/unit/sessions/test_heartbeat_release_owner.py new file mode 100644 index 00000000000..880dc7a507f --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_heartbeat_release_owner.py @@ -0,0 +1,220 @@ +"""The shutdown beat: a departing runner hands its `owner:session:` affinity key back. + +`claim_owner` never steals, and nothing released the key, so a replica that exited while +holding claims locked each of those sessions out of its replacement for the rest of +OWNER_TTL_SECONDS. On the local sandbox provider that is a two-minute outage after every +runner restart, because the replacement refuses to cold-start a session it does not own. + +`release_owner` is deliberately narrow, and these tests pin exactly how narrow: it releases +only while the caller still owns the session, it touches no turn lock and no stream row, and a +beat from a replica that lost the session is a no-op rather than a takeover in reverse. +""" + +from typing import Optional +from unittest.mock import patch +from uuid import UUID, uuid4 + +import pytest +import pytest_asyncio + +from oss.src.core.sessions.streams.dtos import ( + SessionHeartbeatRequest, + SessionStream, +) +from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.dbs.redis.sessions.locks import ( + get_alive_owner, + get_owner, + get_running_owner, +) + +from unit.sessions.test_project_scoped_locks import _FakeRedis + + +_PROJECT = uuid4() +_SESSION = "session_shutdown" + + +class _FakeDAO: + """Records every write, so a test can assert the release beat wrote nothing.""" + + def __init__(self, existing: Optional[SessionStream] = None): + self.row = existing + self.creates = 0 + self.updates = 0 + + async def get_by_session_id(self, *, project_id: UUID, session_id: str): + return self.row + + async def create(self, *, project_id, user_id, stream): + self.creates += 1 + self.row = SessionStream( + id=uuid4(), + project_id=project_id, + session_id=stream.session_id, + flags=stream.flags, + ) + return self.row + + async def update(self, *, project_id, user_id, session_id, stream): + self.updates += 1 + self.row = SessionStream( + id=self.row.id if self.row else uuid4(), + project_id=project_id, + session_id=session_id, + flags=stream.flags, + ) + return self.row + + async def delete_by_session_id(self, *, project_id, session_id): + return True + + +@pytest_asyncio.fixture +async def lock_engine(): + from oss.src.dbs.redis.shared.engine import LockEngine + + eng = LockEngine() + with patch.object(eng, "_client", return_value=_FakeRedis()): + yield eng + + +def _service(lock_engine, dao): + return SessionStreamsService(streams_dao=dao, lock_engine=lock_engine) + + +def _beat(replica: str, turn: str, running: bool = True) -> SessionHeartbeatRequest: + return SessionHeartbeatRequest( + session_id=_SESSION, replica_id=replica, turn_id=turn, is_running=running + ) + + +def _shutdown_beat(replica: str) -> SessionHeartbeatRequest: + """What the runner sends per owned session as it exits: no turn, no liveness.""" + return SessionHeartbeatRequest( + session_id=_SESSION, replica_id=replica, release_owner=True + ) + + +@pytest.mark.asyncio +async def test_owner_release_drops_the_affinity_key(lock_engine): + dao = _FakeDAO() + svc = _service(lock_engine, dao) + pid = str(_PROJECT) + + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-a")) + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == ( + "replica-a" + ) + + await svc.heartbeat(project_id=_PROJECT, request=_shutdown_beat("replica-a")) + + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) is None, ( + "the departing replica still owns the session" + ) + + +@pytest.mark.asyncio +async def test_the_next_replica_can_claim_the_session_at_once(lock_engine): + """The whole point: no waiting out OWNER_TTL_SECONDS after a restart.""" + dao = _FakeDAO() + svc = _service(lock_engine, dao) + pid = str(_PROJECT) + + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-a")) + await svc.heartbeat(project_id=_PROJECT, request=_shutdown_beat("replica-a")) + + result = await svc.heartbeat( + project_id=_PROJECT, request=_beat("replica-b", "turn-b") + ) + + assert result.replica_id == "replica-b" + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == ( + "replica-b" + ) + + +@pytest.mark.asyncio +async def test_release_touches_no_turn_lock_and_no_row(lock_engine): + dao = _FakeDAO() + svc = _service(lock_engine, dao) + pid = str(_PROJECT) + + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-a")) + writes_before = dao.creates + dao.updates + + await svc.heartbeat(project_id=_PROJECT, request=_shutdown_beat("replica-a")) + + assert await get_alive_owner(lock_engine, project_id=pid, session_id=_SESSION) == ( + "turn-a" + ), "the release beat cleared the alive lock" + assert await get_running_owner( + lock_engine, project_id=pid, session_id=_SESSION + ) == ("turn-a"), "the release beat cleared the running lock" + assert dao.creates + dao.updates == writes_before, ( + "the release beat stamped the stream row" + ) + + +@pytest.mark.asyncio +async def test_a_replica_that_lost_the_session_releases_nothing(lock_engine): + """Release-if-owner: a stale runner must not free a session a live one now holds.""" + dao = _FakeDAO() + svc = _service(lock_engine, dao) + pid = str(_PROJECT) + + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-a")) + + result = await svc.heartbeat( + project_id=_PROJECT, request=_shutdown_beat("replica-b") + ) + + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == ( + "replica-a" + ), "replica B released a session it never owned" + assert result.replica_id == "replica-a", "the loser must learn the true owner" + + +@pytest.mark.asyncio +async def test_release_is_idempotent(lock_engine): + dao = _FakeDAO() + svc = _service(lock_engine, dao) + pid = str(_PROJECT) + + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-a")) + await svc.heartbeat(project_id=_PROJECT, request=_shutdown_beat("replica-a")) + result = await svc.heartbeat( + project_id=_PROJECT, request=_shutdown_beat("replica-a") + ) + + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) is None + assert result.replica_id == "replica-a", "an unowned session reports the caller" + assert result.is_current_turn is False, "a release beat refreshes no turn" + + +@pytest.mark.asyncio +async def test_release_of_a_session_nobody_owns_is_harmless(lock_engine): + dao = _FakeDAO() + svc = _service(lock_engine, dao) + + result = await svc.heartbeat( + project_id=_PROJECT, request=_shutdown_beat("replica-a") + ) + + assert result.stream is None + assert dao.creates + dao.updates == 0 + + +@pytest.mark.asyncio +async def test_an_ordinary_beat_still_claims(lock_engine): + """The default must not change: `release_owner` is False unless a caller asks for it.""" + dao = _FakeDAO() + svc = _service(lock_engine, dao) + pid = str(_PROJECT) + + assert _beat("replica-a", "turn-a").release_owner is False + await svc.heartbeat(project_id=_PROJECT, request=_beat("replica-a", "turn-a")) + + assert await get_owner(lock_engine, project_id=pid, session_id=_SESSION) == ( + "replica-a" + ) diff --git a/api/oss/tests/pytest/unit/sessions/test_heartbeat_turn_handover.py b/api/oss/tests/pytest/unit/sessions/test_heartbeat_turn_handover.py index 68fc63e4573..e206cb7076d 100644 --- a/api/oss/tests/pytest/unit/sessions/test_heartbeat_turn_handover.py +++ b/api/oss/tests/pytest/unit/sessions/test_heartbeat_turn_handover.py @@ -174,28 +174,15 @@ async def test_overlapping_beats_of_the_same_turn_stay_current(lock_engine): lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-1" ) - cancels: list[str] = [] - - async def _spy_force_cancel(engine, *, project_id, session_id): - cancels.append(session_id) - return None - # refresh_alive returning False while the key holds OUR id is exactly the interleaving: # the GET raced the concurrent beat's write. - with ( - patch( - "oss.src.core.sessions.streams.service.refresh_alive", - new=AsyncMock(return_value=False), - ), - patch( - "oss.src.core.sessions.streams.service.force_cancel_alive", - new=_spy_force_cancel, - ), + with patch( + "oss.src.core.sessions.streams.service.refresh_alive", + new=AsyncMock(return_value=False), ): result = await svc.heartbeat(project_id=_PROJECT, request=_beat("turn-1")) assert result.is_current_turn is True - assert cancels == [], "we already own `alive`; there is nothing to hand over" assert await _alive(lock_engine) == "turn-1" diff --git a/api/oss/tests/pytest/unit/sessions/test_interaction_cancel_records.py b/api/oss/tests/pytest/unit/sessions/test_interaction_cancel_records.py new file mode 100644 index 00000000000..743d734d4b5 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_interaction_cancel_records.py @@ -0,0 +1,205 @@ +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +import pytest +from fastapi import FastAPI, HTTPException, Request + +from oss.src.apis.fastapi.sessions.models import SessionInteractionRespondRequest +from oss.src.apis.fastapi.sessions.router import InteractionsRouter +from oss.src.core.sessions.interactions.dtos import ( + SessionInteraction, + SessionInteractionKind, + SessionInteractionStatus, +) +from oss.src.core.sessions.interactions.service import SessionInteractionsService + + +class _RecordingPublisher: + def __init__(self, journal): + self.journal = journal + self.calls = [] + + async def interaction(self, *, project_id, session_id, status): + self.journal.append("publish") + self.calls.append((project_id, session_id, status)) + + +class _RecordingRecordsService: + def __init__(self, journal): + self.journal = journal + self.events = [] + + async def append_many(self, *, events): + self.journal.append("records") + self.events.extend(events) + return [] + + +class _FailingRecordsService: + async def append_many(self, *, events): + raise RuntimeError("records unavailable") + + +def _interaction(*, project_id, token, turn_id="turn-1"): + return SessionInteraction( + id=uuid4(), + project_id=project_id, + session_id="sess-1", + turn_id=turn_id, + token=token, + kind=SessionInteractionKind.user_approval, + status=SessionInteractionStatus.cancelled, + ) + + +@pytest.mark.asyncio +async def test_stop_cancel_writes_one_record_per_cancelled_interaction_before_publish(): + project_id = uuid4() + command_id = uuid4() + cancelled = [ + _interaction(project_id=project_id, token="gate-1"), + _interaction(project_id=project_id, token="gate-2"), + ] + dao = AsyncMock() + dao.cancel_session_pending = AsyncMock(return_value=cancelled) + journal = [] + records = _RecordingRecordsService(journal) + publisher = _RecordingPublisher(journal) + service = SessionInteractionsService( + interactions_dao=dao, + records_service=records, + watch_publisher=publisher, + ) + + count = await service.cancel_session_pending( + project_id=project_id, + session_id="sess-1", + only_turn_id="turn-1", + command_id=command_id, + ) + + assert count == 2 + assert len(records.events) == 2 + assert len({event.record_id for event in records.events}) == 2 + for event, interaction in zip(records.events, cancelled): + assert event.record_type == "interaction_response" + assert event.record_source == "agent" + assert event.turn_id == "turn-1" + assert event.attributes == { + "type": "interaction_response", + "id": interaction.token, + "kind": "user_approval", + "payload": { + "outcome": "cancelled", + "turnId": "turn-1", + "commandId": str(command_id), + }, + } + assert journal == ["records", "publish"] + assert publisher.calls == [(str(project_id), "sess-1", "resolved")] + + +@pytest.mark.asyncio +async def test_stop_cancel_writes_no_record_when_nothing_was_pending(): + project_id = uuid4() + dao = AsyncMock() + dao.cancel_session_pending = AsyncMock(return_value=[]) + journal = [] + records = _RecordingRecordsService(journal) + publisher = _RecordingPublisher(journal) + service = SessionInteractionsService( + interactions_dao=dao, + records_service=records, + watch_publisher=publisher, + ) + + count = await service.cancel_session_pending( + project_id=project_id, + session_id="sess-1", + only_turn_id="turn-1", + command_id=uuid4(), + ) + + assert count == 0 + assert records.events == [] + assert publisher.calls == [] + assert journal == [] + + +@pytest.mark.asyncio +async def test_record_failure_does_not_block_interaction_resolution_publish(): + project_id = uuid4() + dao = AsyncMock() + dao.cancel_session_pending = AsyncMock( + return_value=[_interaction(project_id=project_id, token="gate-1")] + ) + journal = [] + publisher = _RecordingPublisher(journal) + service = SessionInteractionsService( + interactions_dao=dao, + records_service=_FailingRecordsService(), + watch_publisher=publisher, + ) + + count = await service.cancel_session_pending( + project_id=project_id, + session_id="sess-1", + only_turn_id="turn-1", + command_id=uuid4(), + ) + + assert count == 1 + assert journal == ["publish"] + assert publisher.calls == [(str(project_id), "sess-1", "resolved")] + + +@pytest.mark.asyncio +async def test_answer_after_stop_returns_the_terminal_interaction_409_contract(): + project_id = uuid4() + user_id = uuid4() + interaction_id = uuid4() + interactions_service = AsyncMock() + interactions_service.fetch_interaction.return_value = SessionInteraction( + id=interaction_id, + project_id=project_id, + session_id="sess-1", + turn_id="turn-1", + token="gate-1", + kind=SessionInteractionKind.user_approval, + status=SessionInteractionStatus.cancelled, + ) + respond_task = AsyncMock() + respond_task.kiq = AsyncMock() + router = InteractionsRouter( + interactions_service=interactions_service, + workflows_service=AsyncMock(), + respond_task=respond_task, + ) + request = Request( + { + "type": "http", + "method": "POST", + "path": f"/sessions/interactions/{interaction_id}/respond", + "headers": [], + "app": FastAPI(), + } + ) + request.state.project_id = project_id + request.state.user_id = user_id + + with patch( + "oss.src.apis.fastapi.sessions.router.check_action_access", + new_callable=AsyncMock, + return_value=True, + ): + with pytest.raises(HTTPException) as exc_info: + await router.respond_interaction( + request=request, + interaction_id=interaction_id, + body=SessionInteractionRespondRequest(answer={"approved": True}), + ) + + assert exc_info.value.status_code == 409 + assert exc_info.value.detail == "Interaction is no longer pending" + interactions_service.transition_interaction.assert_not_awaited() + respond_task.kiq.assert_not_awaited() diff --git a/api/oss/tests/pytest/unit/sessions/test_late_record_quarantine.py b/api/oss/tests/pytest/unit/sessions/test_late_record_quarantine.py new file mode 100644 index 00000000000..b163fd60c6b --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_late_record_quarantine.py @@ -0,0 +1,572 @@ +"""The ingest guard that keeps one execution to one ending. + +RFC "Required behavior / Execution" item 3: after an execution reaches its terminal outcome, +later non-terminal output for it is rejected or quarantined. `RecordsService.append_many` is +where that is enforced, because ingest is the only place the watchdog and the runner meet. + +The case these tests pin was caught live. A runner wedges past the watchdog's stale-heartbeat +threshold, the watchdog writes the turn's `error` and `done` on its behalf, and the runner +then thaws and submits everything it had buffered: a tool call, its result, a `usage`, and a +second `done`. The reader was left with a failure notice followed by the work the agent went +on to do, and with two endings for one turn. + +Every test here drives the service against a stub DAO, so they run with no Postgres. The +DAO-level half — that a quarantined row is invisible to `get_records` and does not answer +`settled_turns` — lives in `test_late_record_quarantine_dao.py` against a real database. +""" + +from datetime import datetime, timezone +from typing import Dict, List, Optional, Sequence, Set, Tuple +from uuid import UUID, uuid4 + +import pytest + +from oss.src.core.sessions.records.dtos import ( + RECORD_SETTLED_BY_ATTRIBUTE, + SETTLED_BY_WATCHDOG, + SessionRecord, + SessionRecordEvent, +) +from oss.src.core.sessions.records.interfaces import RecordsDAOInterface +from oss.src.core.sessions.records.service import RecordsService +from oss.src.core.sessions.executions.dtos import ( + SessionExecutionSettlement, + SessionExecutionSettlementResult, +) +from oss.src.utils.env import env + + +_PROJECT = UUID("00000000-0000-0000-0000-0000000000aa") +_SESSION = "sess-late-tail" +_TURN = "turn-abc" + + +@pytest.fixture(autouse=True) +def _durable_stop_enabled(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + + +class _StubDAO(RecordsDAOInterface): + """Answers `settled_turns` from a fixed set and remembers what `append_many` was given. + + The settled sets belong to `project`, and the real DAO scopes its query the same way, so a + key from another project is never a hit however it is spelled. + """ + + def __init__( + self, + *, + watchdog_settled: Optional[Set[Tuple[str, str]]] = None, + any_settled: Optional[Set[Tuple[str, str]]] = None, + project: UUID = _PROJECT, + raises: bool = False, + ): + self.watchdog_settled = watchdog_settled or set() + self.any_settled = any_settled or set() + self.project = project + self.raises = raises + self.appended: List[SessionRecordEvent] = [] + self.lookups: List[Dict] = [] + + async def settled_turns( + self, + *, + project_id: UUID, + keys: Sequence[Tuple[str, str]], + settled_by: Optional[str] = None, + ) -> Set[Tuple[str, str]]: + self.lookups.append({"project_id": project_id, "settled_by": settled_by}) + if self.raises: + raise RuntimeError("tracing database is unreachable") + if project_id != self.project: + return set() + source = ( + self.watchdog_settled + if settled_by == SETTLED_BY_WATCHDOG + else self.any_settled + ) + return {key for key in keys if key in source} + + async def append_many( + self, *, events: List[SessionRecordEvent] + ) -> List[SessionRecord]: + self.appended.extend(events) + return [ + SessionRecord( + record_id=event.record_id or uuid4(), + session_id=event.session_id, + project_id=event.project_id, + record_index=event.record_index, + record_type=event.record_type, + record_source=event.record_source, + attributes=event.attributes, + turn_id=event.turn_id, + quarantined_at=event.quarantined_at, + ) + for event in events + ] + + +class _ExecutionSettlements: + def __init__(self, *, raises: bool = False, mark_raises: bool = False): + self.rows: Dict[Tuple[str, str], SessionExecutionSettlement] = {} + self.raises = raises + self.mark_raises = mark_raises + + async def settle( + self, + *, + project_id, + session_id, + execution_id, + terminal_outcome, + settled_by, + settled_at=None, + ): + key = (session_id, execution_id) + if key in self.rows: + return SessionExecutionSettlementResult( + settlement=self.rows[key], won=False + ) + row = SessionExecutionSettlement( + project_id=project_id, + session_id=session_id, + execution_id=execution_id, + terminal_outcome=terminal_outcome, + settled_by=settled_by, + settled_at=settled_at or datetime.now(timezone.utc), + ) + self.rows[key] = row + return SessionExecutionSettlementResult(settlement=row, won=True) + + async def query_settled(self, *, project_id, keys): + if self.raises: + raise RuntimeError("core database is unreachable") + return {key: self.rows[key] for key in keys if key in self.rows} + + async def mark_endings_written(self, *, project_id, keys, written_at=None): + if self.raises or self.mark_raises: + raise RuntimeError("core database is unreachable") + for key in keys: + if key in self.rows and self.rows[key].ending_written_at is None: + self.rows[key] = self.rows[key].model_copy( + update={ + "ending_written_at": written_at or datetime.now(timezone.utc) + } + ) + + +def _event(record_type: str, **over) -> SessionRecordEvent: + base = { + "project_id": _PROJECT, + "session_id": _SESSION, + "record_id": uuid4(), + "record_type": record_type, + "record_source": "agent", + "attributes": {"type": record_type}, + "turn_id": _TURN, + } + base.update(over) + return SessionRecordEvent(**base) + + +def _watchdog_event(record_type: str, **over) -> SessionRecordEvent: + """What `orphan_sweep._lost_turn_records` puts on the stream.""" + event = _event(record_type, **over) + event.attributes = { + **(event.attributes or {}), + RECORD_SETTLED_BY_ATTRIBUTE: SETTLED_BY_WATCHDOG, + } + return event + + +def _quarantined(dao: _StubDAO) -> List[SessionRecordEvent]: + return [event for event in dao.appended if event.quarantined_at is not None] + + +# --------------------------------------------------------------------------- # +# The tail: output produced before termination, delivered after it +# --------------------------------------------------------------------------- # + + +async def test_a_thawed_runners_tail_remains_visible_with_durable_stop_off( + monkeypatch, +): + """The live defect, in one test: four records land after the watchdog's ending.""" + monkeypatch.setattr(env.agenta.sessions, "durable_stop", False) + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService( + records_dao=dao, + executions_dao=_ExecutionSettlements(), + ) + + tail = [ + _event("tool_call"), + _event("tool_result"), + _event("usage"), + _event("done", attributes={"type": "done", "stopReason": "cancelled"}), + ] + results = await service.append_many(events=tail) + + # Flag-off retains the pre-milestone presentation: every late record remains visible. + assert len(results) == 4 + assert _quarantined(dao) == [] + assert all(row.quarantined_at is None for row in results) + + +async def test_reject_policy_drops_a_late_tail(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "late_output", "reject") + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + results = await service.append_many(events=[_event("tool_result"), _event("usage")]) + + assert results == [] + assert dao.appended == [] + + +async def test_watchdog_winner_quarantines_the_runners_records(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + executions = _ExecutionSettlements() + winner = await executions.settle( + project_id=_PROJECT, + session_id=_SESSION, + execution_id=_TURN, + terminal_outcome="lost", + settled_by="watchdog", + ) + assert winner.won is True + dao = _StubDAO() + service = RecordsService(records_dao=dao, executions_dao=executions) + + await service.append_many( + events=[ + _event("usage"), + _event("done", attributes={"type": "done", "stopReason": "cancelled"}), + ] + ) + + assert [event.record_type for event in _quarantined(dao)] == ["usage", "done"] + + +async def test_watchdog_winner_rejects_the_runners_records_when_configured(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + monkeypatch.setattr(env.agenta.sessions, "late_output", "reject") + executions = _ExecutionSettlements() + await executions.settle( + project_id=_PROJECT, + session_id=_SESSION, + execution_id=_TURN, + terminal_outcome="lost", + settled_by="watchdog", + ) + dao = _StubDAO() + service = RecordsService(records_dao=dao, executions_dao=executions) + + results = await service.append_many(events=[_event("usage"), _event("done")]) + + assert results == [] + assert dao.appended == [] + + +async def test_runner_winner_quarantines_the_watchdogs_records(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + executions = _ExecutionSettlements() + winner = await executions.settle( + project_id=_PROJECT, + session_id=_SESSION, + execution_id=_TURN, + terminal_outcome="stopped", + settled_by="runner", + ) + assert winner.won is True + dao = _StubDAO() + service = RecordsService(records_dao=dao, executions_dao=executions) + + await service.append_many( + events=[_watchdog_event("error"), _watchdog_event("done")] + ) + + assert [event.record_type for event in _quarantined(dao)] == ["error", "done"] + + +async def test_output_after_the_runners_own_stop_is_ordinary_history(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + executions = _ExecutionSettlements() + dao = _StubDAO() + service = RecordsService(records_dao=dao, executions_dao=executions) + + await service.append_many(events=[_event("usage"), _event("done")]) + await service.append_many(events=[_event("tool_result")]) + + assert _quarantined(dao) == [] + + +async def test_an_ordinary_completion_row_does_not_make_trailing_usage_late( + monkeypatch, +): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + executions = _ExecutionSettlements() + await executions.settle( + project_id=_PROJECT, + session_id=_SESSION, + execution_id=_TURN, + terminal_outcome="completed", + settled_by="runner", + ) + dao = _StubDAO() + service = RecordsService(records_dao=dao, executions_dao=executions) + + await service.append_many(events=[_event("usage")]) + + assert _quarantined(dao) == [] + + +async def test_execution_lookup_failure_appends_the_batch_unguarded(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + dao = _StubDAO() + service = RecordsService( + records_dao=dao, + executions_dao=_ExecutionSettlements(raises=True), + ) + events = [_event("usage"), _event("done")] + + results = await service.append_many(events=events) + + assert len(results) == 2 + assert dao.appended == events + assert _quarantined(dao) == [] + + +async def test_ingest_does_not_write_a_terminal_execution(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + executions = _ExecutionSettlements() + service = RecordsService(records_dao=_StubDAO(), executions_dao=executions) + + await service.append_many( + events=[_event("done", attributes={"type": "done", "stopReason": "cancelled"})] + ) + + assert executions.rows == {} + + +async def test_ingest_marks_the_runners_terminal_record_written(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + executions = _ExecutionSettlements() + await executions.settle( + project_id=_PROJECT, + session_id=_SESSION, + execution_id=_TURN, + terminal_outcome="stopped", + settled_by="runner", + ) + service = RecordsService(records_dao=_StubDAO(), executions_dao=executions) + + await service.append_many( + events=[_event("done", attributes={"type": "done", "stopReason": "cancelled"})] + ) + + assert executions.rows[(_SESSION, _TURN)].ending_written_at is not None + + +async def test_ending_marker_failure_does_not_fail_record_ingest(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + executions = _ExecutionSettlements(mark_raises=True) + await executions.settle( + project_id=_PROJECT, + session_id=_SESSION, + execution_id=_TURN, + terminal_outcome="stopped", + settled_by="runner", + ) + dao = _StubDAO() + service = RecordsService(records_dao=dao, executions_dao=executions) + + results = await service.append_many(events=[_event("done")]) + + assert len(results) == 1 + assert [event.record_type for event in dao.appended] == ["done"] + assert executions.rows[(_SESSION, _TURN)].ending_written_at is None + + +async def test_the_guard_asks_only_about_watchdog_endings(): + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + await service.append_many(events=[_event("usage")]) + + assert [lookup["settled_by"] for lookup in dao.lookups] == [SETTLED_BY_WATCHDOG] + + +async def test_a_late_terminal_record_is_quarantined_like_the_rest_of_the_tail(): + """One effective ending. The runner's contradicting `done` is kept, but not as history. + + Folding it into the watchdog's ending would rewrite the record the user has already + read, and would hide that two writers disagreed about how the turn finished. + """ + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + await service.append_many( + events=[_event("done", attributes={"type": "done", "stopReason": "cancelled"})] + ) + + assert len(_quarantined(dao)) == 1 + assert _quarantined(dao)[0].record_type == "done" + + +# --------------------------------------------------------------------------- # +# What the guard must never touch +# --------------------------------------------------------------------------- # + + +async def test_an_ordinary_stop_the_watchdog_never_saw_is_untouched(): + """The runner's own honest single ending still lands, unmarked.""" + dao = _StubDAO(watchdog_settled=set()) + service = RecordsService(records_dao=dao) + + ending = [ + _event("usage"), + _event("done", attributes={"type": "done", "stopReason": "cancelled"}), + ] + results = await service.append_many(events=ending) + + assert _quarantined(dao) == [] + assert all(row.quarantined_at is None for row in results) + + +async def test_a_turn_the_runner_settled_itself_does_not_trigger_the_guard(): + """A terminal record is not enough; it has to be the WATCHDOG's. + + A `usage` that trails its own `done` through the stream is ordinary history, and a turn + that reached its own ending never lost the argument with the platform. + """ + dao = _StubDAO(watchdog_settled=set(), any_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + await service.append_many(events=[_event("usage")]) + + assert _quarantined(dao) == [] + + +async def test_the_watchdogs_own_records_are_never_quarantined(): + """Its `error` is not terminal, so without the exemption a redelivery would mark it.""" + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + await service.append_many( + events=[ + _watchdog_event( + "error", attributes={"type": "error", "code": "execution_lost"} + ), + _watchdog_event("done"), + ] + ) + + assert _quarantined(dao) == [] + # And they are not even looked up: a watchdog record can never be late for its own turn. + assert dao.lookups == [] + + +async def test_a_record_with_no_turn_id_is_never_quarantined(): + """Nothing to attribute it to. Old records carry no turn key at all.""" + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + await service.append_many(events=[_event("message", turn_id=None)]) + + assert _quarantined(dao) == [] + + +async def test_another_turn_in_the_same_session_is_untouched(): + """The user sent a new message after the failure; that turn is nobody's tail.""" + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + await service.append_many( + events=[_event("message", turn_id="turn-next"), _event("usage")] + ) + + assert [event.record_type for event in _quarantined(dao)] == ["usage"] + + +# --------------------------------------------------------------------------- # +# Batching, redelivery, and failure +# --------------------------------------------------------------------------- # + + +async def test_a_watchdog_ending_settles_its_turn_for_the_rest_of_its_own_batch(): + """Ingest batches up to fifty messages; the tail can share one with the ending. + + Without this the DB lookup would find nothing — the ending is not committed yet — and the + tail would be appended as ordinary history. + """ + dao = _StubDAO(watchdog_settled=set()) + service = RecordsService(records_dao=dao) + + await service.append_many( + events=[ + _watchdog_event( + "error", attributes={"type": "error", "code": "execution_lost"} + ), + _watchdog_event("done"), + _event("tool_result"), + _event("done", attributes={"type": "done", "stopReason": "cancelled"}), + ] + ) + + assert [event.record_type for event in _quarantined(dao)] == ["tool_result", "done"] + + +async def test_redelivery_quarantines_the_same_records_again(): + """The stream replays on a consumer-group failure; the outcome must not drift. + + The upsert coalesces `quarantined_at`, so the row keeps the instant it was FIRST marked; + what this pins is that the guard's own verdict is the same on every delivery. + """ + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + tail = [_event("tool_call"), _event("usage")] + first = await service.append_many(events=tail) + second = await service.append_many(events=tail) + + assert [row.record_id for row in first] == [row.record_id for row in second] + assert all(row.quarantined_at is not None for row in first + second) + + +async def test_a_failed_lookup_appends_the_batch_rather_than_losing_it(): + """Losing a record is worse than showing one that should have been hidden.""" + dao = _StubDAO(raises=True) + service = RecordsService(records_dao=dao) + + results = await service.append_many(events=[_event("tool_call"), _event("done")]) + + assert len(results) == 2 + assert _quarantined(dao) == [] + + +async def test_an_empty_batch_asks_the_database_nothing(): + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + assert await service.append_many(events=[]) == [] + assert dao.lookups == [] + assert dao.appended == [] + + +async def test_each_project_in_a_batch_gets_its_own_lookup(): + """`settled_turns` is project-scoped; a mixed batch must not ask across the boundary.""" + other_project = UUID("00000000-0000-0000-0000-0000000000bb") + dao = _StubDAO(watchdog_settled={(_SESSION, _TURN)}) + service = RecordsService(records_dao=dao) + + await service.append_many( + events=[_event("usage"), _event("usage", project_id=other_project)] + ) + + assert sorted(str(lookup["project_id"]) for lookup in dao.lookups) == sorted( + [str(_PROJECT), str(other_project)] + ) + # Only the project whose turn the watchdog settled is affected. + assert [event.project_id for event in _quarantined(dao)] == [_PROJECT] diff --git a/api/oss/tests/pytest/unit/sessions/test_late_record_quarantine_dao.py b/api/oss/tests/pytest/unit/sessions/test_late_record_quarantine_dao.py new file mode 100644 index 00000000000..2d90c28626d --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_late_record_quarantine_dao.py @@ -0,0 +1,245 @@ +"""The database half of the late-record guard, against a real Postgres. + +The service decides WHICH records are late (`test_late_record_quarantine.py`, no database +needed). These tests pin what the mark then does, and none of it is visible from a stub: + + - a quarantined row is invisible to `get_records`, which is the read every transcript + reconstruction goes through, so one execution renders one ending; + - a quarantined row does not answer `settled_turns`, so a late `done` can never stand in + for the real ending and suppress the watchdog's next pass; + - `settled_by` narrows `settled_turns` to one writer; + - the upsert coalesces `quarantined_at`, so a redelivery keeps the first mark and can never + resurrect a row into the transcript. + +Requires the tracing_oss chain through oss000000005_add_records_quarantined_at, with +POSTGRES_URI_TRACING pointed at that database. +""" + +import uuid +from datetime import datetime, timedelta, timezone + +import pytest + +from oss.src.core.sessions.records.dtos import ( + RECORD_SETTLED_BY_ATTRIBUTE, + SETTLED_BY_WATCHDOG, + SessionRecordEvent, +) +from oss.src.dbs.postgres.sessions.records.dao import RecordsDAO +import oss.src.dbs.postgres.shared.engine as engine_module +from oss.src.dbs.postgres.shared.engine import get_analytics_engine + + +pytestmark = pytest.mark.integration + + +@pytest.fixture(autouse=True) +async def _fresh_engine_per_test(): + """Each pytest-asyncio test gets its own event loop; the module-level engine singleton + binds its asyncpg pool to the first loop that touches it.""" + engine_module._analytics_engine = None + yield + if engine_module._analytics_engine is not None: + await engine_module._analytics_engine.close() + engine_module._analytics_engine = None + + +def _ids(): + return uuid.uuid4(), f"late-record-test-{uuid.uuid4().hex[:8]}" + + +def _event(project_id, session_id, turn_id, record_type, **over): + base = dict( + project_id=project_id, + session_id=session_id, + record_id=uuid.uuid4(), + record_index=0, + record_type=record_type, + record_source="agent", + attributes={"type": record_type}, + turn_id=turn_id, + ) + base.update(over) + return SessionRecordEvent(**base) + + +def _watchdog_done(project_id, session_id, turn_id): + return _event( + project_id, + session_id, + turn_id, + "done", + record_index=1, + attributes={"type": "done", RECORD_SETTLED_BY_ATTRIBUTE: SETTLED_BY_WATCHDOG}, + ) + + +async def test_a_quarantined_record_is_absent_from_the_transcript(): + project_id, session_id = _ids() + turn_id = f"turn-{uuid.uuid4().hex[:8]}" + dao = RecordsDAO(engine=get_analytics_engine()) + + await dao.append_many( + events=[ + _event(project_id, session_id, turn_id, "message", record_index=0), + _watchdog_done(project_id, session_id, turn_id), + _event( + project_id, + session_id, + turn_id, + "tool_call", + record_index=2, + quarantined_at=datetime.now(timezone.utc), + ), + _event( + project_id, + session_id, + turn_id, + "done", + record_index=3, + quarantined_at=datetime.now(timezone.utc), + ), + ] + ) + + rows = await dao.get_records(project_id=project_id, session_id=session_id) + + assert [row.record_type for row in rows] == ["message", "done"] + # Exactly one ending, and it is the watchdog's. + endings = [row for row in rows if row.record_type == "done"] + assert len(endings) == 1 + assert endings[0].attributes[RECORD_SETTLED_BY_ATTRIBUTE] == SETTLED_BY_WATCHDOG + + +async def test_a_quarantined_terminal_record_does_not_settle_its_turn(): + project_id, session_id = _ids() + turn_id = f"turn-{uuid.uuid4().hex[:8]}" + dao = RecordsDAO(engine=get_analytics_engine()) + + await dao.append_many( + events=[ + _event( + project_id, + session_id, + turn_id, + "done", + quarantined_at=datetime.now(timezone.utc), + ) + ] + ) + + settled = await dao.settled_turns( + project_id=project_id, keys=[(session_id, turn_id)] + ) + + assert settled == set() + + +async def test_settled_by_narrows_the_answer_to_one_writer(): + project_id, session_id = _ids() + runner_turn = f"turn-{uuid.uuid4().hex[:8]}" + watchdog_turn = f"turn-{uuid.uuid4().hex[:8]}" + dao = RecordsDAO(engine=get_analytics_engine()) + + await dao.append_many( + events=[ + _event(project_id, session_id, runner_turn, "done"), + _watchdog_done(project_id, session_id, watchdog_turn), + ] + ) + + keys = [(session_id, runner_turn), (session_id, watchdog_turn)] + + # The watchdog's own idempotency question: has this turn ANY ending? + assert await dao.settled_turns(project_id=project_id, keys=keys) == set(keys) + # The ingest guard's question: did the PLATFORM end this turn? + assert await dao.settled_turns( + project_id=project_id, keys=keys, settled_by=SETTLED_BY_WATCHDOG + ) == {(session_id, watchdog_turn)} + + +async def test_a_redelivery_keeps_the_first_quarantine_instant(): + project_id, session_id = _ids() + turn_id = f"turn-{uuid.uuid4().hex[:8]}" + dao = RecordsDAO(engine=get_analytics_engine()) + + first_mark = datetime(2026, 9, 3, 12, 0, 0, tzinfo=timezone.utc) + event = _event(project_id, session_id, turn_id, "usage", quarantined_at=first_mark) + + await dao.append_many(events=[event]) + later = event.model_copy( + update={"quarantined_at": datetime(2026, 9, 3, 13, 0, 0, tzinfo=timezone.utc)} + ) + rows = await dao.append_many(events=[later]) + + assert rows[0].quarantined_at == first_mark + + +async def test_an_unmarked_redelivery_cannot_resurrect_a_quarantined_record(): + """Quarantine is one-way. A delivery that somehow arrives unguarded must not undo it.""" + project_id, session_id = _ids() + turn_id = f"turn-{uuid.uuid4().hex[:8]}" + dao = RecordsDAO(engine=get_analytics_engine()) + + mark = datetime.now(timezone.utc) + event = _event(project_id, session_id, turn_id, "tool_call", quarantined_at=mark) + await dao.append_many(events=[event]) + + await dao.append_many(events=[event.model_copy(update={"quarantined_at": None})]) + + rows = await dao.get_records(project_id=project_id, session_id=session_id) + assert rows == [] + + +async def test_an_ordinary_record_is_still_written_and_read_unmarked(): + project_id, session_id = _ids() + turn_id = f"turn-{uuid.uuid4().hex[:8]}" + dao = RecordsDAO(engine=get_analytics_engine()) + + await dao.append_many( + events=[ + _event(project_id, session_id, turn_id, "message", record_index=0), + _event(project_id, session_id, turn_id, "done", record_index=1), + ] + ) + + rows = await dao.get_records(project_id=project_id, session_id=session_id) + + assert [row.record_type for row in rows] == ["message", "done"] + assert all(row.quarantined_at is None for row in rows) + + +async def test_a_quarantined_message_never_becomes_the_session_preview(): + project_id, session_id = _ids() + turn_id = f"turn-{uuid.uuid4().hex[:8]}" + dao = RecordsDAO(engine=get_analytics_engine()) + + now = datetime.now(timezone.utc) + await dao.append_many( + events=[ + _event( + project_id, + session_id, + turn_id, + "message", + attributes={"type": "message", "text": "the real last message"}, + timestamp=now, + ), + _event( + project_id, + session_id, + turn_id, + "message", + attributes={"type": "message", "text": "written after the ending"}, + # Newer than the real one: without the filter this would win the preview. + timestamp=now + timedelta(seconds=10), + quarantined_at=now, + ), + ] + ) + + previews = await dao.latest_message_per_session( + project_id=project_id, session_ids=[session_id] + ) + + assert previews[session_id].text == "the real last message" diff --git a/api/oss/tests/pytest/unit/sessions/test_orphan_sweep_clears_redis.py b/api/oss/tests/pytest/unit/sessions/test_orphan_sweep_clears_redis.py index db97b1eb842..7e7d62db863 100644 --- a/api/oss/tests/pytest/unit/sessions/test_orphan_sweep_clears_redis.py +++ b/api/oss/tests/pytest/unit/sessions/test_orphan_sweep_clears_redis.py @@ -9,6 +9,8 @@ """ from contextlib import asynccontextmanager +from typing import Optional + from datetime import datetime, timezone, timedelta import pytest @@ -22,10 +24,17 @@ class _FakeRow: - def __init__(self, *, session_id: str, updated_at: datetime): + def __init__( + self, + *, + session_id: str, + updated_at: datetime, + turn_id: Optional[str] = None, + ): self.session_id = session_id self.project_id = _PROJECT_ID self.id = "stream-1" + self.turn_id = turn_id self.deleted_at = None self.flags = {"is_alive": True, "is_running": True, "is_attached": False} self.updated_at = updated_at @@ -40,8 +49,9 @@ def all(self): class _FakeResult: - def __init__(self, rows): + def __init__(self, rows, *, rowcount=0): self._rows = rows + self.rowcount = rowcount def scalars(self): return _FakeScalars(self._rows) @@ -54,6 +64,30 @@ def __init__(self, rows, seen): async def execute(self, stmt): self._seen.append(stmt) + text = str(stmt) + if text.startswith("UPDATE") and "session_streams" in text: + # Both session_streams writes are Core UPDATEs keyed by row id, never ORM + # attribute writes (finding 7). The collapse binds `id IN (...)`, a list; the + # lost-turn clear binds `id = ...`, a scalar. Apply either to the in-memory rows. + params = stmt.compile().params + flags_val = next( + (v for v in params.values() if isinstance(v, dict) and "is_alive" in v), + None, + ) + ids = set() + for value in params.values(): + if isinstance(value, (list, set, tuple)): + ids.update(x for x in value if isinstance(x, str)) + elif isinstance(value, str): + ids.add(value) + if flags_val is not None: + matched = 0 + for row in self._rows: + if row.id in ids: + row.flags = dict(flags_val) + matched += 1 + return _FakeResult([], rowcount=matched) + return _FakeResult([]) return _FakeResult(self._rows) async def commit(self): @@ -94,6 +128,36 @@ async def delete(self, key): async def expire(self, key, ttl): return True + async def eval(self, script, numkeys, *keys_and_args): + def decode(value): + return value.decode() if isinstance(value, bytes) else str(value) + + keys = [decode(value) for value in keys_and_args[:numkeys]] + argv = [decode(value) for value in keys_and_args[numkeys:]] + assert "AGENTA_WATCHDOG_RELEASE_TURN" in script + alive, running, owner, superseded = keys + expected_turn, expected_owner, _ttl = argv + alive_value = decode(self._store[alive]) if alive in self._store else "" + running_value = decode(self._store[running]) if running in self._store else "" + owner_value = decode(self._store[owner]) if owner in self._store else "" + released_alive = int(bool(expected_turn) and alive_value == expected_turn) + released_running = int(bool(expected_turn) and running_value == expected_turn) + if released_alive: + self._store.pop(alive, None) + if released_running: + self._store.pop(running, None) + foreign_turn = (alive_value and alive_value != expected_turn) or ( + running_value and running_value != expected_turn + ) + released_owner = int( + bool(expected_owner) and owner_value == expected_owner and not foreign_turn + ) + if released_owner: + self._store.pop(owner, None) + if expected_turn: + self._store[superseded] = b"1" + return [released_alive, released_running, released_owner] + @pytest.fixture def anyio_backend(): @@ -114,10 +178,14 @@ async def test_orphan_sweep_clears_alive_lock_and_unblocks_send(anyio_backend): await lock_engine.set( f"running:{_PROJECT_ID}:session:{_SESSION_ID}", b"turn-1", ex=3600 ) + await lock_engine.set( + f"owner:{_PROJECT_ID}:session:{_SESSION_ID}", b"replica-legacy", ex=120 + ) stale_row = _FakeRow( session_id=_SESSION_ID, updated_at=datetime.now(timezone.utc) - timedelta(seconds=600), + turn_id=None, ) pg_engine = _FakeTransactionsEngine([stale_row]) @@ -141,6 +209,7 @@ async def test_orphan_sweep_clears_alive_lock_and_unblocks_send(anyio_backend): lock_engine, project_id=_PROJECT_ID, session_id=_SESSION_ID ) assert liveness_after == {"alive": False, "running": False, "attached": False} + assert await lock_engine.get(f"owner:{_PROJECT_ID}:session:{_SESSION_ID}") is None # SEND gate logic (service.py:99-101): would raise if alive were still true. def _send_gate(liveness): @@ -168,6 +237,7 @@ async def test_orphan_sweep_tombstones_the_turn_it_swept(anyio_backend): stale_row = _FakeRow( session_id=_SESSION_ID, updated_at=datetime.now(timezone.utc) - timedelta(seconds=600), + turn_id=None, ) await run_orphan_sweep(_FakeTransactionsEngine([stale_row]), lock_engine) diff --git a/api/oss/tests/pytest/unit/sessions/test_orphan_sweep_thresholds.py b/api/oss/tests/pytest/unit/sessions/test_orphan_sweep_thresholds.py index 20f197cadfe..3d9d8558b20 100644 --- a/api/oss/tests/pytest/unit/sessions/test_orphan_sweep_thresholds.py +++ b/api/oss/tests/pytest/unit/sessions/test_orphan_sweep_thresholds.py @@ -30,6 +30,7 @@ UnaryExpression, ) from sqlalchemy.sql.functions import Function +from sqlalchemy.sql.dml import Update from oss.src.tasks.asyncio.sessions.orphan_sweep import ( IDLE_THRESHOLD_SECONDS, @@ -80,8 +81,16 @@ def _evaluate(node, row) -> Optional[bool]: left, right = _value(node.left, row), _value(node.right, row) if node.operator is operators.is_: return left is right + if node.operator is operators.is_not: + # `turn_id IS NOT NULL`, from the ending-only selection. Postgres `IS NOT` is a + # total predicate: it never returns NULL, so neither does this. + return left is not right if node.operator is operators.lt: return None if left is None or right is None else left < right + if node.operator is operators.eq: + return None if left is None or right is None else left == right + if node.operator is operators.in_op: + return None if left is None else left in right if getattr(node.operator, "opstring", None) == "@>": return _contains(left, right) raise AssertionError( @@ -113,10 +122,18 @@ def _value(node, row): class _FakeRow: - def __init__(self, *, session_id: str, flags: Optional[dict], age_seconds: int): + def __init__( + self, + *, + session_id: str, + flags: Optional[dict], + age_seconds: int, + turn_id: Optional[str] = None, + ): self.session_id = session_id self.project_id = _PROJECT_ID self.id = session_id + self.turn_id = turn_id self.deleted_at = None self.flags = flags self.created_at = datetime.now(timezone.utc) - timedelta(days=1) @@ -132,18 +149,32 @@ def all(self): class _FakeResult: - def __init__(self, rows): + def __init__(self, rows, *, rowcount=0): self._rows = rows + self.rowcount = rowcount def scalars(self): return _FakeScalars(self._rows) class _FakePgSession: - def __init__(self, rows): + def __init__(self, rows, before_update=None): self._rows = rows + self._before_update = before_update async def execute(self, stmt): + if isinstance(stmt, Update): + if self._before_update is not None: + self._before_update() + self._before_update = None + matched = [ + row for row in self._rows if _evaluate(stmt.whereclause, row) is True + ] + for row in matched: + for column, value in stmt._values.items(): + key = column if isinstance(column, str) else column.key + setattr(row, key, _value(value, row)) + return _FakeResult([], rowcount=len(matched)) matched = [ row for row in self._rows if _evaluate(stmt.whereclause, row) is True ] @@ -154,12 +185,13 @@ async def commit(self): class _FakeTransactionsEngine: - def __init__(self, rows): + def __init__(self, rows, before_update=None): self._rows = rows + self._before_update = before_update @asynccontextmanager async def session(self): - yield _FakePgSession(self._rows) + yield _FakePgSession(self._rows, self._before_update) class _FakeRedis: @@ -182,6 +214,36 @@ async def delete(self, key): async def expire(self, key, ttl): return True + async def eval(self, script, numkeys, *keys_and_args): + def decode(value): + return value.decode() if isinstance(value, bytes) else str(value) + + keys = [decode(value) for value in keys_and_args[:numkeys]] + argv = [decode(value) for value in keys_and_args[numkeys:]] + assert "AGENTA_WATCHDOG_RELEASE_TURN" in script + alive, running, owner, superseded = keys + expected_turn, expected_owner, _ttl = argv + alive_value = decode(self._store[alive]) if alive in self._store else "" + running_value = decode(self._store[running]) if running in self._store else "" + owner_value = decode(self._store[owner]) if owner in self._store else "" + released_alive = int(bool(expected_turn) and alive_value == expected_turn) + released_running = int(bool(expected_turn) and running_value == expected_turn) + if released_alive: + self._store.pop(alive, None) + if released_running: + self._store.pop(running, None) + foreign_turn = (alive_value and alive_value != expected_turn) or ( + running_value and running_value != expected_turn + ) + released_owner = int( + bool(expected_owner) and owner_value == expected_owner and not foreign_turn + ) + if released_owner: + self._store.pop(owner, None) + if expected_turn: + self._store[superseded] = b"1" + return [released_alive, released_running, released_owner] + def _swept(row: _FakeRow) -> bool: return row.flags == {"is_alive": False, "is_running": False, "is_attached": False} @@ -192,6 +254,32 @@ def anyio_backend(): return "asyncio" +class _OrderedCommandsService: + def __init__(self) -> None: + self.calls = [] + + async def settle_abandoned_commands(self, *, now): + self.calls.append("settle") + return 0 + + async def repair_terminal_redis(self): + self.calls.append("repair") + return 0 + + +@pytest.mark.anyio +async def test_redis_repair_runs_after_the_sweeps_main_work(anyio_backend): + commands = _OrderedCommandsService() + + await run_orphan_sweep( + _FakeTransactionsEngine([]), + _FakeRedis(), + commands_service=commands, + ) + + assert commands.calls == ["settle", "repair"] + + @pytest.mark.anyio async def test_running_row_is_swept_at_the_short_threshold(anyio_backend): row = _FakeRow( @@ -241,8 +329,13 @@ async def test_idle_row_is_swept_at_the_long_threshold(anyio_backend): @pytest.mark.anyio -async def test_thresholds_are_five_and_thirty_minutes(anyio_backend): - assert (ORPHAN_THRESHOLD_SECONDS, IDLE_THRESHOLD_SECONDS) == (300, 1800) +async def test_default_running_threshold_uses_durable_stop(anyio_backend): + """Three missed 30-second heartbeats settle a running turn by default. + + Idle sessions retain the 30-minute approval TTL. Explicit flag-off behavior + is covered by the session cancellation configuration tests. + """ + assert (ORPHAN_THRESHOLD_SECONDS, IDLE_THRESHOLD_SECONDS) == (90, 1800) @pytest.mark.anyio @@ -295,9 +388,61 @@ async def test_sweep_clears_redis_for_the_long_threshold_branch(anyio_backend): session_id=session_id, flags={"is_alive": True, "is_running": False, "is_attached": False}, age_seconds=IDLE_THRESHOLD_SECONDS + 60, + turn_id="turn-1", ) await run_orphan_sweep(_FakeTransactionsEngine([row]), redis) assert await redis.get(f"alive:{_PROJECT_ID}:session:{session_id}") is None assert await redis.get(f"owner:{_PROJECT_ID}:session:{session_id}") is None + + +@pytest.mark.anyio +async def test_turn_advance_during_sweep_prevents_collapse_and_redis_cleanup( + anyio_backend, +): + session_id = "sess-advanced-during-sweep" + row = _FakeRow( + session_id=session_id, + flags={"is_alive": True, "is_running": True, "is_attached": False}, + age_seconds=360, + turn_id="turn-old", + ) + redis = _FakeRedis() + await redis.set(f"alive:{_PROJECT_ID}:session:{session_id}", b"turn-new") + await redis.set(f"running:{_PROJECT_ID}:session:{session_id}", b"turn-new") + await redis.set(f"owner:{_PROJECT_ID}:session:{session_id}", b"runner-new") + + def advance_row(): + row.turn_id = "turn-new" + row.updated_at = datetime.now(timezone.utc) + + await run_orphan_sweep( + _FakeTransactionsEngine([row], before_update=advance_row), redis + ) + + assert row.flags["is_alive"] is True + assert row.flags["is_running"] is True + assert await redis.get(f"alive:{_PROJECT_ID}:session:{session_id}") == b"turn-new" + assert await redis.get(f"running:{_PROJECT_ID}:session:{session_id}") == b"turn-new" + assert await redis.get(f"owner:{_PROJECT_ID}:session:{session_id}") == b"runner-new" + + +@pytest.mark.anyio +async def test_heartbeat_during_sweep_prevents_collapse(anyio_backend): + row = _FakeRow( + session_id="sess-heartbeat-during-sweep", + flags={"is_alive": True, "is_running": True, "is_attached": False}, + age_seconds=360, + turn_id="turn-current", + ) + + def heartbeat(): + row.updated_at = datetime.now(timezone.utc) + + await run_orphan_sweep( + _FakeTransactionsEngine([row], before_update=heartbeat), _FakeRedis() + ) + + assert row.flags["is_alive"] is True + assert row.flags["is_running"] is True diff --git a/api/oss/tests/pytest/unit/sessions/test_owner_claim.py b/api/oss/tests/pytest/unit/sessions/test_owner_claim.py index ba9e0b5892b..7c3538646b8 100644 --- a/api/oss/tests/pytest/unit/sessions/test_owner_claim.py +++ b/api/oss/tests/pytest/unit/sessions/test_owner_claim.py @@ -78,12 +78,16 @@ async def eval(self, script, numkeys, *keys_and_args): ) if script == CLAIM_OWNER_LUA: - replica_id, ex = argv + owner_value, ex = argv current = self._values.get(key) - replica_id_bytes = self._val(replica_id) - if current is None or current == replica_id_bytes: - await self.set(key, replica_id_bytes, ex=int(ex)) - return replica_id_bytes + owner_value_bytes = self._val(owner_value) + from oss.src.dbs.redis.sessions.contract import owner_replica_id + + if current is None or owner_replica_id( + current.decode() + ) == owner_replica_id(owner_value_bytes.decode()): + await self.set(key, owner_value_bytes, ex=int(ex)) + return owner_value_bytes return current if script == RELEASE_IF_OWNER_LUA: (owner,) = argv @@ -165,6 +169,37 @@ async def test_claim_owner_same_replica_refreshes_without_stealing(fake_redis): assert ttl <= OWNER_TTL_SECONDS +@pytest.mark.asyncio +async def test_claim_owner_same_replica_refreshes_to_the_new_turn_generation( + fake_redis, +): + from oss.src.dbs.redis.sessions.contract import make_owner_value, owner_key + from oss.src.dbs.redis.sessions.locks import claim_owner + + engine, client = fake_redis + session_id = _session_id() + + await claim_owner( + engine, + project_id=_PROJECT_ID, + session_id=session_id, + replica_id="replica-a", + turn_id="turn-a", + ) + await claim_owner( + engine, + project_id=_PROJECT_ID, + session_id=session_id, + replica_id="replica-a", + turn_id="turn-b", + ) + + assert ( + await client.get(owner_key(_PROJECT_ID, session_id)) + == make_owner_value(replica_id="replica-a", turn_id="turn-b").encode() + ) + + @pytest.mark.asyncio async def test_claim_owner_different_replica_does_not_steal(fake_redis): """The core S7 guarantee: a second replica's claim on an owned session never steals it.""" diff --git a/api/oss/tests/pytest/unit/sessions/test_project_scoped_locks.py b/api/oss/tests/pytest/unit/sessions/test_project_scoped_locks.py index e355d8d836b..90bfe149a85 100644 --- a/api/oss/tests/pytest/unit/sessions/test_project_scoped_locks.py +++ b/api/oss/tests/pytest/unit/sessions/test_project_scoped_locks.py @@ -24,12 +24,16 @@ ) from oss.src.dbs.redis.sessions.locks import ( acquire_alive, + acquire_running, claim_owner, force_cancel_alive, force_clear_owner, get_alive_owner, get_owner, + get_running_owner, get_session_liveness, + is_turn_superseded, + reconcile_stopped_turn, ) @@ -44,6 +48,7 @@ class _FakeRedis: def __init__(self): self._values: dict[str, bytes] = {} self._ttl: dict[str, int] = {} + self.now_ms = 1_000_000 @staticmethod def _norm(key) -> str: @@ -82,12 +87,74 @@ async def expire(self, key, ttl): async def ttl(self, key): return self._ttl.get(self._norm(key), -2) + async def time(self): + return divmod(self.now_ms * 1000, 1_000_000) + async def publish(self, channel, payload): return 0 async def eval(self, script, numkeys, *keys_and_args): - key = self._norm(keys_and_args[0]) + keys = [self._norm(key) for key in keys_and_args[:numkeys]] argv = [self._norm(a) for a in keys_and_args[numkeys:]] + if "AGENTA_ACQUIRE_ALIVE_WITH_START" in script: + if keys[0] in self._values: + return 0 + self._values[keys[0]] = argv[0].encode() + self._ttl[keys[0]] = int(argv[1]) + if keys[1] not in self._values: + self._values[keys[1]] = str(self.now_ms).encode() + self._ttl[keys[1]] = int(argv[2]) + return 1 + if "AGENTA_DISPLACE_TURNS" in script: + alive = self._values.get(keys[0], b"").decode() + running = self._values.get(keys[1], b"").decode() + expected = argv[0] + arrived_at_ms = int(argv[1]) if argv[1] else None + running_only = argv[5] == "1" + + def mismatches(owner: str) -> bool: + if not owner: + return False + if expected: + return owner != expected + started = self._values.get(f"{argv[3]}{owner}") + return bool( + arrived_at_ms is not None + and started is not None + and int(started.decode()) > arrived_at_ms + ) + + if not running_only and mismatches(alive): + return [0, alive.encode()] + if (running_only or running != alive) and mismatches(running): + return [0, running.encode()] + seen = set() + displaced = ( + (running, expected) if running_only else (alive, running, expected) + ) + for turn_id in displaced: + if turn_id and turn_id not in seen: + key = f"{argv[2]}{turn_id}" + self._values[key] = b"1" + self._ttl[key] = int(argv[4]) + seen.add(turn_id) + if not running_only or (alive and alive == running): + self._values.pop(keys[0], None) + self._ttl.pop(keys[0], None) + self._values.pop(keys[1], None) + self._ttl.pop(keys[1], None) + returned_alive = "" if running_only else alive + return [1, returned_alive.encode(), running.encode(), expected.encode()] + if "AGENTA_RECONCILE_STOPPED_TURN" in script: + self._values[keys[1]] = b"1" + self._ttl[keys[1]] = int(argv[1]) + if self._values.get(keys[0], b"").decode() == argv[0]: + self._values.pop(keys[0], None) + self._ttl.pop(keys[0], None) + return 1 + return 0 + + key = keys[0] current = self._values.get(key) current_s = current.decode() if current else None if "DEL" in script: # RELEASE_IF_OWNER_LUA @@ -96,7 +163,11 @@ async def eval(self, script, numkeys, *keys_and_args): return 1 return 0 # CLAIM_OWNER_LUA - if current_s is None or current_s == argv[0]: + from oss.src.dbs.redis.sessions.contract import owner_replica_id + + if current_s is None or owner_replica_id(current_s) == owner_replica_id( + argv[0] + ): self._values[key] = argv[0].encode() self._ttl[key] = int(argv[1]) return argv[0] @@ -193,6 +264,36 @@ async def test_tenant_cannot_clear_another_tenants_owner(engine): ) == "replica-b" +@pytest.mark.asyncio +async def test_durable_stop_reconciliation_preserves_alive_and_a_new_running_turn( + engine, +): + await acquire_alive( + engine, project_id=_TENANT_A, session_id=_SESSION, turn_id="turn-old" + ) + await acquire_running( + engine, project_id=_TENANT_A, session_id=_SESSION, turn_id="turn-new" + ) + + released = await reconcile_stopped_turn( + engine, + project_id=_TENANT_A, + session_id=_SESSION, + turn_id="turn-old", + ) + + assert released is False + assert ( + await get_alive_owner(engine, project_id=_TENANT_A, session_id=_SESSION) + ) == "turn-old" + assert ( + await get_running_owner(engine, project_id=_TENANT_A, session_id=_SESSION) + ) == "turn-new" + assert await is_turn_superseded( + engine, project_id=_TENANT_A, session_id=_SESSION, turn_id="turn-old" + ) + + # --------------------------------------------------------------------------- # # kill's owner drop (the 120s lockout) # --------------------------------------------------------------------------- # diff --git a/api/oss/tests/pytest/unit/sessions/test_records_config.py b/api/oss/tests/pytest/unit/sessions/test_records_config.py new file mode 100644 index 00000000000..a67cf8a3d6a --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_records_config.py @@ -0,0 +1,36 @@ +import pytest +from pydantic import ValidationError + +from oss.src.utils.env import SessionsRecordsConfig + + +def test_session_record_retry_bounds_accept_the_minimum_values(): + config = SessionsRecordsConfig(reclaim_idle_ms=0, max_deliveries=1) + + assert config.reclaim_idle_ms == 0 + assert config.max_deliveries == 1 + + +@pytest.mark.parametrize( + ("field", "value"), + [("reclaim_idle_ms", -1), ("max_deliveries", 0), ("max_deliveries", -1)], +) +def test_session_record_retry_bounds_reject_invalid_values(field, value): + with pytest.raises(ValidationError): + SessionsRecordsConfig(**{field: value}) + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("AGENTA_RECORDS_RECLAIM_IDLE_MS", "-1"), + ("AGENTA_RECORDS_MAX_DELIVERIES", "0"), + ], +) +def test_session_record_retry_bounds_validate_environment_defaults( + monkeypatch, name, value +): + monkeypatch.setenv(name, value) + + with pytest.raises(ValidationError): + SessionsRecordsConfig() diff --git a/api/oss/tests/pytest/unit/sessions/test_records_worker_durability.py b/api/oss/tests/pytest/unit/sessions/test_records_worker_durability.py new file mode 100644 index 00000000000..d287d822d53 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_records_worker_durability.py @@ -0,0 +1,517 @@ +"""Records must not be acknowledged before Postgres has them (#5496, #5594). + +`RecordsWorker.process_batch` used to add every decoded Redis message id to its +acknowledged list DURING deserialization, before `append_many` ran. A failed write logged +and continued, and the shared consumer loop then acknowledged and deleted those messages +from the stream. Every Postgres hiccup was therefore permanent, silent record loss, and one +record Postgres rejected took its whole batch with it. + +These tests pin three properties: + +* a message id is acknowledged only after its rows commit, and the redelivered batch is + written exactly once; +* one bad record does not discard the rest of its batch; +* a message that never writes is dropped loudly and counted, instead of holding the + pending list forever. + +The redelivery tests run against fakeredis so the pending-list bookkeeping is real Redis +consumer-group behaviour, not a mock of it. +""" + +import asyncio +import zlib +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import fakeredis.aioredis as fakeredis +import pytest +from orjson import dumps +from sqlalchemy.exc import IntegrityError + +from oss.src.core.sessions.records.dtos import SessionRecord +from oss.src.core.sessions.records.service import RecordsService +from oss.src.tasks.asyncio.sessions import records_worker +from oss.src.tasks.asyncio.sessions.records_worker import RecordsWorker + +STREAM = "streams:records" +GROUP = "worker-records" + + +def _payload(*, project_id, session_id, record_id, record_type="message", turn_id=None): + message = { + "organization_id": None, + "project_id": str(project_id), + "record_event": { + "project_id": str(project_id), + "session_id": session_id, + "record_id": str(record_id), + "record_type": record_type, + "turn_id": turn_id, + }, + } + return zlib.compress(dumps(message)) + + +class FakeRecordsDAO: + """Records what committed, and fails the events the caller names.""" + + def __init__(self, *, poison_ids=(), fail_calls=0, transient_error=None): + self.poison_ids = {str(record_id) for record_id in poison_ids} + self.fail_calls = fail_calls + self.transient_error = transient_error or ConnectionError("postgres is down") + self.calls = 0 + self.committed: list[str] = [] + + async def append_many(self, *, events): + self.calls += 1 + if self.calls <= self.fail_calls: + raise self.transient_error + if any(str(event.record_id) in self.poison_ids for event in events): + # `append_many` is one statement in one transaction: a rejected row takes the + # whole call with it, and nothing in the call commits. + raise IntegrityError("INSERT records", {}, ValueError("record rejected")) + for event in events: + self.committed.append(str(event.record_id)) + return [ + SessionRecord( + record_id=event.record_id, + session_id=event.session_id, + project_id=event.project_id, + ) + for event in events + ] + + +def _worker(dao, *, redis_client=None, max_deliveries=5): + return RecordsWorker( + service=RecordsService(records_dao=dao), + redis_client=redis_client, + stream_name=STREAM, + consumer_group=GROUP, + consumer_name="test-consumer", + reclaim_min_idle_ms=0, + max_deliveries=max_deliveries, + ) + + +def _batch(*, project_id, record_ids): + return [ + ( + f"{index}-0".encode(), + { + b"data": _payload( + project_id=project_id, session_id="sess-1", record_id=record_id + ) + }, + ) + for index, record_id in enumerate(record_ids) + ] + + +@pytest.mark.asyncio +async def test_failed_batch_acknowledges_nothing(): + project_id = uuid4() + record_ids = [uuid4(), uuid4()] + dao = FakeRecordsDAO(fail_calls=99) + + appended, acked_ids = await _worker(dao).process_batch( + _batch(project_id=project_id, record_ids=record_ids) + ) + + assert appended == 0 + # Nothing committed, so nothing may be acknowledged: the shared consumer loop deletes + # every id this list carries. + assert acked_ids == [] + assert dao.committed == [] + assert dao.calls == 1 + + +@pytest.mark.asyncio +async def test_timeout_leaves_the_whole_batch_pending_without_single_row_retries(): + project_id = uuid4() + dao = FakeRecordsDAO(fail_calls=99, transient_error=asyncio.TimeoutError()) + + appended, acked_ids = await _worker(dao).process_batch( + _batch(project_id=project_id, record_ids=[uuid4(), uuid4(), uuid4()]) + ) + + assert appended == 0 + assert acked_ids == [] + assert dao.calls == 1 + + +@pytest.mark.asyncio +async def test_redelivered_batch_is_acknowledged_once_and_written_once(): + project_id = uuid4() + record_ids = [uuid4(), uuid4()] + batch = _batch(project_id=project_id, record_ids=record_ids) + # The whole batch stays pending on the connection failure, then Postgres is back. + dao = FakeRecordsDAO(fail_calls=1) + worker = _worker(dao) + + _, first_acked = await worker.process_batch(batch) + assert first_acked == [] + + appended, second_acked = await worker.process_batch(batch) + + assert appended == 2 + assert second_acked == [msg_id for msg_id, _ in batch] + assert dao.committed == [str(record_id) for record_id in record_ids] + assert worker.dropped_messages == 0 + + +@pytest.mark.asyncio +async def test_one_bad_record_does_not_discard_its_batch(): + project_id = uuid4() + good_a, poison, good_b = uuid4(), uuid4(), uuid4() + batch = _batch(project_id=project_id, record_ids=[good_a, poison, good_b]) + dao = FakeRecordsDAO(poison_ids=[poison]) + + appended, acked_ids = await _worker(dao).process_batch(batch) + + assert appended == 2 + assert dao.committed == [str(good_a), str(good_b)] + # Only the two good ids are acknowledged. The rejected record stays pending. + assert acked_ids == [batch[0][0], batch[2][0]] + + +@pytest.mark.asyncio +async def test_undecodable_message_is_acknowledged_and_counted(): + dao = FakeRecordsDAO() + worker = _worker(dao) + + appended, acked_ids = await worker.process_batch([(b"1-0", {b"data": b"not-zlib"})]) + + assert appended == 0 + # A message that does not decode will not decode on redelivery, so it is dropped on + # purpose rather than left to hold the pending list. + assert acked_ids == [b"1-0"] + assert worker.dropped_messages == 1 + + +@pytest.mark.asyncio +async def test_watch_and_gate_reconciliation_see_only_committed_records(): + project_id = uuid4() + good, poison = uuid4(), uuid4() + batch = [ + ( + b"1-0", + { + b"data": _payload( + project_id=project_id, + session_id="sess-good", + record_id=good, + record_type="done", + turn_id="turn-good", + ) + }, + ), + ( + b"2-0", + { + b"data": _payload( + project_id=project_id, + session_id="sess-poison", + record_id=poison, + record_type="done", + turn_id="turn-poison", + ) + }, + ), + ] + + watch_publisher = AsyncMock() + interactions_service = AsyncMock() + interactions_service.cancel_session_pending = AsyncMock(return_value=0) + + worker = RecordsWorker( + service=RecordsService(records_dao=FakeRecordsDAO(poison_ids=[poison])), + redis_client=None, + stream_name=STREAM, + consumer_group=GROUP, + watch_publisher=watch_publisher, + interactions_service=interactions_service, + ) + + await worker.process_batch(batch) + + # A record that never committed must not wake a client or cancel a gate: the reader it + # would send to Postgres cannot see the row. + notified = { + call.kwargs["session_id"] + for call in watch_publisher.records_changed.await_args_list + } + assert notified == {"sess-good"} + reconciled = { + call.kwargs["session_id"] + for call in interactions_service.cancel_session_pending.await_args_list + } + assert reconciled == {"sess-good"} + + +async def _seed(redis_client, payloads): + await redis_client.xgroup_create( + name=STREAM, groupname=GROUP, id="0", mkstream=True + ) + for payload in payloads: + await redis_client.xadd(name=STREAM, fields={"data": payload}) + + +@pytest.mark.asyncio +async def test_unacknowledged_entry_comes_back_through_the_reclaim_pass(): + project_id = uuid4() + record_id = uuid4() + redis_client = fakeredis.FakeRedis() + await _seed( + redis_client, + [_payload(project_id=project_id, session_id="s", record_id=record_id)], + ) + + dao = FakeRecordsDAO(fail_calls=1) + worker = _worker(dao, redis_client=redis_client) + + batch = await worker.read_batch() + assert len(batch) == 1 + _, acked_ids = await worker.process_batch(batch) + assert acked_ids == [] + + # `read_batch` only ever asks for `>`, so without the reclaim pass this entry is invisible + # to every later read and the "leave it pending" fix would lose it silently. + assert await worker.read_batch() == [] + + await asyncio.sleep(0.01) + reclaimed = await worker.reclaim_batch() + assert [msg_id for msg_id, _ in reclaimed] == [msg_id for msg_id, _ in batch] + + _, acked_ids = await worker.process_batch(reclaimed) + assert acked_ids == [batch[0][0]] + await worker.ack_and_delete(acked_ids) + + assert dao.committed == [str(record_id)] + assert await redis_client.xlen(STREAM) == 0 + pending = await redis_client.xpending_range( + name=STREAM, groupname=GROUP, min="-", max="+", count=10 + ) + assert pending == [] + + +@pytest.mark.asyncio +async def test_a_record_that_never_writes_is_dropped_loudly_and_counted(caplog): + project_id = uuid4() + good, poison = uuid4(), uuid4() + redis_client = fakeredis.FakeRedis() + await _seed( + redis_client, + [ + _payload(project_id=project_id, session_id="s", record_id=good), + _payload( + project_id=project_id, + session_id="doomed-session", + record_id=poison, + record_type="done", + ), + ], + ) + + dao = FakeRecordsDAO(poison_ids=[poison]) + worker = _worker(dao, redis_client=redis_client, max_deliveries=3) + + batch = await worker.read_batch() + _, acked_ids = await worker.process_batch(batch) + await worker.ack_and_delete(acked_ids) + + for _ in range(6): + await asyncio.sleep(0.01) + reclaimed = await worker.reclaim_batch() + if not reclaimed: + break + _, acked_ids = await worker.process_batch(reclaimed) + await worker.ack_and_delete(acked_ids) + + assert dao.committed == [str(good)] + assert worker.dropped_messages == 1 + pending = await redis_client.xpending_range( + name=STREAM, groupname=GROUP, min="-", max="+", count=10 + ) + # The poison entry is gone, so it stops costing a write attempt every window. + assert pending == [] + + dropped = [ + record + for record in caplog.records + if "Dropping messages after repeated delivery failures" in record.getMessage() + ] + assert dropped, "the loss must be logged at error level" + assert dropped[0].levelname == "ERROR" + # The log names the lost record so the loss is traceable after the fact. + assert worker.describe_message(batch[1][1]) == f"doomed-session:{poison}:done" + + +@pytest.mark.asyncio +async def test_nothing_is_dropped_while_the_write_path_is_down(): + """A long outage must not consume the drop budget. + + The delivery counter cannot tell a rejected record apart from an unreachable database, so + dropping on the count alone would delete every record in flight once an outage outlasts + `max_deliveries` windows. That is exactly the loss this worker exists to prevent. + """ + project_id = uuid4() + record_ids = [uuid4(), uuid4()] + redis_client = fakeredis.FakeRedis() + await _seed( + redis_client, + [ + _payload(project_id=project_id, session_id="s", record_id=record_id) + for record_id in record_ids + ], + ) + + dao = FakeRecordsDAO(fail_calls=99) + worker = _worker(dao, redis_client=redis_client, max_deliveries=2) + + batch = await worker.read_batch() + await worker.process_batch(batch) + + for _ in range(6): + await asyncio.sleep(0.01) + reclaimed = await worker.reclaim_batch() + assert len(reclaimed) == 2 + await worker.process_batch(reclaimed) + + assert worker.dropped_messages == 0 + assert await redis_client.xlen(STREAM) == 2 + + # Postgres comes back. Both records land, and neither was deleted meanwhile. + dao.fail_calls = 0 + await asyncio.sleep(0.01) + reclaimed = await worker.reclaim_batch() + _, acked_ids = await worker.process_batch(reclaimed) + await worker.ack_and_delete(acked_ids) + + assert dao.committed == [str(record_id) for record_id in record_ids] + assert await redis_client.xlen(STREAM) == 0 + + +@pytest.mark.asyncio +async def test_recovery_with_new_traffic_keeps_the_over_budget_backlog(): + project_id = uuid4() + old_record, new_record = uuid4(), uuid4() + redis_client = fakeredis.FakeRedis() + await _seed( + redis_client, + [_payload(project_id=project_id, session_id="s", record_id=old_record)], + ) + + dao = FakeRecordsDAO(fail_calls=99) + worker = _worker(dao, redis_client=redis_client, max_deliveries=2) + + old_batch = await worker.read_batch() + await worker.process_batch(old_batch) + for _ in range(3): + await asyncio.sleep(0.01) + reclaimed = await worker.reclaim_batch() + assert [msg_id for msg_id, _ in reclaimed] == [ + msg_id for msg_id, _ in old_batch + ] + await worker.process_batch(reclaimed) + + dao.fail_calls = 0 + await redis_client.xadd( + name=STREAM, + fields={ + "data": _payload( + project_id=project_id, + session_id="s", + record_id=new_record, + ) + }, + ) + new_batch = await worker.read_batch() + _, acked_ids = await worker.process_batch(new_batch) + await worker.ack_and_delete(acked_ids) + + await asyncio.sleep(0.01) + reclaimed = await worker.reclaim_batch() + assert [msg_id for msg_id, _ in reclaimed] == [msg_id for msg_id, _ in old_batch] + _, acked_ids = await worker.process_batch(reclaimed) + await worker.ack_and_delete(acked_ids) + + assert dao.committed == [str(new_record), str(old_record)] + assert worker.dropped_messages == 0 + assert await redis_client.xlen(STREAM) == 0 + + +@pytest.mark.asyncio +async def test_describe_message_survives_an_undecodable_payload(): + assert _worker(FakeRecordsDAO()).describe_message({b"data": b"not-zlib"}) is None + + +def _fake_ee(monkeypatch, *, allowed=True, raises=False): + """Run the EE quota branch of `process_batch` without an EE build.""" + + async def check_entitlements(**_): + if raises: + raise RuntimeError("entitlements unreachable") + return allowed, None, None + + monkeypatch.setattr(records_worker, "is_ee", lambda: True) + monkeypatch.setattr( + records_worker, "check_entitlements", check_entitlements, raising=False + ) + monkeypatch.setattr( + records_worker, + "Counter", + SimpleNamespace(RECORDS_INGESTED="records"), + raising=False, + ) + monkeypatch.setattr( + records_worker, "scope_from", lambda **kwargs: kwargs, raising=False + ) + + +def _org_batch(*, organization_id, project_id, record_id): + message = { + "organization_id": str(organization_id), + "project_id": str(project_id), + "record_event": { + "project_id": str(project_id), + "session_id": "sess-1", + "record_id": str(record_id), + "record_type": "message", + }, + } + return [(b"1-0", {b"data": zlib.compress(dumps(message))})] + + +@pytest.mark.asyncio +async def test_over_quota_org_is_acknowledged_and_counted(monkeypatch): + _fake_ee(monkeypatch, allowed=False) + dao = FakeRecordsDAO() + worker = _worker(dao) + + _, acked_ids = await worker.process_batch( + _org_batch(organization_id=uuid4(), project_id=uuid4(), record_id=uuid4()) + ) + + # Over quota is a deliberate product drop, so redelivering it would spin forever. + assert acked_ids == [b"1-0"] + assert worker.dropped_messages == 1 + assert dao.committed == [] + + +@pytest.mark.asyncio +async def test_unreachable_quota_meter_leaves_the_record_pending(monkeypatch): + _fake_ee(monkeypatch, raises=True) + dao = FakeRecordsDAO() + worker = _worker(dao) + + _, acked_ids = await worker.process_batch( + _org_batch(organization_id=uuid4(), project_id=uuid4(), record_id=uuid4()) + ) + + # The meter was unreachable, not exceeded. Deleting the record would turn an + # entitlements outage into a deleted conversation. + assert acked_ids == [] + assert worker.dropped_messages == 0 + assert dao.committed == [] diff --git a/api/oss/tests/pytest/unit/sessions/test_runner_client_kill.py b/api/oss/tests/pytest/unit/sessions/test_runner_client_kill.py index 1966fdbeff5..2650cee59c4 100644 --- a/api/oss/tests/pytest/unit/sessions/test_runner_client_kill.py +++ b/api/oss/tests/pytest/unit/sessions/test_runner_client_kill.py @@ -9,7 +9,11 @@ import httpx import pytest -from oss.src.core.sessions.streams.runner_client import kill_runner_sandbox +from oss.src.core.sessions.streams.runner_client import ( + RunnerCancelResult, + cancel_runner_execution, + kill_runner_sandbox, +) class _FakeRunnerEnv: @@ -120,3 +124,44 @@ async def post(self, *a, **kw): result = await kill_runner_sandbox(project_id="proj-1", session_id="sess-1") assert result is False + + +@pytest.mark.asyncio +async def test_cancel_accepts_non_object_json_without_crashing(): + class _FakeResponse: + status_code = 200 + + @staticmethod + def json(): + return ["accepted"] + + class _FakeClient: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def post(self, *args, **kwargs): + return _FakeResponse() + + with ( + patch("oss.src.core.sessions.streams.runner_client.env") as mock_env, + patch( + "oss.src.core.sessions.streams.runner_client.httpx.AsyncClient", + return_value=_FakeClient(), + ), + ): + mock_env.runner = _FakeRunnerEnv( + internal_url="http://runner:8765", token="shared-secret" + ) + result = await cancel_runner_execution( + command_id="command-1", + project_id="project-1", + session_id="session-1", + target_turn_id="turn-1", + created_at="2026-09-04T00:00:00Z", + ) + + assert result.status == RunnerCancelResult.accepted + assert result.replica_id is None diff --git a/api/oss/tests/pytest/unit/sessions/test_session_cancel_admission.py b/api/oss/tests/pytest/unit/sessions/test_session_cancel_admission.py new file mode 100644 index 00000000000..35c66fbb1f5 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_session_cancel_admission.py @@ -0,0 +1,1528 @@ +"""What a Stop request decides before anything durable is written. + +Admission is where a Stop can go wrong in the two ways that matter to a user. It can miss the +run they meant, and it can kill a run they never meant. These pin the rules that stop both: + + * the arrival time is stamped BEFORE any read, and stored as the row's `created_at`, so the + value the guard compared is the value the runner can re-compare; + * a stale `expected_execution_id` is refused and writes nothing at all; + * an execution that started AFTER the request arrived is never targeted; + * only a named Stop can reach a parked session, which holds `alive` and not `running`; + * two Stops in a row collapse onto one command; + * Redis is not written at admission, so the stopping execution keeps its locks while it stops. +""" + +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional +from unittest.mock import AsyncMock, patch +from uuid import UUID, uuid4 + +import pytest +import pytest_asyncio +import uuid_utils.compat as uuid + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandOutcome, + SessionCommandState, +) +from oss.src.core.sessions.commands.interfaces import ( + CommandCreateResult, + DeliveryReceipt, +) +from oss.src.core.sessions.commands import service as commands_service_module +from oss.src.core.sessions.commands.service import SessionCommandsService +from oss.src.core.sessions.commands.types import ( + ExecutionExpectationFailed, + SessionCommandIdempotencyConflict, + SessionCommandNotClaimable, +) +from oss.src.core.sessions.executions.dtos import ( + SessionExecutionSettlement, + SessionExecutionSettlementResult, +) +from oss.src.core.sessions.streams.dtos import ( + CommandMode, + SessionStream, + SessionStreamCommandResponse, + SessionStreamFlags, +) +from oss.src.core.sessions.streams.types import SessionTurnMismatch +from oss.src.dbs.redis.sessions.locks import ( + acquire_alive, + acquire_running, + get_alive_owner, + get_running_owner, + get_session_liveness, + is_turn_superseded, + release_running, +) +from oss.src.utils.env import env +from oss.src.tasks.asyncio.sessions.orphan_sweep import _repair_terminal_redis + +from unit.sessions.test_project_scoped_locks import _FakeRedis + + +_PROJECT = uuid4() +_USER = uuid4() +_SESSION = "session_cancel_admission" + + +class _FakeCommandsDAO: + """Enough of the DAO to observe what admission wrote, and how many times.""" + + def __init__(self) -> None: + self.rows: List[SessionCommand] = [] + self.stopping_turn_ids: List[Optional[str]] = [] + self.claims: List[Dict] = [] + self.abandoned: List[SessionCommand] = [] + + @asynccontextmanager + async def transaction(self): + yield object() + + async def create_command( + self, *, user_id, command: SessionCommandCreate, stopping_turn_id=None + ): + row = SessionCommand( + id=uuid.uuid7(), + project_id=command.project_id, + session_id=command.session_id, + kind=command.kind, + target_turn_id=command.target_turn_id, + expected_turn_id=command.expected_turn_id, + data=command.data, + state=command.state, + outcome=command.outcome, + settled_at=command.settled_at, + idempotency_key=command.idempotency_key, + created_at=command.created_at, + ) + self.rows.append(row) + self.stopping_turn_ids.append(stopping_turn_id) + return row + + async def create_command_with_status( + self, *, user_id, command: SessionCommandCreate, stopping_turn_id=None + ): + if command.idempotency_key is not None: + for row in self.rows: + if ( + row.project_id == command.project_id + and row.session_id == command.session_id + and row.idempotency_key == command.idempotency_key + ): + return CommandCreateResult(command=row, inserted=False) + row = await self.create_command( + user_id=user_id, + command=command, + stopping_turn_id=stopping_turn_id, + ) + return CommandCreateResult(command=row, inserted=True) + + async def fetch_by_idempotency_key( + self, *, project_id, session_id, idempotency_key + ): + for row in self.rows: + if ( + row.project_id == project_id + and row.session_id == session_id + and row.idempotency_key == idempotency_key + ): + return row + return None + + async def fetch_open_command(self, *, project_id, session_id, kind, target_turn_id): + for row in reversed(self.rows): + if ( + row.project_id == project_id + and row.session_id == session_id + and row.kind == kind + and row.target_turn_id == target_turn_id + and row.state + in (SessionCommandState.pending, SessionCommandState.claimed) + ): + return row + return None + + async def fetch_command(self, *, command_id, project_id=None): + for row in self.rows: + if row.id == command_id: + return row + return None + + async def claim_for_delivery( + self, *, project_id, command_id, replica_id, lease_seconds + ): + # A copy, never a mutation of the object the caller holds — the real DAO returns a + # fresh row from RETURNING *, so admission's own view of the command stays as it was. + self.claims.append({"command_id": command_id, "replica_id": replica_id}) + for index, row in enumerate(self.rows): + if row.id == command_id and row.state == SessionCommandState.pending: + claimed = row.model_copy( + update={ + "state": SessionCommandState.claimed, + "claimed_by": replica_id, + } + ) + self.rows[index] = claimed + return claimed + return None + + async def record_delivery_attempt( + self, *, project_id, command_id, now, max_deliveries + ): + for index, row in enumerate(self.rows): + if ( + row.id == command_id + and row.state + in (SessionCommandState.pending, SessionCommandState.claimed) + and row.claim_count < max_deliveries + ): + attempted = row.model_copy( + update={ + "state": SessionCommandState.pending, + "claimed_by": None, + "claim_expires_at": None, + "claim_count": row.claim_count + 1, + "updated_at": now, + } + ) + self.rows[index] = attempted + return attempted + return None + + async def claim_commands(self, **_): + return [] + + async def settle_command(self, *, settle, transaction=None): + for index, row in enumerate(self.rows): + if row.id == settle.command_id and row.state in settle.expected_states: + # Mirrors the real guard: a `pending` row holds no claim, so a null + # `claimed_by` passes; a claimed row must be claimed by the reporter. + if ( + settle.replica_id is not None + and row.claimed_by is not None + and row.claimed_by != settle.replica_id + ): + return None + settled = row.model_copy( + update={ + "state": settle.state, + "outcome": settle.outcome, + "settled_at": datetime.now(timezone.utc), + } + ) + self.rows[index] = settled + return settled + return None + + async def clear_stopping_turn(self, *, project_id, session_id, turn_id=None): + self.stopping_turn_ids.append(None) + + async def expire_claims(self, *, now, max_deliveries, pending_before=None): + return self.abandoned + + +class _FakeStreamsService: + """The reads admission makes, plus the row settlement writes. + + `mirrored` stands in for the `session_streams` row. It records the nest exactly as the real + `_mirror_flags` would read it — from Redis, at the moment settlement calls — so a test can + assert what the ROW says and not merely that a call happened. `query_streams`, which is what + the product's liveness polls read, serves that row and never looks at Redis. + """ + + def __init__( + self, stream: Optional[SessionStream] = None, lock_engine=None + ) -> None: + self.stream = stream + self.ended: List[str] = [] + self.lock_engine = lock_engine + self.mirrored: List[Dict[str, bool]] = [] + + async def fetch_header(self, *, project_id: UUID, session_id: str): + return self.stream + + async def command(self, *, project_id, user_id, request): + actual = self.stream.turn_id if self.stream is not None else None + if ( + request.expected_execution_id is not None + and actual != request.expected_execution_id + ): + raise SessionTurnMismatch( + request.session_id, + expected_turn_id=request.expected_execution_id, + actual_turn_id=actual, + ) + return SessionStreamCommandResponse( + mode=CommandMode.cancel, + session_id=request.session_id, + turn_id=actual, + detached=True, + ) + + async def publish_session_ended(self, *, project_id: UUID, session_id: str): + self.ended.append(session_id) + + async def mirror_liveness(self, *, project_id: UUID, session_id: str, user_id=None): + snap = await get_session_liveness( + self.lock_engine, project_id=str(project_id), session_id=session_id + ) + self.mirrored.append( + { + "is_alive": snap["alive"], + "is_running": snap["running"], + "is_attached": snap["attached"], + } + ) + + async def settle_command( + self, + *, + project_id, + session_id, + turn_id, + mirror_stopped, + transaction=None, + ): + if mirror_stopped and self.stream is not None: + self.stream = self.stream.model_copy( + update={ + "flags": self.stream.flags.model_copy(update={"is_running": False}) + } + ) + + +class _FakeInteractionsService: + def __init__(self, *, cancelled_count: int = 1) -> None: + self.cancelled: List[Optional[str]] = [] + self.command_ids: List[Optional[UUID]] = [] + self.published_cancelled: List[str] = [] + self.cancelled_count = cancelled_count + + async def cancel_session_pending( + self, + *, + project_id, + session_id, + only_turn_id=None, + command_id=None, + **_, + ): + self.cancelled.append(only_turn_id) + self.command_ids.append(command_id) + return self.cancelled_count + + async def publish_session_pending_cancelled( + self, *, project_id, session_id + ) -> None: + self.published_cancelled.append(session_id) + + +class _RecordingDelivery: + def __init__(self, status: str = "accepted") -> None: + self.status = status + self.delivered: List[SessionCommand] = [] + + async def deliver(self, *, command: SessionCommand) -> DeliveryReceipt: + self.delivered.append(command) + return DeliveryReceipt(status=self.status, replica_id="runner-1") + + async def acknowledge(self, *, command_id, replica_id) -> None: + return None + + +class _FakeExecutionsDAO: + def __init__(self) -> None: + self.rows: Dict[tuple[str, str], SessionExecutionSettlement] = {} + self.commands = None + self.interactions = None + + async def settle( + self, + *, + project_id, + session_id, + execution_id, + terminal_outcome, + settled_by, + settled_at=None, + transaction=None, + ): + key = (session_id, execution_id) + if key in self.rows: + return SessionExecutionSettlementResult( + settlement=self.rows[key], won=False + ) + row = SessionExecutionSettlement( + project_id=project_id, + session_id=session_id, + execution_id=execution_id, + terminal_outcome=terminal_outcome, + settled_by=settled_by, + settled_at=settled_at or datetime.now(timezone.utc), + ) + self.rows[key] = row + return SessionExecutionSettlementResult(settlement=row, won=True) + + async def list_redis_unreconciled(self, *, limit): + return [ + row + for row in self.rows.values() + if row.settled_by == "runner" + and row.terminal_outcome == "stopped" + and row.redis_reconciled_at is None + ][:limit] + + async def mark_redis_reconciled(self, *, project_id, session_id, execution_id): + key = (session_id, execution_id) + self.rows[key] = self.rows[key].model_copy( + update={"redis_reconciled_at": datetime.now(timezone.utc)} + ) + + +def _stream( + turn_id: Optional[str], turn_started_at: Optional[datetime] +) -> SessionStream: + return SessionStream( + id=uuid4(), + project_id=_PROJECT, + session_id=_SESSION, + turn_id=turn_id, + turn_started_at=turn_started_at, + flags=SessionStreamFlags(is_alive=True, is_running=True), + updated_at=datetime.now(timezone.utc), + ) + + +@pytest_asyncio.fixture +async def lock_engine(): + from oss.src.dbs.redis.shared.engine import LockEngine + + eng = LockEngine() + with patch.object(eng, "_client", return_value=_FakeRedis()): + yield eng + + +def _service( + lock_engine, + *, + dao=None, + streams=None, + interactions=None, + delivery=None, + executions=None, +): + streams = streams or _FakeStreamsService() + # The fake mirrors from Redis, so it reads the same engine the service writes through. + if streams.lock_engine is None: + streams.lock_engine = lock_engine + commands = dao or _FakeCommandsDAO() + interactions = interactions or _FakeInteractionsService() + if executions is not None: + executions.commands = commands + executions.interactions = interactions + return SessionCommandsService( + commands_dao=commands, + streams_service=streams, + interactions_service=interactions, + lock_engine=lock_engine, + delivery=delivery or _RecordingDelivery(), + executions_dao=executions, + ) + + +async def _run_turn(lock_engine, turn_id: str) -> None: + await acquire_alive( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id=turn_id + ) + await acquire_running( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id=turn_id + ) + + +@pytest.mark.asyncio +async def test_stop_on_a_running_turn_is_accepted_and_pins_the_target(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + started = datetime.now(timezone.utc) - timedelta(seconds=30) + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService(_stream("turn-A", started)), + delivery=delivery, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is True + assert admission.execution_id == "turn-A" + assert admission.command.state == SessionCommandState.pending + assert admission.command.target_turn_id == "turn-A" + # The row and the session marker are written together. + assert dao.stopping_turn_ids == ["turn-A"] + assert len(delivery.delivered) == 1 + + +@pytest.mark.asyncio +async def test_admission_does_not_touch_redis(lock_engine): + await _run_turn(lock_engine, "turn-A") + svc = _service( + lock_engine, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + ) + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + + # The stopping execution keeps both locks WHILE it stops, which is what prevents a second + # message from starting underneath it. + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-A" + ) + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-A" + ) + + +@pytest.mark.asyncio +async def test_stop_when_nothing_runs_is_settled_at_once(lock_engine): + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service(lock_engine, dao=dao, delivery=delivery) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is False + assert admission.execution_id is None + assert admission.command.state == SessionCommandState.obsolete + assert admission.command.outcome == SessionCommandOutcome.not_running + assert delivery.delivered == [], "nothing to deliver to" + assert dao.stopping_turn_ids == [None], "no session is stopping" + + +@pytest.mark.asyncio +async def test_stale_expected_execution_id_is_refused_and_writes_nothing(lock_engine): + await _run_turn(lock_engine, "turn-B") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-B", datetime.now(timezone.utc) - timedelta(seconds=5)) + ), + delivery=delivery, + ) + + with pytest.raises(ExecutionExpectationFailed) as excinfo: + await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + ) + + assert excinfo.value.current == "turn-B" + assert dao.rows == [], "a refused Stop must insert nothing" + assert delivery.delivered == [] + + +@pytest.mark.asyncio +async def test_legacy_cancel_keeps_the_expected_execution_guard(lock_engine): + await _run_turn(lock_engine, "turn-B") + svc = _service( + lock_engine, + streams=_FakeStreamsService( + _stream("turn-B", datetime.now(timezone.utc) - timedelta(seconds=5)) + ), + ) + + with pytest.raises(ExecutionExpectationFailed) as excinfo: + await svc.request_cancel_legacy( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + ) + + assert excinfo.value.current == "turn-B" + assert ( + await get_running_owner( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + ) + == "turn-B" + ) + + +@pytest.mark.asyncio +async def test_a_turn_that_started_after_the_request_is_never_targeted(lock_engine): + # The race: the user presses Stop, turn one ends, turn two starts, and only then does the + # request get applied. Turn two must not hear about it. + await _run_turn(lock_engine, "turn-two") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-two", datetime.now(timezone.utc) + timedelta(seconds=5)) + ), + delivery=delivery, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is False + assert admission.execution_id is None + assert admission.command.target_turn_id is None + assert admission.command.outcome == SessionCommandOutcome.superseded_by_newer_turn + assert delivery.delivered == [], "the newer turn is never contacted" + # And its locks are untouched. + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-two" + ) + + +@pytest.mark.asyncio +async def test_the_guard_does_not_fire_when_the_start_time_is_unknown(lock_engine): + # A row written before `turn_started_at` existed yields no comparison. Failing this way + # round is deliberate: refusing every Stop we cannot verify would break the common case. + await _run_turn(lock_engine, "turn-A") + svc = _service(lock_engine, streams=_FakeStreamsService(_stream("turn-A", None))) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is True + assert admission.command.target_turn_id == "turn-A" + + +@pytest.mark.asyncio +async def test_the_stored_created_at_is_the_value_that_was_compared(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + before = datetime.now(timezone.utc) + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + ) + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + after = datetime.now(timezone.utc) + + stored = dao.rows[0].created_at + assert stored is not None + # Stamped by the service, not defaulted by the server: the runner repeats this comparison. + assert before <= stored <= after + + +@pytest.mark.asyncio +async def test_unfenced_stop_before_new_turn_admission_ignores_the_parked_owner( + lock_engine, +): + # Turn B was submitted by the browser but has not established `running` yet. + await acquire_alive( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-parked", + ) + delivery = _RecordingDelivery() + dao = _FakeCommandsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService(_stream("turn-parked", None)), + delivery=delivery, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is False + assert admission.execution_id is None + assert admission.command.outcome == SessionCommandOutcome.not_running + assert delivery.delivered == [] + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-parked" + ) + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + is None + ) + assert not await is_turn_superseded( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-parked", + ) + + +@pytest.mark.asyncio +async def test_a_named_stop_reaches_a_parked_approval(lock_engine): + """The Stop the browser actually sends, on the session state Stop exists to reach. + + A parked approval has released `running` and still holds `alive` under the same turn id. + The browser always sends `expected_execution_id`, because it knows the id it streamed. If + the expectation is compared against `running` alone it is None here, so the named Stop is + refused with a conflict while the identical Stop without an expectation is accepted — the + guard firing on the one case it exists to allow, and the gate left pending. + """ + await acquire_alive( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-parked", + ) + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + streams=_FakeStreamsService(_stream("turn-parked", None)), + delivery=delivery, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-parked", + ) + + assert admission.accepted is True + assert admission.execution_id == "turn-parked" + assert len(delivery.delivered) == 1 + + +@pytest.mark.asyncio +async def test_a_named_stop_on_a_parked_session_still_refuses_a_different_turn( + lock_engine, +): + """The guard must keep working on the fallback, not merely stop firing. + + A user looking at a turn that finished, on a session now parked under a NEWER turn, must + still be refused: the id they named is not the one that would be stopped. + """ + await acquire_alive( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-new", + ) + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService(_stream("turn-new", None)), + delivery=delivery, + ) + + with pytest.raises(ExecutionExpectationFailed) as excinfo: + await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-old", + ) + + assert excinfo.value.current == "turn-new" + assert dao.rows == [] + assert delivery.delivered == [] + + +@pytest.mark.asyncio +async def test_two_stops_in_a_row_collapse_onto_one_command(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + ) + + first = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + second = await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + idempotency_key="a-different-key", + ) + + assert len(dao.rows) == 1, "one intent, one command" + assert second.command.id == first.command.id + assert second.accepted is True + + +@pytest.mark.asyncio +async def test_reused_idempotency_key_replays_the_original_turn_without_redelivery( + lock_engine, +): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + svc = _service( + lock_engine, + dao=dao, + streams=streams, + delivery=delivery, + ) + + first = await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + idempotency_key="same-request", + ) + dao.rows[0] = dao.rows[0].model_copy( + update={ + "state": SessionCommandState.applied, + "outcome": SessionCommandOutcome.stopped, + } + ) + await release_running( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-A", + ) + await acquire_running( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-B", + ) + streams.stream = _stream( + "turn-B", datetime.now(timezone.utc) - timedelta(seconds=5) + ) + + replay = await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + idempotency_key="same-request", + ) + + assert replay.command.id == first.command.id + assert replay.command.state == SessionCommandState.applied + assert replay.execution_id == "turn-A" + assert replay.accepted is True + assert len(delivery.delivered) == 1, "an idempotent replay must not target turn-B" + + +@pytest.mark.asyncio +async def test_reused_idempotency_key_rejects_a_different_expected_execution( + lock_engine, +): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + delivery=delivery, + ) + await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + idempotency_key="same-key-different-request", + ) + + with pytest.raises(SessionCommandIdempotencyConflict): + await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-B", + idempotency_key="same-key-different-request", + ) + + assert len(dao.rows) == 1 + assert len(delivery.delivered) == 1 + + +@pytest.mark.asyncio +async def test_a_reachable_runner_that_does_not_hold_the_session_settles_at_once( + lock_engine, +): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + # A row that has not beaten for a long time: the session really did end, so `not_running` + # is the honest answer rather than the wrong-replica failure. + streams.stream.updated_at = datetime.now(timezone.utc) - timedelta(minutes=30) + interactions = _FakeInteractionsService() + svc = _service( + lock_engine, + dao=dao, + streams=streams, + interactions=interactions, + delivery=_RecordingDelivery(status="not_held"), + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is True, "the caller still gets a durable command" + assert dao.rows[0].state == SessionCommandState.obsolete + assert dao.rows[0].outcome == SessionCommandOutcome.not_running + assert interactions.cancelled == ["turn-A"] + assert streams.ended == [_SESSION] + + +@pytest.mark.asyncio +async def test_not_held_on_a_beating_session_is_reported_as_lost_not_finished( + lock_engine, +): + # The wrong-replica failure. The user must be told the Stop failed, never that the work had + # already finished. `_run_turn` holds `running`, which is the discriminator: an execution is + # being run somewhere, and it is not by the process we called. + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + svc = _service( + lock_engine, + dao=dao, + streams=streams, + delivery=_RecordingDelivery(status="not_held"), + ) + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + + assert dao.rows[0].outcome == SessionCommandOutcome.lost + + +@pytest.mark.asyncio +async def test_not_held_on_a_turn_that_just_ended_is_not_running_not_lost(lock_engine): + """The everyday late Stop: the answer landed, the user pressed Stop a moment after. + + The turn released `running` and left `alive` and a fresh heartbeat behind it, exactly as a + RUNNING turn would, so a beating-row test calls this a failed Stop and tells the user their + Stop was lost. Nothing was lost: the work finished. `running` is what separates the two, + because a session nobody is executing has no `running` owner at all. + """ + # `alive` only, which is what a turn leaves when it ends. + await acquire_alive( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-A" + ) + dao = _FakeCommandsDAO() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + # Beating, and recently: the turn ended seconds ago, not half an hour ago. + streams.stream.updated_at = datetime.now(timezone.utc) + svc = _service( + lock_engine, + dao=dao, + streams=streams, + delivery=_RecordingDelivery(status="not_held"), + ) + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + + assert dao.rows[0].state == SessionCommandState.obsolete + assert dao.rows[0].outcome == SessionCommandOutcome.not_running + + +@pytest.mark.asyncio +async def test_an_unreachable_runner_leaves_the_command_open(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + delivery=_RecordingDelivery(status="unreachable"), + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + # Admission still succeeded. The command is durable, so a later delivery or the settlement + # sweep gives the user a terminal state instead of a Stop that vanished. + assert admission.accepted is True + assert dao.rows[0].state == SessionCommandState.pending + + +@pytest.mark.asyncio +async def test_settlement_releases_running_and_leaves_alive_alone(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + interactions = _FakeInteractionsService() + svc = _service(lock_engine, dao=dao, streams=streams, interactions=interactions) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert dao.rows[0].state == SessionCommandState.applied + assert dao.rows[0].outcome == SessionCommandOutcome.stopped + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + is None + ), "running is released under an owner check" + # THE assertion that pins warm resume. Force-deleting `alive` is what makes today's cancel + # read as a session teardown; Stop must leave the session as a finished turn leaves it. + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-A" + ) + assert interactions.cancelled == ["turn-A"] + assert interactions.command_ids == [admission.command.id] + assert streams.ended == [_SESSION] + + +@pytest.mark.asyncio +async def test_settlement_writes_the_row_as_alive_and_not_running(lock_engine): + """The ROW, not only Redis — the row is the only thing the product's liveness polls read. + + Redis is already right the moment settlement returns, and the test above pins that. The row + is a separate write, and nothing else performs it: settlement tombstones the execution first, + so the runner's own final `is_running=false` heartbeat is refused before it reaches the + heartbeat's mirror write. Left unwritten, the row says `is_running: true` until the orphan + sweep collapses it minutes later, and the tab that pressed Stop shows its own session as + running somewhere else for that whole time. + """ + await _run_turn(lock_engine, "turn-A") + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + svc = _service(lock_engine, streams=streams) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + # Written once, and written AFTER `running` was released — a mirror taken before the release + # would have recorded `is_running: True` and been exactly the bug. + assert streams.mirrored == [ + {"is_alive": True, "is_running": False, "is_attached": False} + ] + # And the mirror is the state a normally finished turn leaves behind, which is what makes + # the session read as resumable rather than as torn down. + assert streams.mirrored[-1]["is_alive"] is True + + +@pytest.mark.asyncio +async def test_a_settlement_that_stops_nothing_does_not_touch_the_row(lock_engine): + """`not_running` changes no lock, so it must not write the row either. + + An obsolete Stop lands here: the turn it named had already finished, a NEWER turn may hold + the nest, and a mirror write from this path would be a write the settlement has no business + making. The row is left to the live turn's own heartbeats. + """ + dao = _FakeCommandsDAO() + streams = _FakeStreamsService(None) + svc = _service(lock_engine, dao=dao, streams=streams) + + # Nothing running and nothing parked: admission settles the command at insert. + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is False + assert dao.rows[0].outcome == SessionCommandOutcome.not_running + assert streams.mirrored == [] + + +@pytest.mark.asyncio +async def test_an_outcome_that_beats_the_claim_still_settles(lock_engine): + """The race the runner wins on a fast abort, driven at the exact instant it happens. + + Admission inserts the command `pending`, hands it to the runner, and writes `claimed` only + after the runner answers. A runner that aborts inside that window reports its outcome while + the row still says `pending`. Guarded on `claimed` alone that report was refused with a + conflict, the command sat open, and the sweep later recorded a Stop that actually worked as + lost — with the user watching "stopping" for the whole sweep window. + + The delivery double below reports from inside `deliver`, which is precisely where the real + runner's report lands relative to the claim. + """ + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + holder: Dict[str, SessionCommandsService] = {} + + class _ReportsBeforeTheClaimCommits: + def __init__(self) -> None: + self.delivered: List[SessionCommand] = [] + self.state_at_report: Optional[SessionCommandState] = None + + async def deliver(self, *, command): + self.delivered.append(command) + # The window. Nothing has written `claimed` yet, and the runner is already done. + self.state_at_report = dao.rows[0].state + await holder["svc"].report_outcome( + command_id=command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + return DeliveryReceipt(status="accepted", replica_id="runner-1") + + async def acknowledge(self, *, command_id, replica_id): + return None + + delivery = _ReportsBeforeTheClaimCommits() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=5)) + ), + delivery=delivery, + ) + holder["svc"] = svc + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + + assert delivery.state_at_report == SessionCommandState.pending, ( + "the test is only meaningful if the report really did beat the claim" + ) + assert dao.rows[0].state == SessionCommandState.applied + assert dao.rows[0].outcome == SessionCommandOutcome.stopped + # And the claim that arrives afterwards must not resurrect a settled command. + assert dao.rows[0].state == SessionCommandState.applied + + +@pytest.mark.asyncio +async def test_an_outcome_from_a_replica_that_does_not_hold_the_claim_is_refused( + lock_engine, +): + """Widening the guard to `pending` must not weaken it for a row that IS claimed. + + A claimed row names its holder, and only that holder may write the outcome. The null + `claimed_by` this change now admits exists solely for the unclaimed row. + """ + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=5)) + ), + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + assert dao.rows[0].state == SessionCommandState.claimed + + from oss.src.core.sessions.commands.types import SessionCommandNotClaimable + + with pytest.raises(SessionCommandNotClaimable): + await svc.report_outcome( + command_id=admission.command.id, + replica_id="a-different-replica", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert dao.rows[0].state == SessionCommandState.claimed + + +@pytest.mark.asyncio +async def test_a_second_outcome_report_changes_nothing(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + interactions = _FakeInteractionsService() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + interactions=interactions, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + from oss.src.core.sessions.commands.types import SessionCommandNotClaimable + + with pytest.raises(SessionCommandNotClaimable): + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert interactions.cancelled == ["turn-A"], "the side effects run exactly once" + + +@pytest.mark.asyncio +async def test_runner_outcome_settles_the_execution_authority(lock_engine): + await _run_turn(lock_engine, "turn-A") + executions = _FakeExecutionsDAO() + interactions = _FakeInteractionsService() + svc = _service( + lock_engine, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + interactions=interactions, + executions=executions, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + winner = executions.rows[(_SESSION, "turn-A")] + assert winner.terminal_outcome == "stopped" + assert winner.settled_by == "runner" + assert interactions.published_cancelled == [_SESSION] + + +@pytest.mark.asyncio +async def test_atomic_settlement_does_not_publish_when_no_gate_was_cancelled( + lock_engine, +): + await _run_turn(lock_engine, "turn-A") + interactions = _FakeInteractionsService(cancelled_count=0) + svc = _service( + lock_engine, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + interactions=interactions, + executions=_FakeExecutionsDAO(), + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert interactions.cancelled == ["turn-A"] + assert interactions.published_cancelled == [] + + +@pytest.mark.asyncio +async def test_watchdog_cannot_replace_the_runners_terminal_outcome(lock_engine): + executions = _FakeExecutionsDAO() + svc = _service(lock_engine, executions=executions) + first = await executions.settle( + project_id=_PROJECT, + session_id=_SESSION, + execution_id="turn-A", + terminal_outcome="stopped", + settled_by="runner", + ) + assert first.won is True + + won = await svc.settle_execution_lost( + project_id=_PROJECT, + session_id=_SESSION, + execution_id="turn-A", + settled_at=datetime.now(timezone.utc), + ) + + assert won is False + assert executions.rows[(_SESSION, "turn-A")].terminal_outcome == "stopped" + + +@pytest.mark.asyncio +async def test_next_sweep_repairs_a_post_commit_redis_failure(lock_engine, monkeypatch): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + executions = _FakeExecutionsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + executions=executions, + ) + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + reconcile = AsyncMock(side_effect=RuntimeError("injected after commit")) + monkeypatch.setattr(commands_service_module, "reconcile_stopped_turn", reconcile) + + with pytest.raises(RuntimeError, match="injected after commit"): + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert dao.rows[0].state == SessionCommandState.applied + assert executions.rows[(_SESSION, "turn-A")].redis_reconciled_at is None + + reconcile.side_effect = None + repaired = await _repair_terminal_redis(svc) + + assert repaired == 1 + assert executions.rows[(_SESSION, "turn-A")].redis_reconciled_at is not None + + +@pytest.mark.asyncio +async def test_successful_redis_projection_is_not_offered_for_repair(lock_engine): + await _run_turn(lock_engine, "turn-A") + executions = _FakeExecutionsDAO() + svc = _service( + lock_engine, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + executions=executions, + ) + admission = await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + ) + + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert executions.rows[(_SESSION, "turn-A")].redis_reconciled_at is not None + assert await svc.repair_terminal_redis() == 0 + + +def _abandoned_command(*, claim_count: int = 1) -> SessionCommand: + return SessionCommand( + id=uuid.uuid7(), + project_id=_PROJECT, + session_id=_SESSION, + kind="cancel", + target_turn_id="turn-A", + state=SessionCommandState.pending, + claim_count=claim_count, + created_at=datetime.now(timezone.utc) - timedelta(minutes=5), + ) + + +@pytest.mark.asyncio +async def test_a_pending_command_is_redelivered_while_the_session_beats(lock_engine): + command = _abandoned_command() + dao = _FakeCommandsDAO() + dao.rows = [command] + dao.abandoned = [command] + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService(_stream("turn-A", datetime.now(timezone.utc))), + delivery=delivery, + ) + + settled = await svc.settle_abandoned_commands(now=datetime.now(timezone.utc)) + + assert settled == 0 + assert [row.id for row in delivery.delivered] == [command.id] + assert dao.rows[0].claim_count == command.claim_count + 1 + + +@pytest.mark.asyncio +async def test_a_pending_command_is_settled_lost_when_the_runner_is_gone(lock_engine): + command = _abandoned_command() + dao = _FakeCommandsDAO() + dao.rows = [command] + dao.abandoned = [command] + delivery = _RecordingDelivery() + executions = _FakeExecutionsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream( + "turn-A", + datetime.now(timezone.utc) - timedelta(minutes=5), + ).model_copy( + update={"updated_at": datetime.now(timezone.utc) - timedelta(minutes=5)} + ) + ), + delivery=delivery, + executions=executions, + ) + + settled = await svc.settle_abandoned_commands(now=datetime.now(timezone.utc)) + + assert settled == 1 + assert delivery.delivered == [] + assert dao.rows[0].state == SessionCommandState.obsolete + assert dao.rows[0].outcome == SessionCommandOutcome.lost + winner = executions.rows[(_SESSION, "turn-A")] + assert winner.terminal_outcome == "lost" + assert winner.settled_by == "watchdog" + + with pytest.raises(SessionCommandNotClaimable): + await svc.report_outcome( + command_id=command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert dao.rows[0].state == SessionCommandState.obsolete + assert dao.rows[0].outcome == SessionCommandOutcome.lost + assert executions.rows[(_SESSION, "turn-A")] == winner + + +@pytest.mark.asyncio +async def test_redelivery_stops_at_the_configured_maximum(lock_engine, monkeypatch): + maximum = 2 + monkeypatch.setattr(env.agenta.sessions.commands, "max_deliveries", maximum) + command = _abandoned_command(claim_count=maximum) + dao = _FakeCommandsDAO() + dao.rows = [command] + dao.abandoned = [command] + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService(_stream("turn-A", datetime.now(timezone.utc))), + delivery=delivery, + ) + + settled = await svc.settle_abandoned_commands(now=datetime.now(timezone.utc)) + + assert settled == 1 + assert delivery.delivered == [] + assert dao.rows[0].outcome == SessionCommandOutcome.lost + + +@pytest.mark.asyncio +async def test_a_superseded_report_leaves_the_newer_turns_locks_alone(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + interactions = _FakeInteractionsService() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + interactions=interactions, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="obsolete", + execution_id="turn-A", + execution_state="superseded_by_newer_turn", + ) + + assert dao.rows[0].outcome == SessionCommandOutcome.superseded_by_newer_turn + # Nothing was stopped, so nothing is released and no gate is cancelled. + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-A" + ) + assert interactions.cancelled == [] diff --git a/api/oss/tests/pytest/unit/sessions/test_session_cancel_feature_flag.py b/api/oss/tests/pytest/unit/sessions/test_session_cancel_feature_flag.py new file mode 100644 index 00000000000..163794b204a --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_session_cancel_feature_flag.py @@ -0,0 +1,136 @@ +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest +from fastapi import HTTPException + +from oss.src.apis.fastapi.sessions import router as router_module +from oss.src.apis.fastapi.sessions.models import SessionCancelRequest +from oss.src.apis.fastapi.sessions.router import SessionControlRouter +from oss.src.core.sessions.commands.dtos import SessionCommandState +from oss.src.core.sessions.streams.dtos import CommandMode, SessionStreamCommandResponse +from oss.src.utils.env import env +from oss.src.utils.env import _parse_sessions_late_output +from oss.src.utils.env import _parse_sessions_watchdog_stale_heartbeat_seconds + + +_PROJECT = UUID("00000000-0000-0000-0000-0000000000aa") +_USER = UUID("00000000-0000-0000-0000-0000000000bb") + + +def test_unknown_late_output_policy_falls_back_to_quarantine(monkeypatch): + monkeypatch.setenv("AGENTA_SESSIONS_LATE_OUTPUT", "typo") + + with pytest.warns(UserWarning, match="behaving as 'quarantine'"): + value = _parse_sessions_late_output() + + assert value == "quarantine" + + +@pytest.mark.parametrize( + ("durable_stop", "expected"), + [(None, 90), ("", 90), ("false", 300), ("true", 90)], +) +def test_watchdog_default_respects_durable_stop_setting( + monkeypatch, durable_stop, expected +): + if durable_stop is None: + monkeypatch.delenv("AGENTA_SESSIONS_DURABLE_STOP", raising=False) + else: + monkeypatch.setenv("AGENTA_SESSIONS_DURABLE_STOP", durable_stop) + monkeypatch.delenv( + "AGENTA_SESSIONS_WATCHDOG_STALE_HEARTBEAT_SECONDS", raising=False + ) + + assert _parse_sessions_watchdog_stale_heartbeat_seconds() == expected + + +def _request(): + return SimpleNamespace( + state=SimpleNamespace(project_id=_PROJECT, user_id=_USER), + headers={}, + ) + + +async def test_cancel_route_uses_legacy_path_when_durable_stop_is_off(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", False) + monkeypatch.setattr( + router_module, "check_action_access", AsyncMock(return_value=True) + ) + service = SimpleNamespace( + request_cancel_legacy=AsyncMock( + return_value=SessionStreamCommandResponse( + mode=CommandMode.cancel, + session_id="session-1", + turn_id="turn-1", + detached=True, + ) + ), + request_cancel=AsyncMock(), + ) + + response = await SessionControlRouter( + commands_service=service + ).cancel_session_execution( + _request(), + "session-1", + SessionCancelRequest(expected_execution_id="turn-1"), + ) + + service.request_cancel_legacy.assert_awaited_once_with( + project_id=_PROJECT, + user_id=_USER, + session_id="session-1", + expected_execution_id="turn-1", + ) + service.request_cancel.assert_not_awaited() + assert response.status_code == 200 + assert json.loads(response.body) == { + "mode": "cancel", + "session_id": "session-1", + "turn_id": "turn-1", + "watcher_id": None, + "detached": True, + "cancelled_turn_ids": [], + } + + +async def test_cancel_route_uses_durable_path_when_flag_is_on(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + monkeypatch.setattr( + router_module, "check_action_access", AsyncMock(return_value=True) + ) + command = SimpleNamespace( + id=UUID("00000000-0000-0000-0000-0000000000cc"), + state=SessionCommandState.pending, + ) + service = SimpleNamespace( + request_cancel_legacy=AsyncMock(), + request_cancel=AsyncMock( + return_value=SimpleNamespace( + command=command, + execution_id="turn-1", + accepted=True, + ) + ), + ) + + response = await SessionControlRouter( + commands_service=service + ).cancel_session_execution(_request(), "session-1") + + service.request_cancel.assert_awaited_once() + service.request_cancel_legacy.assert_not_awaited() + assert response.status_code == 202 + + +def test_runner_token_rejects_non_ascii_credentials_as_unauthorized(monkeypatch): + monkeypatch.setattr(env.runner, "token", "shared-secret") + request = SimpleNamespace(headers={"X-Agenta-Runner-Token": "nøt-the-token"}) + + with pytest.raises(HTTPException) as exc_info: + router_module._assert_runner_token(request) + + assert exc_info.value.status_code == 401 diff --git a/api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py b/api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py new file mode 100644 index 00000000000..acca6be0006 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py @@ -0,0 +1,861 @@ +"""The compare-and-set rules that make a command safe under concurrency. + +These run against a real Postgres, because what is being tested IS the database's behaviour: +a unique constraint, a partial index's predicate, `FOR UPDATE SKIP LOCKED`, and an `UPDATE ... +WHERE RETURNING *` that must be won by exactly one caller. + +The rule every one of them protects: one execution reaches exactly one terminal outcome, +written by exactly one writer. +""" + +import asyncio +import uuid +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import text + +from oss.src.core.sessions.commands.dtos import ( + SessionCommandCreate, + SessionCommandKind, + SessionCommandOutcome, + SessionCommandSettle, + SessionCommandState, +) +from oss.src.core.sessions.commands.interfaces import SessionScope +from oss.src.core.sessions.commands.service import SessionCommandsService +from oss.src.core.sessions.interactions.service import SessionInteractionsService +from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.dbs.postgres.sessions.commands.dao import SessionCommandsDAO +from oss.src.dbs.postgres.sessions.executions.dao import SessionExecutionsDAO +from oss.src.dbs.postgres.sessions.interactions.dao import SessionInteractionsDAO +from oss.src.dbs.postgres.sessions.streams.dao import SessionStreamsDAO +import oss.src.dbs.postgres.shared.engine as engine_module +from oss.src.dbs.postgres.shared.engine import get_transactions_engine +import oss.src.models.db_models # noqa: F401 + + +pytestmark = pytest.mark.integration + + +@pytest.fixture(autouse=True) +async def _fresh_engine_per_test(): + if engine_module._transactions_engine is not None: + await engine_module._transactions_engine.close() + engine_module._transactions_engine = None + yield + if engine_module._transactions_engine is not None: + await engine_module._transactions_engine.close() + engine_module._transactions_engine = None + + +@pytest.fixture +async def command_scope(): + engine = get_transactions_engine() + user_id = uuid.uuid4() + organization_id = uuid.uuid4() + workspace_id = uuid.uuid4() + project_id = uuid.uuid4() + session_id = f"cmd-dao-{project_id.hex[:12]}" + + async with engine.session() as session: + await session.execute( + text( + "INSERT INTO users (id, uid, username, email) " + "VALUES (:id, :uid, :username, :email)" + ), + { + "id": user_id, + "uid": str(user_id), + "username": "command-dao-test", + "email": f"command-dao-{user_id.hex[:8]}@example.com", + }, + ) + await session.execute( + text( + "INSERT INTO organizations (id, name, owner_id) " + "VALUES (:id, :name, :owner_id)" + ), + { + "id": organization_id, + "name": "command-dao-test-org", + "owner_id": user_id, + }, + ) + await session.execute( + text( + "INSERT INTO workspaces (id, name, organization_id) " + "VALUES (:id, :name, :organization_id)" + ), + { + "id": workspace_id, + "name": "command-dao-test-workspace", + "organization_id": organization_id, + }, + ) + await session.execute( + text( + "INSERT INTO projects " + "(id, project_name, workspace_id, organization_id) " + "VALUES (:id, :project_name, :workspace_id, :organization_id)" + ), + { + "id": project_id, + "project_name": "command-dao-test-project", + "workspace_id": workspace_id, + "organization_id": organization_id, + }, + ) + # The session row the command's `stopping_turn_id` is stamped on. + await session.execute( + text( + "INSERT INTO session_streams (id, project_id, session_id, turn_id) " + "VALUES (:id, :project_id, :session_id, :turn_id)" + ), + { + "id": uuid.uuid4(), + "project_id": project_id, + "session_id": session_id, + "turn_id": "turn-A", + }, + ) + await session.commit() + + yield { + "engine": engine, + "project_id": project_id, + "user_id": user_id, + "session_id": session_id, + } + + +def _create(scope, **overrides) -> SessionCommandCreate: + payload = dict( + project_id=scope["project_id"], + session_id=scope["session_id"], + kind=SessionCommandKind.cancel, + target_turn_id="turn-A", + state=SessionCommandState.pending, + created_at=datetime.now(timezone.utc), + ) + payload.update(overrides) + return SessionCommandCreate(**payload) + + +async def _stopping_turn_id(scope) -> str: + async with scope["engine"].session() as session: + result = await session.execute( + text( + "SELECT stopping_turn_id FROM session_streams " + "WHERE project_id = :project_id AND session_id = :session_id" + ), + {"project_id": scope["project_id"], "session_id": scope["session_id"]}, + ) + return result.scalar() + + +async def test_the_command_and_the_stopping_marker_are_written_together(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + + command = await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope), + stopping_turn_id="turn-A", + ) + + assert command.state == SessionCommandState.pending + # A session that renders as plainly running while a command exists to stop it is a session + # nothing later reconciles, so the two writes share one transaction. + assert await _stopping_turn_id(command_scope) == "turn-A" + + +async def test_a_repeated_idempotency_key_returns_the_first_row(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + + first = await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope, idempotency_key="retry-me"), + ) + second = await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope, idempotency_key="retry-me"), + ) + + assert second.id == first.id + assert ( + await dao.count_open( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + ) + == 1 + ) + + +async def test_two_open_commands_for_one_execution_collapse_to_one(command_scope): + # Two Stops for the same execution are one intent, even with no idempotency key and even + # when admission's own read cannot see the other because it has not committed yet. The + # database refuses the second insert and the DAO answers with the command that exists. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + + first = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + second = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + assert second.id == first.id + assert ( + await dao.count_open( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + ) + == 1 + ) + + +async def test_two_concurrent_admissions_still_yield_one_command(command_scope): + # The race the unique index exists for: both inserts run before either commits. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + + first, second = await asyncio.wait_for( + asyncio.gather( + dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ), + dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ), + return_exceptions=True, + ), + timeout=30, + ) + + ids = {r.id for r in (first, second) if not isinstance(r, Exception)} + assert len(ids) == 1, f"expected one command, got {first!r} and {second!r}" + + +async def test_a_settled_command_does_not_block_a_new_one(command_scope): + # The unique index is partial on the OPEN states, so once a Stop has settled the next Stop + # against the same execution is a fresh command, not a constraint violation. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + first = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=first.id, + state=SessionCommandState.applied, + outcome=SessionCommandOutcome.stopped, + expected_states=[SessionCommandState.pending], + replica_id=None, + ) + ) + + second = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + assert second.id != first.id + + +async def test_the_open_command_read_finds_only_the_same_target(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope, target_turn_id="turn-A"), + ) + + same = await dao.fetch_open_command( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + kind=SessionCommandKind.cancel, + target_turn_id="turn-A", + ) + other = await dao.fetch_open_command( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + kind=SessionCommandKind.cancel, + target_turn_id="turn-B", + ) + + assert same is not None + assert other is None, "a different execution is a different intent" + + +async def test_a_settled_command_is_no_longer_open(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], + command=_create( + command_scope, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.not_running, + settled_at=datetime.now(timezone.utc), + ), + ) + + assert command.state == SessionCommandState.obsolete + assert ( + await dao.fetch_open_command( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + kind=SessionCommandKind.cancel, + target_turn_id="turn-A", + ) + is None + ) + + +async def test_two_concurrent_claims_of_one_command_yield_exactly_one_winner( + command_scope, +): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + scopes = [ + SessionScope( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + ) + ] + + # Bounded: both calls contend for the same row on separate pooled connections, so a + # regression that drops SKIP LOCKED would hang the run rather than fail it. + first, second = await asyncio.wait_for( + asyncio.gather( + dao.claim_commands( + sessions=scopes, replica_id="replica-1", lease_seconds=90, limit=10 + ), + dao.claim_commands( + sessions=scopes, replica_id="replica-2", lease_seconds=90, limit=10 + ), + ), + timeout=30, + ) + + assert len(first) + len(second) == 1, ( + "a command is delivered to one replica, not two" + ) + + +async def test_a_claim_ignores_sessions_the_caller_did_not_declare(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + claimed = await dao.claim_commands( + sessions=[ + SessionScope( + project_id=command_scope["project_id"], session_id="a-different-session" + ) + ], + replica_id="replica-1", + lease_seconds=90, + limit=10, + ) + + assert claimed == [] + + +async def test_the_claim_records_the_lease_and_counts_the_delivery(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + attempted = await dao.record_delivery_attempt( + project_id=command_scope["project_id"], + command_id=command.id, + now=datetime.now(timezone.utc), + max_deliveries=3, + ) + assert attempted is not None + + claimed = await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-1", + lease_seconds=90, + ) + + assert claimed is not None + assert claimed.state == SessionCommandState.claimed + assert claimed.claimed_by == "replica-1" + assert claimed.claim_count == 1 + assert claimed.claim_expires_at is not None + assert claimed.claim_expires_at > datetime.now(timezone.utc) + timedelta(seconds=60) + + +async def test_a_second_delivery_claim_finds_nothing_to_take(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-1", + lease_seconds=90, + ) + + again = await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-2", + lease_seconds=90, + ) + + assert again is None + + +async def test_only_the_replica_holding_the_claim_may_settle(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-1", + lease_seconds=90, + ) + + wrong = await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=command.id, + state=SessionCommandState.applied, + outcome=SessionCommandOutcome.stopped, + replica_id="replica-2", + ) + ) + + assert wrong is None + stored = await dao.fetch_command(command_id=command.id) + assert stored.state == SessionCommandState.claimed, "the stored state is unchanged" + + +async def test_settling_an_already_terminal_command_changes_nothing(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-1", + lease_seconds=90, + ) + settled = await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=command.id, + state=SessionCommandState.applied, + outcome=SessionCommandOutcome.stopped, + replica_id="replica-1", + ) + ) + assert settled is not None + + repeat = await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=command.id, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.failed, + replica_id="replica-1", + ) + ) + + assert repeat is None, "one execution, one terminal outcome, one writer" + stored = await dao.fetch_command(command_id=command.id) + assert stored.outcome == SessionCommandOutcome.stopped + + +async def test_the_api_can_settle_a_pending_command_nobody_took(command_scope): + # The `not_held` case: a reachable runner said it does not hold the session, so there is no + # claim to guard on and the API settles it itself. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + settled = await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=command.id, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.not_running, + expected_states=[SessionCommandState.pending], + replica_id=None, + ) + ) + + assert settled is not None + assert settled.outcome == SessionCommandOutcome.not_running + + +async def test_the_runner_can_find_a_command_without_a_project_id(command_scope): + # The runner reports an outcome with the command id alone; it holds no project credential. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + found = await dao.fetch_command(command_id=command.id) + + assert found is not None + assert found.project_id == command_scope["project_id"] + + +async def test_clearing_the_stopping_marker_is_scoped_to_the_turn_it_set(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope), + stopping_turn_id="turn-A", + ) + + # A settlement for an OLDER turn must not clear a newer Stop's marker. + await dao.clear_stopping_turn( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + turn_id="turn-older", + ) + assert await _stopping_turn_id(command_scope) == "turn-A" + + await dao.clear_stopping_turn( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + turn_id="turn-A", + ) + assert await _stopping_turn_id(command_scope) is None + + +async def test_expire_claims_returns_only_leases_that_have_passed(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + fresh = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=fresh.id, + replica_id="replica-1", + lease_seconds=90, + ) + + # The sweep is deliberately NOT project-scoped: it settles every abandoned claim in the + # deployment, so assert on this command's presence rather than on the whole result. + now = datetime.now(timezone.utc) + assert fresh.id not in { + row.id for row in await dao.expire_claims(now=now, max_deliveries=3) + }, "a lease that has not passed is not swept" + # An hour later the same lease has passed, and the settlement sweep sees it. + later = await dao.expire_claims(now=now + timedelta(hours=1), max_deliveries=3) + assert fresh.id in {row.id for row in later} + + +async def test_old_pending_commands_are_returned_for_redelivery(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + now = datetime.now(timezone.utc) + command = await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope, created_at=now - timedelta(minutes=5)), + ) + + rows = await dao.expire_claims( + now=now, + max_deliveries=3, + pending_before=now - timedelta(seconds=90), + ) + + assert command.id in {row.id for row in rows} + + +async def test_delivery_attempts_are_bounded_in_the_database(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + now = datetime.now(timezone.utc) + + first = await dao.record_delivery_attempt( + project_id=command_scope["project_id"], + command_id=command.id, + now=now, + max_deliveries=1, + ) + second = await dao.record_delivery_attempt( + project_id=command_scope["project_id"], + command_id=command.id, + now=now + timedelta(seconds=1), + max_deliveries=1, + ) + + assert first is not None + assert first.claim_count == 1 + assert second is None + + +async def test_runner_and_watchdog_have_one_terminal_winner(command_scope): + dao = SessionExecutionsDAO(engine=command_scope["engine"]) + + runner, watchdog = await asyncio.gather( + dao.settle( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + execution_id="turn-A", + terminal_outcome="stopped", + settled_by="runner", + ), + dao.settle( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + execution_id="turn-A", + terminal_outcome="lost", + settled_by="watchdog", + ), + ) + + assert sum(result.won for result in (runner, watchdog)) == 1 + assert runner.settlement == watchdog.settlement + + +async def test_repeating_the_same_execution_settlement_reports_only_the_insert_as_winner( + command_scope, +): + dao = SessionExecutionsDAO(engine=command_scope["engine"]) + + first = await dao.settle( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + execution_id="turn-A", + terminal_outcome="lost", + settled_by="watchdog", + ) + repeated = await dao.settle( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + execution_id="turn-A", + terminal_outcome="lost", + settled_by="watchdog", + ) + + assert first.won is True + assert repeated.won is False + assert repeated.settlement == first.settlement + + +async def test_execution_ending_marker_is_one_way(command_scope): + dao = SessionExecutionsDAO(engine=command_scope["engine"]) + await dao.settle( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + execution_id="turn-A", + terminal_outcome="stopped", + settled_by="runner", + ) + written_at = datetime.now(timezone.utc) + + await dao.mark_endings_written( + project_id=command_scope["project_id"], + keys=[(command_scope["session_id"], "turn-A")], + written_at=written_at, + ) + await dao.mark_endings_written( + project_id=command_scope["project_id"], + keys=[(command_scope["session_id"], "turn-A")], + written_at=written_at + timedelta(seconds=1), + ) + + stored = await dao.query_settled( + project_id=command_scope["project_id"], + keys=[(command_scope["session_id"], "turn-A")], + ) + assert ( + stored[(command_scope["session_id"], "turn-A")].ending_written_at == written_at + ) + + +async def test_terminal_core_facts_commit_in_one_transaction(command_scope): + commands = SessionCommandsDAO(engine=command_scope["engine"]) + executions = SessionExecutionsDAO(engine=command_scope["engine"]) + streams = SessionStreamsDAO(engine=command_scope["engine"]) + interactions = SessionInteractionsDAO(engine=command_scope["engine"]) + command = await commands.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope), + stopping_turn_id="turn-A", + ) + await commands.record_delivery_attempt( + project_id=command_scope["project_id"], + command_id=command.id, + now=datetime.now(timezone.utc), + max_deliveries=3, + ) + await commands.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="runner-1", + lease_seconds=90, + ) + interaction_id = uuid.uuid4() + async with command_scope["engine"].session() as session: + await session.execute( + text( + "UPDATE session_streams SET flags = " + '\'{"is_alive": true, "is_running": true, ' + '"is_attached": true}\'::jsonb ' + "WHERE project_id = :project_id AND session_id = :session_id" + ), + { + "project_id": command_scope["project_id"], + "session_id": command_scope["session_id"], + }, + ) + await session.execute( + text( + "INSERT INTO session_interactions " + "(project_id, id, session_id, turn_id, token, kind, status) " + "VALUES (:project_id, :id, :session_id, 'turn-A', " + "'token-A', 'user_approval', 'pending')" + ), + { + "project_id": command_scope["project_id"], + "id": interaction_id, + "session_id": command_scope["session_id"], + }, + ) + + transition = SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=command.id, + state=SessionCommandState.applied, + outcome=SessionCommandOutcome.stopped, + expected_states=[SessionCommandState.claimed], + replica_id="runner-1", + ) + async with commands.transaction() as transaction: + settled = await commands.settle_command( + settle=transition, + transaction=transaction, + ) + execution = await executions.settle( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + execution_id="turn-A", + terminal_outcome="stopped", + settled_by="runner", + transaction=transaction, + ) + await streams.settle_command( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + turn_id="turn-A", + mirror_stopped=True, + transaction=transaction, + ) + await interactions.cancel_session_pending( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + only_turn_id="turn-A", + transaction=transaction, + ) + + assert settled is not None + assert execution.won is True + async with command_scope["engine"].session() as session: + row = ( + await session.execute( + text( + "SELECT c.state, c.outcome, s.stopping_turn_id, " + "s.flags->>'is_running', s.flags->>'is_attached', i.status, " + "e.terminal_outcome " + "FROM session_commands c " + "JOIN session_streams s ON s.project_id = c.project_id " + "AND s.session_id = c.session_id " + "JOIN session_interactions i ON i.project_id = c.project_id " + "AND i.session_id = c.session_id " + "JOIN session_executions e ON e.project_id = c.project_id " + "AND e.session_id = c.session_id " + "AND e.execution_id = c.target_turn_id " + "WHERE c.project_id = :project_id AND c.id = :command_id" + ), + { + "project_id": command_scope["project_id"], + "command_id": command.id, + }, + ) + ).one() + assert tuple(row) == ( + "applied", + "stopped", + None, + "false", + "true", + "cancelled", + "stopped", + ) + + +async def test_execution_conflict_rolls_back_the_command_transition(command_scope): + commands = SessionCommandsDAO(engine=command_scope["engine"]) + executions = SessionExecutionsDAO(engine=command_scope["engine"]) + streams = SessionStreamsDAO(engine=command_scope["engine"]) + interactions = SessionInteractionsDAO(engine=command_scope["engine"]) + command = await commands.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope), + stopping_turn_id="turn-A", + ) + await commands.record_delivery_attempt( + project_id=command_scope["project_id"], + command_id=command.id, + now=datetime.now(timezone.utc), + max_deliveries=3, + ) + await commands.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="runner-1", + lease_seconds=90, + ) + await executions.settle( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + execution_id="turn-A", + terminal_outcome="lost", + settled_by="watchdog", + ) + + service = SessionCommandsService( + commands_dao=commands, + streams_service=SessionStreamsService( + streams_dao=streams, + lock_engine=None, + ), + interactions_service=SessionInteractionsService( + interactions_dao=interactions, + ), + lock_engine=None, + delivery=None, + executions_dao=executions, + ) + settled = await service.settle( + command_id=command.id, + project_id=command_scope["project_id"], + replica_id="runner-1", + expected_states=[SessionCommandState.claimed], + state=SessionCommandState.applied, + outcome=SessionCommandOutcome.stopped, + execution_id="turn-A", + ) + + assert settled is None + stored = await commands.fetch_command(command_id=command.id) + assert stored is not None + assert stored.state == SessionCommandState.claimed + assert stored.outcome is None diff --git a/api/oss/tests/pytest/unit/sessions/test_watch_interactions_publish.py b/api/oss/tests/pytest/unit/sessions/test_watch_interactions_publish.py index 9835cb98452..790190c38a1 100644 --- a/api/oss/tests/pytest/unit/sessions/test_watch_interactions_publish.py +++ b/api/oss/tests/pytest/unit/sessions/test_watch_interactions_publish.py @@ -115,7 +115,12 @@ async def test_failed_transition_publishes_nothing(): @pytest.mark.asyncio async def test_cancel_sweep_publishes_resolved_only_when_it_cancelled(): dao = AsyncMock() - dao.cancel_session_pending = AsyncMock(return_value=2) + dao.cancel_session_pending = AsyncMock( + return_value=[ + _interaction("sess-1"), + _interaction("sess-1").model_copy(update={"token": "tok-2"}), + ] + ) svc, publisher = _service(dao) cancelled = await svc.cancel_session_pending( @@ -125,7 +130,7 @@ async def test_cancel_sweep_publishes_resolved_only_when_it_cancelled(): assert publisher.interaction_calls == [(str(_PROJECT), "sess-1", "resolved")] # No-op sweep: nothing was pending, nothing changed, nothing to notify. - dao.cancel_session_pending = AsyncMock(return_value=0) + dao.cancel_session_pending = AsyncMock(return_value=[]) publisher.interaction_calls.clear() await svc.cancel_session_pending(project_id=_PROJECT, session_id="sess-1") assert publisher.interaction_calls == [] diff --git a/api/oss/tests/pytest/unit/sessions/test_watch_lifecycle_publish.py b/api/oss/tests/pytest/unit/sessions/test_watch_lifecycle_publish.py index 1859c655f30..00603bf3464 100644 --- a/api/oss/tests/pytest/unit/sessions/test_watch_lifecycle_publish.py +++ b/api/oss/tests/pytest/unit/sessions/test_watch_lifecycle_publish.py @@ -139,6 +139,16 @@ async def test_cancel_publishes_lifecycle_ended(lock_engine): svc, publisher = _service(lock_engine) session_id = _session_id() + await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest( + session_id=session_id, + data=WorkflowServiceRequestData(inputs={"messages": ["hi"]}), + ), + ) + publisher.lifecycle_calls.clear() + await svc.command( project_id=_PROJECT, user_id=_USER, diff --git a/api/oss/tests/pytest/unit/sessions/test_watch_publish.py b/api/oss/tests/pytest/unit/sessions/test_watch_publish.py index ff0de2f57bd..31717408ae0 100644 --- a/api/oss/tests/pytest/unit/sessions/test_watch_publish.py +++ b/api/oss/tests/pytest/unit/sessions/test_watch_publish.py @@ -136,11 +136,11 @@ async def test_worker_skips_publish_when_append_fails(): assert total_appended == 0 assert publisher.calls == [] - # `process_batch` acknowledges at parse time, before the append, so a failed append is still - # acked and dropped by the shared consumer loop. That predates this change and is shared by - # every worker on `BaseStreamConsumer`; the relay tee neither causes it nor repairs it. This - # assertion pins the tee's scope, not an endorsement of the acknowledgement rule. - assert len(processed_ids) == 1 + # A failed append acknowledges nothing, so the record stays in the Redis pending list and + # the reclaim pass writes it later. `process_batch` used to acknowledge at parse time, + # before the append, which made every Postgres failure permanent record loss (#5496). + # `test_records_worker_durability.py` pins that rule; this line pins the tee's scope. + assert processed_ids == [] @pytest.mark.asyncio diff --git a/api/oss/tests/pytest/unit/sessions/test_watchdog_collapse_persistence.py b/api/oss/tests/pytest/unit/sessions/test_watchdog_collapse_persistence.py new file mode 100644 index 00000000000..ea48e847019 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_watchdog_collapse_persistence.py @@ -0,0 +1,588 @@ +"""The watchdog's collapse must PERSIST against a real Postgres, in the same pass that +settles the command. + +Finding 7 (run 2d, session 6721d762): the sweep logged the collapse, but the row still read +is_running true afterwards with no other writer. Cause: the collapse mutated ORM row objects +(`row.flags = ...`), but those objects had been detached from the task-scoped session by the +nested `engine.session()` calls the pass makes (the records lookup, the command settlement) -- +each opens the SAME current-task-scoped session and closes it in its `finally`. A detached +object's mutation is tracked by no session, so `session.commit()` never emits the flags +UPDATE, while the command settle's Core UPDATE (stopping_turn_id) still lands. A unit test +with fakes cannot catch this: it needs the real async_scoped_session + close semantics, so +this test drives a real Postgres. + +It replays the real pass end to end with the real DAOs on a FRESH, isolated database (created +per test on the same server, dropped after), so the global sweep sees only the seeded row and +nothing is polluted. It seeds one alive+running stream naming a turn, one pending Stop for it, +and a stale heartbeat; runs one real sweep pass; then reads the row back through a fresh +session and asserts the collapse persisted, the execution was settled lost, and the command +went obsolete/lost. + +Only the SERVER in POSTGRES_URI_CORE is used. The database named in that URI is never written: +the fixture creates its own and drops it. Point it at any reachable core Postgres, for example + cd api && POSTGRES_URI_CORE=postgresql+asyncpg://username:password@localhost:5432/agenta_oss_core \ + uv run --no-sync pytest oss/tests/pytest/unit/sessions/test_watchdog_collapse_persistence.py -q +""" + +import asyncio +import uuid +from datetime import datetime, timezone, timedelta +from urllib.parse import urlparse, urlunparse + +import asyncpg +import pytest +from sqlalchemy import event, text +from sqlalchemy.ext.asyncio import create_async_engine + +import oss.src.models.db_models # noqa: F401 (register auth/org tables on Base) + +# Register the session tables on the shared Base so create_all builds them. +from oss.src.dbs.postgres.sessions.streams.dbes import SessionStreamDBE # noqa: F401 +from oss.src.dbs.postgres.sessions.executions.dbes import SessionExecutionDBE # noqa: F401 +from oss.src.dbs.postgres.sessions.commands.dbes import SessionCommandDBE # noqa: F401 +from oss.src.dbs.postgres.sessions.records.dbes import RecordDBE # noqa: F401 +from oss.src.dbs.postgres.sessions.interactions.dbes import ( # noqa: F401 + SessionInteractionDBE, +) +from oss.src.dbs.postgres.shared.base import Base + +from oss.src.core.sessions.streams.dtos import ( + SessionStreamEdit, + SessionStreamFlags, +) +from oss.src.utils.env import env +from oss.src.dbs.postgres.shared.engine import TransactionsEngine +from oss.src.dbs.postgres.sessions.streams.dao import SessionStreamsDAO +from oss.src.dbs.postgres.sessions.commands.dao import SessionCommandsDAO +from oss.src.dbs.postgres.sessions.executions.dao import SessionExecutionsDAO +from oss.src.dbs.postgres.sessions.interactions.dao import SessionInteractionsDAO +from oss.src.dbs.postgres.sessions.records.dao import RecordsDAO +from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.core.sessions.interactions.service import SessionInteractionsService +from oss.src.core.sessions.commands.service import SessionCommandsService +from oss.src.core.sessions.records.service import RecordsService +from oss.src.dbs.http.sessions.control_delivery_direct import DirectControlDelivery +from oss.src.tasks.asyncio.sessions import orphan_sweep + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def anyio_backend(): + return "asyncio" + + +class _FakeLock: + """In-memory Redis stand-in — the DB persistence is what this test is about.""" + + def __init__(self): + self._s = {} + + async def get(self, k): + return self._s.get(k) + + async def set(self, k, v, nx=False, ex=None): + if nx and k in self._s: + return None + self._s[k] = v + return True + + async def delete(self, k): + self._s.pop(k, None) + return 1 + + async def expire(self, k, ttl): + return True + + async def eval(self, script, numkeys, *keys_and_args): + def decode(value): + return value.decode() if isinstance(value, bytes) else str(value) + + keys = [decode(value) for value in keys_and_args[:numkeys]] + argv = [decode(value) for value in keys_and_args[numkeys:]] + if "AGENTA_WATCHDOG_RELEASE_TURN" in script: + alive, running, owner, superseded = keys + expected_turn, expected_owner, _ttl = argv + alive_value = decode(self._s[alive]) if alive in self._s else "" + running_value = decode(self._s[running]) if running in self._s else "" + owner_value = decode(self._s[owner]) if owner in self._s else "" + released_alive = int(bool(expected_turn) and alive_value == expected_turn) + released_running = int( + bool(expected_turn) and running_value == expected_turn + ) + if released_alive: + self._s.pop(alive, None) + if released_running: + self._s.pop(running, None) + foreign_turn = (alive_value and alive_value != expected_turn) or ( + running_value and running_value != expected_turn + ) + released_owner = int( + bool(expected_owner) + and owner_value == expected_owner + and not foreign_turn + ) + if released_owner: + self._s.pop(owner, None) + if expected_turn: + self._s[superseded] = b"1" + return [released_alive, released_running, released_owner] + + k = keys[0] + v = argv[0] + cur = self._s.get(k) + if isinstance(cur, bytes): + cur = cur.decode() + if len(argv) > 1: + from oss.src.dbs.redis.sessions.contract import owner_replica_id + + if cur is None or owner_replica_id(cur) == owner_replica_id(v): + self._s[k] = v.encode() + return v.encode() + return cur.encode() if cur else None + if cur == v: + self._s.pop(k, None) + return 1 + return 0 + + +async def _noop_publish(*, project_id, record_event): + return False + + +def _admin_dsn() -> str: + parsed = urlparse(env.postgres.uri_core) + # asyncpg DSN (no +asyncpg driver tag), connect to the maintenance db. + return urlunparse(("postgresql", parsed.netloc, "/postgres", "", "", "")) + + +def _sqlalchemy_url_for(db_name: str) -> str: + parsed = urlparse(env.postgres.uri_core) + return urlunparse(("postgresql+asyncpg", parsed.netloc, f"/{db_name}", "", "", "")) + + +@pytest.fixture +async def wd_engine(monkeypatch): + """A TransactionsEngine bound to a fresh, isolated database with the full schema.""" + db_name = f"agenta_wd_rca_{uuid.uuid4().hex[:12]}" + admin = await asyncpg.connect(dsn=_admin_dsn()) + await admin.execute(f'CREATE DATABASE "{db_name}"') + await admin.close() + + seed = await asyncpg.connect(dsn=_admin_dsn().replace("/postgres", f"/{db_name}")) + for ext in ("pgcrypto", "ltree"): + await seed.execute(f'CREATE EXTENSION IF NOT EXISTS "{ext}"') + await seed.close() + + # Only the tables this pass touches; the full metadata carries unrelated tables with + # foreign keys to modules we do not import here. + needed = [ + Base.metadata.tables[name] + for name in ( + "users", + "organizations", + "workspaces", + "projects", + "session_streams", + "session_executions", + "session_commands", + "records", + "session_interactions", + ) + ] + schema_engine = create_async_engine(_sqlalchemy_url_for(db_name)) + async with schema_engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all, tables=needed) + await schema_engine.dispose() + + # Point the real TransactionsEngine at the fresh DB so its exact async_scoped_session + + # close semantics (the trigger for the detach bug) are what runs. + monkeypatch.setattr(env.postgres, "uri_core", _sqlalchemy_url_for(db_name)) + engine = TransactionsEngine() + try: + engine._wd_db_name = db_name + yield engine + finally: + await engine.close() + admin = await asyncpg.connect(dsn=_admin_dsn()) + await admin.execute( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + "WHERE datname=$1 AND pid<>pg_backend_pid()", + db_name, + ) + await admin.execute(f'DROP DATABASE IF EXISTS "{db_name}"') + await admin.close() + + +async def _seed_tenant(s): + """One user, organization, workspace and project. Returns the project id.""" + project_id = uuid.uuid4() + uid, org, ws = uuid.uuid4(), uuid.uuid4(), uuid.uuid4() + await s.execute( + text("INSERT INTO users (id, uid, username, email) VALUES (:i,:u,:n,:e)"), + {"i": uid, "u": str(uid), "n": "wd", "e": f"wd-{uid.hex[:8]}@e.com"}, + ) + await s.execute( + text("INSERT INTO organizations (id, name, owner_id) VALUES (:i,:n,:o)"), + {"i": org, "n": "wd", "o": uid}, + ) + await s.execute( + text("INSERT INTO workspaces (id, name, organization_id) VALUES (:i,:n,:o)"), + {"i": ws, "n": "wd", "o": org}, + ) + await s.execute( + text( + "INSERT INTO projects (id, project_name, organization_id, workspace_id) " + "VALUES (:i,:n,:o,:w)" + ), + {"i": project_id, "n": "wd", "o": org, "w": ws}, + ) + return project_id + + +async def _seed_scenario(engine, *, session_id, turn_id): + stale = datetime.now(timezone.utc) - timedelta(hours=1) + async with engine.session() as s: + project_id = await _seed_tenant(s) + await s.execute( + text( + "INSERT INTO session_streams " + "(id, project_id, session_id, turn_id, flags, stopping_turn_id, created_at, updated_at) " + "VALUES (:i,:p,:s,:t, CAST(:f AS JSONB), :st, :c, :u)" + ), + { + "i": uuid.uuid4(), + "p": project_id, + "s": session_id, + "t": turn_id, + "f": '{"is_alive": true, "is_running": true, "is_attached": false}', + "st": turn_id, + "c": stale, + "u": stale, + }, + ) + await s.execute( + text( + "INSERT INTO session_commands " + "(id, project_id, session_id, kind, target_turn_id, state, claim_count, created_at) " + "VALUES (:i,:p,:s,'cancel',:t,'pending',0,:c)" + ), + { + "i": uuid.uuid4(), + "p": project_id, + "s": session_id, + "t": turn_id, + "c": stale, + }, + ) + await s.commit() + return project_id + + +async def _seed_lost_execution_scenario(engine, *, session_id, turn_id): + """A row the ORPHAN query never returns, whose turn is owed an ending. + + The stream row beats normally (a fresh `updated_at`), so it is not stale and is not + collapsed. Its turn is already settled `lost` with no ending written, which is what puts it + in `newly_lost`: the branch that clears `is_running` and keeps `is_alive`. + """ + fresh = datetime.now(timezone.utc) + stale = fresh - timedelta(hours=1) + async with engine.session() as s: + project_id = await _seed_tenant(s) + await s.execute( + text( + "INSERT INTO session_streams " + "(id, project_id, session_id, turn_id, flags, created_at, updated_at) " + "VALUES (:i,:p,:s,:t, CAST(:f AS JSONB), :c, :u)" + ), + { + "i": uuid.uuid4(), + "p": project_id, + "s": session_id, + "t": turn_id, + "f": '{"is_alive": true, "is_running": true, "is_attached": true}', + "c": fresh, + "u": fresh, + }, + ) + await s.execute( + text( + "INSERT INTO session_executions " + "(project_id, session_id, execution_id, terminal_outcome, settled_by, settled_at) " + "VALUES (:p,:s,:t,'lost','watchdog',:a)" + ), + {"p": project_id, "s": session_id, "t": turn_id, "a": stale}, + ) + await s.commit() + return project_id + + +def _build_services(engine): + lock = _FakeLock() + streams_service = SessionStreamsService( + streams_dao=SessionStreamsDAO(engine), lock_engine=lock + ) + interactions_service = SessionInteractionsService( + interactions_dao=SessionInteractionsDAO(engine) + ) + executions_dao = SessionExecutionsDAO(engine) + commands_service = SessionCommandsService( + commands_dao=SessionCommandsDAO(engine), + streams_service=streams_service, + interactions_service=interactions_service, + lock_engine=lock, + delivery=DirectControlDelivery(), + executions_dao=executions_dao, + ) + records_service = RecordsService(RecordsDAO(engine), executions_dao) + return lock, records_service, commands_service + + +@pytest.mark.anyio +async def test_a_lost_pass_persists_the_collapse_against_real_postgres( + anyio_backend, wd_engine, monkeypatch +): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + + session_id = "wd-" + uuid.uuid4().hex[:12] + turn_id = str(uuid.uuid4()) + await _seed_scenario(wd_engine, session_id=session_id, turn_id=turn_id) + + lock, records_service, commands_service = _build_services(wd_engine) + await orphan_sweep.run_orphan_sweep( + wd_engine, + lock, + records_service=records_service, + watch_publisher=None, + commands_service=commands_service, + publish=_noop_publish, + ) + + # Read back through a FRESH session so the assertions see committed DB state, not any + # in-memory ORM object the pass held. + async with wd_engine.session() as s: + flags, stopping = ( + await s.execute( + text( + "SELECT flags, stopping_turn_id FROM session_streams WHERE session_id=:s" + ), + {"s": session_id}, + ) + ).one() + ex = ( + await s.execute( + text( + "SELECT terminal_outcome, settled_by FROM session_executions " + "WHERE session_id=:s AND execution_id=:t" + ), + {"s": session_id, "t": turn_id}, + ) + ).one_or_none() + cmd = ( + await s.execute( + text( + "SELECT state, outcome FROM session_commands " + "WHERE session_id=:s AND target_turn_id=:t" + ), + {"s": session_id, "t": turn_id}, + ) + ).one() + + # The collapse persisted: this is the finding-7 assertion. + assert flags["is_alive"] is False + assert flags["is_running"] is False + assert stopping is None + # The execution reached its durable terminal outcome, settled by the watchdog. + assert ex is not None + assert ex[0] == "lost" + assert ex[1] == "watchdog" + # The Stop command was settled, not left pending. + assert cmd[0] == "obsolete" + assert cmd[1] == "lost" + + +@pytest.mark.anyio +async def test_b_lost_turn_clear_persists_after_a_nested_session_close( + anyio_backend, wd_engine, monkeypatch +): + """The `newly_lost` is_running clear survives a nested session between load and write. + + Same failure mode as finding 7, one branch up. The owner lookup is patched to open an + `engine.session()`, whose `finally` closes the shared task-scoped session before settlement + and the lost-turn update. Core writes must still reopen that session and persist. + + This never failed in production: before the fix the write sat immediately after the load, + with nothing nested in between. The test pins the property rather than a past bug. Make the + write an ORM attribute assignment again and it fails on `is_running` still true. + """ + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + + session_id = "wd-" + uuid.uuid4().hex[:12] + turn_id = str(uuid.uuid4()) + await _seed_lost_execution_scenario( + wd_engine, session_id=session_id, turn_id=turn_id + ) + + real_get_owner_value = orphan_sweep.get_owner_value + nested_sessions = [] + + async def _get_owner_value_through_a_nested_session(*args, **kwargs): + # Open and close the shared task-scoped session, exactly as a DAO call would. + async with wd_engine.session(): + nested_sessions.append(1) + return await real_get_owner_value(*args, **kwargs) + + monkeypatch.setattr( + orphan_sweep, + "get_owner_value", + _get_owner_value_through_a_nested_session, + ) + + lock, records_service, commands_service = _build_services(wd_engine) + await orphan_sweep.run_orphan_sweep( + wd_engine, + lock, + records_service=records_service, + watch_publisher=None, + commands_service=commands_service, + publish=_noop_publish, + ) + + # The pass must actually have reached the branch under test. + assert nested_sessions, "the lost-turn branch never ran, so nothing was proven" + + async with wd_engine.session() as s: + flags = ( + await s.execute( + text("SELECT flags FROM session_streams WHERE session_id=:s"), + {"s": session_id}, + ) + ).scalar_one() + + # is_running cleared and PERSISTED; is_alive kept, so the session stays resumable. + assert flags["is_running"] is False + assert flags["is_alive"] is True + + +@pytest.mark.anyio +async def test_c_heartbeat_blocked_on_sweep_cannot_revive_collapsed_row( + anyio_backend, wd_engine +): + """A heartbeat whose UPDATE snapshot predates the sweep commit must lose its CAS.""" + session_id = "wd-" + uuid.uuid4().hex[:12] + turn_id = str(uuid.uuid4()) + project_id = await _seed_scenario(wd_engine, session_id=session_id, turn_id=turn_id) + + parsed = urlparse(env.postgres.uri_core) + dsn = urlunparse( + ("postgresql", parsed.netloc, f"/{wd_engine._wd_db_name}", "", "", "") + ) + sweep = await asyncpg.connect(dsn=dsn) + observer = await asyncpg.connect(dsn=dsn) + sweep_transaction = sweep.transaction() + heartbeat = None + committed = False + heartbeat_rowcounts = [] + + def capture_heartbeat_rowcount( + _connection, + clauseelement, + _multiparams, + _params, + _execution_options, + result, + ): + if getattr(clauseelement, "is_update", False): + table = getattr(clauseelement, "table", None) + if table is not None and table.name == "session_streams": + heartbeat_rowcounts.append(result.rowcount) + + event.listen( + wd_engine._engine.sync_engine, "after_execute", capture_heartbeat_rowcount + ) + try: + await sweep_transaction.start() + await sweep.execute( + "UPDATE session_streams " + "SET flags=$1::jsonb, updated_at=NOW() " + "WHERE project_id=$2 AND session_id=$3", + '{"is_alive": false, "is_running": false, "is_attached": false}', + project_id, + session_id, + ) + await sweep.execute( + "INSERT INTO session_executions " + "(project_id, session_id, execution_id, terminal_outcome, settled_by, settled_at) " + "VALUES ($1,$2,$3,'lost','watchdog',NOW())", + project_id, + session_id, + turn_id, + ) + + heartbeat = asyncio.create_task( + SessionStreamsDAO(wd_engine).update( + project_id=project_id, + user_id=None, + session_id=session_id, + stream=SessionStreamEdit( + flags=SessionStreamFlags( + is_alive=True, is_running=True, is_attached=False + ), + turn_id=turn_id, + expected_turn_id=turn_id, + ), + ) + ) + + async def heartbeat_is_blocked_on_the_sweep(): + while True: + blocked = await observer.fetchval( + "SELECT EXISTS (" + "SELECT 1 FROM pg_stat_activity " + "WHERE datname=current_database() " + "AND wait_event_type='Lock' " + "AND query LIKE 'UPDATE session_streams%')" + ) + if blocked: + return + await asyncio.sleep(0.01) + + await asyncio.wait_for(heartbeat_is_blocked_on_the_sweep(), timeout=5) + await sweep_transaction.commit() + committed = True + heartbeat_result = await asyncio.wait_for(heartbeat, timeout=5) + finally: + event.remove( + wd_engine._engine.sync_engine, "after_execute", capture_heartbeat_rowcount + ) + if heartbeat is not None and not heartbeat.done(): + heartbeat.cancel() + await asyncio.gather(heartbeat, return_exceptions=True) + if not committed: + await sweep_transaction.rollback() + await observer.close() + await sweep.close() + + assert heartbeat_result is None + assert heartbeat_rowcounts == [0] + + async with wd_engine.session() as s: + flags, outcome = ( + await s.execute( + text( + "SELECT ss.flags, se.terminal_outcome " + "FROM session_streams ss JOIN session_executions se " + "ON se.project_id=ss.project_id AND se.session_id=ss.session_id " + "AND se.execution_id=ss.turn_id WHERE ss.session_id=:s" + ), + {"s": session_id}, + ) + ).one() + + assert outcome == "lost" + assert flags == { + "is_alive": False, + "is_running": False, + "is_attached": False, + } diff --git a/api/oss/tests/pytest/unit/sessions/test_watchdog_lifespan_wiring.py b/api/oss/tests/pytest/unit/sessions/test_watchdog_lifespan_wiring.py new file mode 100644 index 00000000000..aa0ee88ab77 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_watchdog_lifespan_wiring.py @@ -0,0 +1,50 @@ +import asyncio +import importlib +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + + +@pytest.mark.asyncio +async def test_lifespan_wires_the_commands_service_into_the_watchdog(monkeypatch): + with patch("alembic.script.ScriptDirectory.from_config", return_value=object()): + routers = importlib.import_module("entrypoints.routers") + + transactions_engine = SimpleNamespace(close=AsyncMock()) + monkeypatch.setattr(routers, "_transactions_engine", transactions_engine) + monkeypatch.setattr( + routers, "_analytics_engine", SimpleNamespace(close=AsyncMock()) + ) + monkeypatch.setattr(routers, "_streams_engine", SimpleNamespace(close=AsyncMock())) + monkeypatch.setattr(routers, "_lock_engine", object()) + monkeypatch.setattr( + routers, + "_triggers_broker", + SimpleNamespace(startup=AsyncMock(), shutdown=AsyncMock()), + ) + monkeypatch.setattr(routers, "_composio_adapters", {}) + monkeypatch.setattr(routers, "_composio_connections_adapters", {}) + monkeypatch.setattr(routers, "_composio_triggers_adapters", {}) + monkeypatch.setattr(routers.env.store, "bucket", None) + monkeypatch.setattr(routers.env, "composio", SimpleNamespace(enabled=False)) + monkeypatch.setattr(routers, "check_for_new_core_migrations", AsyncMock()) + monkeypatch.setattr(routers, "check_for_new_tracing_migrations", AsyncMock()) + monkeypatch.setattr(routers, "warn_deprecated_env_vars", lambda: None) + monkeypatch.setattr(routers, "validate_required_env_vars", lambda: None) + monkeypatch.setattr(routers, "validate_platform_runtime_key", lambda: None) + + watchdog = AsyncMock() + monkeypatch.setattr(routers, "orphan_sweep_loop", watchdog) + monkeypatch.setattr(routers, "attachment_sweep_loop", AsyncMock()) + + async with routers.lifespan(): + await asyncio.sleep(0) + watchdog.assert_awaited_once_with( + transactions_engine, + routers._lock_engine, + records_service=routers.records_service, + watch_publisher=routers._sessions_watch_publisher, + commands_service=routers.session_commands_service, + ) + assert routers.session_commands_service is not None diff --git a/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py b/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py index 5f44f2b18f3..2b4a36438b6 100644 --- a/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py +++ b/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py @@ -226,6 +226,58 @@ async def test_interaction_transition_preserves_data_and_optionally_adds_resolut assert transitioned_without_resolution.data.resolution is None +async def test_cancel_pending_returns_exactly_the_rows_it_transitioned( + interactions_dao, project +): + project_id = project["project_id"] + session_id = f"interaction-cancel-returning-{uuid.uuid4().hex[:8]}" + + for token in ("pending-1", "pending-2", "already-answered"): + await interactions_dao.create_interaction( + project_id=project_id, + user_id=None, + interaction=SessionInteractionCreate( + project_id=project_id, + session_id=session_id, + turn_id="turn-1", + token=token, + kind=SessionInteractionKind.user_approval, + ), + ) + + await interactions_dao.transition_interaction( + transition=SessionInteractionTransition( + project_id=project_id, + session_id=session_id, + token="already-answered", + status=SessionInteractionStatus.responded, + ) + ) + + cancelled = await interactions_dao.cancel_session_pending( + project_id=project_id, + session_id=session_id, + only_turn_id="turn-1", + ) + + assert {interaction.token for interaction in cancelled} == { + "pending-1", + "pending-2", + } + assert all( + interaction.status == SessionInteractionStatus.cancelled + for interaction in cancelled + ) + assert ( + await interactions_dao.cancel_session_pending( + project_id=project_id, + session_id=session_id, + only_turn_id="turn-1", + ) + == [] + ) + + # --------------------------------------------------------------------------- # SessionInteractionsDAO.delete_by_session_id — new hard delete # --------------------------------------------------------------------------- diff --git a/api/oss/tests/pytest/unit/test_multilogger.py b/api/oss/tests/pytest/unit/test_multilogger.py new file mode 100644 index 00000000000..3228424c7da --- /dev/null +++ b/api/oss/tests/pytest/unit/test_multilogger.py @@ -0,0 +1,46 @@ +"""MultiLogger must expose `exception`, like the stdlib logger. + +The application logger returned by `get_module_logger` is a `MultiLogger`. It used to define +every level method except `exception`, so `log.exception(...)` -- the natural call inside an +`except` block -- raised AttributeError from inside the handler and took the caller down. The +execution watchdog died exactly this way. These tests hold the contract that closed that gap: +the method exists, it does not raise when called from an `except` block, and it forwards to +the wrapped logger's `error` with the active traceback. +""" + +from oss.src.utils.logging import MultiLogger, get_module_logger + + +class _Spy: + """A stand-in wrapped logger that records the `error` calls MultiLogger forwards to it.""" + + def __init__(self): + self.calls = [] + + def error(self, *args, **kwargs): + self.calls.append((args, kwargs)) + + +def test_multilogger_has_an_exception_method(): + assert hasattr(MultiLogger(), "exception") + + +def test_real_module_logger_exposes_exception(): + log = get_module_logger(__name__) + assert hasattr(log, "exception") + + +def test_exception_from_an_except_block_does_not_raise_and_logs_with_traceback(): + spy = _Spy() + log = MultiLogger(spy) + + try: + raise RuntimeError("boom") + except RuntimeError: + # Before the fix this raised AttributeError instead of logging. + log.exception("something failed") + + assert spy.calls, "exception() must forward to the wrapped logger's error()" + args, kwargs = spy.calls[0] + assert args[0] == "something failed" + assert kwargs.get("exc_info") is True diff --git a/api/pyproject.toml b/api/pyproject.toml index 758a09a8b2d..41ca3cf042b 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "api" -version = "0.114.8" +version = "0.115.0" description = "Agenta API" requires-python = ">=3.11,<3.14" authors = [ diff --git a/api/uv.lock b/api/uv.lock index 0fd3cba4e94..edb71039b1b 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -8,7 +8,7 @@ resolution-markers = [ [[package]] name = "agenta" -version = "0.114.8" +version = "0.115.0" source = { editable = "../sdks/python" } dependencies = [ { name = "agenta-client" }, @@ -72,7 +72,7 @@ dev = [ [[package]] name = "agenta-client" -version = "0.114.8" +version = "0.115.0" source = { editable = "../clients/python" } dependencies = [ { name = "httpx" }, @@ -276,7 +276,7 @@ wheels = [ [[package]] name = "api" -version = "0.114.8" +version = "0.115.0" source = { virtual = "." } dependencies = [ { name = "agenta" }, diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 62637e9c919..87936ae2efc 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "agenta-client" -version = "0.114.8" +version = "0.115.0" description = "Fern-generated Python client for the Agenta API." requires-python = ">=3.11,<3.14" authors = [ diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 2e3a03a995d..31df6c3d91e 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.11, <3.14" [[package]] name = "agenta-client" -version = "0.114.8" +version = "0.115.0" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/docs/design/session-control-and-live-events/README.md b/docs/design/session-control-and-live-events/README.md new file mode 100644 index 00000000000..61f6cd0b024 --- /dev/null +++ b/docs/design/session-control-and-live-events/README.md @@ -0,0 +1,35 @@ +# Session control and live events + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +This folder holds the design work for session execution control, shared live output, +durable event replay, and durable commands. + +## Reading order + +1. [Context](context.md) explains the user problems and the current system boundary. +2. [Requirements](requirements.md) lists the open issues and draft system requirements. +3. [Decisions](decisions.md) separates confirmed decisions from proposals and open questions. +4. [Plan](plan.md) defines the design tracks and the order of discussion. +5. [Research](research.md) records verified repository findings and external dependency checks. +6. [Record properties](records-invariants.md) evaluates the existing record model before storage + options are compared. +7. [RFC](rfc.md) is the living architecture proposal. It remains incomplete until each track is discussed. +8. [Status](status.md) records current progress and the next discussion. +9. [Tonight handoff](tonight-handoff.md) contains independent spike and implementation briefs. + +## Terms under review + +- **Session:** One durable conversation and its workspace. +- **Execution:** One runner attempt that can start, pause, complete, fail, or be cancelled. +- **Conversation turn:** One user message and the resulting agent response. One conversation turn + can contain several executions when an approval pauses and resumes work. +- **Runner:** The service that starts a sandbox and drives the coding harness. +- **Harness:** The coding-agent program inside the sandbox, such as Pi or Claude Code. +- **Live frame:** A temporary output update, such as a text delta or tool progress update. +- **Durable event:** An append-only saved fact used for replay and recovery. +- **Command:** A saved request to send, cancel, approve, queue, or steer. +- **Lease:** Temporary proof that one runner owns a session or execution. + +The names are provisional. The contract discussion must settle how these terms map to the +existing `turn_id` and `turn_index` fields. diff --git a/docs/design/session-control-and-live-events/api-design.md b/docs/design/session-control-and-live-events/api-design.md new file mode 100644 index 00000000000..76e6fef6095 --- /dev/null +++ b/docs/design/session-control-and-live-events/api-design.md @@ -0,0 +1,469 @@ +# API design: the routes version one exposes + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +This file holds the route contracts considered for the durable-command work. Version one adds the +public Cancel route, the internal outcome route, and the runner's direct Cancel route; the +long-poll claim contract is explicitly deferred. Everything else in the RFC's public interface +section stays in [the RFC](rfc.md). + +The design behind these routes is in +[the durable command design](spike-b-durable-commands-design.md). Read that first for the state +machine, the lease, the settlement rule and the failure cases. + +Version one ships the direct-call adapter behind the replaceable control-delivery port. The two +long-poll routes remain future contracts; selecting `long_poll` currently fails startup rather +than silently choosing an unimplemented transport. + +Conventions taken from the existing code, not invented here: + +- Request and response models live in `api/oss/src/apis/fastapi/sessions/models.py`, are plain + Pydantic models, and set `model_config = ConfigDict(extra="forbid")` on new request bodies + (`SessionQueryRequest`, `models.py:59`). +- List responses carry `count` plus the list (`SessionsResponse`, `models.py:105`). +- Domain errors are typed exceptions in a `types.py`, mapped to status codes by one decorator on the + router (`_handle_session_exceptions`, `router.py:181`). +- Field names are `lower_snake_case`. Header names keep their standard spelling. The runner's own + HTTP surface uses `camelCase`, matching its existing `/kill` body + (`services/runner/src/server.ts:704`). + +--- + +## 1. Interface review + +Every field is classified before it is written down, as the `design-interfaces` skill requires. The +architecture review's section 4 fixed four of these shapes; where it did, that is noted. + +### Public Cancel request + +| Field | Concretely | Owner | Changes | Role | Placement | +|---|---|---|---|---|---| +| `session_id` | Which session to act on | Caller | Per call | routing | Path parameter, because it names the resource | +| `expected_execution_id` | The execution the caller believes is running | Caller | Per call | precondition | Body, flat | +| `Idempotency-Key` | Retry identity for this request | Caller | Per call | protocol context | Header | + +Three decisions fall out of that table. + +- **The public Cancel body stays flat.** The review examined this exact shape and ruled that it is + correct and should not change: `expected_execution_id` is per-call context named as the guard it + is, in the style of an HTTP `If-Match`. The grouping under `target` applies to the internal + envelope, where a resolved `target.turn_id` needs a home next to the asserted one. A public body + with one field does not. +- **`Idempotency-Key` stays a header** with its standard spelling. It describes the delivery of the + request, not the intent inside it. The stored column is `idempotency_key`, matching + `session_attachments.idempotency_key` (`api/oss/src/dbs/postgres/sessions/attachments/dbas.py:25`). +- **No `force` flag.** `force` on the current stream endpoint is what makes one route mean four + things (`api/oss/src/core/sessions/streams/service.py:7`). Cancel means cancel. + +The field stays optional, as decision D-010 requires, and first-party clients must always send it. +Today the desktop sends nothing (`web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts:505`, +verified), which is the third guard of the design document's section 4 left switched off. + +### Public Cancel response + +| Field | Concretely | Role | +|---|---|---| +| `command.id` | The durable command's id | identity, for the caller's own retries and logs | +| `command.state` | `pending` or `obsolete` at admission time | delivery | +| `execution.id` | The execution this Cancel targets, null when nothing ran | routing | +| `execution.state` | `stopping` or `idle` | result | + +`command` and `execution` are separate objects because they answer different questions and settle at +different times. A client drawing a button reads `execution`. A client retrying safely reads +`command.id`. This is decision D-016 expressed in the response shape. + +### The internal command envelope + +The review's corrected shape, adopted here: + +| Group | Fields | Role | +|---|---|---| +| top level | `id`, `project_id`, `session_id`, `kind`, `created_at` | identity, routing, metadata | +| `target` | `turn_id` (resolved at admission), `expected_turn_id` (as the caller sent it) | context | +| `input` | `text`, `attachments` | input data, absent for `cancel` | +| `policy` | `on_busy` | policy, absent for `cancel` | +| `delivery` | `claimed_by`, `claim_expires_at`, `attempt` | delivery bookkeeping | + +Four rules this applies. + +- **Delivery bookkeeping is grouped and never merged with the result.** That is decision D-016, and + it is easier to hold when the shapes are separate objects. +- **`replica_id` is not a top-level routing field.** It is delivery bookkeeping, it is logical rather + than an address, and it lives under `delivery` as `claimed_by`. +- **There is no `runner_url` field of any kind.** An address in a durable record is an + implementation detail with a longer lifetime than the thing it points at. +- **`input` is an object from the start**, not a bare `message` string. A turn already carries text + plus attachments (`services/runner/src/server.ts:565`), so a string could not grow into that + without a breaking change. `cancel` omits the group entirely rather than sending it empty. + +`created_at` is on the envelope because the runner needs it: it refuses to abort an execution that +started after the command was created. + +### Internal claim request + +| Field | Concretely | Owner | Role | +|---|---|---|---| +| `replica_id` | Which runner is asking, for `claimed_by` | Runner | delivery bookkeeping | +| `sessions` | The sessions this runner holds warm right now | Runner | routing | +| `wait_seconds` | How long the caller accepts being held | Runner | protocol context of this call | +| `limit` | How many commands to return at most | Runner | protocol context of this call | + +`sessions` is the routing input, not `replica_id`. The runner declares what it holds, so the API +never has to guess from an expiring Redis key, and a parked session keeps receiving commands after +its heartbeat stops. A claim is a query over durable state, never a cursor or a stream position. + +### Internal outcome request + +| Field | Concretely | Owner | Role | +|---|---|---|---| +| `replica_id` | Which runner is reporting | Runner | delivery bookkeeping, and the claim guard | +| `result` | The command's terminal state | Runner | delivery | +| `execution.id` | Which execution the runner acted on | Runner | routing | +| `execution.state` | What happened to it | Runner | result | +| `execution.error` | Why it failed, when it did | Runner | result | + +`execution.error` sits under `execution` because it explains one field of that object. + +--- + +## 2. Public: cancel the current execution + +```http +POST /sessions/{session_id}/cancel +Idempotency-Key: 0199a3f2-0000-7000-8000-000000000001 + +{ + "expected_execution_id": "0199a3f1-0000-7000-8000-00000000000a" +} +``` + +Permission: `Permission.RUN_SESSIONS`, the same permission the current cancel path checks +(`api/oss/src/apis/fastapi/sessions/router.py:377`). + +```python +class SessionCancelRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + # Optional stale-request guard (decision D-010). When present, the API cancels only this + # execution and rejects the request if another one is running. When absent, it cancels + # whichever execution is active when the request is applied. A person never types this; + # the browser fills it from the session snapshot, and a first-party client always sends it. + expected_execution_id: Optional[str] = None + + +class SessionCommandRef(BaseModel): + """The durable command an accepted request created. Identity and delivery state only. + A client must not infer execution state from it (decision D-016).""" + + id: UUID + state: Literal["pending", "claimed", "applied", "obsolete"] + + +class SessionExecutionRef(BaseModel): + """What the caller should render. `id` is null when the session was idle.""" + + id: Optional[str] = None + state: Literal["stopping", "idle"] + + +class SessionCancelResponse(BaseModel): + command: SessionCommandRef + execution: SessionExecutionRef +``` + +Responses: + +| Status | When | Body | +|---|---|---| +| 202 Accepted | An execution was running or parked. The command is durable and on its way | `command.state = "pending"`, `execution.state = "stopping"` | +| 200 OK | Nothing was running and no `expected_execution_id` was sent | `command.state = "obsolete"`, `execution.state = "idle"`, `execution.id = null` | +| 200 OK | The running execution started **after** this request arrived, and no `expected_execution_id` was sent | `command.state = "obsolete"`, `execution.state = "idle"`, `execution.id = null`. The newer execution is not touched. See the stale-Stop guard in section 4 of the design document | +| 409 Conflict | `expected_execution_id` does not name the running execution | `detail: {"message": ..., "current_execution_id": }` | +| 422 | The session id fails the allowlist (`SessionIdInvalid`) | `detail: ` | +| 403 | The caller lacks `RUN_SESSIONS` | `FORBIDDEN_EXCEPTION` | + +The two 200 cases are deliberately indistinguishable to the client. Both mean "there is nothing of +yours left to stop", and a client that needs to know which one it hit is reading the wrong signal: +it should read the session's execution state, not this response. The command row keeps the exact +reason in `outcome` for anyone debugging afterwards. + +202 and not 200 for the accepted case, because the work is not done when the response returns. The +caller learns the outcome from the session's own state, not from this response. **A delivery failure +does not change the status**: the command is inserted and committed before any adapter is called, so +an unreachable runner still yields 202 and the watchdog settles the command. + +Repeating the request with the same `Idempotency-Key` returns the same `command.id` and the same +status. Repeating it without a key also returns the same command while one is still open, because +admission collapses onto an open command for the same target execution. + +New domain exceptions in `api/oss/src/core/sessions/commands/types.py`, mapped by a +`_handle_command_exceptions()` decorator alongside the existing one: + +```python +class SessionCommandError(Exception): ... + +class ExecutionExpectationFailed(SessionCommandError): + """expected_execution_id does not name the running execution.""" + def __init__(self, session_id: str, expected: str, current: Optional[str]): ... +``` + +--- + +## 3. Deferred: claim commands (future long-poll adapter) + +```http +POST /sessions/control/commands/claim +X-Agenta-Runner-Token: + +{ + "replica_id": "runner-7f3c", + "sessions": [ + {"project_id": "1f0a4b2c-0000-4000-8000-000000000002", "session_id": "sess-42"} + ], + "wait_seconds": 25, + "limit": 10 +} +``` + +Not a product API. It is excluded from the public schema with `include_in_schema=False`, the +treatment the admin routers already get (`api/entrypoints/routers.py:1502`). + +Authentication is the shared runner token, not a user credential: the loop belongs to the process +and spans many projects, and a run's credential expires while the process keeps polling. The path +prefix `/sessions/control/` is added to `_PUBLIC_ENDPOINTS` (`api/oss/src/middlewares/auth.py:52`) +so the project-scoped middleware does not reject a request with no user credential, and the route +then compares the presented token to `env.runner.token` in constant time. If that setting is unset +the route answers 503 and serves nothing. Scope comes from the declared `(project_id, session_id)` +pairs and the rows themselves, never from a header. + +```python +class SessionScope(BaseModel): + model_config = ConfigDict(extra="forbid") + + project_id: UUID + session_id: SessionId + + +class SessionControlClaimRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + # Delivery bookkeeping: this becomes `claimed_by` so a settle can be matched to its claim. + # Not routing, and not an address. + replica_id: str = Field(min_length=1, max_length=128) + # The routing input: every session this runner holds warm right now, including sessions + # parked awaiting an approval. Most recently used first. + sessions: List[SessionScope] = Field(min_length=1, max_length=200) + # How long the API may hold this request. Clamped server-side to the configured hold. + wait_seconds: int = Field(default=25, ge=0, le=60) + limit: int = Field(default=10, ge=1, le=50) + + +class SessionCommandTarget(BaseModel): + # Resolved once at admission; the runner aborts only this execution. + turn_id: Optional[str] = None + # What the caller asserted, kept so a 409 stays explainable after the fact. + expected_turn_id: Optional[str] = None + + +class SessionCommandDelivery(BaseModel): + claimed_by: str + claim_expires_at: datetime + attempt: int + + +class SessionCommandEnvelope(BaseModel): + """One command as the runner receives it. Every transport delivers this same shape, + so the runner has one parser, one set of guards and one applier.""" + + id: UUID + project_id: UUID + session_id: str + kind: Literal["cancel"] + target: SessionCommandTarget + delivery: SessionCommandDelivery + # The runner refuses to abort an execution that started after this time. + created_at: datetime + # Absent for `cancel`. Present for the kinds that carry them, so a reader never has to + # interpret an empty object. + input: Optional[SessionCommandInput] = None + policy: Optional[SessionCommandPolicy] = None + + +class SessionControlClaimResponse(BaseModel): + count: int = 0 + commands: List[SessionCommandEnvelope] = Field(default_factory=list) +``` + +Responses: + +| Status | When | +|---|---| +| 200 OK | At least one command was claimed. The body is never an empty list | +| 204 No Content | The hold expired with nothing to deliver | +| 401 Unauthorized | The token is absent or wrong | +| 422 | `sessions` is empty or over the cap | +| 503 Service Unavailable | `AGENTA_RUNNER_TOKEN` is not configured on the API | + +204 rather than an empty 200 keeps the common case cheap and gives the runner an unambiguous "claim +again now" signal. + +--- + +## 4. Internal: report a command's outcome + +Used by **both** adapters. Settlement has one path on every transport. + +```http +POST /sessions/control/commands/{command_id}/outcome +X-Agenta-Runner-Token: + +{ + "replica_id": "runner-7f3c", + "result": "applied", + "execution": { + "id": "0199a3f1-0000-7000-8000-00000000000a", + "state": "stopped" + } +} +``` + +```python +class SessionExecutionOutcome(BaseModel): + model_config = ConfigDict(extra="forbid") + + # The execution the runner acted on. Null when it held none. + id: Optional[str] = None + # stopped: cancelled as asked. not_running: no such execution here. + # superseded_by_newer_turn: the held execution started after the command arrived. + # failed: the cancel itself failed. + state: Literal["stopped", "failed", "not_running", "superseded_by_newer_turn"] + # Short, human-readable, present only when `state` is "failed". + error: Optional[str] = Field(default=None, max_length=2000) + + +class SessionControlOutcomeRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + replica_id: str = Field(min_length=1, max_length=128) + # The command's terminal state. `applied` means the runner did the work; `obsolete` + # means there was nothing to do. + result: Literal["applied", "obsolete"] + execution: SessionExecutionOutcome + + +class SessionCommandSettlement(BaseModel): + id: UUID + state: Literal["applied", "obsolete"] + outcome: Literal["stopped", "not_running", "superseded_by_newer_turn", "failed", "lost"] + settled_at: datetime + + +class SessionControlOutcomeResponse(BaseModel): + command: SessionCommandSettlement +``` + +Responses: + +| Status | When | Body | +|---|---|---| +| 200 OK | The command was `claimed` by this replica and is now settled | The settlement | +| 409 Conflict | The claim expired, or another actor settled the command | The stored settlement, so the runner stops instead of retrying | +| 404 Not Found | No command with that id in any project | `detail` | +| 401, 503 | As for the claim route | | + +The API does the settlement side effects inside the same request: it clears +`session_streams.stopping_turn_id`, tombstones the stopped execution, releases the Redis `running` +key under an owner check, leaves `alive` to its own time to live, cancels that execution's pending +interactions, and publishes the existing `lifecycle: ended` watch notification. The full ordering is +in section 7 of the design document. + +--- + +## 5. Internal: the runner's cancel route (direct-call adapter) + +This is the runner's own HTTP surface, not the API's. It sits beside the existing `POST /kill` +(`services/runner/src/server.ts:704`, verified) and shares its token gate, its capped body reader and +its scoping rule. The API calls it the way `kill_runner_sandbox` already calls `/kill` +(`api/oss/src/core/sessions/streams/runner_client.py:30`, verified). + +```http +POST /cancel +Authorization: Bearer + +{ + "commandId": "0199a3f2-0000-7000-8000-000000000001", + "projectId": "1f0a4b2c-0000-4000-8000-000000000002", + "sessionId": "sess-42", + "targetTurnId": "0199a3f1-0000-7000-8000-00000000000a", + "createdAt": "2026-09-02T22:09:01Z" +} +``` + +`camelCase` because the runner's existing routes use it. `projectId` and `sessionId` are both +required, for the same reason `/kill` requires both: a pool key is always project-scoped, so a +single-tenant scope needs the pair. + +Responses: + +| Status | When | Meaning to the API adapter | +|---|---|---| +| 202 Accepted | The runner holds this session and accepted the command | `accepted`; the outcome will arrive on the outcome route | +| 404 Not Found | The runner does not hold this session | `not_held`; the service settles the command at once | +| 400 | `sessionId` or `projectId` missing | `unreachable`, and a bug to fix | +| 401 | Token mismatch | `unreachable`, and a deployment error to log loudly | + +**The response is an acknowledgement, not an outcome.** The runner reports what happened to the +execution through the outcome route in section 4, so both adapters settle through one path. + +**404 is ambiguous, and the API must disambiguate it.** `not_held` is the honest answer both when the +session really has ended and when the call reached the wrong replica. The API tells them apart with +data it already has: a `not_held` for a session whose row says `is_alive` with a heartbeat younger +than one interval is the wrong-replica failure. It is logged at error level, counted, and settled as +`lost` rather than `not_running`, so the user is told the Stop failed instead of being told the work +had already finished. Section 9 of the design document has the rule and the optional preventive +configuration check. + +**The runner resolves a parked session through the pool, not the execution registry.** A Stop against +a parked approval has no in-flight execution, so `/cancel` falls back to +`SessionPool.awaitingApproval(sessionId)` +(`services/runner/src/engines/sandbox_agent/session-pool.ts:117`, verified) before answering 404. + +--- + +## 6. One field added to an existing contract + +The heartbeat response grows one field. Nothing else about `POST /sessions/streams/heartbeat` +changes. + +```python +class SessionHeartbeatResult(BaseModel): + stream: Optional[SessionStream] = None + replica_id: str + is_current_turn: bool = True + # Commands for THIS session only, claimed by this beat under the same compare-and-set + # the claim route uses. Empty in the normal case. + commands: List[SessionCommandEnvelope] = Field(default_factory=list) +``` + +The field is additive and defaults to an empty list, so a runner build that does not know about it is +unaffected. + +This fallback reaches only a session with a live turn. The heartbeat stops when a turn ends or parks +(`services/runner/src/server.ts:618` and `services/runner/src/sessions/alive.ts:241`, verified), so +it is not the delivery path for a parked session and must not be relied on as one. + +--- + +## 7. What does not change in version one + +- `POST /sessions/streams/` keeps its current four-mode behavior until the last migration step, when + its cancel branch becomes a thin wrapper over the same command. See section 10 of the design + document. +- `DELETE /sessions/streams/` (kill) is untouched. Stop and Delete stay different operations + (decision D-008). +- `POST /sessions/interactions/{interaction_id}/respond` is untouched. Turning interaction responses + into commands is later work, and so is the `continuation` field the architecture review asks for on + its response. +- No new public read route. Clients keep using `GET /sessions/streams/` and the watch stream. +- Steer stays out. The `input` and `policy` groups are reserved in the envelope so it does not need a + breaking change later, but no route accepts them in version one. diff --git a/docs/design/session-control-and-live-events/context.md b/docs/design/session-control-and-live-events/context.md new file mode 100644 index 00000000000..84aa2fc1037 --- /dev/null +++ b/docs/design/session-control-and-live-events/context.md @@ -0,0 +1,64 @@ +# Context + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +## Current user experience + +The browser that sends a message owns the live invoke response. Other clients receive saved +record changes later. Stop uses the session control endpoint, but the runner learns about normal +cancellation through its next heartbeat. Session records use upserts and do not provide a durable +per-session replay cursor. + +These behaviors cause several visible problems: + +- Stop can update the browser while execution continues in the runner. +- A failed or missing terminal signal can leave a session shown as running. +- A second message can race with the active execution and break the session. +- Another browser cannot receive the same live text stream as the sender. +- A reconnecting browser cannot request all durable changes after a stable cursor. +- Approval, cancellation, and resume races can leave an interaction or session unusable. +- A re-sent record can change the apparent reading order because records are mutable upserts. + +## Design scope + +The final design must cover four independent paths: + +1. **Live output:** runner to API to every connected reader. +2. **Durable facts:** append-only session history with stable ordering and replay. +3. **Commands:** client to API to the execution owner, with durable admission where required. +4. **Ownership:** one active execution owner, renewed through a temporary lease. + +The read path and control path can progress in parallel. Stop is not blocked on the live relay. +The live relay is not blocked on the final Stop behavior. + +## Goals + +- Stop reaches active execution promptly and produces one terminal outcome. +- Normal Stop preserves the resumable session and sandbox when the harness supports this. +- Multiple clients receive live frames from the same execution. +- Refreshing or closing the sender does not stop execution. +- Clients can recover durable changes after a cursor. +- A second message has an explicit server-side delivery policy. +- Steer saves the new message before it interrupts current work. +- Approval state remains correct across pause, Stop, refresh, and resume. +- Records and events have stable ordering that retries cannot change. +- Runner failure eventually releases ownership and leaves a terminal durable outcome. + +## Non-goals for the first RFC pass + +- Selecting a new broker before current Redis options are evaluated. +- Storing every token permanently in Postgres. +- Replacing every frontend session view in the first implementation. +- Solving all harness limitations through one common behavior. +- Treating the current issue grouping as a confirmed roadmap priority. + +## Design process + +Each track will follow the same sequence: + +1. Review current behavior and linked failures. +2. Agree on the invariant and user-visible requirement. +3. Compare the high-level options. +4. Record the decision and rejected alternatives. +5. Add the approved design to the living RFC. +6. Define one live-stack test that proves the track. diff --git a/docs/design/session-control-and-live-events/decisions.md b/docs/design/session-control-and-live-events/decisions.md new file mode 100644 index 00000000000..d2f1a57656e --- /dev/null +++ b/docs/design/session-control-and-live-events/decisions.md @@ -0,0 +1,288 @@ +# Decisions + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +## Confirmed process decisions + +### D-001: Start from bugs and system requirements + +**Status:** Confirmed by Mahmoud on 2026-09-02. + +The design starts with the open issue inventory and the requirements the final system must +satisfy. Architecture options must link back to these requirements. + +### D-002: Discuss one track at a time + +**Status:** Confirmed by Mahmoud on 2026-09-02. + +For each track, first present the high-level design and important questions. Record the answers +and decisions in the RFC after discussion. + +### D-003: Keep the read path and control path independent + +**Status:** Confirmed direction on 2026-09-02. + +Shared reading and immediate control touch different directions and can progress in parallel: + +- Read path: runner to API to clients. +- Control path: client to API to runner. + +Stop must not wait for the live relay, replay, or sender-as-reader work to finish. + +### D-004: Preserve live token output in the target experience + +**Status:** Confirmed direction on 2026-09-02. + +Moving readers behind the API must not reduce the sender to paragraph-only updates. The final +system must deliver live frames to every connected reader. + +### D-005: Keep temporary frames separate from permanent facts + +**Status:** Confirmed direction on 2026-09-02. + +Live text fragments can have bounded retention. Completed messages, lifecycle facts, tools, and +interactions require durable recovery. One raw ingress can feed both consumers. + +### D-006: Investigate sandbox-agent cancellation before selecting Stop semantics + +**Status:** Confirmed process decision on 2026-09-02. + +The Stop track starts with a focused sandbox-agent investigation. It must determine whether one +execution can be cancelled while the harness session and sandbox remain resumable. It must also +identify any required vendored patch and Daytona snapshot rebuild. This investigation can proceed +in parallel with the API control-path design. + +### D-007: Use five seconds as the provisional Stop delivery target + +**Status:** Provisional product direction from Mahmoud on 2026-09-02. + +Within five seconds of an accepted Stop request, the active execution must stop starting new model +requests and new tool actions. The exact deadline for terminating an already-running provider or +tool operation remains open until harness and tool cancellation capabilities are verified. + +### D-008: Separate Stop from Delete + +**Status:** Confirmed by Mahmoud on 2026-09-02. + +Stop preserves the session, its history, and its resumable sandbox state. Delete permanently +removes the session and its session-scoped resources. The public interface must not overload one +operation to mean both. + +### D-009: Let first-party and external clients use the same session API + +**Status:** Confirmed direction from Mahmoud on 2026-09-02. + +Desktop, mobile, integrations, and external API consumers should use the same public session +contract. Private API-to-runner delivery remains an implementation detail behind that contract. + +### D-010: Make the expected execution guard optional + +**Status:** Confirmed direction from Mahmoud on 2026-09-02. + +A Cancel request can include `expected_execution_id`. When supplied, the API cancels only that +execution and rejects a stale request. When omitted, the API cancels the session's current active +execution. + +### D-011: Keep queued inputs immutable + +**Status:** Confirmed by Mahmoud on 2026-09-02. + +Clients can view and remove a pending input. They cannot edit or reorder it. To change pending +content, a client removes the old input and submits a replacement. The API rejects removal after +the input has been promoted into active work. + +### D-012: Keep design discussions at the architectural level + +**Status:** Confirmed by Mahmoud on 2026-09-02. + +The discussion focuses on resource boundaries, execution ownership, event flow, recovery, and +user-visible behavior. Routine endpoint naming, status codes, defaults, and validation details use +established API conventions during RFC drafting unless they materially change those properties. + +### D-013: A successful submission means durable acceptance + +**Status:** Confirmed by Mahmoud on 2026-09-02. + +The API confirms a submitted input only after it has durably saved the input, its idempotency +identity, its session, and the intent to execute it. Acceptance does not wait for a runner to claim +the work, the harness to start, or the first output frame. If no runner is available, accepted work +remains queued rather than disappearing. + +### D-014: Do not preserve sender-only live visibility as a requirement + +**Status:** Confirmed by Mahmoud on 2026-09-02. + +The shared session stream is available to authorized session viewers. The design does not treat +raw live output as secret to the browser that started the execution. Existing configured +redaction and authorization behavior must be understood, but sender-only visibility is not a +target product rule. + +### D-015: Add the new session interface beside the current endpoints + +**Status:** Confirmed as a fair first draft by Mahmoud on 2026-09-02. + +The new snapshot and replayable event interface is introduced without changing the meaning of the +current stream and watch endpoints. Desktop and mobile migrate before obsolete endpoints are +deprecated. Final endpoint names remain open for a later interface review. + +### D-016: Separate command delivery state from execution state + +**Status:** Confirmed by Mahmoud on 2026-09-02. + +The internal command lifecycle starts with `pending`, `claimed`, `applied`, and `obsolete`. +Claims are temporary and can expire or retry. An execution terminal outcome is durable and cannot +change. Public clients follow execution states such as `running`, `stopping`, `stopped`, `failed`, +and `lost`; they do not infer execution state from internal delivery acknowledgements. + +Accepting Stop durably saves the command and moves the matching execution from `running` to +`stopping` in one transaction. A runner outcome settles both the execution and the command. A +watchdog settles an execution whose runner disappears, but its timeout remains open until the +sandbox cancellation spike. + +Postgres is the admission-state store for both the durable command row and the execution +projection. The API inserts the command and updates the matching execution in one Postgres +transaction. Redis ownership is not part of that transaction. A crash before commit accepts +nothing. A crash after commit leaves a retryable `pending` command that long polling or heartbeat +discovery can deliver until the runner applies it. + +### D-017: Keep current Redis execution ownership for the first version + +**Status:** Confirmed by Mahmoud on 2026-09-02. + +The first version keeps the existing Redis `alive`, `running`, `owner`, and `superseded` model. +It does not add Postgres execution authority, ownership generations, or full stale-writer fencing. +Those changes have low current value because Agenta operates one runner and does not plan near-term +runner scaling. + +Durable commands and direct API-to-runner delivery are in scope. Stop no longer depends on deleting +ownership and waiting for a heartbeat. The current execution keeps its Redis ownership while +stopping and releases it after cancellation settles. Long polling is deferred behind the same +control-delivery port. + +### D-018: Use runner-initiated HTTP long polling for immediate control + +**Status:** Selected for the milestone 1 implementation on 2026-09-04. + +The runner uses HTTP long polling behind a control-delivery port. Durable commands remain +recoverable across disconnection, and the Stop path does not depend on Redis, WebSockets, or direct +runner routing. Heartbeat command discovery remains the fallback delivery path. + +Redis remains an execution lease and routing hint. Postgres command and execution rows are the +durable recovery source after process or Redis failure. + +## Proposed design decisions + +### P-001: Use one raw runner event ingress + +**Status:** Proposed. Not approved. + +The runner sends raw frames once. A shared Redis Stream can feed both the live relay and a durable +projector. The projector combines raw frames into durable events. The live relay forwards raw or +briefly batched frames without waiting for message completion. + +The current sender response and current persistence path can remain during migration. + +### P-002: Keep ownership heartbeats but remove normal control delivery from them + +**Status:** Proposed. Not approved. + +Heartbeats continue to renew runner ownership and detect failures. Immediate control delivery +handles Stop and Steer. Heartbeat detection remains a fallback when direct delivery fails. + +### P-003: Use append-only durable events for replay + +**Status:** Proposed. Requires an explicit decision reversal or separation from records. + +The existing records specification decided to use UUIDv7 ordering and no stored per-session +sequence. The new replay requirement may need an append-only event log with a per-session cursor. +The design must either reopen the existing decision or introduce a separate event-log concept. + +### P-004: Require one active execution and fence stale writers + +**Status:** Proposed. Direction confirmed, mechanism not approved. + +At most one execution can be active for a session. Admission must be atomic. Each accepted owner +receives an increasing ownership generation, also called a fencing token. Every durable write and +effect-producing command carries that generation. The API rejects a write from an older +generation even if the old runner is still alive. + +Redis heartbeats remain useful for leases and crash detection. A lease alone is not the final +correctness guarantee because it can expire during a network partition while the old runner keeps +working. + +## Open decision gates + +### O-001: Vocabulary + +Settle the meanings of `session`, `conversation turn`, and `execution`. Decide how existing +`turn_id` and `turn_index` map to those terms. + +### O-002: Stop behavior inside sandbox-agent + +Verify whether the vendored sandbox-agent can cancel one execution while preserving its harness +session. Warm resume is the required outcome. If current behavior cannot provide it, define the +required patch and whether Daytona needs a rebuilt snapshot. + +### O-003: Durable ordering + +Choose between: + +- A new append-only durable event log with a per-session sequence. +- Append-only records with a new ordering contract. +- Separate record storage and replay-event storage. + +Do not add a sequence column to mutable upserts and call the result append-only. + +### O-004: Raw live transport + +Choose the Redis Stream layout, retention limit, redaction boundary, and browser fan-out model. + +### O-005: Stable record-ID semantics spike + +Before immutable event insertion is implemented, inventory every runner and backend path that +reuses a `record_id`. Separate exact delivery retries from progressive updates and resume +re-emissions. Add regression tests for the final state of tools, interactions, terminal events, +and harness reconstruction. + +### O-006: Immediate runner control + +**Status:** Resolved for version one on 2026-09-03. + +Use a direct API-to-runner HTTP call through the replaceable control-delivery port. Durable storage +precedes the call, so transport failure costs promptness rather than command correctness. Defer +runner-initiated long polling until multi-runner or user-operated routing requires it. + +### O-007: Command boundary + +Decide which actions enter a general command inbox. The working boundary is execution-affecting +intent: Send, Cancel, interaction response, Queue, and Steer. Attach is a read operation. Kill, +rename, archive, and delete remain explicit resource or lifecycle operations unless discussion +shows a need to change that boundary. + +### O-008: Public resource API versus internal command transport + +Decide whether public callers submit every execution action to one command collection or use +clear resource endpoints that translate into internal commands. The current proposal favors clear +public resources with one internal command envelope. + +### O-009: Public Cancel target + +Choose whether Cancel publicly targets: + +- The current work in a session, with no execution ID. +- A specific execution resource. +- The current work in a session plus `expected_execution_id` as a stale-request guard. + +The selected direction combines the first and third options. Cancel targets the current work in a +session. `expected_execution_id` is an optional stale-request guard supplied by clients that know +the current execution. + +### O-010: Busy-message policy names + +Choose the public names and defaults for a message submitted while work is active. The current +working set is `reject`, `queue`, and `steer` under an `on_busy` field. + +### O-011: Pending input ordering + +Pending inputs remain visible in the session snapshot and event stream. The initial contract uses +server-assigned FIFO order. Clients cannot edit or reorder queued inputs. diff --git a/docs/design/session-control-and-live-events/plan.md b/docs/design/session-control-and-live-events/plan.md new file mode 100644 index 00000000000..3882fb0e410 --- /dev/null +++ b/docs/design/session-control-and-live-events/plan.md @@ -0,0 +1,86 @@ +# Design plan + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +## Parallel programs + +The work has three parallel programs. The order below is a discussion order, not a requirement +that one program finish before another starts. + +### Program A: Immediate control + +1. Sandbox-agent cancellation capability and Daytona rebuild impact. +2. Ownership and execution identity. +3. Immediate Stop delivery. +4. Stop settlement and sandbox preservation. +5. Approval and Stop races. + +### Program B: Shared reading + +1. Raw live-frame ingress. +2. Multi-client live relay. +3. Explicit execution lifecycle facts. +4. Append-only durable ordering and replay. +5. Sender becomes an ordinary reader. + +### Program C: Durable input + +1. Durable command admission. +2. Second-message policies: reject, queue, and steer. +3. Approval responses as commands. +4. Steer settlement and promotion. + +## Cross-cutting foundations + +These topics apply to all three programs: + +- Vocabulary and identifier ownership. +- Harness capability reporting. +- Authentication and authorization. +- Redaction and temporary-frame retention. +- Idempotency and duplicate delivery. +- Live-stack tests and failure injection. + +## Proposed discussion order + +The first two discussions can happen in parallel. + +1. **Stop and ownership:** current lease, immediate signal options, sandbox-agent dependency, + terminal settlement, and watchdog behavior. +2. **Live frames:** one raw ingress, Redis Stream layout, multi-client fan-out, and temporary + recovery. +3. **Durable ordering:** append-only event model, cursor allocation, snapshot boundary, and the + conflict with the existing UUIDv7 record-order decision. +4. **Sender detachment:** command acceptance, execution lifetime, and making the sender a reader. +5. **Durable commands:** command states, delivery, retries, and owner routing. +6. **Queue and Steer:** second-message policy, promotion order, interruption boundary, and + interaction races. +7. **Shared client engine:** desktop and mobile state application after the server contracts are + stable. + +Before finalizing the command contract, review the proposed public interface as a whole. The +review must distinguish user-facing resource endpoints from the private command transport used to +reach runners. + +## Definition of a completed track + +Each track must contain: + +- One user problem. +- One invariant. +- One interface or state transition contract. +- The main rejected alternatives. +- One live-stack test that proves the invariant. +- Known harness or deployment limitations. + +## Initial parallel investigation + +The sandbox-agent investigation starts before the Stop interface is fixed. It must answer: + +1. Which protocol request currently ends a prompt or execution? +2. Does that request also close the harness session? +3. Can Pi and Claude Code resume the same native session after cancellation? +4. Does the runner destroy or park the sandbox on each cancellation path? +5. Which source repository owns the required change? +6. Does Daytona need a new snapshot, and how is that snapshot version deployed? +7. What automated test proves cancel followed by warm resume? diff --git a/docs/design/session-control-and-live-events/records-invariants.md b/docs/design/session-control-and-live-events/records-invariants.md new file mode 100644 index 00000000000..fcb000d4569 --- /dev/null +++ b/docs/design/session-control-and-live-events/records-invariants.md @@ -0,0 +1,239 @@ +# Record properties and current violations + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +This note evaluates whether the existing `records` model can become the replayable session event +history. It does not select a storage option. + +## What records do today + +Records are a durable conversation representation. The runner sends raw `AgentEvent` values to +the live response, but coalesces text and thought deltas before record ingest. Tool-family events +can use deterministic IDs. The API publishes records into a dedicated Redis Stream. A worker +writes them into the tracing Postgres database. The frontend fetches the full record collection +and reconstructs `UIMessage[]`. + +Records are therefore closer to a durable transcript projection than a raw transport log. + +## Properties required for the current transcript + +The current transcript and harness reconstruction need these properties: + +1. **Durability.** An acknowledged durable fact survives client, API, runner, and worker restarts + within the configured retention period. +2. **Complete produced order.** Reads preserve the causal order of user messages, assistant + messages, tools, interactions, and terminal markers. +3. **Idempotent retry.** Retrying one logical fact does not create a duplicate or change its place + in history. +4. **Stable correlation.** Messages, tools, interactions, turns, and executions keep stable IDs so + later facts can refer to earlier ones. +5. **Detectable incompleteness.** If retention, truncation, quota, or delivery failure prevents + complete reconstruction, the system reports that condition instead of silently replaying a + partial conversation. +6. **Client independence.** Persistence does not depend on a browser connection. + +## Additional properties required for cursor replay + +A `snapshot + events after cursor` interface adds these requirements: + +1. **Immutable history.** Once a durable event is visible at a cursor, its payload and position do + not change. +2. **Monotonic commit order.** Every committed event gets an order that only moves forward. A + cursor can request all later events without scanning or comparing timestamps. +3. **Atomic visibility.** An event becomes replayable only after its durable write commits. +4. **Replay-to-live handoff.** A reader cannot miss an event between reading history and joining + the live tail. +5. **Stable event identity.** A producer retry maps to the same logical event and does not create a + second cursor entry. + +The sequence does not need to be dense or start at one for each session. It only needs to be +strictly increasing and stable. Gaps are harmless. A plain table-global Postgres sequence is not +enough by itself: Postgres allocates sequence values before commit, so transaction 102 can commit +and become visible before transaction 101. A client that advances to 102 could then miss the late +commit of 101. The write path must preserve commit visibility order, use a committed watermark, or +serialize sequence assignment and commit for each session. + +## Current violations + +### Some rows are mutable + +The primary key is `(project_id, record_id)`. `append` and `append_many` use +`ON CONFLICT DO UPDATE`. A conflict overwrites: + +- `record_type` +- `record_source` +- `timestamp` +- `attributes` +- `turn_id` +- `span_id` + +The runner supplies deterministic UUIDv5 IDs for `tool_call`, `tool_result`, +`interaction_request`, and `interaction_response` families. The stable ID lets repeated snapshots +or retries target one row. The DAO deliberately keeps the last payload. + +This supports a latest-state model. It violates immutable event history. + +### Record IDs do not encode order + +The design document proposed UUIDv7 IDs, but the implementation does not use them: + +- Tool-family records use deterministic UUIDv5 IDs. +- Other records receive backend-generated UUIDv4 IDs. + +UUIDv4 and UUIDv5 values are not time ordered. A client cannot use `record_id` as an `after` +cursor. + +### Current read order is reconstructed from three fields + +The DAO orders records by: + +1. Producer `timestamp`. +2. Database `created_at`. +3. Per-turn `record_index`. + +`record_index` restarts at zero for each execution. `created_at` can be shared by records in one +worker batch. Producer timestamps have clock and resolution limits. The composite order is useful +for transcript rendering, but it is not a stable cursor. + +An upsert also overwrites `timestamp`. A retry or later snapshot can therefore move an existing +row to a different place in the read order. + +### Retry identity is inconsistent + +Tool-family records have deterministic IDs and upsert on retry. Most message, thought, usage, +error, and terminal records omit `record_id`; the API mints a new UUIDv4 for every ingest. + +If Redis accepted the first request but the HTTP response was lost, a runner retry without a +stable ID can create a duplicate durable row. The system therefore uses idempotent retry for some +record types but not all record types. + +### Worker failures can acknowledge unwritten records + +The records worker adds every successfully decoded Redis message ID to `processed_ids` before it +attempts the Postgres batch write. If `append_many` fails, the worker logs the failure and +continues. It still returns those IDs to the shared consumer loop, which acknowledges and deletes +them from Redis. + +This is not an inherent Redis Streams limitation. It is an acknowledgement bookkeeping defect. + +### Runner delivery is bounded and can drop + +The runner retries record ingest a bounded number of times. After the limit, it records an +in-memory failure count and drops the record. The turn-end drain can mark reconstruction unsafe in +that runner process, but the missing fact never reaches the durable history. + +Bounded retry prevents an unavailable API from hanging execution forever. Permanent silent loss +is not required by that constraint. Accepted inputs and terminal outcomes need a recoverable +delivery source outside one runner process. + +### Retention, quotas, and truncation intentionally limit completeness + +Records live in the tracing database and have their own retention policy. Attributes larger than +64 KB are truncated before Redis ingest. Enterprise quota rejection can also skip a batch. + +These are real product and operational constraints. Any design that uses records for session +reconstruction or event replay must define what happens after retention, truncation, or quota +loss. Calling the collection complete without marking these conditions would be incorrect. + +## Structural reasons behind the current design + +### Coalescing is structurally useful + +Persisting every token permanently would increase write volume and storage significantly. The +durable transcript needs completed messages, not every typing-animation fragment. Coalescing raw +text into a completed message is compatible with an append-only durable log. + +### Retries and deduplication are structurally required + +Network and worker delivery is at least once. Stable event IDs and duplicate handling are +required. Mutating an existing row is not required. A final immutable fact can use +`ON CONFLICT DO NOTHING` after every durable event receives a stable producer ID. + +### Progressive tool snapshots do not require mutable durable history + +A live tool call can publish several argument snapshots. Those snapshots can remain temporary. +The durable model can append distinct facts such as `tool.started` and `tool.completed`, or append +one final `tool_call` fact. Reusing one ID and replacing its payload is a chosen projection model, +not a storage necessity. + +### The current stable-ID behavior needs a spike before immutability work + +The same `record_id` can currently mean two different things: + +1. **Delivery retry.** The producer sends the same logical fact and payload again because it did + not receive an acknowledgement. The second insert should be an idempotent no-op. +2. **Progressive update.** The producer sends a later payload for the same logical object. Treating + this as a duplicate no-op would discard the later state and can cause a regression. + +Current runner tests deliberately reuse stable IDs for repeated `tool_result` and +`interaction_response` events. Tool-call argument snapshots share an identity but are currently +coalesced into one final persisted record. The records DAO also documents later snapshots that +replace earlier payloads. These behaviors must be reconciled before changing upserts to immutable +inserts. + +The implementation plan therefore requires a producer-semantics spike. It must inventory every +stable-ID producer, retry path, resume path, and progressive-update path. For each case, it must +classify the repeated write as one of: + +- an identical retry that becomes a no-op; +- a temporary live update that stays outside durable history; +- a new durable fact that receives a new event ID and refers to the same stable tool, message, or + interaction ID. + +The spike is complete when tests cover each classified case and prove that immutable insertion +does not lose a final tool result, interaction response, terminal outcome, or reconstructed +conversation state. Immutable storage changes must not start before this gate passes. + +### A dense per-session counter is not required, but commit order is required + +The earlier design rejected a dense per-session sequence because concurrent writers would need a +counter row, lock, or serializable retry. Cursor replay does not need dense per-session numbers, +but it must not expose a higher cursor while a lower event can still commit later. A plain global +sequence does not provide that guarantee. Viable designs include a per-session transactional +counter and lock, one ordered projector per partition with session affinity, or a separate +committed watermark protocol. The final choice must match the expected write volume. + +### The asynchronous Redis worker is structurally useful + +Redis decouples runner latency from Postgres latency and absorbs bursts. It does not require the +worker to acknowledge failed database writes. Only successfully committed message IDs should be +acknowledged. + +### Retention remains a real constraint + +If session history must outlive tracing retention, the existing records location cannot meet that +requirement without changing retention or storage. If session history follows record retention, +the tracing database remains viable. This is a product decision, not an ordering limitation. + +## Changes that could make records satisfy the properties + +The existing records model could become an append-only replay source if it changes as follows: + +1. Give every durable logical event a producer-generated stable `event_id` before its first send. +2. Make durable inserts immutable. Duplicate `event_id` writes become no-ops or verified identical + duplicates. +3. Add a monotonic cursor whose visibility order matches commit order. Do not use a plain database + sequence without solving out-of-order commits. +4. Keep temporary deltas and progressive snapshots outside permanent records. Append only durable + starts, completions, interaction changes, and execution lifecycle facts. +5. Preserve stable message, tool, interaction, and execution IDs inside event payloads. +6. Acknowledge Redis messages only after their Postgres transaction commits. +7. Store or recover unacknowledged runner output across runner loss for required durable facts. +8. Mark a session history incomplete when truncation, quota, retention, or unrecoverable delivery + loss creates a gap. A replay reader must reject any record whose attributes contain + `_truncated`; it must not pass the partial `text`, `input`, or `output` to reconstruction. +9. Register the live wake-up before reading history so replay-to-live handoff cannot miss a commit. + +These changes are substantial, but there is no proven ordering or retry constraint that forces a +separate event table. The separate-table option must instead justify itself through schema scope, +retention, migration risk, or the desire to keep transcript projections distinct from lifecycle +events. + +## Questions to answer before comparing storage options + +1. Must durable session history outlive tracing-record retention? +2. Should records contain all session lifecycle facts, or only conversation facts? +3. Is the existing records API an internal projection, a public event contract, or both? +4. Can we migrate current upsert rows to immutable events without breaking harness reconstruction? +5. Which cursor assignment method preserves commit order at the expected operational scale? +6. Which durable facts must survive a runner crash before they reach Redis? diff --git a/docs/design/session-control-and-live-events/requirements.md b/docs/design/session-control-and-live-events/requirements.md new file mode 100644 index 00000000000..e0a523117e9 --- /dev/null +++ b/docs/design/session-control-and-live-events/requirements.md @@ -0,0 +1,181 @@ +# Bugs and system requirements + +> AGENT-GENERATED, low weight. Draft for discussion. Issue text is observation. Requirements are +> proposed interpretations until Mahmoud confirms them. + +## Stop and hung executions + +Issues: [#5160](https://github.com/Agenta-AI/agenta/issues/5160), +[#5982](https://github.com/Agenta-AI/agenta/issues/5982), +[#6418](https://github.com/Agenta-AI/agenta/issues/6418), +[#6100](https://github.com/Agenta-AI/agenta/issues/6100), +[#6449](https://github.com/Agenta-AI/agenta/issues/6449), +[#6099](https://github.com/Agenta-AI/agenta/issues/6099), +[#6420](https://github.com/Agenta-AI/agenta/issues/6420), +[#6327](https://github.com/Agenta-AI/agenta/issues/6327), +[#5788](https://github.com/Agenta-AI/agenta/issues/5788), +[#6102](https://github.com/Agenta-AI/agenta/issues/6102), +[#6103](https://github.com/Agenta-AI/agenta/issues/6103), +[#6084](https://github.com/Agenta-AI/agenta/issues/6084), +[#5356](https://github.com/Agenta-AI/agenta/issues/5356), +[#5327](https://github.com/Agenta-AI/agenta/issues/5327), +[#6441](https://github.com/Agenta-AI/agenta/issues/6441), +[#6313](https://github.com/Agenta-AI/agenta/issues/6313). + +Observed examples: + +> “After clicking Stop, the UI reflects the stop action immediately, but backend processing +> continues for several minutes.” ([#5160](https://github.com/Agenta-AI/agenta/issues/5160)) + +> “The turn hangs forever. `runTurn` never resolves, the alive watchdog keeps heartbeating +> `running=true`.” ([#6418](https://github.com/Agenta-AI/agenta/issues/6418)) + +Draft requirements: + +- Normal Stop reaches the active runner within a defined short deadline. +- Every accepted execution reaches exactly one durable terminal outcome. +- The sender and every other reader see the same terminal outcome. +- Runner, sandbox, provider, tool, and adapter failures cannot leave an unbounded running state. +- Normal Stop preserves the session workspace and leaves the harness session warm and resumable. +- A watchdog settles work when the owning runner cannot produce the terminal outcome. +- A slow tool fails with an explicit tool or execution result. It does not disappear silently. + +## Steer and concurrent sends + +Issues: [#6417](https://github.com/Agenta-AI/agenta/issues/6417), +[#6020](https://github.com/Agenta-AI/agenta/issues/6020), +[#5790](https://github.com/Agenta-AI/agenta/issues/5790), +[#5539](https://github.com/Agenta-AI/agenta/issues/5539), +[#5538](https://github.com/Agenta-AI/agenta/issues/5538). + +Observed examples: + +> “I expect the platform to queue the message, or to refuse it with a clear signal. Instead both +> turns die and the session refuses every message for 30 minutes.” +> ([#6417](https://github.com/Agenta-AI/agenta/issues/6417)) + +> “The steering turn itself fails with an error and an empty reply, and every turn I send on that +> session afterwards fails the same way.” ([#6020](https://github.com/Agenta-AI/agenta/issues/6020)) + +Draft requirements: + +- At most one execution is active for a session at one time. +- Only the current execution ownership generation can append events or cause external effects. +- A second message uses an explicit `reject`, `queue`, or `steer` policy. +- The API saves an accepted queue or steer message before interrupting current work. +- The API resolves every execution-affecting command to one execution before delivery. +- Public Stop can optionally name the execution the caller expects. If omitted, it targets the + current execution. +- An older runner cannot reclaim ownership or write after replacement. +- A failed steer leaves the saved message visible and recoverable. + +## Reattach and multiple readers + +Issues: [#5609](https://github.com/Agenta-AI/agenta/issues/5609), +[#5542](https://github.com/Agenta-AI/agenta/issues/5542), +[#6404](https://github.com/Agenta-AI/agenta/issues/6404), +[#5611](https://github.com/Agenta-AI/agenta/issues/5611), +[#5443](https://github.com/Agenta-AI/agenta/issues/5443), +[#5384](https://github.com/Agenta-AI/agenta/issues/5384), +[#6397](https://github.com/Agenta-AI/agenta/issues/6397), +[#5990](https://github.com/Agenta-AI/agenta/issues/5990), +[#6388](https://github.com/Agenta-AI/agenta/issues/6388), +[#6468](https://github.com/Agenta-AI/agenta/issues/6468), +[#5950](https://github.com/Agenta-AI/agenta/issues/5950). + +Observed examples: + +> “A tab that never regains focus misses a run started in another browser.” +> ([#5609](https://github.com/Agenta-AI/agenta/issues/5609)) + +> “Reload the page. After the reload: The approval card is gone entirely.” +> ([#5542](https://github.com/Agenta-AI/agenta/issues/5542)) + +Draft requirements: + +- Every authorized client can follow one execution concurrently. +- Every connected client receives live frames, not only completed messages. +- Refresh, navigation, and sender disconnection do not stop the execution. +- A snapshot declares the durable event cursor it represents. +- A reader can replay durable events after that cursor and then follow new events. +- Missed temporary frames are repaired by the next durable checkpoint. +- Pending interactions remain visible and actionable after reload. +- Session identity is stable in URLs and across client caches. + +## Record durability and ordering + +Issues: [#5496](https://github.com/Agenta-AI/agenta/issues/5496), +[#5594](https://github.com/Agenta-AI/agenta/issues/5594). + +Observed examples: + +> “The session-records pipeline loses records permanently in three separate ways, and reports +> success while doing it.” ([#5496](https://github.com/Agenta-AI/agenta/issues/5496)) + +> “The records worker rejects the whole batch.” +> ([#5594](https://github.com/Agenta-AI/agenta/issues/5594)) + +Draft requirements: + +- A successful ingest acknowledgment has a precise durability meaning. +- One bad record cannot silently discard unrelated records in the same batch. +- Retries are idempotent and cannot change established event order. +- Durable replay uses append-only facts with a stable cursor. +- A detected persistence gap marks the session history incomplete. +- The runner drains required durable writes before terminal settlement. + +## Approvals and pauses + +Issues: [#6315](https://github.com/Agenta-AI/agenta/issues/6315), +[#6316](https://github.com/Agenta-AI/agenta/issues/6316), +[#6106](https://github.com/Agenta-AI/agenta/issues/6106), +[#5907](https://github.com/Agenta-AI/agenta/issues/5907), +[#5592](https://github.com/Agenta-AI/agenta/issues/5592), +[#5638](https://github.com/Agenta-AI/agenta/issues/5638), +[#5545](https://github.com/Agenta-AI/agenta/issues/5545), +[#5097](https://github.com/Agenta-AI/agenta/issues/5097). + +Observed examples: + +> “The playground keeps rendering an actionable card whose buttons do nothing.” +> ([#6315](https://github.com/Agenta-AI/agenta/issues/6315)) + +> “When I answer a parked approval and the resumed run fails to start, the approval is gone.” +> ([#5592](https://github.com/Agenta-AI/agenta/issues/5592)) + +Draft requirements: + +- An interaction has one visible state: pending, resolved, denied, or cancelled. +- Stop cancels pending interactions for the stopped execution. +- A late answer cannot resume a cancelled or replaced execution. +- An answer is not consumed until its continuation has a recoverable outcome. +- Side-effecting tools do not run twice after pause and resume. +- One user-visible conversation turn remains traceable across approval resumes. + +## Session list and identity + +Issues: [#6419](https://github.com/Agenta-AI/agenta/issues/6419), +[#6463](https://github.com/Agenta-AI/agenta/issues/6463), +[#5969](https://github.com/Agenta-AI/agenta/issues/5969), +[#6457](https://github.com/Agenta-AI/agenta/issues/6457), +[#6031](https://github.com/Agenta-AI/agenta/issues/6031), +[#6214](https://github.com/Agenta-AI/agenta/issues/6214). + +Observed example: + +> “The session rail shows a session titled with my message, and the conversation is empty.” +> ([#6419](https://github.com/Agenta-AI/agenta/issues/6419)) + +Draft requirements: + +- A user message accepted by the API is never lost when execution fails to start. +- A visible session has an explicit origin and owner type. +- Session list updates converge without requiring a full page reload. +- Rename and archive operations have observable success or failure. +- Session identity does not depend on the browser that created it. + +## Requirement status + +This file does not yet state priority or implementation order. Some issues may share a cause, and +some may fall outside the final RFC. Each design-track discussion must confirm which requirements +it owns and which linked issues it expects to close. diff --git a/docs/design/session-control-and-live-events/research.md b/docs/design/session-control-and-live-events/research.md new file mode 100644 index 00000000000..9abe25ff861 --- /dev/null +++ b/docs/design/session-control-and-live-events/research.md @@ -0,0 +1,202 @@ +# Research notes + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +## Verified current behavior + +### Normal message delivery + +The desktop sends messages through the workflow invoke transport. The response carries the live +event stream for that sender. The desktop Send path does not yet use the session command endpoint. + +### Normal Stop + +The desktop aborts its local response, then posts to `/sessions/streams/` with `session_id`, no +inputs, and `force=false`. The API classifies this as Cancel. It marks the current Redis turn owner +as superseded and clears the `alive` and `running` keys. The runner learns that it lost ownership +when a heartbeat returns `is_current_turn=false`, then aborts locally. + +### Hard kill + +`DELETE /sessions/streams/?session_id=...` is separate from normal Cancel. It contacts the runner +and tears down the sandbox. The session remains resumable after Cancel but not after Kill. + +The v1 direct client uses one configured `runner.internal_url`. It is correct for a single runner, +or when the URL fronts an owner-aware router. Redis separately stores the logical owner +`replica_id`, but the direct client does not resolve that identity to a replica-specific address. +A request that reaches the wrong replica returns not found and must not be treated as success. + +Immediate Cancel remains durable, so an unavailable owner can recover and apply it later or be +settled as lost. Kill is best effort through the same configured URL. Until owner-aware forwarding +exists, a multi-runner Kill cannot guarantee immediate teardown; authoritative session state is +cleared and sandbox lease or orphan cleanup provides the fallback. + +### Heartbeat + +The runner posts `session_id`, `replica_id`, `turn_id`, and `is_running` to +`/sessions/streams/heartbeat`. The heartbeat renews temporary ownership, mirrors liveness to the +session row, and currently carries the delayed cancellation result back to the runner. + +### Records + +The runner forwards raw events to the sender and performs message and tool coalescing before +durable ingest. Durable records travel through a Redis Stream and worker into Postgres. Record +writes use upsert behavior. + +### Watch relay + +The current SSE watch endpoint relays change notifications through Redis Pub/Sub. A reader then +refetches durable records. It does not relay raw tokens and cannot replay missed Pub/Sub messages. + +## Existing design decision that must be revisited + +`docs/designs/sessions/records/specs.md` states: + +> Ordering = uuid7 `id`, no stored `seq`. + +The same document describes records as append-only, but current implementation uses stable record +IDs and upserts. A retry can therefore update an existing row. The RFC must define whether replay +uses a new append-only event log or changes the record model. + +## Dependency to verify early + +Another design review reports that the vendored sandbox-agent cannot cancel an execution while +preserving the harness session, and that a patch would require a Daytona snapshot rebuild. This +has not yet been verified in this workspace. It is the first research task for the Stop track. + +## Current command endpoint is not a durable command system + +`POST /sessions/streams/` derives four modes from the presence of inputs and the `force` flag: + +| Inputs | `force` | Derived mode | +|---|---:|---| +| Present | `false` | Send | +| Present | `true` | Steer | +| Absent | `false` | Cancel | +| Absent | `true` | Attach | + +The endpoint edits Redis coordination state and the session stream row. Its own DTO states that it +runs nothing. Normal desktop Send still uses the workflow invoke path. Desktop Stop uses the Cancel +mode. Attach acquires watcher bookkeeping but does not deliver live frames. Interaction responses +use their own endpoint and worker path. Kill uses `DELETE /sessions/streams/`. + +This means the current endpoint does not provide a durable inbox, command status, retry handling, +or a single route for all execution-affecting actions. + +## Current interaction response path + +The frontend calls `POST /sessions/interactions/{interaction_id}/respond`. The API checks that the +interaction is pending and atomically changes it to `responded`. The winning responder enqueues a +TaskIQ job. The interaction dispatcher reconstructs the resume conversation from durable records +and calls the workflow invoke service in detached mode. Approval response therefore already uses a +resource-specific public endpoint followed by an internal invoke. + +## Current runner routing information + +Redis stores a logical `replica_id` for the runner that owns a session. The API hard-kill client +does not resolve this identifier to an address. It calls one configured runner service URL with +`project_id` and `session_id`. A normal load-balanced request is not sufficient when only one +replica holds the live sandbox, unless the runner service provides its own owner routing. + +## Existing design references + +- `docs/design/agent-workflows/projects/sessions-takeover/architecture.md` +- `docs/design/agent-workflows/projects/sessions-takeover/opencode-comparison.md` +- `docs/design/agenta-mobile/plans/2026-07-27-m3-live-relay.md` +- `docs/design/agenta-mobile/plans/2026-07-27-mobile-approvals-steering.md` +- `docs/designs/sessions/records/specs.md` +- `docs/designs/sessions/interactions/specs.md` + +## Public API comparison + +This comparison uses public vendor documentation. It describes interface shapes, not internal +implementations. + +### Gumloop + +Gumloop models one workflow execution as a run: + +- `POST /api/v1/start_pipeline` starts work and returns `run_id`. +- `GET /api/v1/get_pl_run?run_id=...` returns the run state, logs, and outputs. +- `POST /api/v1/kill_pipeline` accepts `run_id` and stops that run. + +The kill operation is a POST. It does not delete the workflow definition. The caller uses the +`run_id` returned by the start request. + +Sources: + +- https://docs.gumloop.com/api-reference/running-an-automation/start-automation +- https://docs.gumloop.com/api-reference/running-an-automation/retrieve-run-details +- https://docs.gumloop.com/api-reference/running-an-automation/kill-automation + +### OpenAI Responses background mode + +OpenAI models one background execution as a Response: + +- `POST /v1/responses` with `background: true` starts work and returns a Response with an ID. +- `GET /v1/responses/{response_id}` retrieves its current state and result. +- `POST /v1/responses/{response_id}/cancel` cancels it. Repeating Cancel is idempotent. +- Creating with both `background: true` and `stream: true` provides live events. A disconnected + reader can reconnect with `starting_after=`. + +This is the closest public example to the target read model. Execution continues independently +of the first stream. The same response ID identifies retrieval, cancellation, and resumed +streaming. + +Source: https://developers.openai.com/api/docs/guides/background + +### Claude Managed Agents + +Claude Managed Agents models control as events sent to a persistent session: + +- A `user.message` event starts or continues work. +- A `user.interrupt` event stops current work. +- Sending `user.interrupt` followed by `user.message` redirects the session. +- `GET /v1/sessions/{session_id}/events/stream` provides session events. Optional delta events + provide live text previews. Buffered message events remain authoritative. +- Tool confirmation is another event, `user.tool_confirmation`, tied to the pending tool event ID. +- Deleting a session is separate. Deletion permanently removes its events and sandbox. + +The public interrupt targets a session. The service resolves which internal execution must stop. +Claude also documents that model output can stop immediately while an active tool can take longer. + +Sources: + +- https://platform.claude.com/docs/en/managed-agents/events-and-streaming +- https://platform.claude.com/docs/en/managed-agents/session-operations + +## Findings from the public comparison + +The three interfaces use different names, but they agree on four points: + +1. Starting work returns or uses a stable public identifier. +2. Reading status is separate from stopping work. +3. Stop is an action. It does not mean deleting the session or workflow. +4. Deletion remains a separate destructive operation. + +They differ on the Stop target: + +- Gumloop and OpenAI target a specific execution ID. +- Claude targets the session and lets the service interrupt its current work. + +Agenta can support both safety and convenience. The browser can send a session-scoped Cancel with +an `expected_execution_id` prefilled from state. A human never types the execution ID. The API +rejects the Cancel if that execution already ended and another one started. + +## Public queue visibility + +The reviewed Gumloop public API exposes a run state of `QUEUED`, but its documented run API does +not expose an editable per-session message queue. It starts runs, retrieves run state, and kills a +run. This is a workflow-run queue rather than a conversation input queue. + +OpenAI background Responses expose a `queued` execution status and allow cancellation. The public +background-mode documentation does not expose editing or reordering queued conversation inputs. + +Claude Managed Agents comes closer to a conversation inbox. User events are persisted in order. +Each event has `processed_at=null` while it waits behind earlier events, and past events can be +listed. The reviewed documentation does not describe patching or reordering an already-sent user +event. + +The proposed Agenta pending-input API therefore goes beyond these reviewed public interfaces. It +addresses a product-specific need: Queue currently exists in browser state, and multiple clients +need one visible shared copy. The initial design keeps queued inputs immutable. diff --git a/docs/design/session-control-and-live-events/rfc.md b/docs/design/session-control-and-live-events/rfc.md new file mode 100644 index 00000000000..223fd5ad766 --- /dev/null +++ b/docs/design/session-control-and-live-events/rfc.md @@ -0,0 +1,484 @@ +# RFC: Session control and live events + +> AGENT-GENERATED, low weight. Draft for discussion. No architecture is approved yet. + +## Status + +Pre-design. The problem inventory and process decisions exist. Technical sections will be written +after each design-track discussion. + +## Problem statement + +Agenta currently couples live output to the sending request, uses a heartbeat response as the +normal cancellation signal, and lacks an append-only replay cursor for session changes. This makes +multi-client reading, fast Stop, durable queueing, and reliable reconnect difficult to compose. + +## Required properties + +See [Requirements](requirements.md). The RFC will include only requirements confirmed during the +track discussions. + +## Proposed architecture + +Pending discussion. + +### Public interface boundary + +The working public interface separates four operations. Every route in this section is a proposed +new public contract, not a description of an existing endpoint and not yet an approved API. + +1. Send intent to a session. +2. Read the current session snapshot. +3. Follow session changes. +4. Delete a session permanently. + +The proposal does not require one generic public command endpoint. Clear resource-specific +endpoints can all feed one private command-delivery mechanism. + +Create or send session work: + +```http +POST /sessions/{session_id}/commands +Idempotency-Key: + +{ + "type": "send", + "message": "Explain this failure", + "on_busy": "reject" +} +``` + +Stop the current execution, but only if it is still the execution the caller observed: + +```http +POST /sessions/{session_id}/cancel + +{ + "expected_execution_id": "execution-12" +} +``` + +The browser learns `execution-12` from the session snapshot or the `execution.started` event. The +person pressing Stop never enters it. This field prevents a delayed Stop request from cancelling +new work that started after the button was pressed. The field is optional. Without it, the API +uses Redis arrival and turn-start timestamps and refuses the request if the active execution began +after the request arrived. A client that needs unconditional session-scoped cancellation must use +a future command contract. + +Respond to an interaction through a resource-specific public endpoint: + +```http +POST /sessions/{session_id}/interactions/{interaction_id}/responses + +{ + "answer": {"approved": true}, + "expected_execution_id": "execution-12" +} +``` + +The API can translate the response into the same internal command envelope used by Send, Cancel, +Queue, and Steer. The public caller does not need to understand internal runner routing. + +Read current state. This is an ordinary query, not an event endpoint: + +```http +GET /sessions/{session_id} +``` + +This is not the current `GET /sessions/streams/?session_id=...`, which returns only coordination +and liveness data. + +Follow durable events and live frames: + +```http +GET /sessions/{session_id}/events?after= +Accept: text/event-stream +``` + +This is not the current `GET /sessions/streams/watch`, which sends change notifications and asks +the client to refetch records. The proposed endpoint sends replayable session events and then live +events. + +Rename, archive, delete, and hard termination remain explicit session resource or lifecycle +operations. Attach is replaced by reading the snapshot and event stream. + +### Current and proposed public behavior + +| Operation | Today | Proposed direction | Change | +|---|---|---|---| +| Send | Invoke a workflow and read its response stream | Keep this during migration. Later accept work independently and return an execution ID | Later change | +| Stop | `POST /sessions/streams/` with no inputs and `force=false` | `POST /sessions/{id}/cancel` with an optional expected execution ID | Clearer endpoint and faster delivery | +| Hard kill | `DELETE /sessions/streams/?session_id=...`; destroys the sandbox | Keep as a separate destructive operation with an explicit name | Rename or reshape only | +| Answer approval | `POST /sessions/interactions/{interaction_id}/respond` | Keep a resource-specific response endpoint. Improve acknowledgement and resume guarantees internally | Public shape mostly unchanged | +| Queue while busy | Browser-local queue | Save the message on the server with `on_busy: queue` | Changes ownership from browser to server | +| Steer while busy | Ambiguous `force=true` coordination mode; normal send still uses invoke | Save the message, request interruption, then start the saved message | Behavior becomes explicit and durable | +| Attach | `force=true` without inputs records watcher state but does not provide live output | Remove the command. Load a snapshot, then follow events | Replaced by read operations | +| Load current state | Several queries for records, liveness, and pending interactions | One versioned session snapshot, or a documented composition of existing queries | Open design choice | +| Follow changes | SSE sends change notifications; the browser refetches records | Replay events after a cursor, then continue with live frames | Changes from invalidation to replay plus live tail | +| Delete | Separate destructive behavior exists through the stream API | Explicit session deletion after work is stopped | Public naming changes | + +### Busy-message policies + +The words `reject`, `queue`, and `steer` apply only when a new user message arrives while an +execution is already running: + +- `reject`: return a conflict response. Do not save or start the new message. +- `queue`: save the new message. Start it after current work stops normally. +- `steer`: save the new message. Interrupt current work, then start the new message. + +When the session is idle, all accepted messages start normally. The contract calls this field +`on_busy` so its purpose is clear. + +### Visible pending messages + +Once Queue moves from the browser to the server, every client must be able to see the same pending +messages. A session snapshot can include them: + +```json +{ + "pending_inputs": [ + { + "id": "input-24", + "type": "user_message", + "content": "Then check the database", + "position": 1, + "status": "pending" + } + ] +} +``` + +The event stream announces changes: + +```text +input.queued +input.removed +input.promoted +``` + +Queued inputs are immutable. The management interface only needs removal: + +```http +DELETE /sessions/{session_id}/inputs/{input_id} +``` + +To change a pending message, the client removes it and submits a replacement. DELETE rejects the +request after the input was promoted into active work. The server processes pending inputs in FIFO +order, which means first in, first out. The initial interface does not support reordering. + +This keeps clients synchronized. A message is no longer hidden inside one browser's local queue. + +### One public interface for all clients + +Agenta desktop, mobile, bots, and external API users should call the same public session API. A +first-party browser must not depend on a separate privileged execution endpoint. + +The runner still needs a private protocol because it performs trusted internal work. That private +protocol carries claims, heartbeats, event frames, acknowledgements, and control wake-ups. It is +not a second product API. + +### Interaction responses + +Moving interaction response under the session URL does not itself improve correctness. It only +makes session ownership and authorization visible in the path. The current endpoint can remain: + +```http +POST /sessions/interactions/{interaction_id}/respond +``` + +or the clean public contract can use: + +```http +POST /sessions/{session_id}/interactions/{interaction_id}/responses +``` + +The material change is internal. The API must durably accept the response, make one response win, +and expose whether continuation is pending, running, or failed. URL nesting is a consistency +choice, not the reason for changing approval handling. + +### Private control path + +The public Cancel request does not need a runner address. A simple internal flow is: + +1. The browser sends Cancel to the API. +2. The API records that execution 12 must stop. +3. The API sends a private wake-up to the runner that owns execution 12. +4. The runner stops local work and reports `execution.cancelled`. +5. Every browser receives that event. + +The API already knows the logical runner owner as `replica_id`. It does not yet know a reliable +network address for that replica. The implementation must add one of these private delivery +mechanisms: + +- The runner keeps an outbound connection open to the API. The API sends control messages on it. +- The runner subscribes to a private per-runner broker channel. +- The runner service adds owner-aware routing behind one internal URL. + +This private choice does not change the public Cancel endpoint. A heartbeat remains useful for +renewing ownership and detecting a crashed runner. It stops being the normal way to deliver +Cancel. + +### Execution identity and ownership + +Current Redis state identifies a logical runner replica and the current `turn_id`. The API does +not currently map that replica identifier to a replica-specific network address. The hard-kill +path calls one configured runner service URL. The RFC must select an immediate-control routing +mechanism before it can define fast Cancel delivery. + +The recommended routing pattern for discussion is: + +1. The API saves or atomically records the command. +2. The API identifies the logical owner `replica_id`. +3. A private control channel wakes that runner immediately. +4. The runner acknowledges and applies the command. +5. Heartbeat or periodic recovery finds commands whose wake-up was lost. + +The runner can initiate the control connection to the API. This would support possible future +user-operated runners behind firewalls and keep Redis credentials behind the API boundary. That +future deployment model is a consideration, not a confirmed requirement. + +The simplest first implementation is durable long polling. The runner makes an authenticated +request that the API holds briefly until a command is available. The runner receives the command, +acknowledges it, and immediately opens the next request. A disconnected runner reconnects and +claims commands that remain durable. Redis or Postgres notifications may wake API replicas +internally, but the runner never connects to either system. + +Credential-bearing long polls require HTTPS with normal certificate validation. The client must +disable redirects or reject any redirect whose origin differs from the configured API origin, and +it must never forward runner credentials across origins. + +A persistent WebSocket or bidirectional stream can later reduce repeated requests and carry richer +runner status. It is not required for the first contract. Direct API calls into runner pods and +per-runner Redis subscriptions are poor fits for user-operated runners because they require inbound +reachability or infrastructure credentials. + +Control delivery must sit behind an internal port. Session command handling depends on this port, +not on a particular transport: + +```text +deliver(owner, command) +acknowledge(command_id, owner) +recover(owner) +``` + +Initial adapter: authenticated long polling. Possible later adapters: persistent WebSocket, +private Redis delivery, or direct managed-runner routing. Durable command state, authorization, +idempotency, execution fencing, and terminal settlement remain outside the adapter. Replacing the +adapter must not change the public session API or command state machine. + +The required invariant is stronger than “the second start usually gets a conflict”: at most one +execution is active for a session, and only the current owner can write or cause external effects. +The current Redis `alive` lease, owner affinity, heartbeat refresh, and superseded markers reduce +overlap. They do not fully enforce this invariant after lease expiry or a network partition because +record ingest does not reject a stale ownership generation. + +The target design therefore separates two jobs: + +1. **Admission and fencing.** The API atomically accepts one execution and assigns an increasing + ownership generation. Every runner event carries the execution ID and generation. The API + rejects stale generations. Settlement releases ownership only when both values match. +2. **Failure detection.** A heartbeat renews the active lease. If it expires, recovery can mark the + execution lost and assign a newer generation. The heartbeat detects failure, but it is not the + only protection against two writers. + +The RFC does not yet choose whether admission state belongs in Postgres, Redis with a durable +command record, or a transaction across projections. The selected design must prove atomic +concurrent admission and stale-write rejection. + +### Immediate control + +The existing `/sessions/streams/` endpoint is a coordination-state edit, not a durable command +inbox. It derives Send, Steer, Cancel, and Attach from inputs plus a `force` flag. Normal desktop +Send does not use this endpoint. A future explicit command contract must replace the ambiguous +shape without silently changing existing invoke behavior. + +### Command delivery and execution settlement + +Command delivery and execution lifecycle are separate state machines: + +```text +command: pending -> claimed -> applied + -> obsolete + +execution: running -> stopping -> stopped + -> failed + -> lost +``` + +The API accepts Stop by durably creating the command and moving the matching execution to +`stopping` in one transaction. `expected_execution_id` remains optional. A command claim has a +lease and can be delivered again after disconnection. In a fenced design, the runner deduplicates +by `command_id` and validates both the execution ID and ownership generation before applying it. +The v1 direct-delivery adapter has no generation token; it validates the target execution ID and +requires the addressed runner replica to own that execution. + +Claiming or acknowledging a command does not prove that execution stopped. Public clients follow +execution state. The runner normally reports the terminal outcome and the API settles the command +and execution together. If the runner disappears, a watchdog records `lost`; another runner cannot +claim that it stopped work on the missing machine. The settlement deadline will be selected after +the sandbox cancellation spike. + +### First-version ownership scope + +The first version retains Redis as the execution ownership authority. It does not introduce a new +Postgres execution table, ownership generation, or general fencing migration. + +When Stop is accepted, the API saves the durable command but does not immediately free the current +`alive` lock. Long polling delivers the command. The heartbeat can discover the same pending +command as a fallback. The runner releases owner-checked `running` and `alive` keys only after +cancellation settles, so new work cannot start during normal cancellation. + +This scope accepts the current network-partition limitation. Full multi-runner correctness and +stale-writer fencing remain future work. The command and control-delivery ports must not depend on +Redis-specific ownership details, so that later work can replace the ownership adapter. + +### Live frame ingress and relay + +The working model has one raw runner event ingress. The API acknowledges a frame only after it is +accepted into the shared Redis Stream. Live readers consume temporary frames from that stream. +The durable projector consumes the same source and commits permanent facts. + +Browser delivery never blocks the runner. A slow reader is disconnected and later recovers from +durable state. With multiple API replicas, the runner and readers can connect to different +replicas because Redis and Postgres hold the shared state. A runner-to-API disconnect does not +stop execution; the runner reconnects and resends unacknowledged frames. + +### Durable events and replay + +The durable history requires immutable event IDs, an order whose visibility matches database +commit order, idempotent retries, and a replay-to-live handoff that cannot miss a commit. A plain +Postgres `BIGSERIAL` is insufficient by itself because sequence allocation can precede an +out-of-order transaction commit. + +Two storage options remain under consideration. + +#### Option A: Repair records into the session event history + +Change records so every durable fact has a stable producer ID, immutable payload, and commit-safe +session cursor. Duplicate delivery becomes a no-op. Add execution, input, and interaction +lifecycle facts so the same append-only history can build the transcript and the session snapshot. + +Benefits: + +- One permanent history to write, retain, query, and debug. +- Existing transcript and harness reconstruction already read records. +- No consistency problem between two permanent logs. + +Costs and risks: + +- Changes the existing upsert contract and tool snapshot behavior. +- Expands a conversation-oriented tracing record into the public session event contract. +- Requires a migration story for old rows without cursors and current record retention. +- Requires commit-safe ordering and reliable delivery changes regardless of table reuse. + +This option has a mandatory discovery gate. Today a repeated stable `record_id` can be an exact +transport retry or a later snapshot with changed payload. Only the exact retry becomes a no-op. +The producer-semantics spike in `records-invariants.md` must classify every reuse and add regression +tests before the upsert contract changes. + +#### Option B: Keep records as a transcript projection and add a session event log + +Keep current records for conversation and harness reconstruction. Add an immutable session event +history for input, execution, tool, interaction, and message lifecycle events. Build session +snapshots from that history or projections updated in the same transaction. + +Benefits: + +- Leaves the current transcript and harness path largely intact during migration. +- Gives the public event contract its own schema and retention policy. +- Separates mutable or coalesced transcript projections from immutable lifecycle facts. + +Costs and risks: + +- Two permanent representations of some conversation facts. +- The projector must keep records and session events consistent. +- Debugging and recovery must define which representation is authoritative. +- More schema, storage, migrations, and cleanup machinery. + +#### Redis as permanent history + +Redis remains the temporary ingress and delivery buffer. It is not a permanent session history in +this draft because the Stream is bounded, entries are acknowledged and deleted, and Redis does not +match the existing Postgres retention, query, and recovery model. + +#### Snapshot and stream consistency + +The snapshot is a durable projection through cursor N. The event endpoint replays durable events +after N and then follows newly committed events. Snapshot data and cursor must be read from one +consistent database view, or the projection and cursor must update in the same transaction. + +Temporary live frames do not advance the durable cursor. The next durable completion repairs +missed previews. A reader subscribes to commit wake-ups before reading replay history so a commit +cannot fall between the historical read and live tail. + +The delivery chain must handle these failure boundaries: + +1. Runner to API: retry unacknowledged frames after reconnect. +2. API to Redis: acknowledge only after `XADD` succeeds. +3. Redis to projector: leave failed work pending for retry. +4. Projector to Postgres: append events and update projections in one transaction. +5. Postgres to live wake-up: a lost wake-up is repaired by querying after the cursor. +6. API to browser: reconnect after the last durable cursor. + +### Detached sender + +Starting work and watching work are separate operations. The API durably accepts an input and +returns without waiting for a runner claim, harness start, first output frame, or reader +connection. The execution then proceeds independently of the submitting HTTP request. + +The durable acceptance boundary includes: + +- The submitted input. +- Its idempotency identity. +- Its session association. +- Its accepted execution intent. + +The sender then reads the same session event stream as desktop, mobile, bots, and external +clients. Disconnecting any reader does not cancel or park the execution. A convenience request +may submit and begin streaming in one call, but that response remains a reader of an independently +accepted execution. + +During migration, the current invoke response can continue serving the sender while the shared +read path is introduced. The final client model removes this privileged sender path. + +### Durable commands + +Working scope for discussion: + +- Send a user message. +- Cancel an expected execution. +- Respond to an interaction. +- Queue a message. +- Steer with a saved message. + +Attach belongs to the read path. Kill, rename, archive, and delete remain separate lifecycle or +resource operations in the working model. + +The internal command transport does not require every public action to use one generic endpoint. +Public resource endpoints can validate domain-specific input and then create the common internal +command. + +### Queue and Steer + +Pending discussion. + +### Approvals and pauses + +Pending discussion. + +### Client state application + +Pending discussion. + +## Migration + +Pending discussion. The migration must preserve the current sender stream until the shared read +path passes its live-stack tests. + +## Test plan + +Pending discussion. Each architecture section must add one invariant and one live-stack test. + +## Rejected alternatives + +Pending discussion. diff --git a/docs/design/session-control-and-live-events/slice-admission.md b/docs/design/session-control-and-live-events/slice-admission.md new file mode 100644 index 00000000000..1aa2633389e --- /dev/null +++ b/docs/design/session-control-and-live-events/slice-admission.md @@ -0,0 +1,373 @@ +# Slice: single-turn admission + +Status: built and verified live on 2026-09-02. Branch `feat/session-single-turn-admission`. +Not pushed, no pull request. + +This slice makes one invariant true: **at most one execution runs per session, decided in one +place.** A second message sent while a turn is running is refused before anything is destroyed. +That is the `on_busy: reject` policy. Queue and steer are not in this slice. + +Closes [#6417](https://github.com/Agenta-AI/agenta/issues/6417), +[#5539](https://github.com/Agenta-AI/agenta/issues/5539), and +[#5538](https://github.com/Agenta-AI/agenta/issues/5538). + +--- + +## What happens today, and why + +A user sends a second message while the agent is still answering the first. Both turns die and +the session stays locked for about thirty minutes. Every step below is **verified** in code. + +1. A desktop Send does not go through the session coordination endpoint. It goes to the workflow + invoke path, `POST /services/agent/v0/invoke` + (`web/packages/agenta-playground/src/state/execution/agentRequest.ts:400`). The only caller of + `commandSessionStream` in the web tree is Stop. +2. The runner mints its own turn id for that request + (`services/runner/src/server.ts:189`). +3. The runner starts the turn's alive watchdog **before** it touches any sandbox + (`services/runner/src/server.ts:519` versus the run at `:621`). That watchdog's first heartbeat + is an atomic `nx` acquire of the session's `alive` lock in the API + (`api/oss/src/core/sessions/streams/service.py:513`). +4. The second turn loses that acquire, because a different turn holds `running` + (`api/oss/src/core/sessions/streams/service.py:534`). The API answers `is_current_turn: false`. + **The arbiter was already correct.** +5. The runner read that answer only as "abort this run later" + (`services/runner/src/sessions/alive.ts:217`), then carried on into the keepalive pool, found + the first turn's environment busy, and **destroyed it**: + the `evict (supersede-busy)` branch, now at + `services/runner/src/lifecycle/session-coordinator.ts:1343` and no longer reachable by a live + turn. The first turn lost its sandbox mid-answer. +6. The second turn then aborted on its own watchdog signal. Both turns were dead, and the session + read as alive under a dead turn's lock until the lease expired. + +So the fix is not a new subsystem. It is reading an answer the platform already gives, before +acting on the session. + +--- + +## What changed + +Seven commits on `feat/session-single-turn-admission`. + +### 1. The runner reads the admission answer (`7675eb0dc7`) + +| File | Change | +|---|---| +| `services/runner/src/sessions/admission.ts` | New. Holds the stable code `session_turn_in_use` and the one line the user reads. The decision is not made here; this is only how the runner reports it. | +| `services/runner/src/sessions/alive.ts:190` | `startAliveWatchdog` now returns `admitted`, the FIRST beat's answer. A later `is_current_turn: false` is a Stop or steer and still travels the `onInterrupted` to abort path. | +| `services/runner/src/server.ts:534` | A refused turn stops at the edge and returns. | +| `services/runner/src/lifecycle/session-coordinator.ts:1320` | A `busy` pool entry is refused, never evicted. A `destroyed` entry still evicts and cold-starts. | +| `services/runner/src/engines/sandbox_agent/errors.ts:69` | `session_turn_in_use` added to `RunErrorCode`. | + +The refusal in `server.ts` sits above three things it must not do, and this ordering is the +point: + +- `cancelStaleInteractions` (`server.ts:573`) cancels the session's unanswered approval gates. A + refused turn running it would cancel the **live** turn's approval card. +- The persisting emitter (`server.ts:587`) would write the refused message into the durable + transcript, so it would come back on reload as a message the user never sent. +- `run()` (`server.ts:621`) is what reaches the keepalive pool. + +The refusal streams as an `error` event carrying the code, then a failed terminal result. That is +the path every runner failure already takes to the browser, so no new transport is involved. + +The first heartbeat is the admission decision and fails closed unless the coordination plane +confirms ownership. Later heartbeat failures remain best effort for a turn that was already +admitted. The coordinator stays as a same-runner backstop and refuses a competing `busy` pool entry +without destroying the live environment. + +### 2. The browser keeps the user's text (`bdd7116520`) + +A naive refusal is worse than the bug for the person typing. The composer clears synchronously on +submit (`web/packages/agenta-ui/src/RichChatInput/assets/submit.ts:31`), so without this change +their text is simply gone. + +| File | Change | +|---|---| +| `web/packages/agenta-chat/src/model/error.ts` | The refusal constants, `isSessionBusyRefusal`, and a stable class on the parsed error. | +| `web/packages/agenta-chat/src/hooks/useAgentChatQueue.ts:98` | Remembers the message handed to `sendQueued` and hands it back once through `takeLastSent`. | +| `web/oss/src/components/AgentChatSlice/AgentConversation.tsx:433` | Puts that text back in the composer on a refusal. | +| `web/oss/src/components/AgentChatSlice/components/AgentMessage.tsx:204` | The bubble says "Message not sent" rather than "The agent run failed", and offers no retry. | + +The message is **not** re-queued. The queue releases on a settled `"error"` status +(`useAgentChatQueue.ts:68`, and the release effect below it), which for a refusal would re-send and be refused again in a tight +loop. The user decides when to send again. + +The refusal message text is the contract between the runner and the browser. It is produced once, +in `services/runner/src/sessions/admission.ts`, and reaches the browser verbatim: the SDK's +`sanitize_runner_error` passes a clean one-line error through unchanged +(`sdks/python/agenta/sdk/agents/utils/wire.py:60`) and the Vercel egress puts it on the stream as +`errorText` with the code beside it +(`sdks/python/agenta/sdk/agents/adapters/vercel/stream.py:954`). The two constants must stay +byte-identical. + +Mobile shares the queue hook and the error model, so it gets the refusal class. It has its own +composer and its own copy of the error effect, so it does not get the text restore. See the open +questions. + +### 3. The client learns which execution it is watching (`ce0f1e12da`, `ca600cb1e6`) + +Added on request from the Stop guard lane, which found that no first-party client can send +`expected_execution_id` on the public Cancel: the runner mints the turn id per execution +(`services/runner/src/server.ts:189`) and never tells anyone, so a Stop can only mean "whatever +is running now", never "the turn I was watching". + +**The `start` frame cannot carry it.** It is built and sent by the SDK's Vercel egress before the +runner replies at all: the `start` yield is the first statement of the projection +(`sdks/python/agenta/sdk/agents/adapters/vercel/stream.py:459-464`), and the runner is not +consulted until the loop below it. Putting the id there would mean moving the mint out of the +runner and threading a new correlation id through the normalizer, the response models and the +routing layer for **every** workflow, not just agent ones. That is a much larger change than the +problem needs. + +The earliest frame that can carry it is the one right after: + +| File | Change | +|---|---| +| `services/runner/src/protocol.ts:479` | New `{type: "turn", turnId}` agent event. | +| `services/runner/src/server.ts:579` | Emitted as the first event of a session-owned run, immediately after admission, through `liveEmit` and never the persisting emitter. It is transport correlation, not conversation, and must not become a session record. | +| `sdks/python/agenta/sdk/agents/adapters/vercel/stream.py:361` and `:663` | Forwarded onto the MESSAGE METADATA, in both the live and dev-twin projections. | + +The client reads it as `message.metadata.turnId`, beside the `sessionId` the `start` frame already +sets and the `traceId` and `usage` the `finish` frame adds, rather than scanning parts for it. +A `message-metadata` chunk is a first-class chunk in the pinned `ai@6.0.0-beta.150`. + +That is safe **because the AI SDK merges metadata rather than replacing it** (`mergeObjects`), so +the `finish` frame's own metadata lands beside the turn id rather than over it. A test pins that +the two carry disjoint keys and the turn id is written first. If the SDK ever changed to replace, +a client would lose the id exactly when a late Stop needs it. + +A missing, empty or non-string id emits no frame, so a client is never handed a guard value that +names nothing. A refused turn emits none either: it runs nothing, so there is nothing to stop. + +Verified live on the stack below. The frame arrives third, after `start` and `start-step` and +before any content: + +``` +["start", "start-step", "message-metadata", "text-start"] +``` + +and its id is the one holding the session's alive lock, cross-checked against the runner log: + +``` +message-metadata turnId=6a49ff3f-2165-4e4b-bbe8-c9f7192fabb3 +[sessions] stream sessionOwned=true sessionId=2fa74edd-… turnId=6a49ff3f-2165-4e4b-bbe8-c9f7192fabb3 +``` + +The first version of this (`ce0f1e12da`) used a `data-agent-turn` part instead. `ca600cb1e6` +replaced it rather than adding to it: one fact should travel one channel, and nothing consumed the +part yet. The Stop guard lane adds the browser half on its own branch. + +### 4. API tests only (`8b1a45e5a6`) + +No API code changed. Three cases now pin the answers the runner depends on, in +`api/oss/tests/pytest/unit/sessions/test_heartbeat_is_current_turn.py`. + +--- + +## Approvals still resume + +This is the case a naive "is anything alive on this session?" gate breaks, and it was checked +before the design was chosen. + +A turn parked awaiting approval still holds `alive`, which is what makes the session +reattachable, but its turn-end beat released `running` +(`api/oss/src/core/sessions/streams/service.py:590`). The approval resume arrives as a new turn. +The heartbeat sees stale `alive` with **no** `running` owner, treats it as a legitimate handover, +tombstones the parked turn and admits the resume +(`api/oss/src/core/sessions/streams/service.py:536-561`). + +So `running` is the discriminator, not `alive`. Both cases are now tested, at the API and end to +end at the runner. + +--- + +## Tests + +| Suite | Command | Result | +|---|---|---| +| Runner unit | `cd services/runner && pnpm test` | 2639 passed, 4 failed | +| Chat package | `cd web/packages/agenta-chat && pnpm test` | 626 passed | +| SDK agents unit | `pytest oss/tests/pytest/unit/agents/` | 1198 passed, 4 skipped | +| API sessions unit | `pytest unit/sessions/` | 328 passed, 41 skipped | +| Web lint | `cd web && pnpm lint-fix` | 25 tasks, 0 errors | +| Web typecheck | `tsc --noEmit` on `@agenta/oss` and `@agenta/chat` | clean | + +The four runner failures are **pre-existing**, all in +`tests/unit/gateway-run-turn-composition.test.ts`. Confirmed by stashing this slice's changes and +re-running: the same four fail on the branch tip. + +The 11 collection errors in the API run came from a virtual environment that resolved `agenta` +from a different checkout. They are import errors in unrelated files. + +New tests: + +- `services/runner/tests/unit/session-admission.test.ts` (7 tests). A real runner HTTP server + driven over a socket against a fake platform API. Covers: a refused turn never calls `run()`, + the error event carries the code, no interaction sweep or attachment claim happens, the end + beat names the refused turn, an admitted turn proceeds, a resume-shaped request is admitted, + and an unreachable platform fails closed before `run()`. +- `services/runner/tests/unit/session-alive-interrupt.test.ts` (+4). `admitted` semantics: first + beat only, fail-closed without confirmation, and a later interruption does not un-admit. +- `services/runner/tests/unit/session-keepalive-dispatch.test.ts` (+1, 1 rewritten). A busy entry + refuses with no eviction and no cold acquire; a destroyed entry still evicts. +- `services/runner/tests/unit/session-steer-mount-loss.test.ts` (3 rewritten). These pinned the + old supersede outcome. They now pin the refusal, and one new case asserts the live turn's + environment is never torn down. +- `web/packages/agenta-chat/tests/unit/model/error.test.ts` (+4). +- `web/packages/agenta-chat/tests/unit/hooks/useAgentChatQueue.test.ts` (+4). +- `api/oss/tests/pytest/unit/sessions/test_heartbeat_is_current_turn.py` (+3). +- `services/runner/tests/unit/session-admission.test.ts` (+3, the turn-id frame): it arrives + first, it is the id the alive lock was acquired under, and a refused turn emits none. +- `sdks/python/oss/tests/pytest/unit/agents/adapters/test_vercel_stream_conformance.py` (+4): the + egress forwards the id verbatim exactly once in both projections, before any content; the + `finish` frame's metadata does not displace it; and a frame with no usable id emits nothing. + +One rewritten test was found to be passing for the wrong reason. The `destroyed`-entry case in +`session-keepalive-dispatch.test.ts` called `pool.destroyAll`, which clears the map, so the +assertion ran against a `miss` rather than a `destroyed` entry. The runner typecheck caught the +argument-type error that exposed it. It now marks the entry directly, because every public route +that destroys a session also removes it. + +--- + +## Live verification + +### The stack + +A standalone EE development stack built from this worktree at `:`. The deployment +used a current EE development environment file with isolated ports and project name. Dev-mode bind +mounts confirmed that the containers ran this worktree's source. + +When host and container users differ, dependency ownership can prevent the web entrypoint from +updating generated binaries. Repair only the affected dependency or generated paths with targeted +ownership or ACL changes, then restart the container. Never make the whole web tree world-writable. + +Sandbox provider: `local`. Harness: `pi_core`. The model credential came from the stack's test +vault; no key or secret is part of this record. + +### The scenario + +The verification driver worked at wire level, asserted on SSE frame types, and never used model +prose as evidence. Its environment-specific path and credentials are intentionally not recorded. + +1. Turn A starts on a fresh session and runs `sleep 40 && echo DONE_A` as a shell tool. +2. Fifteen seconds in, turn B sends "What is 2 + 2?" to the same session. +3. After A settles, turn C sends a third message. + +The agent config sets `runner.permissions.default` to `allow`, so the long tool runs instead of +parking on an approval card. The first attempt without it proved nothing: the tool parked, turn A +ended after eleven seconds, and the two turns never overlapped. + +### Results + +| Turn | HTTP | Duration | Outcome | +|---|---|---|---| +| A, long turn | 200 | 50.3 s | finished, `finishReason: stop`, reply "Finished.", ran its tool | +| B, second send | 200 | **0.18 s** | refused, no assistant text, no tool call | +| C, after | 200 | 2.0 s | ran, reply "READY" | + +Turn B's error frames, verbatim: + +```json +{"code": "session_turn_in_use", "errorText": "This session is already running a turn. Your message was not sent. Wait for the reply, or stop the turn, then send again."} +{"type": "error", "errorText": "This session is already running a turn. Your message was not sent. Wait for the reply, or stop the turn, then send again."} +``` + +Runner log for the test session, in order: + +``` +[sessions] stream sessionOwned=true sessionId=081a1fe7-… turnId=444d272b-… cred=present +[sessions/alive] heartbeat OK session=081a1fe7-… turn=444d272b-… running=true +[keepalive] miss key=01a063ea-…:081a1fe7-…; cold +[keepalive] reserve key=01a063ea-…:081a1fe7-… poolSize=2 +[sessions] stream sessionOwned=true sessionId=081a1fe7-… turnId=0e4a90c0-… cred=present +[sessions/alive] heartbeat OK session=081a1fe7-… turn=0e4a90c0-… running=true INTERRUPTED +[sessions] admission REFUSED session=081a1fe7-… turn=0e4a90c0-…; another turn owns this session. No pool resolve, no eviction. +[sessions/alive] heartbeat OK session=081a1fe7-… turn=0e4a90c0-… running=false +[sessions/alive] heartbeat OK session=081a1fe7-… turn=444d272b-… running=true +[sandbox-agent] complete OK session=081a1fe7-… turn=0 +[keepalive] park key=01a063ea-…:081a1fe7-… ttl=60000ms state=idle (re-park) poolSize=1 +[sessions] stream sessionOwned=true sessionId=081a1fe7-… turnId=42fa2fa0-… cred=present +[keepalive] hit-continue key=01a063ea-…:081a1fe7-… +[sandbox-agent] complete OK session=081a1fe7-… turn=1 +``` + +Three things to read from that log: + +- There is **no** `evict (supersede-…)` line. The refused turn touched the pool not at all. +- Turn A ran to `complete OK` and then `park … state=idle`, so it kept its sandbox. +- Turn C got `hit-continue`, which means it continued the **warm** session A parked. The warm + sandbox and the native harness session survived the second send. That is the constraint this + slice was bound by, checked rather than assumed. + +Use the matching edition, image mode, and environment file to tear down the isolated stack: + +```bash +bash ./hosting/docker-compose/run.sh --ee --dev --down +``` + +Add `--nuke` only when the isolated volumes should also be removed. + +### Not verified + +The browser behaviour was **not** verified in a browser. The composer restore, the "Message not +sent" bubble and the mobile path are covered by unit tests and a typecheck only. The web app on +this stack is serving (`/w` answers 200), so a UI pass is available and is worth doing before +this ships. Reproducing the refusal by hand needs two browser tabs on one session, or one tab +plus a curl invoke while a turn runs. + +--- + +## What is left for queue and steer + +Refusing needs no storage. Queue and steer both do, and that is the whole reason they are not in +this slice. + +- **Queue** needs a durable pending-input store, because a saved message has to survive the turn + it is waiting on and a browser reload. The client-side queue in `useAgentChatQueue` is a + per-tab convenience; it is lost on reload and invisible to any other reader of the session. +- **Steer** needs the same store plus a decision that this slice deliberately does not make: is + steer reject-with-message, keeping the turn and the warm session, or interrupt-and-restart? The + RFC (`rfc.md:125`) says interrupt-and-restart, which reverses the ruling of 2026-07-22 without + saying so, and interrupt-and-restart is the shape that loses warm state today. +- **The 409 shape.** The API's `_start_turn` already raises `SessionTurnInUse` and the router + already maps it to 409 (`api/oss/src/apis/fastapi/sessions/router.py:192`). This slice does not + route Send through that endpoint, because the runner's own heartbeat already performs the same + atomic acquire one step earlier and the invoke path does not otherwise touch the API. If Send + ever moves onto the coordination endpoint, the refusal should become the 409 and the runner's + edge check becomes a second line of defence. +- **The watchdog** is untouched by this slice and remains the highest-value next change. It + bounds every hang rather than only the double send. + +--- + +## Open questions for Mahmoud + +1. **Should the refused message be queued instead of handed back to the composer?** + *Recommendation: keep handing it back for now.* Queueing reads better, but the queue + auto-releases on a settled error status, so a refusal would re-send and be refused in a loop + until the running turn ends. Fixing that needs a refusal-aware release gate, which is queue + work, not admission work. + +2. **Should mobile also restore the text?** Today it gets the refusal class but not the restore, + because its composer and its error effect are separate files from desktop's. + *Recommendation: yes, in a follow-up.* The precedent already exists at + `web/mobile/src/features/chat/Composer.tsx:98`, which puts text back and shows a composer-level + rejection strip. It is a few lines, but it is a second host to QA and this slice is already + wide. + +3. **Is the composer-level rejection strip a better home for this than a red transcript bubble?** + Mobile already has one. A refusal is a fact about the message the user just typed, not about + the conversation. *Recommendation: move it there once someone looks at it in a browser.* The + current bubble is honest but it sits in the transcript, which is where run failures live. + +4. **Should initial admission fail closed when the API is unreachable?** *Decision: yes.* At-most-one + execution has to hold across replicas. Later watchdog failures remain best effort so an already + admitted healthy turn is not aborted by a transient API failure. + +5. **Should `--build` have been skipped?** The brief said to skip it if the images were under + three hours old, and they were fifteen minutes old. The live results therefore depend on dev + mode bind-mounting this worktree's source, which the runner log confirms it did (the + `admission REFUSED` line only exists in this branch). *Recommendation: no action.* Flagged only + so the evidence is auditable. diff --git a/docs/design/session-control-and-live-events/slice-durable-cancel.md b/docs/design/session-control-and-live-events/slice-durable-cancel.md new file mode 100644 index 00000000000..cd9478acb79 --- /dev/null +++ b/docs/design/session-control-and-live-events/slice-durable-cancel.md @@ -0,0 +1,255 @@ +# Slice: the durable Stop command, with the direct-call adapter + +> AGENT-GENERATED, low weight. Built and verified live. Mahmoud makes final decisions. + +Branch `feat/session-durable-cancel`, rebased onto `spike/session-cancel-warm` at `f5b1ae6244`. +It implements [the durable command design](spike-b-durable-commands-design.md) at `86281fa313` +and [the route contracts](api-design.md), with the direct-call adapter of that design's +section 9. The long-poll adapter is not built. + +Every claim below is marked **verified** (observed on the running stack, or read in this +branch's code with a `path:line`) or **reported** (taken from a document). + +--- + +## What a Stop does now + +**Verified live.** A user Stop reaches the running turn in 82 milliseconds, ends it, and leaves +the sandbox and the native harness session warm. Before this branch it reached the runner on the +next heartbeat, up to 30 seconds later. + +| Step | Observed at | After the Stop request | +|---|---|---| +| The browser's request arrives, the command row commits, the API calls the runner | 00:12:45.624 | 0 | +| The runner aborts the execution | 00:12:45.706 | 82 ms | +| The harness confirms it stopped | 00:12:45.730 | 106 ms | +| The runner reports, and the API settles the command and the execution | 00:12:45.750 | 126 ms | +| The sandbox is parked warm, not deleted | 00:12:46.617 | 993 ms | + +The 5 second budget in the design is met with two orders of magnitude to spare. The next message +on that session recalled a codeword from the stopped turn, which is warm resume measured from +the product rather than from a timer. + +--- + +## What changed, with references + +### The record + +`session_commands` holds one row per durable request to change an execution +(`api/oss/src/dbs/postgres/sessions/commands/dbes.py:14`, migration +`api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py`). +Two columns are never merged: `state` says where the COMMAND is (`pending`, `claimed`, +`applied`, `obsolete`) and `outcome` says what happened to the EXECUTION (`stopped`, +`not_running`, `superseded_by_newer_turn`, `failed`, `lost`). + +`session_streams` gains `stopping_turn_id` and `turn_started_at` +(`api/oss/src/dbs/postgres/sessions/streams/dbes.py:73` and `:82`). The start time is stamped +only when the turn id actually changes +(`api/oss/src/dbs/postgres/sessions/streams/mappings.py`, the edit mapper), so the heartbeat that +restamps the same id every 30 seconds never moves it. + +Every transition is one `UPDATE ... WHERE RETURNING *` decided by +`scalar_one_or_none()` (`api/oss/src/dbs/postgres/sessions/commands/dao.py`). Two API replicas +cannot both win a claim or both write a terminal outcome. + +### Admission + +`SessionCommandsService.request_cancel` +(`api/oss/src/core/sessions/commands/service.py:111`) stamps the arrival time before it reads +anything, resolves the target once from Redis `running` falling back to `alive` +(`service.py:217`), applies the three late-Stop guards, then writes the command and the session's +`stopping_turn_id` in one transaction. **Redis is not written at admission**, so the stopping +execution keeps both locks while it stops, which is what prevents a second message from starting +underneath it. + +### Settlement + +`SessionCommandsService.settle` (`service.py:419`) settles the command and the execution +together, guarded on the command's state so a repeat changes nothing. For a `stopped` outcome it +tombstones the turn, then releases `running` under an owner check, then cancels that execution's +pending interactions, then publishes the existing `lifecycle: ended` notification. **It leaves +`alive` to its own time to live**, exactly as the end of an ordinary turn does. That single +decision is what makes Stop a stop rather than a session teardown. + +### Delivery + +`ControlDeliveryPort` (`api/oss/src/core/sessions/commands/interfaces.py`) is the port. The one +adapter is `DirectControlDelivery` +(`api/oss/src/dbs/http/sessions/control_delivery_direct.py`), which posts to the runner's own +`/cancel` beside the existing `kill_runner_sandbox` +(`api/oss/src/core/sessions/streams/runner_client.py`). The command row is committed BEFORE the +runner is called, and a delivery failure never fails the request. + +### The runner + +`POST /cancel` sits beside `POST /kill` behind the same token gate +(`services/runner/src/server.ts:821`). It resolves a live execution through a module-level +registry (`services/runner/src/sessions/execution-registry.ts`), falls back to the keep-alive +pool for a parked approval, and answers 404 when it holds neither. + +**The abort carries the user-stop label.** `shouldPark` parks only an abort the runner can prove +was a cooperative Stop (`services/runner/src/sessions/stop-signal.ts`, from Spike A), so the +registry aborts with `USER_STOP_ABORT_REASON`. Without it a Stop delivered as a command ends the +turn `cancelled` and then DESTROYS the sandbox, which is the failure Stop exists to avoid. Two +tests pin both directions, and the live run after the rebase logs `park-cancelled`. The applier +sits above the transport (`services/runner/src/sessions/control-channel.ts`) with the +deduplication set beside the session pool (`services/runner/src/sessions/applied-commands.ts`), +so a long-poll loop would reuse every guard unchanged. + +### The routes + +`POST /sessions/{session_id}/cancel` and +`POST /sessions/control/commands/{command_id}/outcome`, both on `SessionControlRouter` +(`api/oss/src/apis/fastapi/sessions/router.py:1909`). The public route checks +`Permission.RUN_SESSIONS` and is deliberately **not** behind `check_runner_concurrency_limit`: +refusing to STOP work because a project is at its run limit is the wrong answer to a busy +project. The internal route authenticates with the shared runner token +(`router.py:2027`) and resolves the project from the command id, so the auth exemption +(`api/oss/src/middlewares/auth.py`, the `/sessions/control/` prefix) widens no tenant boundary. + +`POST /sessions/streams/` is untouched. Its cancel branch becomes a thin wrapper over this +command in a later change, together with the mobile client; do both in one change so one revert +restores one behaviour. + +### The desktop + +The Stop button posts the new route +(`web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts`, `stopCurrentExecution`), +awaits it, and refreshes the session state on the answer. It names the execution it means, read +FRESH from the session row rather than from the project-wide liveness poll, which is up to +15 seconds stale; a stale id is refused with a conflict and the Stop would silently do nothing. +The client is `cancelSessionExecution` +(`web/packages/agenta-entities/src/session/api/api.ts`), written against raw axios because the +Fern client does not know the route yet. Mobile is untouched. + +--- + +## Four defects the live run and the rebase found + +None were visible in unit tests. The first three were found by pressing Stop against a real +agent turn; the fourth by rebasing onto Spike A's final tip. Each is committed with its own fix. + +1. **The execution registry never held the session, so every Stop got a 404 and the turn ran to + completion.** The entry was keyed by `:`, but the project scope is not + known when a run starts: `runContext.project.id` is empty on the live invoke path and the + scope that forms the pool key comes from the signed mount, which the coordinator resolves + after the run is in flight (`services/runner/src/lifecycle/session-coordinator.ts:281`, + verified). The registry is now keyed by session id and the coordinator fills the project in + through `onScopeResolved`. A lookup with a disagreeing project is refused; an entry whose + project is not known yet matches, because refusing every Stop in the first moments of a run is + the bug being replaced. +2. **The outcome report was refused with a 409, leaving the command `claimed` and the session + marked stopping forever.** The API claimed on the runner's behalf under a placeholder while + the runner reported under its own replica id, and the settle guard compares the two. The + runner's acknowledgement now carries its replica id and the API claims under that. +3. **The multi-replica census refused delivery for five minutes after every runner restart.** A + runner mints a fresh replica id at boot when `AGENTA_RUNNER_REPLICA_ID` is unset + (`services/runner/src/sessions/alive.ts:31`, verified), so its previous id is still inside the + window and the count reads two, which broke Stop after every ordinary deploy. **The census is + now removed entirely**, on the revised design's guidance that it is optional and the exact + detector is the one to build. That deletes a Redis write on every heartbeat, two settings and + a module. What remains is the detector that cannot be fooled: a `not_held` for a session whose + row says alive with a heartbeat younger than one interval means some process is running that + session and it is not the one we called. It logs at error level naming the owner replica from + the Redis `owner` key, and settles the command `lost` rather than `not_running`, so the user + is told the Stop failed instead of that the work had already finished + (`api/oss/src/core/sessions/commands/service.py`, `_settle_not_held`). + +4. **The control-plane abort carried no label, so after the rebase every Stop would have + destroyed the sandbox.** Spike A's `96012e8d8e` made `shouldPark` require proof that an abort + was a cooperative Stop, because inferring it from the stop reason alone would let any future + `controller.abort()` park a sandbox nobody had checked. The registry handed the applier a bare + `controller.abort()`. It now aborts with `USER_STOP_ABORT_REASON`, and two tests pin both + directions of the contract. + +A fifth, smaller one: two Stops **in the same instant** both inserted, because admission reads +for an open command and then inserts and neither request can see a row the other has not +committed. Sequential Stops always collapsed. A unique partial index over the open states now +makes the database decide, and the losing insert reads the winner back. + +--- + +## Live verification + +Stack: `http://144.76.237.122:9180`, project `agenta-ee-dev-session-cancel`, EE, dev images, +built from this worktree. The agent ran the `pi_core` harness on the local sandbox with an +OpenAI model. + +| Scenario | Result | Evidence | +|---|---|---| +| 1. Stop during a 60 s tool call | **Pass**, re-verified after the rebase. Turn ends at 26.2 s instead of 77.6 s. Command `pending` to `applied`, outcome `stopped`, settled 116 ms after the request. Runner logs `aborted`, then `harness_cancel sent=true settled=true elapsed_ms=17`, then `park-cancelled`. Next message recalled the codeword. | command `01a0641f-b775-75c1-bfe1-32a80e85f85e` | +| 2. Stop when nothing runs | **Pass.** 200, one row inserted already settled: `obsolete` with outcome `not_running`, no target, no Redis write. | command `01a0641f-5535-7130-a6be-537d287b6d9b` | +| 3. Stop with a stale `expected_execution_id` | **Pass.** 409 naming the current execution, and no row inserted. | `detail.current_execution_id` returned the live turn | +| 4. Two Stops in a row | **Pass.** Two simultaneous requests return the same command id and one row exists. Sequentially, the second now correctly reports nothing running, because a Stop settles in about 100 ms. | command `01a06423-c067-7c80-9b68-636953655698` returned to both | +| 5. Stop a turn parked for approval | **Pass.** The interaction goes `pending` to `cancelled`, the command settles `applied` with `not_running` in 68 ms, the pool keeps the entry, and the next message recalled the codeword. | command `01a06424-102b-76d0-a7cf-9e7d25c88041` | +| 6. Runner gone while a command is open | **Not settled, as expected.** No sweep exists in this slice. | see below | + +**Redis after a Stop, verified by direct inspection:** `running` gone, `alive` still present and +by then held by the resuming turn, and `superseded::session::turn:` +written. That is the same shape an ordinary turn end leaves, which is the point. + +**Scenario 6 in detail.** A command that is claimed and never reported stays `claimed`, and the +session's `stopping_turn_id` stays set, indefinitely. Observed directly: command +`01a0641a-d3c0-7980-8675-5349d0e3a118` sat `claimed` for over ten minutes with nothing to settle +it, and two session rows were left marked stopping. **This slice does not build the settlement +sweep.** The DAO exposes `expire_claims(now, max_deliveries)` and `settle_command` for it. The +handoff is to the branch `feat/session-execution-watchdog`, and the rule both sides must obey is +that one execution reaches exactly one terminal outcome from exactly one writer. It has to be +agreed before either lands; a second sweep racing the first is a worse bug than the one being +fixed. + +### Tests + +| Suite | Result | +|---|---| +| `api/oss/tests/pytest/unit/sessions/test_session_cancel_admission.py` | 15 pass. Admission guards, the arrival-time stamp, the collapse, the settlement, and the assertion that pins warm resume: `alive` survives a Stop. | +| `api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py` | 17 pass against a real Postgres. Two concurrent claims yield one winner, two concurrent admissions yield one command, the settle guard refuses a foreign replica, a terminal command cannot be settled twice. | +| `api/oss/tests/pytest/unit/sessions` (whole directory) | 553 pass. Four failures in `test_records_turn_span_dao.py` are a DNS failure reaching the tracing database from the host, unrelated to this branch. | +| `cd services/runner && pnpm test` | 2663 pass, 4 fail. All four are `gateway-run-turn-composition.test.ts`, verified failing on the base commit before any change here. | +| `cd web && pnpm lint-fix` | 25 tasks, no errors. | +| `ruff format` and `ruff check` in `api/` | Clean, run with the CI-pinned 0.15.12. | + +--- + +## What is left + +- **The settlement sweep.** Named above. It is the difference between "a Stop the runner missed + settles in two minutes" and "it never settles". +- **The long-poll adapter.** Not built. `AGENTA_SESSIONS_CONTROL_ADAPTER` defaults to `direct` + and any other value refuses to boot (`api/entrypoints/routers.py`) rather than falling back + silently to a transport the operator did not choose. Building it changes one file plus one + runner module, and no route, data shape, or transition. +- **The wrapper.** `POST /sessions/streams/` still does what it always did. Its cancel branch + becomes a call to `request_cancel` in the same change that flips mobile, so released clients + get the new behaviour with no client change. +- **The Fern client.** The desktop calls the new route through raw axios. Move it when the API + client is next regenerated. +- **Mobile.** Untouched, as the brief asked. + +--- + +## Open questions for Mahmoud + +1. **Is the exact `not_held` detector enough on its own, with no replica census?** Settled in + the revised design and built that way. Recommendation: **yes**. Reason: the census could not + tell two live replicas from one that had restarted and broke Stop after every deploy, while + the `not_held` condition is produced by nothing but the wrong-replica failure. Listed here + only so the removal is on the record. +2. **Who owns settling an abandoned command?** Recommendation: **the execution watchdog**, using + the DAO methods this slice exposes. Reason: one execution must reach exactly one terminal + outcome from one writer, and two sweeps racing to write `lost` is worse than the bug. Until + it exists, a Stop the runner never reports leaves the session reading "stopping" forever. +3. **Does the desktop read the execution id with an extra request?** Recommendation: **yes, as + built.** Reason: the cached liveness poll is up to 15 seconds stale and a stale id is refused + with a conflict, which would make Stop silently do nothing. The extra read costs about 30 + milliseconds inside a budget of five seconds. The alternative is to send no expectation, which + switches off the cheapest late-Stop guard. +4. **Should a Stop settle before the sandbox has finished parking?** Recommendation: **yes, as + built.** The runner reports as soon as it has issued the abort, about 70 milliseconds in, + while the park completes around a second later. Reason: the command's job is to deliver the + Stop, and waiting for the teardown would make a Stop that worked look stuck. The cost is that + `outcome = stopped` means "the cancel was delivered", not "the sandbox is parked". +5. **Do we keep `session_commands` rows forever?** Recommendation: **delete settled rows seven + days after `settled_at`**, as the design says. Not built here, because it belongs with the + sweep. Commands are operational state; durable history stays in `session_records`. diff --git a/docs/design/session-control-and-live-events/slice-records-ack.md b/docs/design/session-control-and-live-events/slice-records-ack.md new file mode 100644 index 00000000000..96a3e9220f7 --- /dev/null +++ b/docs/design/session-control-and-live-events/slice-records-ack.md @@ -0,0 +1,188 @@ +# Slice: the records worker acknowledges only what Postgres has + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +This slice closes the durability half of GitHub issue +[#5496](https://github.com/Agenta-AI/agenta/issues/5496) and all of +[#5594](https://github.com/Agenta-AI/agenta/issues/5594). It changes the records stream worker +and the shared stream consumer it runs on. It does not touch the runner, the records DAO, or +the records table. + +Every claim below marked **verified** was read in code at the cited `path:line`, proven by a +test in this branch, or observed in the live run in "What I verified". Nothing here is +reported from another document without saying so. + +## The answer + +Three defects deleted records and reported success. All three are fixed. + +| Defect | What happened | Where it is fixed | +| --- | --- | --- | +| The worker acknowledged records before the write | Every failed Postgres write deleted its records from Redis | `records_worker.py:277`, `:341`, `:349`, `:383` | +| One rejected record discarded its whole batch | A batch of fifty lost forty-nine good records to one bad one | `records_worker.py:192-232` | +| A record left unacknowledged was never redelivered | `read_batch` only asks for new entries, so "leave it pending" meant "lose it silently" | `consumer.py:181-268` | + +A fourth defect is fixed in its own commit because it is one line and unrelated to the worker. +Enterprise record retention referenced `RecordDBE.id`, an attribute the model does not have, so +the retention statement raised before it deleted anything. Records were never aged out. Fixed at +`api/ee/src/dbs/postgres/sessions/records/dao.py:107` and `:121`. Verified: `hasattr(RecordDBE, +"id")` is `False`, the key is `(project_id, record_id)` +(`api/oss/src/dbs/postgres/sessions/records/dbes.py:18`), and the corrected statement compiles +against the Postgres dialect. + +## What happened before this change + +The worker added every decoded Redis message id to its acknowledged list during +deserialization, before it tried the Postgres write. A failed `append_many` logged an error and +continued. The ids were still returned, and the shared consumer loop acknowledged and deleted +them from the stream. Verified in the previous revision of `records_worker.py` at lines 177, +184, 236-246 and 278, and in `shared/consumer.py:143-155`. + +Two consequences followed. + +1. Any Postgres failure deleted the records of the turn that was running. The user saw a + complete conversation on screen, and the durable transcript kept a hole. The runner rebuilt + later turns from that incomplete transcript. +2. `append_many` writes one statement in one transaction, so one record Postgres rejected took + its whole batch with it. Up to fifty unrelated records were lost per rejection. This is + #5594. + +A third problem was hidden underneath. The obvious fix, "do not acknowledge a failed batch", does +not work on its own. `read_batch` reads only `>`, which means new entries +(`consumer.py:118`, `:143`). An entry that is never acknowledged is invisible to every later +read of that consumer group. Without a reclaim pass, not acknowledging turns silent loss into a +pending list that grows forever and still never writes. Verified by test: +`test_unacknowledged_entry_comes_back_through_the_reclaim_pass` asserts that a second +`read_batch` returns nothing. + +## What the worker does now + +An id enters the acknowledged list for exactly three reasons. + +1. Its rows committed. +2. It could not be decoded, so a redelivery cannot help. Counted as a loss. +3. Its organization is over its records quota, which is a deliberate product drop. Counted as a + loss. + +Everything else stays pending and comes back. + +**The write path.** `_append_committed` (`records_worker.py:192`) calls `append_many` for the +whole project group. If that commits, every id in the group is acknowledged. If it fails and the +group holds more than one record, the worker writes the group one record at a time and +acknowledges only the records that committed. A rejected record stays pending on its own. + +**The reclaim pass.** `reclaim_batch` (`consumer.py:181`) runs at the top of the worker loop +(`consumer.py:333`). It asks Redis for the group's pending entries with `XPENDING`, claims them +with `XCLAIM`, and hands them back to `process_batch`. It is opt-in through `reclaim_pending`, +which is off for the tracing and events workers and always on for records +(`records_worker.py:98`). It runs at most once per idle window, so a busy stream does not add a +round trip per loop turn. + +**The retry bound.** Redis counts deliveries per entry. After `max_deliveries` deliveries the +worker drops the entry, logs at error with the session id, record id and record type, and +increments `dropped_messages` (`consumer.py:270`). The drop is data loss, and the log line is +what makes it countable. + +**The guard on the bound.** The delivery counter alone cannot tell a record Postgres will never +accept apart from a Postgres that is simply down. Both fail every delivery. Dropping on the +count alone therefore deletes every record in flight as soon as an outage outlasts +`max_deliveries` windows, which is the loss this slice exists to prevent. So the worker drops an +over-budget entry only while other records are committing (`consumer.py:167`, +`records_worker.py:181`). While nothing at all is writing, over-budget entries are kept and the +worker logs a warning instead. This is safe because a pending entry in a Redis stream does not +block later entries: `read_batch` keeps delivering new records the whole time. + +I found this hole in the live run, not in review. The first live run dropped all five records of +the second turn because the outage lasted ten reclaim windows. See "What I verified". + +## The retry policy, and why + +| Setting | Default | Environment variable | Meaning | +| --- | --- | --- | --- | +| `reclaim_idle_ms` | 30000 | `AGENTA_RECORDS_RECLAIM_IDLE_MS` | How long a failed record waits before the worker tries it again | +| `max_deliveries` | 5 | `AGENTA_RECORDS_MAX_DELIVERIES` | Deliveries after which a record is dropped, but only while other records are committing | + +Both live in `api/oss/src/utils/env.py:528` and `:532`, and are wired in the composition root at +`api/entrypoints/worker_streams.py:101-102`. + +Three choices are worth stating. + +**One record at a time, not a binary split.** A split costs about `2 log2(n)` calls when one +record is bad and about `2n` calls when Postgres is down. Writing one record at a time costs `n` +calls in both cases, and Postgres being down is the common case. The simpler rule is also the +cheaper one where it matters. + +**The reclaim lives in the shared consumer, not in the records worker.** It belongs next to +`read_batch` and `ack_and_delete`, which are the two halves it completes, and the tracing and +events workers have the same defect waiting for them. It is off by default, so this change alters +no other worker's behaviour. + +**A failed entitlements check now defers instead of dropping.** An over-quota organization is a +deliberate drop and is still acknowledged. An entitlements service that cannot be reached is a +transient failure, and its records now stay pending (`records_worker.py:334`). This is the same +defect class as the main bug, so I fixed it here rather than filing it. + +## What I verified + +**Unit tests.** `api/oss/tests/pytest/unit/sessions/test_records_worker_durability.py`, 11 tests. +The redelivery tests run against `fakeredis`, so the pending-list bookkeeping is real consumer +group behaviour rather than a mock of it. + +I also updated one assertion in +`api/oss/tests/pytest/unit/sessions/test_watch_publish.py:137-144`. That test pinned the old +acknowledge-before-write rule, and its own comment said it was not an endorsement of it. + +Full API unit suite, OSS and Enterprise: 3248 passed, 74 skipped, 0 failed. The skips need a +Postgres or an external key that this environment does not have. None of them cover the records +worker. `ruff format` and `ruff check` are clean at version 0.15.12, which is what continuous +integration pins. + +**Live run against a real Redis 8.** I did not deploy a stack. Swap on the box was fully used +(31 GB of 31 GB) and three other agent stacks were already running, so a fourth stack would have +put the others at risk. Instead I ran the real `RecordsWorker.run` loop against a throwaway +`redis:8` container on port 6399, with a write path that fails on demand. The script is at +`/tmp/claude-1000/-home-mahmoud-code-agenta-2/7c724667-82cd-41a6-ba0b-e47bc96b4f67/scratchpad/verify_records_ack.py`. +The container is stopped and removed. + +| Step | Result | +| --- | --- | +| Turn one, three records, healthy write path | 3 committed, `XLEN` 0 | +| Turn two, five records published while the write path is down for 20 seconds | 0 committed, `XLEN` 5, `XPENDING` 5, 0 acknowledged | +| Write path restored | All 8 records present, 0 duplicates, `XLEN` 0, `XPENDING` 0, 0 dropped | +| One always-rejected record among three good ones | The 3 good records committed, the rejected one stayed pending | +| Traffic resumes | The rejected record dropped at its budget, logged at error naming `sess-2::message`, `XLEN` 0, `XPENDING` 0 | + +This covers the substance of the scenario in the brief. It does not cover the real +`RecordsDAO.append_many` against a real Postgres, or the runner and the browser. Those are not +verified. + +## What I did not do + +- I did not deploy a docker compose stack, for the memory reason above. +- I did not change the runner's bounded retry, the records DAO upsert rule, or the records table. +- I did not add a metric or an alert for `dropped_messages`. It is a counter on the worker object + and a log line, nothing more. +- The tracing and events workers still acknowledge before their write. The mechanism to fix them + now exists, and turning it on for them is one constructor argument each. I left it off. + +## Open questions for Mahmoud + +1. **Is 5 deliveries over 30 second windows the right bound?** Recommendation: keep it. With the + health guard, the bound only applies while other records are committing, so it now measures + "this record is bad" rather than "the database is slow". Both values are environment + variables if a deployment disagrees. +2. **Should a dropped record raise an alert, not just a log line?** Recommendation: add one when + the observability plane is next touched, not now. The counter and the error log make the loss + countable, and Agenta runs one records worker, so the volume is small. +3. **Should the tracing and events workers get the same treatment?** Recommendation: yes, but as + a separate change. They have the same acknowledge-before-write defect, and the machinery is + already shared and off by default. Traces and events are less costly to lose than a + conversation, so they do not need to ride with this one. +4. **Enterprise record retention starts deleting records the day this ships.** It has never run + successfully, so old records have accumulated since the feature landed. Recommendation: + check the row count and the configured cutoff on the first deployment before the job runs, so + the first sweep is not a surprise. +5. **The reclaim pass makes a lost record land late rather than never.** A record can now be + written a minute or more after its turn ended. Recommendation: accept it. The runner rebuilds + history at the start of the next turn, not at the end of the previous one, so a late write is + still in time for the reader that matters. diff --git a/docs/design/session-control-and-live-events/slice-stop-guard.md b/docs/design/session-control-and-live-events/slice-stop-guard.md new file mode 100644 index 00000000000..00d4daf3638 --- /dev/null +++ b/docs/design/session-control-and-live-events/slice-stop-guard.md @@ -0,0 +1,363 @@ +# Slice: the Stop guard and pending cancel + +Branch `feat/session-stop-guard`. Three changes on the existing cancel path, no new transport, no +new table, no runner change. + +## What happens today + +Stop is `POST /sessions/streams/` with no inputs and `force=false`. The service classifies that as +CANCEL and calls `_displace_turns`, which tombstones whichever turn holds `alive` or `running` at +that instant and clears both keys. + +Two things follow from that, and both are bugs. + +1. **A Stop applied after its turn ended kills the next turn.** Nothing recorded which turn the + Stop meant, so the tombstone lands on whatever is there. The tombstone lives for 3600 s and its + TTL is refreshed on every read (`api/oss/src/dbs/redis/sessions/locks.py:147-153`), so the + session stays wedged. This is review finding H-3 and a plausible mechanism for #6417. +2. **A stopped session keeps a live approval card.** Kill cancels pending interactions + (`api/oss/src/apis/fastapi/sessions/router.py:441-444`); cancel did not. The card's buttons then + answer a turn that no longer exists (#6315). `requirements.md:149` asks for this and no work + package owned it (review open question 5). + +## What changed + +### 1. The cancel guard + +| Change | Where | +|---|---| +| `expected_execution_id` on the cancel request | `api/oss/src/core/sessions/streams/dtos.py:150-166` | +| `SessionTurnMismatch`, the refusal | `api/oss/src/core/sessions/streams/types.py:41-73` | +| The guard itself | `api/oss/src/core/sessions/streams/service.py:174-222` | +| `_displace_turns` takes the guard and reports what it killed | `service.py:224-297` | +| The cancel branch passes both guards | `service.py:384-411` | +| 409 mapping with both ids in the body | `api/oss/src/apis/fastapi/sessions/router.py:203-212` | + +`expected_execution_id` keeps the RFC's public name. Internally it is a turn id, which is the +coordination plane's word for one execution of a session, and the service maps the two at the +boundary. It stays optional in the contract, per D-010. A whitespace-only value is read as absent, +which is the safe failure. + +With the id present, cancel touches that turn or nothing. Another turn holding the session means the +turn the caller meant is already gone, so the request returns 409 and no key is written. A named turn +that holds nothing is still tombstoned, so a beat still in flight for it cannot re-take the session. + +With no id, cancel refuses a turn whose start is later than the request's arrival. Read the next +section before relying on that. + +### 2. Turn start times + +Nothing recorded when a turn started, and it cannot be derived. `session_turns.start_time` is +written by the runner after the fact, and a browser turn's id is a runner-minted uuid4 +(`services/runner/src/server.ts:188`), so it carries no timestamp. The slice adds one API-side Redis +key, in the same shape as the existing tombstone key and with the same lifetime as `alive`. + +| Change | Where | +|---|---| +| `started::session::turn:` | `api/oss/src/dbs/redis/sessions/contract.py:81-93` | +| `record_turn_start` (write-once) and `get_turn_start` | `api/oss/src/dbs/redis/sessions/locks.py:171-221` | +| Stamped when the API mints a turn | `service.py:1079-1086` | +| Stamped on a runner-minted turn's first beat | `service.py:601-613` | + +An absent record means unknown, never old, so a turn from before this shipped stays stoppable. + +This branch adds **no migration and no column**. `feat/session-durable-cancel` owns +`session_streams.turn_started_at`; when that lands it replaces this key and the two helpers in +`locks.py` can go. The Redis key is here because the alternative offered, comparing the turn id read +at the start of the cancel handler with the one read at the end, only catches a turn that changes +inside the handler, which is microseconds wide and catches nothing real. + +### 3. Stop cancels the stopped turn's pending interactions + +The cancel response now reports every turn it tombstoned (`cancelled_turn_ids` on +`SessionStreamCommandResponse`, `dtos.py:175-178`). The route reads it and calls +`cancel_session_pending` once per turn, scoped with the existing `only_turn_id` argument +(`router.py:413-433`). That helper already publishes the `interaction: resolved` watch event, so an +open browser refetches and re-renders. A cancel that ended no turn cancels every pending gate on the +session, because nothing holds the session and nothing can ever answer them. That is kill's +reasoning. + +The runner writes a gate with `request.turnId` (`services/runner/src/engines/sandbox_agent/run-turn.ts:708-714`), +which is the same id it heartbeats with, so the scoping matches what the runner produces. Verified in +code. + +### 4. The browser renders a cancelled gate as closed + +Replay already did: `settleApprovalPart` maps a `cancelled` interaction row to `output-denied` +(`web/packages/agenta-chat/src/assets/transcriptToMessages.ts:240-243`). The live path did not. The +in-memory pending list was not gated on `stopped`, unlike the two docks beside it, so a live card with +working buttons and hot keyboard shortcuts stayed up until a reload. + +`getLivePendingApprovals` (`web/packages/agenta-chat/src/model/approvals.ts:68-82`) holds the rule for +both clients. Desktop reads it at `web/oss/src/components/AgentChatSlice/AgentConversation.tsx:378-387` +and mobile at `web/mobile/src/features/chat/LiveConversation.tsx:191-196`. `stopped` clears on the next +send, so a new turn's gates appear normally. + +### 5. The concurrency limit no longer refuses a Stop + +Added scope, raised after the first pass. `check_runner_concurrency_limit` gated every mode, so a +project at its per-project run limit could not stop the very runs holding the limit: the one request +that frees capacity was the one refused with 429. Cancel starts nothing, so it is now exempt +(`api/oss/src/apis/fastapi/sessions/router.py:407-413`). + +The route needs the mode before the service runs, so the inputs-by-force matrix moved into +`derive_command_mode` (`api/oss/src/core/sessions/streams/service.py:144-160`) and both the route and +the service call it. One derivation, so the two cannot disagree about what a cancel is. + +### 6. A refused Stop reaches the user + +Added scope. The desktop Stop was fire-and-forget and `callFern` logs and returns null for every +non-abort failure, so a Stop the server refused was invisible: the transcript said "Stopped" while +the run continued and kept billing. Now the outcome is read. + +`cancelSessionStream` (`web/packages/agenta-entities/src/session/api/api.ts:629-690`) returns one of +three answers, `cancelled`, `stale`, or `failed`, carrying the server's own 409 message. It is a +separate function rather than a flag on `commandSessionStream` because the other callers of that +function deliberately ignore the result and use a null check that widening would break. + +The desktop reads it at `web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts:505-524`: +on a refusal it withdraws the local "Stopped" marker, shows a short notice, and invalidates the +liveness query so the running-elsewhere strip tells the truth. + +### 7. The mobile composer Stop calls the server + +Added scope. Mobile had the server-calling Stop only on the running-elsewhere strip, the button that +appears when the turn is NOT this device's. The composer's own Stop called `conversation.stop`, which +aborts this device's fetch and nothing else, so stopping your own turn on mobile left the run going +and billing. + +`stopHere` (`web/mobile/src/features/chat/LiveConversation.tsx:197-213`, wired at `:482`) now aborts +locally and sends the same cancel the desktop sends, with the same refusal handling. The strip's +`StopButton` moved to the same helper (`web/mobile/src/features/chat/StopButton.tsx:15-38`) and shows +the stale message instead of "try again", which would have sent the user round the same refusal. + +### 8. The browser sends the guard + +The client half of the turn id. The runner sends it as a `message-metadata` chunk, so it lands on +`message.metadata.turnId` beside the `sessionId` the start frame sets. It arrives third, before any +content, and the SDK merges metadata, so the finish frame's `traceId` does not overwrite it. It +cannot ride on the start frame itself: the SDK egress emits `start` before the runner is consulted. +The runner half is runner commit `ca600cb1e6` on `feat/session-single-turn-admission`; until it +lands no metadata arrives, nothing is stored, and Stop sends no guard, exactly as before. + +| Change | Where | +|---|---| +| `getMessageTurnId` and `latestTurnId`, the strict readers | `web/packages/agenta-chat/src/assets/agentTurn.ts:19-37` | +| The per-session store, cleared with the session's ephemera | `web/packages/agenta-chat/src/state/sessionEphemera.ts:33-56` | +| Desktop keeps the id and sends it | `web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts:353-360`, `:524-531` | +| Mobile keeps it (shared hook) and sends it | `web/packages/agenta-chat/src/hooks/useAgentConversation.ts:343-350`, `web/mobile/src/features/chat/LiveConversation.tsx:200-207` | +| `cancelSessionStream` carries it | `web/packages/agenta-entities/src/session/api/api.ts:629-680` | +| The typed client gained the field | `web/packages/agenta-api-client/src/generated/.../SessionStreamCommandRequest.ts` | + +The store is a Map beside the composer drafts rather than an atom, because nothing renders the id: +it is written once per turn and read once, when Stop is pressed. It is deliberately kept past the end +of the turn. A turn parked on an approval has finished streaming and is still the turn a Stop means, +which is exactly the state review finding H-2 is about. + +**Why an effect on the messages and not a callback.** The pinned `ai@6.0.0-beta.150` gives the +client chat exactly four callbacks: `onError`, `onToolCall`, `onFinish` and `onData` (`ChatInit` in +that package's `dist/index.d.ts:3121-3157`). There is no metadata callback. `onData` takes a +`DataUIPart` (`:3101`), so it never sees a `message-metadata` chunk, and `onFinish` is too late for +Stop, which happens mid-turn. `messageMetadataSchema` is a validation schema, not a hook. Reading +the merged metadata off the streaming message is therefore the only channel this version exposes. +Verified in the installed package, not assumed. + +It is in memory and never persisted with the messages, which is what makes it safe. Message metadata +does round-trip through the browser's message cache, so reading `metadata.turnId` straight off the +transcript at Stop time would name a turn from a previous page load. The Map starts empty after a +reload, so Stop then sends no guard, which is the old behavior rather than a wrong refusal. + +Three rules the code holds to, each because the wrong id refuses a Stop that is correct: + +- The readers are strict. A missing id, a blank id, or a non-string yields null, and null means send + nothing. `latestTurnId` consults only the NEWEST assistant message and does not fall back to an + older one, because an older message carries an older turn's id. +- The field is omitted, never sent as null, when the client never learned an id. The server then + falls back to its own arrival-time check. +- The running-elsewhere button on mobile sends no guard at all + (`web/mobile/src/features/chat/StopButton.tsx:15-21`). That turn runs on another device, so this + device never saw its metadata, and naming a turn it watched earlier would refuse a correct Stop. + +**The typed client was regenerated**, against this stack's own OpenAPI +(`clients/scripts/generate.sh --language typescript --url /api/openapi.json`). +The diff is two fields and nothing else: `expected_execution_id` on the request and +`cancelled_turn_ids` on the response. The checked-in client was otherwise already in sync. + +One transition hazard, stated rather than guarded: if a session's first turn carried the metadata and +a later turn did not, the stored id would be stale and that Stop would be refused. It needs a runner +version change in the middle of one session, and the refusal is visible and says why. The code does +not retry without the guard, because retrying unguarded is exactly the behavior #6417 is about. + +## The honest limit of the arrival-time guard + +**The arrival-time check does not close #6417 on its own. `expected_execution_id` does.** Both +first-party clients now send it, and the id reaches them only once the runner half lands on +`feat/session-single-turn-admission`. Until then the measurement below is what Stop does. + +Measured, not argued. Fourteen runs of the real race against the live stack: turn one takes the +session, then a Stop with no id and the next Send are fired together, Stop first. Results below. + +| Measurement | Result | +|---|---| +| Stops refused by the arrival-time guard | 0 of 14 | +| Runs where turn two was tombstoned | 1 of 14 | + +The guard never fired because in every run where turn two died, the Stop genuinely reached the API +after turn two had started. The check only catches a request that arrives before the turn starts and +is processed after it. That window is the permission check plus the concurrency check, both database +round trips, which is why the stamp is taken at the route's first line +(`router.py:385-391`) rather than inside the service. It is still small next to the client's own +network latency, which is the larger half of the race and which the server cannot see. + +The mechanism itself works. Forcing one turn's recorded start five seconds into the future and then +sending a Stop with no id returns 409 and leaves the turn holding `alive` and `running`, untombstoned. +That protocol is under "Live verification" below. + +A client-supplied age would close the gap without a clock-skew problem: the browser sends how many +milliseconds ago the button was pressed, and the server subtracts that from arrival. It is not in the +RFC and it is not in this slice. It is open question 2 below. + +## Live verification + +The scenarios ran against an access-controlled EE development deployment with a local sandbox +provider. Endpoint, project, container, and host-path identifiers are omitted from the repository. +The raw transcript is retained in the restricted test record. + +### (a) A Stop naming a turn that has ended is refused, and the new turn keeps running + +Turn one took the session, a steer replaced it with turn two, then a Stop named turn one. + +``` +--- STALE STOP: expected_execution_id = T1 --- +{"detail":{"message":"Session '' is running turn '', not the expected turn ''. Nothing was cancelled.", + "expected_execution_id":"", + "actual_execution_id":""}} +HTTP=409 +--- state after the refused stop --- +alive -> +running -> +tombstone(T2) exists -> 0 +tombstone(T1) exists -> 1 +``` + +A Stop naming turn two was then accepted, returned `cancelled_turn_ids`, and cleared `alive`. + +### (b) A Stop with no id does not tombstone a turn that started after it + +Constructed, because the timing cannot be forced from outside the process. One turn was started +normally, its recorded start was moved five seconds into the future, and a Stop with no id was sent. + +``` +forced start -> +--- Stop with NO expected_execution_id --- +{"detail":{"message":"Session '' started turn '' + after this cancel arrived, so the cancel is stale. Nothing was cancelled. + Send `expected_execution_id` to cancel a specific turn.", ...}} +HTTP=409 +alive -> +running -> +tombstone(T2) -> 0 +``` + +The unconstructed version of this scenario is the 14-run race above, which the guard did not catch. + +### (c) Stop cancels a pending gate and a late answer is refused + +The gate was created through `POST /sessions/interactions/`, the endpoint and body the runner uses, +with the same `turn_id` as the running turn. + +``` +status before Stop = pending turn_id = +=== STOP === +{"mode":"cancel","session_id":"","turn_id":"", + "detached":true,"cancelled_turn_ids":[""]} +HTTP=200 +status after Stop = cancelled +=== late answer === +{"detail":"Interaction is no longer pending"} +HTTP=409 +``` + +An open browser sees the refresh signal. The watch stream for the same sequence: + +``` +event: ready +event: interaction data: {"type": "interaction", "session_id": "...", "status": "pending"} +event: lifecycle data: {"type": "lifecycle", "session_id": "...", "state": "ended"} +event: interaction data: {"type": "interaction", "session_id": "...", "status": "resolved"} +``` + +Not verified live: a gate raised by a real agent turn rather than by the same endpoint the runner +posts to. The turn id the runner uses was checked in code, not on the wire. + +### (d) A project at its concurrency limit can still Stop + +The API was recreated with `AGENTA_SESSIONS_REDIS_CONCURRENCY_LIMIT=1`, driven, then recreated with +the setting removed. The stack is back on the default. + +``` +=== a SEND takes the one slot === HTTP=200 +=== a second SEND is refused === HTTP=429 + {"detail":"Concurrency limit of 1 concurrent runs reached for this project."} +=== STOP on the running session === HTTP=200 + {"mode":"cancel", ... "cancelled_turn_ids":[""]} +=== the freed slot lets the next SEND through === HTTP=200 +``` + +Before the change the third line was a 429. + +Not verified live: the desktop and mobile notices in a browser. Both need an agent run with a model +key, which this stack has no key for. The three outcomes of `cancelSessionStream` are unit-tested, +all four touched packages typecheck, and the web container compiled the chat route clean +(`✓ Compiled /w`). + +## Tests + +| Suite | File | Result | +|---|---|---| +| The guard, the start record, steer staying unguarded | `api/oss/tests/pytest/unit/sessions/test_cancel_stop_guard.py` | 12 passed | +| The route: pending gates and the concurrency exemption | `api/oss/tests/pytest/unit/sessions/test_cancel_cancels_pending_interactions.py` | 11 passed | +| The live approval rule | `web/packages/agenta-chat/tests/unit/model/liveApprovals.test.ts` | 3 passed | +| The Stop outcomes and the guard on the wire | `web/packages/agenta-entities/tests/unit/session-cancel-stream.test.ts` | 8 passed | +| The turn-id readers and their store | `web/packages/agenta-chat/tests/unit/assets/agentTurn.test.ts` | 9 passed | + +`api/oss/tests/pytest/unit/sessions/` as a whole: 505 passed, 41 skipped. The `@agenta/entities` +suite is 1472 passed and `@agenta/chat` is 634 passed. `pnpm lint-fix` in `web/` is clean, `ruff +format` and `ruff check` in `api/` are clean, and `@agenta/entities`, `@agenta/chat`, +`@agenta/mobile` and `@agenta/oss` all typecheck. + +## What is left + +- **The guard is inert until the runner sends the turn id in the message metadata.** Both clients read it and send it; + no runner on this branch emits it. The runner half is commit `ca600cb1e6` on + `feat/session-single-turn-admission` (commit `ca600cb1e6`). The stream row's `turn_id` was rejected as a source: it + reaches the browser through a 15 s liveness poll, and a stale id refuses a legitimate Stop of the + current turn, which is worse than the bug. +- The residual in-handler race: a turn that takes `alive` between `_displace_turns` reading the owners + and clearing them is still tombstoned. Microseconds wide, and closing it needs a Lua script or the + fencing that D-017 defers. +- The desktop and mobile notices, and the guard actually travelling from a browser, were not seen in + a browser. All of it is unit-tested and typechecked, and the wire body the client now sends was + driven by hand against the live API. Seeing it end to end needs the runner half plus an agent run + with a model key, and this stack has no key. + +## Open questions for Mahmoud + +1. **Should the browser's Stop carry how long ago the button was pressed?** Recommendation: yes, one + optional integer. Reason: it is the only thing that closes #6417 before a turn id reaches the + browser, it needs no clock agreement between client and server, and the measurement above shows the + server-side arrival stamp catches nothing on its own. +2. **Should the same turn id also guard the interaction responses?** Recommendation: yes, once the + runner half lands. Reason: `rfc.md:68-75` already asks for `expected_execution_id` on a response, + the browser will now have the id in hand, and an approval answered against a turn that has ended is + the same class of bug as a stale Stop. +3. **Should a refused Stop be a 409 or a quiet success?** Recommendation: 409 with both ids, as built. + Reason: the browser can retry with the id in the body, and a silent success would tell the user the + run stopped when it did not. +4. **Should Stop keep cancelling every pending gate when it ended no turn?** Recommendation: keep it. + Reason: nothing holds the session in that state, so no gate can ever be answered, and leaving them + pending reproduces #6315 for the case where the turn had already lapsed. +5. **Should closing a chat tab keep sending a cancel?** Recommendation: no. Reason: + `AgentChatPanel.tsx:138` is now the only Stop that still discards its outcome, and it fires on tab + close, which contradicts `requirements.md:98` and surprises anyone who closes a tab to reopen the + session elsewhere. If it stays, it should say so in the requirements and use the same helper. diff --git a/docs/design/session-control-and-live-events/slice-watchdog.md b/docs/design/session-control-and-live-events/slice-watchdog.md new file mode 100644 index 00000000000..2e5be47db1a --- /dev/null +++ b/docs/design/session-control-and-live-events/slice-watchdog.md @@ -0,0 +1,465 @@ +# Slice: the execution watchdog + +Branch `feat/session-execution-watchdog`. Commits `59fb1a7864` and `5bbd5a36df`. + +This slice makes one RFC requirement true: **every accepted execution reaches exactly one +durable terminal outcome within a bounded time** (`requirements.md:36`, D-016 at +`decisions.md:129-139`). It adds no table, no transport, and no new subsystem. + +## What happens today + +A turn can run out of ways to end. + +The runner writes its terminal record downstream of `await run(...)` +(`services/runner/src/server.ts:622`), and releases its alive watchdog in the `finally` around +that same await (`services/runner/src/server.ts:618`). Both are correct on every path where +`run()` returns. Neither happens when it does not. An await inside the run that never settles +leaves the heartbeat announcing `running=true` every thirty seconds for good, and each beat +re-arms a Redis lease whose TTL is an hour. + +The user sees a session that is running, refuses a new message, and never finishes. The only +exits were the thirty-minute idle threshold and pressing Stop. + +Three ways in, all reported: + +- The sandbox dies under the turn ([#6418](https://github.com/Agenta-AI/agenta/issues/6418)). + Verified: the agent-to-client half of the ACP channel is a long-lived SSE `GET`; when the peer + dies the transport's read loop swallows the severed stream and never fails the readable + (`services/runner/node_modules/acp-http-client/dist/index.js:335-339`), so the pending + `session/prompt` request is structurally incapable of settling. +- The runner itself is gone: a container restart, a crash, an OOM kill. Nothing on the runner + can write an outcome, because there is no runner. +- A write failure is swallowed and the turn beats on + ([#6100](https://github.com/Agenta-AI/agenta/issues/6100), + [#5327](https://github.com/Agenta-AI/agenta/issues/5327), + [#6099](https://github.com/Agenta-AI/agenta/issues/6099)). + +The existing run limits do not cover these. Time-to-first-byte (2 min) catches a sandbox that +dies before the first token and idle (30 min) catches one that dies mid-stream, but +`notePaused()` retires every timer permanently the moment a turn parks for a human +(`services/runner/src/engines/sandbox_agent/run-limits.ts:207-210`) — which is exactly when a +long turn is most likely to outlive its sandbox. Verified. + +An embryonic watchdog already existed. `orphan_sweep.py` found stale rows and cleared their +Redis nest, but it wrote nothing to the transcript and told no open browser, so a swept +session's conversation simply stopped mid-turn. Verified before this change. + +## What this slice changes + +Two halves, and **they close different bugs**. Neither one alone is enough, and it is worth +being precise about which does what, because the obvious reading is wrong. + +| Failure | Detected by | Why the other half cannot | +|---|---|---| +| The sandbox dies under the turn ([#6418](https://github.com/Agenta-AI/agenta/issues/6418)) | The runner's sandbox liveness probe | The runner is healthy and keeps beating, so its heartbeat never goes stale and the API scan never sees the row | +| The runner is gone: restart, crash, OOM | The API watchdog | There is no runner left to detect anything | + +**The API watchdog does not close [#6418](https://github.com/Agenta-AI/agenta/issues/6418), and +cannot.** It keys off heartbeat age, and a wedged turn's own heartbeat stays perfectly fresh — +only the machine underneath is gone. The runner-side probe is what closes that one, and it is +proved live in Scenario B below. This was flagged from the Stop map before the work started and +it held up in the live test: at the moment of the kill the runner logged `ECONNREFUSED` on the +ACP socket and `heartbeat OK ... running=true` in the same second. + +### API: settle an execution whose runner cannot report one + +`api/oss/src/tasks/asyncio/sessions/orphan_sweep.py`, extended rather than duplicated. A second +job scanning the same rows would race this one: whichever collapsed the flags first would hide +the row from the other, and the terminal record would sometimes never be written. + +For each stale row that claims a running turn, in this order: + +1. Write the two records the dead runner owed, `_lost_turn_records` at `orphan_sweep.py:112`. +2. Collapse the row's flags so the session reads as ended. +3. Clear the Redis nest and tombstone the turn, so a late beat cannot re-nest it. +4. Publish the watch notification on the session channel and the project channel. + +Step 1 is deliberately first. A crash between the steps leaves the row a candidate for the next +pass, which is recoverable; collapsing the flags first would hide the row forever with no +ending ever written. + +**The records mirror the runner's own error path exactly**: an `error` event carrying the class +a client can act on, then the terminal `done`. A lone `done` would render as a clean finish, +which is the opposite of what happened. The message is character-for-character the runner's +`EXECUTION_LOST_MESSAGE`, so one outcome never reaches the user in two wordings. + +**Idempotent twice over.** A stable `uuid5` per (turn, record) (`orphan_sweep.py:94`) means the +ingest upsert writes the same two rows however many passes or replicas see the turn. And +`RecordsDAO.settled_turns` (`api/oss/src/dbs/postgres/sessions/records/dao.py:233`) asks, in one +query per project, which turns already carry a terminal record — because a runner can die +*after* writing its outcome but *before* its final `is_running=false` beat lands. That turn is +already settled; its row still needs collapsing, but a second, contradictory ending would +corrupt the transcript. The records table lives in the tracing database and the stream rows in +the core database, so this is a two-phase read, never a join. + +If that lookup fails, the pass writes nothing and still collapses the row. Saying nothing is +better than inventing a second ending. + +**Only a turn that still claims `is_running` is eligible**, and that is what protects a parked +approval. A turn that parks for a human sends one final beat with `is_running: false` +(`services/runner/src/sessions/alive.ts:241-252`) and then stops beating on purpose, so its +heartbeat goes stale within seconds. It is also the state we most need to keep: the sandbox is +warm and the user is about to answer. Such a row never becomes a candidate for a terminal +record however long it sits. It is still reclaimed after thirty minutes, which is the +pre-existing sweep behaviour keyed to the approval TTL, but no ending is written for it. +Pinned by `test_a_parked_approval_is_never_settled`. + +### Runner: never wait on a run forever + +`services/runner/src/sessions/turn-settle.ts` (new). `awaitTurnOrAbandon` wraps the run in +`server.ts`. It waits normally, and gives up when the platform says the turn is no longer +current, or when the hard deadline elapses. Giving up is two steps: `abort()` first, because +most hangs do unwind from an abort, and only if the run is still pending after the grace window +does the request write the outcome itself and stop waiting. + +This closes the loop with the API half. When the watchdog settles a turn it tombstones it, so +the wedged runner's next heartbeat answers `is_current_turn: false`, which already aborts the +run (`services/runner/src/sessions/alive.ts:207`). Where that abort lands somewhere the signal +is observed, the turn ends cleanly. Where it does not, the grace window ends the request anyway. + +`services/runner/src/engines/sandbox_agent/sandbox-liveness.ts` (new) covers the case the API +cannot see: a dead sandbox under a runner that is still beating happily. It probes the daemon's +own health route, a different socket from the wedged ACP channel, and trips the existing +run-limit path after three consecutive failures. Any HTTP status counts as alive, 401 and 404 +included: the question is whether something is listening, and only a transport failure answers +it. + +The turn is closed to further events once the request has written its outcome +(`services/runner/src/server.ts`, the `gatedEmit` wrapper). An abandoned run that unwinds +minutes later must not append a second ending. + +The heartbeat itself gained a request timeout and an in-flight guard +(`services/runner/src/sessions/alive.ts`). Both beats used a bare `fetch` with no signal, so a +stalled socket never settled: beats piled up behind it, and the final beat in `release()` could +hold the whole request open after the turn had ended. + +### Web + +Two small changes, both found by tracing what a browser does when the records land. + +- `execution_lost` joins `RETRYABLE_CODES` + (`web/oss/src/components/AgentChatSlice/components/AgentMessage.tsx:154`). The retry wiring + already existed; the code was simply in no branch, so the failed turn offered no action. +- The desktop watch now listens for `lifecycle` + (`web/oss/src/components/AgentChatSlice/hooks/useSessionRecordsWatch.ts`). It previously + registered only `ready`, `records-changed` and `interaction`, so the watchdog's `ended` event + was received by the EventSource and discarded, and the session kept *looking* alive until the + next fifteen-second liveness poll. Mobile already did this. + +The error itself needed no frontend change: the replay adapter already folds +`{type: "error", message, code}` onto the interrupted turn and `done` already closes it. + +## The timeouts, and how to change them + +Every value is a setting. Nothing here needs a redesign to tune. + +| Setting | Default | Environment variable | +|---|---|---| +| Heartbeat age before a running turn is lost | 90 s | `AGENTA_SESSIONS_WATCHDOG_STALE_HEARTBEAT_SECONDS` | +| Grace before an alive-but-idle row is settled | 1800 s | `AGENTA_SESSIONS_WATCHDOG_IDLE_GRACE_SECONDS` | +| How often the watchdog runs | 60 s | `AGENTA_SESSIONS_WATCHDOG_INTERVAL_SECONDS` | +| Rows settled per pass | 500 | `AGENTA_SESSIONS_WATCHDOG_BATCH_SIZE` | +| Sandbox probe interval | 30 s | `AGENTA_RUNNER_SANDBOX_PROBE_INTERVAL_MS` | +| Sandbox probe timeout | 10 s | `AGENTA_RUNNER_SANDBOX_PROBE_TIMEOUT_MS` | +| Consecutive probe failures before the sandbox is declared gone | 3 | `AGENTA_RUNNER_SANDBOX_PROBE_FAILURES` | +| Hard per-turn deadline | 11.5 h | `AGENTA_RUNNER_TURN_HARD_DEADLINE_MS` | +| Grace after an abort before the request stops waiting | 60 s | `AGENTA_RUNNER_TURN_ABANDON_GRACE_MS` | +| Heartbeat request timeout | 15 s | `AGENTA_RUNNER_HEARTBEAT_TIMEOUT_MS` | + +Definitions live in `api/oss/src/utils/env.py:564` (`SessionWatchdogConfig`), +`services/runner/src/engines/sandbox_agent/sandbox-liveness.ts` and +`services/runner/src/sessions/turn-settle.ts`. + +Two of these deserve their reasoning stated. + +**The rule is heartbeat age, not lease expiry, and the difference is an hour.** The Redis +`alive` and `running` keys carry a 3600-second TTL (`api/oss/src/utils/env.py:1416-1422`), so a +watchdog phrased as "settle shortly after the lease expires" would leave a dead turn running for +an hour. The runner beats every 30 seconds +(`services/runner/src/sessions/contract.ts:18`) and the beat is mirrored onto +`session_streams.updated_at`, so the age of that column is the real signal. The threshold is 90 +seconds of it: three missed beats. It was a flat 300 seconds, which was defensible while the +sweep only collapsed flags and nobody saw the result, and is too long now that it writes an +ending a user reads. + +**The hard per-turn deadline sits ABOVE the longest legitimate run, not below it.** The run +limits already own when a real turn should stop, and users have asked for longer runs, not +shorter ones ([#6084](https://github.com/Agenta-AI/agenta/issues/6084), +[#5356](https://github.com/Agenta-AI/agenta/issues/5356)). A turn that reaches this deadline is +one whose own limits already tripped and failed to end it. `AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS` +is unchanged. + +## Tests + +**API**, `api/oss/tests/pytest/unit/sessions/test_execution_watchdog.py`, 8 tests, all passing. +A lost turn gets an `error` then a `done`; a second pass writes no second ending; record ids are +stable across passes; an idle row owes no ending; a running row with no turn id is settled +silently; open readers are told the session ended; the Redis nest follows the settled row; a +failed lookup never invents an ending. + +The existing `test_orphan_sweep_thresholds.py` and `test_orphan_sweep_clears_redis.py` still +pass. Their fixtures gained the `turn_id` column, and the threshold assertion now names 120 +seconds with the reason written down. + +**Runner**, vitest, 14 tests, all passing. +`services/runner/tests/unit/sandbox-liveness.test.ts` (6): the threshold of consecutive +failures, tolerance of a single blip, a probe that *hangs* counted as a failure, and firing at +most once and never after dispose. `services/runner/tests/unit/turn-settle.test.ts` (8): the +happy path leaves no timer armed, a rejecting run still reaches the caller's own catch, an +interruption aborts first, a run that will not unwind hands back a reason, the hard deadline +works with no interruption signal at all, and an abort that throws does not break the settle. + +Full suites: `services/runner` 2642 unit tests pass. `api/oss/tests/pytest/unit/sessions` 333 +pass. 11 modules in that directory error on import with +`cannot import name 'InvalidHarnessKindError' from 'agenta.sdk.agents'`; that is pre-existing, +confirmed by running the same command on the unmodified tree, and comes from borrowing the main +checkout's virtual environment, whose SDK is installed from a different tree. + +Commands: + +``` +cd services/runner && pnpm exec vitest run --project unit +cd api && PYTHONPATH=$PWD python -m pytest oss/tests/pytest/unit/sessions/ -q +``` + +## Live verification + +Stack: `agenta-ee-dev-session-watchdog` at **http://144.76.237.122:8880**, EE, dev images, +local sandbox provider, its own Postgres on 5442. Deployed from this worktree at commit +`59fb1a7864`; the runner picked up `5bbd5a36df` by hot reload. Images were 40 minutes old at +deploy time, so `--build` was skipped as the brief allows. + +### Scenario A: the runner is gone + +A turn was opened by beating `POST /sessions/streams/heartbeat` once with +`is_running: true` — the runner's only liveness contribution — and then going silent, which is +byte-for-byte what a runner that died produces. + +Before: the Redis lease had 3586 seconds left, a second turn asking for the session got +`is_current_turn = False`, and the session had zero records. + +``` +2026-09-02T21:26:29.624Z [WARN.] watchdog: settled a session_stream whose runner went silent + extra={'session_id': 'wd-scenario-a-161c2d24', + 'stream_id': '01a06402-25b6-7072-97ea-9164efb69baf', + 'turn_id': 'e79207c5-813c-4913-98b8-a12d244afefb', 'lost': True} +2026-09-02T21:26:29.643Z [INFO.] watchdog: settled 1 sessions (1 turns marked lost) +``` + +The row was created at 21:24:17 and settled at 21:26:29, so 132 seconds. That run used the +earlier 120-second threshold; at the 90-second threshold this slice now ships, the same case +settles between 90 and 150 seconds depending on where the sweep tick falls. The behaviour under +test is unchanged — only the constant moved. + +After, all four verified by reading the stores: + +| Check | Result | +|---|---| +| Records for the turn | `error` (`code: execution_lost`) at `21:26:29.623`, then `done` at `21:26:29.624` | +| Stream row flags | `is_alive: false, is_running: false, is_attached: false` | +| Redis `alive` / `running` / `owner` | all empty | +| Redis `superseded:...:turn:` | `1` | +| A new turn on the same session | `is_current_turn = True` | + +### Scenario A1: re-run at the 90-second threshold, beside a parked approval + +Run again after the threshold changed from 120 seconds to 90, on a redeployed stack, with two +sessions opened in the same second so the two rules are tested against each other: + +- one turn beating `is_running: true` and then going silent, which is a runner that died; +- one turn sending a final beat with `is_running: false` and then going silent on purpose, + which is a turn parked for a human. + +Both were opened at 21:56:24. The constants in the running container, read from the live process: + +``` +running threshold (heartbeat age): 90 seconds +idle threshold: 1800 seconds +sweep interval: 60 seconds +``` + +``` +2026-09-02T21:58:02.911Z [WARN.] watchdog: settled a session_stream whose runner went silent + extra={'session_id': 'wd-90s-cdb259fb', ..., 'lost': True} +2026-09-02T21:58:02.926Z [INFO.] watchdog: settled 1 sessions (1 turns marked lost) +``` + +One session, not two. 98 seconds from the last beat, which is the 90-second threshold plus the +part of a sweep interval that had still to run. + +| Session | Records written | Row after | +|---|---|---| +| Runner died | `error` (`code: execution_lost`), then `done` | `is_alive: false, is_running: false` | +| Parked for a human | none | `is_alive: true, is_running: false` | + +The parked session kept its warm, resumable state and was given no ending, while sitting on a +heartbeat that had been stale for the same 98 seconds. That is the whole point of eligibility +resting on `is_running` rather than on silence alone. + +### Scenario A2: the runner wrote its outcome but lost its final beat + +The idempotency guard, on a real deployment. A turn was opened, the runner's own `done` record +was ingested, and the beating stopped. + +``` +2026-09-02T21:29:29.654Z [WARN.] watchdog: settled a session_stream whose runner went silent + extra={'session_id': 'wd-already-settled-4863aef0', ..., 'lost': False} +2026-09-02T21:29:29.663Z [INFO.] watchdog: settled 2 sessions (1 turns marked lost) +``` + +`lost: False`, and the session still holds exactly one record: the runner's own `done`. The row +was collapsed and the Redis nest cleared, with no second ending invented. The other session +settled in the same pass was a genuinely different lost turn, and it got its own single +`error` + `done` pair. + +### Scenario B: the sandbox dies under the turn + +A real agent turn on the local sandbox provider (codex harness, OpenAI through the vault), asked +to run `sleep 240`. Once the tool call was in flight, the sandbox's process group was killed +from outside the runner. + +The kill produced exactly the reported failure shape, and this is what made the first attempt +worth having: + +``` +Error: connect ECONNREFUSED 127.0.0.1:35171 + at async StreamableHttpAcpTransport.postMessage (acp-http-client/src/index.ts:406:21) +[sandbox-agent] unhandledRejection: TypeError: fetch failed +[sessions/alive] heartbeat OK session=wd-sandbox-gone-3eb8bb02 turn=0bc24bf1... running=true +``` + +The ACP socket was refusing every write while the heartbeat kept reporting the turn as running, +and the turn never ended. **The first version of the probe did not catch it**, because it called +`SandboxAgent.getSession()`, which reads a local persist driver and never touches the daemon — +so it answered happily while the sandbox was dead. That is fixed in `5bbd5a36df` and written +into the module docstring so nobody reaches for it again. + +Re-run with the corrected probe. The sandbox was killed at 21:32:08, mid tool call: + +``` +[sandbox-agent] [sandbox-liveness] probe failed (1/3): fetch failed +[sessions/alive] heartbeat OK session=wd-sandbox-gone-75399fc3 turn=c682ebb5... running=true +[sandbox-agent] [sandbox-liveness] probe failed (2/3): fetch failed +[sessions/alive] heartbeat OK session=wd-sandbox-gone-75399fc3 turn=c682ebb5... running=true +[sandbox-agent] [sandbox-liveness] probe failed (3/3): fetch failed +[sandbox-agent] [sandbox-liveness] sandbox is gone: 3 consecutive liveness probes failed (last: fetch failed) +[sessions/alive] heartbeat OK session=wd-sandbox-gone-75399fc3 turn=c682ebb5... running=false +``` + +The turn ended at 21:33:30, 82 seconds after the kill, and the last line is the point of the +whole exercise: the beat that used to say `running=true` for ever now says `running=false` once +and stops. + +The client's stream carried a real ending rather than closing on a broken pipe: + +``` +error: {"type": "error", "errorText": "The sandbox running this session stopped responding, + so the run was ended. Send the message again to start a fresh sandbox."} +finish: {"type": "finish", "messageMetadata": {...}} +``` + +And the durable transcript for that turn, read back from the records endpoint: + +| Record | Content | +|---|---| +| `message` | the user's prompt | +| `message` | "I'm running the command and will report its output when it completes." | +| `tool_call` | `sleep 240 && echo finished` | +| `usage` | the turn's token accounting | +| `error` | `code: sandbox_gone`, with the line above | +| `done` | terminal | + +The stream row ended as `is_alive: true, is_running: false`. That is the intended result and not +an oversight: the turn is over, and the session stays alive and reattachable. Only the runner +being gone entirely makes a session not alive. + +Without this change the same kill produced, and stopped at, this — captured on the first attempt: + +``` +Error: connect ECONNREFUSED 127.0.0.1:35171 +[sandbox-agent] unhandledRejection: TypeError: fetch failed +[sessions/alive] heartbeat OK session=... running=true <- for ever +``` + +### Reproducing it + +```bash +docker exec agenta-ee-dev-session-watchdog-runner-1 sh -c 'ps -eo pid,args | grep "[s]andbox-agent server"' +docker exec agenta-ee-dev-session-watchdog-runner-1 sh -c 'kill -9 -' +docker logs -f agenta-ee-dev-session-watchdog-runner-1 2>&1 | grep -E "sandbox-liveness|turn-settle" +docker logs -f agenta-ee-dev-session-watchdog-api-1 2>&1 | grep -i watchdog +``` + +**The stack has been torn down.** It ran on port 8880 as `agenta-ee-dev-session-watchdog` while +the scenarios above were recorded, and was stopped with `--down` once they were, to give the box +back its memory. Volumes were kept, so a rebuild is a redeploy rather than a fresh database. + +To bring it back, from this worktree: + +```bash +set -a && . hosting/docker-compose/ee/.env.ee.dev.watchdog && set +a +bash ./hosting/docker-compose/run.sh --license ee --dev --env-file .env.ee.dev.watchdog --no-tunnel +``` + +Two notes for whoever does. The env file is gitignored and carries a stack-local +`AGENTA_SERVICES_INTERNAL_KEY`, which is not in the template and which the deploy refuses to +start without. And the QA OpenAI key lives in that stack's vault, never in this repository. + +## What this slice does not do + +- It does not change `shouldPark` or any teardown rule. An abandoned run keeps its environment + and still runs its own teardown if it ever unwinds. Reclaiming machines stays with the + keep-alive pool. +- It does not close the turns ledger. `session_turns.end_time` still stays NULL on a lost turn. + `SessionTurnsDAO.complete` is idempotent and safe to call, but it needs the turn index, which + is an extra read per row, and nothing in the transcript depends on it. +- It does not hold a distributed lock across API replicas, because deterministic record ids make + a concurrent pass harmless rather than merely unlikely. Two replicas would each do the work; + neither would write a duplicate. +- It does not fix the originating tab. `refreshFromRecords` deliberately early-returns while the + tab is busy, so a tab holding an open-but-dead HTTP stream ignores the watchdog's records until + its own stream errors. Other tabs and a reload see the settled turn immediately. + +## Handover: command settlement + +The durable-cancel slice writes a command's terminal outcome and deliberately does not sweep +expired claims. Its DAO exposes `expire_claims(now, max_deliveries)` and `settle_command` for +this watchdog to call, on the principle that one execution reaches one terminal outcome from +one writer, and that the watchdog is that writer. + +**Agreed, and not built here.** Those functions do not exist on this branch, so code written +against them could be neither compiled nor tested, and a second sweep beside this one is exactly +the race this slice avoided. The work is small and belongs in the pass that already exists: once +the commands slice lands, extend `run_orphan_sweep` to expire claims and settle each expired +command in the same loop that settles its execution. + +## Open questions for Mahmoud + +1. **Is 90 seconds of heartbeat silence the right time to declare a turn lost?** + *Recommendation: ship it and watch.* It is three missed beats at the runner's 30-second + cadence, and it is a setting rather than a constant, so a wrong answer costs a restart rather + than a redesign. The old 300 was chosen when the sweep only collapsed flags and nobody saw the + result. The risk to watch for is the opposite of the obvious one: not settling a turn too + late, but settling a live turn whose runner was merely slow to beat. + +2. **Should the watchdog also close the turns ledger?** *Recommendation: not in this slice.* + `session_turns.end_time` stays NULL on a lost turn, which is a real inconsistency, but nothing + reads it for the transcript and closing it costs a query per row. Worth doing when something + actually reports on turn durations. + +3. **Should the sandbox probe run on Daytona too, given it cannot tell a deleted sandbox from a + proxy error?** *Recommendation: yes, leave it on.* It is a strict improvement where the proxy + does refuse, it costs one request per turn per thirty seconds, and the API watchdog is the + backstop where the proxy answers for a sandbox that is gone. + +4. **Does a lost turn deserve a distinct look in the transcript, rather than the same red + callout as a model failure?** *Recommendation: leave it as it is for now.* The copy and the + Try again button say the useful part, and a new visual state is worth designing only once we + know how often users see this. + +5. **The runner's SSE read loop swallows a severed stream instead of failing the pending + request** (`acp-http-client/dist/index.js:335-339`). That is the true root cause of + [#6418](https://github.com/Agenta-AI/agenta/issues/6418), and this slice bounds it rather than + fixing it. *Recommendation: raise it upstream rather than growing the local patch.* The patch + file already carries four changes, and a fifth in the read path is the kind that breaks + quietly on the next version bump. diff --git a/docs/design/session-control-and-live-events/spike-a-sandbox-cancel.md b/docs/design/session-control-and-live-events/spike-a-sandbox-cancel.md new file mode 100644 index 00000000000..c125b9533db --- /dev/null +++ b/docs/design/session-control-and-live-events/spike-a-sandbox-cancel.md @@ -0,0 +1,422 @@ +# Spike A: cancelling a turn without losing the warm sandbox + +> AGENT-GENERATED, low weight. Findings and a first implementation. Mahmoud makes final decisions. + +Status: the six questions are answered, the runner change is written and unit tested, and the live +scenario passed on the local sandbox for two harnesses. The Claude harness is not tested, because +this stack has no Anthropic key. + +**Codex process reaping is best effort after a settled Stop.** Pi kills its child; Codex does not. +The runner attempts to reap the Codex child and records cleanup misses for QA. A cleanup miss does +not revoke warm reuse or continuity; the 600-second stopped-session window bounds any leftover +process. + +## The answer in one paragraph + +A user Stop can keep the sandbox warm today, and the change to do it is small. The runner already +receives the Stop through its heartbeat and already ends the turn as `cancelled` rather than as an +error. Two things were missing. First, nothing told the harness to stop: the abort only made the +runner stop waiting, so the harness kept an open prompt and a running tool, and only the teardown +that was already deleting the sandbox ever stopped it. Second, `shouldPark` answered `false` for +every aborted run, so a Stop always deleted the sandbox. The fix sends the ACP `session/cancel` +notification, waits for the harness to answer its open prompt, and parks when it does. Live, Pi +answered in 14 ms and Codex in 22 ms, and the next message reused the same sandbox and the same +native harness session. + +## What was tested, and on what + +One table for the whole spike, so nobody has to infer coverage from the prose. "Live" means the +scenario in "The live test" ran against a real deployment; everything else is a code read. + +| Harness | Live test | What `session/cancel` does to the in-flight tool | Evidence | +| --- | --- | --- | --- | +| Pi (`pi_core`) | yes, local sandbox | harness answers the prompt in 14 to 31 ms, and the shell child is GONE | live, process probe returned `NO_SLEEP_PROCESS` | +| Codex | yes, local sandbox | harness answers the prompt in 22 ms; the runner reaps the shell child before parking | live process tree captured the leak; the runner reap is covered at the turn boundary | +| Claude Code | no, this stack has no Anthropic key | not measured | expected to match, from code: the runner branches on capabilities, never on harness name, and sends the same ACP notification to all three | + +| Sandbox provider | Live test | Note | +| --- | --- | --- | +| local | yes, every run | The "sandbox" is a process tree in the runner container. | +| Daytona | no | The park-versus-delete decision costs real money here, so it belongs in the release gate. No snapshot rebuild is needed for the runner-side cancel; a Codex bridge fix would need one. | + +Before this change, the abort sent NO cancel to Claude Code or Codex at all: it resolved a local +promise and left the harness working (`services/runner/src/engines/sandbox_agent/run-turn.ts`, the +cancel race). Only Pi sent one, and only as a side effect of its trace-flush path calling +`destroySession`. All three now get a real cancel. + +## The six questions + +### 1. Which request cancels a running prompt, and where is the guard? + +The request is the ACP `session/cancel` notification. It is the same request for all three +harnesses, because the runner talks to every harness through the same Agent Client Protocol +adapter. There is no per-harness cancel. + +The guard is in the vendored TypeScript client only. `sandbox-agent`'s `SandboxAgent` refuses a +caller-sent cancel: + +```js +var MANUAL_CANCEL_ERROR = "Manual session/cancel calls are not allowed. Use destroySession(sessionId) instead."; +... +async sendSessionMethodInternal(sessionId, method, params, options, allowManagedCancel) { + if (method === SESSION_CANCEL_METHOD && !allowManagedCancel) { + throw new Error(MANUAL_CANCEL_ERROR); + } +``` + +`services/runner/node_modules/sandbox-agent/dist/chunk-TVCDKGSM.js:561` and `:1550` (verified). +The public `rawSendSessionMethod` passes `allowManagedCancel: false`; only `destroySession` passes +`true` (`:1407`). + +The guard is NOT in the daemon. The daemon is a Rust binary +(`@sandbox-agent/cli-`, resolved at `services/runner/src/engines/sandbox_agent/daemon.ts:26`) +that proxies ACP over HTTP. The client sends the cancel as a plain notification with no response +envelope (`services/runner/node_modules/acp-http-client/dist/index.js:115`), and the runner already +sends exactly this notification on every teardown through `destroySession` +(`services/runner/src/environment/harness-session-lifecycle.ts:163`). Verified live: the new +cancel reached the adapter and both harnesses answered. + +`destroySession` is misleadingly named. It sends the cancel, resolves the client's pending +permission requests, and stamps `destroyedAt` on its own local record. It does not tell the daemon +to drop the session, and `resumeSession` clears `destroyedAt` again +(`node_modules/sandbox-agent/dist/chunk-TVCDKGSM.js:1364`). + +### 2. Does the cancel preserve the native harness session? + +Yes, verified live for Pi and Codex. ACP requires the agent to end the open `session/prompt` with +`stopReason: "cancelled"` after a cancel, and both harnesses did: the runner logged +`prompt stopReason=cancelled` in every run. The ACP session stays bound, so the next turn on the +same environment prompts the same native session with no reopen. The live proof is the second +turn recalling a codeword from the first, with no `create_session` stage in the log. + +What the harness reports is the prompt's own answer, not a separate frame. The runner reads the +settlement as "the prompt promise resolved", which is the harness saying it is idle again. + +### 3. What happens to a running tool call and a partial message? + +**In the transcript, the same on every harness.** The runner closes it honestly: on `cancelled` it +drains the queued ACP frames, keeps any real tool completion that already arrived, and settles +every still-open tool call with the `INTERRUPTED_BY_USER` sentinel +(`services/runner/src/engines/sandbox_agent/run-turn.ts:1305`, verified). No orphaned running part +and no invented success. Live, the browser-visible stream for the cancelled turn ended +`tool-input-available`, `tool-output-error`, `finish-step`, `finish`, and the partial assistant text +that had already streamed stayed in the stream. + +**In the sandbox, the harnesses differ, and this is the finding that needs a decision.** The +transcript says the tool was interrupted. Whether the PROCESS actually stopped is a separate +question, and the answer is not the same for both harnesses. Measured by cancelling a running +`sleep`, then asking the next turn to run `ps -eo pid,etimes,args | grep '[s]leep '`: + +| Harness | Cancel answered | Shell child after the Stop | +| --- | --- | --- | +| Pi (`pi_core`) | 14 to 31 ms | gone (`NO_SLEEP_PROCESS`) | +| Codex | 22 ms | still running | + +The Codex reading is unambiguous. One probe returned two leftovers at once, `sleep 120` at 84 +seconds elapsed and `sleep 300` at 31 seconds elapsed, which are the cancelled turns of two +different sessions, so the child survives its own turn AND the session that spawned it. + +**Parking can expose the original leak.** The runner performs a best-effort cleanup in +`reap-exec.ts`: after the cancelled prompt settles, it finds the `codex app-server` below this +sandbox's daemon, selects only descendants started during the stopped turn, and checks that +`kill -9` exits successfully before reporting them reaped. The turn-boundary test pins the order as +cancel, process scan, reap, then park. Failed or unknown cleanup is recorded for QA, while the +settled Stop still preserves the sandbox and native session for the 600-second stopped window. + +**What reaches the API.** The turn's `message`, `tool_call` and `tool_result` rows, a `usage` row, +and the terminal `done` row were present in the live runs. The terminal record carries +`stopReason: "cancelled"` (see below). When the harness confirms cancellation, the runner completes +the turn ledger row and preserves the native-session continuity record. Reap outcomes do not alter +that confirmation. + +### 3b. A stopped turn is now distinguishable from a completed one + +The runner used to drop `stopReason` from the terminal `done` record unless it was exactly +`"paused"`, so nothing downstream could tell a Stop from a normal finish. The record now carries +`"cancelled"` too (`services/runner/src/tracing/otel.ts`, an explicit two-value allowlist rather +than passing the harness's reason through, so `end_turn` cannot start appearing there by accident). + +Verified in Postgres on the live stack, one stopped turn and one completed turn of the same session: + +``` + record_index | record_type | attributes + 4 | done | {"type": "done", "traceId": "a278...", "stopReason": "cancelled"} + 3 | done | {"type": "done", "traceId": "65b4..."} +``` + +### 4. Does the runner park or destroy on every cancellation path today? + +Before this change: it destroyed on every one of them. `shouldPark` opened with +`if (signal?.aborted) return false`, and every Stop reaches the runner as an abort. The path is: + +1. The API Stop tears the `alive` and `running` locks off the turn + (`api/oss/src/core/sessions/streams/service.py:169`, `:288`). +2. The runner's next heartbeat reads `is_current_turn: false` and calls the interrupt callback + (`services/runner/src/sessions/alive.ts:100`, `:205`). +3. The callback aborts the run signal, the turn races to `CANCELLED`, and the result carries + `stopReason: "cancelled"` with `ok: true`. +4. `shouldPark` answered `false`, so the session coordinator evicted with + `no-park:cancelled` and the teardown reason `aborted`, which deletes + (`services/runner/src/engines/sandbox_agent/teardown.ts`, `aborted` is not in the parkable set). + +The keepalive pool never saw a cancelled turn park. Verified live in the negative control run: +`evict key=... reason=no-park:cancelled`, then a cold rebuild on the next message. + +Other teardown reasons are unaffected. A failed turn still destroys, a pause still parks under its +own approval path, and a client disconnect still destroys. + +**The park decision now asks WHY the run aborted, not just whether it did.** Reading +`signal.aborted` cannot tell a cooperative Stop from any other abort, and inferring the Stop from +`stopReason === "cancelled"` would be worse than it looks: the turn sets that value whenever the +signal aborts, whatever aborted it. Any future `controller.abort()` anywhere in the runner would +then silently start parking sandboxes nobody had checked, which is exactly the failure the teardown +allowlist exists to prevent. So the one call site that means a Stop labels its abort +(`server.ts`, the heartbeat interrupt) and `shouldPark` requires that label. The mechanism is the +standard `AbortController.abort(reason)`, so nothing new is threaded through the engine, the +coordinator or the turn. See `services/runner/src/sessions/stop-signal.ts`. + +Today only one call site could have produced a false park, and it is guarded another way: a +non-session run aborts on client disconnect (`server.ts`), but such a run is never `resumable`, so +`runSandboxAgent` would not have parked it. The label is what keeps that true tomorrow. + +**Cancel, steer and kill are indistinguishable to the runner today**, because all three reach it as +the same "you lost the alive lock" heartbeat. That is safe rather than merely tolerable. A steer +WANTS the warm environment for the turn it starts, and a kill separately calls the runner's `/kill`, +which destroys the pool entry by key whether or not it was parked first +(`services/runner/src/server.ts`, the `/kill` route). Naming the actual operation needs the durable +command plane, which is work package B. + +### 5. Is a sandbox-agent patch needed? + +Yes, and it is eight lines. The guard is client-side, so the patch adds one method that sends the +managed cancel without stamping the session record destroyed. It is appended to the existing +`services/runner/patches/sandbox-agent@0.4.2.patch` through the normal pnpm patch flow: + +```js + async cancelSession(id) { + this.cancelPendingPermissionsForSession(id); + await this.sendSessionMethodInternal(id, SESSION_CANCEL_METHOD, {}, {}, true); + } +``` + +plus the matching line in `dist/index.d.ts`. + +Calling `destroySession` instead would also work at the wire level, and would need no patch. It is +the wrong call for two reasons. It marks the session destroyed when it is not, and on the Pi path +it aborts `env.mcpAbort`, which belongs to the ENVIRONMENT rather than the turn, so a parked +environment would come back with a dead tool-MCP server. The runner therefore uses `cancelSession` +and treats a client without it as "cannot cancel cleanly, so destroy". + +### 6. Does Daytona need a rebuilt snapshot? + +No. The daemon is baked into the snapshot +(`services/runner/images/sandbox/daytona/build_snapshot.py:53`, base image +`rivetdev/sandbox-agent:0.5.0-rc.2-full`, snapshot name `agenta-agent-sandbox-v1`, +selected by `AGENTA_RUNNER_DAYTONA_SNAPSHOT`). The change touches only the client library, which +lives in the runner image, and the daemon needs no new behavior: it already forwards this exact +notification on every teardown. Reported, not verified live, because this stack ran the local +sandbox provider. See the release-gate plan below. + +## What the change does + +Five files, one new module, one patch. + +| File | Change | +| --- | --- | +| `services/runner/src/engines/sandbox_agent/cancel-turn.ts` | New. Sends the cancel, waits for the harness, reports whether it settled. | +| `services/runner/src/engines/sandbox_agent/run-turn.ts:1271` | On `cancelled`, cancel the harness first, then record `cancelSettled`. | +| `services/runner/src/sessions/stop-signal.ts` | New. Labels the Stop abort so the park policy can tell it from every other abort. | +| `services/runner/src/server.ts` | The heartbeat interrupt aborts WITH that label. | +| `services/runner/src/engines/sandbox_agent/engine.ts:28` | `shouldPark` parks a labelled, settled Stop. `clientGone` moved above the abort check. | +| `services/runner/src/tracing/otel.ts` | The terminal `done` record carries `stopReason: "cancelled"`. | +| `services/runner/src/engines/sandbox_agent/session-identity.ts` | New `stoppedTtlMs` park window. | +| `services/runner/src/engines/sandbox_agent/teardown.ts:35` | New parkable teardown reason `cancelled`. | +| `services/runner/src/lifecycle/session-coordinator.ts:773` | Both park paths use the stopped window and log `park-cancelled`. | +| `services/runner/src/protocol.ts` | `AgentRunResult.cancelSettled`. | +| `services/runner/patches/sandbox-agent@0.4.2.patch` | Adds `cancelSession(id)`. | + +The rule is: only a CONFIRMED stop parks, and three separate things must be true. The abort must +carry the user-Stop label, the turn must have ended `cancelled`, and the harness must have answered +its prompt inside the budget. A cancel that cannot be sent, a cancel that throws, a prompt that +rejects on the transport, an unlabelled abort, and a harness that stays silent all fail at least one +of the three, and every one of them destroys. This keeps the teardown allowlist's discipline: a new +situation deletes until somebody proves its sandbox is safe to reuse. + +Two deliberate non-changes: + +- **`clientGone` still always destroys.** The check moved above the abort check so the disconnect + verdict cannot be overridden by a settled cancel. One line, and it keeps today's behavior exactly. +- **The cancel does not abort `env.mcpAbort`.** That controller is the environment's, not the + turn's. The approval-park path already skips it for the same reason + (`services/runner/src/engines/sandbox_agent/run-turn.ts:491`). A teardown that does happen still + aborts it through `teardownRuntimeInFlight`. + +## The park window for a stopped session + +A Stop asks a different question from an ordinary idle park. The ordinary window asks how long a +conversation might keep going by itself. A Stop is a button the user just pressed, so the answer is +known: they are about to type. On the 60 second local idle window the sandbox can be thrown away +while they are still writing, which is the cold start this change exists to remove. Mahmoud decided +on 2026-09-05 that a settled Stop uses the same 600 second window as an approval card on both +providers. A stopped Daytona sandbox can therefore remain billed for up to ten minutes. + +Current windows, all from `services/runner/src/engines/sandbox_agent/session-identity.ts`: + +| Window | Local | Daytona | Env override | +| --- | --- | --- | --- | +| Idle (a clean finished turn) | 60 s | 120 s | `AGENTA_RUNNER_SESSION_TTL_MS`, `AGENTA_RUNNER_DAYTONA_SESSION_IDLE_TTL_MS` | +| Awaiting approval | 600 s | 120 s | `AGENTA_RUNNER_SESSION_APPROVAL_TTL_MS` | +| Stopped by the user (new) | 600 s | 600 s | `AGENTA_RUNNER_SESSION_STOPPED_TTL_MS` | + +The stopped window has its own environment override so operators can choose a different retention +and billing trade-off without changing the ordinary idle or approval windows. The 600 second value +was exercised live and logged `park-cancelled key=... ttl=600000ms`. + +## The settlement timeout (RFC D-016) + +**Recommendation: 10 seconds, overridable with `AGENTA_RUNNER_HARNESS_CANCEL_SETTLE_MS`.** + +Measured settlement, local sandbox, both cancelling a running `sleep 90`: + +| Harness | Time from cancel sent to prompt answered | +| --- | --- | +| Pi (`pi_core`) | 14 ms, 31 ms | +| Codex | 22 ms | +| Claude | not measured, no Anthropic key on this stack | + +Ten seconds is about three hundred times the measured cost, which leaves room for a harness that +has to kill a child process, flush a partial turn, or answer over a Daytona network hop. It is +also short enough that a Stop which genuinely wedges gives up before a user gives up. Raise it only +against a measurement, because every extra second is a second the Stop looks unfinished. Do not +lower it below about one second: the budget also absorbs a slow network to a remote sandbox. + +The timeout is not the user-visible Stop latency. That is dominated by the 30 second heartbeat +interval, which work package B replaces with long polling. + +## The live test + +An isolated EE development stack built from the spike branch used the local sandbox provider and +development images. + +Protocol, driven by `spike_cancel_live.py` in the evidence folder: + +1. Mint an account through `POST /admin/simple/accounts/` and stock the vault with an OpenAI key. +2. Create a workflow, a variant and a revision. The agent config sets + `runner.permissions.default = "allow"`, so no approval gate can end the turn before the Stop + lands. +3. Turn 1: ask the agent to run `sleep 90` through its shell tool, streamed over SSE. +4. At 30 seconds, send the Stop: `POST /api/sessions/streams/` with `{"session_id": ..., "force": false}`. + The API answers `{"mode":"cancel", ...}`. +5. Turn 2: same session, replay the cancelled turn's assistant message, then ask for the codeword + from turn 1. + +Results: + +| Harness | Cancel settled | Sandbox after Stop | Turn 2 | Turn 2 wall clock | Recalled turn 1 | +| --- | --- | --- | --- | --- | --- | +| Pi (`pi_core`) | yes, 14 ms | parked | same sandbox, `hit-continue` | 2.3 s | yes | +| Codex | yes, 22 ms | parked | same sandbox, `hit-continue` | 12.2 s | yes | +| Pi, budget forced to 1 ms | no, timeout | destroyed | new sandbox, cold | 8.0 s | yes, from replay | + +The negative control is also the "before" picture: with the cancel unable to settle, the log reads +`evict key=... reason=no-park:cancelled` and the next turn pays a full rebuild. That is what every +Stop did before this change. + +The scenario was re-run after the review changes landed, and the park now shows the stopped window: +`park-cancelled key=... ttl=600000ms`, then `hit-continue` on the next turn. + +Codex's 12.2 second second turn is the model, not a cold start: the log shows `hit-continue` and +no `sandbox_start`, and the turn spent its time on reasoning tokens and two file reads. + +Log lines and raw run output: `~/agenta-qa-evidence/2026-09-02-spike-a-sandbox-cancel/`. + +**One trap worth writing down.** The first attempt looked like a failure and was not. The keepalive +pool matches a warm session on a fingerprint over the prior user texts AND the tool-call ids the +previous turn emitted (`services/runner/src/engines/sandbox_agent/session-identity.ts:436`). A +resume that omits the cancelled turn's assistant message therefore mismatches on history and +rebuilds cold, no matter how well the cancel worked. The browser sends that message, so the product +path is fine, but any test driver must replay it. + +## Unit tests + +`services/runner/tests/unit/harness-cancel-park.test.ts` (new) pins four rules: + +- The cancel helper's settled, timed-out, rejected, unpatched-client and throwing cases. +- The Stop label: only the labelled abort counts, and a look-alike value cannot forge it. +- `shouldPark` parking a labelled settled Stop, destroying an unsettled one, destroying an + UNLABELLED abort even when the cancel settled, destroying a failed turn, and still destroying on + client disconnect. +- The park windows, their env override, and the teardown reason stopping rather than deleting. +- The terminal `done` record: a Stop carries `cancelled`, a pause still carries `paused`, and a + completed turn plus every harness-reported reason carry nothing. The last case is the point of + the two-value allowlist. + +`services/runner/tests/unit/teardown.test.ts` gains the `cancelled` row, and +`services/runner/tests/unit/session-pool.test.ts` gains the new config field. + +On the frontend, `web/packages/agenta-chat/tests/unit/assets/transcriptToMessages.test.ts` pins that +a cancelled `done` closes the turn like a completed one and does not mark it paused. Reconstruction +reads only `"paused"` (`transcriptToMessages.ts`), so the new value is inert there, which is a claim +worth a test rather than a comment. + +Full suite: `cd services/runner && pnpm test` gives 159 files passed, 2651 tests passed. The +agenta-chat transcript suite gives 52 passed. + +## What is not done + +- **Claude is untested.** This stack has no Anthropic key. The cancel is the same ACP notification + for every harness and the runner branches on capabilities rather than harness name, so the + expectation is that Claude behaves like the other two. It is an expectation, not a measurement. +- **Daytona is untested.** Every live run used the local sandbox provider. The Daytona park path is + the one where park versus delete costs real money, so it belongs in the release gate. +- **A settled Stop preserves the continuity record.** The durable row carries the native session + ID and an end time, so a runner restart can load the same native conversation from its mounted + transcript instead of discarding the Stop as an invalid resume point. +- **The Stop still takes up to 30 seconds to reach the runner.** That is work package B. +- **The Codex orphan is reaped by the runner.** Live Daytona verification remains part of the + pair-level release gate; the runner-side fix needs no vendored bridge or snapshot rebuild. + +## Live test plan for the release gate + +Add one cell, run per harness and on both sandbox providers. + +1. Start a turn that runs a long shell command, on a fresh session. +2. Wait until a `tool-input-available` frame for that command has arrived, then send the Stop. +3. Assert on the stream: the turn ends with `finish`, its open tool call settles as + `tool-output-error`, and no `error` frame claims the run failed. +4. Assert on the runner log: `stage=harness_cancel sent=true settled=true`, then, for Codex, + `stage=harness_reap killed=...`, then `prompt stopReason=cancelled`, then `park-cancelled`. Record + `cleanup_miss=true` as QA evidence, but fail the warm-reuse cell only on `no-park:cancelled`. +5. Send a second message on the same session, replaying the cancelled turn's assistant message. +6. Assert on the runner log: `hit-continue` for the same pool key, and NO `stage=sandbox_start` + between the two turns. On Daytona, additionally assert the sandbox id is unchanged. +7. Assert the second turn's answer references something only turn 1 said. +8. Assert the stopped turn's terminal `done` record carries `stopReason: "cancelled"` and the + completed turn's does not. +9. When reaping succeeds, assert that no leftover process from the cancelled command survives into + the second turn. When reaping fails or is unknown, record the cleanup miss and still assert warm + parking and native-session continuity; the stopped TTL bounds the leftover process to 600 seconds. + +The negative leg is worth keeping too: with `AGENTA_RUNNER_HARNESS_CANCEL_SETTLE_MS=1` the same +scenario must log `settled=false` and `no-park:cancelled`. That proves the guard still guards. + +## Open questions for Mahmoud + +1. **How should a failed Codex reap affect parking?** Decision: keep the settled Stop parked. Reaping + is best effort, cleanup misses are QA evidence, and the 600-second stopped TTL bounds leftovers + without sacrificing warm reuse or native-session continuity. +2. **Ten seconds for the settle budget?** Recommendation: yes, ship it. The measured cost is + 14 to 31 ms, so the budget is not a latency cost in the normal case, and it only ever delays a + Stop that is already going badly. +3. **Should the Stop also settle the turn ledger row, rather than leaving the turn incomplete?** + Recommendation: yes, in work package C. The terminal record now says `cancelled`, so a reader can + tell a Stop from a completion, but the ledger row still looks like a turn that never finished. +4. **Do we test Claude and Daytona before the RFC is accepted, or at the release gate?** + Recommendation: at the release gate, with the cell above. Blocking the design on an Anthropic key + tonight buys little, because the cancel is one protocol request shared by every harness, and the + Codex result shows the interesting variation is in what the harness does with it, not whether it + accepts it. + +Every settled Stop preserves the continuity row and native session, regardless of its best-effort +Codex reap outcome. Only a harness cancel that does not settle invalidates continuity and falls back +to cold replay. A plain `clientGone` still destroys because a disconnect is not a Stop. diff --git a/docs/design/session-control-and-live-events/spike-b-durable-commands-design.md b/docs/design/session-control-and-live-events/spike-b-durable-commands-design.md new file mode 100644 index 00000000000..65f82bd3806 --- /dev/null +++ b/docs/design/session-control-and-live-events/spike-b-durable-commands-design.md @@ -0,0 +1,1355 @@ +# Spike B: durable commands and control delivery + +> AGENT-GENERATED, low weight. Implementation-ready design for discussion. Mahmoud makes final +> decisions. + +Scope: reliable API-to-runner commands, version one. The only command kind in version one is +Cancel, which the product calls Stop. The design keeps Redis execution ownership as it is, adds no +Postgres execution authority, no ownership generations, no stale-writer fencing, and no +multi-runner routing. + +Every claim below is marked **verified** (read in the code of this worktree, with `path:line`) or +**reported** (taken from a document, named at the point of use). + +This revision answers the architecture review at `review-architecture.md`, sections 3 and 4. The +holes it names are addressed here: H-2 in sections 5 and 7, H-3 in sections 4 and 7, H-4 in section +4, H-5 in section 4, H-6 in section 5, and the interface corrections in sections 2, 5 and 9. H-1, +the `shouldPark` change, belongs to Work package A and is named as a dependency in section 7. + +Terms used here: + +- **Execution:** one runner attempt at one user message. In the code today its identifier is the + `turn_id` the runner mints (`services/runner/src/server.ts:190`). This design does not rename it. +- **Command:** one durable request to change an execution. +- **Held session:** a session this runner process holds warm, whether it is running a turn, idle in + the keep-alive pool, or parked awaiting an approval. + +--- + +## 1. What happens today when a user presses Stop + +**Verified.** The browser stops its own stream at once. The runner learns nothing until its next +heartbeat, which is up to 30 seconds later. The sandbox is then deleted, so the next message is a +cold start. + +The chain, in order: + +1. `handleStop` marks the turn stopped locally and aborts the client fetch + (`web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts:480`). +2. The browser posts `POST /sessions/streams/` with no inputs and no `force`, and **with no + execution id** (`useAgentChatSession.ts:505`, which passes only `{sessionId, projectId}`). + Mobile posts the same call (`web/mobile/src/features/chat/StopButton.tsx:18`). +3. The route runs `set_session_stream` (`api/oss/src/apis/fastapi/sessions/router.py:369`), which + calls `SessionStreamsService.command` (`api/oss/src/core/sessions/streams/service.py:229`). +4. No inputs and no `force` resolves to `CommandMode.cancel` + (`api/oss/src/core/sessions/streams/service.py:288`). +5. Cancel calls `_displace_turns` (`api/oss/src/core/sessions/streams/service.py:169`). It writes a + supersession tombstone for the current `alive` and `running` owners, then force-deletes both keys + (`service.py:190` and `service.py:193`). It marks the row ended and publishes the `ended` + lifecycle event. **The API never contacts the runner.** +6. The runner finds out on its next heartbeat. The beat runs on a 30 second interval + (`services/runner/src/sessions/alive.ts:221`, `HEARTBEAT_INTERVAL_SECONDS = 30` in + `services/runner/src/sessions/contract.ts:18`). +7. The beat returns `is_current_turn: false` + (`api/oss/src/core/sessions/streams/service.py:452`), the runner reads it as `interrupted` + (`services/runner/src/sessions/alive.ts:105`), and the watchdog fires `onInterrupted` once + (`alive.ts:207`), which `server.ts:519` wires to `controller.abort()`. +8. The abort makes `shouldPark` return false, so the environment is destroyed rather than parked + (`services/runner/src/engines/sandbox_agent/engine.ts:26`). The sandbox and the native harness + session are gone. + +### The delay chain + +| Step | Where | Cost | +|---|---|---| +| Browser aborts its own stream | `useAgentChatSession.ts:480` | immediate | +| Cancel request returns | `router.py:369` | one API round trip | +| Redis keys cleared, row marked ended | `streams/service.py:190` | inside that call | +| Runner notices | `alive.ts:221` | **0 to 30 seconds** | +| Run aborts | `server.ts:519` | immediate after the beat | +| Harness cancel and sandbox teardown | `engine.ts:26` | seconds, and the sandbox is deleted | + +The 30 second wait is the whole problem. Four further defects ride on it: + +- **A Stop can be lost silently.** A heartbeat that returns a non-2xx status yields + `interrupted: false` by design (`services/runner/src/sessions/alive.ts:92`). A run whose platform + credential expired or was dropped can never be stopped. The credential states are logged at + `services/runner/src/server.ts:445`. +- **A parked session has no channel at all.** When a turn parks awaiting an approval, the request + handler's `finally` calls `aliveWatchdog.release()` (`services/runner/src/server.ts:618`), which + clears the heartbeat interval and sends one last beat with `is_running: false` + (`services/runner/src/sessions/alive.ts:241`). From that moment the runner sends no heartbeat for + that session, so the only existing control channel is gone. This is review hole H-2, and it is why + section 5 makes the poll session-scoped rather than turn-scoped. +- **A late Stop can kill the next turn.** `_displace_turns` reads whoever holds `alive` and + `running` at the moment it runs, so a Stop applied 300 ms after the turn ended tombstones the turn + that started in between. The tombstone lasts an hour and every read refreshes it + (`api/oss/src/dbs/redis/sessions/locks.py:147`). This is review hole H-3. +- **Stop is not free.** Because the abort path destroys the environment, Stop today costs the warm + sandbox and the native harness session. Work package A owns the fix. This design assumes it + delivers a warm park on Stop. + +--- + +## 2. The command record + +### Placement + +| Question | Answer | +|---|---| +| Database | Core Postgres (`env.postgres.uri_core`, `TransactionsEngine`), the same database as `session_streams`, `session_turns`, `session_interactions`. Verified at `api/oss/src/dbs/postgres/shared/engine.py:29`. | +| Table | `session_commands` | +| Core module | `api/oss/src/core/sessions/commands/` with `dtos.py`, `interfaces.py`, `service.py`, `types.py`, matching the layout of `core/sessions/interactions/` | +| Storage module | `api/oss/src/dbs/postgres/sessions/commands/` with `dbas.py`, `dbes.py`, `dao.py`, `mappings.py` | +| Migration | `api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py`, revising `oss000000021` (verified: `oss000000021_add_session_streams_references.py` is the current head of that chain) | + +Not tracing. The tracing database holds spans, and a command is coordination state that the +sessions plane owns. + +### Columns + +The mixins are the house ones from `api/oss/src/dbs/postgres/shared/dbas.py`: `ProjectScopeDBA`, +`IdentifierDBA`, `LifecycleDBA`, `DataDBA`, `FlagsDBA`, `TagsDBA`, `MetaDBA`. That is the same set +`SessionInteractionDBA` uses (`api/oss/src/dbs/postgres/sessions/interactions/dbas.py:14`). + +| Column | Type | Role | Meaning | +|---|---|---|---| +| `project_id` | UUID, not null | scope | Tenant boundary. Foreign key to `projects.id`, `ON DELETE CASCADE`. | +| `id` | UUID, not null, uuid7 | identity | The `command_id`. The API mints it. | +| `session_id` | String, not null | routing | Which session the command acts on. A bare correlator, not a foreign key, like every other sessions table. | +| `kind` | String, not null | routing | `cancel` in version one. | +| `target_turn_id` | String, null | target | The execution this command must reach, resolved once at admission. Null only when nothing was running or parked. | +| `expected_turn_id` | String, null | target | The caller's `expected_execution_id`, stored as sent. Null when the caller supplied none. | +| `data` | JSON, null | input | The command's own arguments, shaped `{"input": {"text": ..., "attachments": [...]}, "policy": {"on_busy": ...}}`. Empty for `cancel`. | +| `state` | String, not null | delivery | `pending`, `claimed`, `applied`, `obsolete`. | +| `claimed_by` | String, null | delivery | The replica that holds the current claim. Bookkeeping, not an address. | +| `claim_expires_at` | TIMESTAMP tz, null | delivery | When the claim may be delivered again. | +| `claim_count` | Integer, not null, default 0 | delivery | Deliveries so far. Caps re-delivery. | +| `outcome` | String, null | result | What happened to the execution: `stopped`, `not_running`, `superseded_by_newer_turn`, `failed`, `lost`. Null while open. | +| `idempotency_key` | String, null | context | The caller's `Idempotency-Key` header, stored verbatim. | +| `settled_at` | TIMESTAMP tz, null | metadata | When the command reached a terminal state. | +| `flags`, `tags`, `meta` | JSONB / JSON, null | metadata | House mixins. Unused in version one, present for consistency. | +| `created_at`, `updated_at`, `deleted_at`, `created_by_id`, `updated_by_id`, `deleted_by_id` | `LifecycleDBA` | metadata | House lifecycle columns. `created_at` carries a guard: it is the "do not supersede a newer turn" comparison of section 4. | + +Four grouping rules from the interface review are applied here. + +- **Delivery bookkeeping is one group.** `state`, `claimed_by`, `claim_expires_at` and `claim_count` + are the delivery record. On the wire they are nested under `delivery`. In the table they are flat + columns because a claim query filters and orders on them, and a JSON blob cannot be indexed for + that. The names carry the grouping. +- **Delivery is never merged with the result.** `state` says where the command is; `outcome` says + what happened to the execution. That separation is the whole point of decision D-016. +- **The target has its own two columns.** `expected_turn_id` is what the caller asserted; + `target_turn_id` is what the API resolved. Keeping both makes a 409 explainable after the fact and + gives a future `target.execution_id` an obvious home. +- **There is no `owner_replica_id` and no `runner_url`.** The first revision routed commands by the + owner replica. Section 5 replaces that with session-scoped claims, so the record needs no routing + identity at all, and an address in a durable record would be an implementation detail with a + lifetime longer than the thing it points at. + +### Two columns added to `session_streams` + +**`stopping_turn_id`**, String, nullable. It names the execution that an accepted Stop is waiting on. +It is written in the same transaction as the command insert, and cleared at settlement. + +**`turn_started_at`**, TIMESTAMP tz, nullable. It records when the row's current `turn_id` started. +It exists for one reason: the stale-Stop guard in section 4 needs to compare a command's arrival +time with the current execution's start time, and **there is nowhere to read that today**. The +options were checked, and none of them works: + +| Candidate | Why it does not serve | +|---|---| +| `session_streams.updated_at` | It is the heartbeat timestamp and moves every 30 seconds. Verified: the mirror write is unconditional (`api/oss/src/core/sessions/streams/service.py:618`). | +| The turn id itself | API-minted turns use uuid7 and are time-ordered (`streams/service.py:940`), but the runner mints its own with `randomUUID()`, which is uuid4 and carries no time (`services/runner/src/server.ts:190`, verified). Every browser turn today is runner-minted. | +| Redis `running` or `alive` | The value is the bare turn id, and the release-if-owner script compares the whole value (`api/oss/src/dbs/redis/sessions/contract.py:153`). Packing a timestamp into it would break that compare and the golden fixture the runner shares. | +| `session_turns.start_time` | It is written, from `turnStartedAt` captured at `services/runner/src/engines/sandbox_agent/run-turn.ts:192` and sent at `:469`. But the append is fire-and-forget (`.catch(() => {})`) and it needs a stream id and a continuity index, so a turn can be running with no row at all. It is a good secondary source, not a guard. | + +So add the column. It is written wherever `turn_id` is written, in the same statement, and only when +the id actually changes: + +```sql +UPDATE session_streams + SET turn_id = :turn_id, + turn_started_at = CASE + WHEN turn_id IS DISTINCT FROM :turn_id THEN now() + ELSE turn_started_at + END, + ... +``` + +That form is idempotent under the repeated heartbeats that stamp the same id every 30 seconds, and +it needs no new writer: both `_start_turn` (`streams/service.py:940`) and the heartbeat's +`durable_turn_id` stamp already go through `SessionStreamEdit`. + +Both are columns and not bits inside `flags` because `flags` is the Redis mirror. Every heartbeat +rewrites it (`api/oss/src/core/sessions/streams/service.py:618`), so a value stored there would be +erased on the next beat. `SessionStreamEdit` carries only `flags`, `tags`, `meta` and `turn_id` +(`api/oss/src/core/sessions/streams/dtos.py:73`), so the heartbeat path cannot touch +`stopping_turn_id` by accident, and it touches `turn_started_at` only through the guarded `CASE`. + +### Indexes and constraints + +```python +__table_args__ = ( + ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + PrimaryKeyConstraint("project_id", "id"), + UniqueConstraint( + "project_id", "session_id", "idempotency_key", + name="uq_session_commands_idempotency", + ), + CheckConstraint("kind IN ('cancel')", name="ck_session_commands_kind"), + CheckConstraint( + "state IN ('pending', 'claimed', 'applied', 'obsolete')", + name="ck_session_commands_state", + ), + Index( + "ix_session_commands_open", + "project_id", "session_id", "created_at", + postgresql_where=text("state IN ('pending', 'claimed') AND deleted_at IS NULL"), + ), + Index( + "ix_session_commands_claims", + "claim_expires_at", + postgresql_where=text("state = 'claimed' AND deleted_at IS NULL"), + ), + Index( + "ix_session_commands_project_session", + "project_id", "session_id", "created_at", + ), +) +``` + +`ix_session_commands_open` is the claim query's index. It leads with `(project_id, session_id)` +because a claim asks for the commands of a named set of sessions, and it is partial on the open +states because a settled command is never claimed again. It also serves the open-command collapse +read at admission. + +The check constraints copy the shape of `ck_session_attachments_state` +(`api/oss/src/dbs/postgres/sessions/attachments/dbes.py:35`). + +### Idempotency, in two layers + +1. **Client key.** `uq_session_commands_idempotency` on `(project_id, session_id, + idempotency_key)`, the same triple `uq_session_attachments_idempotency` uses + (`api/oss/src/dbs/postgres/sessions/attachments/dbes.py:29`). An insert that hits the constraint + is caught, the existing row is read back, and it is returned to the caller. That is the pattern + `SessionInteractionsDAO.create_interaction` already uses + (`api/oss/src/dbs/postgres/sessions/interactions/dao.py:60`). A null key never collides, because + Postgres treats nulls as distinct in a unique index. +2. **Open-command collapse.** Even with no client key, admission first looks for an open command + (`state IN ('pending','claimed')`) of the same `kind` for the same `(project_id, session_id, + target_turn_id)`. If one exists, the API returns it instead of creating a second. This is what + makes "two Stops in a row" correct without asking the browser to send a key. + +The server `command_id` is the idempotency identity of every later step. A settle for a command that +already reached a terminal state returns the stored state and changes nothing. + +### Retention + +Settled rows (`state IN ('applied','obsolete')`) are deleted 7 days after `settled_at` by the sweep +described in section 4. Commands are operational state, not session history. Durable session history +stays in `session_records`. Open rows are never deleted by the sweep; the watchdog settles them +first. + +--- + +## 3. The state machine + +```text + admission + | + v + +------------> pending ------------------------+ + | | | + | claim expired, | claim (poll, direct call, | nothing to do + | session still | or heartbeat) | + | beating v v + | claimed --------> applied obsolete + | | runner + +-----------------+ reports + | + | claim expired and the session stopped beating + v + obsolete (outcome = lost) +``` + +`applied` and `obsolete` are terminal. There is no transition out of either. + +Every transition is one `UPDATE ... WHERE ... RETURNING *` whose `WHERE` names the state it expects. +`scalar_one_or_none()` decides the winner, so two API replicas cannot both win. This is exactly the +pattern `SessionInteractionsDAO.transition_interaction` already uses +(`api/oss/src/dbs/postgres/sessions/interactions/dao.py:120`). Verified. + +| Transition | Who does it | Guard | +|---|---|---| +| none to `pending` | The API, on an accepted Cancel | `INSERT`, protected by `uq_session_commands_idempotency` and by the open-command collapse read in the same transaction | +| none to `obsolete` | The API, when nothing is running or parked | Same insert, with `state='obsolete'`, `outcome='not_running'`, `settled_at=now()` | +| `pending` to `claimed` | The API, serving a claim, a direct call, or a heartbeat | `WHERE state = 'pending'` | +| `claimed` to `pending` | The command sweep, when a lease expired and the session is still beating | `WHERE state = 'claimed' AND claim_expires_at < now() AND claim_count < :max_deliveries` | +| `claimed` to `applied` | The API, on the runner's outcome report | `WHERE state = 'claimed' AND claimed_by = :replica_id` | +| `claimed` to `obsolete` | The API, on a report of `not_running` or `superseded_by_newer_turn` | Same guard | +| `claimed` to `obsolete` (`lost`) | The command sweep, when the lease expired and the session stopped beating | `WHERE state = 'claimed' AND claim_expires_at < now()`, plus the heartbeat-age test of section 4 | +| `pending` to `obsolete` (`lost`) | The command sweep, when nobody ever claimed it | `WHERE state = 'pending' AND created_at < :admission_deadline` | + +The claim statement, in the form the DAO writes it: + +```sql +UPDATE session_commands + SET state = 'claimed', + claimed_by = :replica_id, + claim_expires_at = now() + make_interval(secs => :lease_seconds), + claim_count = claim_count + 1, + updated_at = now() + WHERE (project_id, id) IN ( + SELECT project_id, id + FROM session_commands + WHERE state = 'pending' + AND deleted_at IS NULL + AND (project_id, session_id) IN :held_sessions + ORDER BY created_at + LIMIT :limit + FOR UPDATE SKIP LOCKED + ) +RETURNING *; +``` + +`:held_sessions` is the set of sessions the calling runner holds warm, sent with the request. See +section 5. `FOR UPDATE SKIP LOCKED` is what lets two API replicas serve two claims at the same time +without either blocking or double-claiming. + +The settle statement: + +```sql +UPDATE session_commands + SET state = :result, outcome = :outcome, settled_at = now(), updated_at = now() + WHERE project_id = :project_id + AND id = :command_id + AND state = 'claimed' + AND claimed_by = :replica_id +RETURNING *; +``` + +Zero rows means the claim had already expired or another actor settled it. The route then reads the +row and answers 409 with its stored state, so the runner learns the truth instead of retrying. + +--- + +## 4. The claim lease and the settlement rule + +| Setting | Value | Reason | Environment variable | +|---|---|---|---| +| Lease duration | 90 seconds | Three heartbeat intervals, the window the review picked for H-4 | `AGENTA_SESSIONS_COMMAND_LEASE_SECONDS` | +| Maximum deliveries | 3 | Bounds a delivery loop when a runner accepts but never reports | `AGENTA_SESSIONS_COMMAND_MAX_DELIVERIES` | +| Sweep interval | 10 seconds | Fine enough that a lost Stop settles inside two minutes | `AGENTA_SESSIONS_COMMAND_SWEEP_SECONDS` | +| Admission deadline | 90 seconds | A command nobody ever claimed is a runner that is not there | `AGENTA_SESSIONS_COMMAND_ADMISSION_TIMEOUT_SECONDS` | + +All four go in a new `SessionsCommandsConfig` block in `api/oss/src/utils/env.py`, read through the +shared `env` object. Do not call `os.getenv` in the service (`AGENTS.md`, "Environment config"). + +**Renewal: none in version one.** A claim is not renewed while the runner works. It expires and is +either delivered again or settled. This is safe because applying a Cancel is idempotent, and because +the runner deduplicates. A renewal route is the first thing to add if a harness cancel is ever +slower than the lease, and the column that would carry it (`claim_expires_at`) already exists. + +### The settlement rule when the runner is gone (H-4) + +**The Redis time to live cannot be the signal.** `alive` and `running` both hold 3600 seconds +(`api/oss/src/utils/env.py:1417` and `:1421`, verified). A `stopping` state that waits for those +keys to expire is a `stopping` state that lasts an hour. Settlement must key off **heartbeat age**, +which is `session_streams.updated_at`, the column the heartbeat writes on every beat and the one the +orphan sweep already filters on (`api/oss/src/tasks/asyncio/sessions/orphan_sweep.py:57`, verified). + +The rule, evaluated by the sweep for every command whose `claim_expires_at` has passed: + +| Heartbeat age for that session | Attempts left | Action | +|---|---|---| +| Under 90 seconds (the runner is alive, the report was lost) | yes | Re-arm to `pending` and deliver again | +| Under 90 seconds | no | Settle `obsolete`, `outcome='lost'`, and run the settlement side effects | +| 90 seconds or more (the runner is gone) | either | Settle `obsolete`, `outcome='lost'`, and run the settlement side effects | + +A session parked awaiting an approval stops beating on purpose (`server.ts:618`), so it would look +"gone" by heartbeat age alone. Exclude it: a command whose target session has an open interaction, +or whose stream row is `alive` but not `running`, uses the admission deadline rather than the +heartbeat-age test. That is the same distinction the orphan sweep already draws between its 300 +second running threshold and its 1800 second idle threshold +(`api/oss/src/tasks/asyncio/sessions/orphan_sweep.py:33` and `:37`, verified). + +**The watchdog owns settlement, not this design.** A separate agent is building the execution +watchdog on branch `feat/session-execution-watchdog`. This design does not build a second one. The +command sweep described here is either that watchdog with the command rules folded in, or a caller +of it. The single rule both must obey: **one execution reaches exactly one terminal outcome, written +by exactly one writer.** If the watchdog marks an execution `lost`, it must settle that execution's +open commands in the same transaction, and vice versa. Decide the ownership before either lands. + +The side effects of a `lost` settlement are the same as a normal settlement (section 7, step 10 to +13), with one difference: Redis keys are cleared with the force variants rather than the +owner-checked ones, because the owning process is gone. + +### Deduplication on the runner (H-5) + +The applied-command set must outlive the poll loop, because a loop restart with an empty set would +apply a Stop a second time, and by then the session may be running a newer turn. + +- The set lives in the same module as the session state the runner already keeps across turns, next + to `SessionPool` (`services/runner/src/engines/sandbox_agent/session-pool.ts:90`), keyed by + `${projectId}:${sessionId}` with a bounded list of applied command ids and their apply times, kept + for 30 minutes. It is not owned by the poll loop and does not reset when the loop restarts. +- Both delivery paths call one `applyCommand(command)` entry point that consults the set first. +- **Applying an already-applied command is a no-op that re-sends the acknowledgement.** It does not + abort anything, and it does report the stored outcome, so a lost acknowledgement is repaired + without a second abort. + +### The three guards on the target execution (H-3) + +A Stop that arrives after its turn ended must not touch the next turn. Three guards, in order of +strength: + +1. **The API compares arrival time with the current turn's start time.** This is the guard that + closes the reported race, so it is spelled out below. +2. **The target is pinned at admission.** The API resolves `target_turn_id` once and never + re-resolves it. A turn that starts later has a different id, so a pinned command cannot reach it. +3. **The runner repeats the comparison locally.** The envelope carries the command's arrival time. + The runner refuses to abort an execution that started after it, and settles the command + `obsolete` with `outcome='superseded_by_newer_turn'`. The runner holds its own execution's start + time in memory, so this check is exact even when the API's is not. +4. **First-party clients always send `expected_execution_id`.** The field stays optional in the + contract, as decision D-010 requires, but the desktop and mobile Stop buttons must send it. Today + the desktop sends nothing (`useAgentChatSession.ts:505`, verified). Treat an omitted id from a + first-party client as a bug, not as a supported mode. + +#### The arrival-time comparison, when no expected execution id was sent + +The race: the user presses Stop at t=0 while turn one is running. Turn one ends at t=0.1. Turn two +starts at t=0.2. The request is applied at t=0.3, reads Redis, finds turn two, and targets a turn the +user never meant to stop. + +The rule, applied at admission before anything is inserted: + +1. The service stamps `received_at = now()` as its **first** action, before it reads Redis. It later + writes that same value as the row's `created_at` rather than letting the server default fill it, + so the value it compared is the value it stored. +2. It reads the current running owner from Redis and the session's row, which gives `turn_id` and + `turn_started_at` in one query the admission path already makes. +3. If `turn_started_at > received_at`, the current execution began after the user pressed Stop. + Insert the command already settled: `state='obsolete'`, + `outcome='superseded_by_newer_turn'`, `settled_at=now()`, `target_turn_id=null`. Return 200 with + `execution.state = "idle"`. **Do not target that turn and do not touch Redis.** +4. Otherwise proceed normally. + +This runs only when `expected_execution_id` is absent. When the caller sent one, the 409 comparison +already settles the question and is stricter. + +**When `turn_started_at` is null, the guard does not fire.** A row written before this column +existed, or a turn whose stamp was lost, yields no comparison. The API then targets the turn as it +does today and leaves the decision to guard 3, which is exact because the runner reads its own +memory. Failing this way round is deliberate: a guard that refuses to Stop whenever it lacks data +would break the common case to protect a rare one. + +`session_turns.start_time` is a useful secondary source when the row exists, but the design does not +depend on it, for the reasons in the table in section 2. + +The `expected_execution_id` check itself happens twice, for two different reasons. At admission the +API compares it to the Redis running owner and answers 409 if they differ. At application the runner +applies the command only to a local execution whose `turnId` equals `target_turn_id`, and settles +`obsolete` with `outcome='not_running'` when it holds no such execution. + +--- + +## 5. The claim contract + +### The loop is session-scoped and lives as long as the session is warm (H-2) + +This is the single most important correction from the review. The first revision started one poll +per runner process and routed by owner replica. That has two faults: it cannot say which sessions +the runner actually holds, and a per-turn loop would go silent exactly when a turn parks. + +The rule: + +- **One loop per runner process.** Not one per turn and not one per session. +- **The loop declares the sessions it holds.** Every claim carries the current set. That set is the + union of the execution registry (turns in flight) and the keep-alive pool keys, which are already + `${projectId}:${sessionId}` strings and already include parked entries + (`SessionPool.keys()` and `SessionPool.snapshot()`, + `services/runner/src/engines/sandbox_agent/session-pool.ts:108` and `:127`, verified; a parked + entry is seated as `awaiting_approval` at + `services/runner/src/lifecycle/session-coordinator.ts:764`, verified). +- **A session leaves the set only when the runner stops holding it warm.** A parked approval stays + in the set, so a Stop reaches it. That is H-2 closed. +- **Claims are queries over durable state, never a stream position** (H-6). The request declares a + set of sessions and the API answers with whatever is pending for them right now. There is no + cursor, no offset and no resume token, so a command created while the connection was down is + picked up by the next claim like any other. + +### Routes + +| Route | Method | Caller | Purpose | +|---|---|---|---| +| `/sessions/control/commands/claim` | POST | Runner | Claim the pending commands for the sessions this runner holds, waiting up to the hold if there are none | +| `/sessions/control/commands/{command_id}/outcome` | POST | Runner | Report the terminal outcome | + +Both live on a new `SessionControlRouter` in `api/oss/src/apis/fastapi/sessions/router.py`, included +with no prefix like the streams router (`api/entrypoints/routers.py:1354`, verified), and excluded +from the public schema. + +### Authentication + +The runner authenticates its per-run calls as the invoke caller, using the ephemeral platform +credential from the run (`services/runner/src/sessions/alive.ts:60`, verified). That credential +cannot carry these routes: the loop belongs to the process and spans many projects, and a run's +credential expires while the process keeps polling. + +So both routes use the shared runner token, `AGENTA_RUNNER_TOKEN`, which both sides already hold +(`api/oss/src/utils/env.py:1161` as `env.runner.token`, and `services/runner/src/server.ts:104`). +Verified. It is the same secret the existing API-to-runner hop uses in the other direction +(`api/oss/src/core/sessions/streams/runner_client.py:44`). + +Mechanics: + +- Add the prefix `/sessions/control/` to `_PUBLIC_ENDPOINTS` + (`api/oss/src/middlewares/auth.py:52`), so the project-scoped auth middleware does not reject a + request that carries no user credential. This is the same treatment the OAuth callback and the + Composio event routes already get. +- The route then does its own check, with a constant-time comparison against `env.runner.token`, + accepting `X-Agenta-Runner-Token: ` first and `Authorization: Bearer ` second. That + is the header pair and the comparison the runner itself already implements + (`services/runner/src/server.ts:127`). +- **Fail closed.** If `env.runner.token` is unset or blank, both routes answer 503 and serve nothing. + Being exempt from the middleware makes the route's own check the only gate, so it must never + default to open. +- The project scope of every command comes from the row and from the declared session set, never + from a header. A runner can only receive commands for sessions it named, and a session id is + meaningful only inside its project, so the pair is the scope. + +### Request and response bodies + +Claim request: + +```json +{ + "replica_id": "runner-7f3c", + "sessions": [ + {"project_id": "1f0a4b2c-0000-4000-8000-000000000002", "session_id": "sess-42"}, + {"project_id": "1f0a4b2c-0000-4000-8000-000000000002", "session_id": "sess-77"} + ], + "wait_seconds": 25, + "limit": 10 +} +``` + +`replica_id` is delivery bookkeeping: it becomes `claimed_by` so a settle can be matched to its +claim. It is not routing, and it is not an address. `sessions` is the routing input, capped at 200 +entries and ordered most recently used first. `wait_seconds` is bounded server-side to +`[0, AGENTA_SESSIONS_CONTROL_POLL_HOLD_SECONDS]`, default 25. `limit` is bounded to `[1, 50]`, +default 10. + +Claim response, 200: + +```json +{ + "count": 1, + "commands": [ + { + "id": "0199a3f2-0000-7000-8000-000000000001", + "project_id": "1f0a4b2c-0000-4000-8000-000000000002", + "session_id": "sess-42", + "kind": "cancel", + "target": { + "turn_id": "0199a3f1-0000-7000-8000-00000000000a", + "expected_turn_id": "0199a3f1-0000-7000-8000-00000000000a" + }, + "delivery": { + "claimed_by": "runner-7f3c", + "claim_expires_at": "2026-09-02T22:10:31Z", + "attempt": 1 + }, + "created_at": "2026-09-02T22:09:01Z" + } + ] +} +``` + +`count` plus a list is the house response envelope (`SessionsResponse`, +`api/oss/src/apis/fastapi/sessions/models.py:105`). A `cancel` carries no `input` and no `policy`; +both appear only for the kinds that have them, so a reader never has to interpret an empty object. +`created_at` is on the envelope because the runner needs it for guard 3 of section 4. + +Claim response, 204: the hold expired with nothing to deliver. No body. + +Outcome request: + +```json +{ + "replica_id": "runner-7f3c", + "result": "applied", + "execution": { + "id": "0199a3f1-0000-7000-8000-00000000000a", + "state": "stopped" + } +} +``` + +`result` is the command's terminal state, `applied` or `obsolete`. `execution.state` is one of +`stopped`, `failed`, `not_running`, `superseded_by_newer_turn`. `execution.error` is a short string, present only +when the state is `failed`. The two objects are separate because they answer different questions and +have different owners: `result` is delivery bookkeeping the runner controls, `execution` is a +product fact the user sees. + +Outcome response, 200: + +```json +{ + "command": { + "id": "0199a3f2-0000-7000-8000-000000000001", + "state": "applied", + "outcome": "stopped", + "settled_at": "2026-09-02T22:09:12Z" + } +} +``` + +Outcome response, 409: the claim was not held by this replica. The body carries the same `command` +object with its stored state, so the runner can stop and move on rather than retry. + +### How the hold works + +The route subscribes to one Redis Pub/Sub channel per declared session on the durable plane, then +loops: + +1. Claim once, without waiting. Return 200 if anything came back. +2. Wait on the subscription with a one second timeout, so the loop can re-check the shutdown flag. +3. On a message, or every second, try the claim again. +4. When the hold budget runs out, return 204. + +Three details are not optional: + +- **Add `control_channel(project_id, session_id)` to the Redis contract** + (`api/oss/src/dbs/redis/sessions/contract.py`), with the payload `{"type": "command-pending"}` and + nothing else. It is project-scoped like every other key in that file, and it carries no tenant data + because the claim re-queries Postgres, which is the authority. +- **Reuse the watch endpoint's shutdown release.** `api/oss/src/apis/fastapi/sessions/watch.py:50` + installs a hook on uvicorn's exit path because a held response blocks graceful shutdown for ever. + A held claim has exactly the same failure. Import `request_shutdown` and the same threading event, + or move both into a small shared helper. +- **A new session mid-hold ends the hold.** When the runner starts holding a session that was not in + the declared set, the loop aborts its in-flight request locally and re-issues the claim with the + new set. That is one in-process event, not a server concern. + +### What the runner does + +| Result | What the runner does | +|---|---| +| 200 with commands | Apply each through `applyCommand`, report each outcome, then claim again at once | +| 204 | Claim again at once | +| Read timeout with no response | Claim again after the backoff floor | +| Network error, 502, 503, 504 | Back off: 1 s, 2 s, 4 s, 8 s, 16 s, then 30 s, with 20 percent jitter. Reset on the first success | +| 401 or 403 | Log once at error level and retry every 60 s. This is a deployment misconfiguration and must be loud, not a tight loop | +| 429 | Back off as for a network error | +| API restart | The held connection closes. This is the network error case. Nothing is lost, and the next claim is a fresh query over durable state, not a resumed cursor | +| Empty session set | Do not call. Wait for the next session to be held | + +The client timeout must exceed the hold: set the fetch timeout to `hold_seconds + 10`. + +### After a reconnect, the runner asks again; it never resumes a position + +This is worth stating on its own, because getting it wrong loses commands silently. + +A claim is a **query over durable state**. The runner sends the sessions it currently holds and the +API answers with whatever is pending for them at that moment. There is no cursor, no offset, no +sequence number, no resume token and no server-side per-runner queue position. + +So after any break, whether the connection dropped, the API replica restarted, the runner process +restarted, or the loop was switched off and on, the runner simply issues the next claim with its +current session set. A command created while nothing was listening is `pending` in Postgres, and the +next claim returns it like any other. Nothing has to be replayed, and nothing can be skipped by +starting from the wrong place, because there is no place to start from. + +The one thing this requires: the session set must be rebuilt from what the process actually holds, +not cached from before the break. After a runner restart the set comes from the rebuilt pool and the +live execution registry, both of which reflect reality rather than history. + +--- + +## 6. The heartbeat fallback + +One field is added to the heartbeat response DTO `SessionHeartbeatResult` +(`api/oss/src/core/sessions/streams/dtos.py:180`): + +```python +class SessionHeartbeatResult(BaseModel): + stream: Optional[SessionStream] = None + replica_id: str + is_current_turn: bool = True + # Commands for THIS session, claimed by this beat under the same compare-and-set the + # claim route uses. Empty when there is nothing to deliver, which is the normal case. + commands: List[SessionCommandEnvelope] = Field(default_factory=list) +``` + +`SessionCommandEnvelope` is the same model the claim route returns, so the runner has one parser and +one applier. + +Rules: + +- The beat serves only commands for its own `(project_id, session_id)`, and only those whose + `target.turn_id` matches the beat's `turn_id` or is null. It never serves another session's + commands, because the beat is authenticated with the run's project-scoped credential. +- It claims them under the same statement as the claim route, so a command cannot be delivered by + both paths at once. One of the two wins the compare-and-set; the other sees zero rows. +- The runner deduplicates by `command_id` in the set described in section 4, so a command delivered + by the claim route and offered again by a beat is acknowledged again but applied once. + +**Know what this fallback cannot do.** It covers only a session with a live turn, because the +heartbeat stops when a turn ends or parks (`services/runner/src/server.ts:618` and +`services/runner/src/sessions/alive.ts:241`, verified). It is not a substitute for the session-scoped +loop, and it must not be treated as the delivery path for a parked session. It exists for two cases: +the primary adapter is switched off, and the primary adapter is failing while the run's own +heartbeat still works. + +The runner reads the new field in `sendHeartbeat` (`services/runner/src/sessions/alive.ts:96`) and +hands each entry to `applyCommand`. The existing fail-open rule at `alive.ts:92` is unchanged: a +non-2xx beat returns nothing. That is one more reason the primary path does not depend on a run's +credential. + +--- + +## 7. Stop, end to end + +### Case 1: the normal Stop + +1. The browser posts `POST /sessions/{session_id}/cancel` with `expected_execution_id` filled in + from its own state, and an optional `Idempotency-Key` header. It marks its own view "stopping" + and stops rendering. It does not abort anything server-side by itself. +2. The API authorizes the caller with `Permission.RUN_SESSIONS`, the same permission the current + cancel path uses (`api/oss/src/apis/fastapi/sessions/router.py:377`). +3. The API resolves the target once. It stamps `received_at` first, then reads + `get_running_owner`, falling back to `get_alive_owner`, both already imported by the streams + service (`api/oss/src/core/sessions/streams/service.py:39`), and reads the session row for + `turn_started_at`. Call the result `turn_id`. Three outcomes: if `expected_execution_id` was sent + and differs, stop with 409; if no expected id was sent and `turn_started_at > received_at`, stop + with a settled `superseded_by_newer_turn` command and 200 (section 4); otherwise continue. +4. **One transaction.** Insert the command with `state='pending'`, `kind='cancel'`, + `target_turn_id=turn_id`, `expected_turn_id=`, and set + `session_streams.stopping_turn_id = turn_id` on the same session's row. The DAO method takes an + optional `AsyncSession` so both writes share one session, the pattern `RecordsDAO.append` already + uses (`api/oss/src/dbs/postgres/sessions/records/dao.py:33`). +5. **Redis is not touched.** No tombstone, no `force_cancel_alive`, no `clear_running`. The current + execution keeps `alive` and `running` while it stops, which is what stops a second message from + starting underneath it. This is decision D-017. +6. The API delivers through the configured adapter: the direct call posts to the runner (section 9), + the long-poll adapter publishes on the session's control channel. Either way the API then returns + 202 with the command id and the target execution id. **Delivery failure does not fail the + request**, because the command is already durable. +7. The runner receives the command, on its held claim or on the direct route. +8. `applyCommand` checks the deduplication set, checks that it holds an execution with + `target.turn_id`, checks that the execution did not start after `created_at`, and then aborts it. + The abort must be a harness cancel that keeps the sandbox and the native harness session warm. + **This step is Work package A's deliverable, and it is not free today.** `shouldPark` returns + false whenever the signal is aborted (`services/runner/src/engines/sandbox_agent/engine.ts:26`, + verified), so the environment is destroyed. The review's proposed fix, which this design assumes: + thread a cancel reason to the runner so a user Stop is distinguishable from a disconnect abort, + and let `shouldPark` park when the result is a clean cancellation caused by a user Stop. Nothing + in this design can deliver a warm Stop without that change. +9. The runner posts `POST /sessions/control/commands/{command_id}/outcome` with + `result: "applied"` and `execution: {"id": turn_id, "state": "stopped"}`. +10. The API settles both, in one transaction: + - Command: `state='applied'`, `outcome='stopped'`, `settled_at=now()`, guarded on + `state='claimed' AND claimed_by=`. + - Stream row: clear `stopping_turn_id`. +11. The API releases ownership, in this order: + - `mark_turn_superseded(turn_id)`, so a late beat from the stopped execution cannot re-arm the + locks. + - `release_running(turn_id)`, owner-checked, so it can only release its own execution's key. + - **`alive` is left alone.** It expires on its own time to live, exactly as it does at the end + of a normal turn (`api/oss/src/core/sessions/streams/service.py:590`, verified). This is the + deliberate difference from today's cancel, which force-deletes `alive` and is a large part of + why Stop currently reads as a session teardown. Warm resume is the required outcome, so Stop + must leave the session in the state a finished turn leaves it in. +12. The API cancels the stopped execution's pending interactions, the same call the kill route + already makes (`api/oss/src/apis/fastapi/sessions/router.py:441`), scoped with `only_turn_id` so + it touches only this execution's gates. +13. The API publishes the existing watch notification `lifecycle: ended` on the session channel + (`api/oss/src/core/sessions/streams/service.py:202`), which every open browser already listens + to. +14. Browsers refetch through their current query paths and show the turn as stopped. + +Steps 1 to 8 are the five second budget. Steps 9 to 14 follow the runner's own cancel time. + +### Case 2: Stop when nothing runs + +At step 3 there is no running owner and no alive owner. + +- If the caller sent no `expected_execution_id`: the API inserts the command already settled, + `state='obsolete'`, `outcome='not_running'`, `settled_at=now()`, and returns 200. No Redis write, + no delivery. The caller gets a stable command id, so a retry with the same idempotency key returns + the same record. +- The stream row is not touched, because nothing is stopping. + +### Case 3: Stop with a stale `expected_execution_id` + +The caller sent an execution id that is not the current running owner. The API returns 409 with a +body naming the current execution id, or null when nothing runs. Nothing is inserted and nothing is +delivered. The browser learns that the run it was looking at already ended and refreshes. + +### Case 4: Stop while an interaction is pending and the sandbox is parked + +This is the case with no channel today. A parked approval means the runner is running no turn: the +coordinator seats the environment as `awaiting_approval` +(`services/runner/src/lifecycle/session-coordinator.ts:764`, verified) and the request handler's +`finally` has already released the alive watchdog (`services/runner/src/server.ts:618`, verified), +so the heartbeat has stopped. Redis holds `alive` but not `running`, because the last beat carried +`is_running: false` (`api/oss/src/core/sessions/streams/service.py:590`, verified). + +1. Step 3 finds no `running` owner and does find an `alive` owner. `target_turn_id` takes the alive + owner's value. +2. The command is created `pending` and delivered. **The session is in the runner's declared set**, + because the parked pool entry is one of `SessionPool.keys()`, so the held claim delivers it. With + the direct adapter the process is reachable regardless. +3. `applyCommand` finds no live execution for that turn. It resolves the parked entry instead, + settles the command `applied` with `execution.state = "not_running"`, and leaves the parked + environment in the pool so the session stays warm. It does not destroy the park: Stop ends the + work, not the session. +4. The API settles as in case 1. **Step 12 is the visible part here:** the pending interaction is + cancelled, so the approval card stops rendering as actionable. That closes the class of bugs where + an approval survives a Stop and its buttons do nothing. + +### Case 5: two Stops in a row + +The second request finds an open command for the same `(project_id, session_id, target_turn_id)` +and returns it unchanged, with the same command id. If the second request carries a different +`Idempotency-Key`, the open-command collapse still wins, because it runs before the insert. If the +first command has already settled and a new execution has started, the second Stop is a fresh +command against the new execution, which is what the user meant. + +### Case 6: a Stop that arrives after its turn ended + +The user presses Stop at t=0 while turn one runs. Turn one ends at t=0.1, turn two starts at t=0.2, +and the request is applied at t=0.3. Today `_displace_turns` would tombstone turn two before its +first output, and that tombstone lasts an hour because every read refreshes it +(`api/oss/src/dbs/redis/sessions/locks.py:147`, verified). The four guards of section 4 answer this +case in order. + +1. **Guard 1, at admission.** The API compares `received_at` with the row's `turn_started_at`. Turn + two started after the request arrived, so the API inserts a command that is already settled, + `state='obsolete'` with `outcome='superseded_by_newer_turn'`, targets nothing, touches no Redis + key, and returns 200 with `execution.state = "idle"`. **Turn two never hears about it.** This is + the guard that closes the case; the rest are for what it cannot see. +2. **Guard 2** covers the ordinary late Stop, where turn one simply ended and nothing replaced it. + The command names a turn that no longer exists, so the runner settles `obsolete` with + `not_running`. +3. **Guard 3** covers the residual window where turn two took over between the API's Redis read and + its insert, or where `turn_started_at` was null and guard 1 could not fire. The runner sees an + execution that started after the command's arrival time and settles `obsolete` with + `superseded_by_newer_turn` rather than aborting it. This check is exact, because the runner reads + its own memory. +4. **Guard 4** removes the whole class for first-party clients, which send `expected_execution_id` + and get a 409 naming the current execution. + +No guard writes a Redis tombstone, so nothing can be killed for an hour the way `_displace_turns` +can today. + +### Case 7: the runner is gone + +No claim arrives, or the command was claimed and never settled. The sweep applies the table in +section 4, keyed off heartbeat age rather than the 3600 second Redis time to live. It settles the +command `obsolete` with `outcome='lost'`, force-clears the Redis keys, cancels the pending +interactions, and publishes `ended`. The user sees a terminal state within about two minutes instead +of an hour of "stopping". + +--- + +## 8. The control-delivery port + +There are two ports, one on each side. They are named separately because they are implemented in +different languages by different components, and only one of them is the RFC's `deliver / +acknowledge / recover`. + +### API side, Python + +`api/oss/src/core/sessions/commands/interfaces.py`: + +```python +class ControlDeliveryPort(ABC): + """How the API reaches the runner that holds a session. Transport only. + + Durability, authorization, idempotency, the state machine, and terminal settlement + live in SessionCommandsService and must not move into an adapter. + """ + + @abstractmethod + async def deliver(self, *, command: SessionCommand) -> DeliveryReceipt: + """Make `command` reachable by whoever holds its session, promptly. + + Best effort: a failure here never fails admission, because the command is already + durable and both the sweep and the fallback recover it. The receipt says only what + the transport learned, never what happened to the execution. + """ + ... + + @abstractmethod + async def acknowledge(self, *, command_id: UUID, replica_id: str) -> None: + """Record that a replica took the command, for adapters that keep their own + delivery bookkeeping.""" + ... + + @abstractmethod + async def recover( + self, *, sessions: List[SessionScope], limit: int + ) -> List[SessionCommand]: + """Open commands for these sessions. The claim route, the direct-call retry and the + heartbeat fallback all go through this.""" + ... +``` + +```python +class DeliveryReceipt(BaseModel): + # What the transport learned. Not an execution outcome. + status: Literal["accepted", "unreachable", "not_held"] +``` + +`accepted` means a runner took the command and will report. `unreachable` means the transport +failed, so the sweep or a later claim will handle it. `not_held` means a reachable runner said it +does not hold that session, which lets the service settle the command at once instead of waiting for +the deadline. + +A later adapter must provide prompt, at-least-once delivery to whoever holds the named session. It +may reorder. It may deliver twice. It must not transform or interpret a command, must not settle +one, and must not be the only record that a command exists. Replacing it must change no route, no +DTO, and no state transition. + +### Runner side, TypeScript + +`services/runner/src/sessions/control-channel.ts`: + +```ts +/** One command as the API delivers it. The same shape arrives on every transport. */ +export interface ControlCommand { + id: string; + projectId: string; + sessionId: string; + kind: "cancel"; + target: { turnId: string | null; expectedTurnId: string | null }; + createdAt: string; +} + +export interface ControlOutcome { + /** The command's terminal state. */ + result: "applied" | "obsolete"; + execution: { + id: string | null; + state: "stopped" | "failed" | "not_running" | "superseded_by_newer_turn"; + error?: string; + }; +} + +/** The transport. `control-poll.ts` implements it over long polling; the direct route + * in `server.ts` feeds the same applier without implementing this at all. */ +export interface ControlChannel { + /** Block until a command arrives for one of `sessions`, or the hold expires. */ + receive(sessions: SessionScope[], signal: AbortSignal): Promise; + settle(command: ControlCommand, outcome: ControlOutcome): Promise; +} +``` + +`applyCommand(command)` sits above the channel, not inside it, so every path shares one applier, one +set of guards and one deduplication set. + +The runner also needs an execution registry, because the abort controller is a local variable inside +`runAndStreamWithApiBaseResolved` today (`services/runner/src/server.ts:450`, verified). Add a +module-level map from `${projectId}:${sessionId}` to `{ turnId, startedAt, abort(): void }`, +registered when the run starts and removed in the same `finally` that releases the watchdog +(`services/runner/src/server.ts:618`). `startedAt` is what guard 3 of section 4 compares. This +mirrors `inFlightSandboxes` (`services/runner/src/engines/sandbox_agent/environment.ts:239`). + +--- + +## 9. The direct-call adapter as an alternative first adapter + +This is the section the architecture review asked for as 8b. It sits here, directly after the port, +because that is what it is: the second adapter behind the same port, and a candidate for being the +**first** one built. + +The product review argues that with one runner, the authenticated API-to-runner hop that already +carries hard kill can carry Cancel today, and that long polling is machinery for a second runner that +does not exist. The RFC's own text agrees that direct managed-runner routing is a legitimate adapter +behind the port (`rfc.md`, "Control delivery must sit behind an internal port"). That argument is +correct on its own terms, and this design makes both adapters cheap so Mahmoud can pick either in +the morning without changing anything else. + +### What already exists + +- **The API side.** `kill_runner_sandbox` posts `{sessionId, projectId}` with + `Authorization: Bearer ` to `env.runner.internal_url` and swallows every + failure (`api/oss/src/core/sessions/streams/runner_client.py:30`, verified). It is 33 lines. +- **The runner side.** `POST /kill` sits behind the same token gate, reads a capped body, resolves + the pool scope and tears the session down (`services/runner/src/server.ts:704`, verified). + +### What the direct adapter adds + +**The runner: `POST /cancel`, beside `/kill`.** Same token gate, same capped body reader, same +scoping rule. Body: + +```json +{ + "commandId": "0199a3f2-0000-7000-8000-000000000001", + "projectId": "1f0a4b2c-0000-4000-8000-000000000002", + "sessionId": "sess-42", + "targetTurnId": "0199a3f1-0000-7000-8000-00000000000a", + "createdAt": "2026-09-02T22:09:01Z" +} +``` + +It builds a `ControlCommand` from that body and hands it to the same `applyCommand`. It answers 202 +when it holds the session and has accepted the command, and 404 when it does not. It does **not** +return the execution outcome: the runner reports that through the settle route, so settlement has +one path on every transport. Roughly 40 lines beside the existing kill branch. + +**The API: `cancel_runner_execution`, beside `kill_runner_sandbox`.** The same 30 lines with a +different path and body. The adapter maps the response: 202 to `accepted`, 404 to `not_held`, +anything else and every exception to `unreachable`. One file, +`api/oss/src/dbs/http/sessions/control_delivery_direct.py`, implementing `ControlDeliveryPort`. +`acknowledge` is a no-op, because the claim compare-and-set is the acknowledgement. `recover` runs +the same query the claim route runs, and the service calls it from the sweep. + +**The durable command is still inserted first.** The order is not negotiable and it is the whole +difference between this adapter and a bare remote call: + +1. Admit and insert the command, with `stopping_turn_id`, in one transaction. Commit. +2. Only then call the runner. +3. Whatever the call returns, the user's request has already succeeded. A `not_held` lets the + service settle at once; an `unreachable` leaves the command `pending` for the sweep or for a + later retry. **Neither changes the 202.** + +Inverting those two steps, calling first and recording afterwards, would give back every failure the +record exists to close, because a crash between the call and the insert leaves an aborted execution +with no terminal outcome written anywhere. + +### What it cannot do + +- **Reach a session it cannot resolve locally.** A Stop against a parked approval has no entry in the + execution registry, because no turn is running. The runner must fall back to the keep-alive pool, + which already has the lookup for exactly this: `SessionPool.awaitingApproval(sessionId)` + (`services/runner/src/engines/sandbox_agent/session-pool.ts:117`, verified). That is a few lines, + but it is not free, and it is needed by both adapters. Do not treat the parked case as covered + just because the process is reachable. +- **Survive a second runner replica.** `env.runner.internal_url` is one service address + (`api/oss/src/core/sessions/streams/runner_client.py:44`, verified). Behind a load balancer the + call lands on whichever replica answers, which is the right one only by luck. +- **Reach a user-operated runner.** It needs inbound reachability from the API to the runner. A + runner behind a firewall cannot be called at all. The RFC treats that deployment as a + consideration rather than a requirement, so this is a real but not yet binding limit. + +### Making the wrong-replica failure loud + +The silent-failure worry is fair, and there are two ways to close it. Build the first; the second is +optional. + +**Primary, and exact: treat a contradictory `not_held` as an error.** A mis-routed call is not +actually silent at the protocol level. The runner answers 404 `not_held` when it does not hold the +session, so the API always learns that delivery did not land. What makes it dangerous is that +`not_held` is also the **legitimate** answer when the session really has ended, so the two cases look +alike. They are easy to tell apart with data the API already has: + +> A `not_held` for a session whose `session_streams` row says `is_alive` **and** whose heartbeat age +> is under one interval means some process is running that session and it is not the one we just +> called. That is the wrong-replica failure, and nothing else produces it. + +On that condition, log at error level with the session id, the target turn id and the replica id +from the Redis `owner` key, count it on a metric, and settle the command `obsolete` with +`outcome='lost'` rather than `not_running`, so the user is told the Stop failed instead of being +told the work had already finished. This needs no new storage and no census. + +**Optional, preventive: refuse the configuration.** Two parts, both cheap: + +- A required flag. The direct adapter refuses to start unless + `AGENTA_SESSIONS_CONTROL_DIRECT_SINGLE_REPLICA=true` is set, so choosing it is a deliberate + statement about the deployment rather than a default someone inherited. Optionally let the operator + name the replica instead, `AGENTA_SESSIONS_CONTROL_DIRECT_REPLICA_ID=`, and refuse delivery + when the session's owner key names a different one. +- A replica census. The heartbeat handler already computes the owning `replica_id` on every beat + (`api/oss/src/core/sessions/streams/service.py:458`). Have it also run one `ZADD` into a sorted set + keyed by replica id and scored by timestamp. The sweep then reads `ZCOUNT` over the last 10 + minutes and, if the direct adapter is configured and the count exceeds one, logs an error every + pass naming the replicas it saw. One write per beat, one read per sweep, no key scan. + +Do not add a retry across the load balancer in the hope of hitting the right process. It converts a +diagnosable failure into a lottery, and it multiplies load exactly when a deployment is already +misconfigured. + +### What the durable command record adds beyond a bare direct call + +The direct call alone would be an HTTP request with no memory. The record buys four things, and each +one is a bug the current system has: + +1. **Recovery.** The runner can be restarting, deploying, or briefly unreachable. A bare call fails + and the Stop is gone; the user pressed a button and nothing happened. With the record the command + survives, the sweep settles it as `lost` with a terminal outcome the user sees, and a returning + runner picks it up on its next claim. +2. **Idempotency.** Two Stops, a retried request, or a browser that resends on reconnect all collapse + onto one command. A bare call would abort twice, and the second abort can land on a newer turn. + That is review hole H-3 in its cheapest form. +3. **One terminal outcome per execution.** The record is where `stopped`, `not_running`, + `superseded_by_newer_turn`, `failed` and `lost` are written down, and where the watchdog and the runner agree + on who wrote it. A bare call has nowhere to record that the execution really ended. +4. **Audit and the next command kinds.** Who stopped what, when, and what happened. Steer and Queue + need exactly this record, so building it now is not speculative: it is the part of version one + that version two does not have to redo. + +The honest counter-argument, stated plainly: for a single Stop that succeeds on the first try, the +record adds a table and two writes and changes nothing the user sees. Its value is entirely in the +failure cases. + +### Choosing the adapter + +One setting, `AGENTA_SESSIONS_CONTROL_ADAPTER`, with values `direct` and `long_poll`, read through +`env`. The service depends only on the port. Neither adapter changes a route, a DTO, or a state +transition. + +| | Direct call | Long poll | +|---|---|---| +| New code | One runner route, one API client, both small | A runner loop, an API route with a hold, a Redis channel | +| Reaches a parked session | Yes, with the pool lookup above | Yes, the parked session is in the declared set | +| Two or more runner replicas | Wrong process gets the call. Loud with the `not_held` rule above, silent without it | Correct, because the runner declares what it holds | +| Runner behind a firewall | Impossible | Works | +| Runner restarting | The call fails, the sweep settles or a later claim delivers | The claim resumes on reconnect | +| Held connections | None | One per runner process | + +**If `direct` is the default, PR 3b in section 10 is deferred** and the session-scoped loop is not +built at all. H-2 is then closed by the direct route plus the pool lookup rather than by the loop, +and the heartbeat fallback stays as the second path for a session with a live turn. Everything else +in this design is unchanged, which is the point of the port. + +### Recommendation + +**Build the direct adapter first.** Three reasons, in order of weight: + +1. **It removes the largest piece of new machinery from the first release.** No held connection, no + poll loop, no per-session Redis channel, no uvicorn shutdown interaction. The parts that carry the + correctness, the record, the state machine, the guards and the settlement rule, are identical + either way, and they are the parts worth reviewing carefully. +2. **The deployment it fails on does not exist yet.** Agenta runs one runner. The failure mode is + real, and the `not_held` rule above makes it loud rather than silent, which is what turns a + dangerous limitation into a known one. +3. **The port makes the switch small.** Long polling stays one file plus one runner module. When a + second replica or a user-operated runner becomes real, the change is a configuration value and a + module, not a redesign. + +The cost of being wrong is bounded and visible: if a second replica appears before the long-poll +adapter is built, Stop starts failing loudly on the wrong-replica condition and the fix is already +designed. The cost of building long polling first is a larger first release for a deployment that +does not exist. Take the smaller one. + +--- + +## 10. Migration sequence + +Eight pull requests. Each names the files it touches so parallel agents do not collide. Ordering +constraints are stated; anything not constrained can go in any order. + +| PR | Title | Files | Depends on | +|---|---|---|---| +| 1 | Add the session command record | `api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py`, `api/oss/src/dbs/postgres/sessions/commands/{dbas,dbes,dao,mappings}.py`, `api/oss/src/core/sessions/commands/{dtos,interfaces,service,types}.py`, `api/oss/src/utils/env.py`, `api/entrypoints/routers.py` (wiring only), `api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py` | none | +| 2 | Runner execution registry and applier | `services/runner/src/sessions/control-channel.ts`, `services/runner/src/sessions/execution-registry.ts`, `services/runner/src/sessions/applied-commands.ts`, `services/runner/src/server.ts` (register and unregister), runner unit tests | none | +| 3a | Direct-call adapter | `services/runner/src/server.ts` (the `/cancel` route and the parked-pool lookup), `api/oss/src/dbs/http/sessions/control_delivery_direct.py` (including the wrong-replica detector of section 9) | 1, 2 | +| 3b | Long-poll adapter | `api/oss/src/apis/fastapi/sessions/router.py` (`SessionControlRouter`), `api/oss/src/apis/fastapi/sessions/models.py`, `api/oss/src/middlewares/auth.py` (one prefix), `api/oss/src/dbs/redis/sessions/contract.py`, `api/oss/src/dbs/redis/sessions/control_delivery.py`, `services/runner/src/sessions/control-poll.ts` | 1, 2 | +| 4 | Public Cancel creates a command | `api/oss/src/apis/fastapi/sessions/router.py`, `api/oss/src/apis/fastapi/sessions/models.py`, `api/oss/src/core/sessions/commands/service.py`, migration `oss000000023` for `session_streams.stopping_turn_id` **and** `session_streams.turn_started_at`, `api/oss/src/dbs/postgres/sessions/streams/{dbas,dbes,dao}.py` (the `CASE` that stamps the start time), `api/oss/src/core/sessions/streams/service.py` (`_start_turn` and the heartbeat stamp) | 1 | +| 5 | Heartbeat command discovery | `api/oss/src/core/sessions/streams/{dtos,service}.py`, `services/runner/src/sessions/alive.ts` | 3a or 3b, and 4 | +| 6 | Command settlement in the watchdog | `api/oss/src/tasks/asyncio/sessions/command_sweep.py` or the equivalent file on `feat/session-execution-watchdog`, `api/entrypoints/routers.py` (lifespan) | 1, and agreement with the watchdog author | +| 7 | Point the clients at the command | `web/packages/agenta-entities/src/session/api/api.ts`, `web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts` (send `expected_execution_id`), `web/mobile/src/features/chat/StopButton.tsx`, `api/oss/src/core/sessions/streams/service.py` (the cancel branch becomes a wrapper) | 4, 5 | + +3a and 3b are alternatives, not a sequence. Build whichever Mahmoud picks; the other becomes optional +later work. + +Conflict notes: + +- PRs 1, 3b, 4 and 6 touch `api/entrypoints/routers.py`. Keep each edit to its own block and land + them in order. +- PRs 3b and 4 both touch `router.py` and `models.py`. Land 3b first; 4 adds a separate router class. +- PR 2 must land before 3a or 3b, because both need the registry and the applier. +- PRs 1 and 4 each add a migration and must not both claim `oss000000022`. +- PR 6 must be agreed with the agent on `feat/session-execution-watchdog` before either lands. Two + independent writers of an execution's terminal outcome is a worse bug than the one being fixed. +- **Work package A's `shouldPark` change is a hard dependency of the user-visible result.** Landing + PRs 1 to 7 without it gives a fast Stop that still destroys the sandbox. + +**Keeping the current Stop working.** `POST /sessions/streams/` with no inputs and no `force` keeps +its exact current behavior through PRs 1 to 6. Nothing about `CommandMode.cancel` changes. Released +browsers and the current mobile build keep working unchanged. + +**When it becomes a wrapper.** In PR 7. At that point `SessionStreamsService.command`'s cancel +branch (`api/oss/src/core/sessions/streams/service.py:288`) stops calling `_displace_turns` and +instead calls `SessionCommandsService.request_cancel(...)` with no expected execution id, then +returns the same `SessionStreamCommandResponse` shape it returns today. That gives every old client +the new behavior with no client change, and it is also the point at which the old teardown of +`alive` and the hour-long tombstone disappear. Do it in the same PR that flips the browser, so one +revert restores one consistent behavior. + +--- + +## 11. Test plan + +### Unit tests + +| Component | Test | Passes when | +|---|---|---| +| Commands DAO | Two concurrent claims of one pending command | Exactly one returns a row; the other returns none | +| Commands DAO | Insert with a repeated `Idempotency-Key` | The second insert returns the first row, and one row exists | +| Commands DAO | Settle with the wrong `replica_id` | Returns no row; the stored state is unchanged | +| Commands DAO | Settle a command that is already `applied` | Returns no row; the caller reads the terminal state | +| Commands DAO | Claim with a session set that excludes the command's session | Returns nothing | +| Commands service | Admission with a stale `expected_execution_id` | Raises the conflict type; no row inserted | +| Commands service | Admission with nothing running or parked | One row, `state='obsolete'`, `outcome='not_running'` | +| Commands service | Admission when `turn_started_at` is later than `received_at` | One row, `state='obsolete'`, `outcome='superseded_by_newer_turn'`, `target_turn_id` null, no Redis write | +| Commands service | Admission when `turn_started_at` is null | The guard does not fire; the command targets the current turn | +| Commands service | Admission when `turn_started_at` is earlier than `received_at` | Normal admission, `state='pending'` | +| Commands service | The stored `created_at` equals the `received_at` that was compared | The two values match exactly, not merely closely | +| Streams DAO | The same `turn_id` stamped by ten heartbeats | `turn_started_at` is written once and never moves | +| Streams DAO | A new `turn_id` stamped over an old one | `turn_started_at` moves to the new turn's time | +| Commands service | Admission twice with no idempotency key | One row; the second call returns the first | +| Commands service | Admission writes the command and `stopping_turn_id` | Both are visible after one commit, neither after a rollback | +| Command sweep | Claim expired, session beating, attempts left | Back to `pending` | +| Command sweep | Claim expired, session silent for 90 s | `obsolete`, `outcome='lost'`, keys force-cleared, `ended` published | +| Command sweep | Claim expired, session parked with an open interaction | Not settled as lost; the admission deadline applies instead | +| Command sweep | Redis `alive` still holds its 3600 s value | Settlement still happens, because the rule reads heartbeat age, not the key | +| Direct adapter | Runner answers 404 for a session whose row is not alive | Receipt is `not_held`; the command settles `obsolete` with `not_running` | +| Direct adapter | Runner answers 404 for a session that is alive and beating | Logged at error level, counted, and settled `obsolete` with `lost`, never `not_running` | +| Direct adapter | Runner unreachable | Receipt is `unreachable`; admission still succeeded and returned 202 | +| Direct adapter | The command row exists before the runner is called | A crash injected between the two leaves a `pending` command, never an aborted execution with no record | +| Long-poll adapter | `deliver` when Redis is down | Admission still succeeds; the failure is logged, not raised | +| Runner claim loop | 204, then 200, then a network error | Immediate re-claim, apply, then the backoff sequence with jitter | +| Runner claim loop | 401 | One error log, then a 60 second retry, no tight loop | +| Runner claim loop | Session set includes a parked pool entry | The parked session appears in the request body | +| Runner applier | A command for a `turnId` this process does not hold | Settles `obsolete` with `not_running`; nothing is aborted | +| Runner applier | The held execution started after the command's `created_at` | Settles `obsolete` with `superseded_by_newer_turn`; nothing is aborted | +| Runner applier | The same `command_id` delivered twice | Aborted once, acknowledged twice | +| Runner applier | The deduplication set survives a loop restart | A command applied before the restart is not applied again | +| Runner registry | The run's `finally` runs | The entry is removed even when the run threw | + +The runner suite is `cd services/runner && pnpm test` (vitest). The API unit tests sit under +`api/oss/tests/pytest/unit/sessions/`, next to `test_command_matrix_inputs_data.py`. + +### One API integration test + +`api/oss/tests/pytest/integration/sessions/test_stop_command_delivery.py`, against a real Postgres +and a real Redis, with a fake runner: + +1. Establish a session with `alive` and `running` held by `turn-A`, exactly as a heartbeat does. +2. Call the public Cancel route with `expected_execution_id = 'turn-A'`. Assert 202, one `pending` + row, and `session_streams.stopping_turn_id = 'turn-A'`. +3. Call the claim route as `replica-1`, declaring that session. Assert 200, one command, + `state='claimed'`. +4. Call the claim route again. Assert 204 within the hold. +5. Post the outcome with `result='applied'` and `execution.state='stopped'`. Assert 200. +6. Assert: the command is `applied` with `outcome='stopped'`; `stopping_turn_id` is null; the Redis + `running` key is gone; **the Redis `alive` key is still present**; `superseded:...:turn-A` exists; + the session's pending interactions are cancelled; one `lifecycle: ended` message was published on + the session watch channel. + +Step 6's `alive` assertion is the one that pins warm resume at the API layer. If a later change +starts clearing `alive` on Stop, this test fails. + +Add a second integration case for the parked path: park the session (no `running`, `alive` held, one +pending interaction), Stop it, and assert the command is delivered, the interaction is cancelled, and +`alive` still holds. + +### One live-stack wire test + +Add a cell to the agent release gate, next to the existing W5 steer cell +(`.agents/skills/agent-release-gate/resources/`), driving a deployed stack over the product +endpoints only: + +1. Start a turn with a prompt that runs for at least 60 seconds. +2. Wait for the first agent output frame, then record the wall clock and press Stop through + `POST /sessions/{id}/cancel`. +3. **Pass criterion one:** the runner reports the outcome, and the session's `running` flag goes + false, within **5 seconds** of the Stop request. Measure from the request, not from the frame. +4. **Pass criterion two:** `session_turns` for the stopped turn still names the same `sandbox_id` + and `agent_session_id` as before the Stop, and the session's `alive` flag is still true. +5. Send a second message on the same session. +6. **Pass criterion three:** the second turn reuses the same `sandbox_id` and `agent_session_id`. + That is warm resume, measured from stored rows rather than from timing. +7. **Pass criterion four:** the stopped turn's records end with a cancelled outcome, not an error + record. + +A second cell for the parked path: run a prompt that triggers an approval, wait for the gate, press +Stop, and assert that the outcome lands within 5 seconds, the interaction reads `cancelled`, and the +next message still resumes warm. That cell is the regression test for H-2 and it fails on today's +code for a reason no timing change can fix. + +Criteria 2, 3 and 4 depend on Work package A. Criterion 1 does not, and can be gated as soon as PR 7 +lands. + +--- + +## 12. Rejected alternatives + +**Shorten the heartbeat interval.** Dropping `HEARTBEAT_INTERVAL_SECONDS` from 30 to 2 would cut the +Stop delay with no new machinery. It fails on four counts. It multiplies heartbeat load by fifteen +for every live session, and each beat is a Postgres write plus four Redis operations +(`api/oss/src/core/sessions/streams/service.py:406`). It cannot deliver a Stop to a run whose +credential was dropped, because the beat itself is what fails (`alive.ts:92`). It cannot deliver a +Stop to a parked session at any interval, because the heartbeat has stopped (`server.ts:618`). And it +leaves the control signal encoded as the absence of a lock, which is what makes today's cancel a +session teardown rather than an execution cancel. + +**Route commands by owner replica instead of by declared session.** This was the first revision's +design and it is worse. The Redis `owner` key expires after 120 seconds +(`api/oss/src/dbs/redis/sessions/contract.py:40`), so a parked session's owner can lapse and its +commands become unroutable. It also cannot tell whether the named replica still holds the session, +which is exactly the question delivery needs answered. Letting the runner declare what it holds +turns a guess into a fact, and it removes a column from the durable record. + +**Subscribe the runner to Redis directly.** The runner could subscribe to a per-session Pub/Sub +channel and skip the claim. It is the least code. It fails on the boundary the codebase already +enforces: the API is the single Redis writer and the runner reaches the coordination plane only over +HTTP (`services/runner/src/sessions/alive.ts:13` and `sessions/contract.ts:25`, both explicit about +this). Handing the runner Redis credentials reverses a deliberate decision, and Pub/Sub has no +replay, so a disconnected runner loses every command sent while it was away. + +**A persistent WebSocket or bidirectional stream.** It removes the repeated request and can carry +richer runner status. It is deferred, not wrong. It needs connection lifecycle handling, ping and +pong, reconnect with backoff, and a message framing contract, none of which the command state +machine needs to be correct. Because delivery sits behind the port in section 8, it becomes a later +adapter rather than a rewrite. + +**Skip the durable record and make Stop a bare direct call.** This is the product review's position +and it is the strongest alternative. Note what is and is not rejected here. The **direct call** is +not rejected at all: it is section 9, it is a first-class adapter behind the port, and it is the +recommended first adapter. What is rejected is dropping the **record**, for the four reasons set out +in section 9: no recovery when the runner is unreachable, no idempotency against a double Stop +landing on a newer turn, no place to write the one terminal outcome the watchdog and the runner must +agree on, and no foundation for Steer and Queue. Insert first, then call. + +--- + +## 13. Open questions for Mahmoud + +1. **Which adapter is the default, `direct` or `long_poll`?** Recommendation: **`direct`** for + version one, with the wrong-replica detector from section 9 built in the same PR. Reason: you run + one runner, the hop is authenticated and in production today, it reaches a parked session once the + pool lookup is added, and it removes a held connection and a poll loop from the first release. The + port keeps long polling one file away for the day a second replica or a user-operated runner is + real. The condition on the recommendation: the detector is not optional, because without it the + two-replica failure is silent, and with it the choice is reversible on a metric rather than on a + bug report. + +2. **Who owns execution settlement, this design or the watchdog branch?** Recommendation: **the + watchdog owns it, and the command rules move into it.** Reason: one execution must reach exactly + one terminal outcome from exactly one writer, and two sweeps racing to write `lost` is a worse + bug than the one being fixed. This needs deciding before PR 6 and before the watchdog branch + lands. + +3. **Does Stop leave the Redis `alive` key in place?** Recommendation: **yes, leave it**, exactly as + a normal turn end does. Reason: force-deleting `alive` is what makes today's cancel read as a + session teardown, and warm resume is the required outcome. This is a deliberate deviation from + the phrase "Redis `running` and `alive` released" in the work package brief, so it needs an + explicit yes or no. + +4. **Do first-party clients always send `expected_execution_id`?** Recommendation: **yes, and treat + an omission as a bug.** Reason: it is the cheapest of the three H-3 guards and the only one that + works before the request reaches the server. The field stays optional in the contract for + external callers, as decision D-010 requires. + +5. **Do we cancel the pending interaction when Stop hits a parked session?** Recommendation: + **yes, cancel it, and keep the parked environment.** Reason: an approval card whose execution was + stopped is exactly the "actionable card whose buttons do nothing" bug, and the kill route already + makes this call (`api/oss/src/apis/fastapi/sessions/router.py:441`). Keeping the environment is + what makes the next message warm, and it is what distinguishes Stop from Delete. diff --git a/docs/design/session-control-and-live-events/status.md b/docs/design/session-control-and-live-events/status.md new file mode 100644 index 00000000000..da73284e795 --- /dev/null +++ b/docs/design/session-control-and-live-events/status.md @@ -0,0 +1,63 @@ +# Status + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +## Current state + +- Isolated branch created: `agent/session-execution-rfc`. +- Problem inventory created from 48 open GitHub issues. +- Current Stop, heartbeat, records, and watch paths checked against the repository. +- Confirmed process decisions recorded. +- Proposed architecture choices kept separate from confirmed decisions. +- Living RFC created with empty sections for track-by-track discussion. +- Current command endpoint and runner routing boundary verified. +- Sandbox-agent cancellation investigation promoted to the first parallel task. +- Five seconds recorded as the provisional Stop delivery target. +- Public resource API separated from the proposed internal command transport. +- Current interaction response path documented. +- Public APIs from Gumloop, OpenAI background Responses, and Claude Managed Agents compared. +- Each current operation mapped to its proposed behavior and degree of change. +- Stop and Delete distinction confirmed. +- Optional `expected_execution_id` guard recorded. +- One public session API for first-party and external clients recorded. +- Visible server-side pending inputs added to the interface discussion. +- Queued inputs made immutable. Clients can remove and replace them, but cannot edit or reorder. +- Detailed API mechanics delegated to established conventions unless they affect architecture. +- Durable acceptance defined independently from runner claim and execution start. +- Sender-only visibility explicitly excluded from the target requirements. +- Proposed snapshot and event routes explicitly marked as new contracts, not changed meanings of + current stream routes. +- Side-by-side endpoint migration accepted as the first draft. Final naming deferred. +- Existing record properties, violations, structural constraints, and repair options traced before + selecting a replay storage design. +- Corrected the cursor analysis: plain Postgres sequences do not guarantee commit visibility order. +- Added the repaired-records and separate-event-log options with trade-offs. Redis-only permanent + history excluded from the draft. +- Added a mandatory stable-ID producer spike before immutable record changes. +- Made single active execution and stale-writer fencing explicit requirements. +- Kept the public Stop execution guard optional. +- Added possible future user-operated runners as a control-transport consideration, not a + requirement. +- Implemented direct control delivery behind a replaceable adapter for version one. +- Recorded warm sandbox and harness resume as the required Stop outcome. +- Confirmed the minimal internal command lifecycle and its separation from public execution state. +- Left the Stop settlement timeout for the sandbox cancellation spike. +- Confirmed that the first version keeps current Redis execution ownership. +- Kept durable commands and direct delivery in scope; deferred long polling and full fencing. + +## Branch + +- Branch: `agent/session-execution-rfc` +- The branch is pushed to `Agenta-AI/agenta` after each design exchange. + +## Next discussion + +Start with **Stop and ownership**: + +1. Start the sandbox-agent capability investigation. +2. Confirm the user-visible Stop requirements and latency target. +3. Validate the direct runner-control transport and its failure behavior. +4. Define terminal settlement and watchdog responsibility. +5. Decide which current issues this track is expected to close. + +The **live-frame ingress** discussion can proceed independently after that or in parallel. diff --git a/docs/design/session-control-and-live-events/tonight-handoff.md b/docs/design/session-control-and-live-events/tonight-handoff.md new file mode 100644 index 00000000000..cbda2972594 --- /dev/null +++ b/docs/design/session-control-and-live-events/tonight-handoff.md @@ -0,0 +1,84 @@ +# Tonight handoff + +> AGENT-GENERATED, low weight. Draft execution handoff. Mahmoud makes final decisions. + +## Fixed direction + +- Keep current Redis execution ownership for version one. +- Add durable commands with `pending`, `claimed`, `applied`, and `obsolete` states. +- Use direct API-to-runner HTTP behind a replaceable control-delivery port for version one. +- Keep `expected_execution_id` optional on public Stop. +- Keep the Redis ownership lock until Stop settles. +- Keep durable storage and settlement independent of the delivery transport. +- Use heartbeat command discovery as delivery fallback. +- Require same-sandbox and native-session resume only for harnesses and environments that expose + resumable cancellation. Run this release-gate cell for every supported harness and + sandbox-provider pair; record an explicit cold-start result where resume is unavailable. +- Keep live-frame work independent from Stop work. +- Park the repaired-records versus separate-event-table decision for review. + +## Work package A: sandbox cancellation spike + +**Goal:** Identify which cancellation paths preserve warm resume and qualify the requirement by +capability. + +Answer: + +1. Which request cancels a prompt in each supported harness? +2. Does it preserve the native harness session? +3. What happens to a running tool and partial message? +4. Does the runner park or destroy the sandbox on every cancellation path? +5. Is a sandbox-agent patch required? +6. Does Daytona need a rebuilt snapshot? + +Deliver a code-traced report, a characterization test, the smallest patch proposal, and a live test +plan for start, Stop, and resume. Require the same sandbox and native session only where the harness +and environment report that capability. Do not redesign ownership, commands, or public endpoints. + +## Work package B: durable command and direct-delivery design + +**Goal:** Produce an implementation-ready design for reliable API-to-runner commands. + +Define the command schema, idempotency, direct-delivery acknowledgement, failure recovery, adapter +boundary, and how Redis ownership remains held until Stop settles. Keep long-poll claim semantics +as a deferred transport. Do not implement a new execution ownership model. + +## Work package C: current Stop implementation map + +**Goal:** Remove uncertainty before changing Stop. + +Trace the browser request, API stream mutation, Redis key changes, heartbeat response, runner abort, +sandbox cleanup, records, interactions, and frontend refresh. List every branch that means cancel, +kill, steer, or approval interruption. Deliver a sequence diagram and file-by-file change map. Do +not implement changes. + +## Work package D: stable record-ID spike + +**Goal:** Make the later immutable-history decision safe. + +Inventory every stable `record_id` producer and classify repeated IDs as exact retries, +progressive updates, or resume re-emissions. Add or propose regression tests for final tool state, +interaction responses, terminal events, and harness reconstruction. Do not select repaired records +or a separate event table. + +## First implementation after the spikes + +1. Add the durable command repository and service behind interfaces. +2. Add the direct API-to-runner adapter and authenticated runner route. +3. Let Stop create a durable command with an optional expected-execution guard. +4. Let the runner apply Stop through its active abort controller. +5. Preserve Redis ownership until cancellation settles. +6. Make heartbeat discover pending Stop as fallback. +7. Emit the durable cancellation outcome and publish the existing watch notification. +8. Prove Stop delivery within five seconds and warm resume on the live stack. + +## Deferred explicitly + +- Postgres execution authority. +- Ownership generations and full fencing. +- Multiple-runner routing guarantees. +- User-operated runner requirements. +- Final records versus event-table selection. +- Final public endpoint naming. +- WebSocket or gRPC control transport. +- Runner-initiated long-poll control transport. diff --git a/hosting/docker-compose/ee/docker-compose.dev.yml b/hosting/docker-compose/ee/docker-compose.dev.yml index b8be5619aa1..855fb1360eb 100644 --- a/hosting/docker-compose/ee/docker-compose.dev.yml +++ b/hosting/docker-compose/ee/docker-compose.dev.yml @@ -498,6 +498,9 @@ services: # Worst-case turn duration. Keep AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS (API) above it # plus 60s of skew, or every dispatch rebuilds the environment cold. AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS:-} + # Defaults: 30 min without progress; 30 min for one tool call. + AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS:-} + AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS: ${AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS:-} AGENTA_RUNNER_DAYTONA_API_KEY: ${AGENTA_RUNNER_DAYTONA_API_KEY:-} AGENTA_RUNNER_DAYTONA_API_URL: ${AGENTA_RUNNER_DAYTONA_API_URL:-} AGENTA_RUNNER_DAYTONA_TARGET: ${AGENTA_RUNNER_DAYTONA_TARGET:-} diff --git a/hosting/docker-compose/ee/docker-compose.gh.local.yml b/hosting/docker-compose/ee/docker-compose.gh.local.yml index f1d9d123be1..52fa32fb570 100644 --- a/hosting/docker-compose/ee/docker-compose.gh.local.yml +++ b/hosting/docker-compose/ee/docker-compose.gh.local.yml @@ -332,6 +332,9 @@ services: # Worst-case turn duration. Keep AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS (API) above it # plus 60s of skew, or every dispatch rebuilds the environment cold. AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS:-} + # Defaults: 30 min without progress; 30 min for one tool call. + AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS:-} + AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS: ${AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS:-} PI_CODING_AGENT_DIR: ${PI_CODING_AGENT_DIR:-/pi-agent} AGENTA_RUNNER_DAYTONA_API_KEY: ${AGENTA_RUNNER_DAYTONA_API_KEY:-} AGENTA_RUNNER_DAYTONA_API_URL: ${AGENTA_RUNNER_DAYTONA_API_URL:-} diff --git a/hosting/docker-compose/ee/docker-compose.gh.yml b/hosting/docker-compose/ee/docker-compose.gh.yml index 965261db52e..b1d8086e097 100644 --- a/hosting/docker-compose/ee/docker-compose.gh.yml +++ b/hosting/docker-compose/ee/docker-compose.gh.yml @@ -334,6 +334,9 @@ services: # Worst-case turn duration. Keep AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS (API) above it # plus 60s of skew, or every dispatch rebuilds the environment cold. AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS:-} + # Defaults: 30 min without progress; 30 min for one tool call. + AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS:-} + AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS: ${AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS:-} PI_CODING_AGENT_DIR: ${PI_CODING_AGENT_DIR:-/pi-agent} AGENTA_RUNNER_DAYTONA_API_KEY: ${AGENTA_RUNNER_DAYTONA_API_KEY:-} AGENTA_RUNNER_DAYTONA_API_URL: ${AGENTA_RUNNER_DAYTONA_API_URL:-} diff --git a/hosting/docker-compose/ee/env.ee.dev.example b/hosting/docker-compose/ee/env.ee.dev.example index 411db1efddf..ce26143d5aa 100644 --- a/hosting/docker-compose/ee/env.ee.dev.example +++ b/hosting/docker-compose/ee/env.ee.dev.example @@ -136,6 +136,9 @@ AGENTA_RUNNER_DEFAULT_SANDBOX_PROVIDER=local # Smart truncation preserves the structure of a record whose body exceeds the API size # cap (higher-fidelity reconstruction). Still opt-in, default off. # AGENTA_RECORDS_SMART_TRUNCATION=true +# Durable Stop is enabled by default; set false to use legacy cancellation. +AGENTA_SESSIONS_DURABLE_STOP=true +# AGENTA_SESSIONS_LATE_OUTPUT=quarantine # --- Attachment limits (files attached to an agent chat turn) --- # Per-file caps in bytes, by kind: 10 MB, except audio at 15 MB. Read by the api. diff --git a/hosting/docker-compose/oss/docker-compose.dev.yml b/hosting/docker-compose/oss/docker-compose.dev.yml index 26f04cd1b43..cc9b9da708d 100644 --- a/hosting/docker-compose/oss/docker-compose.dev.yml +++ b/hosting/docker-compose/oss/docker-compose.dev.yml @@ -463,6 +463,9 @@ services: # Worst-case turn duration. Keep AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS (API) above it # plus 60s of skew, or every dispatch rebuilds the environment cold. AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS:-} + # Defaults: 30 min without progress; 30 min for one tool call. + AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS:-} + AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS: ${AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS:-} AGENTA_RUNNER_DAYTONA_API_KEY: ${AGENTA_RUNNER_DAYTONA_API_KEY:-} AGENTA_RUNNER_DAYTONA_API_URL: ${AGENTA_RUNNER_DAYTONA_API_URL:-} AGENTA_RUNNER_DAYTONA_TARGET: ${AGENTA_RUNNER_DAYTONA_TARGET:-} diff --git a/hosting/docker-compose/oss/docker-compose.gh.local.yml b/hosting/docker-compose/oss/docker-compose.gh.local.yml index 7ea9ce66690..9bea93b297b 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.local.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.local.yml @@ -328,6 +328,9 @@ services: # Worst-case turn duration. Keep AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS (API) above it # plus 60s of skew, or every dispatch rebuilds the environment cold. AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS:-} + # Defaults: 30 min without progress; 30 min for one tool call. + AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS:-} + AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS: ${AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS:-} PI_CODING_AGENT_DIR: ${PI_CODING_AGENT_DIR:-/pi-agent} AGENTA_RUNNER_DAYTONA_API_KEY: ${AGENTA_RUNNER_DAYTONA_API_KEY:-} AGENTA_RUNNER_DAYTONA_API_URL: ${AGENTA_RUNNER_DAYTONA_API_URL:-} diff --git a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml index 756d780c0f7..c07ddf88a43 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml @@ -355,6 +355,9 @@ services: # Worst-case turn duration. Keep AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS (API) above it # plus 60s of skew, or every dispatch rebuilds the environment cold. AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS:-} + # Defaults: 30 min without progress; 30 min for one tool call. + AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS:-} + AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS: ${AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS:-} AGENTA_RUNNER_DAYTONA_API_KEY: ${AGENTA_RUNNER_DAYTONA_API_KEY:-} AGENTA_RUNNER_DAYTONA_API_URL: ${AGENTA_RUNNER_DAYTONA_API_URL:-} AGENTA_RUNNER_DAYTONA_TARGET: ${AGENTA_RUNNER_DAYTONA_TARGET:-} diff --git a/hosting/docker-compose/oss/docker-compose.gh.yml b/hosting/docker-compose/oss/docker-compose.gh.yml index e95002913ed..897730d30d4 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.yml @@ -352,6 +352,9 @@ services: # Worst-case turn duration. Keep AGENTA_MOUNTS_CREDENTIALS_TTL_SECONDS (API) above it # plus 60s of skew, or every dispatch rebuilds the environment cold. AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_TOTAL_TIMEOUT_MS:-} + # Defaults: 30 min without progress; 30 min for one tool call. + AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS: ${AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS:-} + AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS: ${AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS:-} PI_CODING_AGENT_DIR: ${PI_CODING_AGENT_DIR:-/pi-agent} AGENTA_RUNNER_DAYTONA_API_KEY: ${AGENTA_RUNNER_DAYTONA_API_KEY:-} AGENTA_RUNNER_DAYTONA_API_URL: ${AGENTA_RUNNER_DAYTONA_API_URL:-} diff --git a/hosting/docker-compose/oss/env.oss.dev.example b/hosting/docker-compose/oss/env.oss.dev.example index 6f65ca6c913..d863d07bcde 100644 --- a/hosting/docker-compose/oss/env.oss.dev.example +++ b/hosting/docker-compose/oss/env.oss.dev.example @@ -142,6 +142,9 @@ NEXT_PUBLIC_AGENT_FILE_UPLOADS=true # Smart truncation preserves the structure of a record whose body exceeds the API size # cap (higher-fidelity reconstruction). Still opt-in, default off. # AGENTA_RECORDS_SMART_TRUNCATION=true +# Durable Stop is enabled by default; set false to use legacy cancellation. +AGENTA_SESSIONS_DURABLE_STOP=true +# AGENTA_SESSIONS_LATE_OUTPUT=quarantine # --- Attachment limits (files attached to an agent chat turn) --- # Per-file caps in bytes, by kind: 10 MB, except audio at 15 MB. Read by the api. diff --git a/hosting/kubernetes/helm/Chart.yaml b/hosting/kubernetes/helm/Chart.yaml index 577e56d9158..faaafe67830 100644 --- a/hosting/kubernetes/helm/Chart.yaml +++ b/hosting/kubernetes/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: agenta description: A Helm chart for deploying Agenta (OSS or EE) on Kubernetes type: application -version: 0.114.8 -appVersion: "v0.114.8" +version: 0.115.0 +appVersion: "v0.115.0" keywords: - agenta - llm diff --git a/hosting/railway/oss/scripts/configure.sh b/hosting/railway/oss/scripts/configure.sh index 65abd3cba53..4752b371520 100755 --- a/hosting/railway/oss/scripts/configure.sh +++ b/hosting/railway/oss/scripts/configure.sh @@ -444,6 +444,8 @@ main() { POSTGRES_URI_CORE="$pg_async_core" \ POSTGRES_URI_TRACING="$pg_async_tracing" \ POSTGRES_URI_SUPERTOKENS="$pg_sync_supertokens" \ + AGENTA_RUNNER_INTERNAL_URL="$agent_runner_url" \ + AGENTA_RUNNER_TOKEN="$AGENTA_RUNNER_TOKEN" \ AGENTA_STORE_ENDPOINT_URL="$seaweedfs_endpoint_url" \ AGENTA_STORE_ACCESS_KEY="$AGENTA_STORE_ACCESS_KEY" \ AGENTA_STORE_SECRET_KEY="$AGENTA_STORE_SECRET_KEY" \ diff --git a/hosting/railway/oss/template/template.json b/hosting/railway/oss/template/template.json index 76993d179df..c3be8a90161 100644 --- a/hosting/railway/oss/template/template.json +++ b/hosting/railway/oss/template/template.json @@ -181,6 +181,10 @@ "POSTGRES_URI_CORE": "postgresql+asyncpg://${{Postgres.POSTGRES_USER}}:${{Postgres.POSTGRES_PASSWORD}}@${{Postgres.RAILWAY_PRIVATE_DOMAIN}}:${{Postgres.PGPORT}}/agenta_oss_core", "POSTGRES_URI_TRACING": "postgresql+asyncpg://${{Postgres.POSTGRES_USER}}:${{Postgres.POSTGRES_PASSWORD}}@${{Postgres.RAILWAY_PRIVATE_DOMAIN}}:${{Postgres.PGPORT}}/agenta_oss_tracing", "POSTGRES_URI_SUPERTOKENS": "postgresql://${{Postgres.POSTGRES_USER}}:${{Postgres.POSTGRES_PASSWORD}}@${{Postgres.RAILWAY_PRIVATE_DOMAIN}}:${{Postgres.PGPORT}}/agenta_oss_supertokens", + "AGENTA_RUNNER_INTERNAL_URL": "http://${{runner.RAILWAY_PRIVATE_DOMAIN}}:8765", + "AGENTA_RUNNER_TOKEN": { + "secret": "AGENTA_RUNNER_TOKEN" + }, "AGENTA_STORE_ENDPOINT_URL": "http://${{seaweedfs.RAILWAY_PRIVATE_DOMAIN}}:8333", "AGENTA_STORE_ACCESS_KEY": { "secret": "AGENTA_STORE_ACCESS_KEY" diff --git a/sdks/python/agenta/sdk/agents/adapters/vercel/stream.py b/sdks/python/agenta/sdk/agents/adapters/vercel/stream.py index 5722cffc681..832a95fb655 100644 --- a/sdks/python/agenta/sdk/agents/adapters/vercel/stream.py +++ b/sdks/python/agenta/sdk/agents/adapters/vercel/stream.py @@ -358,6 +358,25 @@ async def _agent_run_to_vercel_parts_impl( failure_code=_runner_failure_code(data.get("code")), ): yield part + elif etype == "turn": + # The runner's admitted execution id, forwarded onto the MESSAGE METADATA so the + # client reads it as `message.metadata.turnId`, beside the `sessionId` the `start` + # frame already carries and the `traceId`/`usage` the `finish` frame adds. + # + # It cannot ride the `start` frame itself: that frame is emitted before the runner + # replies at all (see the `start` yield above), so a runner-minted id does not + # exist yet. A `message-metadata` chunk is the same channel one frame later, and + # the AI SDK MERGES metadata rather than replacing it (`mergeObjects`), so the id + # survives the `finish` frame's own metadata to the end of the turn. + # + # A client keeps it to name the execution it means to Stop + # (`expected_execution_id`) instead of cancelling "whatever runs now". + turn_id = data.get("turnId") + if isinstance(turn_id, str) and turn_id: + yield { + "type": "message-metadata", + "messageMetadata": {"turnId": turn_id}, + } elif etype == "done": # Last non-null stop reason wins; see the routing-layer twin's `done` note. reason = data.get("stopReason") @@ -641,6 +660,25 @@ async def _agent_stream_to_vercel_stream_impl( failure_code=_runner_failure_code(data.get("code")), ): yield part + elif etype == "turn": + # The runner's admitted execution id, forwarded onto the MESSAGE METADATA so the + # client reads it as `message.metadata.turnId`, beside the `sessionId` the `start` + # frame already carries and the `traceId`/`usage` the `finish` frame adds. + # + # It cannot ride the `start` frame itself: that frame is emitted before the runner + # replies at all (see the `start` yield above), so a runner-minted id does not + # exist yet. A `message-metadata` chunk is the same channel one frame later, and + # the AI SDK MERGES metadata rather than replacing it (`mergeObjects`), so the id + # survives the `finish` frame's own metadata to the end of the turn. + # + # A client keeps it to name the execution it means to Stop + # (`expected_execution_id`) instead of cancelling "whatever runs now". + turn_id = data.get("turnId") + if isinstance(turn_id, str) and turn_id: + yield { + "type": "message-metadata", + "messageMetadata": {"turnId": turn_id}, + } elif etype == "done": # Prefer the LAST non-null stop reason. The handler appends a corrective # terminal `done` after the runner's `done` when the authoritative result diff --git a/sdks/python/oss/tests/pytest/unit/agents/adapters/test_vercel_stream_conformance.py b/sdks/python/oss/tests/pytest/unit/agents/adapters/test_vercel_stream_conformance.py index 474f3f4598f..aaa3cde0e79 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/adapters/test_vercel_stream_conformance.py +++ b/sdks/python/oss/tests/pytest/unit/agents/adapters/test_vercel_stream_conformance.py @@ -423,3 +423,109 @@ def test_vendored_version_matches_package_pin() -> None: # CI-grep-able tripwire: bump this const (and re-audit the shape above) whenever # web/oss/package.json's "ai" pin changes. assert _AI_PACKAGE_VERSION == "6.0.0-beta.150" + + +# --------------------------------------------------------------------------- +# The turn id pass-through. +# +# The runner mints the turn id per execution and, until this, told no one. The `start` frame is +# built and emitted before the runner replies at all, so it CANNOT carry a runner-minted id — +# which is why `expected_execution_id` on the public Cancel had no first-party caller able to fill +# it. The runner now emits a `turn` event as its first frame and the egress forwards it unchanged +# as `data-agent-turn`, the earliest part that can carry it. +# --------------------------------------------------------------------------- + +_TURN_ID = "d3b4a1c2-0000-4000-8000-abcdefabcdef" + +# The runner's AgentEvent is FLAT (`{type, turnId}`, like `{type, message, code}` for an error), +# and each path wraps it differently. The live handler yields `{"type", "data"}` where `data` is +# the whole flat runner event; `AgentStream` (the dev twin) hands the flat record through +# `Event.from_wire`, which also sets `data` to the whole record. Both fixtures below are the real +# shapes, not a convenient one — a fixture that reshapes the event tests nothing about the wire. +_TURN_EVENTS_LIVE: List[Dict[str, Any]] = [ + {"type": "turn", "data": {"type": "turn", "turnId": _TURN_ID}}, + {"type": "message", "data": {"text": "hello"}}, + {"type": "done", "data": {"stopReason": "stop"}}, +] +_TURN_EVENTS_RUN: List[Dict[str, Any]] = [ + {"type": "turn", "turnId": _TURN_ID}, + {"type": "message", "text": "hello"}, + {"type": "done", "stopReason": "stop"}, +] + +_TURN_METADATA = {"type": "message-metadata", "messageMetadata": {"turnId": _TURN_ID}} + + +@pytest.mark.asyncio +async def test_live_projection_puts_the_turn_id_on_message_metadata() -> None: + parts = [ + part + async for part in agent_stream_to_vercel_stream( + _records(_TURN_EVENTS_LIVE), trace_id="t1" + ) + ] + for part in parts: + assert_conforms(part) + + metadata_parts = [p for p in parts if p["type"] == "message-metadata"] + assert metadata_parts == [_TURN_METADATA], ( + "the egress must forward the runner's id verbatim, exactly once, as message metadata" + ) + + # It must land before any content, so a client that Stops early already holds the id. + turn_index = next(i for i, p in enumerate(parts) if p["type"] == "message-metadata") + first_text = next( + (i for i, p in enumerate(parts) if p["type"].startswith("text-")), None + ) + assert first_text is None or turn_index < first_text + + +@pytest.mark.asyncio +async def test_the_finish_frames_metadata_does_not_displace_the_turn_id() -> None: + """The whole reason `message-metadata` is a safe carrier. + + The AI SDK merges metadata rather than replacing it (`mergeObjects` in ai@6), so the + `finish` frame's own `messageMetadata` (traceId, usage) lands BESIDE the turn id rather than + over it. If that ever changed, a client would lose the id exactly when a late Stop needs it, + so pin that the two carry disjoint keys and the turn id is written first. + """ + parts = [ + part + async for part in agent_stream_to_vercel_stream( + _records(_TURN_EVENTS_LIVE), trace_id="t1" + ) + ] + turn_index = next(i for i, p in enumerate(parts) if p["type"] == "message-metadata") + finish = next(p for p in parts if p["type"] == "finish") + finish_index = parts.index(finish) + + assert turn_index < finish_index + assert "turnId" not in (finish.get("messageMetadata") or {}), ( + "the finish frame must not restate the turn id; it merges beside it" + ) + + +@pytest.mark.asyncio +async def test_dev_twin_projection_puts_the_turn_id_on_message_metadata() -> None: + run = _run_with(_TURN_EVENTS_RUN, result={"output": "hello"}) + parts = [part async for part in agent_run_to_vercel_parts(run)] + for part in parts: + assert_conforms(part) + assert _TURN_METADATA in parts + + +@pytest.mark.asyncio +async def test_a_turn_event_with_no_usable_id_emits_nothing() -> None: + # An older runner, or a malformed frame, must not put an empty id on the stream: a client + # would send it as `expected_execution_id` and cancel nothing, or worse, read it as "no + # guard". Dropping it leaves the client in the honest "I do not know the id" state. + for bad in ({}, {"turnId": None}, {"turnId": ""}, {"turnId": 7}): + parts = [ + part + async for part in agent_stream_to_vercel_stream( + _records([{"type": "turn", "data": bad}]), trace_id="t1" + ) + ] + assert not [p for p in parts if p["type"] == "message-metadata"], ( + f"a turn event with data={bad!r} must emit no metadata frame" + ) diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 12c5a799e81..a763a300aca 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "agenta" -version = "0.114.8" +version = "0.115.0" description = "Agenta is the open-source workspace for your agents. Build agents through chat, improve them with feedback, and share them with your team." readme = "README.md" requires-python = ">=3.11,<3.14" diff --git a/sdks/python/uv.lock b/sdks/python/uv.lock index a5f124c7300..4aed276d248 100644 --- a/sdks/python/uv.lock +++ b/sdks/python/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.11, <3.14" [[package]] name = "agenta" -version = "0.114.8" +version = "0.115.0" source = { editable = "." } dependencies = [ { name = "agenta-client" }, @@ -85,7 +85,7 @@ dev = [ [[package]] name = "agenta-client" -version = "0.114.8" +version = "0.115.0" source = { editable = "../../clients/python" } dependencies = [ { name = "httpx" }, diff --git a/services/pyproject.toml b/services/pyproject.toml index a8945c09eb5..22359eb55ad 100644 --- a/services/pyproject.toml +++ b/services/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "services" -version = "0.114.8" +version = "0.115.0" description = "Agenta Services (Chat & Completion)" requires-python = ">=3.11,<3.14" authors = [ diff --git a/services/runner/patches/sandbox-agent@0.4.2.patch b/services/runner/patches/sandbox-agent@0.4.2.patch index 610ead15d1e..b69a898d7f0 100644 --- a/services/runner/patches/sandbox-agent@0.4.2.patch +++ b/services/runner/patches/sandbox-agent@0.4.2.patch @@ -1,5 +1,5 @@ diff --git a/dist/chunk-TVCDKGSM.js b/dist/chunk-TVCDKGSM.js -index 29a0a22210d39ae9d886c0ccd7059cd9af0e26f0..7d4585271c700e6222d24bd391604e7007e81b99 100644 +index 29a0a22210d39ae9d886c0ccd7059cd9af0e26f0..60becacde92447d50c0d29609953b93be245d40d 100644 --- a/dist/chunk-TVCDKGSM.js +++ b/dist/chunk-TVCDKGSM.js @@ -738,6 +738,7 @@ var LiveAcpConnection = class _LiveAcpConnection { @@ -123,7 +123,18 @@ index 29a0a22210d39ae9d886c0ccd7059cd9af0e26f0..7d4585271c700e6222d24bd391604e70 const updated = { ...existing, agentSessionId: recreated.sessionId, -@@ -2504,6 +2545,25 @@ function normalizeSessionInit(value, cwdShorthand, providerDefaultCwd) { +@@ -1363,6 +1404,10 @@ var SandboxAgent = class _SandboxAgent { + } + return this.createSession(request); + } ++ async cancelSession(id) { ++ this.cancelPendingPermissionsForSession(id); ++ await this.sendSessionMethodInternal(id, SESSION_CANCEL_METHOD, {}, {}, true); ++ } + async destroySession(id) { + this.cancelPendingPermissionsForSession(id); + try { +@@ -2504,6 +2549,25 @@ function normalizeSessionInit(value, cwdShorthand, providerDefaultCwd) { mcpServers: value.mcpServers ?? [] }; } @@ -149,6 +160,18 @@ index 29a0a22210d39ae9d886c0ccd7059cd9af0e26f0..7d4585271c700e6222d24bd391604e70 function mapSessionParams(params, agentSessionId) { return { ...params, +diff --git a/dist/index.d.ts b/dist/index.d.ts +index e67d588032a085d28adc82252199e389722b86d2..c75a8efa3ad663abc4497e6853c0b97835549cf8 100644 +--- a/dist/index.d.ts ++++ b/dist/index.d.ts +@@ -3174,6 +3174,7 @@ declare class SandboxAgent { + createSession(request: SessionCreateRequest): Promise; + resumeSession(id: string): Promise; + resumeOrCreateSession(request: SessionResumeOrCreateRequest): Promise; ++ cancelSession(id: string): Promise; + destroySession(id: string): Promise; + setSessionMode(sessionId: string, modeId: string): Promise<{ + session: Session; diff --git a/dist/providers/local.js b/dist/providers/local.js index 3e68d70c340ded6b2f99cf3142e951b804a4a8a2..103851397d0f575f103644002a94ab46306034d5 100644 --- a/dist/providers/local.js diff --git a/services/runner/pnpm-lock.yaml b/services/runner/pnpm-lock.yaml index a6ddf79637c..9e58cdd64b1 100644 --- a/services/runner/pnpm-lock.yaml +++ b/services/runner/pnpm-lock.yaml @@ -23,7 +23,7 @@ patchedDependencies: hash: e30d9db3a9981a7f844a83795d7ceef6d86919eede6e2adf8a9bf0024d5ff49c path: patches/pi-acp@0.0.29.patch sandbox-agent@0.4.2: - hash: ade0985e7ab79fab885a4cd818c0790d5cf319730931bd0a049775c7fbdeb7a4 + hash: 91ccdae91dc68390329197a105ecd8cf394f680dd3be2a26731ebe1c2bd58c3f path: patches/sandbox-agent@0.4.2.patch importers: @@ -74,7 +74,7 @@ importers: version: 0.0.29(patch_hash=e30d9db3a9981a7f844a83795d7ceef6d86919eede6e2adf8a9bf0024d5ff49c) sandbox-agent: specifier: 0.4.2 - version: 0.4.2(patch_hash=ade0985e7ab79fab885a4cd818c0790d5cf319730931bd0a049775c7fbdeb7a4)(@daytona/sdk@0.198.0(ws@8.21.0))(zod@4.4.3) + version: 0.4.2(patch_hash=91ccdae91dc68390329197a105ecd8cf394f680dd3be2a26731ebe1c2bd58c3f)(@daytona/sdk@0.198.0(ws@8.21.0))(zod@4.4.3) undici: specifier: 8.9.0 version: 8.9.0 @@ -4985,7 +4985,7 @@ snapshots: safer-buffer@2.1.2: {} - sandbox-agent@0.4.2(patch_hash=ade0985e7ab79fab885a4cd818c0790d5cf319730931bd0a049775c7fbdeb7a4)(@daytona/sdk@0.198.0(ws@8.21.0))(zod@4.4.3): + sandbox-agent@0.4.2(patch_hash=91ccdae91dc68390329197a105ecd8cf394f680dd3be2a26731ebe1c2bd58c3f)(@daytona/sdk@0.198.0(ws@8.21.0))(zod@4.4.3): dependencies: '@sandbox-agent/cli-shared': 0.4.2 acp-http-client: 0.4.2(patch_hash=a673c410af2021d9bb5f05c899522b66e6bcbe67134a92f506f0aef23fcf090d)(zod@4.4.3) diff --git a/services/runner/src/engines/sandbox_agent/acp-fetch.ts b/services/runner/src/engines/sandbox_agent/acp-fetch.ts index dcee85d9b4d..e3df15b4b7c 100644 --- a/services/runner/src/engines/sandbox_agent/acp-fetch.ts +++ b/services/runner/src/engines/sandbox_agent/acp-fetch.ts @@ -1,5 +1,7 @@ import { Agent, fetch as undiciFetch } from "undici"; +import { sandboxGoneReason } from "./sandbox-gone.ts"; + /** * HITL pauses keep the ACP HTTP connection open for human-timescale delays: when a tool call needs * approval, the runner holds the in-flight `prompt` request while it waits for the human to @@ -52,12 +54,50 @@ export function createAcpDispatcher(): Agent { }); } +export interface AcpFetchOptions { + /** + * Called when a response proves the sandbox is gone (see `sandbox-gone.ts`). + * + * This fetch is the socket the turn runs on, so it sees a deleted remote sandbox seconds before + * any poll can. It cannot end the turn itself: the ACP transport swallows the failure (it errors + * its readable, and the protocol SDK's read loop never rejects the pending `session/prompt`), so + * the promise the turn awaits stays pending forever. Reporting it here is what lets the liveness + * probe end the turn at once instead of one probe interval later, or never. + */ + onSandboxGone?: (reason: string) => void; +} + +/** + * Wrap a `fetch` so a response that names the sandbox as gone is reported once. + * + * The response is passed through untouched, body included: this wrapper reads only the status and + * the headers, because draining the body here would break every caller. With no + * `onSandboxGone` it is the identity, so nothing is inspected on a path that cannot act on it. + */ +export function withSandboxGoneReport( + inner: typeof fetch, + options: AcpFetchOptions = {}, +): typeof fetch { + const report = options.onSandboxGone; + if (!report) return inner; + return (async (input: any, init?: any) => { + const response = await inner(input, init); + const reason = sandboxGoneReason(response); + if (reason) report(reason); + return response; + }) as unknown as typeof fetch; +} + /** * A `fetch` for the ACP HTTP client backed by {@link createAcpDispatcher}. We use undici's own * `fetch` so the `dispatcher` option is honored regardless of how the global dispatcher is set. * The `sandbox-agent` SDK accepts a custom `fetch`; we hand it this one on every path. */ -export function createAcpFetch(dispatcher: Agent = createAcpDispatcher()): typeof fetch { - return ((input: any, init?: any) => +export function createAcpFetch( + dispatcher: Agent = createAcpDispatcher(), + options: AcpFetchOptions = {}, +): typeof fetch { + const bound = ((input: any, init?: any) => undiciFetch(input, { ...init, dispatcher })) as unknown as typeof fetch; + return withSandboxGoneReport(bound, options); } diff --git a/services/runner/src/engines/sandbox_agent/agent-mount.ts b/services/runner/src/engines/sandbox_agent/agent-mount.ts index 41ca4767ec8..7a63378cacb 100644 --- a/services/runner/src/engines/sandbox_agent/agent-mount.ts +++ b/services/runner/src/engines/sandbox_agent/agent-mount.ts @@ -16,6 +16,7 @@ import { type SandboxExec, type SignMountDeps, } from "./mount.ts"; +import { throwIfAcquireAborted } from "../../environment/acquire-abort.ts"; export const AGENT_MOUNT_ENV_VAR = "AGENTA_AGENT_MOUNT_DIR"; export const AGENT_README_NAME = "README.md"; @@ -47,6 +48,10 @@ export async function signAgentMountCredentials( ): Promise { const log = deps.log ?? defaultLog; const doFetch = deps.fetchImpl ?? fetch; + const timeoutSignal = AbortSignal.timeout(10_000); + const signal = deps.signal + ? AbortSignal.any([deps.signal, timeoutSignal]) + : timeoutSignal; const url = `${deps.apiBase}/mounts/agents/sign?artifact_id=${encodeURIComponent(artifactId)}&name=${encodeURIComponent(name)}`; try { const res = await doFetch(url, { @@ -57,7 +62,7 @@ export async function signAgentMountCredentials( }, // Bound the sign so a hung endpoint fails open (null mount) instead of // stalling environment acquisition on the agent mount forever. - signal: AbortSignal.timeout(10_000), + signal, }); if (!res.ok) { log( @@ -98,6 +103,7 @@ export async function signAgentMountCredentials( : undefined, }; } catch (err) { + throwIfAcquireAborted(deps.signal); log( `sign failed artifact=${artifactId}: ${String(err instanceof Error ? err.message : err).slice(0, 160)}`, ); diff --git a/services/runner/src/engines/sandbox_agent/cancel-turn.ts b/services/runner/src/engines/sandbox_agent/cancel-turn.ts new file mode 100644 index 00000000000..72e773adb33 --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/cancel-turn.ts @@ -0,0 +1,155 @@ +/** + * Cancel the harness turn so the sandbox can be PARKED instead of deleted. + * + * WHAT THIS FIXES. A user Stop aborts the run signal. The turn then ends with + * `stopReason: "cancelled"`, and `shouldPark` used to answer `false` for every aborted run, so + * the sandbox was deleted and the next message paid a cold start. The abort alone never told the + * harness anything: it only made the runner stop waiting. The harness kept its prompt open, + * possibly with a tool still running, and the only thing that ever stopped it was the teardown + * that was already deleting the sandbox. + * + * WHAT THIS DOES INSTEAD. Send the ACP `session/cancel` notification for the live session, then + * wait a bounded time for the harness to answer the open `session/prompt`. ACP requires the agent + * to end that prompt with `stopReason: "cancelled"` after a cancel, so a settled prompt promise is + * the harness saying "I am idle again". Only a settled cancel may park. A cancel that cannot be + * sent, or that the harness never answers in time, leaves the environment in an unknown state, and + * unknown means delete. + * + * WHY THE CLIENT NEEDS A PATCH. `sandbox-agent`'s `SandboxAgent` refuses a manual `session/cancel` + * ("Manual session/cancel calls are not allowed. Use destroySession(sessionId) instead."). The + * guard is in the TypeScript client only; the daemon inside the sandbox proxies ACP and holds no + * such rule. The existing pnpm patch adds `cancelSession(id)`, which sends the same managed cancel + * `destroySession` sends but does NOT mark the session record destroyed. The `?.` below keeps this + * module honest against an unpatched client: no method, no clean cancel, no park. + * + * WHY IT DOES NOT ABORT THE ENVIRONMENT'S MCP CONTROLLER. `env.mcpAbort` belongs to the + * ENVIRONMENT, not the turn. Aborting it kills the tool-MCP server for every later turn, which is + * exactly what a parked environment must keep. The approval-park path already skips it for the + * same reason (see `run-turn.ts`, the `approvalParkMode` early return). The turn's own tool relay + * is stopped separately, and a teardown that does happen still aborts the controller through + * `teardownRuntimeInFlight`. + */ + +import { envTimerMs } from "../../env.ts"; + +export const CANCEL_SETTLE_TIMEOUT_ENV = + "AGENTA_RUNNER_HARNESS_CANCEL_SETTLE_MS"; + +/** + * How long to wait for the harness to answer the cancelled prompt. + * + * Ten seconds is a starting value, not a measured one. It has to cover the adapter aborting the + * tool it is running and writing its partial turn, and it has to stay well under the user's + * patience for a second message. Raise it only with a measurement that shows a harness needing + * more; every extra second is a second the Stop looks unfinished. + */ +export const DEFAULT_CANCEL_SETTLE_MS = 10_000; + +export interface CancelHarnessTurnInput { + /** The live sandbox client. `cancelSession` is absent on an unpatched `sandbox-agent`. */ + sandbox: { cancelSession?: (id: string) => Promise } | undefined; + /** The harness session id to cancel. */ + sessionId: string | undefined; + /** The still-open `session/prompt` promise for this turn. */ + promptPromise: Promise | undefined; + timeoutMs?: number; + log: (message: string) => void; + /** Test seam. Defaults to a real timer. */ + wait?: (ms: number) => Promise; + now?: () => number; +} + +export interface CancelHarnessTurnResult { + /** True only when the cancel was sent AND the harness answered the prompt in time. */ + settled: boolean; + /** True when the cancel notification left the runner, whatever the harness did next. */ + requested: boolean; + /** Milliseconds from sending the cancel to the harness answering, when it answered. */ + elapsedMs: number; +} + +export function resolveCancelSettleMs(): number { + return envTimerMs(CANCEL_SETTLE_TIMEOUT_ENV, DEFAULT_CANCEL_SETTLE_MS, { + min: 1, + }); +} + +/** + * Ask the harness to stop the current prompt and wait for it to say it did. + * + * Never throws. Every failure answers `settled: false`, which the caller reads as "destroy". + */ +export async function cancelHarnessTurn( + input: CancelHarnessTurnInput, +): Promise { + const unsettled = { settled: false, requested: false, elapsedMs: 0 }; + const cancelSession = input.sandbox?.cancelSession; + if (!cancelSession || !input.sessionId || !input.promptPromise) { + input.log( + "stage=harness_cancel sent=false reason=" + + (!cancelSession + ? "client-has-no-cancelSession" + : !input.sessionId + ? "no-session" + : "no-open-prompt"), + ); + return unsettled; + } + + const now = input.now ?? (() => Date.now()); + const startedAt = now(); + const timeoutMs = input.timeoutMs ?? resolveCancelSettleMs(); + const wait = + input.wait ?? + ((ms: number) => + new Promise((resolve) => { + const handle = setTimeout(resolve, ms); + handle.unref?.(); + })); + const TIMED_OUT = Symbol("cancel-settle-timeout"); + + try { + const requested = await Promise.race([ + cancelSession.call(input.sandbox, input.sessionId).then(() => true), + wait(timeoutMs).then(() => TIMED_OUT), + ]); + if (requested === TIMED_OUT) { + input.log( + `stage=harness_cancel sent=false reason=request-timeout budget_ms=${timeoutMs}`, + ); + return unsettled; + } + } catch (error) { + input.log( + "stage=harness_cancel sent=false error=" + + (error instanceof Error ? error.message : String(error)).slice(0, 160), + ); + return unsettled; + } + + // A RESOLVED prompt is the harness reporting its own `stopReason`. A REJECTED one means the + // prompt died on the transport instead, which says nothing about whether the harness stopped, + // so it counts as unsettled and the environment is destroyed. + const settledOk = await Promise.race([ + input.promptPromise.then( + () => true, + () => false, + ), + wait(timeoutMs).then(() => TIMED_OUT), + ]); + const elapsedMs = now() - startedAt; + + if (settledOk === true) { + input.log( + `stage=harness_cancel sent=true settled=true elapsed_ms=${elapsedMs}`, + ); + return { settled: true, requested: true, elapsedMs }; + } + input.log( + `stage=harness_cancel sent=true settled=false elapsed_ms=${elapsedMs} ` + + (settledOk === TIMED_OUT + ? `reason=timeout budget_ms=${timeoutMs}` + : "reason=prompt-rejected"), + ); + return { settled: false, requested: true, elapsedMs }; +} diff --git a/services/runner/src/engines/sandbox_agent/credential-preflight.ts b/services/runner/src/engines/sandbox_agent/credential-preflight.ts index 3cf866b0e37..6f9facba853 100644 --- a/services/runner/src/engines/sandbox_agent/credential-preflight.ts +++ b/services/runner/src/engines/sandbox_agent/credential-preflight.ts @@ -108,6 +108,11 @@ * runner call that the provider's own auth endpoint answered with 200. */ +import { + throwIfAcquireAborted, + waitForAcquire, +} from "../../environment/acquire-abort.ts"; + /** * Does this acquire deliver the run's MODEL credential as a Daytona Secret? * @@ -508,6 +513,8 @@ function createFetchControlProbe(fetchImpl: typeof fetch): ControlProbe { export async function awaitCredentialSubstitution( input: CredentialPreflightInput, ): Promise { + const signal = input.signal; + throwIfAcquireAborted(signal); const now = input.now ?? Date.now; const sleep = input.sleep ?? @@ -600,15 +607,23 @@ export async function awaitCredentialSubstitution( const script = sandboxProbeScript(shape, input.apiKeyVar, probeSeconds); let stdout: string | undefined; try { - const result = await input.sandbox.runProcess({ - command: "sh", - args: ["-c", script], - // Capped by the same deadline. The exec channel gets its usual slack over curl's - // own ceiling only while the grace can pay for it. - timeoutMs: Math.max(1, Math.min((probeSeconds + 4) * 1000, leftMs)), - }); + const result = await waitForAcquire( + () => + input.sandbox.runProcess({ + command: "sh", + args: ["-c", script], + // Capped by the same deadline. The exec channel gets its usual slack over curl's + // own ceiling only while the grace can pay for it. + timeoutMs: Math.max( + 1, + Math.min((probeSeconds + 4) * 1000, leftMs), + ), + }), + signal, + ); stdout = result?.stdout; } catch (error) { + throwIfAcquireAborted(signal); // The exec channel itself failed (sandbox tearing down, daemon hiccup): fail open. log( `[credential-preflight] probe errored, proceeding: ${String( @@ -701,7 +716,10 @@ export async function awaitCredentialSubstitution( return "stuck"; } log(`[credential-preflight] ${evidence.probeLine}`); - await sleep(Math.min(pollMs, remainingMs())); + await waitForAcquire( + () => sleep(Math.min(pollMs, remainingMs())), + signal, + ); } } finally { // Nothing reads the runner's call after this point, on any exit path. diff --git a/services/runner/src/engines/sandbox_agent/daytona.ts b/services/runner/src/engines/sandbox_agent/daytona.ts index 973f5d3f915..fccc7f9e80d 100644 --- a/services/runner/src/engines/sandbox_agent/daytona.ts +++ b/services/runner/src/engines/sandbox_agent/daytona.ts @@ -1,6 +1,10 @@ import { join } from "node:path"; -import { createAcpFetch } from "./acp-fetch.ts"; +import { + createAcpFetch, + withSandboxGoneReport, + type AcpFetchOptions, +} from "./acp-fetch.ts"; import { resolvePiToolSpecsDelivery, uploadPiExtensionToSandbox, @@ -293,11 +297,17 @@ export async function prepareDaytonaPiAssets({ * required" / 502. The sandbox-agent SDK accepts a custom fetch, so we hand it this one. * * It layers on {@link createAcpFetch} (the long-timeout ACP dispatcher) so a paused HITL turn - * over Daytona is not reaped by undici's default `headersTimeout` either. + * over Daytona is not reaped by undici's default `headersTimeout` either, and so `options` (the + * sandbox-gone report) reaches the one place that inspects every ACP response. Daytona is the + * provider whose proxy answers for a deleted sandbox, so this is the path that needs it most. */ export function createCookieFetch( - inner: typeof fetch = createAcpFetch(), + inner?: typeof fetch, + options: AcpFetchOptions = {}, ): typeof fetch { + const base = inner + ? withSandboxGoneReport(inner, options) + : createAcpFetch(undefined, options); const jar = new Map>(); // host -> (name -> "name=value") return async (input: any, init?: any) => { const url = new URL(typeof input === "string" ? input : input.url); @@ -312,7 +322,7 @@ export function createCookieFetch( if (existing) merged.unshift(existing); headers.set("cookie", merged.join("; ")); } - const response = await inner(input, { ...init, headers }); + const response = await base(input, { ...init, headers }); const setCookies = typeof (response.headers as any).getSetCookie === "function" ? (response.headers as any).getSetCookie() diff --git a/services/runner/src/engines/sandbox_agent/engine.ts b/services/runner/src/engines/sandbox_agent/engine.ts index 671add8328d..569a73fa7c8 100644 --- a/services/runner/src/engines/sandbox_agent/engine.ts +++ b/services/runner/src/engines/sandbox_agent/engine.ts @@ -3,6 +3,7 @@ import { type AgentRunResult, type EmitEvent, } from "../../protocol.ts"; +import { isUserStopAbort } from "../../sessions/stop-signal.ts"; import { acquireEnvironment } from "./environment.ts"; import { runCredential } from "./runtime-policy.ts"; import { loadDurableDecisions } from "../../sessions/interactions.ts"; @@ -13,18 +14,53 @@ import { } from "./runtime-contracts.ts"; /** - * Whether a completed turn's environment may be parked: never on abort, client disconnect, - * pause, or failure. Session-owned streams survive disconnect WITHOUT aborting the run signal - * (server policy), so the disconnect check needs the separate `clientGone` flag. A wedged - * sandbox that failed its turn must be destroyed, not reconnected on the next one. + * Whether a completed turn's environment may be parked: never on client disconnect, pause, or + * failure. Session-owned streams survive disconnect WITHOUT aborting the run signal (server + * policy), so the disconnect check needs the separate `clientGone` flag. A wedged sandbox that + * failed its turn must be destroyed, not reconnected on the next one. + * + * A USER STOP IS THE ONE ABORT THAT MAY PARK. Stop and Delete are different operations: Stop + * keeps the session, the sandbox, and the harness session resumable. Three things must all be + * true, and each answers a different question: + * + * - `isUserStopAbort(signal)` — WAS this abort a cooperative Stop? The signal is labelled at + * the one call site that means it (`server.ts`, the heartbeat interrupt). Reading + * `signal.aborted` alone cannot answer this, and inferring it from the stop reason would let + * any future `controller.abort()` park a sandbox nobody checked. See `sessions/stop-signal.ts`. + * - `result.stopReason === "cancelled"` — did the TURN actually end as a cancel? + * - `result.cancelSettled` — did the HARNESS confirm it stopped? See `cancel-turn.ts`. + * + * Every other abort leaves the environment in an unknown state and still destroys. + * + * A SETTLED USER STOP IS CHECKED BEFORE `clientGone`, AND THAT ORDER IS THE WHOLE POINT. + * `clientGone` used to be read first, which read well and broke the product on every real Stop. + * The browser's Stop button aborts its own chat stream in the SAME tick it sends the durable + * cancel command (`web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts`, + * `handleStop`), so the disconnect and the labelled abort always arrive together. With the + * disconnect read first, every Stop fell into the destroy branch: the sandbox was deleted, the + * native harness session went with it, and the next message replayed cold. Observed on the + * increment-6 stack on 2026-09-04, three Stops, three evictions, no warm park. + * + * The disconnect rule loses nothing it was written for. It exists so an UNATTENDED session is + * never kept warm on a guess, and a Stop is not a guess: it is an authenticated command the API + * recorded durably, from a user who is still on the page and about to type. Every other + * disconnect — mid-turn tab close, a dropped connection, a failed turn — still destroys, and the + * parked entry still expires on its own TTL. */ export function shouldPark( result: AgentRunResult, signal: AbortSignal | undefined, clientGone: (() => boolean) | undefined, ): boolean { - if (signal?.aborted) return false; // aborted run: destroy, do not park + // The harness is idle and the sandbox is worth keeping warm, whatever the stream did. + const settledUserStop = + isUserStopAbort(signal) && + result.ok === true && + result.stopReason === "cancelled" && + result.cancelSettled === true; + if (settledUserStop) return true; if (clientGone?.()) return false; // client disconnected mid-turn: destroy, do not park + if (signal?.aborted) return false; // any other abort: unknown state, destroy if (!result.ok) return false; // failed turn: teardown as today if (result.stopReason === "paused") return false; // a plain pause never parks return true; @@ -55,6 +91,7 @@ export async function runSandboxAgent( try { result = await runTurn(env, request, emit, signal, { loaded: env.loadedFromContinuity, + nativeHistoryVerified: env.nativeHistoryVerified, ...turnOptions, // After the spread so a caller-supplied set wins, and short-circuited so we never CLAIM // rows the spread would then discard — a claimed row is spent even if it is thrown away. @@ -76,7 +113,10 @@ export async function runSandboxAgent( shouldPark(result, signal, undefined); await env.destroy({ reason: cleanResumable - ? "clean-resumable" + ? // A settled Stop parks under its own reason, so the log says WHY the sandbox survived. + result?.stopReason === "cancelled" + ? "cancelled" + : "clean-resumable" : signal?.aborted ? "aborted" : "failed-turn", diff --git a/services/runner/src/engines/sandbox_agent/environment-setup.ts b/services/runner/src/engines/sandbox_agent/environment-setup.ts index 75003877d9a..fc2519fbddd 100644 --- a/services/runner/src/engines/sandbox_agent/environment-setup.ts +++ b/services/runner/src/engines/sandbox_agent/environment-setup.ts @@ -64,6 +64,7 @@ export async function prepareEnvironmentSetup( request: AgentRunRequest, deps: SandboxAgentDeps = {}, presignedMount?: MountCredentials | null, + signal?: AbortSignal, ) { const logger = deps.log ?? defaultLog; const acquireStartedAt = Date.now(); @@ -116,6 +117,7 @@ export async function prepareEnvironmentSetup( apiBase: apiBase(), authorization: runCred, log: logger, + signal, }) : null; // A session-owned run expects a durable session cwd mount. When signing returns nothing the run @@ -136,6 +138,7 @@ export async function prepareEnvironmentSetup( apiBase: apiBase(), authorization: runCred, log: logger, + signal, }) : null; // A workflow-artifact run expects an agent mount; same structured degrade signal when unsigned. @@ -390,6 +393,10 @@ export async function prepareEnvironmentSetup( mountProjectId: mountCreds?.projectId, projectScopeId: projectScopeFor(request, mountCreds?.projectId)?.id, loadedFromContinuity: false, + nativeHistoryVerified: false, + // Daytona keeps its established per-harness transcript mounts. Local becomes durable only + // after its cwd mount succeeds; the Pi transcript directory lives underneath that cwd. + nativeHistoryDurable: plan.isDaytona, resumable: false, continuityTurnIndex: undefined, sessionDestroyRequested: false, diff --git a/services/runner/src/engines/sandbox_agent/environment.ts b/services/runner/src/engines/sandbox_agent/environment.ts index bedd98c6801..5b6410e2f71 100644 --- a/services/runner/src/engines/sandbox_agent/environment.ts +++ b/services/runner/src/engines/sandbox_agent/environment.ts @@ -34,6 +34,8 @@ import { mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { apiBase } from "../../apiBase.ts"; +import { abortableSandboxProvider } from "../../environment/abortable-sandbox-provider.ts"; +import { throwIfAcquireAborted } from "../../environment/acquire-abort.ts"; import { InMemorySessionPersistDriver, @@ -49,6 +51,7 @@ import { } from "../../protocol.ts"; import { advertisedToolSpecs } from "../../tools/public-spec.ts"; import { createAcpFetch } from "./acp-fetch.ts"; +import { createSandboxGoneLatch } from "./sandbox-gone.ts"; import { assert, assertRequiredCapabilities, @@ -426,7 +429,14 @@ async function acquireEnvironmentOnce( data: { phase: "environment_starting" }, transient: true, }); - const setup = await prepareEnvironmentSetup(request, deps, presignedMount); + throwIfAcquireAborted(signal); + const setup = await prepareEnvironmentSetup( + request, + deps, + presignedMount, + signal, + ); + throwIfAcquireAborted(signal); if (!setup.ok) return setup; const { acquireStartedAt, @@ -592,6 +602,7 @@ async function acquireEnvironmentOnce( signMount, signAgentMount, daytonaPiDir: DAYTONA_PI_DIR, + signal, }; const mountLocalDurableCwd = (reason: string) => mountLocalDurableCwdUnit(ctx, mountDeps, reason); @@ -650,25 +661,49 @@ async function acquireEnvironmentOnce( // mount-success path add guidance/env atomically, while a failed mount starts a normal // scratch-only harness with no false durable-storage signal. if (environment.mountCreds && !plan.isDaytona) { - await mountLocalDurableCwd("initial"); + const mounted = await mountLocalDurableCwd("initial"); + if (mounted && piSessionDir) environment.nativeHistoryDurable = true; + throwIfAcquireAborted(signal); } if (environment.agentMountCreds && !plan.isDaytona) { await mountLocalAgentCwd(); + throwIfAcquireAborted(signal); } // INVARIANT 1: the provider takes `env` and `piExtEnv` BY REFERENCE and hands them to the // daemon, after which the daemon environment is fixed. Every local mount had to land above // this line. From here a `writeDaemonEnv` is a programming-order bug and throws. ctx.freezeDaemonEnv(); - sandboxProvider = (deps.buildSandboxProvider ?? buildSandboxProvider)( - plan.sandboxId, - env, - binaryPath, - piExtEnv, - plan.credentials.modelEnvironment, - plan.sandboxPermission, - plan.credentials.daytonaSecretPlan, - inheritedLease ? { inheritedLease } : {}, + sandboxProvider = abortableSandboxProvider( + (deps.buildSandboxProvider ?? buildSandboxProvider)( + plan.sandboxId, + env, + binaryPath, + piExtEnv, + plan.credentials.modelEnvironment, + plan.sandboxPermission, + plan.credentials.daytonaSecretPlan, + inheritedLease ? { inheritedLease } : {}, + ), + signal, + logger, ); + // The turn's own socket is the first thing to learn that a remote sandbox was deleted, and it + // cannot end a turn by itself (the ACP transport swallows the failure and the pending prompt + // never settles). It notes the death here; `run-turn.ts` hands this latch to the liveness + // probe, which ends the turn. See `sandbox-gone.ts`. + // + // ARMED ONLY AFTER ACQUIRE. The same fetch also carries the SDK's health wait, which polls a + // sandbox that is still coming up and tolerates a provider error by design. On a warm resume + // the provider's proxy can lag its own control plane and answer for a sandbox it has not + // finished re-exposing. A report during acquire would latch a HEALTHY sandbox as dead and kill + // its first turn, and the latch is one-way, so the window has to be closed before it rather + // than reasoned about after. Acquire already has its own failure path for a sandbox that + // genuinely never comes up. + const sandboxGone = createSandboxGoneLatch(); + environment.sandboxGone = sandboxGone; + const acpFetchOptions = { + onSandboxGone: (reason: string) => sandboxGone.note(reason), + }; const startOptions = { sandbox: sandboxProvider, persist, @@ -678,8 +713,11 @@ async function acquireEnvironmentOnce( // Long-timeout undici dispatcher so a paused HITL turn is not reaped by undici's default // headersTimeout; Daytona additionally carries the per-sandbox auth cookie. fetch: plan.isDaytona - ? (deps.createCookieFetch ?? createCookieFetch)() - : (deps.createAcpFetch ?? createAcpFetch)(), + ? (deps.createCookieFetch ?? createCookieFetch)( + undefined, + acpFetchOptions, + ) + : (deps.createAcpFetch ?? createAcpFetch)(undefined, acpFetchOptions), }; // SandboxLifecycle owns the reconnect ladder, the fresh-create fallback, and both // `sandbox_start` timing marks. See `environment/sandbox-lifecycle.ts`. @@ -703,7 +741,11 @@ async function acquireEnvironmentOnce( }, ); environment.sandbox = acquiredSandbox.sandbox; + throwIfAcquireAborted(signal); environment.resumable = acquiredSandbox.resumable; + // The sandbox is up and the reconnect ladder is done, so a "sandbox not found" from here on is + // a real death rather than a proxy that has not caught up. See the latch above. + sandboxGone.arm(); // Read AFTER the sandbox is acquired, because the port is bound to a sandbox: the provider has // no allocation to deliver against until create (or reconnect) has settled. Undefined for // every provider that cannot deliver a credential to a live sandbox, which is what routes a @@ -772,6 +814,9 @@ async function acquireEnvironmentOnce( return "ok" as const; }) : undefined; + // The preflight runs concurrently with the rest of acquire. Attach a rejection observer now + // so an early Stop cannot become an unhandled rejection before the final await reaches it. + void credentialPreflight?.catch(() => {}); // On Daytona, push the harness login, the extension, and AGENTS.md into the remote sandbox. // For a non-Pi harness with executable tools, also push the in-sandbox stdio MCP shim @@ -864,6 +909,7 @@ async function acquireEnvironmentOnce( ? undefined : ((await (deps.discoverTunnelEndpoint ?? discoverTunnelEndpoint)({ log: logger, + signal, })) ?? undefined); const refusal = mountRefusal(storeEndpoint, endpoint); const canMount = !refusal; @@ -886,6 +932,7 @@ async function acquireEnvironmentOnce( { endpoint, log: logger, + signal, }, )) ) { @@ -913,6 +960,7 @@ async function acquireEnvironmentOnce( apiBase: apiBase(), authorization: runCred, log: logger, + signal, }, ); } @@ -933,6 +981,7 @@ async function acquireEnvironmentOnce( ? undefined : ((await (deps.discoverTunnelEndpoint ?? discoverTunnelEndpoint)({ log: logger, + signal, })) ?? undefined); const refusal = mountRefusal(storeEndpoint, endpoint); const canMount = !refusal; @@ -955,7 +1004,7 @@ async function acquireEnvironmentOnce( environment.sandbox, mountPath, environment.agentMountCreds, - { endpoint, log: logger }, + { endpoint, log: logger, signal }, )) ) { environment.agentMountedPath = mountPath; @@ -975,6 +1024,7 @@ async function acquireEnvironmentOnce( logger(`remote agent mount active for artifact=${artifactId}`); } } catch (err) { + throwIfAcquireAborted(signal); logger( `remote agent mount failed artifact=${artifactId}: ${conciseError(err, plan.harness)}`, ); @@ -1231,6 +1281,7 @@ async function acquireEnvironmentOnce( cwd: plan.workspace.cwd, sessionInit, priorAgentSessionId, + nativeHistoryDurable: environment.nativeHistoryDurable, localSessionId, continuitySessionKey, log: logger, @@ -1238,6 +1289,7 @@ async function acquireEnvironmentOnce( }); environment.session = opened.session; environment.loadedFromContinuity = opened.loadedFromContinuity; + environment.nativeHistoryVerified = opened.nativeHistoryVerified; // The reopen capability, captured here because this is the only scope holding the persist // driver, the session-init payload and the local session key together. Same pattern as // `destroy`: the environment carries a closure rather than the ingredients. @@ -1250,6 +1302,7 @@ async function acquireEnvironmentOnce( cwd: plan.workspace.cwd, sessionInit, priorAgentSessionId: environment.session?.agentSessionId, + nativeHistoryDurable: environment.nativeHistoryDurable, localSessionId, continuitySessionKey, log: logger, @@ -1260,6 +1313,7 @@ async function acquireEnvironmentOnce( if (result.ok) { environment.session = result.session; environment.loadedFromContinuity = result.loadedFromContinuity; + environment.nativeHistoryVerified = result.nativeHistoryVerified; } return result; }; @@ -1334,6 +1388,8 @@ async function acquireEnvironmentOnce( } } + throwIfAcquireAborted(signal); + timingLog("acquire_total", acquireStartedAt); emit?.({ type: "data", diff --git a/services/runner/src/engines/sandbox_agent/errors.ts b/services/runner/src/engines/sandbox_agent/errors.ts index 563c2774fdf..35265ac6d77 100644 --- a/services/runner/src/engines/sandbox_agent/errors.ts +++ b/services/runner/src/engines/sandbox_agent/errors.ts @@ -57,13 +57,45 @@ function keyHintFor( * `runner_error` is the catch-all every unclassified failure keeps, matching what the SDK stamped * on runner-reported errors before the runner had a say. */ +/** + * Markers the runner puts in an error message so `classifyRunError` can set the class. + * + * Both are strings only this runner produces, so a match needs no corroboration. They live + * here, next to the codes they map to, and are imported by the modules that raise them. + */ +export const SANDBOX_GONE_MARKER = "sandbox is gone"; +export const ABANDONED_TURN_MARKER = "execution abandoned"; + +/** The line the user reads when the machine running their turn disappeared. */ +export const SANDBOX_GONE_MESSAGE = + "The sandbox running this session stopped responding, so the run was ended. " + + "Send the message again to start a fresh sandbox."; + +/** The line the user reads when the run never produced an outcome of its own. */ +export const EXECUTION_LOST_MESSAGE = + "The agent stopped responding and the run was closed. Send the message again to retry."; + export type RunErrorCode = | "runner_error" | "starter_credits_exhausted" | "starter_credits_program_paused" | "starter_credits_unavailable" | "credential_delivery_failed" - | "rate_limited"; + | "rate_limited" + // Not a failure: the turn was REFUSED before it started because another turn already owns + // this session. Nothing ran, nothing was destroyed, and the user's message was never sent. + // Clients render it as a "not sent, try again" state and keep the text, never as a run error. + // Produced by `sessions/admission.ts`, not by this module's classifier. + | "session_turn_in_use" + // The sandbox died under a running turn: its liveness probe stopped answering, so the turn + // was ended rather than left holding a machine that no longer exists. See + // `sandbox-liveness.ts`. + | "sandbox_gone" + // The execution never produced an outcome of its own, so one was written for it. Two + // producers: this runner, when a turn will not unwind after its abort (`sessions/ + // turn-settle.ts`), and the platform's execution watchdog, when the runner itself is gone + // (`api/oss/src/tasks/asyncio/sessions/orphan_sweep.py`). + | "execution_lost"; /** One failed run, condensed: the line the user reads plus the class a client can act on. */ export interface ClassifiedRunError { @@ -336,6 +368,14 @@ export function classifyRunError( code: "credential_delivery_failed", }; } + // First, and self-evidencing: this marker is produced by our own liveness probe and by + // nothing else, so it needs no corroboration and must not be re-read as a provider fault. + if (raw.includes(SANDBOX_GONE_MARKER)) { + return { message: SANDBOX_GONE_MESSAGE, code: "sandbox_gone" }; + } + if (raw.includes(ABANDONED_TURN_MARKER)) { + return { message: EXECUTION_LOST_MESSAGE, code: "execution_lost" }; + } // A budget refusal is checked first: it is the most specific reading of a 429, and its body also // trips the rate-limit and quota matchers below. if (BUDGET_REFUSAL.test(raw)) { diff --git a/services/runner/src/engines/sandbox_agent/mount.ts b/services/runner/src/engines/sandbox_agent/mount.ts index 974775ea773..c3588389ee8 100644 --- a/services/runner/src/engines/sandbox_agent/mount.ts +++ b/services/runner/src/engines/sandbox_agent/mount.ts @@ -17,6 +17,11 @@ import { execFile, spawn } from "node:child_process"; import { promisify } from "node:util"; +import { + throwIfAcquireAborted, + waitForAcquire, +} from "../../environment/acquire-abort.ts"; + const pExecFile = promisify(execFile); /** POSIX single-quote escaping for values interpolated into `sh -c` strings. */ @@ -51,6 +56,7 @@ export interface SignMountDeps { /** Injectable for tests; defaults to global fetch. */ fetchImpl?: typeof fetch; log?: (msg: string) => void; + signal?: AbortSignal; } function defaultLog(msg: string): void { @@ -81,6 +87,7 @@ export async function signSessionMountCredentials( "content-type": "application/json", authorization: deps.authorization, }, + signal: deps.signal, }); if (!res.ok) { // 503 = storage not configured (mounts disabled). Any non-2xx → run without this mount. @@ -124,6 +131,7 @@ export async function signSessionMountCredentials( : undefined, }; } catch (err) { + throwIfAcquireAborted(deps.signal); log( `sign failed session=${sessionId}: ${String(err instanceof Error ? err.message : err).slice(0, 160)}`, ); @@ -286,6 +294,7 @@ export interface MountStorageDeps { /** Injectable command/probe seams while retaining production unmountStorage behavior. */ unmountDeps?: UnmountStorageDeps; log?: (msg: string) => void; + signal?: AbortSignal; } /** @@ -304,6 +313,8 @@ export async function mountStorage( ): Promise { const log = deps.log ?? defaultLog; const checkMounted = deps.checkMounted ?? ((c: string) => isMounted(c, log)); + const signal = deps.signal; + throwIfAcquireAborted(signal); log( `mountStorage begin cwd=${cwd} bucket=${creds.bucket} prefix=${creds.prefix} ` + @@ -311,7 +322,7 @@ export async function mountStorage( `expiresAt=${creds.expiresAt ?? "(none)"}`, ); - if (await checkMounted(cwd)) { + if (await waitForAcquire(() => checkMounted(cwd), signal)) { log(`already mounted (verified alive): ${cwd}`); return true; } @@ -322,6 +333,7 @@ export async function mountStorage( ...deps.unmountDeps, log, }); + throwIfAcquireAborted(signal); if (!staleMountDetached) { throw new Error( "pre-mount detach could not be confirmed for " + @@ -388,11 +400,16 @@ export async function mountStorage( let failure: unknown; try { log(`geesefs mount argv: ${args.join(" ")}`); - const started = await run(args, env); + const started = await waitForAcquire(() => run(args, env), signal, { + onLateSuccess: async (lateAttempt) => { + await lateAttempt?.stop(); + await unmountStorage(cwd, { ...deps.unmountDeps, log }); + }, + }); attempt = started || undefined; // Confirm the new mount actually serves I/O — a still-not-alive cwd means geesefs failed // to mount (invalid STS creds, store unreachable) or did not come up within the poll window. - if (!(await checkMounted(cwd))) { + if (!(await waitForAcquire(() => checkMounted(cwd), signal))) { failure = new Error( `mount reported success but cwd is NOT alive ${creds.bucket}:${creds.prefix} -> ${cwd} ` + `— likely expired/invalid STS creds or store unreachable`, @@ -405,6 +422,18 @@ export async function mountStorage( failure = err; } + if (signal?.aborted) { + // Cleanup must not hold the Stop response open. A late `runGeesefs` result has its own hook + // above; an already-returned attempt is stopped here, and both paths confirm the detach. + void Promise.resolve() + .then(async () => { + await attempt?.stop(); + await unmountStorage(cwd, { ...deps.unmountDeps, log }); + }) + .catch(() => {}); + throwIfAcquireAborted(signal); + } + // Never detach/fallback while a failed geesefs attempt may still attach later. await attempt?.stop(); const detached = await unmountStorage(cwd, { ...deps.unmountDeps, log }); @@ -502,6 +531,7 @@ export interface TunnelDeps { ngrokApi?: string; fetchImpl?: typeof fetch; log?: (msg: string) => void; + signal?: AbortSignal; } /** @@ -523,7 +553,7 @@ export async function discoverTunnelEndpoint( process.env.AGENTA_MOUNTS_TUNNEL_API ?? "http://ngrok:4040"; try { - const res = await doFetch(`${api}/api/tunnels`); + const res = await doFetch(`${api}/api/tunnels`, { signal: deps.signal }); if (!res.ok) { log(`tunnel discovery HTTP ${res.status}`); return null; @@ -539,6 +569,7 @@ export async function discoverTunnelEndpoint( const any = tunnels.find((t) => !!t.public_url)?.public_url; return https ?? any ?? null; } catch (err) { + throwIfAcquireAborted(deps.signal); log( `tunnel discovery failed: ${String(err instanceof Error ? err.message : err).slice(0, 160)}`, ); @@ -569,6 +600,7 @@ export interface MountStorageRemoteDeps { */ aliveAttempts?: number; log?: (msg: string) => void; + signal?: AbortSignal; } /** @@ -586,25 +618,35 @@ async function remoteMountAlive( sandbox: SandboxExec, cwd: string, attempts: number, + signal?: AbortSignal, ): Promise { let consecutiveThrows = 0; for (let i = 0; i < attempts; i++) { + throwIfAcquireAborted(signal); try { - const res = await sandbox.runProcess({ - command: "sh", - args: [ - "-c", - `mountpoint -q ${shellQuote(cwd)} && ls ${shellQuote(cwd)} >/dev/null 2>&1`, - ], - timeoutMs: 5_000, - }); + const res = await waitForAcquire( + () => + sandbox.runProcess({ + command: "sh", + args: [ + "-c", + `mountpoint -q ${shellQuote(cwd)} && ls ${shellQuote(cwd)} >/dev/null 2>&1`, + ], + timeoutMs: 5_000, + }), + signal, + ); consecutiveThrows = 0; if (res?.exitCode === 0) return true; } catch { + throwIfAcquireAborted(signal); consecutiveThrows += 1; if (consecutiveThrows >= 2) break; } - await new Promise((r) => setTimeout(r, 500)); + await waitForAcquire( + () => new Promise((resolve) => setTimeout(resolve, 500)), + signal, + ); } return false; } @@ -649,28 +691,45 @@ export async function mountStorageRemote( deps: MountStorageRemoteDeps, ): Promise { const log = deps.log ?? defaultLog; + throwIfAcquireAborted(deps.signal); try { // A reattached running sandbox may still hold a FUSE mount with expired credentials. Detach // it before remounting; on a fresh sandbox this is one fast best-effort no-op. - await unmountRemoteDeadMount(sandbox, cwd, log); + await waitForAcquire( + () => unmountRemoteDeadMount(sandbox, cwd, log), + deps.signal, + ); // Ensure the directory exists before mounting. - await sandbox.runProcess({ - command: "sh", - args: ["-c", `mkdir -p ${shellQuote(cwd)}`], - timeoutMs: 30_000, - }); + await waitForAcquire( + () => + sandbox.runProcess({ + command: "sh", + args: ["-c", `mkdir -p ${shellQuote(cwd)}`], + timeoutMs: 30_000, + }), + deps.signal, + ); // Background geesefs with its logs to a file so the RPC returns immediately. const args = geesefsArgs(creds, cwd, deps.endpoint, false); const logFile = "/tmp/geesefs-mount.log"; const quotedArgs = args.map(shellQuote).join(" "); const geefsCmd = `geesefs --log-file ${shellQuote(logFile)} ${quotedArgs} >>${shellQuote(logFile)} 2>&1 &`; log(`remote geesefs argv: ${args.join(" ")}`); - const res = await sandbox.runProcess({ - command: "sh", - args: ["-c", geefsCmd], - env: credEnv(creds), - timeoutMs: deps.mountTimeoutMs ?? 60_000, - }); + const res = await waitForAcquire( + () => + sandbox.runProcess({ + command: "sh", + args: ["-c", geefsCmd], + env: credEnv(creds), + timeoutMs: deps.mountTimeoutMs ?? 60_000, + }), + deps.signal, + { + onLateSuccess: async () => { + await unmountRemoteDeadMount(sandbox, cwd, log); + }, + }, + ); if (res?.exitCode !== 0) { log( `remote mount exit=${res?.exitCode}: ${String(res?.stderr).slice(-300)}`, @@ -678,12 +737,23 @@ export async function mountStorageRemote( return false; } // The daemon backgrounds before the FUSE channel serves I/O, so wait for it. - if (!(await remoteMountAlive(sandbox, cwd, deps.aliveAttempts ?? 12))) { - const tail = await sandbox.runProcess({ - command: "sh", - args: ["-c", "tail -5 /tmp/geesefs-mount.log 2>/dev/null"], - timeoutMs: 10_000, - }); + if ( + !(await remoteMountAlive( + sandbox, + cwd, + deps.aliveAttempts ?? 12, + deps.signal, + )) + ) { + const tail = await waitForAcquire( + () => + sandbox.runProcess({ + command: "sh", + args: ["-c", "tail -5 /tmp/geesefs-mount.log 2>/dev/null"], + timeoutMs: 10_000, + }), + deps.signal, + ); log( `remote mount not alive ${creds.bucket}:${creds.prefix} -> ${cwd}` + `; geesefs: ${String(tail?.result ?? tail?.stderr ?? "").slice(-400)}`, @@ -698,6 +768,10 @@ export async function mountStorageRemote( ); return true; } catch (err) { + if (deps.signal?.aborted) { + void unmountRemoteDeadMount(sandbox, cwd, log); + throwIfAcquireAborted(deps.signal); + } log( `remote mount failed: ${String(err instanceof Error ? err.message : err).slice(0, 200)}`, ); @@ -715,6 +789,7 @@ export interface MountHarnessSessionDirsDeps { log?: (msg: string) => void; signSessionMountCredentials?: typeof signSessionMountCredentials; mountStorageRemote?: typeof mountStorageRemote; + signal?: AbortSignal; } /** @@ -747,6 +822,7 @@ export async function mountHarnessSessionDirs( authorization: deps.authorization, fetchImpl: deps.fetchImpl, log, + signal: deps.signal, }, dir.name, ); @@ -761,6 +837,7 @@ export async function mountHarnessSessionDirs( await mountRemote(sandbox, dir.path, creds, { endpoint: tunnelEndpoint, log, + signal: deps.signal, }); } } diff --git a/services/runner/src/engines/sandbox_agent/provider.ts b/services/runner/src/engines/sandbox_agent/provider.ts index 06710bd3eab..f312c23aa30 100644 --- a/services/runner/src/engines/sandbox_agent/provider.ts +++ b/services/runner/src/engines/sandbox_agent/provider.ts @@ -24,6 +24,35 @@ import { type DaytonaSecretPlan, } from "./daytona-secret-plan.ts"; +/** The port the Daytona provider passes to `sandbox-agent server`. */ +export const DAYTONA_SANDBOX_AGENT_PORT = 3_000; + +/** + * Recover the daemon port from the public sandbox handle id. + * + * Local ids are the daemon's `host:port`; Daytona ids are opaque, so use the explicit port this + * module gives that provider. Unknown providers stay undefined rather than borrowing a port. + */ +export function sandboxAgentServerPort( + sandboxId: string | undefined, +): number | undefined { + if (!sandboxId) return undefined; + const separator = sandboxId.indexOf("/"); + if (separator <= 0) return undefined; + const provider = sandboxId.slice(0, separator); + if (provider === "daytona") return DAYTONA_SANDBOX_AGENT_PORT; + if (provider !== "local") return undefined; + + try { + const port = Number(new URL(`http://${sandboxId.slice(separator + 1)}`).port); + return Number.isInteger(port) && port > 0 && port <= 65_535 + ? port + : undefined; + } catch { + return undefined; + } +} + /** * Translate the Layer 2 network policy into Daytona create fields. Daytona enforces egress * at the sandbox boundary: `networkBlockAll` blocks all outbound, `networkAllowList` is a @@ -191,6 +220,7 @@ export function buildSandboxProvider( daytonaWithLifecycle( { ...(image ? { image } : {}), + agentPort: DAYTONA_SANDBOX_AGENT_PORT, create: { ...createFields, ...(Object.keys(secretAttachments).length > 0 diff --git a/services/runner/src/engines/sandbox_agent/reap-exec.ts b/services/runner/src/engines/sandbox_agent/reap-exec.ts new file mode 100644 index 00000000000..fb489cad88e --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/reap-exec.ts @@ -0,0 +1,313 @@ +/** + * Kill the shell command a STOPPED Codex turn left running inside the parked sandbox. + * + * WHY THIS EXISTS. `cancel-turn.ts` makes a Stop keep the sandbox warm. Parking is what makes + * this leak visible: before it, every Stop deleted the sandbox, and the delete killed whatever + * the turn had started. Measured on the integration stack, local sandbox provider, 2026-09-03: + * + * | Harness | ACP `session/cancel` answered | The shell child after the Stop | + * | --- | --- | --- | + * | Pi (`pi_core`) | yes | gone inside 0.2 s | + * | Claude Code | yes | gone inside 0.2 s | + * | Codex | yes, in 48 ms | STILL RUNNING, until the park window closed at 60 s | + * + * WHY CODEX DIFFERS, AND WHY THE FIX CANNOT LIVE IN THE BRIDGE. Pi and Claude run their shell + * tool inside a process the ACP adapter owns, so the adapter holds the child's pid and kills it + * when the run's `AbortSignal` fires. Codex does not: `@agentclientprotocol/codex-acp` is a thin + * JavaScript bridge over a Rust `codex app-server` subprocess, the shell child is a DIRECT child + * of that Rust process, and the bridge's `cancel()` only sends the `turn/interrupt` JSON-RPC + * request. Measured parent chain of the leaked child: + * + * python3 -c ... <- the leak + * codex app-server <- the Rust core, spawns and abandons it + * node .../codex.js <- the JS launcher + * node .../codex-acp <- the ACP bridge, holds NO pid for the shell + * sandbox-agent server <- the daemon + * + * The interrupt itself works: the prompt settles `cancelled` in about 48 ms. What the Rust core + * does not do is kill the exec it started, and that core is a stripped vendored binary we pin + * rather than build. A patch to the JS bridge would have to do the same `/proc` walk this module + * does, in a bundle that is installed into the sandbox image and therefore needs a Daytona + * SNAPSHOT REBUILD to ship. This module does it from the runner instead, through the sandbox + * daemon's one-off process API, so it ships in the runner image alone and behaves identically on + * the local and the Daytona provider. + * + * WHY IT IS SAFE FOR A WARM SESSION. The reap never touches the daemon, the ACP bridge, or the + * `codex app-server` itself, so the native harness session survives exactly as it did before. Two + * rules keep it off anything else the app-server legitimately owns, an stdio MCP server most of + * all: only DESCENDANTS of the app-server are candidates, and only those younger than the turn + * that was just stopped. An MCP server starts when the session is created, before the prompt, so + * it is always older than the turn and is never selected. + * + * WHY A FAILURE DESTROYS. A parked sandbox must not retain a command from the stopped turn. Only a + * successful kill or a successful inspection that finds nothing to reap proves parking is safe. + */ + +/** One row of `ps -eo pid=,ppid=,etimes=,args=`. */ +export interface ProcRow { + pid: number; + ppid: number; + /** Seconds since the process started. */ + etimes: number; + args: string; +} + +export const PS_ARGS = ["-eo", "pid=,ppid=,etimes=,args="]; + +/** + * How many processes one reap may kill. A Stop leaks one command; anything near this number means + * the anchor matched something it should not have, so the reap gives up rather than guessing. + */ +export const MAX_REAPED = 32; + +/** Parse `ps -eo pid=,ppid=,etimes=,args=`. An unparseable line is dropped, never guessed at. */ +export function parseProcessTable(stdout: string): ProcRow[] { + const rows: ProcRow[] = []; + for (const line of stdout.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S.*)$/.exec(line); + if (!match) continue; + rows.push({ + pid: Number(match[1]), + ppid: Number(match[2]), + etimes: Number(match[3]), + args: match[4], + }); + } + return rows; +} + +/** Find exactly one `sandbox-agent server` whose `--port` value is this sandbox's port. */ +export function findSandboxAgentServerPid( + rows: ProcRow[], + port: number | undefined, +): number | undefined { + if (!Number.isInteger(port) || (port ?? 0) <= 0) return undefined; + const expectedPort = String(port); + const matches = rows.filter((row) => { + const [executable, ...rest] = row.args.split(/\s+/); + if (!executable) return false; + const basename = executable.split("/").pop(); + const portIndex = rest.indexOf("--port"); + return ( + basename === "sandbox-agent" && + rest.includes("server") && + portIndex >= 0 && + rest[portIndex + 1] === expectedPort + ); + }); + return matches.length === 1 ? matches[0].pid : undefined; +} + +/** + * Find the `codex app-server` process beneath this sandbox's daemon. + * + * The match is deliberately narrow: the executable's basename must be exactly `codex` AND the + * command must carry the `app-server` subcommand. The JS launcher (`node .../codex.js app-server`) + * also carries the subcommand, which is why the basename check is on the executable rather than + * anywhere in the string. Returns `undefined` when there is not exactly one match below the + * daemon, because killing on a guess is worse than leaving a `sleep` running for the park window. + */ +export function findAppServerPid( + rows: ProcRow[], + sandboxAgentPid: number, +): number | undefined { + const childrenOf = new Map(); + for (const row of rows) { + const siblings = childrenOf.get(row.ppid); + if (siblings) siblings.push(row); + else childrenOf.set(row.ppid, [row]); + } + + const descendants: ProcRow[] = []; + const seen = new Set([sandboxAgentPid]); + const queue = [sandboxAgentPid]; + while (queue.length > 0) { + const parent = queue.shift() as number; + for (const child of childrenOf.get(parent) ?? []) { + if (seen.has(child.pid) || child.pid <= 1) continue; + seen.add(child.pid); + queue.push(child.pid); + descendants.push(child); + } + } + + const matches = descendants.filter((row) => { + const [executable, ...rest] = row.args.split(/\s+/); + if (!executable) return false; + const basename = executable.split("/").pop(); + return basename === "codex" && rest.includes("app-server"); + }); + return matches.length === 1 ? matches[0].pid : undefined; +} + +/** + * The pids a settled Codex Stop may kill. + * + * A candidate must be a descendant of the `codex app-server` process and must have started no + * earlier than the stopped turn. Everything else, the app-server included, is left alone. + */ +export function selectLeakedExecPids( + rows: ProcRow[], + input: { appServerPid: number; turnElapsedSeconds: number }, +): number[] { + const childrenOf = new Map(); + for (const row of rows) { + const siblings = childrenOf.get(row.ppid); + if (siblings) siblings.push(row); + else childrenOf.set(row.ppid, [row]); + } + + const selected: number[] = []; + const seen = new Set([input.appServerPid]); + const queue = [input.appServerPid]; + while (queue.length > 0) { + const parent = queue.shift() as number; + for (const child of childrenOf.get(parent) ?? []) { + if (seen.has(child.pid) || child.pid <= 1) continue; + seen.add(child.pid); + queue.push(child.pid); + // `etimes` is whole seconds, so a child started in the same second as the prompt reads + // equal to the turn's elapsed time. `<=` keeps that child; anything OLDER than the turn + // predates the prompt and belongs to the session, not to the turn that was stopped. + if (child.etimes <= input.turnElapsedSeconds) selected.push(child.pid); + } + } + return selected; +} + +export interface ReapSandbox { + runProcess?: (request: { + command: string; + args?: string[]; + timeoutMs?: number; + maxOutputBytes?: number; + }) => Promise<{ stdout: string; exitCode?: number | null }>; +} + +export interface ReapLeakedExecInput { + sandbox: ReapSandbox | undefined; + /** Port passed to this sandbox's `sandbox-agent server --port`. */ + sandboxAgentPort: number | undefined; + /** Milliseconds from the prompt being issued to the cancel settling. */ + turnElapsedMs: number; + log: (message: string) => void; + timeoutMs?: number; +} + +export interface ReapResult { + /** How many processes the reap killed. */ + killed: number; + /** Why nothing was killed, when nothing was. */ + skipped?: + | "no-run-process" + | "ps-failed" + | "no-app-server" + | "nothing-to-reap" + | "too-many" + | "kill-failed"; +} + +/** True when best-effort cleanup needs QA follow-up. */ +export function reapResultHasCleanupMiss( + result: ReapResult | undefined, +): boolean { + return ( + !result || (result.killed === 0 && result.skipped !== "nothing-to-reap") + ); +} + +/** + * Best effort. Never throws, and every outcome is one log line the release gate can assert on. + */ +export async function reapLeakedExecChildren( + input: ReapLeakedExecInput, +): Promise { + const runProcess = input.sandbox?.runProcess; + if (!runProcess) { + input.log("stage=harness_reap killed=0 skipped=no-run-process"); + return { killed: 0, skipped: "no-run-process" }; + } + const timeoutMs = input.timeoutMs ?? 2_000; + + let rows: ProcRow[]; + try { + const listing = await runProcess.call(input.sandbox, { + command: "ps", + args: PS_ARGS, + timeoutMs, + maxOutputBytes: 256 * 1024, + }); + rows = parseProcessTable(listing.stdout ?? ""); + if (rows.length === 0) throw new Error("no parseable rows"); + } catch (error) { + // A sandbox image without a compatible `ps` cannot prove that parking is safe. + input.log( + "stage=harness_reap killed=0 skipped=ps-failed error=" + + (error instanceof Error ? error.message : String(error)).slice(0, 120), + ); + return { killed: 0, skipped: "ps-failed" }; + } + + const sandboxAgentPid = findSandboxAgentServerPid( + rows, + input.sandboxAgentPort, + ); + const appServerPid = + sandboxAgentPid === undefined + ? undefined + : findAppServerPid(rows, sandboxAgentPid); + if (appServerPid === undefined) { + input.log("stage=harness_reap killed=0 skipped=no-app-server"); + return { killed: 0, skipped: "no-app-server" }; + } + + // FLOOR, not round or ceil. Every rounding error must make the reap kill LESS. On a cold first + // turn the session's own helpers (Codex clones its plugin repo) start barely a second before + // the prompt, so one second of generosity here is one second of overlap with processes the + // session owns. A child born in the first second of a turn is not physically possible: the + // model has to emit a tool call first. + const turnElapsedSeconds = Math.floor( + Math.max(0, input.turnElapsedMs) / 1000, + ); + const pids = selectLeakedExecPids(rows, { + appServerPid, + turnElapsedSeconds, + }); + if (pids.length === 0) { + input.log( + `stage=harness_reap killed=0 skipped=nothing-to-reap app_server=${appServerPid}`, + ); + return { killed: 0, skipped: "nothing-to-reap" }; + } + if (pids.length > MAX_REAPED) { + input.log( + `stage=harness_reap killed=0 skipped=too-many candidates=${pids.length} ` + + `limit=${MAX_REAPED} app_server=${appServerPid}`, + ); + return { killed: 0, skipped: "too-many" }; + } + + try { + const result = await runProcess.call(input.sandbox, { + command: "kill", + args: ["-9", ...pids.map(String)], + timeoutMs, + maxOutputBytes: 4 * 1024, + }); + if (result.exitCode != null && result.exitCode !== 0) { + throw new Error(`kill exited with status ${result.exitCode}`); + } + } catch (error) { + input.log( + "stage=harness_reap killed=0 skipped=kill-failed error=" + + (error instanceof Error ? error.message : String(error)).slice(0, 120), + ); + return { killed: 0, skipped: "kill-failed" }; + } + + input.log( + `stage=harness_reap killed=${pids.length} pids=${pids.join(",")} ` + + `app_server=${appServerPid} turn_elapsed_s=${turnElapsedSeconds}`, + ); + return { killed: pids.length }; +} diff --git a/services/runner/src/engines/sandbox_agent/reconstruct-history.ts b/services/runner/src/engines/sandbox_agent/reconstruct-history.ts index cacc4d2e5bf..dcfd3188c3e 100644 --- a/services/runner/src/engines/sandbox_agent/reconstruct-history.ts +++ b/services/runner/src/engines/sandbox_agent/reconstruct-history.ts @@ -30,6 +30,16 @@ export interface ReconstructHistoryOptions { restore?: (messages: ChatMessage[]) => Promise; } +function isLegacyWholeBodyTruncation(row: { attributes?: unknown }): boolean { + const attributes = row.attributes; + return ( + !!attributes && + typeof attributes === "object" && + (attributes as { _truncated?: unknown })._truncated === true && + !("type" in attributes) + ); +} + // Compose passes `${AGENTA_SESSIONS_RECONSTRUCT:-}`, so an empty value must mean on just like an // absent value. Only the literal "false" disables reconstruction. function reconstructEnabled(): boolean { @@ -100,6 +110,11 @@ export async function reconstructHistoryIfNeeded( const prior = currentTurnId ? records.filter((row) => row.turn_id !== currentTurnId) : records; + if (prior.some(isLegacyWholeBodyTruncation)) { + throw new Error( + `session ${sessionId} contains a truncated durable record; refusing to rebuild an incomplete conversation`, + ); + } // Reachable in practice: a caller that builds its answer from the durable interaction row can // echo the row's stored `turn_id`, which drops exactly the turn that parked. if (prior.length === 0) { diff --git a/services/runner/src/engines/sandbox_agent/run-limits.ts b/services/runner/src/engines/sandbox_agent/run-limits.ts index d360c3e43b6..a8b8c34d867 100644 --- a/services/runner/src/engines/sandbox_agent/run-limits.ts +++ b/services/runner/src/engines/sandbox_agent/run-limits.ts @@ -37,9 +37,11 @@ export const TOOL_CALL_TIMEOUT_ENV = "AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS"; // every mount-backed warm session rebuild cold. The ~1h gap under the 12h lease is // the warm parking window. export const DEFAULT_TOTAL_DEADLINE_MS = 11 * 60 * 60_000; // 11 hours -export const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60_000; // 30 min +// 30 minutes; override with AGENTA_RUNNER_RUN_IDLE_TIMEOUT_MS. +export const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60_000; export const DEFAULT_TTFB_TIMEOUT_MS = 2 * 60_000; // 2 min -export const DEFAULT_TOOL_CALL_TIMEOUT_MS = 30 * 60_000; // 30 min +// 30 minutes; override with AGENTA_RUNNER_TOOL_CALL_TIMEOUT_MS. +export const DEFAULT_TOOL_CALL_TIMEOUT_MS = 30 * 60_000; /** Every field is a usable timer delay (integer ms, at least 1, within Node's timer range) — * `resolveRunLimits` guarantees it, so callers can arm any of them without re-checking. */ diff --git a/services/runner/src/engines/sandbox_agent/run-turn.ts b/services/runner/src/engines/sandbox_agent/run-turn.ts index b72265165ab..dfa7f09702b 100644 --- a/services/runner/src/engines/sandbox_agent/run-turn.ts +++ b/services/runner/src/engines/sandbox_agent/run-turn.ts @@ -67,6 +67,14 @@ import { CREDENTIAL_RACE_REPORTS_PER_SESSION, withinCredentialPropagationWindow, } from "./errors.ts"; +import { noteExecutionSettled } from "../../sessions/execution-registry.ts"; +import { isUserStopAbort } from "../../sessions/stop-signal.ts"; +import { cancelHarnessTurn } from "./cancel-turn.ts"; +import { + reapLeakedExecChildren, + reapResultHasCleanupMiss, +} from "./reap-exec.ts"; +import { sandboxAgentServerPort } from "./provider.ts"; import { PAUSED, PendingApprovalPauseController } from "./pause.ts"; import { capturePiTranscriptCursor, @@ -80,6 +88,12 @@ import { createCommitAuthorizationState, } from "./approved-content.ts"; import { createRunLimits, resolveRunLimits } from "./run-limits.ts"; +import { + httpLivenessProbe, + resolveSandboxLivenessLimits, + sandboxHealthUrl, + startSandboxLivenessProbe, +} from "./sandbox-liveness.ts"; import { RUN_LIMIT_TRIPPED, sendLastMessageOnly, @@ -147,6 +161,12 @@ export async function runTurn( // heartbeat aborts `signal`). Distinct from PAUSED/RUN_LIMIT_TRIPPED so the turn ends CLEANLY // (honest interrupted transcript, keep-warm) instead of falling through to the error catch. const CANCELLED = Symbol("cancelled"); + /** + * Did the harness confirm it stopped? Set only on the cancelled path, and only when the ACP + * cancel was sent AND the harness answered its open prompt inside the settle budget. It rides + * out on the result because it is the one fact that decides park versus delete for a Stop. + */ + let cancelSettled = false; const continuityStore = deps.sessionContinuityStore ?? sessionContinuityStore; /** * Should a credential refusal this turn be reported as a delivery race rather than a bad key? @@ -212,7 +232,8 @@ export async function runTurn( // A fresh turn never inherits an approval. Only a resume may consume records minted before // the park; anything else starts empty, so no call can execute on the strength of an approval // raised for an earlier turn. - if (!opts.resume) env.commitAuthorization = undefined; + if (!opts.resume && !opts.settleApprovalsThenPrompt) + env.commitAuthorization = undefined; env.nonParkablePauseCount = 0; // Hoisted so the catch can flush a partial trace (mirroring the pre-split `otel?` handling — // a createOtel throw must still return `{ ok: false }`, not propagate raw) and the finally can @@ -251,6 +272,33 @@ export async function runTurn( runLimitTrip?.(); }); + // The run limits above cannot see a sandbox that DIED under the turn: the ACP prompt they + // race against never settles once the peer is gone, and `notePaused()` retires them entirely + // while a turn waits for a human. So probe the sandbox's own HTTP surface, independently of + // the wedged ACP channel, and end the turn through the same trip path any other limit uses. + // See `sandbox-liveness.ts` and issue #6418. + // A remote sandbox does not refuse the socket when it dies: its provider's proxy answers for it + // with "sandbox not found" indefinitely, which the poll reads as alive. So the turn's own + // ACP transport reports that answer on `env.sandboxGone`, and the probe ends the turn on it at + // once. That path needs no health URL, so it is wired even when the poll is disabled. + const sandboxHealth = sandboxHealthUrl(env.sandbox); + const sandboxLiveness = startSandboxLivenessProbe({ + ...(sandboxHealth ? { probe: httpLivenessProbe(sandboxHealth) } : {}), + goneSignal: env.sandboxGone, + limits: resolveSandboxLivenessLimits(logger), + onGone: (reason: string) => { + runLimitReason = reason; + runLimitTrip?.(); + }, + log: logger, + }); + if (!sandboxHealth) { + logger( + "[sandbox-liveness] no health URL on this sandbox; polling disabled " + + "(the transport's own sandbox-gone report still ends the turn)", + ); + } + try { // AGENTA_SESSIONS_RECONSTRUCT defaults on so minimal-history clients keep their conversation; // only the literal "false" opts out. The compose default supplies an empty string, not "true". @@ -1076,10 +1124,15 @@ export async function runTurn( // byte-exact args). Either way, on a HITL pause the prompt resolves cancelled or never // resolves, and the pause signal ends the turn. let promptPromise: Promise; - if (opts.resume) { + // When the prompt was issued, so a reap after a Stop can tell a process this turn started + // from one the SESSION started earlier (an stdio MCP server). A resumed turn keeps the + // resume's own start, which only ever makes the reap more conservative. See `reap-exec.ts`. + let promptStartedAtMs = Date.now(); + const approvalTransition = opts.resume ?? opts.settleApprovalsThenPrompt; + if (approvalTransition) { // The resume turn owns continued events; each decision answers one parked gate by id. // Carried gates keep the shared original prompt pending until a later answer. - const decisions = opts.resume.decisions; + const decisions = approvalTransition.decisions; promptPromise = Promise.resolve(decisions[0]?.promptPromise); promptPromise.catch(() => {}); for (const seed of carriedApprovedExecutions) { @@ -1145,7 +1198,7 @@ export async function runTurn( // refresh the carried gates' approval TTL. Pi is exempt on purpose: it prepares the whole // batch before executing any call, so while a carried sibling gate is pending closure is // impossible and the paused-settle's park-and-carry branch owns those spans. - if (opts.resume.carriedForward.length > 0) { + if (opts.resume && opts.resume.carriedForward.length > 0) { if (!plan.isPi) { const answeredAllowedIds = decisions .filter((decision) => decision.reply === "once") @@ -1159,13 +1212,20 @@ export async function runTurn( pause.pause(); } } else { + promptStartedAtMs = Date.now(); promptPromise = Promise.resolve(env.session.prompt(promptBlocks)); promptPromise.catch(() => {}); } - // A user Stop aborts `signal`, which severs the harness fetch (rejecting the prompt). We want a - // clean cancel, not an error: resolve the race to CANCELLED both when the abort event lands first - // AND when the prompt rejection lands first while already aborted, so the outcome is deterministic - // regardless of ordering. A real (non-abort) prompt rejection is re-thrown into the shared catch. + // A user Stop aborts `signal`. That abort does NOT reach the harness: the signal is handed to + // `SandboxAgent.start` for its health wait only, never to the ACP transport or the prompt + // request, so the prompt promise below stays pending and the harness keeps working. (An earlier + // comment here claimed the abort severed the harness fetch. It does not, which is why the + // cancelled branch has to send a real `session/cancel` — see `cancel-turn.ts`.) + // + // So the race is won by the abort event itself. Resolve to CANCELLED both when the abort lands + // first AND when the prompt rejection lands first while already aborted, so the outcome is + // deterministic regardless of ordering. A real (non-abort) prompt rejection is re-thrown into + // the shared catch. const cancelled = new Promise((resolve) => { if (signal?.aborted) resolve(CANCELLED); else @@ -1173,26 +1233,55 @@ export async function runTurn( once: true, }); }); - const raced = await Promise.race([ - promptPromise.then( - (value) => value, - (err) => (signal?.aborted ? CANCELLED : Promise.reject(err)), - ), - pause.signal.then(() => PAUSED), - runLimitTripped.then(() => RUN_LIMIT_TRIPPED), - cancelled, - ]); + const racePrompt = (pending: Promise) => + Promise.race([ + pending.then( + (value) => value, + (err) => (signal?.aborted ? CANCELLED : Promise.reject(err)), + ), + pause.signal.then(() => PAUSED), + runLimitTripped.then(() => RUN_LIMIT_TRIPPED), + cancelled, + ]); + let raced = await racePrompt(promptPromise); + if ( + opts.settleApprovalsThenPrompt && + raced !== PAUSED && + raced !== RUN_LIMIT_TRIPPED && + raced !== CANCELLED && + !pause.active + ) { + // The request ends in a NEW user turn. Finish applying the interaction decision to the old + // prompt first, then make the request's actual work a regular prompt. Without this second + // prompt the runner silently answers the old denied tool call and drops the new text. The + // old prompt was raced above, so a harness that opened another gate after the denial pauses + // this turn instead of hanging unwatched. `continuation` makes promptBlocks the fresh tail. + promptStartedAtMs = Date.now(); + promptPromise = Promise.resolve(env.session.prompt(promptBlocks)); + promptPromise.catch(() => {}); + raced = await racePrompt(promptPromise); + } // A tripped run-limit ends the turn as an error: throw into the shared catch below so the // trace is flushed and the caller's teardown reclaims the (wedged) sandbox. if (raced === RUN_LIMIT_TRIPPED) { throw new Error(runLimitReason ?? "run limit tripped"); } - const stopReason = + let stopReason = raced === CANCELLED ? "cancelled" : raced === PAUSED || pause.active ? "paused" : (raced as any)?.stopReason; + // THE TURN'S OWN WORK IS OVER HERE. Everything below is teardown: draining gates, writing + // the transcript, exporting the trace, deciding whether to park. That takes hundreds of + // milliseconds, and the execution stays registered for all of it, so a Stop arriving now + // would abort a run that has already finished. The abort would change no outcome and would + // still make the teardown treat the run as aborted, which DESTROYS the warm environment + // instead of parking it. Marked here rather than where the caller awaits this function, + // because that window is precisely what lies between the two. + if (stopReason !== "paused" && request.sessionId && request.turnId) { + noteExecutionSettled(request.sessionId, request.turnId); + } // Terminalization drains queued gates, classifies pause-time completions, and gives allowed // executions their original per-call bound before the orphan sweep closes the turn. if (stopReason === "paused") { @@ -1210,11 +1299,19 @@ export async function runTurn( const openAllowedExecutions = openToolCallIds().filter( (id) => pause.isAllowedExecution(id) && !pause.isPausedToolCall(id), ); + // NOT scoped to a resume. Pi batches on the FIRST turn too, and the first turn is where a + // user meets it: the model asks for a Read and a Bash together, the Read answers `allow` + // and the Bash parks, and the allowed Read then never closes because Pi will not execute + // any call in the batch while a sibling gate is open. With `opts.resume` in this predicate + // that turn took the wait below and sat on the 30-minute per-tool-call bound. It never + // parked, never emitted `done`, and its alive watchdog kept beating `running=true`, so + // every durable continuation aimed at the next turn was refused for want of ownership. + // (Browser pass 2026-09-04, sessions d66e2920 at 17:32Z and 6d06f624 at 17:57Z. A healthy + // gated turn shows the Read's `tool_result` BEFORE the Bash gate — sequential, so nothing + // is open at pause time. The two failures show the two gates back to back with no result + // between them, which is the parallel batch.) const piBatchBlockedByApproval = Boolean( - opts.resume && - plan.isPi && - opts.approvalParkMode && - env.parkedApprovals.size > 0, + plan.isPi && opts.approvalParkMode && env.parkedApprovals.size > 0, ); if (piBatchBlockedByApproval) { // Pi prepares every call in a parallel batch before it executes any of them. While a @@ -1260,8 +1357,57 @@ export async function runTurn( unexpectedOpenToolCallIds.join(","), ); } + + if (isUserStopAbort(signal)) { + stopReason = "cancelled"; + } + if (request.sessionId && request.turnId) { + noteExecutionSettled(request.sessionId, request.turnId); + } } if (stopReason === "cancelled") { + env.parkedApprovals.clear(); + env.parkedApproval = undefined; + env.approvalGateCount = 0; + parkedApprovedExecutions.clear(); + // Tell the HARNESS to stop before anything else. The abort only made the runner stop + // waiting; without this the harness still holds an open prompt and a running tool, and the + // sandbox could never be parked. A settled cancel is what earns the warm park below; see + // `cancel-turn.ts`. + const cancel = await cancelHarnessTurn({ + sandbox: env.sandbox, + sessionId: env.session?.id, + promptPromise, + log: logger, + }); + cancelSettled = cancel.settled; + // Codex leaves its shell child running inside the sandbox we are about to park; Pi and + // Claude kill theirs. Reap it here, never in the bridge: the Codex shell is a child of a + // vendored Rust binary the JS bridge holds no pid for, and a bridge patch would ship only + // through a Daytona snapshot rebuild. This cleanup is best effort; the stopped TTL bounds + // leftovers without changing the harness-confirmed park and continuity decision. + if (cancel.settled && plan.acpAgent === "codex") { + let reapError: unknown; + const reap = await reapLeakedExecChildren({ + sandbox: env.sandbox, + sandboxAgentPort: sandboxAgentServerPort(env.sandbox?.sandboxId), + turnElapsedMs: Date.now() - promptStartedAtMs, + log: logger, + }).catch((error) => { + reapError = error; + return undefined; + }); + if (reapResultHasCleanupMiss(reap)) { + logger( + `stage=harness_reap cleanup_miss=true skipped=${reap?.skipped ?? "unknown"}` + + (reapError ? ` error=${String(reapError).slice(0, 120)}` : ""), + ); + } + } + // The harness has been asked to stop, so the Pi trace port and the environment teardown must + // not ask again. Their `destroySession` also aborts `env.mcpAbort`, which belongs to the + // ENVIRONMENT and must survive a park (the approval-park path skips it for the same reason). + if (cancel.requested) env.sessionDestroyRequested = true; // The user Stopped the turn: let any in-flight frames settle, honor real completions that // already arrived, then settle every STILL-open tool call with the interrupt sentinel so the // transcript closes HONESTLY — no orphaned "running" parts, no synthetic success. A deliberate @@ -1374,11 +1520,27 @@ export async function runTurn( return { ok: false, error: swallowedError }; } - // A pause has not finished authoring the turn, so only a completed execution can advance the - // in-memory resume pointer or complete the durable ledger row. + // Which endings are a faithful resume point, and may therefore advance the in-memory resume + // pointer and complete the durable ledger row. + // + // - A completed execution, as it always has been. + // - A user Stop the HARNESS confirmed. `cancelSettled` is the same proof that earns the warm + // park in `shouldPark`: the harness answered the cancelled prompt, so it is idle and its + // native transcript holds a short but FINISHED turn. Nothing more will be written into it. + // + // A stopped turn has to take this path, not just the park, because the park alone is + // process-local. `hydrateHarnessSessionFromDurable` refuses to re-seed the store from a row + // without `end_time`, so a Stop used to leave its row forever incomplete and the session lost + // its native harness session on the next runner restart or pool eviction: the rebuild went + // cold and the conversation survived only as replayed text. + // + // Still dropped, unchanged: a pause has not finished authoring the turn, and an UNSETTLED + // cancel leaves the harness in an unknown state, possibly still writing. Both fall back to + // cold replay, which is the always-correct floor. + const turnIsResumePoint = + stopReason !== "paused" && (stopReason !== "cancelled" || cancelSettled); if ( - stopReason !== "paused" && - stopReason !== "cancelled" && + turnIsResumePoint && env.continuityTurnIndex !== undefined && sessionId ) { @@ -1405,7 +1567,8 @@ export async function runTurn( ).catch(() => {}); } } else if (stopReason === "paused" || stopReason === "cancelled") { - // A pause/cancel stopped mid-turn, after the harness may have written a partial turn natively. + // A pause, or a cancel the harness never confirmed: the turn stopped mid-write, so the + // native transcript may hold a partial turn nobody can describe. invalidateContinuity(sessionId, plan.harness, deps); } @@ -1416,6 +1579,7 @@ export async function runTurn( events: emit ? [] : run.events(), usage, stopReason, + ...(stopReason === "cancelled" ? { cancelSettled } : {}), capabilities: { ...env.capabilities, streamingDeltas: !!emit && env.capabilities.streamingDeltas, @@ -1472,6 +1636,8 @@ export async function runTurn( void settleInBandInteractions?.(); // Release every run-limits timer (idempotent, never re-arms on a late event) on EVERY path. runLimits.dispose(); + // Same contract for the sandbox liveness probe: one timer, released on EVERY path. + sandboxLiveness?.dispose(); // This turn owns its relay: stop it on EVERY exit path (the happy path already stopped it // after the prompt; stop is safe to repeat, matching the old finally). Null it afterwards so // a later `destroy()` — possibly after the dispatch cleared the sink — cannot double-stop or diff --git a/services/runner/src/engines/sandbox_agent/runtime-contracts.ts b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts index 4efc7110120..c3217532990 100644 --- a/services/runner/src/engines/sandbox_agent/runtime-contracts.ts +++ b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts @@ -190,12 +190,18 @@ export interface RunTurnOptions { continuation?: boolean; /** * The session was rehydrated via `session/load` (the patched `resumeSession`), so the harness - * already holds the prior turns natively. Like `continuation`, the prompt is only the new user - * text; `buildTurnText` must not run. Distinct field from `continuation` because the two arrive - * through different acquire paths (live pool checkout vs a fresh cold acquire that loaded an - * old session) — `runTurn` treats them identically for the text-selection decision. + * accepted the prior native session id. This is deliberately weaker than proof that prior turns + * were replayed; `nativeHistoryVerified` supplies that proof. Distinct from `continuation` + * because the two arrive through different acquire paths (live pool checkout vs a fresh cold + * acquire that attempted to load an old session). */ loaded?: boolean; + /** + * The native load produced observable prior-message events. `loaded` alone only proves the + * adapter accepted the requested id; without this proof the reconstructed transcript remains + * authoritative and must be replayed. + */ + nativeHistoryVerified?: boolean; /** * Keep-alive approval park mode: on a parkable ACP permission gate the pause keeps the session * alive (no settle/abort/destroy) so a later resume can answer it. A non-parkable pause (Pi @@ -211,15 +217,25 @@ export interface RunTurnOptions { decisions: ResumeApprovalInput[]; carriedForward: ParkedApproval[]; }; + /** + * Settle the parked gate first, then send this request's fresh user tail as a normal prompt on + * the same warm session. Unlike `resume`, this does not make the old prompt the request's turn: + * its decision is context for the new prompt rather than the turn's terminal interaction. + */ + settleApprovalsThenPrompt?: { + decisions: ResumeApprovalInput[]; + }; } /** * Send only the new user text (not the full cold transcript) when the harness already holds the - * prior turns: a live continuation, or a session rehydrated via `session/load`. `runTurn` calls - * this, so a test that pins it pins the shipped decision. + * prior turns: a live continuation, or a `session/load` that emitted observable prior-message + * events. `runTurn` calls this, so a test that pins it pins the shipped decision. */ export function sendLastMessageOnly(opts: RunTurnOptions): boolean { - return Boolean(opts.continuation || opts.loaded); + return Boolean( + opts.continuation || (opts.loaded && opts.nativeHistoryVerified), + ); } /** @@ -280,6 +296,13 @@ export interface SessionEnvironment { plan: RunPlan; logger: Log; deps: SandboxAgentDeps; + /** + * Set once this environment's sandbox is known to be gone, by the ACP transport that talks to + * it. A remote provider answers for a deleted sandbox instead of refusing the socket, so this + * report is often the only evidence of the death that arrives at all. `run-turn.ts` hands the + * latch to the liveness probe, which is what ends the turn. See `sandbox-gone.ts`. + */ + sandboxGone?: import("./sandbox-gone.ts").SandboxGoneLatch; sandbox: any; session: any; sessionId: string; @@ -311,6 +334,10 @@ export interface SessionEnvironment { projectScopeId?: string; /** This acquire resumed the harness's native session via `session/load` (not cold). */ loadedFromContinuity: boolean; + /** The load emitted at least one prior conversation event, proving native history is present. */ + nativeHistoryVerified: boolean; + /** The native transcript path survives this environment's teardown and a later cold rebuild. */ + nativeHistoryDurable: boolean; /** A remote, session-owned run whose sandbox can be parked (warm) rather than deleted at end. */ resumable: boolean; /** diff --git a/services/runner/src/engines/sandbox_agent/sandbox-gone.ts b/services/runner/src/engines/sandbox_agent/sandbox-gone.ts new file mode 100644 index 00000000000..7e8580461aa --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/sandbox-gone.ts @@ -0,0 +1,148 @@ +/** + * Recognise a provider answer that says THIS SANDBOX no longer exists. + * + * A local sandbox announces its death by refusing the socket: the probe's `fetch` rejects and the + * liveness counter climbs. A REMOTE sandbox never does that. Daytona keeps its proxy host alive + * after the sandbox is deleted and answers every request for it with a normal HTTP error that + * names the sandbox: + * + * 404, `x-daytona-error-code: SANDBOX_NOT_FOUND`, + * "not found: sandbox not found, it may have been deleted or stopped" + * + * The liveness probe reads any HTTP status as alive on purpose (see `sandbox-liveness.ts`), so + * that answer used to mean "still there" and the turn hung until the runner process died. That is + * the blind spot this module closes. The answer is authoritative in a way a status alone is not: + * the provider's own control plane is telling us the machine is gone, so it counts as death on + * the FIRST sighting rather than after the usual three failures. + * + * Recognition is deliberately narrow, because a false positive ends a healthy turn: + * - The answer must be an HTTP ERROR (>= 400). A 200 body that merely quotes this prose, such as + * an agent describing its own earlier failure, is not evidence of anything. + * - Either the provider's own error-code header names the sandbox, or the error body does. A + * bare 404 stays "alive": the daemon's health route may simply not exist on an older image, + * and reading that as death would end healthy turns. + */ + +/** The shape both the probe's `fetch` and the ACP transport's response satisfy. */ +export interface SandboxAnswer { + status: number; + headers: { get(name: string): string | null }; +} + +/** Provider headers that carry a machine-readable error code for the sandbox itself. */ +const GONE_CODE_HEADERS = ["x-daytona-error-code"] as const; + +/** + * Error codes that mean the sandbox is GONE, not that the request was bad and not that the sandbox + * is merely between states. + * + * `SANDBOX_STOPPED` and `SANDBOX_ARCHIVED` are deliberately absent. Both are RESUMABLE states the + * provider itself handles, and the reconnect ladder can legitimately meet either one while it + * brings a parked sandbox back. Reading them as death would end a turn on a sandbox that is about + * to answer. + */ +const GONE_CODE = /^SANDBOX_(NOT_FOUND|DELETED|DESTROYED)$/i; + +/** + * The same verdict in prose, for a proxy that sends no code header. "may have been deleted or + * stopped" is Daytona's own wording for a sandbox it cannot find, so it stays even though a + * `SANDBOX_STOPPED` code does not count. + */ +const GONE_BODY = + /sandbox\s+\S+\s+not found|may have been deleted or stopped|sandbox\s+\S+\s+(?:has been |was )?(?:deleted|destroyed)/i; + +/** + * The reason this answer proves the sandbox is gone, or undefined when it proves nothing. + * + * `bodyText` is optional: the ACP transport must not drain the response body it is about to hand + * to its caller, so it passes headers only. The liveness probe owns its response and passes the + * body too. + */ +export function sandboxGoneReason( + response: SandboxAnswer, + bodyText?: string, +): string | undefined { + if (response.status < 400) return undefined; + for (const header of GONE_CODE_HEADERS) { + const code = response.headers.get(header)?.trim(); + if (code && GONE_CODE.test(code)) { + return `provider reports the sandbox is gone (HTTP ${response.status}, ${header}: ${code})`; + } + } + if (bodyText && GONE_BODY.test(bodyText)) { + return `provider reports the sandbox is gone (HTTP ${response.status}: ${bodyText.slice(0, 200)})`; + } + return undefined; +} + +/** + * A one-way latch shared by everything that talks to one sandbox. + * + * The ACP transport sees the death first — it is the socket carrying the turn — but it has no way + * to end a turn. The liveness probe can end a turn but only wakes every 30 seconds. The latch is + * the seam between them: the transport notes the reason, the probe fires on it at once. First + * reason wins; later notes are ignored, so one death yields one outcome. + */ +export interface SandboxGoneLatch { + /** + * Open the latch. Every `note` before this is DISCARDED. + * + * The latch starts closed because the same fetch that carries a turn also carries the SDK's + * health wait during acquire, and that wait polls a sandbox which is still coming up. A + * provider proxy that lags its own control plane can answer "not found" for a sandbox it has + * not finished re-exposing, which is a normal step of a warm resume rather than a death. The + * latch is one-way, so a report from that window has to be discarded rather than reasoned about + * later. The owner of the environment arms it once the sandbox is acquired. + */ + arm(): void; + /** + * Record that the sandbox is gone. Idempotent; only the first reason after `arm()` is kept, and + * a note before `arm()` is ignored. + */ + note(reason: string): void; + /** The recorded reason, or undefined while the sandbox still answers. */ + reason(): string | undefined; + /** + * Call `listener` when the sandbox is declared gone, or immediately when it already was. At + * most one call per listener. + * + * Returns an unsubscribe function the caller MUST call when its turn ends. A warm environment + * outlives every turn that runs on it, so a turn that leaves its listener behind leaks one dead + * closure per turn and would end up calling a finished turn's `onGone`. + */ + subscribe(listener: (reason: string) => void): () => void; +} + +export function createSandboxGoneLatch(): SandboxGoneLatch { + let armed = false; + let reason: string | undefined; + const listeners = new Set<(reason: string) => void>(); + return { + arm(): void { + armed = true; + }, + note(next: string): void { + if (!armed || reason) return; + reason = next; + for (const listener of listeners) { + try { + listener(next); + } catch { + // A listener fault must not stop the others, nor the request that noticed the death. + } + } + listeners.clear(); + }, + reason: () => reason, + subscribe(listener: (next: string) => void): () => void { + if (reason) { + listener(reason); + return () => {}; + } + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + }; +} diff --git a/services/runner/src/engines/sandbox_agent/sandbox-liveness.ts b/services/runner/src/engines/sandbox_agent/sandbox-liveness.ts new file mode 100644 index 00000000000..7d364381e8d --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/sandbox-liveness.ts @@ -0,0 +1,281 @@ +/** + * Detect that the sandbox died UNDER a running turn, so the turn ends instead of hanging. + * + * The runner talks to the sandbox agent over ACP, a JSON-RPC channel whose agent-to-client half + * is a long-lived SSE `GET`. When the sandbox process disappears that stream is severed, but the + * transport's read loop swallows the error and never fails the readable, so the pending + * `session/prompt` request is structurally incapable of settling. The turn then holds its + * sandbox, its mount and its slot forever, while the alive watchdog keeps telling the platform + * `running=true` every 30 seconds. That is issue #6418. + * + * The existing run limits do not cover it. Time-to-first-byte (2 min) catches a sandbox that + * dies before the first token, and idle (30 min) catches one that dies mid-stream — but + * `notePaused()` retires every one of them for good the moment the turn parks for a human, and a + * sandbox that dies during a pause therefore has no deadline at all. + * + * So probe the sandbox directly, over its own HTTP surface, which is a different socket from the + * wedged ACP channel: it answers while the sandbox lives and refuses once it is gone. + * `failureThreshold` consecutive failures — not one — is what separates a dead sandbox from a + * slow network, and each probe carries its own timeout because a vanished host can hang a + * request rather than refuse it. + * + * What counts as alive is deliberately weak: ANY HTTP response, including 401 or 404. The + * question is whether something is listening, not whether we are authorised or whether the + * route exists, and only a transport failure answers that with certainty. + * + * The ONE exception is an answer that names the SANDBOX as gone, which `sandbox-gone.ts` + * recognises. Behind a remote provider's proxy the transport failure never arrives: Daytona keeps + * the proxy host up after the sandbox is deleted and answers "sandbox not found" for it + * indefinitely, so the weak rule alone read a dead sandbox as alive and the turn hung until the + * runner process died. That answer is the provider's own verdict rather than a network symptom, + * so it ends the turn on the FIRST sighting instead of after three failures. + * + * `goneSignal` is the other half of the same fix. The ACP transport carrying the turn sees that + * answer seconds before any poll can, and it cannot end a turn on its own, so it notes the death + * on a shared latch and this probe fires on the latch at once. + * + * NOTE on what NOT to probe: `SandboxAgent.getSession()` looks like a liveness check and is not + * one. It reads the local persist driver and never touches the daemon, so it answers happily + * while the sandbox is dead — verified live on 2026-09-02, where a killed daemon logged + * `ECONNREFUSED` on the ACP socket while every `getSession` succeeded. + * + * The probe deliberately keeps running while the turn is paused. A pause is a legitimate wait for + * a human; it is not a reason to stop noticing that the machine underneath is gone. + */ + +import { envInt, envTimerMs } from "../../env.ts"; +import { SANDBOX_GONE_MARKER } from "./errors.ts"; +import { sandboxGoneReason, type SandboxGoneLatch } from "./sandbox-gone.ts"; + +export const PROBE_INTERVAL_ENV = "AGENTA_RUNNER_SANDBOX_PROBE_INTERVAL_MS"; +export const PROBE_TIMEOUT_ENV = "AGENTA_RUNNER_SANDBOX_PROBE_TIMEOUT_MS"; +export const PROBE_FAILURES_ENV = "AGENTA_RUNNER_SANDBOX_PROBE_FAILURES"; +export const PROBE_DISABLED_ENV = "AGENTA_RUNNER_SANDBOX_PROBE_DISABLED"; + +// One probe per heartbeat interval. Anything faster buys latency the user cannot perceive and +// costs a request per sandbox per tick. +export const DEFAULT_PROBE_INTERVAL_MS = 30_000; +// A live daemon answers a session read in milliseconds; ten seconds is a generous ceiling that +// still bounds a hung request well inside one interval. +export const DEFAULT_PROBE_TIMEOUT_MS = 10_000; +// Three consecutive failures, so a single dropped request or a brief network stall is not a +// death sentence. At the defaults that is about 90 seconds before a turn is ended. +export const DEFAULT_PROBE_FAILURES = 3; + +export interface SandboxLivenessLimits { + intervalMs: number; + timeoutMs: number; + failureThreshold: number; +} + +export interface Clock { + setTimeout(fn: () => void, ms: number): NodeJS.Timeout; + clearTimeout(handle: NodeJS.Timeout): void; +} + +const realClock: Clock = { + setTimeout: (fn, ms) => setTimeout(fn, ms), + clearTimeout: (handle) => clearTimeout(handle), +}; + +/** Read the probe's limits from env, with wide defaults. */ +export function resolveSandboxLivenessLimits( + log: (message: string) => void = () => {}, +): SandboxLivenessLimits { + return { + intervalMs: envTimerMs(PROBE_INTERVAL_ENV, DEFAULT_PROBE_INTERVAL_MS, { + log, + }), + timeoutMs: envTimerMs(PROBE_TIMEOUT_ENV, DEFAULT_PROBE_TIMEOUT_MS, { log }), + failureThreshold: envInt(PROBE_FAILURES_ENV, DEFAULT_PROBE_FAILURES, { + min: 1, + log, + }), + }; +} + +/** + * The daemon's health URL, derived from the only public handle on the agent that carries its + * base address. `inspectorUrl` is `/ui/`; the health route is `/v1/health`. + * + * Returns undefined when the agent exposes no usable URL, which disables the probe rather than + * guessing — a probe pointed at the wrong host would end healthy turns. + */ +export function sandboxHealthUrl(sandbox: unknown): string | undefined { + const inspector = (sandbox as { inspectorUrl?: unknown } | undefined) + ?.inspectorUrl; + if (typeof inspector !== "string" || !inspector) return undefined; + const base = inspector.replace(/\/ui\/?$/, "").replace(/\/+$/, ""); + if (!/^https?:\/\//.test(base)) return undefined; + return `${base}/v1/health`; +} + +/** + * A failure the provider itself confirmed: the sandbox is gone, so waiting for two more probes + * would only delay an outcome that is already certain. + */ +export class SandboxGoneError extends Error { + constructor(reason: string) { + super(reason); + this.name = "SandboxGoneError"; + } +} + +/** + * The default probe: one unauthenticated GET at the daemon's health route. + * + * Resolves on any HTTP status, except one whose headers or body name the sandbox as gone — that + * rejects with {@link SandboxGoneError}. Otherwise it rejects only when the request never became a + * response, which is what "nothing is listening any more" looks like from here. + * + * The body is read only for an HTTP error, so a healthy answer costs nothing extra and a 200 that + * happens to quote the provider's prose can never be misread as death. + */ +export function httpLivenessProbe(url: string): () => Promise { + return async () => { + const response = await fetch(url, { method: "GET" }); + const bodyText = + response.status >= 400 + ? await response.text().catch(() => "") + : undefined; + const reason = sandboxGoneReason(response, bodyText); + if (reason) throw new SandboxGoneError(reason); + return response.status; + }; +} + +export interface SandboxLivenessHandle { + /** Release the probe's timer. Always call this once the turn ends, on every path. */ + dispose(): void; + /** Consecutive failures observed so far; for tests and diagnostics. */ + failures(): number; +} + +export interface SandboxLivenessOptions { + /** + * One liveness check. Resolves when the sandbox answered, rejects or hangs when it did not. + * + * Optional: a sandbox that exposes no health URL still gets the `goneSignal` route, which needs + * no polling at all. + */ + probe?: () => Promise; + limits: SandboxLivenessLimits; + /** Called at most once, with a human-readable reason, when the sandbox is declared gone. */ + onGone: (reason: string) => void; + /** + * The latch the turn's ACP transport writes to when a response names the sandbox as gone. It + * ends the turn on the spot, without waiting for the next probe interval. + */ + goneSignal?: SandboxGoneLatch; + clock?: Clock; + log?: (message: string) => void; +} + +/** + * Start probing. Returns immediately; the first probe runs one interval later, because a turn + * that just acquired its environment has already proved the sandbox was up. + */ +export function startSandboxLivenessProbe({ + probe, + limits, + onGone, + goneSignal, + clock = realClock, + log = () => {}, +}: SandboxLivenessOptions): SandboxLivenessHandle { + let disposed = false; + let fired = false; + let inFlight = false; + let failures = 0; + let timer: NodeJS.Timeout | undefined; + + /** Declare the sandbox gone, at most once for the life of this handle. */ + const fire = (reason: string): void => { + if (fired || disposed) return; + fired = true; + if (timer) clock.clearTimeout(timer); + timer = undefined; + log(`[sandbox-liveness] ${reason}`); + onGone(reason); + }; + + const schedule = (): void => { + if (disposed || fired) return; + timer = clock.setTimeout(() => void tick(), limits.intervalMs); + }; + + const withTimeout = async (): Promise => { + let timeoutHandle: NodeJS.Timeout | undefined; + try { + await Promise.race([ + probe!(), + new Promise((_resolve, reject) => { + timeoutHandle = clock.setTimeout( + () => + reject(new Error(`probe timed out after ${limits.timeoutMs}ms`)), + limits.timeoutMs, + ); + }), + ]); + } finally { + if (timeoutHandle) clock.clearTimeout(timeoutHandle); + } + }; + + const tick = async (): Promise => { + // A probe still running when the next tick lands means the sandbox is not answering; let + // the in-flight one reach its own timeout rather than stacking requests on a dead host. + if (disposed || fired || inFlight) { + schedule(); + return; + } + inFlight = true; + try { + await withTimeout(); + failures = 0; + } catch (err) { + failures += 1; + const detail = err instanceof Error ? err.message : String(err); + // The provider answering "that sandbox does not exist" is a verdict, not a symptom, so it + // needs no corroboration from two more probes. + if (err instanceof SandboxGoneError) { + log(`[sandbox-liveness] probe failed (definitive): ${detail}`); + fire(`${SANDBOX_GONE_MARKER}: ${detail}`); + return; + } + log( + `[sandbox-liveness] probe failed (${failures}/${limits.failureThreshold}): ${detail}`, + ); + if (failures >= limits.failureThreshold) { + fire( + `${SANDBOX_GONE_MARKER}: ${failures} consecutive liveness probes failed ` + + `(last: ${detail})`, + ); + return; + } + } finally { + inFlight = false; + } + schedule(); + }; + + // The transport's report is not a poll, so `PROBE_DISABLED_ENV` does not silence it: that switch + // exists to stop the runner making a request per sandbox per tick, not to make the runner ignore + // a death it was told about. The latch belongs to the ENVIRONMENT, which outlives this turn on a + // warm sandbox, so `dispose` must hand the listener back or every turn leaves one behind. + const unsubscribeGone = goneSignal?.subscribe((reason) => { + fire(`${SANDBOX_GONE_MARKER}: ${reason}`); + }); + + if (probe && !process.env[PROBE_DISABLED_ENV]) schedule(); + + return { + dispose() { + disposed = true; + if (timer) clock.clearTimeout(timer); + timer = undefined; + unsubscribeGone?.(); + }, + failures: () => failures, + }; +} diff --git a/services/runner/src/engines/sandbox_agent/session-identity.ts b/services/runner/src/engines/sandbox_agent/session-identity.ts index 50897e1fad3..1d9011b10e9 100644 --- a/services/runner/src/engines/sandbox_agent/session-identity.ts +++ b/services/runner/src/engines/sandbox_agent/session-identity.ts @@ -26,6 +26,19 @@ export interface KeepaliveConfig { enabled: boolean; ttlMs: number; approvalTtlMs: number; + /** + * The idle window for a session PARKED BY A USER STOP. + * + * Defaults to 600 s for both providers, matching the local approval window because both waits + * begin when a human is about to act. This deliberately differs from the ordinary 60 s local + * and 120 s Daytona idle windows. The trade-off is that a stopped Daytona sandbox can remain + * billed for up to ten minutes. Override with AGENTA_RUNNER_SESSION_STOPPED_TTL_MS. + * + * Optional so a hand-built config (every test fixture) keeps meaning what it always meant: + * omitted reads as "same as the idle window". `readKeepaliveConfig`, the only production + * source, always sets it. + */ + stoppedTtlMs?: number; poolMax: number; } @@ -34,6 +47,7 @@ export type KeepaliveProviderName = "local" | "daytona"; const KEEPALIVE_ENV = "AGENTA_RUNNER_SESSION_KEEPALIVE"; const TTL_ENV = "AGENTA_RUNNER_SESSION_TTL_MS"; const APPROVAL_TTL_ENV = "AGENTA_RUNNER_SESSION_APPROVAL_TTL_MS"; +const STOPPED_TTL_ENV = "AGENTA_RUNNER_SESSION_STOPPED_TTL_MS"; const POOL_MAX_ENV = "AGENTA_RUNNER_SESSION_POOL_MAX"; const DEFAULT_TTL_MS = 60_000; @@ -46,6 +60,7 @@ const DEFAULT_TTL_MS = 60_000; // (never fails the turn), and an awaiting_approval entry keeps holding a pool slot — override // via AGENTA_RUNNER_SESSION_APPROVAL_TTL_MS if warm slots are contended. const DEFAULT_APPROVAL_TTL_MS = 600_000; +const DEFAULT_STOPPED_TTL_MS = 600_000; const DEFAULT_POOL_MAX = 8; const DAYTONA_TTL_ENV = "AGENTA_RUNNER_DAYTONA_SESSION_IDLE_TTL_MS"; const DAYTONA_POOL_MAX_ENV = "AGENTA_RUNNER_DAYTONA_SESSION_MAX_WARM"; @@ -97,6 +112,9 @@ export function readKeepaliveConfig( // pool never sees an awaiting_approval park for Daytona today because parkedApproval is // only set by ACP gates. approvalTtlMs: ttlMs, + // A stopped Daytona session is deliberately held for the same human-response window as a + // local one, even though the sandbox remains billed. Zero remains a valid operator override. + stoppedTtlMs: nonNegativeIntEnv(STOPPED_TTL_ENV, DEFAULT_STOPPED_TTL_MS), // This budgets billed compute (idle warm sandboxes), deliberately separate from the local // pool's host-memory budget; Slice 4 adds the strict warm-slot accounting semantics. poolMax: positiveIntEnv(DAYTONA_POOL_MAX_ENV, DEFAULT_DAYTONA_POOL_MAX), @@ -106,6 +124,8 @@ export function readKeepaliveConfig( enabled: boolEnv(KEEPALIVE_ENV, true), ttlMs: positiveIntEnv(TTL_ENV, DEFAULT_TTL_MS), approvalTtlMs: positiveIntEnv(APPROVAL_TTL_ENV, DEFAULT_APPROVAL_TTL_MS), + // A settled Stop gets the same ten-minute human-response window as a pending approval. + stoppedTtlMs: positiveIntEnv(STOPPED_TTL_ENV, DEFAULT_STOPPED_TTL_MS), poolMax: positiveIntEnv(POOL_MAX_ENV, DEFAULT_POOL_MAX), }; } @@ -504,15 +524,34 @@ export function approvalDecisionForToolCall( toolCallId: string, ): "allow" | "deny" | undefined { if (!toolCallId) return undefined; - for (const message of request.messages ?? []) { + const messages = request.messages ?? []; + if (messages.length === 0) return undefined; + + // A pure interaction reply carries its decision at the request tail. A fresh user turn can + // carry a rewritten `output-denied` tool part in its history; only the LAST assistant message + // is relevant there. Scanning the whole transcript lets an older denial bind to a newer gate + // that reused the id and incorrectly diverts the new user text into approval-resume. + let message: ChatMessage | undefined; + if (!tailIsFreshUserMessage(request)) { + message = messages[messages.length - 1]; + } else { + for (let i = messages.length - 2; i >= 0; i--) { + if (messages[i]?.role === "assistant") { + message = messages[i]; + break; + } + } + } + if (message) { const content = message?.content; - if (!Array.isArray(content)) continue; - for (const block of content) { - if (block?.type !== "tool_result" || block.toolCallId !== toolCallId) { - continue; + if (Array.isArray(content)) { + for (const block of content) { + if (block?.type !== "tool_result" || block.toolCallId !== toolCallId) { + continue; + } + const decision = approvalDecisionOf(block); + if (decision !== undefined) return decision; } - const decision = approvalDecisionOf(block); - if (decision !== undefined) return decision; } } return undefined; diff --git a/services/runner/src/engines/sandbox_agent/teardown.ts b/services/runner/src/engines/sandbox_agent/teardown.ts index 456c691e250..7a4ce69e977 100644 --- a/services/runner/src/engines/sandbox_agent/teardown.ts +++ b/services/runner/src/engines/sandbox_agent/teardown.ts @@ -31,6 +31,8 @@ export type TeardownReason = | "kill" | "failed-turn" | "aborted" + /** A user Stop whose harness cancel SETTLED. The daemon is idle and sound, so park it. */ + | "cancelled" /** @deprecated Name the failing layer instead. Kept so an unclassified call site fails safe. */ | "compatibility-mismatch" | "session-incompatible" @@ -61,6 +63,10 @@ const PARKABLE_REASONS: ReadonlySet = new Set([ "idle-expiry", "capacity-eviction", "shutdown-idle", + // A settled Stop. The harness answered its cancelled prompt, so nothing inside the daemon is + // mid-flight and nothing baked into it is stale. An UNSETTLED Stop never reaches this reason: + // it stays `aborted`, which deletes. + "cancelled", // The two incompatibilities whose daemon is still sound. See the module comment. "session-incompatible", "continuity-invalid", diff --git a/services/runner/src/environment/abortable-sandbox-provider.ts b/services/runner/src/environment/abortable-sandbox-provider.ts new file mode 100644 index 00000000000..f083aaa40f1 --- /dev/null +++ b/services/runner/src/environment/abortable-sandbox-provider.ts @@ -0,0 +1,109 @@ +import type { SandboxProvider } from "sandbox-agent"; + +import { waitForAcquire } from "./acquire-abort.ts"; + +type ProviderMethod = (...args: any[]) => Promise; + +async function cleanupCreatedSandbox( + provider: SandboxProvider, + sandboxId: string, + log: (message: string) => void, +): Promise { + try { + await provider.destroy(sandboxId); + log(`cancelled acquire cleaned late-created sandbox=${sandboxId}`); + } catch (error) { + log( + `cancelled acquire cleanup failed sandbox=${sandboxId}: ${String( + error instanceof Error ? error.message : error, + ).slice(0, 160)}`, + ); + } +} + +async function cleanupReconnectedSandbox( + provider: SandboxProvider, + sandboxId: string, + log: (message: string) => void, +): Promise { + try { + if (provider.pause) await provider.pause(sandboxId); + else await provider.destroy(sandboxId); + log(`cancelled acquire cleaned late-reconnected sandbox=${sandboxId}`); + } catch (error) { + log( + `cancelled reconnect cleanup failed sandbox=${sandboxId}: ${String( + error instanceof Error ? error.message : error, + ).slice(0, 160)}`, + ); + } +} + +/** + * Make the provider-owned part of `SandboxAgent.start` observe the turn signal. + * + * `sandbox-agent` forwards its signal only to the client health wait; provider `create()` and + * `reconnect()` have no signal parameter. This proxy races those calls without changing provider + * identity or hiding provider-specific methods. A fresh sandbox that appears after cancellation + * is deleted; a late reconnect is returned to its parked state when the provider supports pause. + */ +export function abortableSandboxProvider( + provider: T, + signal: AbortSignal | undefined, + log: (message: string) => void, +): T { + if (!signal) return provider; + + return new Proxy(provider, { + get(target, property, receiver) { + const value = Reflect.get(target, property, receiver); + if (typeof value !== "function") return value; + + if (property === "create") { + return (...args: unknown[]) => + waitForAcquire( + () => Reflect.apply(value as ProviderMethod, target, args), + signal, + { + onLateSuccess: (sandboxId: string) => + cleanupCreatedSandbox(target, sandboxId, log), + }, + ); + } + + if (property === "reconnect") { + return (sandboxId: string, ...args: unknown[]) => + waitForAcquire( + () => + Reflect.apply(value as ProviderMethod, target, [ + sandboxId, + ...args, + ]), + signal, + { + onLateSuccess: () => + cleanupReconnectedSandbox(target, sandboxId, log), + onLateFailure: () => + cleanupReconnectedSandbox(target, sandboxId, log), + }, + ); + } + + // These calls happen after a raw sandbox id exists. `SandboxAgent.start` owns compensation + // if one is cancelled, so they need only become promptly abortable here. + if ( + property === "ensureServer" || + property === "getUrl" || + property === "getFetch" + ) { + return (...args: unknown[]) => + waitForAcquire( + () => Reflect.apply(value as ProviderMethod, target, args), + signal, + ); + } + + return value.bind(target); + }, + }); +} diff --git a/services/runner/src/environment/acquire-abort.ts b/services/runner/src/environment/acquire-abort.ts new file mode 100644 index 00000000000..cb6b2b691cf --- /dev/null +++ b/services/runner/src/environment/acquire-abort.ts @@ -0,0 +1,96 @@ +/** + * Cancellation helpers for environment acquisition. + * + * A user Stop can arrive while a provider or mount call is still pending. Waiting for that call + * before observing the signal makes the control delivery time out. Racing without compensating + * cleanup is worse: a provider may finish creating a sandbox after the turn has already ended. + * These helpers provide the shared race and the late-success cleanup hook used by those stages. + */ + +/** The stable error shape returned when acquisition is interrupted by its turn signal. */ +export class AcquireAbortedError extends Error { + constructor() { + super("Sandbox acquisition was aborted."); + this.name = "AbortError"; + } +} + +export function throwIfAcquireAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new AcquireAbortedError(); +} + +export interface AbortableAcquireHooks { + /** Cleanup for a resource that materialized after the caller already observed cancellation. */ + onLateSuccess?: (value: T) => void | Promise; + /** Cleanup for a known resource whose operation failed after cancellation. */ + onLateFailure?: (error: unknown) => void | Promise; +} + +function runLateHook( + hook: ((value: T) => void | Promise) | undefined, + value: T, +): void { + if (!hook) return; + void Promise.resolve() + .then(() => hook(value)) + .catch(() => {}); +} + +/** + * Start one acquire operation and reject as soon as `signal` aborts. The underlying operation is + * not assumed to support AbortSignal, so a resource that resolves later is handed to the cleanup + * hook instead of being leaked or published to the cancelled caller. + */ +export function waitForAcquire( + start: () => Promise, + signal?: AbortSignal, + hooks: AbortableAcquireHooks = {}, +): Promise { + if (!signal) return start(); + throwIfAcquireAborted(signal); + + return new Promise((resolve, reject) => { + let cancelled = false; + let settled = false; + const onAbort = () => { + if (settled || cancelled) return; + cancelled = true; + reject(new AcquireAbortedError()); + }; + signal.addEventListener("abort", onAbort, { once: true }); + + let operation: Promise; + try { + operation = start(); + } catch (error) { + settled = true; + signal.removeEventListener("abort", onAbort); + reject(error); + return; + } + + operation.then( + (value) => { + settled = true; + signal.removeEventListener("abort", onAbort); + if (cancelled) { + runLateHook(hooks.onLateSuccess, value); + return; + } + resolve(value); + }, + (error) => { + settled = true; + signal.removeEventListener("abort", onAbort); + if (cancelled) { + runLateHook(hooks.onLateFailure, error); + return; + } + reject(error); + }, + ); + + // Cover an abort that raced the listener registration and operation start. + if (signal.aborted) onAbort(); + }); +} diff --git a/services/runner/src/environment/harness-session-lifecycle.ts b/services/runner/src/environment/harness-session-lifecycle.ts index 24525e6f463..605f23db7f4 100644 --- a/services/runner/src/environment/harness-session-lifecycle.ts +++ b/services/runner/src/environment/harness-session-lifecycle.ts @@ -59,7 +59,14 @@ export interface OpenSessionInput { createSession: (request: unknown) => Promise<{ id: string }>; }; /** The session persist driver. Typed loosely so this unit does not restate the SDK's record. */ - persist: { updateSession: (record: never) => Promise }; + persist: { + updateSession: (record: never) => Promise; + listEvents?: (request: { + sessionId: string; + cursor?: string; + limit?: number; + }) => Promise<{ items: unknown[]; nextCursor?: string }>; + }; acpAgent: string; harness: string; cwd: string; @@ -67,6 +74,8 @@ export interface OpenSessionInput { sessionInit: Record; /** The native session id to resume, when the store says one is eligible. */ priorAgentSessionId: string | undefined; + /** Whether the native transcript path is backed by durable storage for this acquire. */ + nativeHistoryDurable: boolean; /** The runner-local key both modes use for the persist record. */ localSessionId: string | undefined; /** For the continuity log line only. */ @@ -87,9 +96,56 @@ export interface OpenSessionResult { * reopen may claim continuity; this unit reports what it can observe and no more. */ loadedFromContinuity: boolean; + /** Whether `session/load` emitted prior conversation content, not merely accepted the id. */ + nativeHistoryVerified: boolean; mode: "load" | "create"; } +const HISTORY_SESSION_UPDATES = new Set([ + "user_message_chunk", + "agent_message_chunk", + "agent_thought_chunk", + "tool_call", + "tool_call_update", +]); + +/** + * `sandbox-agent` persists every ACP envelope observed while `session/load` runs. A real native + * replay therefore leaves at least one conversation update behind; an adapter that merely accepts + * the id leaves none. This is the positive proof the id comparison cannot provide. + */ +async function loadedHistoryWasObserved( + persist: OpenSessionInput["persist"], + localSessionId: string, + eventCountBeforeLoad: number, +): Promise { + if (!persist.listEvents) return false; + const items: unknown[] = []; + let cursor: string | undefined; + do { + const page = await persist.listEvents({ + sessionId: localSessionId, + cursor, + limit: 100, + }); + items.push(...page.items); + cursor = page.nextCursor; + } while (cursor); + return items.slice(eventCountBeforeLoad).some((item) => { + const event = item as { + sender?: unknown; + payload?: { method?: unknown; params?: { update?: { sessionUpdate?: unknown } } }; + }; + return ( + event.sender === "agent" && + event.payload?.method === "session/update" && + HISTORY_SESSION_UPDATES.has( + String(event.payload.params?.update?.sessionUpdate ?? ""), + ) + ); + }); +} + /** * Open the harness session: load the native conversation when one is eligible, otherwise create * a fresh one. @@ -102,6 +158,7 @@ export async function openSession( ): Promise { let session: { id: string; agentSessionId?: string } | undefined; let loadedFromContinuity = false; + let nativeHistoryVerified = false; if (input.priorAgentSessionId && input.localSessionId) { await input.persist.updateSession({ @@ -114,12 +171,55 @@ export async function openSession( } as never); const createSessionStartedAt = Date.now(); try { + let eventCountBeforeLoad: number | undefined; + if (input.nativeHistoryDurable && input.persist.listEvents) { + try { + const page = await input.persist.listEvents({ + sessionId: input.localSessionId, + limit: 100, + }); + eventCountBeforeLoad = page.items.length; + let cursor = page.nextCursor; + while (cursor) { + const next = await input.persist.listEvents({ + sessionId: input.localSessionId, + cursor, + limit: 100, + }); + eventCountBeforeLoad += next.items.length; + cursor = next.nextCursor; + } + } catch (err) { + input.log( + `[continuity] native history baseline failed: ${conciseError(err, input.harness)}`, + ); + } + } session = await input.sandbox.resumeSession(input.localSessionId); loadedFromContinuity = session.agentSessionId === input.priorAgentSessionId; + if ( + loadedFromContinuity && + input.nativeHistoryDurable && + eventCountBeforeLoad !== undefined + ) { + try { + nativeHistoryVerified = await loadedHistoryWasObserved( + input.persist, + input.localSessionId, + eventCountBeforeLoad, + ); + } catch (err) { + input.log( + `[continuity] native history verification failed: ${conciseError(err, input.harness)}`, + ); + } + } input.log( `[continuity] session/load attempted session=${input.continuitySessionKey} ` + - `harness=${input.harness} loaded=${loadedFromContinuity}`, + `harness=${input.harness} loaded=${loadedFromContinuity} ` + + `historyDurable=${input.nativeHistoryDurable} ` + + `historyVerified=${nativeHistoryVerified}`, ); } catch (err) { input.log( @@ -143,10 +243,20 @@ export async function openSession( } finally { input.timingLog("create_session", createSessionStartedAt, " mode=create"); } - return { session, loadedFromContinuity, mode: "create" }; + return { + session, + loadedFromContinuity, + nativeHistoryVerified, + mode: "create", + }; } - return { session, loadedFromContinuity, mode: "load" }; + return { + session, + loadedFromContinuity, + nativeHistoryVerified, + mode: "load", + }; } /** @@ -222,6 +332,7 @@ export type ReopenResult = ok: true; session: { id: string; agentSessionId?: string }; loadedFromContinuity: boolean; + nativeHistoryVerified: boolean; } | { ok: false; reason: "history-unverifiable" | "reopen-failed" }; @@ -251,6 +362,7 @@ export async function reopen(input: ReopenInput): Promise { ok: true, session: opened.session, loadedFromContinuity: opened.loadedFromContinuity, + nativeHistoryVerified: opened.nativeHistoryVerified, }; } catch (err) { input.log(`reopen failed: ${conciseError(err, input.harness)}`); diff --git a/services/runner/src/environment/mount-lifecycle.ts b/services/runner/src/environment/mount-lifecycle.ts index 493317e8fc6..be45fce5745 100644 --- a/services/runner/src/environment/mount-lifecycle.ts +++ b/services/runner/src/environment/mount-lifecycle.ts @@ -62,6 +62,7 @@ import { writeSystemPromptLocal, } from "../engines/sandbox_agent/pi-assets.ts"; import { containsTransportEndpointDisconnected } from "../engines/sandbox_agent/runtime-policy.ts"; +import { throwIfAcquireAborted } from "./acquire-abort.ts"; import { rethrowIfInvariant, type AcquireContext } from "./acquire-context.ts"; /** The Pi agent directory inside a Daytona sandbox. Injected so this unit stays import-light. */ @@ -77,6 +78,8 @@ export interface MountDeps { ) => Promise; /** The remote Pi directory constant, passed in rather than imported. */ daytonaPiDir: string; + /** The turn signal that must preempt a mount during environment acquisition. */ + signal?: AbortSignal; } /** @@ -203,10 +206,11 @@ export async function mountLocalDurableCwd( const mounted = await (deps.mountStorage ?? mountStorage)( plan.workspace.cwd, creds, - { log: ctx.log }, + { log: ctx.log, signal: deps.signal }, ); if (mounted) { ctx.commitLocalMount("cwd", plan.workspace.cwd, creds); + throwIfAcquireAborted(deps.signal); // Session-local links belong to the mount's lifecycle, not to first acquire: this mount is // object storage, which has no symlinks, so a remount hands back a 0-byte file where the link // was. Re-materialize the subscription Codex login link here, AFTER the mount is live @@ -220,6 +224,7 @@ export async function mountLocalDurableCwd( } return true; } + throwIfAcquireAborted(deps.signal); // A false result means mountStorage stopped the attempt and CONFIRMED the path detached. ctx.markCwdDetachConfirmed(); return false; @@ -240,6 +245,7 @@ export async function mountLocalAgentCwd( if ( !(await (deps.mountStorage ?? mountStorage)(mountPath, creds, { log: ctx.log, + signal: deps.signal, })) ) { // false means mountStorage confirmed detach is safe. This path is a sibling of the session @@ -248,6 +254,7 @@ export async function mountLocalAgentCwd( return false; } ctx.commitLocalMount("agent", mountPath, creds); + throwIfAcquireAborted(deps.signal); await seedAgentReadme(mountPath, { log: ctx.log }); await linkAgentFiles(plan.workspace.cwd, mountPath, { log: ctx.log }); await activateAgentMountGuidance(ctx, deps); diff --git a/services/runner/src/lifecycle/session-coordinator.ts b/services/runner/src/lifecycle/session-coordinator.ts index 4ab7ca810d4..05622589062 100644 --- a/services/runner/src/lifecycle/session-coordinator.ts +++ b/services/runner/src/lifecycle/session-coordinator.ts @@ -40,6 +40,7 @@ import { type SessionEnvironment, } from "../engines/sandbox_agent.ts"; import type { MountCredentials } from "../engines/sandbox_agent/mount.ts"; +import { SESSION_TURN_IN_USE_MESSAGE } from "../sessions/admission.ts"; import { teardownDisposition, type TeardownReason, @@ -192,6 +193,14 @@ export interface KeepaliveContext { clientGone?: () => boolean; /** Latest session credential accessor supplied by the alive watchdog. */ credential?: () => string; + /** + * Called once with this run's project scope, as soon as it is known. + * + * The scope can only be resolved here: `runContext.project.id` is empty on the live invoke + * path, so the project comes from the signed mount, which is signed inside this function. The + * transport needs it to route a control command to the right tenant's session. + */ + onScopeResolved?: (projectId: string) => void; /** * Test seam for the credential-propagation hold. Production waits for real: the hold is what * keeps applied state from advancing over a value the provider's egress layer has probably not @@ -292,6 +301,10 @@ export async function runWithKeepalive( } const key = scope.key; klog(`scope=${scope.source} key=${key} session=${sessionId}`); + // Tell the transport which project this run belongs to. Until this lands, a control command + // cannot tell one tenant's session from another's, because the request itself often carries + // no project and the scope was only just derived from the signed mount. + ctx.onScopeResolved?.(scope.key.slice(0, scope.key.lastIndexOf(":"))); // The mount may be null here (store unconfigured, 503, ephemeral fallback) or undefined (the // sign attempt threw) when the run-context scope produced the key. A mount-less session still @@ -557,6 +570,14 @@ export async function runWithKeepalive( } }; + /** + * The idle window a clean park gets. A user Stop gets the longer stopped window, because the + * user is about to type the next message; every other clean turn gets the ordinary one. See + * `KeepaliveConfig.stoppedTtlMs` for how to collapse the two. + */ + const parkTtlMs = (stopped: boolean): number => + stopped ? (config.stoppedTtlMs ?? config.ttlMs) : config.ttlMs; + const resultTeardownReason = (result: AgentRunResult): TeardownReason => shouldPark(result, signal, clientGone) ? "clean-resumable" @@ -768,7 +789,12 @@ export async function runWithKeepalive( watchParkedPrompt(env); } } else if (shouldPark(result, signal, clientGone)) { - if (!(await seat(config.ttlMs, "idle"))) { + // A settled user Stop parks like any clean turn, but on the LONGER stopped window: the + // user is about to type. Logged so the live evidence shows the sandbox surviving a Stop + // rather than a `no-park:cancelled` eviction. + const stopped = result.stopReason === "cancelled"; + if (stopped) klog(`park-cancelled key=${key} ttl=${parkTtlMs(stopped)}ms`); + if (!(await seat(parkTtlMs(stopped), "idle"))) { await drop("park-refused", "clean-resumable"); } else { await notifyParkedLive(env); @@ -826,7 +852,9 @@ export async function runWithKeepalive( watchParkedPrompt(env); } } else if (shouldPark(result, signal, clientGone)) { - if (!(await pool.repark(live, update, config.ttlMs))) { + const stopped = result.stopReason === "cancelled"; + if (stopped) klog(`park-cancelled key=${key} ttl=${parkTtlMs(stopped)}ms`); + if (!(await pool.repark(live, update, parkTtlMs(stopped)))) { await live.teardown("failed-turn"); } else { await notifyParkedLive(env); @@ -891,6 +919,7 @@ export async function runWithKeepalive( result = await engine.runTurn(env, request, trackedEmit, signal, { approvalParkMode: true, loaded: env.loadedFromContinuity, + nativeHistoryVerified: env.nativeHistoryVerified, ...turnCredential, }); } catch (err) { @@ -1156,6 +1185,7 @@ export async function runWithKeepalive( const parkedList = [...existing.environment.parkedApprovals.values()]; const resumeDecisions: ResumeApprovalInput[] = []; const carriedForward: ParkedApproval[] = []; + const freshUserTail = tailIsFreshUserMessage(request); let mismatch: string | undefined; if (parkedList.length === 0) { mismatch = "no-parked-gate"; @@ -1205,7 +1235,14 @@ export async function runWithKeepalive( // session; the history check only guards a client that DID assert a transcript. const clientAssertsHistory = !carriesApprovalReplyOnly(request); if (!mismatch) { - if (clientAssertsHistory && priorFp !== existing.historyFingerprint) { + if (freshUserTail && carriedForward.length > 0) { + // A new prompt cannot start while any old gate still holds the harness's original prompt. + // Only a complete decision set can settle that prompt and keep this environment warm. + mismatch = "fresh-prompt-unanswered-gate"; + } else if ( + clientAssertsHistory && + priorFp !== existing.historyFingerprint + ) { mismatch = "history"; } else if (mountCredentialsExpired(existing.credentialEpoch)) { mismatch = "credentials-expired"; @@ -1261,21 +1298,25 @@ export async function runWithKeepalive( const live = pool.checkoutApproval(key); if (live) { - shadowRoute(existing, "reuse", "approval-resume"); + const decisionRoute = freshUserTail + ? "approval-decision-then-prompt" + : "approval-resume"; + shadowRoute(existing, "reuse", decisionRoute); const approveCount = resumeDecisions.filter( (d) => d.reply === "once", ).length; const rejectCount = resumeDecisions.length - approveCount; klog( - `resume key=${key} gates=${parkedList.length} answered=${resumeDecisions.length} ` + + `${freshUserTail ? "decision-then-prompt" : "resume"} key=${key} ` + + `gates=${parkedList.length} answered=${resumeDecisions.length} ` + `carried=${carriedForward.length} ` + `approve=${approveCount} reject=${rejectCount} tool=${parked?.toolName ?? "?"}`, ); let result: AgentRunResult; try { - // Answer the parked gate on the SAME live session; the original prompt continues and this - // (new) turn owns streaming + tracing. The gated tool runs with its original byte-exact - // args — no model re-issues anything, so argument drift/task restart cannot happen. + // A pure decision resumes the original prompt. A decision followed by fresh user text + // settles that gate first and then sends the text as a normal continuation prompt on the + // same warm session; the decision becomes context instead of swallowing the new turn. result = await engine.runTurn( live.environment, request, @@ -1283,7 +1324,14 @@ export async function runWithKeepalive( signal, { approvalParkMode: true, - resume: { decisions: resumeDecisions, carriedForward }, + ...(freshUserTail + ? { + continuation: true, + settleApprovalsThenPrompt: { decisions: resumeDecisions }, + } + : { + resume: { decisions: resumeDecisions, carriedForward }, + }), ...turnCredential, }, ); @@ -1316,12 +1364,29 @@ export async function runWithKeepalive( return result; } // checkout lost a race; fall through to cold. + } else if (existing && existing.state === "busy") { + // A LIVE turn is streaming on this environment right now, in this process. Refuse; never + // destroy it. + // + // This branch used to `evict` and cold-start ("supersede-busy"), which is the second half of + // the double-send bug (#6417, #5539, #5538): a second message on a running session tore the + // sandbox out from under the first turn, so both turns died and the session stayed locked + // until the 30-minute lease expired. Admission (`sessions/admission.ts`, decided by the API's + // atomic `nx` acquire on the turn's first heartbeat) now refuses the second turn at the edge, + // so in normal operation nothing reaches here at all. + // + // What still reaches here is the fail-open window: the heartbeat fails open on a network or + // HTTP error, so an API blip can admit two turns. Local state is the more specific truth in + // that window — a busy entry means a turn is demonstrably in flight on this box — so this is + // the backstop that keeps the invariant true when the arbiter is unreachable. Only a + // `checkoutIdle` continuation and a freshly `reserve`d cold turn leave a busy entry; + // `checkoutApproval` REMOVES its session, so an in-flight approval resume is never found here. + klog(`refuse (busy) key=${key}; another turn owns this session`); + return { ok: false, error: SESSION_TURN_IN_USE_MESSAGE }; } else if (existing) { - // Busy / destroyed: two turns racing one session. Only a checkoutIdle continuation leaves a - // busy entry in the map (checkoutApproval REMOVES its session, so an in-flight approval - // resume can never be found — a duplicate approval misses the pool and runs cold, and its - // environment can never be destroyed by this branch). Supersede — destroy the parked one and - // cold-start — awaited so its teardown cannot overlap our acquire. + // `destroyed`: a dead entry left by a drain (`destroyAll`) or a teardown that has already + // run. Nothing is in flight on it, so clearing the key and cold-starting is correct and + // costs nothing warm. klog(`evict (supersede-${existing.state}) key=${key}; cold`); await pool.evict(key, `supersede-${existing.state}`, "failed-turn"); } else { diff --git a/services/runner/src/protocol.ts b/services/runner/src/protocol.ts index 4e13fcba146..53d4e78d04a 100644 --- a/services/runner/src/protocol.ts +++ b/services/runner/src/protocol.ts @@ -465,6 +465,18 @@ export type AgentEvent = total?: number; cost?: number; } + /** + * This turn's ADMITTED execution id, emitted once at the start of a session-owned run. + * + * The runner mints the turn id per execution (`resolveTurnId`), so before this the browser had + * no way to learn it: the client's `start` frame is built and sent before the runner replies at + * all. Without the id no first-party client can name the execution it means to act on, which is + * why `expected_execution_id` on the public Cancel has never had a caller that could fill it. + * + * Emitted LIVE only, never through the persisting emitter: it is transport correlation, not + * conversation, and it must not become a record in the session's history. + */ + | { type: "turn"; turnId: string } | { type: "error"; message: string; @@ -795,6 +807,12 @@ export interface AgentRunResult { usage?: AgentUsage; /** Why the turn ended (harness-reported when available). */ stopReason?: string; + /** + * Only on `stopReason: "cancelled"`. True when the harness was told to stop AND confirmed it + * stopped inside the settle budget, which is what lets the sandbox be parked warm instead of + * deleted. Absent or false means the harness never confirmed, so the environment is destroyed. + */ + cancelSettled?: boolean; /** What the harness was probed to support this run. */ capabilities?: HarnessCapabilities; sessionId?: string; diff --git a/services/runner/src/server.ts b/services/runner/src/server.ts index 56fc89d5cd8..794548d600a 100644 --- a/services/runner/src/server.ts +++ b/services/runner/src/server.ts @@ -8,6 +8,7 @@ * GET /subscription-status -> one login state per harness (no paths, no credentials) * POST /stream -> body is an AgentRunRequest, NDJSON event stream (alias: POST /run) * POST /kill -> best-effort, idempotent teardown, scoped to one { sessionId, projectId } + * POST /cancel -> stop the CURRENT TURN of one session and keep it warm * * Uses Node's built-in http server (no framework dependency). * @@ -16,6 +17,10 @@ */ import { apiBase, runWithRequestApiBase } from "./apiBase.ts"; import { loadDurableDecisions } from "./sessions/interactions.ts"; +import { + isUserStopAbort, + USER_STOP_ABORT_REASON, +} from "./sessions/stop-signal.ts"; import { randomUUID, timingSafeEqual } from "node:crypto"; import { createServer, @@ -46,6 +51,10 @@ import { type ParkedApproval, type SessionEnvironment, } from "./engines/sandbox_agent.ts"; +import { + cancelHarnessTurn, + resolveCancelSettleMs, +} from "./engines/sandbox_agent/cancel-turn.ts"; import { isMounted, type MountCredentials, @@ -54,6 +63,7 @@ import type { TeardownReason } from "./engines/sandbox_agent/teardown.ts"; import { approvalDecisionForToolCall, poolKeyFor, + projectScopeFor, readKeepaliveConfig, tailIsFreshUserMessage, type KeepaliveConfig, @@ -76,7 +86,34 @@ import { import { applyDaytonaSdkEnv } from "./engines/sandbox_agent/daytona-provider.ts"; import { isEntrypoint } from "./entry.ts"; import { insecureEgressAllowed } from "./tools/ssrf-guard.ts"; -import { startAliveWatchdog } from "./sessions/alive.ts"; +import { + SESSION_TURN_IN_USE_CODE, + SESSION_TURN_IN_USE_MESSAGE, +} from "./sessions/admission.ts"; +import { + REPLICA_ID, + releaseOwnedSessions, + startAliveWatchdog, +} from "./sessions/alive.ts"; +import { + applyCommand, + holdsSession, + type ControlCommand, + type ParkedSessionControl, +} from "./sessions/control-channel.ts"; +import { + noteExecutionProject, + registerExecution, + unregisterExecution, +} from "./sessions/execution-registry.ts"; +import { + awaitTurnOrAbandon, + resolveTurnSettleLimits, +} from "./sessions/turn-settle.ts"; +import { + ABANDONED_TURN_MARKER, + type RunErrorCode, +} from "./engines/sandbox_agent/errors.ts"; import { buildWorkflowReferenceList, cancelStaleInteractions, @@ -285,6 +322,7 @@ const realKeepaliveEngine: KeepaliveEngine = { try { result = await runTurn(acquired.env, request, emit, signal, { loaded: acquired.env.loadedFromContinuity, + nativeHistoryVerified: acquired.env.nativeHistoryVerified, ...(credential ? { credential } : {}), seededDecisions: await loadDurableDecisions( acquired.env.sessionId, @@ -349,6 +387,15 @@ const runAgent: RunAgent = (request, emit, signal, options) => { config, clientGone: options?.clientGone, credential: options?.credential, + // The coordinator is the first place that knows this run's project, because the scope can + // come from the signed mount rather than the request. A control command needs it to tell + // one tenant's session from another's. + onScopeResolved: (projectId) => { + const sessionId = request.sessionId?.trim(); + const turnId = request.turnId?.trim(); + if (sessionId && turnId) + noteExecutionProject(sessionId, turnId, projectId); + }, }); }; @@ -449,6 +496,13 @@ async function runAndStreamWithApiBaseResolved( // runs abort on disconnect (original behavior: caller drives, disconnect = cancel). const controller = new AbortController(); let clientDisconnected = false; + // Resolves when the platform tells us this turn is no longer current — a Stop, a takeover, + // or the API's own execution watchdog having declared the turn lost. `awaitTurnOrAbandon` + // uses it to stop waiting on a run that may never return. See `sessions/turn-settle.ts`. + let markInterrupted: ((reason: string) => void) | undefined; + const interrupted = new Promise((resolve) => { + markInterrupted = resolve; + }); if (!sessionOwned) { // Listen on the response, not the request: the request body is already fully read, so // its `close` can fire early on a keep-alive connection. `res` `close` fires when the @@ -485,8 +539,20 @@ async function runAndStreamWithApiBaseResolved( // For session-owned runs: wrap the live emitter so every event is also persisted // producer-side, independent of whether the client is still connected. let emitFn: EmitEvent = liveEmit; + // Closed once this request has written the turn's terminal outcome. An abandoned run may + // still unwind minutes later and emit its own `error`/`done` through the same emitter; the + // turn already has an ending, and a second one would put two endings in one transcript. + let turnClosed = false; + const gatedEmit: EmitEvent = (event) => { + if (turnClosed) return; + emitFn(event); + }; let flushPersist: (() => Promise) | undefined; - let persistError: ((message: string) => void) | undefined; + let persistError: + | ((message: string, code?: RunErrorCode) => void) + | undefined; + let persistTerminal: ((stopReason?: string) => void) | undefined; + let terminalRecordEmitted = false; let aliveWatchdog: | { release: () => Promise; @@ -494,97 +560,226 @@ async function runAndStreamWithApiBaseResolved( } | undefined; - if (sessionOwned) { - // The request's api base (if any) is already scoped for this call via - // runWithRequestApiBase in the outer runAndStream — apiBase() below sees it. - // The runner authenticates session calls AS the invoke caller (the run credential), - // refreshing it for the turn's lifetime — never the admin key. Project scope is - // resolved server-side from the credential, so no project_id rides the request. - // - // onInterrupted (W7.4): a cancel/steer/kill against this session (via - // `POST /sessions/streams/` or the runner's own `/kill`) drops this turn's alive lock. - // The next heartbeat surfaces that as `is_current_turn: false`; wiring it to - // `controller.abort()` is what makes the control-plane signal actually reach this - // in-flight run — before this, a session-owned run's controller was never aborted. - // Awaited (WP3) so the first heartbeat's stream_id is ready before the turn starts. - // - // The beat also proposes the two things a headless session otherwise never gets: a name - // (no browser ever renders it, and the browser is the only other title writer) and the - // run's workflow references (they ride only a fire-and-forget turn append today, so a - // dropped append leaves a row the UI cannot open). Both are fill-once server-side. - const watchdog = await startAliveWatchdog( - sessionId, - turnId, - platformCredentialForRequest(request), - () => controller.abort(), - { - name: proposeSessionName(request), - references: buildWorkflowReferenceList(request.runContext?.workflow), - }, - ); - aliveWatchdog = watchdog; - // The heartbeat response already carries the session_streams row id — free, no extra - // round-trip. Thread it onto the request so the engine's turn-append write has it. - request.streamId = watchdog.streamId(); - // A new turn supersedes any prior turn's unanswered gate: cancel stale pending - // interactions (sparing this turn's own, plus a parked gate this turn answers in-band — - // the resume resolves that one). Best-effort, never blocks the turn. - const answeredTokens = inBandAnswerTokens(request); - void cancelStaleInteractions( - sessionId, - turnId, - answeredTokens, - watchdog.credential, - ); - // Deny-set from THIS run's typed credential material (model connection credentials + - // materialized environment values + MCP connection credentials) and the run credential — - // not process env, which never holds them. A credential value a model echoes back must - // never reach the durable session records unredacted. - const { - emit: persistingEmit, - persist, - flush, - } = buildPersistingEmitter( - sessionId, - watchdog.credential, - liveEmit, - seedForRun(request), - turnId, - request.runContext?.trace?.span_id, - ); - // Record the inbound user turn first so the session record is the full conversation, not just - // agent output. Guard on `tailIsFreshUserMessage`: an approval RESUME's tail is the tool_result - // envelope, so it must not re-persist the ORIGINAL prompt as a duplicate user row. The guard - // writes the prompt only on the turn that first introduced it. - if (tailIsFreshUserMessage(request)) { - persist( - { type: "message", text: turn.text, attachments: turn.attachments }, - "user", + try { + if (sessionOwned) { + // The request's api base (if any) is already scoped for this call via + // runWithRequestApiBase in the outer runAndStream — apiBase() below sees it. + // The runner authenticates session calls AS the invoke caller (the run credential), + // refreshing it for the turn's lifetime — never the admin key. Project scope is + // resolved server-side from the credential, so no project_id rides the request. + // + // onInterrupted (W7.4): a cancel/steer/kill against this session (via + // `POST /sessions/streams/` or the runner's own `/kill`) drops this turn's alive lock. + // The next heartbeat surfaces that as `is_current_turn: false`; wiring it to + // `controller.abort()` is what makes the control-plane signal actually reach this + // in-flight run — before this, a session-owned run's controller was never aborted. + // Awaited (WP3) so the first heartbeat's stream_id is ready before the turn starts. + // + // The beat also proposes the two things a headless session otherwise never gets: a name + // (no browser ever renders it, and the browser is the only other title writer) and the + // run's workflow references (they ride only a fire-and-forget turn append today, so a + // dropped append leaves a row the UI cannot open). Both are fill-once server-side. + const watchdog = await startAliveWatchdog( + sessionId, + turnId, + platformCredentialForRequest(request), + () => { + markInterrupted?.( + "the platform reported this turn is no longer current (stopped, taken over, or " + + "declared lost)", + ); + // LABELLED, not a bare abort: `shouldPark` parks only an abort it can prove was a + // cooperative Stop. See `sessions/stop-signal.ts`. + controller.abort(USER_STOP_ABORT_REASON); + }, + { + name: proposeSessionName(request), + references: buildWorkflowReferenceList(request.runContext?.workflow), + }, ); - if (turn.attachments.length > 0) { - // A failed claim is accepted as graceful loss: the worst case is that the sweeper - // reclaims the attachment and cold replay renders it as no longer available. - await claimAttachments( - sessionId, - turn.attachments.map((attachment) => attachment.attachmentId), - watchdog.credential, + aliveWatchdog = watchdog; + // The heartbeat response already carries the session_streams row id — free, no extra + // round-trip. Thread it onto the request so the engine's turn-append write has it. + request.streamId = watchdog.streamId(); + + // ADMISSION. That first beat asked the platform's atomic `nx` acquire whether this turn may + // run, and `admitted: false` means a DIFFERENT turn already holds the session. Stop here. + // + // Everything below this point has a side effect that a refused turn must not have: + // `cancelStaleInteractions` would cancel the LIVE turn's unanswered approval gate, the + // persisting emitter would write this message into the durable transcript, and `run()` would + // reach the keepalive pool and destroy the live turn's warm environment. That last one is + // the double-send bug (#6417, #5539, #5538): the arbiter's answer was already correct, the + // runner simply never read it before acting. + // + // The refusal travels as an `error` EVENT with a stable code plus a failed terminal result, + // which is the path every runner failure already takes to the browser. Nothing is persisted, + // so the refused message never appears in the session's history — the client keeps the text. + if (!watchdog.admitted) { + process.stderr.write( + `[sessions] admission REFUSED session=${sessionId} turn=${turnId}; ` + + `another turn owns this session. No pool resolve, no eviction.\n`, ); + // Stops the heartbeat interval and releases the credential lease. Its final + // `is_running: false` beat is owner-scoped server-side, so it cannot clear the live + // turn's `running` lock or stamp its own turn id on the session row. + await watchdog.release().catch(() => {}); + unregisterExecution(sessionId, turnId); + liveEmit({ + type: "error", + message: SESSION_TURN_IN_USE_MESSAGE, + code: SESSION_TURN_IN_USE_CODE, + }); + writeRecord({ + kind: "result", + result: { ok: false, error: SESSION_TURN_IN_USE_MESSAGE, events: [] }, + }); + res.end(); + return; } + + // A refused contender must never replace the admitted execution's Stop handle. + registerExecution({ + projectId: projectScopeFor(request, undefined)?.id, + sessionId, + turnId, + startedAt: Date.now(), + abort: () => controller.abort(USER_STOP_ABORT_REASON), + }); + + // Admitted. Tell the client which execution it is watching, before anything else streams. + // + // The runner mints the turn id (`resolveTurnId`), and until now it never told anyone: the + // client's `start` frame is built and sent before the runner replies at all, so it cannot + // carry a runner-minted id. That is why `expected_execution_id` on the public Cancel has had + // no first-party caller able to fill it — a Stop could only mean "whatever is running now", + // never "the turn I was watching". This is the earliest frame that can carry it. + // + // Deliberately on `liveEmit`, not the persisting emitter that replaces it below: this is + // transport correlation, not conversation, and it must never become a session record. + liveEmit({ type: "turn", turnId }); + + // A new turn supersedes any prior turn's unanswered gate: cancel stale pending + // interactions (sparing this turn's own, plus a parked gate this turn answers in-band — + // the resume resolves that one). Best-effort, never blocks the turn. + const answeredTokens = inBandAnswerTokens(request); + void cancelStaleInteractions( + sessionId, + turnId, + answeredTokens, + watchdog.credential, + ); + // Deny-set from THIS run's typed credential material (model connection credentials + + // materialized environment values + MCP connection credentials) and the run credential — + // not process env, which never holds them. A credential value a model echoes back must + // never reach the durable session records unredacted. + const { + emit: persistingEmit, + persist, + flush, + } = buildPersistingEmitter( + sessionId, + watchdog.credential, + liveEmit, + seedForRun(request), + turnId, + request.runContext?.trace?.span_id, + ); + // Record the inbound user turn first so the session record is the full conversation, not just + // agent output. Guard on `tailIsFreshUserMessage`: an approval RESUME's tail is the tool_result + // envelope, so it must not re-persist the ORIGINAL prompt as a duplicate user row. The guard + // writes the prompt only on the turn that first introduced it. + if (tailIsFreshUserMessage(request)) { + persist( + { type: "message", text: turn.text, attachments: turn.attachments }, + "user", + ); + if (turn.attachments.length > 0) { + // A failed claim is accepted as graceful loss: the worst case is that the sweeper + // reclaims the attachment and cold replay renders it as no longer available. + await claimAttachments( + sessionId, + turn.attachments.map((attachment) => attachment.attachmentId), + watchdog.credential, + ); + } + } + emitFn = (event) => { + if (event.type === "done") terminalRecordEmitted = true; + persistingEmit(event); + }; + flushPersist = flush; + persistError = (message, code) => + persist({ type: "error", message, ...(code ? { code } : {}) }, "agent"); + persistTerminal = (stopReason) => { + terminalRecordEmitted = true; + persist( + { + type: "done", + ...(stopReason === "cancelled" ? { stopReason } : {}), + }, + "agent", + ); + }; } - emitFn = persistingEmit; - flushPersist = flush; - persistError = (message) => persist({ type: "error", message }, "agent"); + } catch (error) { + if (aliveWatchdog) await aliveWatchdog.release().catch(() => {}); + if (sessionOwned) unregisterExecution(sessionId, turnId); + throw error; } let result: AgentRunResult; try { - result = await run(request, emitFn, controller.signal, { - clientGone: () => clientDisconnected, - credential: aliveWatchdog?.credential, + // Not a bare `await run(...)`: an await inside the run that never settles would keep this + // function parked forever, and with it the terminal record below AND the alive watchdog's + // release in the `finally` — the turn would announce `running=true` every 30s for good. + // `awaitTurnOrAbandon` returns either the run's own result or a reason to write one + // without it, so this request always produces exactly one terminal outcome. + const outcome = await awaitTurnOrAbandon({ + run: run(request, gatedEmit, controller.signal, { + clientGone: () => clientDisconnected, + credential: aliveWatchdog?.credential, + }), + abort: () => controller.abort(), + interrupted: sessionOwned ? interrupted : undefined, + limits: resolveTurnSettleLimits((message) => + process.stderr.write(`${message}\n`), + ), + log: (message) => process.stderr.write(`${message}\n`), }); - // A failed engine run ({ok:false}) already emitted its own error EVENT through the - // persisting emitter, so no extra persist here (it would duplicate the record). Drain - // all queued persists before the sandbox tears down. + if (outcome.settled) { + result = outcome.value; + // `runTurn` normally emits `done` itself. Acquisition can fail before `runTurn` starts, + // though, and a cooperative Stop during a cold sandbox create reaches exactly that path. + // Close any failed run that emitted no terminal record; preserve the Stop marker when the + // labelled control-plane abort caused it. A genuine acquire failure never reached runTurn's + // error emitter, so preserve its error before the done backstop instead of making the empty + // turn look successful. Both records use the same ordered persistence chain as runTurn's + // emitter but stay off the live stream, whose result envelope is unchanged. + if ( + !terminalRecordEmitted && + persistTerminal && + (!result.ok || isUserStopAbort(controller.signal)) + ) { + const userStopped = isUserStopAbort(controller.signal); + if (!userStopped && !result.ok && persistError) { + persistError(result.error ?? "Agent run failed."); + } + persistTerminal(userStopped ? "cancelled" : undefined); + } + } else { + // The run is still pending and may never settle. Give the turn the ending the runner + // owes it, and let the abandoned run keep its own teardown if it ever unwinds. + turnClosed = true; + const message = `${ABANDONED_TURN_MARKER}: ${outcome.reason}`; + process.stderr.write( + `[sessions] ABANDONED session=${sessionId ?? "-"} turn=${turnId ?? "-"}: ${outcome.reason}\n`, + ); + if (persistError) persistError(message, "execution_lost"); + result = { ok: false, error: message }; + } + // Drain the terminal backstop or abandonment marker and all prior persists before the + // sandbox tears down. if (flushPersist) await flushPersist(); } catch (err) { const message = err instanceof Error ? err.message : String(err); @@ -599,6 +794,9 @@ async function runAndStreamWithApiBaseResolved( // A throw escaping run() itself (outside the engine's own try/catch) emitted no error // event — persist it here as the backstop. if (persistError) persistError(message); + if (!terminalRecordEmitted && persistTerminal) { + persistTerminal(isUserStopAbort(controller.signal) ? "cancelled" : undefined); + } if (flushPersist) await flushPersist().catch(() => {}); result = { ok: false, error: message }; } finally { @@ -616,6 +814,10 @@ async function runAndStreamWithApiBaseResolved( } } if (aliveWatchdog) await aliveWatchdog.release().catch(() => {}); + // Same `finally` as the watchdog release, so a run that threw still leaves the registry + // clean. Scoped to this turn id, so a turn that finishes after its successor registered + // cannot unregister the successor. + if (sessionOwned) unregisterExecution(sessionId, turnId); } // Streaming delivered the events live, so don't echo them in the terminal record. @@ -682,6 +884,131 @@ function readBodyCapped( }); } +/** `/cancel`'s payload is five short strings. */ +const CANCEL_BODY_MAX_BYTES = 16 * 1024; + +/** A non-empty trimmed string, or null. Used for every id `/cancel` reads. */ +function readRequiredId(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed ? trimmed : null; +} + +/** + * Does the keep-alive pool hold this session parked awaiting an approval? + * + * A Stop against a parked approval has no entry in the execution registry, because no turn is + * running. Without this lookup the runner would answer 404 for exactly the case that has no + * control channel at all today: a parked session stops heartbeating, so the only existing Stop + * signal never reaches it. + */ +function parkedSessionControl( + projectId: string, + sessionId: string, +): ParkedSessionControl | undefined { + const key = `${projectId}:${sessionId}`; + for (const provider of Object.keys( + keepalivePools, + ) as KeepaliveProviderName[]) { + const pool = keepalivePools[provider]; + const parked = pool.get(key); + if (!parked || parked.state !== "awaiting_approval") continue; + return { + stop: async () => { + // Checkout makes the transition exclusive: a racing request cannot consume the same + // permission gate while Stop is releasing it. + const live = pool.checkoutApproval(key); + if (!live) throw new Error("parked approval was already checked out"); + await stopParkedApprovalSession({ + environment: live.environment, + repark: () => + pool.repark( + live, + { + historyFingerprint: live.historyFingerprint, + historyAsserted: live.historyAsserted, + credentialEpoch: live.credentialEpoch, + }, + keepaliveConfigs[provider].stoppedTtlMs ?? + keepaliveConfigs[provider].ttlMs, + ), + teardown: () => + pool.evictIfCurrent( + live, + "stop-approval-failed", + "failed-turn", + ), + }); + }, + }; + } + return undefined; +} + +interface StopParkedApprovalSessionInput { + environment: SessionEnvironment; + repark: () => Promise; + teardown: () => Promise; + /** Test seams; production uses the operator-configured bound and a real timer. */ + cancelSettleMs?: number; + wait?: (ms: number) => Promise; +} + +/** Reject and cancel a parked prompt before exposing its environment as idle again. */ +export async function stopParkedApprovalSession( + input: StopParkedApprovalSessionInput, +): Promise { + const env = input.environment; + const gates = [...env.parkedApprovals.values()]; + try { + await Promise.all( + gates.map((gate) => + env.session.respondPermission(gate.permissionId, "reject"), + ), + ); + const cancel = await cancelHarnessTurn({ + sandbox: env.sandbox, + sessionId: env.session?.id, + promptPromise: gates[0]?.promptPromise, + timeoutMs: input.cancelSettleMs ?? resolveCancelSettleMs(), + log: env.logger, + wait: input.wait, + }); + if (cancel.requested) env.sessionDestroyRequested = true; + if (!cancel.settled && cancel.requested) { + // The ACP cancel WAS sent but the harness did not confirm it inside the budget. The prompt + // may still be open, so fail closed rather than present a possibly-running turn as idle. + throw new Error("parked approval harness cancel did not settle"); + } + if (!cancel.settled) { + // No ACP cancel could be SENT: a local runtime whose sandbox client has no `cancelSession` + // (`stage=harness_cancel sent=false reason=client-has-no-cancelSession`). The reject above + // is still the stop signal for a parked approval, which runs no turn, and a Stop must NEVER + // evict the warm sandbox. So repark it warm, exactly as the reject-then-repark path did + // before the ACP cancel was added, instead of tearing it down and reporting a failed Stop. + env.logger( + "stage=parked_stop reject-only (client has no cancelSession); reparking warm", + ); + } + + env.parkedApprovals.clear(); + env.parkedApproval = undefined; + env.parkedApprovedExecutions?.clear(); + env.approvalGateCount = 0; + env.nonParkablePauseCount = 0; + env.commitAuthorization = undefined; + env.clearTurn(); + if (!(await input.repark())) { + throw new Error("released approval could not return to the pool"); + } + } catch (error) { + // A partly released or unsettled prompt is not safe to present as idle. Fail closed through + // the normal teardown path; applyCommand reports the failed outcome. + await input.teardown(); + throw error; + } +} + /** Build the HTTP request listener around a given engine runner (the testable seam). */ export function createRequestListener( run: RunAgent, @@ -751,6 +1078,78 @@ export function createRequestListener( return send(res, 200, { ok: true }); } + if (req.method === "POST" && req.url === "/cancel") { + if (!isAuthorized(req)) { + return send(res, 401, { ok: false, error: "Unauthorized" }); + } + // Stop the CURRENT TURN and keep the session warm. This is not `/kill`: the sandbox, + // the native harness session and the keep-alive pool entry all survive, and the next + // message continues the same conversation. + // + // The response is an ACKNOWLEDGEMENT, not an outcome. What happened to the execution + // goes to the API's outcome route, so settlement has one path on every transport. + let cancelBody: { + commandId?: unknown; + projectId?: unknown; + sessionId?: unknown; + targetTurnId?: unknown; + createdAt?: unknown; + }; + try { + const raw = await readBodyCapped(req, CANCEL_BODY_MAX_BYTES); + cancelBody = raw.trim() ? JSON.parse(raw) : {}; + } catch (err) { + if (err instanceof BodyTooLargeError) { + return send(res, 413, { ok: false, error: err.message }); + } + return send(res, 400, { + ok: false, + error: `Invalid JSON: ${err instanceof Error ? err.message : String(err)}`, + }); + } + const commandId = readRequiredId(cancelBody.commandId); + const cancelSessionId = readRequiredId(cancelBody.sessionId); + const cancelProjectId = readRequiredId(cancelBody.projectId); + if (!commandId || !cancelSessionId || !cancelProjectId) { + return send(res, 400, { + ok: false, + error: + "commandId, sessionId and projectId are all required: a pool key is always project-scoped", + }); + } + const command: ControlCommand = { + id: commandId, + projectId: cancelProjectId, + sessionId: cancelSessionId, + kind: "cancel", + target: { + turnId: readRequiredId(cancelBody.targetTurnId), + expectedTurnId: null, + }, + createdAt: + typeof cancelBody.createdAt === "string" + ? cancelBody.createdAt + : "", + }; + if ( + !holdsSession( + cancelProjectId, + cancelSessionId, + parkedSessionControl, + ) + ) { + // 404 is ambiguous on purpose and the API disambiguates it: a `not_held` for a + // session whose row is alive and beating means the call reached the wrong replica. + return send(res, 404, { ok: false, error: "session not held here" }); + } + // Answer before the outcome. The applier reports it separately, and a Stop that takes + // seconds to settle must not hold this request open. + void applyCommand(command, { + isParked: parkedSessionControl, + }).catch(() => {}); + return send(res, 202, { ok: true, replicaId: REPLICA_ID }); + } + // POST /stream is the productized name; /run is kept as a back-compat alias // for one release (the SDK still posts /run). Both share the handler. if ( @@ -889,6 +1288,14 @@ if (isEntrypoint(import.meta.url)) { ), ); await destroyInFlightSandboxes(timeoutMs, "shutdown-in-flight"); + // LAST, and only after the sandboxes are gone: hand back the `owner:session:` + // affinity keys this replica holds. Nothing else releases them, and `claim_owner` never + // steals, so without this the replacement replica is refused every message on those + // sessions for the rest of the 120-second lease. It runs last because a session whose + // sandbox is still being destroyed should not yet look free to another replica, and it + // is bounded so it can never hold the process past the SIGTERM grace period. A SIGKILL + // reaches no handler at all; the lease stays the fallback for that. + await releaseOwnedSessions(timeoutMs); }, }); diff --git a/services/runner/src/sessions/admission.ts b/services/runner/src/sessions/admission.ts new file mode 100644 index 00000000000..e711af76484 --- /dev/null +++ b/services/runner/src/sessions/admission.ts @@ -0,0 +1,28 @@ +/** + * Single-turn admission: at most one execution runs per session, decided in one place. + * + * The decision is NOT made here. It is made by the platform API's atomic `nx` acquire of the + * `alive` Redis lock, which the runner asks for on a turn's first heartbeat + * (`sessions/alive.ts` -> `POST /sessions/streams/heartbeat` -> + * `api/oss/src/core/sessions/streams/service.py`). This module holds only what the runner needs + * to REPORT that decision: the stable code and the one line the user reads. + * + * Why the runner has to stop rather than continue: before this, a second turn that lost the + * acquire still walked into the keepalive pool, found the first turn's environment busy, and + * destroyed it (`lifecycle/session-coordinator.ts`, the old `supersede-busy` branch). Both turns + * then died and the session stayed locked under a dead turn's lease. Refusing at the edge is what + * makes the first turn survive. + */ + +import type { RunErrorCode } from "../engines/sandbox_agent/errors.ts"; + +/** Stable class for a refused turn. Never a display string. */ +export const SESSION_TURN_IN_USE_CODE: RunErrorCode = "session_turn_in_use"; + +/** + * Product copy. The reader is the person in the chat, so it says what happened to THEIR message + * and what to do next, with no lock, turn, or session-id mechanics. It must stay ONE line: the + * SDK's `sanitize_runner_error` keeps only the first line of a runner error. + */ +export const SESSION_TURN_IN_USE_MESSAGE = + "This session is already running a turn. Your message was not sent. Wait for the reply, or stop the turn, then send again."; diff --git a/services/runner/src/sessions/alive.ts b/services/runner/src/sessions/alive.ts index 292ee3d825e..eeeef93baf3 100644 --- a/services/runner/src/sessions/alive.ts +++ b/services/runner/src/sessions/alive.ts @@ -15,13 +15,28 @@ * Key contract constants mirror `sessions/contract.ts`; do not duplicate them. */ +import { envTimerMs } from "../env.ts"; import { apiBase } from "../apiBase.ts"; import { randomUUID } from "node:crypto"; -import { HEARTBEAT_INTERVAL_SECONDS } from "./contract.ts"; +import { HEARTBEAT_INTERVAL_SECONDS, OWNER_TTL_SECONDS } from "./contract.ts"; const REFRESH_INTERVAL_MS = HEARTBEAT_INTERVAL_SECONDS * 1000; +export const HEARTBEAT_TIMEOUT_ENV = "AGENTA_RUNNER_HEARTBEAT_TIMEOUT_MS"; +/** + * A beat that never answers must not outlive its interval. + * + * The beat used a bare `fetch` with no signal, so a stalled socket never settled: beats piled + * up behind it, and the final `is_running: false` beat in `release()` could hold the request + * open after the turn had already ended. Half an interval keeps at most one beat in flight. + */ +export const DEFAULT_HEARTBEAT_TIMEOUT_MS = Math.floor(REFRESH_INTERVAL_MS / 2); + +function heartbeatTimeoutMs(): number { + return envTimerMs(HEARTBEAT_TIMEOUT_ENV, DEFAULT_HEARTBEAT_TIMEOUT_MS); +} + /** * This runner container's stable id, minted once per process. An orchestrator can inject a * meaningful id (pod/container name) via `AGENTA_RUNNER_REPLICA_ID`; otherwise a random @@ -50,6 +65,67 @@ function log(msg: string): void { process.stderr.write(`[sessions/alive] ${msg}\n`); } +// --- owner-claim registry -------------------------------------------------- // +// +// WHY THIS EXISTS. `owner:session:` is claimed by every beat and released by nothing, and +// the API's `claim_owner` deliberately never steals from a live owner. So a runner that exits +// while holding claims leaves each of those sessions unusable by the replacement replica until +// the lease expires — measured at 112 to 123 s against a 120 s TTL, on every restart. The +// registry is the smallest thing that makes the shutdown handler able to hand them back: which +// sessions this process claimed, and a credential that can still speak for each one. +// +// The credential is the run's own ephemeral platform token, the same one every beat already +// carries; it never leaves this process and is never logged. An entry that outlives its token +// simply fails its release call and falls back to the lease, exactly as a killed runner does. +// +// BOUNDED BY THE LEASE ITSELF. Every beat records, so without a bound a long-lived runner would +// accumulate one entry per session it ever served, hold each of their credentials for the +// process lifetime, and fire a useless release for every one of them at shutdown. An entry +// whose last beat is older than `OWNER_TTL_SECONDS` cannot still hold the key, so it is pruned: +// the registry holds only what this replica can plausibly still own. + +interface OwnedSession { + authorization: string; + /** When the API last confirmed this replica owns the session. */ + claimedAt: number; +} + +const ownedSessions = new Map(); + +/** Drop entries whose affinity lease cannot still be held. */ +function pruneExpiredClaims(now: number): void { + const cutoff = now - OWNER_TTL_SECONDS * 1000; + for (const [sessionId, entry] of ownedSessions) { + if (entry.claimedAt < cutoff) ownedSessions.delete(sessionId); + } +} + +/** + * Note that this replica holds (or has just refreshed) the affinity key for `sessionId`, so + * the shutdown handler can release it. Called from every beat that the API confirmed we own. + * Overwrites the stored credential, which keeps the freshest token per session. + */ +export function recordOwnedSession( + sessionId: string, + authorization: string, + now: number = Date.now(), +): void { + if (!sessionId || !authorization) return; + pruneExpiredClaims(now); + ownedSessions.set(sessionId, { authorization, claimedAt: now }); +} + +/** Forget a session (a test hook, and the successful-release path). */ +export function forgetOwnedSession(sessionId: string): void { + ownedSessions.delete(sessionId); +} + +/** How many sessions this replica could still own. Test/inspection hook. */ +export function ownedSessionCount(now: number = Date.now()): number { + pruneExpiredClaims(now); + return ownedSessions.size; +} + /** * Send one heartbeat to keep the `alive` lock and the `session_streams` row live. Carries the * container `replica_id` (refreshes `owner` affinity) and the `turn_id` (proves alive ownership). @@ -60,8 +136,8 @@ function log(msg: string): void { * uuid — the free gift of a call the runner already makes every turn, no new round-trip) and * `interrupted: true` when the API reports `is_current_turn: false` (a cancel/steer/kill took * this turn's alive/running lock since the last beat — W7.4, the control-signal path). A - * network/HTTP failure yields `{ streamId: undefined, interrupted: false }` (fail-open: a - * transient API blip must neither abort a healthy run nor fabricate a stream id). + * network/HTTP failure yields `confirmed: false`; callers use that to fail closed for initial + * admission while later watchdog beats remain best effort for a turn already admitted. */ async function sendHeartbeat( sessionId: string, @@ -69,11 +145,16 @@ async function sendHeartbeat( authorization: string, isRunning = true, proposal?: SessionProposal, -): Promise<{ streamId: string | undefined; interrupted: boolean }> { +): Promise<{ + streamId: string | undefined; + interrupted: boolean; + confirmed: boolean; +}> { try { const url = `${apiBase()}/sessions/streams/heartbeat`; const res = await fetch(url, { method: "POST", + signal: AbortSignal.timeout(heartbeatTimeoutMs()), headers: { "content-type": "application/json", authorization, @@ -91,11 +172,12 @@ async function sendHeartbeat( }); if (!res.ok) { log(`heartbeat HTTP ${res.status} session=${sessionId} turn=${turnId}`); - return { streamId: undefined, interrupted: false }; + return { streamId: undefined, interrupted: false, confirmed: false }; } const body = (await res.json()) as { stream?: { id?: unknown } | null; is_current_turn?: unknown; + replica_id?: unknown; }; const rawStreamId = body.stream?.id; const streamId = @@ -103,15 +185,21 @@ async function sendHeartbeat( ? rawStreamId : undefined; const interrupted = body.is_current_turn === false; + // Record ONLY what the API says we own. The beat claims affinity as a side effect, so this + // is the one place that learns the claim happened; a beat this replica lost records nothing + // and the shutdown release skips it. + if (body.replica_id === REPLICA_ID) { + recordOwnedSession(sessionId, authorization); + } log( `heartbeat OK session=${sessionId} turn=${turnId} running=${isRunning}${interrupted ? " INTERRUPTED" : ""}`, ); - return { streamId, interrupted }; + return { streamId, interrupted, confirmed: true }; } catch (err) { log( `heartbeat failed session=${sessionId} turn=${turnId}: ${String(err instanceof Error ? err.message : err).slice(0, 120)}`, ); - return { streamId: undefined, interrupted: false }; + return { streamId: undefined, interrupted: false, confirmed: false }; } } @@ -146,6 +234,7 @@ export async function claimSessionOwnership( const body = (await res.json()) as { replica_id?: unknown }; const owner = typeof body.replica_id === "string" ? body.replica_id : undefined; + if (owner === REPLICA_ID) recordOwnedSession(sessionId, authorization); return { replicaId: REPLICA_ID, ownerReplicaId: owner }; } catch (err) { log( @@ -173,6 +262,16 @@ export async function claimSessionOwnership( * the caller MUST await in the run's `finally` so the heartbeat stops and the row is marked * `ended`. * + * That first beat is also this turn's ADMISSION request, and `admitted` reports its answer. The + * beat's `nx` acquire of the `alive` lock is the platform's single atomic arbiter of "who runs + * this session" (`api/oss/src/core/sessions/streams/service.py`), and it already refuses a turn + * that arrives while a different turn holds `running`. Reading that answer BEFORE the caller + * touches the sandbox is what makes at-most-one-execution-per-session true: a refused turn stops + * at the edge instead of reaching the keepalive pool and destroying the live turn's environment. + * + * Initial admission fails closed unless the coordination plane confirms this turn owns the lock. + * Later heartbeat failures remain best effort and do not abort an already-admitted healthy turn. + * * `proposal` rides EVERY beat rather than only the first. The server fills each field once, so * repeating them is a no-op, and one payload for all beats beats a "was this the first?" flag. */ @@ -186,6 +285,8 @@ export async function startAliveWatchdog( release: () => Promise; credential: () => string; streamId: () => string | undefined; + /** False when the FIRST beat reported `is_current_turn: false` — another turn owns the session. */ + admitted: boolean; }> { // Session coordination and standalone turns share this lease. The watchdog owns it here so // heartbeat, persistence, and trace export all observe the same current credential. @@ -218,17 +319,29 @@ export async function startAliveWatchdog( ); handleBeat(first); + // One beat in flight at a time. `setInterval` fires unconditionally, so without this a + // slow API stacks a new request every 30s on top of every request already waiting. + let beatInFlight = false; const interval = setInterval(() => { + if (beatInFlight) { + log(`heartbeat skipped (previous still in flight) session=${sessionId}`); + return; + } + beatInFlight = true; void (async () => { - handleBeat( - await sendHeartbeat( - sessionId, - turnId, - credentialLease.credential(), - true, - proposal, - ), - ); + try { + handleBeat( + await sendHeartbeat( + sessionId, + turnId, + credentialLease.credential(), + true, + proposal, + ), + ); + } finally { + beatInFlight = false; + } })(); }, REFRESH_INTERVAL_MS); @@ -238,6 +351,8 @@ export async function startAliveWatchdog( } return { + // Read from the FIRST beat only. Later interruptions travel the abort path instead. + admitted: first.confirmed && !first.interrupted, async release() { clearInterval(interval); credentialLease.release(); @@ -254,3 +369,80 @@ export async function startAliveWatchdog( streamId: () => streamId, }; } + +/** + * Hand this replica's affinity key for one session back to the coordination plane. + * + * The inverse beat: `release_owner: true`, no turn id, no liveness claim. The API releases + * `owner:session:` only while this replica still holds it, so the call can never take a + * session from a live runner and is safe to repeat. + * + * Never throws. A failure leaves the key to expire on its own lease, which is exactly the + * behaviour a killed (SIGKILL) runner already has. + */ +export async function releaseSessionOwnership( + sessionId: string, + authorization: string, + timeoutMs?: number, +): Promise { + try { + const runnerToken = process.env.AGENTA_RUNNER_TOKEN?.trim(); + const res = await fetch(`${apiBase()}/sessions/streams/heartbeat`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization, + ...(runnerToken ? { "x-agenta-runner-token": runnerToken } : {}), + }, + body: JSON.stringify({ + session_id: sessionId, + replica_id: REPLICA_ID, + release_owner: true, + }), + ...(timeoutMs ? { signal: AbortSignal.timeout(timeoutMs) } : {}), + }); + if (!res.ok) { + log(`ownership release HTTP ${res.status} session=${sessionId}`); + return false; + } + forgetOwnedSession(sessionId); + log(`ownership released session=${sessionId}`); + return true; + } catch (err) { + log( + `ownership release failed session=${sessionId}: ${String(err instanceof Error ? err.message : err).slice(0, 120)}`, + ); + return false; + } +} + +/** How long the whole shutdown release may take before the process stops waiting for it. */ +export const DEFAULT_OWNERSHIP_RELEASE_TIMEOUT_MS = 5_000; + +/** + * Release every affinity key this replica holds. Called from the shutdown handler, so it is + * bounded and never rejects: a runner that cannot reach the API must still exit promptly, and + * the 120-second owner lease is the fallback for that case and for a SIGKILL, which reaches no + * handler at all. + * + * The releases run concurrently because they are independent single-key deletes, and the whole + * set races one deadline rather than each call carrying its own budget. + */ +export async function releaseOwnedSessions( + timeoutMs: number = DEFAULT_OWNERSHIP_RELEASE_TIMEOUT_MS, +): Promise { + pruneExpiredClaims(Date.now()); + const held = [...ownedSessions.entries()]; + if (held.length === 0) return; + log(`releasing ${held.length} session ownership claim(s) on shutdown`); + const releases = Promise.all( + held.map(([sessionId, entry]) => + releaseSessionOwnership(sessionId, entry.authorization, timeoutMs), + ), + ); + const deadline = new Promise((resolve) => { + const handle = setTimeout(resolve, timeoutMs); + handle.unref?.(); + }); + await Promise.race([releases.then(() => undefined), deadline]); +} diff --git a/services/runner/src/sessions/applied-commands.ts b/services/runner/src/sessions/applied-commands.ts new file mode 100644 index 00000000000..71e1206b149 --- /dev/null +++ b/services/runner/src/sessions/applied-commands.ts @@ -0,0 +1,91 @@ +/** + * Commands this process has already acted on. + * + * WHY IT MUST OUTLIVE THE DELIVERY PATH. Delivery is at-least-once by design: a lost + * acknowledgement, a retried admission, or a re-armed claim can all bring the same command back. + * Applying a Stop a second time is not harmless — by then the session may be running a NEWER + * turn, and a second abort would kill work the user never asked to stop. + * + * So the set lives at module scope, beside the session pool, not inside a request or a poll + * loop. A loop restart with an empty set would be exactly the bug this prevents. + * + * An already-applied command is a NO-OP THAT STILL ACKNOWLEDGES. It aborts nothing and it + * reports the stored outcome, so a lost acknowledgement is repaired without a second abort. + * + * The entry is written when the command is ACCEPTED, not when the cancel finishes. A duplicate + * that arrives while the first is still cancelling must also be a no-op. + */ + +export interface AppliedCommand { + commandId: string; + /** What this process reported, so a duplicate can repeat the same answer. */ + executionState: string; + executionId: string | null; + result: "applied" | "obsolete"; + appliedAt: number; +} + +/** + * How long an applied command is remembered. Long enough to cover every redelivery path (the + * claim lease is 90 seconds and the sweep runs inside two minutes), short enough that the map + * cannot grow without bound on a long-lived process. + */ +export const APPLIED_COMMAND_TTL_MS = 30 * 60 * 1000; + +/** Hard cap, so a burst cannot grow the map faster than the TTL prunes it. */ +const MAX_APPLIED_COMMANDS = 5000; + +const applied = new Map(); + +function prune(now: number): void { + for (const [id, entry] of applied) { + if (now - entry.appliedAt > APPLIED_COMMAND_TTL_MS) applied.delete(id); + } + while (applied.size > MAX_APPLIED_COMMANDS) { + const oldest = applied.keys().next(); + if (oldest.done) break; + applied.delete(oldest.value); + } +} + +/** What this process already did with `commandId`, if anything. */ +export function recallCommand( + commandId: string, + now: number = Date.now(), +): AppliedCommand | undefined { + const entry = applied.get(commandId); + if (!entry) return undefined; + if (now - entry.appliedAt > APPLIED_COMMAND_TTL_MS) { + applied.delete(commandId); + return undefined; + } + return entry; +} + +/** Record what this process did with a command. Insertion order is the prune order. */ +export function rememberCommand( + entry: Omit, + now: number = Date.now(), +): AppliedCommand { + const stored: AppliedCommand = { ...entry, appliedAt: now }; + applied.delete(entry.commandId); + applied.set(entry.commandId, stored); + prune(now); + return stored; +} + +/** Revise the outcome of a command already accepted, once the cancel settles. */ +export function updateCommandOutcome( + commandId: string, + patch: Pick, +): void { + const entry = applied.get(commandId); + if (!entry) return; + entry.executionState = patch.executionState; + entry.result = patch.result; +} + +/** Test seam. */ +export function resetAppliedCommandsForTest(): void { + applied.clear(); +} diff --git a/services/runner/src/sessions/control-channel.ts b/services/runner/src/sessions/control-channel.ts new file mode 100644 index 00000000000..bc45e57f1ca --- /dev/null +++ b/services/runner/src/sessions/control-channel.ts @@ -0,0 +1,299 @@ +/** + * Applying a control command, and reporting what it did. + * + * The applier sits ABOVE the transport, not inside it, so every delivery path shares one set of + * guards and one deduplication set. Today there is one path, the direct `POST /cancel` route in + * `server.ts`. A long-poll loop would call the same `applyCommand` and change nothing here. + * + * WHAT THE RUNNER DECIDES AND WHAT IT DOES NOT. It decides whether it holds the named execution + * and whether that execution is old enough to be the one the user meant. It does NOT decide the + * command's fate: it reports an outcome to the API, and the API settles the command and the + * execution together. Settlement has one writer, on every transport. + * + * THE THREE ANSWERS. + * + * stopped — this process held the target execution and aborted it. + * not_running — it holds no execution that can still be stopped. A turn whose + * prompt has already settled and is only tearing down answers this. + * An approval-parked turn is still stoppable: its pending gate is + * released and it answers `stopped` like a live execution. + * superseded_by_newer_turn — it holds an execution that STARTED AFTER the command was + * created, so the command was meant for a turn that has since + * ended. Nothing is aborted. This check is exact, because it + * compares against this process's own memory of when it started + * the run. + */ + +import { apiBase } from "../apiBase.ts"; +import { REPLICA_ID } from "./alive.ts"; +import { + recallCommand, + rememberCommand, + updateCommandOutcome, +} from "./applied-commands.ts"; +import { findExecution, type LiveExecution } from "./execution-registry.ts"; + +function log(message: string): void { + process.stderr.write(`[control] ${message}\n`); +} + +/** One command as the API delivers it. The same shape arrives on every transport. */ +export interface ControlCommand { + id: string; + projectId: string; + sessionId: string; + kind: "cancel"; + target: { turnId: string | null; expectedTurnId: string | null }; + /** When the API admitted the command. The late-Stop guard compares against this. */ + createdAt: string; +} + +export type ExecutionState = + | "stopped" + | "failed" + | "not_running" + | "superseded_by_newer_turn"; + +export interface ControlOutcome { + /** The command's terminal state, as the runner sees it. */ + result: "applied" | "obsolete"; + execution: { + id: string | null; + state: ExecutionState; + error?: string; + }; +} + +/** The control operation exposed by one approval-parked session. */ +export interface ParkedSessionControl { + /** Release every gate and return the same environment to the pool as idle. */ + stop(): Promise | void; +} + +/** How the runner reaches a parked session. Injected so tests need no pool. */ +export interface ParkedLookup { + (projectId: string, sessionId: string): ParkedSessionControl | undefined; +} + +export interface ApplyCommandDeps { + /** Overridden in tests. Defaults to the module-level execution registry. */ + findLive?: (projectId: string, sessionId: string) => LiveExecution | undefined; + /** Whether the keep-alive pool holds this session parked awaiting an approval. */ + isParked?: ParkedLookup; + /** Overridden in tests. Defaults to the HTTP report below. */ + report?: (command: ControlCommand, outcome: ControlOutcome) => Promise; + now?: () => number; +} + +/** Does this process hold the session at all? The `/cancel` route answers 404 when it does not. */ +export function holdsSession( + projectId: string, + sessionId: string, + isParked?: ParkedLookup, +): boolean { + if (findExecution(projectId, sessionId)) return true; + return isParked ? isParked(projectId, sessionId) !== undefined : false; +} + +/** + * Apply one command and report its outcome. Never throws. + * + * Returns the outcome it reported, which is what a duplicate delivery repeats. + */ +export async function applyCommand( + command: ControlCommand, + deps: ApplyCommandDeps = {}, +): Promise { + const findLive = deps.findLive ?? findExecution; + const report = deps.report ?? reportOutcome; + const now = deps.now ?? (() => Date.now()); + + const seen = recallCommand(command.id, now()); + if (seen) { + // A no-op that STILL acknowledges. Aborting a second time could kill a newer turn; not + // acknowledging would leave the command open until the settlement sweep gave up on it. + const outcome: ControlOutcome = { + result: seen.result, + execution: { + id: seen.executionId, + state: seen.executionState as ExecutionState, + }, + }; + log( + `duplicate command=${command.id} session=${command.sessionId} state=${seen.executionState}`, + ); + await report(command, outcome).catch(() => {}); + return outcome; + } + + const createdAtMs = Date.parse(command.createdAt); + const live = findLive(command.projectId, command.sessionId); + const parked = live + ? undefined + : deps.isParked?.(command.projectId, command.sessionId); + const outcome = decideOutcome(command, live, parked, createdAtMs); + + // Remember BEFORE aborting. A duplicate that arrives while the first abort is still settling + // must find the command already taken, not start a second one. + rememberCommand( + { + commandId: command.id, + executionId: outcome.execution.id, + executionState: outcome.execution.state, + result: outcome.result, + }, + now(), + ); + + if (outcome.execution.state === "stopped") { + try { + if (live) { + // The abort is the cancel. It makes the turn end `cancelled`, which is what sends the + // ACP `session/cancel` to the harness and lets the environment be PARKED rather than + // deleted (see `cancel-turn.ts` and `shouldPark`). Stop keeps the session warm. + live.abort(); + log( + `aborted command=${command.id} session=${command.sessionId} turn=${live.turnId}`, + ); + } else if (parked) { + // An approval park has no live execution to abort, but its harness still holds the + // original prompt on one or more permission gates. Releasing those gates ends the work + // and returns the SAME environment to the idle pool, so the next user message is a + // normal warm prompt rather than an approval resume. + await parked.stop(); + log( + `released parked approval command=${command.id} session=${command.sessionId}`, + ); + } + } catch (error) { + const message = + error instanceof Error ? error.message : String(error ?? "abort failed"); + outcome.result = "applied"; + outcome.execution.state = "failed"; + outcome.execution.error = message.slice(0, 2000); + updateCommandOutcome(command.id, { result: "applied", executionState: "failed" }); + log(`abort FAILED command=${command.id} session=${command.sessionId}: ${message}`); + } + } + + // Reported as soon as the abort is issued, not after the harness settles. The command's job + // is to deliver the Stop; the turn's own teardown then writes its transcript and parks the + // sandbox on its own clock, which can take seconds. Waiting for it would make a Stop that + // worked look stuck. + await report(command, outcome).catch((error) => { + log( + `outcome report failed command=${command.id}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }); + return outcome; +} + +function decideOutcome( + command: ControlCommand, + live: LiveExecution | undefined, + parked: ParkedSessionControl | undefined, + createdAtMs: number, +): ControlOutcome { + if (!live) { + if (parked) { + return { + result: "applied", + execution: { id: command.target.turnId, state: "stopped" }, + }; + } + // No live or approval-parked turn is held here. There is nothing to stop. + return { + result: "applied", + execution: { id: command.target.turnId, state: "not_running" }, + }; + } + + if (Number.isFinite(createdAtMs) && live.startedAt > createdAtMs) { + // This execution began AFTER the user pressed Stop, so it is not the one they meant. + return { + result: "obsolete", + execution: { id: live.turnId, state: "superseded_by_newer_turn" }, + }; + } + + if (command.target.turnId && command.target.turnId !== live.turnId) { + // A different execution holds the session. The pinned target is gone. + return { + result: "obsolete", + execution: { id: command.target.turnId, state: "not_running" }, + }; + } + + if (live.settled) { + // THE STOP LOST THE RACE BY A MOMENT. The harness prompt already settled and the entry is + // only still here because teardown is running: writing the transcript, exporting the trace, + // parking the environment. There is nothing left to abort. + // + // Doing nothing is not merely tidier, it is the whole fix. `live.abort()` here would abort + // a finished run, and the aborted signal then makes `shouldPark` refuse to park a healthy + // idle environment, so the sandbox is destroyed and the user's next message rebuilds cold. + // The user paid a cold start for pressing Stop as the answer landed. + // + // `obsolete`, not `applied`: the command never stopped anything. `not_running` is the same + // answer a parked approval gets, and it means the same thing here — this process holds no + // execution that can still be stopped. + return { + result: "obsolete", + execution: { id: command.target.turnId ?? live.turnId, state: "not_running" }, + }; + } + + return { + result: "applied", + execution: { id: live.turnId, state: "stopped" }, + }; +} + +/** + * Report a command's outcome to the API. + * + * Authenticates with the shared runner token, not a project credential: the runner holds no + * project credential for a command it was handed, and the command id resolves the project on + * the API side. + */ +export async function reportOutcome( + command: ControlCommand, + outcome: ControlOutcome, +): Promise { + const token = process.env.AGENTA_RUNNER_TOKEN; + if (!token) { + log(`cannot report command=${command.id}: AGENTA_RUNNER_TOKEN is not set`); + return; + } + const url = `${apiBase()}/sessions/control/commands/${encodeURIComponent(command.id)}/outcome`; + const res = await fetch(url, { + method: "POST", + redirect: "error", + headers: { + "content-type": "application/json", + "x-agenta-runner-token": token, + }, + body: JSON.stringify({ + replica_id: REPLICA_ID, + result: outcome.result, + execution: { + id: outcome.execution.id, + state: outcome.execution.state, + ...(outcome.execution.error ? { error: outcome.execution.error } : {}), + }, + }), + }); + if (!res.ok) { + // A 409 means the claim was gone, which is an answer, not a failure to retry: the API has + // already written a terminal outcome for this command. + log( + `outcome HTTP ${res.status} command=${command.id} session=${command.sessionId}`, + ); + return; + } + log( + `outcome reported command=${command.id} session=${command.sessionId} state=${outcome.execution.state}`, + ); +} diff --git a/services/runner/src/sessions/execution-registry.ts b/services/runner/src/sessions/execution-registry.ts new file mode 100644 index 00000000000..0d83a2f3820 --- /dev/null +++ b/services/runner/src/sessions/execution-registry.ts @@ -0,0 +1,133 @@ +/** + * Which executions this runner process is running right now. + * + * WHY IT EXISTS. The abort controller for a session-owned run was a local variable inside the + * request handler in `server.ts`. Nothing outside that closure could reach it, so the only way + * to stop a turn was to take the session's Redis lock away and wait up to 30 seconds for the + * heartbeat to notice. A control command has to reach the running turn directly, and that needs + * a lookup keyed by something the API knows. + * + * THE KEY IS THE SESSION ID, AND THE PROJECT IS CHECKED SEPARATELY. Keying by + * `:` would be tidier, but the project scope is NOT known when a run + * starts: `runContext.project.id` is empty on the live invoke path, and the scope actually used + * for the pool key comes from the signed mount, which the coordinator resolves after the run is + * already in flight (`session-coordinator.ts`, `poolKeyFor(request, signed?.projectId)`). + * Registering under a key that does not exist yet is what made the first version of this + * registry answer "I do not hold that session" for every Stop. + * + * So the entry goes in under the session id at once, and `noteExecutionProject` fills the + * project in as soon as the coordinator knows it. A lookup matches only when the stored project + * agrees, so a Stop from another tenant is REFUSED rather than misrouted. Until the project is + * known the entry matches any project: that window is a few hundred milliseconds at the very + * start of a run, and refusing every Stop in it would reintroduce the bug this comment + * describes. + * + * The limit worth knowing: one entry per session id per process. Two projects running the same + * session id on one runner at the same time keep only the later entry, and the earlier one's + * Stop is then refused with a 404. Refusal is the safe direction, and the keep-alive pool has + * the same shape of key. + * + * `startedAt` is the field that makes a late Stop safe. The API pins the target turn at + * admission and compares its own clock, but the runner's comparison against its OWN memory is + * exact: a command created before an execution started cannot have been meant for it. + * + * Entries are removed in the same `finally` that releases the alive watchdog, so a run that + * threw still leaves the registry clean. + */ + +export interface LiveExecution { + /** Undefined until the coordinator resolves the run's project scope. */ + projectId: string | undefined; + sessionId: string; + /** The execution id, which is the runner's `turn_id`. */ + turnId: string; + /** When this process started the run, in epoch milliseconds. */ + startedAt: number; + /** + * True once the harness prompt has settled, whatever it settled as. + * + * The entry stays registered through teardown, which writes the transcript, exports the + * trace and decides whether to park, and that takes hundreds of milliseconds. A Stop that + * arrives in that window has nothing left to abort, and aborting anyway is actively harmful: + * the abort makes teardown read the run as cancelled-but-unsettled and DESTROY a healthy + * environment that was about to be parked. So the applier reads this flag and does nothing. + */ + settled?: boolean; + /** Stop the run. Aborting is what makes the turn end `cancelled`. */ + abort: () => void; +} + +const executions = new Map(); + +/** + * Register a run as live. A second registration for the same session REPLACES the first, + * because the pool's own supersede path has already torn the previous environment down by the + * time a replacement turn starts. + */ +export function registerExecution(execution: LiveExecution): void { + executions.set(execution.sessionId, execution); +} + +/** + * Fill in the project scope once the coordinator has resolved it. Scoped to the turn id, so a + * late callback from a finished run cannot relabel its successor. + */ +export function noteExecutionProject( + sessionId: string, + turnId: string, + projectId: string, +): void { + const current = executions.get(sessionId); + if (current && current.turnId === turnId) current.projectId = projectId; +} + +/** + * Mark a run's own work as finished, the moment the harness prompt settles and before teardown + * begins. Scoped to the turn id for the same reason `noteExecutionProject` is: a late callback + * from a finished run must not relabel its successor. + * + * Set from inside the turn, not from the request handler that awaits it, because the harmful + * window is exactly the teardown that runs between those two points. + */ +export function noteExecutionSettled(sessionId: string, turnId: string): void { + const current = executions.get(sessionId); + if (current && current.turnId === turnId) current.settled = true; +} + +/** + * Remove a run, but only if it is still the one registered. A turn that finishes after its + * successor registered must not unregister the successor. + */ +export function unregisterExecution(sessionId: string, turnId: string): void { + const current = executions.get(sessionId); + if (current && current.turnId === turnId) executions.delete(sessionId); +} + +/** + * The live execution for a session, when it belongs to the asking project. + * + * A stored project that DISAGREES yields nothing, so a Stop from another tenant is refused. + * A stored project that is not known yet matches, because the run has genuinely not been + * scoped at that point and refusing would drop every Stop in the first moments of a run. + */ +export function findExecution( + projectId: string, + sessionId: string, +): LiveExecution | undefined { + const current = executions.get(sessionId); + if (!current) return undefined; + if (current.projectId !== undefined && current.projectId !== projectId) { + return undefined; + } + return current; +} + +/** Test/inspection snapshot. */ +export function liveExecutions(): LiveExecution[] { + return [...executions.values()]; +} + +/** Test seam: drop everything. Never called by the server. */ +export function resetExecutionsForTest(): void { + executions.clear(); +} diff --git a/services/runner/src/sessions/stop-signal.ts b/services/runner/src/sessions/stop-signal.ts new file mode 100644 index 00000000000..f7428d13ad8 --- /dev/null +++ b/services/runner/src/sessions/stop-signal.ts @@ -0,0 +1,43 @@ +/** + * Labelling the abort so the park policy can tell a user Stop from every other abort. + * + * WHY A LABEL AND NOT THE FLAG. The runner has one `AbortController` per run, and several + * different events end a run through it. Only one of them is a cooperative user Stop: the + * heartbeat reporting `is_current_turn: false` after the API cleared this turn's alive lock + * (`sessions/alive.ts`, wired at `server.ts`). The rest — a client disconnect on a + * non-session run, anything a future call site adds — are not Stops, and their environments + * must still be destroyed. + * + * Before this label, `shouldPark` could only read `signal.aborted`, which cannot answer WHY. + * Inferring the Stop from `stopReason === "cancelled"` would be worse than it looks: the turn + * sets that value whenever the signal aborts, whatever aborted it, so any new + * `controller.abort()` anywhere in the runner would silently start parking sandboxes whose + * state nobody has checked. The teardown allowlist exists precisely to stop that from being + * possible, and this label is what keeps the allowlist honest. + * + * The mechanism is the standard one: `AbortController.abort(reason)` puts the value on + * `signal.reason`, and the same signal object reaches the park decision, so nothing new has to + * be threaded through the engine, the coordinator or the turn. + * + * WHAT THIS LABEL DOES NOT DISTINGUISH. Cancel, steer and hard kill all reach the runner the + * same way today: the API clears the alive lock and the next heartbeat reports it. So all three + * arrive labelled as a user Stop. That is safe rather than merely tolerable. A steer WANTS the + * warm environment for the turn it starts, and a kill separately calls the runner's `/kill`, + * which destroys the pool entry by key whether or not it was parked first. Naming the actual + * operation needs the durable command plane, which is work package B. + */ + +/** + * The `signal.reason` value a cooperative user Stop aborts with. + * + * A plain frozen object, not a string or an `Error`: object identity cannot be produced by + * accident, so nothing can be mistaken for a Stop by writing the same text. + */ +export const USER_STOP_ABORT_REASON = Object.freeze({ + agentaAbort: "user-stop", +} as const); + +/** True when this signal was aborted BY a cooperative user Stop, not by anything else. */ +export function isUserStopAbort(signal: AbortSignal | undefined): boolean { + return signal?.aborted === true && signal.reason === USER_STOP_ABORT_REASON; +} diff --git a/services/runner/src/sessions/turn-settle.ts b/services/runner/src/sessions/turn-settle.ts new file mode 100644 index 00000000000..917748dc1e9 --- /dev/null +++ b/services/runner/src/sessions/turn-settle.ts @@ -0,0 +1,177 @@ +/** + * Guarantee that a turn ends, even when `run()` does not. + * + * The runner's terminal record, and the release of its alive watchdog, both sit downstream of + * `await run(...)`. That is correct for every path where `run()` returns, and it is the whole + * bug where it does not: an await inside the run that never settles leaves the heartbeat + * announcing `running=true` every thirty seconds forever, so the platform holds the session + * open under a turn nobody is running and no terminal record is ever written. See issues #6418, + * #6100 and #5327. + * + * This module bounds that. It waits for `run()` normally, and gives up on it when either: + * + * * the platform says this turn is no longer current (a Stop, a takeover, or the API's own + * execution watchdog declaring the turn lost), or + * * the hard deadline elapses. + * + * Giving up is two steps, never one. First `abort()`, because most hangs DO unwind from an + * abort — the prompt race inside the turn resolves on the signal — and an unwound turn tears + * its sandbox down properly. Only if the run is still pending after `abandonGraceMs` does the + * caller stop waiting and write the outcome itself. + * + * What this deliberately does NOT do: kill the sandbox, or change any teardown rule. The + * abandoned `run()` still owns its environment and still runs its own `finally` if it ever + * settles. This is about the platform always learning the outcome, not about reclaiming + * machines — the keep-alive pool and the API watchdog already own that. + */ + +import { envTimerMs } from "../env.ts"; +import { DEFAULT_TOTAL_DEADLINE_MS } from "../engines/sandbox_agent/run-limits.ts"; + +export const HARD_DEADLINE_ENV = "AGENTA_RUNNER_TURN_HARD_DEADLINE_MS"; +export const ABANDON_GRACE_ENV = "AGENTA_RUNNER_TURN_ABANDON_GRACE_MS"; + +/** + * Half an hour past the longest legitimate run. + * + * This is a backstop, not a policy: it must never be the limit that ends a real turn, because + * the run limits already own that decision and users have asked for LONGER runs, not shorter + * ones (issues #6084, #5356). Keeping it above `DEFAULT_TOTAL_DEADLINE_MS` means a turn that + * reaches it is one whose own deadline already tripped and failed to end it. + */ +export const DEFAULT_HARD_DEADLINE_MS = DEFAULT_TOTAL_DEADLINE_MS + 30 * 60_000; + +/** + * How long a turn may take to unwind after its abort before the caller stops waiting. + * + * Long enough for a normal teardown (flush the trace, settle the interaction rows, destroy or + * park the sandbox), short enough that a user who pressed Stop is not left watching a spinner. + */ +export const DEFAULT_ABANDON_GRACE_MS = 60_000; + +export interface TurnSettleLimits { + hardDeadlineMs: number; + abandonGraceMs: number; +} + +export interface Clock { + setTimeout(fn: () => void, ms: number): NodeJS.Timeout; + clearTimeout(handle: NodeJS.Timeout): void; +} + +const realClock: Clock = { + setTimeout: (fn, ms) => setTimeout(fn, ms), + clearTimeout: (handle) => clearTimeout(handle), +}; + +export function resolveTurnSettleLimits( + log: (message: string) => void = () => {}, +): TurnSettleLimits { + return { + hardDeadlineMs: envTimerMs(HARD_DEADLINE_ENV, DEFAULT_HARD_DEADLINE_MS, { + log, + }), + abandonGraceMs: envTimerMs(ABANDON_GRACE_ENV, DEFAULT_ABANDON_GRACE_MS, { + log, + }), + }; +} + +export type TurnSettleOutcome = + /** `run()` returned. The normal path, and the only one that carries the run's own result. */ + | { settled: true; value: T } + /** `run()` never returned. The caller must write the terminal outcome itself. */ + | { settled: false; reason: string }; + +export interface AwaitTurnOptions { + /** The in-flight run. Never rejected by this function; the caller keeps its own catch. */ + run: Promise; + /** Ask the run to stop. Called once, before the grace window opens. */ + abort: () => void; + /** + * Resolves when the platform says this turn is no longer current — the heartbeat answered + * `is_current_turn: false`. Optional: a non-session run has no such signal. + */ + interrupted?: Promise; + limits: TurnSettleLimits; + clock?: Clock; + log?: (message: string) => void; +} + +/** + * Await `run`, or give up on it and say why. + * + * Resolves as soon as `run` settles on the happy path, with no timer left armed. + */ +export async function awaitTurnOrAbandon({ + run, + abort, + interrupted, + limits, + clock = realClock, + log = () => {}, +}: AwaitTurnOptions): Promise> { + const timers: NodeJS.Timeout[] = []; + const clearTimers = (): void => { + for (const timer of timers) clock.clearTimeout(timer); + timers.length = 0; + }; + + // A tagged sentinel, not a symbol on the value channel: `run` may resolve to anything, + // including a symbol, and the race must be able to tell the two apart with certainty. + type Raced = + | { kind: "resolved"; value: T } + | { kind: "rejected"; error: unknown } + | { kind: "abandon" }; + const trigger: Raced = { kind: "abandon" }; + let triggerReason: string | undefined; + const settled: Promise = run.then( + (value) => ({ kind: "resolved" as const, value }), + (error) => ({ kind: "rejected" as const, error }), + ); + + try { + const deadline = new Promise((resolve) => { + timers.push( + clock.setTimeout(() => { + triggerReason = `hard turn deadline of ${limits.hardDeadlineMs}ms exceeded`; + resolve(trigger); + }, limits.hardDeadlineMs), + ); + }); + const displaced: Promise | undefined = interrupted?.then((reason) => { + triggerReason = reason; + return trigger; + }); + + const first = await Promise.race( + displaced ? [settled, deadline, displaced] : [settled, deadline], + ); + if (first.kind === "resolved") return { settled: true, value: first.value }; + if (first.kind === "rejected") throw first.error; + + // The run must stop. Most hangs unwind from here, so ask before giving up. + const reason = triggerReason ?? "turn abandoned"; + log(`[turn-settle] ${reason}; aborting and waiting ${limits.abandonGraceMs}ms`); + try { + abort(); + } catch (err) { + log(`[turn-settle] abort threw: ${err instanceof Error ? err.message : err}`); + } + + const grace = new Promise((resolve) => { + timers.push(clock.setTimeout(() => resolve(trigger), limits.abandonGraceMs)); + }); + const second = await Promise.race([settled, grace]); + if (second.kind === "resolved") return { settled: true, value: second.value }; + if (second.kind === "rejected") throw second.error; + + log( + `[turn-settle] run did not unwind within ${limits.abandonGraceMs}ms of the abort; ` + + `writing the terminal outcome without it`, + ); + return { settled: false, reason }; + } finally { + clearTimers(); + } +} diff --git a/services/runner/src/tracing/otel.ts b/services/runner/src/tracing/otel.ts index 0b8042a3c2e..b9ea3f01174 100644 --- a/services/runner/src/tracing/otel.ts +++ b/services/runner/src/tracing/otel.ts @@ -2082,11 +2082,19 @@ export function createSandboxAgentOtel( } // Stamp the run's trace id on the turn's terminal event so a persisted transcript can link a // replayed turn back to its trace (undefined only in span-less mode with no valid traceparent). - // Mark a paused turn's terminal record so a cold reload can tell a pause from a real turn + // Mark a non-completing turn's terminal record so a cold reload can tell it from a real turn // boundary (the FE adoption heuristic and hydration read this). A completed turn omits it. + // + // `cancelled` rides here for the same reason `paused` does, and closes a real gap: without + // it a stopped turn is indistinguishable from a finished one in Postgres, so neither the + // frontend nor the release gate can tell a Stop from a completion. Kept as an explicit + // allowlist rather than passing `stopReason` through, so a harness-reported value such as + // `end_turn` or `max_tokens` cannot start appearing on the terminal record by accident. record({ type: "done", - ...(stopReason === "paused" ? { stopReason: "paused" } : {}), + ...(stopReason === "paused" || stopReason === "cancelled" + ? { stopReason } + : {}), ...(runTraceId ? { traceId: runTraceId } : {}), }); if (!emitSpans) return text; diff --git a/services/runner/tests/unit/acquire-abort.test.ts b/services/runner/tests/unit/acquire-abort.test.ts new file mode 100644 index 00000000000..9b2da1e91d4 --- /dev/null +++ b/services/runner/tests/unit/acquire-abort.test.ts @@ -0,0 +1,106 @@ +/** + * A Stop must preempt provider acquisition before the command-delivery timeout. The provider APIs + * do not accept AbortSignal, so the runner races them and compensates resources that arrive late. + * + * Run: pnpm exec vitest run tests/unit/acquire-abort.test.ts + */ +import assert from "node:assert/strict"; +import { describe, it } from "vitest"; + +import { abortableSandboxProvider } from "../../src/environment/abortable-sandbox-provider.ts"; + +function deferred(): { + promise: Promise; + resolve: (value: T) => void; +} { + let resolve!: (value: T) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function mustSettlePromptly(promise: Promise): Promise { + return Promise.race([ + promise, + new Promise((_resolve, reject) => + setTimeout( + () => reject(new Error("acquire did not cancel promptly")), + 4_000, + ), + ), + ]); +} + +describe("abortableSandboxProvider", () => { + for (const providerName of ["local", "daytona"] as const) { + it(`cancels a slow ${providerName} create and deletes the sandbox if it appears late`, async () => { + const created = deferred(); + const cleaned = deferred(); + const destroyed: string[] = []; + const controller = new AbortController(); + const provider = abortableSandboxProvider( + { + name: providerName, + create: () => created.promise, + async destroy(sandboxId: string) { + destroyed.push(sandboxId); + cleaned.resolve(); + }, + async getUrl() { + return "http://sandbox.invalid"; + }, + }, + controller.signal, + () => {}, + ); + + const acquire = provider.create(); + controller.abort(); + await assert.rejects( + () => mustSettlePromptly(acquire), + (error: unknown) => + error instanceof Error && + error.name === "AbortError" && + /acquisition was aborted/.test(error.message), + ); + + created.resolve(`${providerName}-late-id`); + await mustSettlePromptly(cleaned.promise); + assert.deepEqual(destroyed, [`${providerName}-late-id`]); + }); + } + + it("parks a Daytona sandbox whose reconnect finishes after cancellation", async () => { + const reconnected = deferred(); + const cleaned = deferred(); + const controller = new AbortController(); + let paused = 0; + const provider = abortableSandboxProvider( + { + name: "daytona", + async create() { + return "unused"; + }, + async destroy() {}, + reconnect: (_sandboxId: string) => reconnected.promise, + async pause() { + paused += 1; + cleaned.resolve(); + }, + async getUrl() { + return "http://sandbox.invalid"; + }, + }, + controller.signal, + () => {}, + ); + + const acquire = provider.reconnect!("parked-id"); + controller.abort(); + await assert.rejects(() => mustSettlePromptly(acquire), /aborted/); + reconnected.resolve(); + await mustSettlePromptly(cleaned.promise); + assert.equal(paused, 1); + }); +}); diff --git a/services/runner/tests/unit/cancel-continuity.test.ts b/services/runner/tests/unit/cancel-continuity.test.ts new file mode 100644 index 00000000000..61dd41122f5 --- /dev/null +++ b/services/runner/tests/unit/cancel-continuity.test.ts @@ -0,0 +1,409 @@ +/** + * The continuity record a STOPPED turn writes. + * + * A user Stop keeps the sandbox warm (see `harness-cancel-park.test.ts`), but the park is + * process-local: it dies with the runner. What survives a runner restart is the durable turn + * ledger, and `hydrateHarnessSessionFromDurable` re-seeds the in-memory store from it only when + * the latest row carries `end_time` AND `agent_session_id`. A stopped turn used to write neither, + * so a restart after a Stop lost the native harness session and the next message rebuilt cold. + * + * These tests pin the rule that fixes it: the record follows the HARNESS's confirmation, not the + * park decision. A settled cancel means the harness answered the cancelled prompt and is idle, so + * its native transcript holds a short but finished turn — a faithful resume point. An unsettled + * cancel leaves the harness in an unknown state and still falls back to cold replay. + * + * Run: pnpm exec vitest run tests/unit/cancel-continuity.test.ts + */ +import assert from "node:assert/strict"; +import { beforeEach, describe, it } from "vitest"; + +import { runSandboxAgent } from "../../src/engines/sandbox_agent.ts"; +import type { SandboxAgentDeps } from "../../src/engines/sandbox_agent.ts"; +import type { AgentRunRequest } from "../../src/protocol.ts"; +import { SessionContinuityStore } from "../../src/engines/sandbox_agent/session-continuity.ts"; +import { USER_STOP_ABORT_REASON } from "../../src/sessions/stop-signal.ts"; +import { resetRunnerConfigCache } from "../../src/config/runner-config.ts"; + +beforeEach(() => { + process.env.AGENTA_RUNNER_ENABLED_SANDBOX_PROVIDERS = "local,daytona"; + process.env.AGENTA_RUNNER_DAYTONA_API_KEY = "test-key"; + resetRunnerConfigCache(); +}); + +const AGENT_SESSION_ID = "agent-native-7"; + +interface CancelFakeOpts { + /** + * Whether the sandbox client can send `session/cancel` at all. An unpatched client has no + * `cancelSession`, which is the shipped "unsettled" shape: the harness is never told to stop. + */ + cancellable?: boolean; + /** Trigger the test's abort only after acquisition has completed and prompt has started. */ + onPrompt?: () => void; + /** Model the shell child Codex leaves behind after answering a cancelled prompt. */ + leakedCodexChild?: boolean; + /** Force Codex's best-effort post-cancel reap to fail in a known or unexpected way. */ + codexReapFailure?: "failed" | "unknown"; +} + +/** + * A sandbox whose prompt stays open until the cancel arrives — the real shape of a Stop. The + * abort alone never ends the prompt; only `session/cancel` does. + */ +function fakeCancellableSandbox(opts: CancelFakeOpts = {}) { + const continuityStore = new SessionContinuityStore(); + const calls = { + paused: 0, + destroyed: 0, + appended: [] as Array<{ turnIndex: number; agentSessionId?: string }>, + completed: [] as Array<{ + sessionId: string; + turnIndex: number; + agentSessionId?: string; + endTime: string; + }>, + cancelled: [] as string[], + logs: [] as string[], + lifecycle: [] as string[], + }; + let leakedCodexChildRunning = opts.leakedCodexChild === true; + + let answerPrompt: (() => void) | undefined; + const session = { + id: "harness-session-1", + agentSessionId: AGENT_SESSION_ID, + onEvent() {}, + onPermissionRequest() {}, + prompt() { + const response = new Promise((resolve) => { + answerPrompt = () => resolve({ stopReason: "cancelled" }); + }); + opts.onPrompt?.(); + return response; + }, + }; + + const sandbox: any = { + sandboxId: "daytona/sbx-warm", + sandboxProvider: { destroy: async () => {} }, + sandboxProviderRawId: "sbx-warm", + async createSession() { + return session; + }, + async destroySession() {}, + async pauseSandbox() { + calls.lifecycle.push("park"); + calls.paused += 1; + }, + async destroySandbox() { + calls.destroyed += 1; + }, + async dispose() {}, + async runProcess(request: { command: string; args?: string[] }) { + if (request.command === "ps") { + calls.lifecycle.push("ps"); + if (opts.codexReapFailure === "failed") { + throw new Error("ps unavailable"); + } + return { + stdout: [ + "100 1 120 /x/bin/sandbox-agent server --port 3000", + "110 100 119 node /x/codex-acp", + "120 110 118 /x/bin/codex app-server", + ...(leakedCodexChildRunning ? ["130 120 0 sleep 300"] : []), + ].join("\n"), + exitCode: 0, + }; + } + if (request.command === "kill") { + calls.lifecycle.push("kill"); + leakedCodexChildRunning = false; + return { stdout: "", exitCode: 0 }; + } + return { stdout: "", exitCode: 0 }; + }, + }; + if (opts.codexReapFailure === "unknown") { + Object.defineProperty(sandbox, "runProcess", { + get() { + throw new Error("reap inspection unavailable"); + }, + }); + } + if (opts.cancellable !== false) { + sandbox.cancelSession = async (id: string) => { + calls.lifecycle.push("cancel"); + calls.cancelled.push(id); + // The harness answers the cancelled prompt: this is what `settled` measures. + answerPrompt?.(); + }; + } + + const appendSessionTurn: any = async ( + _sessionId: string, + _harness: string, + turnIndex: number, + turn: { agentSessionId?: string }, + ) => { + calls.appended.push({ turnIndex, agentSessionId: turn.agentSessionId }); + }; + appendSessionTurn.complete = async ( + sessionId: string, + turnIndex: number, + turn: { agentSessionId?: string; endTime: string }, + ) => { + calls.completed.push({ + sessionId, + turnIndex, + agentSessionId: turn.agentSessionId, + endTime: turn.endTime, + }); + }; + + const deps: SandboxAgentDeps = { + log: (message) => { + calls.logs.push(message); + }, + createDaytonaCwd: (durable?: string) => durable ?? "/tmp/agenta-fake-cwd", + createLocalCwd: (durable?: string) => durable ?? "/tmp/agenta-fake-cwd", + resolveSkillDirs: () => ({ skills: [], cleanup: () => {} }), + buildDaemonEnv: () => ({}), + resolveDaemonBinary: () => "/bin/sandbox-agent", + buildSandboxProvider: () => + ({ provider: true, deleteSandbox: async () => {} }) as any, + createPersist: () => ({}) as any, + sessionContinuityStore: continuityStore, + hydrateHarnessSessionFromDurable: async () => {}, + appendSessionTurn, + startSandboxAgent: (async () => sandbox) as any, + prepareWorkspace: (async () => ({ cleanup: async () => {} })) as any, + prepareDaytonaPiAssets: async () => true, + discoverTunnelEndpoint: async () => null, + probeCapabilities: async () => + ({ + source: "probed", + capabilities: { + mcpTools: true, + toolCalls: true, + usage: true, + streamingDeltas: true, + }, + }) as any, + applyModel: async (_s, model) => model ?? "resolved-model", + createOtel: (() => ({ + start() {}, + handleUpdate() {}, + emitEvent() {}, + usage: () => ({ input: 0, output: 0, total: 0, cost: 0 }), + setUsage() {}, + finish: () => "partial answer", + recordError() {}, + output: () => "partial answer", + flush: async () => {}, + events: () => [], + settleOpenToolCalls() {}, + traceId: () => "trace-1", + })) as any, + startToolRelay: (() => ({ stop: async () => {} })) as any, + localRelayHost: (() => "local-relay-host") as any, + sandboxRelayHost: (() => "sandbox-relay-host") as any, + responderFactory: () => ({ + async onPermission() { + return { kind: "allow" } as const; + }, + async onClientTool() { + return { kind: "deny" } as const; + }, + }), + readStoredSandboxPointer: async () => ({ sandboxId: "sbx-warm" }), + }; + + return { + calls, + deps, + continuityStore, + leakedCodexChildRunning: () => leakedCodexChildRunning, + }; +} + +const stopRequest: AgentRunRequest = { + harness: "claude", + sandbox: "daytona", + sessionId: "sess-stop", + streamId: "stream-stop", + messages: [{ role: "user", content: "remember the codeword" }], + telemetry: { + exporters: { otlp: { headers: { authorization: "ApiKey abc" } } }, + } as any, +}; + +/** Build the real timing shape: acquire first, then abort when the harness prompt is in flight. */ +function fakeAbortingSandbox( + opts: CancelFakeOpts = {}, + kind: "user-stop" | "plain" = "user-stop", +) { + const controller = new AbortController(); + const fake = fakeCancellableSandbox({ + ...opts, + onPrompt: () => + kind === "user-stop" + ? controller.abort(USER_STOP_ABORT_REASON) + : controller.abort(), + }); + return { ...fake, signal: controller.signal }; +} + +describe("a stopped turn's continuity record", () => { + it("completes the durable ledger row with an end time and the native session id", async () => { + const { calls, deps, signal } = fakeAbortingSandbox(); + + const result = await runSandboxAgent(stopRequest, undefined, signal, deps); + + assert.equal(result.ok, true); + assert.equal(result.stopReason, "cancelled"); + assert.equal(result.cancelSettled, true, "the harness confirmed the stop"); + assert.deepEqual(calls.cancelled, ["harness-session-1"]); + + assert.equal( + calls.completed.length, + 1, + "a settled Stop completes its ledger row exactly once", + ); + const completed = calls.completed[0]; + assert.equal(completed.sessionId, "sess-stop"); + assert.equal(completed.turnIndex, 0, "it completes the row it started"); + assert.equal( + completed.agentSessionId, + AGENT_SESSION_ID, + "the row carries the harness session the next turn must load", + ); + // `hydrateHarnessSessionFromDurable` refuses a row without this field. + assert.ok( + completed.endTime && !Number.isNaN(Date.parse(completed.endTime)), + "end_time is an ISO instant, not empty", + ); + }); + + it("advances the in-memory resume pointer, so the next turn may load by id", async () => { + const { deps, continuityStore, signal } = fakeAbortingSandbox(); + + await runSandboxAgent(stopRequest, undefined, signal, deps); + + assert.deepEqual(continuityStore.get("sess-stop", "claude"), { + agentSessionId: AGENT_SESSION_ID, + turnIndex: 0, + }); + assert.equal( + continuityStore.latestTurn("sess-stop"), + 0, + "the stopped turn consumed its index", + ); + }); + + it("keeps the sandbox warm as well, so both halves of the resume survive", async () => { + const { calls, deps, signal } = fakeAbortingSandbox(); + + await runSandboxAgent(stopRequest, undefined, signal, deps); + + assert.equal(calls.paused, 1, "a confirmed Stop parks"); + assert.equal(calls.destroyed, 0); + }); + + it("reaps the Codex shell child before parking the warm sandbox", async () => { + const controller = new AbortController(); + const fake = fakeCancellableSandbox({ + leakedCodexChild: true, + onPrompt: () => controller.abort(USER_STOP_ABORT_REASON), + }); + + const result = await runSandboxAgent( + { ...stopRequest, harness: "codex" }, + undefined, + controller.signal, + fake.deps, + ); + + assert.equal(result.ok, true); + assert.equal(result.stopReason, "cancelled"); + assert.equal(fake.leakedCodexChildRunning(), false); + assert.deepEqual(fake.calls.lifecycle, ["cancel", "ps", "kill", "park"]); + }); + + for (const codexReapFailure of ["failed", "unknown"] as const) { + it(`keeps a settled Codex Stop warm after a ${codexReapFailure} reap`, async () => { + const { calls, continuityStore, deps, signal } = fakeAbortingSandbox({ + codexReapFailure, + }); + + const result = await runSandboxAgent( + { ...stopRequest, harness: "codex" }, + undefined, + signal, + deps, + ); + + assert.equal(result.ok, true); + assert.equal(result.cancelSettled, true); + assert.equal(calls.paused, 1, "a settled Stop still parks"); + assert.equal(calls.destroyed, 0); + assert.equal(calls.completed.length, 1, "continuity stays durable"); + assert.equal( + continuityStore.get("sess-stop", "codex")?.agentSessionId, + AGENT_SESSION_ID, + ); + assert.ok(calls.logs.some((line) => line.includes("cleanup_miss=true"))); + }); + } + + it("writes the record even when the abort was not a user Stop and the sandbox is deleted", async () => { + // A disconnect deletes the sandbox, but the harness still confirmed it is idle and its + // native session lives on the durable cwd, so the record stays worth keeping: the next turn + // mounts the same durable directory and may `session/load` into a fresh sandbox. + const { calls, deps, continuityStore, signal } = fakeAbortingSandbox( + {}, + "plain", + ); + + const result = await runSandboxAgent(stopRequest, undefined, signal, deps); + + assert.equal(result.ok, true); + assert.equal(calls.destroyed, 1, "an unlabelled abort still deletes"); + assert.equal(calls.paused, 0); + assert.equal(calls.completed.length, 1); + assert.equal( + continuityStore.get("sess-stop", "claude")?.agentSessionId, + AGENT_SESSION_ID, + ); + }); +}); + +describe("an abort the harness never confirmed", () => { + it("drops the record and leaves the ledger row open", async () => { + // An unpatched client cannot send `session/cancel`, so the harness may still be writing. + // This is the unchanged floor: no record, no completion, cold replay next turn. + const { calls, deps, continuityStore, signal } = fakeAbortingSandbox({ + cancellable: false, + }); + + const result = await runSandboxAgent(stopRequest, undefined, signal, deps); + + assert.equal(result.ok, true); + assert.equal(result.cancelSettled, false); + assert.deepEqual(calls.completed, [], "no end_time for an unknown state"); + assert.equal(continuityStore.get("sess-stop", "claude"), undefined); + assert.equal(calls.destroyed, 1, "unknown means delete"); + assert.equal(calls.paused, 0); + }); + + it("still appended the started row, which alone must never look resumable", async () => { + const { calls, deps, signal } = fakeAbortingSandbox({ + cancellable: false, + }); + + await runSandboxAgent(stopRequest, undefined, signal, deps); + + assert.equal(calls.appended.length, 1, "the turn started, so a row exists"); + assert.equal(calls.appended[0].turnIndex, 0); + assert.deepEqual(calls.completed, []); + }); +}); diff --git a/services/runner/tests/unit/continuation.test.ts b/services/runner/tests/unit/continuation.test.ts index ae8fb9747d9..36807c98721 100644 --- a/services/runner/tests/unit/continuation.test.ts +++ b/services/runner/tests/unit/continuation.test.ts @@ -75,10 +75,10 @@ describe("buildTurnText", () => { }); }); -// S3: on any successful resume rung (HOT continuation OR S1 session/load) the ACP prompt is -// last-message-only; buildTurnText only runs on the cold path. This imports `runTurn`'s own -// decision function, so the pin fails if the shipped rule drifts. -describe("S3 skip-flatten: sendLastMessageOnly = continuation || loaded", () => { +// S3: HOT continuation is intrinsically verified because the live harness never went away. A +// cold `session/load` must additionally prove that native history was replayed; accepting the id +// alone is not enough to discard the reconstructed transcript. +describe("S3 skip-flatten: only verified native history uses last-message-only", () => { it("cold turn (neither flag): the full transcript is sent, not last-message-only", () => { assert.equal(sendLastMessageOnly({}), false); }); @@ -87,11 +87,25 @@ describe("S3 skip-flatten: sendLastMessageOnly = continuation || loaded", () => assert.equal(sendLastMessageOnly({ continuation: true }), true); }); - it("S1 session/load rehydration turn: last-message-only", () => { - assert.equal(sendLastMessageOnly({ loaded: true }), true); + it("S1 session/load that only accepted the id: full reconstructed transcript", () => { + assert.equal(sendLastMessageOnly({ loaded: true }), false); + }); + + it("S1 session/load with observed native history: last-message-only", () => { + assert.equal( + sendLastMessageOnly({ loaded: true, nativeHistoryVerified: true }), + true, + ); }); it("both flags set (should not happen, but never double-flattens): still last-message-only", () => { - assert.equal(sendLastMessageOnly({ continuation: true, loaded: true }), true); + assert.equal( + sendLastMessageOnly({ + continuation: true, + loaded: true, + nativeHistoryVerified: false, + }), + true, + ); }); }); diff --git a/services/runner/tests/unit/control-command-apply.test.ts b/services/runner/tests/unit/control-command-apply.test.ts new file mode 100644 index 00000000000..825d0bc3bfa --- /dev/null +++ b/services/runner/tests/unit/control-command-apply.test.ts @@ -0,0 +1,780 @@ +/** + * The rules a control command obeys on the runner. + * + * A Stop reaches the runner as a durable command naming one execution. Four rules decide what + * the runner does with it, and this file pins all four: + * + * 1. It aborts the named execution when it holds it, which is what keeps the sandbox warm + * (the abort ends the turn `cancelled`, and only a cancelled turn takes the park path). + * 2. It aborts NOTHING when it holds an execution that started after the command was created. + * That is the late-Stop guard, and it is exact because it reads this process's own memory. + * 3. A session it holds parked awaiting an approval releases every gate, answers `stopped`, + * and stays warm as an idle session for the next normal prompt. + * 4. The same command delivered twice aborts once and acknowledges twice. + * 5. It aborts NOTHING when the named execution's prompt has already settled and only its + * teardown is still running. That Stop lost the race by a moment, and aborting a finished + * run would destroy the warm environment teardown was about to park. + */ +import assert from "node:assert/strict"; +import { beforeEach, describe, it } from "vitest"; + +import { + applyCommand, + holdsSession, + reportOutcome, + type ControlCommand, + type ControlOutcome, +} from "../../src/sessions/control-channel.ts"; +import { stopParkedApprovalSession } from "../../src/server.ts"; +import { resetAppliedCommandsForTest } from "../../src/sessions/applied-commands.ts"; +import { shouldPark } from "../../src/engines/sandbox_agent/engine.ts"; +import type { + ParkedApproval, + SessionEnvironment, +} from "../../src/engines/sandbox_agent.ts"; +import { + isUserStopAbort, + USER_STOP_ABORT_REASON, +} from "../../src/sessions/stop-signal.ts"; +import { + findExecution, + noteExecutionSettled, + registerExecution, + resetExecutionsForTest, + noteExecutionProject, + unregisterExecution, + type LiveExecution, +} from "../../src/sessions/execution-registry.ts"; + +const PROJECT = "11111111-1111-4111-8111-111111111111"; +const SESSION = "sess-42"; +const TURN = "turn-A"; + +/** t=1000 is "now"; a command created at t=1000 is contemporary with a run started at t=900. */ +const COMMAND_CREATED_AT = new Date(1000).toISOString(); + +function command(overrides: Partial = {}): ControlCommand { + return { + id: "cmd-1", + projectId: PROJECT, + sessionId: SESSION, + kind: "cancel", + target: { turnId: TURN, expectedTurnId: null }, + createdAt: COMMAND_CREATED_AT, + ...overrides, + }; +} + +function liveRun( + overrides: Partial = {}, +): { execution: LiveExecution; aborts: number[] } { + const aborts: number[] = []; + const execution: LiveExecution = { + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => aborts.push(Date.now()), + ...overrides, + }; + return { execution, aborts }; +} + +function collector(): { + reported: ControlOutcome[]; + report: (c: ControlCommand, o: ControlOutcome) => Promise; +} { + const reported: ControlOutcome[] = []; + return { + reported, + report: async (_c, o) => { + reported.push(o); + }, + }; +} + +beforeEach(() => { + resetExecutionsForTest(); + resetAppliedCommandsForTest(); +}); + +describe("applyCommand", () => { + it("aborts the live execution the command names and reports it stopped", async () => { + const { execution, aborts } = liveRun(); + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 1); + assert.equal(outcome.result, "applied"); + assert.equal(outcome.execution.state, "stopped"); + assert.equal(outcome.execution.id, TURN); + assert.deepEqual(reported, [outcome]); + }); + + it("reports not_running when this process holds no execution or parked approval", async () => { + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => undefined, + report, + }); + + assert.equal(outcome.result, "applied"); + assert.equal(outcome.execution.state, "not_running"); + assert.equal(reported.length, 1); + }); + + it("stops a parked approval, clears its gates, and leaves the next prompt warm", async () => { + const { reported, report } = collector(); + const permissionReplies: Array<{ id: string; reply: string }> = []; + const prompts: string[] = []; + const parked = { + state: "awaiting_approval" as "awaiting_approval" | "idle", + gates: new Map([ + ["tool-a", { permissionId: "perm-a" }], + ["tool-b", { permissionId: "perm-b" }], + ]), + session: { + respondPermission: async (id: string, reply: string) => { + permissionReplies.push({ id, reply }); + }, + prompt: async (text: string) => { + prompts.push(text); + }, + }, + }; + + const outcome = await applyCommand(command(), { + findLive: () => undefined, + isParked: (projectId, sessionId) => + projectId === PROJECT && + sessionId === SESSION && + parked.state === "awaiting_approval" + ? { + stop: async () => { + for (const gate of parked.gates.values()) { + await parked.session.respondPermission( + gate.permissionId, + "reject", + ); + } + parked.gates.clear(); + parked.state = "idle"; + }, + } + : undefined, + report, + }); + + assert.equal(outcome.result, "applied"); + assert.equal(outcome.execution.state, "stopped"); + assert.equal(outcome.execution.id, TURN); + assert.deepEqual(permissionReplies, [ + { id: "perm-a", reply: "reject" }, + { id: "perm-b", reply: "reject" }, + ]); + assert.equal(parked.gates.size, 0); + assert.equal(parked.state, "idle"); + + if (parked.state === "idle") { + await parked.session.prompt("what next?"); + } + assert.deepEqual(prompts, ["what next?"]); + assert.deepEqual(reported, [outcome]); + }); + + it("reparks a stopped approval only after the harness cancel settles", async () => { + const journal: string[] = []; + let settlePrompt!: (value: unknown) => void; + const promptPromise = new Promise((resolve) => { + settlePrompt = resolve; + }); + const gate: ParkedApproval = { + gateType: "claude-acp-permission", + permissionId: "perm-a", + toolCallId: "tool-a", + toolName: "commit", + args: {}, + interactionToken: "interaction-a", + promptPromise, + }; + const env = { + sandbox: { + cancelSession: async () => { + journal.push("cancel"); + settlePrompt({ stopReason: "cancelled" }); + }, + }, + session: { + id: "harness-session", + respondPermission: async () => journal.push("reject"), + }, + logger: () => {}, + parkedApprovals: new Map([[gate.toolCallId, gate]]), + parkedApproval: gate, + parkedApprovedExecutions: new Map([["approved", {}]]), + approvalGateCount: 1, + nonParkablePauseCount: 1, + commitAuthorization: {}, + sessionDestroyRequested: false, + clearTurn: () => journal.push("clear"), + } as unknown as SessionEnvironment; + let tornDown = 0; + + await stopParkedApprovalSession({ + environment: env, + repark: async () => { + journal.push("repark"); + return true; + }, + teardown: async () => { + tornDown += 1; + }, + cancelSettleMs: 1, + wait: async () => {}, + }); + + assert.deepEqual(journal, ["reject", "cancel", "clear", "repark"]); + assert.equal(env.parkedApprovals.size, 0); + assert.equal(env.parkedApproval, undefined); + assert.equal(env.sessionDestroyRequested, true); + assert.equal(tornDown, 0); + }); + + it("tears down a stopped approval when the harness cancel does not settle", async () => { + const journal: string[] = []; + const gate: ParkedApproval = { + gateType: "claude-acp-permission", + permissionId: "perm-a", + toolCallId: "tool-a", + toolName: "commit", + args: {}, + interactionToken: "interaction-a", + promptPromise: new Promise(() => {}), + }; + const env = { + sandbox: { + cancelSession: async () => journal.push("cancel"), + }, + session: { + id: "harness-session", + respondPermission: async () => journal.push("reject"), + }, + logger: () => {}, + parkedApprovals: new Map([[gate.toolCallId, gate]]), + parkedApproval: gate, + parkedApprovedExecutions: new Map(), + approvalGateCount: 1, + nonParkablePauseCount: 0, + sessionDestroyRequested: false, + clearTurn: () => journal.push("clear"), + } as unknown as SessionEnvironment; + + await assert.rejects( + stopParkedApprovalSession({ + environment: env, + repark: async () => { + journal.push("repark"); + return true; + }, + teardown: async () => { + journal.push("teardown"); + }, + cancelSettleMs: 1, + wait: async () => { + journal.push("timeout"); + }, + }), + /parked approval harness cancel did not settle/, + ); + + assert.deepEqual(journal, [ + "reject", + "cancel", + "timeout", + "timeout", + "teardown", + ]); + assert.equal(env.parkedApprovals.size, 1); + assert.equal(env.sessionDestroyRequested, true); + }); + + it("reparks a parked approval warm when the sandbox client has no cancelSession", async () => { + // The local provider's sandbox client can lack `cancelSession` (an older runtime), so the + // runner cannot send the ACP session/cancel and `cancelHarnessTurn` answers + // `sent=false reason=client-has-no-cancelSession`. A parked approval runs no turn, and the + // reject below is still the stop signal, so the environment must repark WARM and the Stop must + // report `stopped` — never tear the sandbox down and report a failed cancel. + const journal: string[] = []; + const gate: ParkedApproval = { + gateType: "claude-acp-permission", + permissionId: "perm-a", + toolCallId: "tool-a", + toolName: "commit", + args: {}, + interactionToken: "interaction-a", + // A prompt that never settles: without a cancelSession the runner never waits on it, so a + // pending prompt must not block or fail the repark. + promptPromise: new Promise(() => {}), + }; + const env = { + // No `cancelSession` on the sandbox client. This is the local-runtime case. + sandbox: {}, + session: { + id: "harness-session", + respondPermission: async () => journal.push("reject"), + }, + logger: () => {}, + parkedApprovals: new Map([[gate.toolCallId, gate]]), + parkedApproval: gate, + parkedApprovedExecutions: new Map([["approved", {}]]), + approvalGateCount: 1, + nonParkablePauseCount: 0, + commitAuthorization: {}, + sessionDestroyRequested: false, + clearTurn: () => journal.push("clear"), + } as unknown as SessionEnvironment; + let tornDown = 0; + + await stopParkedApprovalSession({ + environment: env, + repark: async () => { + journal.push("repark"); + return true; + }, + teardown: async () => { + tornDown += 1; + }, + cancelSettleMs: 1, + wait: async () => {}, + }); + + // No cancel was sent, so the environment is reparked straight from the reject and never + // tears down. + assert.deepEqual(journal, ["reject", "clear", "repark"]); + assert.equal(tornDown, 0, "a Stop must never evict the warm sandbox"); + assert.equal(env.parkedApprovals.size, 0); + assert.equal(env.parkedApproval, undefined); + // No cancel notification left the runner, so no destroy was ever requested for the session. + assert.equal(env.sessionDestroyRequested, false); + }); + + it("stops a local parked approval, staying warm, and reports it stopped end to end", async () => { + // The same case as above, but through `applyCommand`, which is what the /cancel route calls. + // It proves the OUTCOME the API settles on: `applied` / `stopped`, which is what writes the + // one terminal `session_executions` row. Before the fix this answered `applied` / `failed`, + // which the API never records as a terminal execution. + const { reported, report } = collector(); + const parked = { state: "awaiting_approval" as "awaiting_approval" | "idle" }; + let reparked = false; + let tornDown = false; + + const env = { + sandbox: {}, // no cancelSession + session: { + id: "harness-session", + respondPermission: async () => {}, + }, + logger: () => {}, + parkedApprovals: new Map([ + [ + "tool-a", + { + gateType: "claude-acp-permission", + permissionId: "perm-a", + toolCallId: "tool-a", + toolName: "commit", + args: {}, + interactionToken: "interaction-a", + promptPromise: new Promise(() => {}), + } as ParkedApproval, + ], + ]), + parkedApproval: undefined, + parkedApprovedExecutions: new Map(), + approvalGateCount: 1, + nonParkablePauseCount: 0, + commitAuthorization: {}, + sessionDestroyRequested: false, + clearTurn: () => {}, + } as unknown as SessionEnvironment; + + const outcome = await applyCommand(command(), { + findLive: () => undefined, + isParked: (projectId, sessionId) => + projectId === PROJECT && + sessionId === SESSION && + parked.state === "awaiting_approval" + ? { + stop: () => + stopParkedApprovalSession({ + environment: env, + repark: async () => { + reparked = true; + parked.state = "idle"; + return true; + }, + teardown: async () => { + tornDown = true; + }, + cancelSettleMs: 1, + wait: async () => {}, + }), + } + : undefined, + report, + }); + + assert.equal(outcome.result, "applied"); + assert.equal(outcome.execution.state, "stopped"); + assert.equal(outcome.execution.id, TURN); + assert.equal(reparked, true, "the warm sandbox returns to the pool"); + assert.equal(tornDown, false, "and is never evicted"); + assert.equal(parked.state, "idle"); + assert.deepEqual(reported, [outcome]); + }); + + it("refuses to abort an execution that started AFTER the command was created", async () => { + const { execution, aborts } = liveRun({ + turnId: "turn-B", + startedAt: 5000, // the command was created at t=1000 + }); + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 0, "a newer turn must never be aborted"); + assert.equal(outcome.result, "obsolete"); + assert.equal(outcome.execution.state, "superseded_by_newer_turn"); + assert.equal(reported.length, 1); + }); + + it("reports not_running when it holds a DIFFERENT, older execution", async () => { + const { execution, aborts } = liveRun({ turnId: "turn-Z", startedAt: 500 }); + const { report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 0); + assert.equal(outcome.result, "obsolete"); + assert.equal(outcome.execution.state, "not_running"); + assert.equal(outcome.execution.id, TURN); + }); + + it("aborts once and acknowledges twice when the same command is delivered twice", async () => { + const { execution, aborts } = liveRun(); + const { reported, report } = collector(); + + await applyCommand(command(), { findLive: () => execution, report }); + await applyCommand(command(), { findLive: () => execution, report }); + + assert.equal(aborts.length, 1, "a second abort could kill a newer turn"); + assert.equal(reported.length, 2, "a lost acknowledgement must be repairable"); + assert.equal(reported[1].execution.state, "stopped"); + }); + + it("remembers the command before aborting, so a duplicate mid-cancel is still a no-op", async () => { + // The abort itself delivers a second copy of the same command, which is what a retried + // admission looks like on the wire. + let nested: ControlOutcome | undefined; + const { report } = collector(); + const aborts: number[] = []; + const execution: LiveExecution = { + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => { + aborts.push(1); + }, + }; + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report: async (c, o) => { + if (nested === undefined) { + nested = await applyCommand(c, { findLive: () => execution, report }); + } + }, + }); + + assert.equal(aborts.length, 1); + assert.equal(outcome.execution.state, "stopped"); + assert.equal(nested?.execution.state, "stopped"); + }); + + it("aborts with the user-stop label, which is what lets the sandbox park", async () => { + // The registry hands the applier whatever abort the transport registered. `shouldPark` + // parks only an abort the runner can prove was a cooperative Stop, so an unlabelled abort + // here would end the turn `cancelled` and then DESTROY the sandbox. This pins the contract + // the applier depends on; `server.ts` is where the label is actually attached. + const controller = new AbortController(); + const execution: LiveExecution = { + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => controller.abort(USER_STOP_ABORT_REASON), + }; + const { report } = collector(); + + await applyCommand(command(), { findLive: () => execution, report }); + + assert.equal(isUserStopAbort(controller.signal), true); + assert.equal( + shouldPark( + { ok: true, stopReason: "cancelled", cancelSettled: true }, + controller.signal, + undefined, + ), + true, + "a Stop delivered as a command must leave the sandbox parkable", + ); + }); + + it("does NOT park when the abort carries no label", () => { + // The regression this guards: the first version of the control route called + // `controller.abort()` with no reason, so every Stop through it destroyed the sandbox. + const controller = new AbortController(); + controller.abort(); + + assert.equal(isUserStopAbort(controller.signal), false); + assert.equal( + shouldPark( + { ok: true, stopReason: "cancelled", cancelSettled: true }, + controller.signal, + undefined, + ), + false, + ); + }); + + it("aborts nothing when the named execution's prompt has already settled", async () => { + // The race the user cannot see: the answer lands, they press Stop a moment later, and the + // entry is still registered because teardown is writing the transcript and parking the + // sandbox. Aborting here stops nothing and makes teardown destroy a healthy environment. + const { execution, aborts } = liveRun({ settled: true }); + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 0, "a finished run must not be aborted"); + assert.equal(outcome.result, "obsolete", "the command stopped nothing"); + assert.equal(outcome.execution.state, "not_running"); + assert.equal(outcome.execution.id, TURN); + assert.deepEqual(reported, [outcome], "and it still acknowledges"); + }); + + it("still aborts an execution whose prompt has NOT settled", async () => { + // The guard must be the flag and not the mere presence of teardown, or every Stop becomes + // a no-op and Stop stops working. + const { execution, aborts } = liveRun({ settled: false }); + const { report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 1); + assert.equal(outcome.execution.state, "stopped"); + }); + + it("parks the environment of a finished turn that a late Stop did not abort", () => { + // The consequence the fix exists for, stated as the teardown sees it. No abort means no + // aborted signal, so a normally finished turn takes the ordinary park path. + const controller = new AbortController(); + assert.equal( + shouldPark( + { ok: true, stopReason: "end_turn" } as never, + controller.signal, + undefined, + ), + true, + "an un-aborted, cleanly finished turn parks", + ); + // And this is what used to happen instead: the late abort fired, and the same finished + // turn was destroyed rather than parked. + controller.abort(USER_STOP_ABORT_REASON); + assert.equal( + shouldPark( + { ok: true, stopReason: "end_turn" } as never, + controller.signal, + undefined, + ), + false, + "which is why the applier must not abort a settled run", + ); + }); + + it("reports the cancel as failed when the abort itself throws", async () => { + const execution: LiveExecution = { + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => { + throw new Error("controller is gone"); + }, + }; + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(outcome.execution.state, "failed"); + assert.equal(outcome.execution.error, "controller is gone"); + assert.equal(reported.length, 1); + }); +}); + +describe("the execution registry", () => { + it("refuses a lookup from another project once the scope is known", () => { + const { execution } = liveRun(); + registerExecution(execution); + + assert.equal(findExecution(PROJECT, SESSION)?.turnId, TURN); + assert.equal( + findExecution("22222222-2222-4222-8222-222222222222", SESSION), + undefined, + "two projects may use the same session id; the project is the tenant boundary", + ); + }); + + it("matches any project until the coordinator has resolved the scope", () => { + // `runContext.project.id` is empty on the live invoke path, so a run is registered before + // its project is known. Refusing every Stop in that window is what made the first version + // of this registry answer 404 for every real Stop. + registerExecution(liveRun({ projectId: undefined }).execution); + + assert.equal(findExecution(PROJECT, SESSION)?.turnId, TURN); + + noteExecutionProject(SESSION, TURN, PROJECT); + assert.equal( + findExecution("22222222-2222-4222-8222-222222222222", SESSION), + undefined, + "once the scope is known, another tenant is refused", + ); + }); + + it("does not let a late scope callback relabel a successor turn", () => { + registerExecution(liveRun({ turnId: "turn-2", projectId: undefined }).execution); + + noteExecutionProject(SESSION, "turn-1", "some-other-project"); + + assert.equal(findExecution(PROJECT, SESSION)?.projectId, undefined); + }); + + it("marks only the turn it names as settled", () => { + registerExecution({ + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => {}, + }); + + // A late callback from a turn that has already been replaced must not mark the successor + // finished, which would make every Stop on the live turn a no-op. + noteExecutionSettled(SESSION, "some-older-turn"); + assert.equal(findExecution(PROJECT, SESSION)?.settled, undefined); + + noteExecutionSettled(SESSION, TURN); + assert.equal(findExecution(PROJECT, SESSION)?.settled, true); + }); + + it("does not let a finished turn unregister its successor", () => { + const first = liveRun({ turnId: "turn-1" }).execution; + const second = liveRun({ turnId: "turn-2" }).execution; + registerExecution(first); + registerExecution(second); + + unregisterExecution(SESSION, "turn-1"); + + assert.equal(findExecution(PROJECT, SESSION)?.turnId, "turn-2"); + }); +}); + +describe("holdsSession", () => { + it("is true for a live execution", () => { + registerExecution(liveRun().execution); + assert.equal(holdsSession(PROJECT, SESSION), true); + }); + + it("is true for a session parked awaiting an approval, which runs no turn", () => { + // This is the case that has no control channel at all today: a parked session stops + // heartbeating, so the existing Stop signal never reaches it. + assert.equal(holdsSession(PROJECT, SESSION), false); + assert.equal( + holdsSession(PROJECT, SESSION, (projectId, sessionId) => + projectId === PROJECT && sessionId === SESSION + ? { stop: () => {} } + : undefined, + ), + true, + ); + }); + + it("does not match a parked session with the same id in another project", () => { + assert.equal( + holdsSession( + PROJECT, + SESSION, + (projectId, sessionId) => + projectId === "22222222-2222-4222-8222-222222222222" && + sessionId === SESSION + ? { stop: () => {} } + : undefined, + ), + false, + ); + }); + + it("is false for a session this process does not hold, which is what answers 404", () => { + assert.equal( + holdsSession(PROJECT, "other-session", () => undefined), + false, + ); + }); +}); + +describe("reportOutcome", () => { + it("rejects redirects so the runner token cannot be forwarded", async () => { + const previousToken = process.env.AGENTA_RUNNER_TOKEN; + const previousFetch = globalThis.fetch; + let captured: RequestInit | undefined; + process.env.AGENTA_RUNNER_TOKEN = "shared-secret"; + globalThis.fetch = (async (_input, init) => { + captured = init; + return new Response("{}", { status: 200 }); + }) as typeof fetch; + + try { + await reportOutcome(command(), { + result: "applied", + execution: { id: TURN, state: "stopped" }, + }); + } finally { + globalThis.fetch = previousFetch; + if (previousToken === undefined) delete process.env.AGENTA_RUNNER_TOKEN; + else process.env.AGENTA_RUNNER_TOKEN = previousToken; + } + + assert.equal(captured?.redirect, "error"); + }); +}); diff --git a/services/runner/tests/unit/credential-preflight.test.ts b/services/runner/tests/unit/credential-preflight.test.ts index 049533405fa..7917509065b 100644 --- a/services/runner/tests/unit/credential-preflight.test.ts +++ b/services/runner/tests/unit/credential-preflight.test.ts @@ -98,6 +98,42 @@ const OPENROUTER: HarnessOptions = { }; describe("awaitCredentialSubstitution", () => { + it("cancels a slow probe promptly when the turn is Stopped", async () => { + const controller = new AbortController(); + let probeStarted!: () => void; + const started = new Promise((resolve) => { + probeStarted = resolve; + }); + const run = awaitCredentialSubstitution({ + sandbox: { + runProcess: async () => { + probeStarted(); + return new Promise(() => {}); + }, + }, + baseUrl: "https://gateway.example/", + apiKeyVar: "OPENAI_API_KEY", + log: () => {}, + signal: controller.signal, + }); + + await started; + controller.abort(); + await assert.rejects( + () => + Promise.race([ + run, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("preflight did not cancel")), + 4_000, + ), + ), + ]), + /acquisition was aborted/, + ); + }); + it("returns ok immediately when the first probe substitutes", async () => { const { run, logs, commands } = harness([ '{"error":{"message":"you must provide a model parameter"}}', diff --git a/services/runner/tests/unit/environment-units.test.ts b/services/runner/tests/unit/environment-units.test.ts index 089fa05d2fc..667fb364b7d 100644 --- a/services/runner/tests/unit/environment-units.test.ts +++ b/services/runner/tests/unit/environment-units.test.ts @@ -18,6 +18,7 @@ import { type AcquireStage, } from "../../src/environment/timing.ts"; import * as workspaceManager from "../../src/environment/workspace-manager.ts"; +import { openSession as openHarnessSession } from "../../src/environment/harness-session-lifecycle.ts"; const SRC = (rel: string) => readFileSync( @@ -492,6 +493,115 @@ describe("harness-session unit: the seam", () => { ); }); + it("does not verify a load that accepted the id but emitted no prior messages", async () => { + const result = await openHarnessSession({ + sandbox: { + resumeSession: async () => ({ id: "local", agentSessionId: "native-1" }), + createSession: async () => ({ id: "must-not-create" }), + }, + persist: { + updateSession: async () => {}, + listEvents: async () => ({ items: [] }), + }, + acpAgent: "pi", + harness: "pi_core", + cwd: "/tmp/session", + sessionInit: {}, + priorAgentSessionId: "native-1", + nativeHistoryDurable: false, + localSessionId: "session-1:pi_core", + continuitySessionKey: "session-1", + log: () => {}, + timingLog: () => {}, + }); + + assert.equal(result.mode, "load"); + assert.equal(result.loadedFromContinuity, true); + assert.equal(result.nativeHistoryVerified, false); + }); + + it("verifies a load only after observing native prior-message events", async () => { + let reads = 0; + const result = await openHarnessSession({ + sandbox: { + resumeSession: async () => ({ id: "local", agentSessionId: "native-1" }), + createSession: async () => ({ id: "must-not-create" }), + }, + persist: { + updateSession: async () => {}, + listEvents: async () => { + reads += 1; + return { + items: + reads === 1 + ? [] + : [ + { + sender: "agent", + payload: { + method: "session/update", + params: { + update: { sessionUpdate: "user_message_chunk" }, + }, + }, + }, + ], + }; + }, + }, + acpAgent: "pi", + harness: "pi_core", + cwd: "/tmp/session", + sessionInit: {}, + priorAgentSessionId: "native-1", + nativeHistoryDurable: true, + localSessionId: "session-1:pi_core", + continuitySessionKey: "session-1", + log: () => {}, + timingLog: () => {}, + }); + + assert.equal(result.mode, "load"); + assert.equal(result.loadedFromContinuity, true); + assert.equal(result.nativeHistoryVerified, true); + }); + + it("does not treat prior prompt events as proof for the current load", async () => { + const priorEvent = { + sender: "agent", + payload: { + method: "session/update", + params: { + update: { sessionUpdate: "user_message_chunk" }, + }, + }, + }; + const result = await openHarnessSession({ + sandbox: { + resumeSession: async () => ({ id: "local", agentSessionId: "native-1" }), + createSession: async () => ({ id: "must-not-create" }), + }, + persist: { + updateSession: async () => {}, + listEvents: async () => ({ items: [priorEvent] }), + }, + acpAgent: "pi", + harness: "pi_core", + cwd: "/tmp/session", + sessionInit: {}, + priorAgentSessionId: "native-1", + nativeHistoryDurable: true, + localSessionId: "session-1:pi_core", + continuitySessionKey: "session-1", + log: () => {}, + timingLog: () => {}, + }); + + assert.equal(result.mode, "load"); + assert.equal(result.loadedFromContinuity, true); + assert.equal(result.nativeHistoryVerified, false); + }); + it("the composer delegates both stages", () => { const source = SRC("engines/sandbox_agent/environment.ts"); assert.ok(source.includes("await probeHarness(")); diff --git a/services/runner/tests/unit/harness-cancel-park.test.ts b/services/runner/tests/unit/harness-cancel-park.test.ts new file mode 100644 index 00000000000..8ccc1d8066b --- /dev/null +++ b/services/runner/tests/unit/harness-cancel-park.test.ts @@ -0,0 +1,338 @@ +/** + * Characterization of the Stop-keeps-warm path. + * + * A user Stop must keep the sandbox and the harness session so the next message resumes warm. + * Three rules make that safe, and this file pins all three: + * + * 1. The runner asks the HARNESS to stop and waits for it to confirm (`cancelHarnessTurn`). + * 2. Only a CONFIRMED stop parks (`shouldPark`); an unconfirmed one still destroys. + * 3. The parked reason is on the teardown allowlist, so the sandbox is stopped, not deleted. + */ +import assert from "node:assert/strict"; +import { afterEach, beforeEach, describe, it } from "vitest"; + +import { + cancelHarnessTurn, + DEFAULT_CANCEL_SETTLE_MS, +} from "../../src/engines/sandbox_agent/cancel-turn.ts"; +import { shouldPark } from "../../src/engines/sandbox_agent/engine.ts"; +import { readKeepaliveConfig } from "../../src/engines/sandbox_agent/session-identity.ts"; +import { + isUserStopAbort, + USER_STOP_ABORT_REASON, +} from "../../src/sessions/stop-signal.ts"; +import { createSandboxAgentOtel } from "../../src/tracing/otel.ts"; +import { teardownDisposition } from "../../src/engines/sandbox_agent/teardown.ts"; +import type { AgentRunResult } from "../../src/protocol.ts"; + +const cancelledTurn = (cancelSettled: boolean): AgentRunResult => ({ + ok: true, + output: "partial answer", + stopReason: "cancelled", + cancelSettled, +}); + +/** An abort that is NOT a user Stop: a disconnect, a future call site, anything unlabelled. */ +const abortedSignal = (): AbortSignal => { + const controller = new AbortController(); + controller.abort(); + return controller.signal; +}; + +/** The cooperative user Stop: the heartbeat interrupt labels its abort. */ +const userStopSignal = (): AbortSignal => { + const controller = new AbortController(); + controller.abort(USER_STOP_ABORT_REASON); + return controller.signal; +}; + +const never = (): Promise => new Promise(() => {}); +const noLog = (): void => {}; + +describe("cancelHarnessTurn", () => { + it("sends the cancel and reports settled when the harness answers the prompt", async () => { + const cancelled: string[] = []; + const result = await cancelHarnessTurn({ + sandbox: { + cancelSession: async (id: string) => { + cancelled.push(id); + }, + }, + sessionId: "sess-1", + promptPromise: Promise.resolve({ stopReason: "cancelled" }), + timeoutMs: 5_000, + wait: never, + log: noLog, + }); + + assert.deepEqual(cancelled, ["sess-1"]); + assert.equal(result.requested, true); + assert.equal(result.settled, true); + }); + + it("reports unsettled when the harness never answers inside the budget", async () => { + const result = await cancelHarnessTurn({ + sandbox: { cancelSession: async () => {} }, + sessionId: "sess-1", + promptPromise: never(), + timeoutMs: 5_000, + wait: async () => {}, + log: noLog, + }); + + assert.equal(result.requested, true); + assert.equal(result.settled, false); + }); + + it("reports unsettled when the prompt rejects instead of answering", async () => { + const result = await cancelHarnessTurn({ + sandbox: { cancelSession: async () => {} }, + sessionId: "sess-1", + promptPromise: Promise.reject(new Error("transport closed")), + timeoutMs: 5_000, + wait: never, + log: noLog, + }); + + assert.equal(result.requested, true); + assert.equal(result.settled, false); + }); + + it("reports neither requested nor settled on an unpatched client", async () => { + const result = await cancelHarnessTurn({ + sandbox: {}, + sessionId: "sess-1", + promptPromise: never(), + timeoutMs: 5_000, + wait: never, + log: noLog, + }); + + assert.equal(result.requested, false); + assert.equal(result.settled, false); + }); + + it("reports unsettled when the cancel itself throws", async () => { + const result = await cancelHarnessTurn({ + sandbox: { + cancelSession: async () => { + throw new Error("daemon gone"); + }, + }, + sessionId: "sess-1", + promptPromise: never(), + timeoutMs: 5_000, + wait: never, + log: noLog, + }); + + assert.equal(result.requested, false); + assert.equal(result.settled, false); + }); + + it("bounds a cancel request that never answers", async () => { + const logs: string[] = []; + const result = await cancelHarnessTurn({ + sandbox: { cancelSession: never }, + sessionId: "sess-1", + promptPromise: Promise.resolve({ stopReason: "cancelled" }), + timeoutMs: 5_000, + wait: async () => {}, + log: (message) => logs.push(message), + }); + + assert.deepEqual(result, { settled: false, requested: false, elapsedMs: 0 }); + assert.ok(logs.some((line) => line.includes("reason=request-timeout"))); + }); + + it("keeps a settle budget a user would wait through", () => { + assert.ok(DEFAULT_CANCEL_SETTLE_MS > 0); + assert.ok(DEFAULT_CANCEL_SETTLE_MS <= 30_000); + }); +}); + +describe("the user-Stop abort label", () => { + it("recognizes only the abort that carries the Stop reason", () => { + assert.equal(isUserStopAbort(userStopSignal()), true); + assert.equal(isUserStopAbort(abortedSignal()), false); + assert.equal(isUserStopAbort(undefined), false); + assert.equal(isUserStopAbort(new AbortController().signal), false); + }); + + it("cannot be forged by a look-alike value", () => { + const controller = new AbortController(); + controller.abort({ agentaAbort: "user-stop" }); + assert.equal(isUserStopAbort(controller.signal), false); + }); +}); + +describe("shouldPark on a user Stop", () => { + it("parks a stopped turn whose harness cancel settled", () => { + assert.equal( + shouldPark(cancelledTurn(true), userStopSignal(), undefined), + true, + ); + }); + + it("destroys a stopped turn whose harness cancel timed out", () => { + assert.equal( + shouldPark(cancelledTurn(false), userStopSignal(), undefined), + false, + ); + }); + + it("destroys an UNLABELLED abort even when the cancel settled", () => { + // The guard that keeps a future `controller.abort()` from silently parking a sandbox + // nobody checked. Only the heartbeat interrupt labels its abort. + assert.equal( + shouldPark(cancelledTurn(true), abortedSignal(), undefined), + false, + ); + }); + + it("destroys an aborted turn that never reported a cancel at all", () => { + const runLimitTrip: AgentRunResult = { ok: false, error: "run limit" }; + assert.equal(shouldPark(runLimitTrip, userStopSignal(), undefined), false); + }); + + it("parks a settled Stop even though the client dropped its stream", () => { + // The case the product actually produces. The browser's Stop button aborts the chat stream + // in the same tick it sends the durable cancel command, so a real Stop ALWAYS reaches this + // predicate with the client already gone. This assertion used to read `false`, and reading + // the disconnect first is what deleted the sandbox on every Stop. + assert.equal( + shouldPark(cancelledTurn(true), userStopSignal(), () => true), + true, + ); + }); + + it("keeps destroying on every disconnect that is not a settled Stop", () => { + // A disconnect with no Stop behind it, an unlabelled abort, and an unconfirmed cancel all + // leave a session nobody asked to keep. The rule the disconnect check exists for is intact. + assert.equal( + shouldPark({ ok: true, stopReason: "end_turn" }, undefined, () => true), + false, + ); + assert.equal( + shouldPark(cancelledTurn(true), abortedSignal(), () => true), + false, + ); + assert.equal( + shouldPark(cancelledTurn(false), userStopSignal(), () => true), + false, + ); + }); + + it("leaves every non-abort verdict as it was", () => { + assert.equal( + shouldPark({ ok: true, stopReason: "end_turn" }, undefined, undefined), + true, + ); + assert.equal( + shouldPark({ ok: false, error: "boom" }, undefined, undefined), + false, + ); + assert.equal( + shouldPark({ ok: true, stopReason: "paused" }, undefined, undefined), + false, + ); + }); +}); + +describe("the cancelled teardown reason", () => { + it("stops the sandbox instead of deleting it", () => { + assert.equal(teardownDisposition("cancelled"), "stop"); + }); + + it("still deletes when clean parking is switched off", () => { + assert.equal(teardownDisposition("cancelled", false), "delete"); + }); + + it("leaves a plain abort deleting", () => { + assert.equal(teardownDisposition("aborted"), "delete"); + }); +}); + +describe("the stopped-session park window", () => { + const ttlEnvNames = [ + "AGENTA_RUNNER_SESSION_TTL_MS", + "AGENTA_RUNNER_SESSION_APPROVAL_TTL_MS", + "AGENTA_RUNNER_SESSION_STOPPED_TTL_MS", + "AGENTA_RUNNER_DAYTONA_SESSION_IDLE_TTL_MS", + ] as const; + let savedTtlEnv: Record; + + beforeEach(() => { + savedTtlEnv = Object.fromEntries( + ttlEnvNames.map((name) => [name, process.env[name]]), + ); + for (const name of ttlEnvNames) delete process.env[name]; + }); + + afterEach(() => { + for (const name of ttlEnvNames) { + const value = savedTtlEnv[name]; + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + // A settled Stop gets the same ten-minute human-response window on both providers. The + // ordinary idle windows remain shorter and continue to govern clean completed turns. + it("defaults a local stopped session to the approval window", () => { + const config = readKeepaliveConfig("local"); + assert.equal(config.ttlMs, 60_000); + assert.equal(config.stoppedTtlMs, 600_000); + assert.equal(config.approvalTtlMs, 600_000); + }); + + it("defaults a Daytona stopped session to the ten-minute human-response window", () => { + const config = readKeepaliveConfig("daytona"); + assert.equal(config.ttlMs, 120_000); + assert.equal(config.stoppedTtlMs, 600_000); + }); + + it("moves with its own env var, without touching the ordinary idle window", () => { + process.env.AGENTA_RUNNER_SESSION_STOPPED_TTL_MS = "300000"; + const local = readKeepaliveConfig("local"); + const daytona = readKeepaliveConfig("daytona"); + assert.equal(local.stoppedTtlMs, 300_000); + assert.equal(local.ttlMs, 60_000); + assert.equal(daytona.stoppedTtlMs, 300_000); + assert.equal(daytona.ttlMs, 120_000); + }); +}); + +describe("the terminal done record", () => { + /** Finish a runner-traced turn and hand back the terminal `done` event it recorded. */ + const doneRecordFor = (stopReason?: string): Record => { + const run = createSandboxAgentOtel({ + harness: "pi", + model: "openai/x", + emitSpans: false, + }); + run.start({ prompt: "hi" }); + run.finish(stopReason); + const done = run.events().find((event) => event.type === "done"); + assert.ok(done, "the turn must record exactly one terminal done event"); + return done as unknown as Record; + }; + + it("carries the stop reason for a user Stop", () => { + // Without this, a stopped turn is indistinguishable from a completed one in Postgres, so + // neither the frontend nor the release gate can tell a Stop from a finish. + assert.equal(doneRecordFor("cancelled").stopReason, "cancelled"); + }); + + it("still carries a pause, which is what this field originally existed for", () => { + assert.equal(doneRecordFor("paused").stopReason, "paused"); + }); + + it("omits the field for a completed turn and for every harness-reported reason", () => { + // An explicit two-value allowlist, so `end_turn` / `max_tokens` / a future harness string + // cannot start appearing on the terminal record by accident. + assert.equal(doneRecordFor("end_turn").stopReason, undefined); + assert.equal(doneRecordFor("max_tokens").stopReason, undefined); + assert.equal(doneRecordFor(undefined).stopReason, undefined); + }); +}); diff --git a/services/runner/tests/unit/mount-lifecycle.test.ts b/services/runner/tests/unit/mount-lifecycle.test.ts new file mode 100644 index 00000000000..69a590cc31e --- /dev/null +++ b/services/runner/tests/unit/mount-lifecycle.test.ts @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "vitest"; + +import type { AcquireContext } from "../../src/environment/acquire-context.ts"; +import { + mountLocalAgentCwd, + mountLocalDurableCwd, + type MountDeps, +} from "../../src/environment/mount-lifecycle.ts"; + +const credentials = { + endpoint: "http://store", + region: "eu-central-1", + bucket: "bucket", + prefix: "prefix", + accessKey: "access", + secretKey: "secret", +}; + +const depsFor = ( + signal: AbortSignal, + mountStorage: MountDeps["mountStorage"], +): MountDeps => ({ + mountStorage, + signMount: async () => null, + signAgentMount: async () => null, + daytonaPiDir: "/tmp/pi", + signal, +}); + +const contextFor = (cwd: string, commits: string[]): AcquireContext => + ({ + plan: { + acpAgent: "pi", + isDaytona: false, + workspace: { cwd }, + }, + env: { + mountCreds: credentials, + agentMountCreds: credentials, + }, + sessionForMount: "session-1", + artifactId: "artifact-1", + log: () => {}, + beginCwdMount: () => {}, + markCwdDetachConfirmed: () => {}, + commitLocalMount: (kind: string) => commits.push(kind), + }) as unknown as AcquireContext; + +describe("local mount cancellation", () => { + it("commits a durable cwd mount before observing an abort", async () => { + const cwd = mkdtempSync(join(tmpdir(), "agenta-mount-cwd-")); + const controller = new AbortController(); + const commits: string[] = []; + + try { + await assert.rejects( + mountLocalDurableCwd( + contextFor(cwd, commits), + depsFor(controller.signal, async () => { + controller.abort(); + return true; + }), + "initial", + ), + { name: "AbortError" }, + ); + assert.deepEqual(commits, ["cwd"]); + } finally { + rmSync(cwd, { recursive: true, force: true }); + } + }); + + it("commits an agent mount before its abort is handled", async () => { + const cwd = mkdtempSync(join(tmpdir(), "agenta-mount-agent-")); + const controller = new AbortController(); + const commits: string[] = []; + + try { + const mounted = await mountLocalAgentCwd( + contextFor(cwd, commits), + depsFor(controller.signal, async () => { + controller.abort(); + return true; + }), + ); + + assert.equal(mounted, false); + assert.deepEqual(commits, ["agent"]); + } finally { + rmSync(cwd, { recursive: true, force: true }); + rmSync(`${cwd}-agent`, { recursive: true, force: true }); + } + }); +}); diff --git a/services/runner/tests/unit/reap-exec.test.ts b/services/runner/tests/unit/reap-exec.test.ts new file mode 100644 index 00000000000..6f190aa0145 --- /dev/null +++ b/services/runner/tests/unit/reap-exec.test.ts @@ -0,0 +1,389 @@ +/** + * The Codex Stop leaves its shell child running; this pins the reap that kills it. + * + * The rules that matter are the two that keep a warm session warm: the `codex app-server` process + * itself is never a candidate, and neither is anything OLDER than the turn that was stopped (an + * stdio MCP server starts with the session, so it always is). Everything else is bookkeeping. + */ +import { describe, expect, it, vi } from "vitest"; + +import { + MAX_REAPED, + findAppServerPid, + findSandboxAgentServerPid, + parseProcessTable, + reapLeakedExecChildren, + reapResultHasCleanupMiss, + selectLeakedExecPids, +} from "../../src/engines/sandbox_agent/reap-exec.ts"; +import { + DAYTONA_SANDBOX_AGENT_PORT, + sandboxAgentServerPort, +} from "../../src/engines/sandbox_agent/provider.ts"; + +const LIVE_PORT = 43_123; + +describe("reapResultHasCleanupMiss", () => { + it("flags failed and unknown cleanup for QA", () => { + expect(reapResultHasCleanupMiss({ killed: 1 })).toBe(false); + expect( + reapResultHasCleanupMiss({ killed: 0, skipped: "nothing-to-reap" }), + ).toBe(false); + expect(reapResultHasCleanupMiss({ killed: 0, skipped: "ps-failed" })).toBe( + true, + ); + expect(reapResultHasCleanupMiss(undefined)).toBe(true); + }); +}); + +/** The real tree, copied from the live probe on the integration stack (2026-09-03). */ +const LIVE_PS = [ + " 1 0 50000 /sbin/docker-init -- docker-entrypoint.sh sh -c node scripts/build-extension.mjs", + " 7 1 49999 node node_modules/.bin/../tsx/dist/cli.mjs watch src/server.ts", + " 58 7 49998 /usr/local/bin/node --require /app/node_modules/.pnpm/tsx@4.19.2/preflight.cjs src/server.ts", + `67965 58 120 /app/node_modules/.pnpm/@sandbox-agent+cli-linux-x64@0.4.2/bin/sandbox-agent server --host 127.0.0.1 --port ${LIVE_PORT}`, + "68015 67965 118 node /root/.local/share/sandbox-agent/bin/agent_processes/codex/node_modules/.bin/codex-acp", + "68022 68015 117 /usr/local/bin/node /root/.local/share/sandbox-agent/bin/agent_processes/codex/node_modules/@openai/codex/bin/codex.js app-server", + "68029 68022 116 /root/.local/share/sandbox-agent/bin/agent_processes/codex/node_modules/@openai/codex-linux-x64/vendor/x86_64-unknown-linux-musl/bin/codex app-server", + "68164 68029 12 python3 -c import time; time.sleep(300.925793)", +].join("\n"); + +describe("sandboxAgentServerPort", () => { + it("reads the allocated port from a local sandbox handle id", () => { + expect(sandboxAgentServerPort(`local/127.0.0.1:${LIVE_PORT}`)).toBe( + LIVE_PORT, + ); + }); + + it("returns the explicit port configured for Daytona", () => { + expect(sandboxAgentServerPort("daytona/sandbox-1")).toBe( + DAYTONA_SANDBOX_AGENT_PORT, + ); + }); +}); + +describe("parseProcessTable", () => { + it("reads pid, ppid, elapsed seconds and the full argv", () => { + const rows = parseProcessTable(LIVE_PS); + expect(rows).toHaveLength(8); + expect(rows.at(-1)).toEqual({ + pid: 68164, + ppid: 68029, + etimes: 12, + args: "python3 -c import time; time.sleep(300.925793)", + }); + }); + + it("drops a line it cannot read rather than guessing at it", () => { + expect(parseProcessTable("PID PPID ELAPSED COMMAND\nnonsense\n")).toEqual( + [], + ); + }); +}); + +describe("findSandboxAgentServerPid", () => { + it("matches the exact --port value, not another port with the same prefix", () => { + const rows = parseProcessTable( + [ + " 10 1 5 /x/bin/sandbox-agent server --port 4312", + " 11 1 5 /x/bin/sandbox-agent server --port 43123", + ].join("\n"), + ); + expect(findSandboxAgentServerPid(rows, 4312)).toBe(10); + }); +}); + +describe("findAppServerPid", () => { + it("finds the Rust core and not the JavaScript launcher that shares its subcommand", () => { + expect(findAppServerPid(parseProcessTable(LIVE_PS), 67965)).toBe(68029); + }); + + it("answers undefined when nothing matches", () => { + const rows = parseProcessTable(" 10 1 5 node server.js"); + expect(findAppServerPid(rows, 1)).toBeUndefined(); + }); + + it("answers undefined when TWO descendants match, rather than picking one", () => { + const rows = parseProcessTable( + [ + " 10 1 5 /x/bin/sandbox-agent server --port 4312", + " 11 10 5 /a/bin/codex app-server", + " 12 10 5 /b/bin/codex app-server", + ].join("\n"), + ); + expect(findAppServerPid(rows, 10)).toBeUndefined(); + }); +}); + +describe("selectLeakedExecPids", () => { + const rows = parseProcessTable(LIVE_PS); + + it("selects the leaked shell child", () => { + expect( + selectLeakedExecPids(rows, { + appServerPid: 68029, + turnElapsedSeconds: 20, + }), + ).toEqual([68164]); + }); + + it("never selects the app-server itself, nor any of its ancestors", () => { + const selected = selectLeakedExecPids(rows, { + appServerPid: 68029, + turnElapsedSeconds: 100000, + }); + for (const pid of [1, 7, 58, 67965, 68015, 68022, 68029]) { + expect(selected).not.toContain(pid); + } + }); + + it("leaves a process the SESSION started alone: an stdio MCP server outlives the turn", () => { + const withMcp = parseProcessTable( + [LIVE_PS, "68100 68029 90 node /app/mcp/stdio-server.js"].join("\n"), + ); + const selected = selectLeakedExecPids(withMcp, { + appServerPid: 68029, + turnElapsedSeconds: 20, + }); + expect(selected).toEqual([68164]); + expect(selected).not.toContain(68100); + }); + + it("keeps a child born in the same whole second as the prompt", () => { + const rows2 = parseProcessTable( + [ + "68029 68022 116 /x/bin/codex app-server", + "68164 68029 20 sleep 300", + ].join("\n"), + ); + expect( + selectLeakedExecPids(rows2, { + appServerPid: 68029, + turnElapsedSeconds: 20, + }), + ).toEqual([68164]); + }); + + it("follows the tree, so a shell that forked its own child loses both", () => { + const rows2 = parseProcessTable( + [ + "68029 68022 116 /x/bin/codex app-server", + "68164 68029 12 /bin/bash -c sleep 300", + "68165 68164 12 sleep 300", + ].join("\n"), + ); + expect( + selectLeakedExecPids(rows2, { + appServerPid: 68029, + turnElapsedSeconds: 20, + }), + ).toEqual([68164, 68165]); + }); +}); + +describe("reapLeakedExecChildren", () => { + function sandboxWith(stdout: string) { + const calls: Array<{ command: string; args?: string[] }> = []; + return { + calls, + sandbox: { + runProcess: vi.fn( + async (request: { command: string; args?: string[] }) => { + calls.push(request); + return { + stdout: request.command === "ps" ? stdout : "", + exitCode: 0, + }; + }, + ), + }, + }; + } + + it("rounds the turn's age DOWN, so a session helper a hair older survives", async () => { + // The `git fetch` Codex runs to sync its plugins starts about a second before the prompt on + // a cold turn. At 28.9 s of turn, a 29 s-old helper must not be a candidate. + const { sandbox, calls } = sandboxWith( + [ + `67965 58 120 /x/bin/sandbox-agent server --port ${LIVE_PORT}`, + "68029 67965 116 /x/bin/codex app-server", + "68100 68029 29 git -C /w/.codex/.tmp/plugins-clone fetch --depth 1", + "68164 68029 22 sleep 300", + ].join("\n"), + ); + const result = await reapLeakedExecChildren({ + sandbox, + sandboxAgentPort: LIVE_PORT, + turnElapsedMs: 28_900, + log: vi.fn(), + }); + expect(result).toEqual({ killed: 1 }); + expect(calls[1]).toMatchObject({ command: "kill", args: ["-9", "68164"] }); + }); + + it("lists, then kills exactly the leaked pid", async () => { + const { sandbox, calls } = sandboxWith(LIVE_PS); + const log = vi.fn(); + const result = await reapLeakedExecChildren({ + sandbox, + sandboxAgentPort: LIVE_PORT, + turnElapsedMs: 20_000, + log, + }); + expect(result).toEqual({ killed: 1 }); + expect(calls[0].command).toBe("ps"); + expect(calls[1]).toMatchObject({ command: "kill", args: ["-9", "68164"] }); + expect(log).toHaveBeenCalledWith(expect.stringContaining("killed=1")); + expect(log).toHaveBeenCalledWith(expect.stringContaining("pids=68164")); + }); + + it("reaps only the stopped turn beneath the daemon on this sandbox's port", async () => { + const rows = [ + "100 1 120 /x/bin/sandbox-agent server --host 127.0.0.1 --port 41001", + "110 100 119 node /x/codex-acp", + "120 110 118 /x/bin/codex app-server", + "130 120 10 sleep 300", + "200 1 120 /x/bin/sandbox-agent server --host 127.0.0.1 --port 41002", + "210 200 119 node /x/codex-acp", + "220 210 118 /x/bin/codex app-server", + "230 220 10 sleep 300", + ].join("\n"); + const { sandbox, calls } = sandboxWith(rows); + const result = await reapLeakedExecChildren({ + sandbox, + sandboxAgentPort: 41002, + turnElapsedMs: 20_000, + log: vi.fn(), + }); + expect(result).toEqual({ killed: 1 }); + expect(calls[1]).toMatchObject({ command: "kill", args: ["-9", "230"] }); + }); + + it("kills nothing, and says why, when the sandbox has no one-off process API", async () => { + const log = vi.fn(); + expect( + await reapLeakedExecChildren({ + sandbox: {}, + sandboxAgentPort: LIVE_PORT, + turnElapsedMs: 1, + log, + }), + ).toEqual({ + killed: 0, + skipped: "no-run-process", + }); + }); + + it("gives up quietly when `ps` is missing or speaks a different dialect", async () => { + const log = vi.fn(); + const sandbox = { + runProcess: vi.fn(async () => { + throw new Error("ps: unrecognized option -eo"); + }), + }; + expect( + await reapLeakedExecChildren({ + sandbox, + sandboxAgentPort: LIVE_PORT, + turnElapsedMs: 1, + log, + }), + ).toEqual({ killed: 0, skipped: "ps-failed" }); + expect(log).toHaveBeenCalledWith( + expect.stringContaining("skipped=ps-failed"), + ); + }); + + it("kills nothing when the app-server cannot be identified", async () => { + const { sandbox } = sandboxWith(" 10 1 5 node other.js"); + expect( + await reapLeakedExecChildren({ + sandbox, + sandboxAgentPort: LIVE_PORT, + turnElapsedMs: 1, + log: vi.fn(), + }), + ).toEqual({ killed: 0, skipped: "no-app-server" }); + }); + + it("kills nothing when the harness already cleaned up after itself", async () => { + const { sandbox, calls } = sandboxWith( + [ + `67965 58 120 /x/bin/sandbox-agent server --port ${LIVE_PORT}`, + "68029 67965 116 /x/bin/codex app-server", + ].join("\n"), + ); + expect( + await reapLeakedExecChildren({ + sandbox, + sandboxAgentPort: LIVE_PORT, + turnElapsedMs: 1, + log: vi.fn(), + }), + ).toEqual({ killed: 0, skipped: "nothing-to-reap" }); + expect(calls).toHaveLength(1); + }); + + it("refuses to fire when the candidate set is implausibly large", async () => { + const rows = [ + `67965 58 120 /x/bin/sandbox-agent server --port ${LIVE_PORT}`, + "68029 67965 116 /x/bin/codex app-server", + ]; + for (let i = 0; i <= MAX_REAPED; i += 1) { + rows.push(`${70000 + i} 68029 1 worker-${i}`); + } + const { sandbox, calls } = sandboxWith(rows.join("\n")); + expect( + await reapLeakedExecChildren({ + sandbox, + sandboxAgentPort: LIVE_PORT, + turnElapsedMs: 5_000, + log: vi.fn(), + }), + ).toEqual({ killed: 0, skipped: "too-many" }); + expect(calls).toHaveLength(1); + }); + + it("reports a failed kill instead of claiming the leak is gone", async () => { + let seen = 0; + const sandbox = { + runProcess: vi.fn(async () => { + seen += 1; + if (seen === 1) return { stdout: LIVE_PS, exitCode: 0 }; + throw new Error("kill: permission denied"); + }), + }; + expect( + await reapLeakedExecChildren({ + sandbox, + sandboxAgentPort: LIVE_PORT, + turnElapsedMs: 20_000, + log: vi.fn(), + }), + ).toEqual({ killed: 0, skipped: "kill-failed" }); + }); + + it("reports a non-zero kill exit instead of claiming the leak is gone", async () => { + let seen = 0; + const log = vi.fn(); + const sandbox = { + runProcess: vi.fn(async () => { + seen += 1; + return seen === 1 + ? { stdout: LIVE_PS, exitCode: 0 } + : { stdout: "", exitCode: 1 }; + }), + }; + expect( + await reapLeakedExecChildren({ + sandbox, + sandboxAgentPort: LIVE_PORT, + turnElapsedMs: 20_000, + log, + }), + ).toEqual({ killed: 0, skipped: "kill-failed" }); + expect(log).toHaveBeenCalledWith( + expect.stringContaining("kill exited with status 1"), + ); + }); +}); diff --git a/services/runner/tests/unit/sandbox-agent-acp-fetch.test.ts b/services/runner/tests/unit/sandbox-agent-acp-fetch.test.ts index 56a53cd3b02..8b58a885f95 100644 --- a/services/runner/tests/unit/sandbox-agent-acp-fetch.test.ts +++ b/services/runner/tests/unit/sandbox-agent-acp-fetch.test.ts @@ -15,6 +15,7 @@ import assert from "node:assert/strict"; import { createAcpDispatcher, createAcpFetch, + withSandboxGoneReport, } from "../../src/engines/sandbox_agent/acp-fetch.ts"; const envKeys = [ @@ -79,3 +80,49 @@ describe("createAcpFetch", () => { assert.equal(typeof acpFetch, "function"); }); }); + +/** + * The turn's own socket is the first thing to learn that a remote sandbox was deleted: Daytona + * answers `404 SANDBOX_NOT_FOUND` from its proxy while the ACP transport swallows the failure and + * the pending prompt never settles. This wrapper is how that death reaches the liveness probe. + */ +describe("withSandboxGoneReport", () => { + const goneResponse = () => + new Response("not found: sandbox a476c238 not found", { + status: 404, + headers: { "x-daytona-error-code": "SANDBOX_NOT_FOUND" }, + }); + + it("reports a provider answer that names the sandbox as gone", async () => { + const reasons: string[] = []; + const wrapped = withSandboxGoneReport( + (async () => goneResponse()) as unknown as typeof fetch, + { onSandboxGone: (reason) => reasons.push(reason) }, + ); + + const response = await wrapped("http://sandbox/v1/acp/session"); + + assert.equal(reasons.length, 1); + assert.ok(reasons[0].includes("SANDBOX_NOT_FOUND")); + // The body must still be readable by the ACP client that asked for it. + assert.ok((await response.text()).includes("a476c238")); + }); + + it("reports nothing for an ordinary answer", async () => { + const reasons: string[] = []; + const wrapped = withSandboxGoneReport( + (async () => + new Response("{}", { status: 200 })) as unknown as typeof fetch, + { onSandboxGone: (reason) => reasons.push(reason) }, + ); + + await wrapped("http://sandbox/v1/acp/session"); + + assert.equal(reasons.length, 0); + }); + + it("is the identity when no reporter is wired", () => { + const inner = (async () => new Response("{}")) as unknown as typeof fetch; + assert.equal(withSandboxGoneReport(inner), inner); + }); +}); diff --git a/services/runner/tests/unit/sandbox-agent-mount.test.ts b/services/runner/tests/unit/sandbox-agent-mount.test.ts index 49bb094b72c..7eaccf7b900 100644 --- a/services/runner/tests/unit/sandbox-agent-mount.test.ts +++ b/services/runner/tests/unit/sandbox-agent-mount.test.ts @@ -245,6 +245,58 @@ function notMountedThenAlive(): (cwd: string) => Promise { } describe("mountStorage", () => { + it("cancels a slow local mount promptly and stops a geesefs handle that arrives late", async () => { + const controller = new AbortController(); + let mountStarted!: () => void; + const started = new Promise((resolve) => { + mountStarted = resolve; + }); + let finishMount!: (attempt: { stop: () => Promise }) => void; + const lateMount = new Promise<{ stop: () => Promise }>((resolve) => { + finishMount = resolve; + }); + let stopped = 0; + const mount = mountStorage("/work/cwd", CREDS, { + signal: controller.signal, + checkMounted: async () => false, + runGeesefs: async () => { + mountStarted(); + return lateMount; + }, + unmountDeps: { + runUnmount: async () => {}, + checkMountpoint: async () => "gone", + }, + log: SILENT, + }); + + await started; + controller.abort(); + await assert.rejects( + () => + Promise.race([ + mount, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("local mount did not cancel")), + 4_000, + ), + ), + ]), + /acquisition was aborted/, + ); + + finishMount({ + stop: async () => { + stopped += 1; + }, + }); + for (let i = 0; i < 10 && stopped === 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + assert.equal(stopped, 1, "the late geesefs process is stopped"); + }); + it("builds the geesefs command with creds in env, not argv", async () => { let seenArgs: string[] = []; let seenEnv: Record = {}; @@ -502,6 +554,87 @@ describe("discoverTunnelEndpoint (remote)", () => { }); describe("mountStorageRemote", () => { + it("cancels a slow Daytona mount command promptly", async () => { + const controller = new AbortController(); + let mountStarted!: () => void; + const started = new Promise((resolve) => { + mountStarted = resolve; + }); + const sandbox = { + runProcess: async (opts: { args?: string[] }) => { + if ((opts.args?.[1] ?? "").includes("geesefs --log-file")) { + mountStarted(); + return new Promise<{ exitCode: number }>(() => {}); + } + return { exitCode: 0 }; + }, + }; + const mount = mountStorageRemote(sandbox, "/home/sandbox/work", CREDS, { + endpoint: "https://abc.ngrok.io", + signal: controller.signal, + log: SILENT, + }); + + await started; + controller.abort(); + await assert.rejects( + () => + Promise.race([ + mount, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("remote mount did not cancel")), + 4_000, + ), + ), + ]), + /acquisition was aborted/, + ); + }); + + it("detaches a remote mount that completes after cancellation", async () => { + const controller = new AbortController(); + const unmountCalls: string[] = []; + let finishMount!: (value: { exitCode: number }) => void; + const mountFinished = new Promise<{ exitCode: number }>((resolve) => { + finishMount = resolve; + }); + let mountStarted!: () => void; + const started = new Promise((resolve) => { + mountStarted = resolve; + }); + const sandbox = { + runProcess: async (opts: { command: string; args?: string[] }) => { + const command = opts.args?.[1] ?? ""; + if (command.includes("geesefs --log-file")) { + mountStarted(); + return mountFinished; + } + if (command.includes("fusermount") || command.includes("umount")) { + unmountCalls.push(command); + } + return { exitCode: 0 }; + }, + }; + const mount = mountStorageRemote(sandbox, "/home/sandbox/work", CREDS, { + endpoint: "https://abc.ngrok.io", + signal: controller.signal, + log: SILENT, + }); + + await started; + controller.abort(); + await assert.rejects(mount, /acquisition was aborted/); + finishMount({ exitCode: 0 }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + assert.equal( + unmountCalls.length, + 3, + "cleans before mounting, on cancellation, and after the mount completes", + ); + }); + it("detaches an existing mount before starting geesefs", async () => { const commands: string[] = []; const sandbox = { diff --git a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts index 4b4ef74564e..75245aceb74 100644 --- a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts +++ b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts @@ -36,20 +36,31 @@ import { shouldSuppressPausedToolCallUpdate, } from "../../src/engines/sandbox_agent/runtime-policy.ts"; import { mountStorage } from "../../src/engines/sandbox_agent/mount.ts"; +import { withSandboxGoneReport } from "../../src/engines/sandbox_agent/acp-fetch.ts"; +import { SANDBOX_GONE_MESSAGE } from "../../src/engines/sandbox_agent/errors.ts"; import { buildPiGateEnvelope } from "../../src/engines/sandbox_agent/pi-gate-envelope.ts"; import { appendPlatformGuidance } from "../../src/engines/sandbox_agent/system-prompt-appendix.ts"; import { platformGuidanceAppendix } from "../../src/engines/sandbox_agent/platform-guidance.ts"; import type { PermissionDecision } from "../../src/responder.ts"; import { + acquireEnvironment, + runTurn, runSandboxAgent, type SandboxAgentDeps, } from "../../src/engines/sandbox_agent.ts"; import { resetRunnerConfigCache } from "../../src/config/runner-config.ts"; +import { USER_STOP_ABORT_REASON } from "../../src/sessions/stop-signal.ts"; import { fakeHarness, flushPromises, type FakeOptions, } from "../utils/sandbox-agent-harness.ts"; +import { + findExecution, + registerExecution, + resetExecutionsForTest, +} from "../../src/sessions/execution-registry.ts"; +import { applyCommand } from "../../src/sessions/control-channel.ts"; // Orchestration cases include Daytona runs: enable it (with a provisioning credential) on top of // the hermetic scrub, then drop the memoized config so the run plan reads the enabled set. @@ -60,6 +71,7 @@ beforeEach(() => { }); afterEach(() => { + resetExecutionsForTest(); vi.unstubAllGlobals(); }); @@ -96,6 +108,79 @@ describe("PendingApprovalPauseController", () => { }); describe("runSandboxAgent orchestration", () => { + for (const providerName of ["local", "daytona"] as const) { + it(`a Stop preempts slow ${providerName} acquisition and cleans a late sandbox`, async () => { + const { deps } = fakeHarness(); + const delegateStart = deps.startSandboxAgent!; + let releaseCreate!: (sandboxId: string) => void; + const slowCreate = new Promise((resolve) => { + releaseCreate = resolve; + }); + let markCreateStarted!: () => void; + const createStarted = new Promise((resolve) => { + markCreateStarted = resolve; + }); + let markCleaned!: () => void; + const cleaned = new Promise((resolve) => { + markCleaned = resolve; + }); + let destroys = 0; + deps.buildSandboxProvider = (() => ({ + name: providerName, + create: () => { + markCreateStarted(); + return slowCreate; + }, + async destroy() { + destroys += 1; + markCleaned(); + }, + async getUrl() { + return "http://sandbox.invalid"; + }, + })) as any; + deps.startSandboxAgent = (async (options: any) => { + await options.sandbox.create(); + return delegateStart(options); + }) as any; + + const controller = new AbortController(); + const acquire = acquireEnvironment( + { + harness: "claude", + sandbox: providerName, + messages: [{ role: "user", content: "start slowly" }], + }, + deps, + controller.signal, + ); + await createStarted; + controller.abort(USER_STOP_ABORT_REASON); + + const result = await Promise.race([ + acquire, + new Promise((_resolve, reject) => + setTimeout( + () => reject(new Error("Stop exceeded the delivery timeout")), + 4_000, + ), + ), + ]); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /acquisition was aborted/); + + releaseCreate(`${providerName}-late-id`); + await Promise.race([ + cleaned, + new Promise((_resolve, reject) => + setTimeout(() => reject(new Error("late sandbox leaked")), 4_000), + ), + ]); + assert.equal(destroys, 1); + }); + } + // NOTE: in-band redaction of the LIVE event stream / result / trace-start input was a // daytona-secret-materialization concept that was not adopted. Redaction happens at the // durable/exported sinks (persisted transcript + exported spans; see redaction-sinks.test.ts), @@ -146,6 +231,74 @@ describe("runSandboxAgent orchestration", () => { assert.equal(calls.workspaceCleanup, 1); }); + it("replays rebuilt history after an evicted local Pi load cannot verify native turns", async () => { + const request: AgentRunRequest = { + harness: "pi_core", + sandbox: "local", + messages: [ + { role: "user", content: "Remember the codeword KIWI-9" }, + { role: "assistant", content: "I will remember it." }, + { role: "user", content: "What was the codeword?" }, + ], + }; + const { calls, deps } = fakeHarness(); + const acquired = await acquireEnvironment(request, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + + try { + const result = await runTurn( + acquired.env, + request, + undefined, + undefined, + { loaded: true, nativeHistoryVerified: false }, + ); + + assert.equal(result.ok, true); + const prompt = calls.promptBlocks?.[0]?.text ?? ""; + assert.match(prompt, /^Conversation so far:/); + assert.match(prompt, /KIWI-9/); + assert.match(prompt, /The user now says:\nWhat was the codeword\?$/); + } finally { + await acquired.env.destroy(); + } + }); + + it("keeps the last-message-only path for a verified Daytona native load", async () => { + const request: AgentRunRequest = { + harness: "pi_core", + sandbox: "daytona", + messages: [ + { role: "user", content: "Remember the codeword KIWI-9" }, + { role: "assistant", content: "I will remember it." }, + { role: "user", content: "What was the codeword?" }, + ], + }; + const { calls, deps } = fakeHarness(); + deps.prepareDaytonaPiAssets = (async () => true) as any; + const acquired = await acquireEnvironment(request, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + + try { + const result = await runTurn( + acquired.env, + request, + undefined, + undefined, + { loaded: true, nativeHistoryVerified: true }, + ); + + assert.equal(result.ok, true); + assert.deepEqual(calls.promptBlocks, [ + { type: "text", text: "What was the codeword?" }, + ]); + } finally { + await acquired.env.destroy(); + } + }); + it("passes the live turn credential provider to the trace exporter", async () => { const { calls, deps } = fakeHarness(); let authorization = "Secret initial"; @@ -615,6 +768,48 @@ describe("runSandboxAgent orchestration", () => { rmSync(cwd, { recursive: true, force: true }); }); + it("backs the local Pi transcript directory with the active durable cwd mount", async () => { + const { calls, deps } = fakeHarness(); + deps.signSessionMountCredentials = async () => ({ + region: "us-east-1", + bucket: "bucket", + prefix: "mounts/project/session", + accessKey: "test-access-key", + secretKey: "test-secret-key", + projectId: "project", + }); + deps.mountStorage = async () => true; + deps.unmountStorage = async () => true; + deps.hydrateHarnessSessionFromDurable = async () => {}; + + const request: AgentRunRequest = { + harness: "pi_core", + sandbox: "local", + sessionId: "session-local-rebuild", + runContext: { project: { id: "project" } }, + telemetry: { + exporters: { + otlp: { headers: { authorization: "ApiKey test" } }, + }, + }, + messages: [{ role: "user", content: "continue" }], + }; + const acquired = await acquireEnvironment(request, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + + try { + assert.equal(acquired.env.nativeHistoryDurable, true); + assert.equal( + (calls.providerArgs[1] as Record) + .PI_CODING_AGENT_SESSION_DIR, + "/tmp/agenta/mounts/project/session/agents/sessions/pi", + ); + } finally { + await acquired.env.destroy(); + } + }); + it("creates the configured Pi transcript directory inside a Daytona cwd", async () => { const { calls, deps } = fakeHarness(); deps.prepareDaytonaPiAssets = (async () => true) as any; @@ -2414,6 +2609,118 @@ describe("runSandboxAgent default ApprovalResponder wiring", () => { assert.deepEqual(calls.permissionReplies, []); }); + it("marks a paused turn settled after its cancellable teardown window", async () => { + const { deps } = depsWithDefaultResponder(); + const sessionId = "conv-paused-registry"; + const turnId = "turn-paused-registry"; + registerExecution({ + projectId: "11111111-1111-4111-8111-111111111111", + sessionId, + turnId, + startedAt: Date.now(), + abort: () => {}, + }); + + const result = await runSandboxAgent( + { + harness: "claude", + sessionId, + turnId, + permissions: { default: "ask" }, + messages: [{ role: "user", content: "edit the file" }], + }, + undefined, + undefined, + deps, + ); + + assert.equal(result.ok, true); + if (!result.ok) return; + assert.equal(result.stopReason, "paused"); + assert.equal( + findExecution("11111111-1111-4111-8111-111111111111", sessionId)?.settled, + true, + ); + }); + + it("converts a Stop during pause teardown into the turn's cancelled outcome", async () => { + let markPauseTeardownStarted!: () => void; + const pauseTeardownStarted = new Promise((resolve) => { + markPauseTeardownStarted = resolve; + }); + let releasePauseTeardown!: () => void; + const pauseTeardownMayFinish = new Promise((resolve) => { + releasePauseTeardown = resolve; + }); + const { deps } = fakeHarness({ + emitPermission: true, + hangPrompt: true, + afterDestroySession: async () => { + markPauseTeardownStarted(); + await pauseTeardownMayFinish; + }, + }); + delete deps.responderFactory; + const startSandboxAgent = deps.startSandboxAgent!; + deps.startSandboxAgent = async (options) => { + const sandbox = await startSandboxAgent(options); + const cancellable = sandbox as unknown as { + destroySession: (id: string) => Promise; + cancelSession?: (id: string) => Promise; + }; + cancellable.cancelSession = (id) => cancellable.destroySession(id); + return sandbox; + }; + + const projectId = "11111111-1111-4111-8111-111111111111"; + const sessionId = "conv-stop-during-pause-teardown"; + const turnId = "turn-stop-during-pause-teardown"; + const controller = new AbortController(); + registerExecution({ + projectId, + sessionId, + turnId, + startedAt: Date.now(), + abort: () => controller.abort(USER_STOP_ABORT_REASON), + }); + + const turn = runSandboxAgent( + { + harness: "claude", + sessionId, + turnId, + permissions: { default: "ask" }, + messages: [{ role: "user", content: "edit the file" }], + }, + undefined, + controller.signal, + deps, + ); + + await pauseTeardownStarted; + const outcome = await applyCommand( + { + id: "command-stop-during-pause-teardown", + projectId, + sessionId, + kind: "cancel", + target: { turnId, expectedTurnId: turnId }, + createdAt: new Date().toISOString(), + }, + { report: async () => {} }, + ); + assert.equal(outcome.execution.state, "stopped"); + + releasePauseTeardown(); + const result = await turn; + + assert.equal(result.ok, true); + if (!result.ok) return; + assert.equal(result.stopReason, "cancelled"); + assert.equal(result.cancelSettled, true); + assert.equal(findExecution(projectId, sessionId)?.settled, true); + }); + it("effective ask with no decision pauses the tool, no harness reply (F-024)", async () => { const { calls, deps } = depsWithDefaultResponder(); @@ -3210,3 +3517,118 @@ describe("runTurn run-limits deadline (split path)", () => { assert.equal(calls.sandboxDestroyed, 1); }); }); + +/** + * The Daytona sandbox-gone path, end to end through the real environment wiring. + * + * On 2026-09-04 an isolated re-run showed the full cost of the gap this closes: the sandbox was + * deleted at 16:26:31, the runner's own socket was told `SANDBOX_NOT_FOUND` at 16:26:37, and the + * turn still beat `running=true` for THIRTY minutes. Nothing detected the death. What finally + * ended the turn was the 30 minute per-tool-call deadline + * (`[run-limits] tool call ... exceeded 1800000ms`), and only then did the turn's error and done + * records persist, 27 minutes after the client had already given up. + * + * Everything downstream of the turn ending is already correct: the error terminal, the records, + * the `running=false` beat that clears the row, the teardown. The only defect was WHEN the turn + * ended. So these tests pin the trigger and the terminal it produces, which is what puts all of + * that 27 minutes earlier. + */ +describe("a sandbox the provider deletes under a running turn", () => { + /** Daytona's real answer for a deleted sandbox, from the runner log of that re-run. */ + const goneAnswer = () => + new Response( + "not found: sandbox 39f3aa96-ddc7-4417-b8ab-71804894edf6 not found, " + + "it may have been deleted or stopped - inspect audit logs for more info", + { status: 404, headers: { "x-daytona-error-code": "SANDBOX_NOT_FOUND" } }, + ); + + /** + * Production's reporter over a fake socket. The double stands in for the network only: the + * wrapper, the latch and the arming are the real ones, so this exercises the wiring rather + * than a copy of it. + */ + function harnessWithGoneSocket(answer: () => Response) { + const fake = fakeHarness({ hangPrompt: true }); + fake.deps.createAcpFetch = ((_dispatcher: unknown, options: any) => + withSandboxGoneReport( + (async () => answer()) as unknown as typeof fetch, + options, + )) as any; + return fake; + } + + /** Let the run reach its prompt, which is where the real turn sits when its sandbox dies. */ + async function waitForStartedTurn(calls: { startOptions: any }) { + for (let i = 0; i < 50 && !calls.startOptions; i += 1) + await flushPromises(); + assert.ok(calls.startOptions, "the run should have started its sandbox"); + await flushPromises(); + } + + it("ends the turn with a sandbox_gone error terminal, from the turn's own socket", async () => { + const { calls, deps, events } = harnessWithGoneSocket(goneAnswer); + + const run = runSandboxAgent( + { + harness: "claude", + sessionId: "conv-sandbox-deleted", + messages: [{ role: "user", content: "run one shell command" }], + } as AgentRunRequest, + undefined, + undefined, + deps, + ); + await waitForStartedTurn(calls); + + // The turn is parked on a prompt that can never settle, exactly as on 2026-09-04. Its own + // socket is the next thing to speak, and what it says is that the sandbox is gone. + await (calls.startOptions.fetch as typeof fetch)( + "http://sandbox/v1/acp/session", + ); + const result = await run; + + // The turn RETURNED rather than hanging for thirty minutes, and it returned as this error. + assert.equal(result.ok, false); + if (result.ok) return; + assert.equal(result.error, SANDBOX_GONE_MESSAGE); + // The terminal the client reads, and the record that persists at this moment. + const errorEvent = events.find((event) => event.type === "error") as any; + assert.ok(errorEvent, "the run should emit an error terminal"); + assert.equal(errorEvent.code, "sandbox_gone"); + // The teardown ran, so the sandbox and its slot are reclaimed here rather than at eviction. + assert.equal(calls.sandboxDestroyed, 1); + }); + + it("keeps running when the same socket merely returns an ordinary error", async () => { + // A 502 from the proxy is a blip, not a death. Nothing must end the turn on it, or a + // transient network fault would kill healthy runs. + const { calls, deps } = harnessWithGoneSocket( + () => new Response("", { status: 502 }), + ); + + const run = runSandboxAgent( + { + harness: "claude", + sessionId: "conv-proxy-blip", + messages: [{ role: "user", content: "run one shell command" }], + } as AgentRunRequest, + undefined, + undefined, + deps, + ); + await waitForStartedTurn(calls); + + await (calls.startOptions.fetch as typeof fetch)( + "http://sandbox/v1/acp/session", + ); + for (let i = 0; i < 20; i += 1) await flushPromises(); + + // Still parked on its prompt: no terminal, no teardown. + assert.equal(calls.sandboxDestroyed, 0); + const settled = await Promise.race([ + run.then(() => "settled" as const), + Promise.resolve("pending" as const), + ]); + assert.equal(settled, "pending"); + }); +}); diff --git a/services/runner/tests/unit/sandbox-gone.test.ts b/services/runner/tests/unit/sandbox-gone.test.ts new file mode 100644 index 00000000000..0c3b52b651d --- /dev/null +++ b/services/runner/tests/unit/sandbox-gone.test.ts @@ -0,0 +1,214 @@ +/** + * A REMOTE sandbox does not refuse the socket when it dies. + * + * Daytona keeps the proxy host up after the sandbox is deleted and answers every request for it + * with `404` + `x-daytona-error-code: SANDBOX_NOT_FOUND`. The liveness probe reads any HTTP answer + * as alive on purpose, so that answer used to mean "still there": on 2026-09-04 a turn whose + * sandbox was deleted under it kept heartbeating `running=true` for five minutes and only stopped + * because the runner process was terminated. + * + * These tests hold the recognition rule. It has to be narrow in both directions: it must catch the + * provider's verdict, and it must not read a healthy answer, an unrelated error, or a 200 body that + * merely quotes the prose as a death. + */ + +import { describe, it, expect, vi } from "vitest"; + +import { + createSandboxGoneLatch, + sandboxGoneReason, +} from "../../src/engines/sandbox_agent/sandbox-gone.ts"; + +/** The shape the probe and the ACP transport both hand to the predicate. */ +function answer(status: number, headers: Record = {}) { + const lower = new Map( + Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v]), + ); + return { + status, + headers: { get: (name: string) => lower.get(name.toLowerCase()) ?? null }, + }; +} + +/** The exact answer Daytona gave for the deleted sandbox on 2026-09-04. */ +const DAYTONA_BODY = + "not found: sandbox a476c238-dfdb-492c-bb4a-0ca15f42fddf not found, " + + "it may have been deleted or stopped - inspect audit logs for more info"; + +describe("sandboxGoneReason", () => { + it("reads the provider's own error code as a death", () => { + const reason = sandboxGoneReason( + answer(404, { "x-daytona-error-code": "SANDBOX_NOT_FOUND" }), + ); + + expect(reason).toBeTruthy(); + expect(reason).toContain("SANDBOX_NOT_FOUND"); + }); + + it("reads the provider's prose as a death when no code header rides along", () => { + expect(sandboxGoneReason(answer(404), DAYTONA_BODY)).toBeTruthy(); + }); + + it("keeps a bare 404 alive: the health route may simply not exist", () => { + expect(sandboxGoneReason(answer(404), "Not Found")).toBeUndefined(); + }); + + it("keeps 401 alive: unauthorised proves something is listening", () => { + expect(sandboxGoneReason(answer(401))).toBeUndefined(); + }); + + it("keeps a 502 alive: a proxy blip is not a deleted sandbox", () => { + expect(sandboxGoneReason(answer(502), "")).toBeUndefined(); + }); + + it("ignores the prose in a SUCCESSFUL answer, which proves the sandbox answered", () => { + expect(sandboxGoneReason(answer(200), DAYTONA_BODY)).toBeUndefined(); + }); + + it("ignores an unrelated provider error code", () => { + expect( + sandboxGoneReason(answer(400, { "x-daytona-error-code": "BAD_REQUEST" })), + ).toBeUndefined(); + }); + + /* + * A stopped or archived sandbox is a RESUMABLE state the provider itself handles, and the + * reconnect ladder can legitimately meet either one while it brings a parked sandbox back. + * Reading them as death would end a turn on a sandbox that is about to answer. + */ + it("keeps a stopped sandbox alive: the provider can resume it", () => { + expect( + sandboxGoneReason( + answer(404, { "x-daytona-error-code": "SANDBOX_STOPPED" }), + ), + ).toBeUndefined(); + }); + + it("keeps an archived sandbox alive, for the same reason", () => { + expect( + sandboxGoneReason( + answer(404, { "x-daytona-error-code": "SANDBOX_ARCHIVED" }), + ), + ).toBeUndefined(); + }); +}); + +/** An armed latch, which is what every caller past acquire holds. */ +function armedLatch() { + const latch = createSandboxGoneLatch(); + latch.arm(); + return latch; +} + +describe("sandbox gone latch", () => { + it("delivers the first reason to a listener that subscribed earlier", () => { + const latch = armedLatch(); + const listener = vi.fn(); + + latch.subscribe(listener); + latch.note("deleted"); + + expect(listener).toHaveBeenCalledTimes(1); + expect(listener).toHaveBeenCalledWith("deleted"); + expect(latch.reason()).toBe("deleted"); + }); + + it("delivers to a listener that subscribed after the death", () => { + const latch = armedLatch(); + const listener = vi.fn(); + + latch.note("deleted"); + latch.subscribe(listener); + + expect(listener).toHaveBeenCalledWith("deleted"); + }); + + it("keeps one death for one sandbox, however many requests observe it", () => { + const latch = armedLatch(); + const listener = vi.fn(); + latch.subscribe(listener); + + latch.note("first"); + latch.note("second"); + latch.note("third"); + + expect(listener).toHaveBeenCalledTimes(1); + expect(latch.reason()).toBe("first"); + }); + + it("reports nothing while the sandbox still answers", () => { + expect(armedLatch().reason()).toBeUndefined(); + }); + + it("drops a listener that unsubscribed, so a warm sandbox keeps no dead turns", () => { + const latch = armedLatch(); + const finishedTurn = vi.fn(); + const currentTurn = vi.fn(); + + const unsubscribe = latch.subscribe(finishedTurn); + unsubscribe(); + latch.subscribe(currentTurn); + latch.note("deleted"); + + expect(finishedTurn).not.toHaveBeenCalled(); + expect(currentTurn).toHaveBeenCalledTimes(1); + }); + + it("survives a listener that throws, and still tells the others", () => { + const latch = armedLatch(); + const other = vi.fn(); + latch.subscribe(() => { + throw new Error("listener fault"); + }); + latch.subscribe(other); + + latch.note("deleted"); + + expect(other).toHaveBeenCalledTimes(1); + expect(latch.reason()).toBe("deleted"); + }); +}); + +/* + * The startup window. The same fetch carries the SDK's health wait during acquire, which polls a + * sandbox that is still coming up and tolerates a provider error by design. On a warm resume the + * proxy can lag its own control plane and answer "not found" for a sandbox it has not finished + * re-exposing. The latch is one-way, so a report from that window must be discarded, or the first + * turn on a healthy sandbox is killed. + */ +describe("sandbox gone latch before it is armed", () => { + it("discards a gone report seen before acquire resolves", () => { + const latch = createSandboxGoneLatch(); + const listener = vi.fn(); + latch.subscribe(listener); + + latch.note("provider reports the sandbox is gone (HTTP 404)"); + + expect(listener).not.toHaveBeenCalled(); + expect(latch.reason()).toBeUndefined(); + }); + + it("latches the SAME report once acquire has resolved", () => { + const latch = createSandboxGoneLatch(); + const listener = vi.fn(); + latch.subscribe(listener); + + latch.note("provider reports the sandbox is gone (HTTP 404)"); + latch.arm(); + latch.note("provider reports the sandbox is gone (HTTP 404)"); + + expect(listener).toHaveBeenCalledTimes(1); + expect(latch.reason()).toBe( + "provider reports the sandbox is gone (HTTP 404)", + ); + }); + + it("does not remember a discarded report: arming alone declares nothing", () => { + const latch = createSandboxGoneLatch(); + + latch.note("seen during acquire"); + latch.arm(); + + expect(latch.reason()).toBeUndefined(); + }); +}); diff --git a/services/runner/tests/unit/sandbox-lifecycle.test.ts b/services/runner/tests/unit/sandbox-lifecycle.test.ts index 4d2add811d9..e77349e8828 100644 --- a/services/runner/tests/unit/sandbox-lifecycle.test.ts +++ b/services/runner/tests/unit/sandbox-lifecycle.test.ts @@ -36,6 +36,8 @@ interface FakeOpts { * pauseSandbox() throws while retaining its provider handles for the delete fallback. */ pauseThrows?: boolean; + /** Abort after environment acquisition, when the harness prompt starts. */ + onPrompt?: () => void; } function fakeSandbox(sandboxId: string | undefined, opts: FakeOpts = {}) { @@ -59,6 +61,7 @@ function fakeSandbox(sandboxId: string | undefined, opts: FakeOpts = {}) { onEvent() {}, onPermissionRequest() {}, async prompt() { + opts.onPrompt?.(); if (opts.promptThrows) throw new Error("harness exploded"); return { stopReason: opts.stopReason ?? "complete", @@ -366,9 +369,10 @@ describe("remote sandbox teardown", () => { }); it("destroys (not parks) when the run is aborted", async () => { - const { calls, deps } = fakeSandbox("sbx-99"); const controller = new AbortController(); - controller.abort(); + const { calls, deps } = fakeSandbox("sbx-99", { + onPrompt: () => controller.abort(), + }); await runSandboxAgent(daytonaRequest, undefined, controller.signal, deps); assert.equal(calls.paused, 0, "an aborted run must not park"); assert.equal(calls.destroyed, 1); diff --git a/services/runner/tests/unit/sandbox-liveness.test.ts b/services/runner/tests/unit/sandbox-liveness.test.ts new file mode 100644 index 00000000000..e1f30c837cb --- /dev/null +++ b/services/runner/tests/unit/sandbox-liveness.test.ts @@ -0,0 +1,325 @@ +/** + * A sandbox that dies under a running turn must end the turn, not hang it. + * + * The ACP prompt the turn is parked on can never settle once the sandbox process is gone: the + * transport's read loop swallows the severed stream and never rejects the pending request. The + * existing run limits do not save it either — `notePaused()` retires all of them the moment the + * turn parks for a human, which is exactly when a long turn is most likely to outlive its + * sandbox. So the runner probes the sandbox's own HTTP surface, independently of the wedged ACP + * channel. These tests hold the probe's contract: it tolerates a blip, it declares death once, + * and it never fires after the turn released it. Issue #6418. + */ + +import { describe, it, expect, vi, afterEach } from "vitest"; + +import { + DEFAULT_PROBE_FAILURES, + DEFAULT_PROBE_INTERVAL_MS, + DEFAULT_PROBE_TIMEOUT_MS, + PROBE_FAILURES_ENV, + PROBE_INTERVAL_ENV, + httpLivenessProbe, + resolveSandboxLivenessLimits, + SandboxGoneError, + sandboxHealthUrl, + startSandboxLivenessProbe, + type Clock, + type SandboxLivenessLimits, +} from "../../src/engines/sandbox_agent/sandbox-liveness.ts"; +import { SANDBOX_GONE_MARKER } from "../../src/engines/sandbox_agent/errors.ts"; +import { createSandboxGoneLatch } from "../../src/engines/sandbox_agent/sandbox-gone.ts"; + +/** A clock whose timers only run when the test says so, in scheduled order. */ +function fakeClock(): Clock & { tick(): Promise; pending(): number } { + let nextId = 1; + const timers = new Map void; at: number }>(); + let now = 0; + + const clock = { + setTimeout(fn: () => void, ms: number) { + const id = nextId++; + timers.set(id, { fn, at: now + ms }); + return id as unknown as NodeJS.Timeout; + }, + clearTimeout(handle: NodeJS.Timeout) { + timers.delete(handle as unknown as number); + }, + pending: () => timers.size, + /** Run the earliest pending timer, then drain the microtask queue. */ + async tick() { + const entries = [...timers.entries()].sort((a, b) => a[1].at - b[1].at); + const next = entries[0]; + if (!next) return; + timers.delete(next[0]); + now = next[1].at; + next[1].fn(); + await new Promise((resolve) => setTimeout(resolve, 0)); + }, + }; + return clock; +} + +const limits: SandboxLivenessLimits = { + intervalMs: 1_000, + timeoutMs: 500, + failureThreshold: 3, +}; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("sandbox liveness probe", () => { + it.each([ + ["local", "http://127.0.0.1:43123/ui/", "http://127.0.0.1:43123/v1/health"], + [ + "Daytona", + "https://3000-sandbox-id.proxy.daytona.works/ui/", + "https://3000-sandbox-id.proxy.daytona.works/v1/health", + ], + ])( + "derives the daemon health route from a %s inspector URL", + (_provider, inspectorUrl, expected) => { + expect(sandboxHealthUrl({ inspectorUrl })).toBe(expected); + }, + ); + + it("declares the sandbox gone after the threshold of consecutive failures", async () => { + const onGone = vi.fn(); + const probe = vi.fn().mockRejectedValue(new Error("ECONNREFUSED")); + const clock = fakeClock(); + + const handle = startSandboxLivenessProbe({ probe, limits, onGone, clock }); + + // Each pass is one interval timer, then the probe's own timeout timer. + for (let i = 0; i < 3; i++) { + await clock.tick(); // interval fires, probe rejects + await clock.tick(); // the (already settled) probe timeout is cleared/drained + } + + expect(probe).toHaveBeenCalledTimes(3); + expect(onGone).toHaveBeenCalledTimes(1); + expect(onGone.mock.calls[0][0]).toContain(SANDBOX_GONE_MARKER); + handle.dispose(); + }); + + it("tolerates a blip: one failure between successes is not a death", async () => { + const onGone = vi.fn(); + const probe = vi + .fn() + .mockRejectedValueOnce(new Error("transient")) + .mockResolvedValue({ id: "session-1" }); + const clock = fakeClock(); + + const handle = startSandboxLivenessProbe({ probe, limits, onGone, clock }); + + for (let i = 0; i < 8; i++) await clock.tick(); + + expect(onGone).not.toHaveBeenCalled(); + expect(handle.failures()).toBe(0); + handle.dispose(); + }); + + it("counts a probe that hangs as a failure, so a vanished host is not waited on forever", async () => { + const onGone = vi.fn(); + // The exact #6418 shape: the request neither answers nor refuses. + const probe = vi.fn().mockImplementation(() => new Promise(() => {})); + const clock = fakeClock(); + + const handle = startSandboxLivenessProbe({ probe, limits, onGone, clock }); + + // interval -> probe hangs -> its timeout fires, three times over. + for (let i = 0; i < 6; i++) await clock.tick(); + + expect(onGone).toHaveBeenCalledTimes(1); + expect(onGone.mock.calls[0][0]).toContain("probe timed out"); + handle.dispose(); + }); + + it("fires at most once, and never after dispose", async () => { + const onGone = vi.fn(); + const probe = vi.fn().mockRejectedValue(new Error("gone")); + const clock = fakeClock(); + + const handle = startSandboxLivenessProbe({ probe, limits, onGone, clock }); + handle.dispose(); + + for (let i = 0; i < 10; i++) await clock.tick(); + + expect(probe).not.toHaveBeenCalled(); + expect(onGone).not.toHaveBeenCalled(); + expect(clock.pending()).toBe(0); + }); +}); + +/** A latch the environment already armed, which is what every turn past acquire holds. */ +function armedLatch() { + const latch = createSandboxGoneLatch(); + latch.arm(); + return latch; +} + +/** + * The Daytona case. The proxy answers for a deleted sandbox, so no probe ever fails the weak way + * and the three-strike counter never moves. Both routes below end the turn instead. + */ +describe("a sandbox the provider says is gone", () => { + it("ends the turn on the FIRST such answer, without waiting for the threshold", async () => { + const onGone = vi.fn(); + const probe = vi + .fn() + .mockRejectedValue(new SandboxGoneError("sandbox a476c238 not found")); + const clock = fakeClock(); + + const handle = startSandboxLivenessProbe({ probe, limits, onGone, clock }); + + await clock.tick(); // one interval, one probe + await clock.tick(); + + expect(probe).toHaveBeenCalledTimes(1); + expect(onGone).toHaveBeenCalledTimes(1); + expect(onGone.mock.calls[0][0]).toContain(SANDBOX_GONE_MARKER); + expect(onGone.mock.calls[0][0]).toContain("a476c238"); + handle.dispose(); + }); + + it("ends the turn the moment the ACP transport reports it, with no probe at all", async () => { + const onGone = vi.fn(); + const goneSignal = armedLatch(); + const clock = fakeClock(); + + const handle = startSandboxLivenessProbe({ + probe: vi.fn().mockResolvedValue(200), + goneSignal, + limits, + onGone, + clock, + }); + goneSignal.note("provider reports the sandbox is gone (HTTP 404)"); + + expect(onGone).toHaveBeenCalledTimes(1); + expect(onGone.mock.calls[0][0]).toContain(SANDBOX_GONE_MARKER); + handle.dispose(); + }); + + it("honours the transport's report on a sandbox with no health URL to poll", () => { + const onGone = vi.fn(); + const goneSignal = armedLatch(); + const clock = fakeClock(); + + const handle = startSandboxLivenessProbe({ + goneSignal, + limits, + onGone, + clock, + }); + + expect(clock.pending()).toBe(0); // nothing to poll, so nothing is scheduled + goneSignal.note("deleted"); + + expect(onGone).toHaveBeenCalledTimes(1); + handle.dispose(); + }); + + it("still reports one death when the probe and the transport both see it", async () => { + const onGone = vi.fn(); + const goneSignal = armedLatch(); + const probe = vi + .fn() + .mockRejectedValue(new SandboxGoneError("sandbox gone per probe")); + const clock = fakeClock(); + + const handle = startSandboxLivenessProbe({ + probe, + goneSignal, + limits, + onGone, + clock, + }); + goneSignal.note("sandbox gone per transport"); + await clock.tick(); + await clock.tick(); + + expect(onGone).toHaveBeenCalledTimes(1); + handle.dispose(); + }); + + it("hands the listener back on dispose, so a warm sandbox keeps no finished turns", () => { + const goneSignal = armedLatch(); + const finishedTurn = vi.fn(); + const currentTurn = vi.fn(); + const clock = fakeClock(); + + // Turn 1 runs and ends. Turn 2 starts on the SAME warm environment, so the same latch. + startSandboxLivenessProbe({ + goneSignal, + limits, + onGone: finishedTurn, + clock, + }).dispose(); + const handle = startSandboxLivenessProbe({ + goneSignal, + limits, + onGone: currentTurn, + clock, + }); + + goneSignal.note("deleted"); + + expect(finishedTurn).not.toHaveBeenCalled(); + expect(currentTurn).toHaveBeenCalledTimes(1); + handle.dispose(); + }); +}); + +describe("httpLivenessProbe", () => { + const originalFetch = globalThis.fetch; + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it("rejects with a definitive error when the provider names the sandbox as gone", async () => { + globalThis.fetch = vi.fn().mockResolvedValue( + new Response("not found: sandbox a476c238 not found", { + status: 404, + headers: { "x-daytona-error-code": "SANDBOX_NOT_FOUND" }, + }), + ) as unknown as typeof fetch; + + await expect( + httpLivenessProbe("http://sandbox/v1/health")(), + ).rejects.toBeInstanceOf(SandboxGoneError); + }); + + it("keeps reading an ordinary 404 as alive", async () => { + globalThis.fetch = vi + .fn() + .mockResolvedValue( + new Response("Not Found", { status: 404 }), + ) as unknown as typeof fetch; + + await expect(httpLivenessProbe("http://sandbox/v1/health")()).resolves.toBe( + 404, + ); + }); +}); + +describe("sandbox liveness limits", () => { + it("defaults to one probe per heartbeat interval and three strikes", () => { + expect(resolveSandboxLivenessLimits()).toEqual({ + intervalMs: DEFAULT_PROBE_INTERVAL_MS, + timeoutMs: DEFAULT_PROBE_TIMEOUT_MS, + failureThreshold: DEFAULT_PROBE_FAILURES, + }); + }); + + it("takes an operator override", () => { + vi.stubEnv(PROBE_INTERVAL_ENV, "5000"); + vi.stubEnv(PROBE_FAILURES_ENV, "2"); + + const resolved = resolveSandboxLivenessLimits(); + + expect(resolved.intervalMs).toBe(5_000); + expect(resolved.failureThreshold).toBe(2); + }); +}); diff --git a/services/runner/tests/unit/server.test.ts b/services/runner/tests/unit/server.test.ts index d452842d530..281cba9b69c 100644 --- a/services/runner/tests/unit/server.test.ts +++ b/services/runner/tests/unit/server.test.ts @@ -19,8 +19,17 @@ import { createAgentServer, normalizeKillProjectId, registerShutdownHandler, + runWithKeepalive, + type KeepaliveEngine, type RunAgent, } from "../../src/server.ts"; +import type { SessionEnvironment } from "../../src/engines/sandbox_agent.ts"; +import { SessionPool } from "../../src/engines/sandbox_agent/session-pool.ts"; +import { HEARTBEAT_INTERVAL_SECONDS } from "../../src/sessions/contract.ts"; +import { + liveExecutions, + resetExecutionsForTest, +} from "../../src/sessions/execution-registry.ts"; const TOKEN_ENV = "AGENTA_RUNNER_TOKEN"; const previousToken = process.env[TOKEN_ENV]; @@ -29,6 +38,7 @@ const LIMIT_ENV = "AGENTA_RUNNER_CONCURRENCY_LIMIT"; const previousLimit = process.env[LIMIT_ENV]; afterEach(() => { + resetExecutionsForTest(); vi.restoreAllMocks(); vi.unstubAllEnvs(); if (previousToken === undefined) delete process.env[TOKEN_ENV]; @@ -524,6 +534,300 @@ describe("createAgentServer", () => { } }); + it("persists one stopped ending when user Stop aborts a slow cold acquire", async () => { + let markAcquireStarted!: () => void; + const acquireStarted = new Promise((resolve) => { + markAcquireStarted = resolve; + }); + let runTurnCalls = 0; + const engine: KeepaliveEngine = { + async resolveKeepaliveMount() { + return null; + }, + async acquireEnvironment(_request, signal) { + markAcquireStarted(); + await new Promise((resolve) => { + if (signal?.aborted) return resolve(); + signal?.addEventListener("abort", () => resolve(), { once: true }); + }); + return { ok: false, error: "sandbox acquisition aborted" }; + }, + async runTurn() { + runTurnCalls += 1; + return { ok: true, output: "must not run" }; + }, + async runCold() { + return { ok: false, error: "must not run cold fallback" }; + }, + }; + const run: RunAgent = (request, emit, signal) => + runWithKeepalive(request, emit, signal, { + engine, + pool: new SessionPool({ poolMax: 1 }), + config: { + enabled: true, + ttlMs: 60_000, + approvalTtlMs: 60_000, + poolMax: 1, + }, + }); + const s = await listen(run); + const realFetch = globalThis.fetch.bind(globalThis); + const ingested: Array> = []; + let heartbeatCount = 0; + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input, init) => { + const url = String(input); + if (url === `${s.url}/run`) return realFetch(input, init); + if (url.endsWith("/sessions/streams/heartbeat")) { + heartbeatCount += 1; + return Response.json({ + stream: { id: "stream-stop-during-acquire" }, + is_current_turn: heartbeatCount === 1, + }); + } + if (url.endsWith("/sessions/records/ingest")) { + ingested.push(JSON.parse(String(init?.body))); + } + return Response.json({}); + }); + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + + try { + const responsePromise = fetchSpy(`${s.url}/run`, { + method: "POST", + headers: { accept: "application/x-ndjson", ...AUTH }, + body: JSON.stringify({ + harness: "pi_core", + sandbox: "local", + sessionId: "session-stop-during-acquire", + runContext: { project: { id: "project-1" } }, + telemetry: { + exporters: { + otlp: { + endpoint: `${s.url}/otlp/v1/traces`, + headers: { authorization: "Test platform authorization" }, + }, + }, + }, + messages: [{ role: "user", content: "start slowly" }], + }), + }); + + await acquireStarted; + await vi.advanceTimersByTimeAsync(HEARTBEAT_INTERVAL_SECONDS * 1000); + const response = await responsePromise; + const records = (await response.text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + + assert.equal(runTurnCalls, 0, "the Stop landed before the turn started"); + const endings = ingested.filter( + (record) => record.record_type === "done", + ); + assert.equal(endings.length, 1, "the transcript has one terminal record"); + assert.equal( + ingested.filter((record) => record.record_type === "error").length, + 0, + "a user Stop does not persist an acquire error", + ); + assert.deepEqual(endings[0].attributes, { + type: "done", + stopReason: "cancelled", + }); + assert.equal( + records.filter((record) => record.kind === "result").length, + 1, + "the run outcome is reported once", + ); + assert.equal(records.at(-1)?.result.ok, false); + } finally { + vi.useRealTimers(); + fetchSpy.mockRestore(); + await s.close(); + } + }); + + it("persists an acquire failure error before exactly one ending", async () => { + const acquireError = "sandbox mount failed"; + let runTurnCalls = 0; + const engine: KeepaliveEngine = { + async resolveKeepaliveMount() { + return null; + }, + async acquireEnvironment() { + return { ok: false, error: acquireError }; + }, + async runTurn() { + runTurnCalls += 1; + return { ok: true, output: "must not run" }; + }, + async runCold() { + return { ok: false, error: "must not run cold fallback" }; + }, + }; + const run: RunAgent = (request, emit, signal) => + runWithKeepalive(request, emit, signal, { + engine, + pool: new SessionPool({ poolMax: 1 }), + config: { + enabled: true, + ttlMs: 60_000, + approvalTtlMs: 60_000, + poolMax: 1, + }, + }); + const s = await listen(run); + const realFetch = globalThis.fetch.bind(globalThis); + const ingested: Array> = []; + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input, init) => { + const url = String(input); + if (url === `${s.url}/run`) return realFetch(input, init); + if (url.endsWith("/sessions/streams/heartbeat")) { + return Response.json({ + stream: { id: "stream-acquire-failure" }, + is_current_turn: true, + }); + } + if (url.endsWith("/sessions/records/ingest")) { + ingested.push(JSON.parse(String(init?.body))); + } + return Response.json({}); + }); + + try { + const response = await fetchSpy(`${s.url}/run`, { + method: "POST", + headers: { accept: "application/x-ndjson", ...AUTH }, + body: JSON.stringify({ + harness: "pi_core", + sandbox: "local", + sessionId: "session-acquire-failure", + runContext: { project: { id: "project-1" } }, + telemetry: { + exporters: { + otlp: { + endpoint: `${s.url}/otlp/v1/traces`, + headers: { authorization: "Test platform authorization" }, + }, + }, + }, + messages: [{ role: "user", content: "fail during acquire" }], + }), + }); + const records = (await response.text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + + assert.equal(runTurnCalls, 0, "the failed acquire never starts the turn"); + const endingRecords = ingested.filter((record) => + ["error", "done"].includes(record.record_type), + ); + assert.deepEqual( + endingRecords.map((record) => record.record_type), + ["error", "done"], + "the transcript preserves the error before its ending", + ); + assert.deepEqual(endingRecords[0].attributes, { + type: "error", + message: acquireError, + }); + assert.deepEqual(endingRecords[1].attributes, { type: "done" }); + assert.equal( + records.filter((record) => record.kind === "result").length, + 1, + "the failed run outcome is reported once", + ); + assert.equal(records.at(-1)?.result.error, acquireError); + } finally { + fetchSpy.mockRestore(); + await s.close(); + } + }); + + it("keeps the normal Stop path at exactly one persisted ending", async () => { + const normalStop: RunAgent = async (_request, emit) => { + emit?.({ type: "done", stopReason: "cancelled" }); + return { ok: true, stopReason: "cancelled", events: [] }; + }; + const s = await listen(normalStop); + const realFetch = globalThis.fetch.bind(globalThis); + const ingested: Array> = []; + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input, init) => { + const url = String(input); + if (url === `${s.url}/run`) return realFetch(input, init); + if (url.endsWith("/sessions/streams/heartbeat")) { + return Response.json({ + stream: { id: "stream-normal-stop" }, + is_current_turn: true, + }); + } + if (url.endsWith("/sessions/records/ingest")) { + ingested.push(JSON.parse(String(init?.body))); + } + return Response.json({}); + }); + + try { + const response = await fetchSpy(`${s.url}/run`, { + method: "POST", + headers: { accept: "application/x-ndjson", ...AUTH }, + body: JSON.stringify({ + harness: "pi_core", + sessionId: "session-normal-stop", + telemetry: { + exporters: { + otlp: { + endpoint: `${s.url}/otlp/v1/traces`, + headers: { authorization: "Test platform authorization" }, + }, + }, + }, + messages: [{ role: "user", content: "stop normally" }], + }), + }); + const records = (await response.text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + + const endings = ingested.filter( + (record) => record.record_type === "done", + ); + assert.equal( + endings.length, + 1, + "the server must not duplicate runTurn's ending", + ); + assert.deepEqual(endings[0].attributes, { + type: "done", + stopReason: "cancelled", + }); + assert.equal( + records.filter( + (record) => record.kind === "event" && record.event?.type === "done", + ).length, + 1, + "the normal Stop still streams its one done event", + ); + assert.equal( + records.filter((record) => record.kind === "result").length, + 1, + "the run outcome is reported once", + ); + } finally { + fetchSpy.mockRestore(); + await s.close(); + } + }); + it("redacts this run's credentials from the stderr stack log when a run throws", async () => { // A per-run provider key rides ONLY the typed request (never process env). When the run // throws with that key captured in the error message/stack (an auth failure echoing it, @@ -578,6 +882,73 @@ describe("createAgentServer", () => { } }); + it("persists one terminal done record when a session-owned run throws", async () => { + const s = await listen(async () => { + throw new Error("engine escaped"); + }); + const realFetch = globalThis.fetch.bind(globalThis); + const ingested: Array> = []; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input, init) => { + const url = String(input); + if (url === `${s.url}/run`) return realFetch(input, init); + if (url.endsWith("/sessions/streams/heartbeat")) { + return Response.json({ + stream: { id: "stream-escaped-run" }, + is_current_turn: true, + }); + } + if (url.endsWith("/sessions/records/ingest")) { + ingested.push(JSON.parse(String(init?.body))); + } + return Response.json({}); + }); + + try { + const response = await fetchSpy(`${s.url}/run`, { + method: "POST", + headers: { accept: "application/x-ndjson", ...AUTH }, + body: JSON.stringify({ + harness: "pi_core", + sessionId: "session-escaped-run", + runContext: { project: { id: "project-1" } }, + telemetry: { + exporters: { + otlp: { + endpoint: `${s.url}/otlp/v1/traces`, + headers: { authorization: "Test platform authorization" }, + }, + }, + }, + messages: [{ role: "user", content: "throw" }], + }), + }); + const records = (await response.text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + + assert.deepEqual( + ingested + .filter((record) => ["error", "done"].includes(record.record_type)) + .map((record) => record.record_type), + ["error", "done"], + ); + assert.equal( + ingested.filter((record) => record.record_type === "done").length, + 1, + ); + assert.equal(records.filter((record) => record.kind === "result").length, 1); + assert.equal(records.at(-1)?.result.error, "engine escaped"); + } finally { + fetchSpy.mockRestore(); + errorSpy.mockRestore(); + await s.close(); + } + }); + it("rejects an over-cap session turn before persistence or attachment claiming", async () => { // Override the cap rather than generating a default-sized batch, so the case stays small. process.env.AGENTA_ATTACHMENTS_MAX_PER_TURN = "2"; @@ -644,6 +1015,7 @@ describe("createAgentServer", () => { records[0].result.error, "A user turn may carry at most 2 attachments.", ); + assert.deepEqual(liveExecutions(), []); } finally { delete process.env.AGENTA_ATTACHMENTS_MAX_PER_TURN; fetchSpy.mockRestore(); diff --git a/services/runner/tests/unit/session-admission.test.ts b/services/runner/tests/unit/session-admission.test.ts new file mode 100644 index 00000000000..6829f267fce --- /dev/null +++ b/services/runner/tests/unit/session-admission.test.ts @@ -0,0 +1,567 @@ +/** + * Single-turn admission at the runner's edge (#6417, #5539, #5538). + * + * ============================================================================================ + * THE BUG THESE PIN + * ============================================================================================ + * + * A second user message that reached the runner while a turn was running on the same session + * killed BOTH turns and left the session locked until the 30-minute lease expired: + * + * 1. The runner started the second turn's alive watchdog. Its first heartbeat asked the API's + * atomic `nx` acquire for the session and LOST, so the API answered `is_current_turn: false`. + * 2. The runner read that only as "abort this run later" and carried on into the keepalive pool, + * which found the first turn's environment busy and DESTROYED it (`supersede-busy`). Turn one + * lost its sandbox mid-answer. + * 3. Turn two then aborted on its own watchdog signal. Both turns were dead, and the session read + * as alive under a dead turn's lock. + * + * The arbiter was always right. The runner acted before reading its answer. These tests pin that + * the runner now stops at the edge: a refused turn resolves no session environment, evicts + * nothing, persists nothing, and returns a clear conflict to the caller. + * + * ============================================================================================ + * WHAT THE FAKE MODELS + * ============================================================================================ + * + * A real runner HTTP server (`createAgentServer`) driven over a real socket, plus a fake platform + * API that answers `POST /sessions/streams/heartbeat`. The fake API models exactly one fact: the + * `is_current_turn` field, which is the whole admission answer. Every other API call the turn + * makes (interaction sweep, attachment claim, credential refresh) is answered 200-and-empty, + * because none of them participate in the decision. + * + * Run: pnpm test (or: pnpm exec vitest run tests/unit/session-admission.test.ts) + */ +import { afterEach, beforeEach, describe, it } from "vitest"; +import assert from "node:assert/strict"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; + +import type { AgentRunRequest, AgentRunResult } from "../../src/protocol.ts"; +import { createAgentServer, type RunAgent } from "../../src/server.ts"; +import { + SESSION_TURN_IN_USE_CODE, + SESSION_TURN_IN_USE_MESSAGE, +} from "../../src/sessions/admission.ts"; + +const TEST_TOKEN = "test-runner-token"; +const AUTH = { authorization: `Bearer ${TEST_TOKEN}` }; +const INTERNAL_ENV = "AGENTA_API_INTERNAL_URL"; + +interface Beat { + session_id?: string; + turn_id?: string; + is_running?: boolean; +} + +/** The fake platform API. `admit` decides what its heartbeat answers for each beat. */ +async function startFakeApi( + admit: (beat: Beat) => boolean | Promise, +): Promise<{ + url: string; + beats: Beat[]; + paths: string[]; + close: () => Promise; +}> { + const beats: Beat[] = []; + const paths: string[] = []; + const server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c) => chunks.push(c as Buffer)); + req.on("end", async () => { + const path = (req.url ?? "").split("?")[0]; + paths.push(path); + let body: Record = {}; + const raw = Buffer.concat(chunks).toString("utf8"); + if (raw.trim()) { + try { + body = JSON.parse(raw) as Record; + } catch { + body = {}; + } + } + if (path.endsWith("/sessions/streams/heartbeat")) { + const beat = body as Beat; + beats.push(beat); + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + stream: { id: "11111111-1111-1111-1111-111111111111" }, + replica_id: body.replica_id ?? null, + // A turn-end beat (`is_running: false`) is never an admission question. + is_current_turn: + beat.is_running === false ? true : await admit(beat), + }), + ); + return; + } + res.writeHead(200, { "content-type": "application/json" }); + res.end("{}"); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + beats, + paths, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +async function startRunner( + run: RunAgent, +): Promise<{ url: string; close: () => Promise }> { + process.env.AGENTA_RUNNER_TOKEN = TEST_TOKEN; + const server: Server = createAgentServer(run); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +/** A session-owned run request: `sessionId` is the whole gate (`isSessionOwned`). */ +function sessionRequest( + overrides: Partial = {}, +): Record { + return { + harness: "claude", + model: "m1", + sessionId: "session-admission-1", + messages: [{ role: "user", content: "hello" }], + ...overrides, + }; +} + +interface StreamRecord { + kind: string; + event?: { type: string; message?: string; code?: string; turnId?: string }; + result?: { ok: boolean; error?: string }; +} + +async function postRun( + runnerUrl: string, + body: Record, +): Promise<{ status: number; records: StreamRecord[] }> { + const res = await fetch(`${runnerUrl}/run`, { + method: "POST", + headers: { accept: "application/x-ndjson", ...AUTH }, + body: JSON.stringify(body), + }); + const text = await res.text(); + const records = text + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line) as StreamRecord); + return { status: res.status, records }; +} + +const previousInternal = process.env[INTERNAL_ENV]; +const previousToken = process.env.AGENTA_RUNNER_TOKEN; + +beforeEach(() => { + delete process.env[INTERNAL_ENV]; +}); + +afterEach(() => { + if (previousInternal === undefined) delete process.env[INTERNAL_ENV]; + else process.env[INTERNAL_ENV] = previousInternal; + if (previousToken === undefined) delete process.env.AGENTA_RUNNER_TOKEN; + else process.env.AGENTA_RUNNER_TOKEN = previousToken; +}); + +describe("runner admission: a refused turn never reaches the session environment", () => { + it("does not call run() when the first heartbeat reports is_current_turn: false", async () => { + const api = await startFakeApi(() => false); + process.env[INTERNAL_ENV] = api.url; + const runCalls: AgentRunRequest[] = []; + const runner = await startRunner(async (request): Promise => { + runCalls.push(request); + return { ok: true, output: "should never run", events: [] }; + }); + try { + const { records } = await postRun(runner.url, sessionRequest()); + + assert.equal( + runCalls.length, + 0, + "the refused turn must not reach run(), which is what resolves the keepalive pool " + + "and is where the live turn's environment used to be destroyed", + ); + const terminal = records.find((r) => r.kind === "result"); + assert.ok(terminal, "a terminal result record is still written"); + assert.equal(terminal!.result!.ok, false); + assert.equal(terminal!.result!.error, SESSION_TURN_IN_USE_MESSAGE); + } finally { + await runner.close(); + await api.close(); + } + }); + + it("emits an error event carrying the stable session_turn_in_use code", async () => { + // The code is what lets the browser render "not sent, keep your text" instead of the generic + // "The agent run failed" bubble. The message is one line, because the SDK's + // `sanitize_runner_error` keeps only the first line of a runner error. + const api = await startFakeApi(() => false); + process.env[INTERNAL_ENV] = api.url; + const runner = await startRunner(async () => ({ + ok: true, + output: "", + events: [], + })); + try { + const { records } = await postRun(runner.url, sessionRequest()); + + const error = records.find( + (r) => r.kind === "event" && r.event?.type === "error", + ); + assert.ok(error, "the refusal is streamed as an error event"); + assert.equal(error!.event!.code, SESSION_TURN_IN_USE_CODE); + assert.equal(error!.event!.message, SESSION_TURN_IN_USE_MESSAGE); + assert.ok( + !SESSION_TURN_IN_USE_MESSAGE.includes("\n"), + "the message must stay one line to survive sanitize_runner_error", + ); + } finally { + await runner.close(); + await api.close(); + } + }); + + it("makes no interaction-sweep or attachment-claim call for a refused turn", async () => { + // `cancelStaleInteractions` cancels the session's unanswered approval gates, sparing only the + // CALLING turn's own. Running it for a turn that was refused would cancel the LIVE turn's + // pending approval card — a second way the double send broke the running turn. + const api = await startFakeApi(() => false); + process.env[INTERNAL_ENV] = api.url; + const runner = await startRunner(async () => ({ + ok: true, + output: "", + events: [], + })); + try { + await postRun(runner.url, sessionRequest()); + // Give any fire-and-forget call a chance to land before asserting it did not. + await new Promise((resolve) => setTimeout(resolve, 50)); + + const nonHeartbeat = api.paths.filter( + (p) => !p.endsWith("/sessions/streams/heartbeat"), + ); + assert.deepEqual( + nonHeartbeat, + [], + `a refused turn touched the platform beyond its own beats: ${nonHeartbeat.join(", ")}`, + ); + } finally { + await runner.close(); + await api.close(); + } + }); + + it("stops the heartbeat with an owner-scoped end beat for its own turn id", async () => { + // The end beat is safe to send: the API releases `running` only for the turn that owns it, so + // a refused turn's final beat cannot clear the LIVE turn's lock. Sending it is what stops the + // heartbeat interval and releases the credential lease. + const api = await startFakeApi(() => false); + process.env[INTERNAL_ENV] = api.url; + const runner = await startRunner(async () => ({ + ok: true, + output: "", + events: [], + })); + try { + await postRun(runner.url, sessionRequest()); + + assert.equal(api.beats.length, 2, "exactly one start beat and one end beat"); + assert.equal(api.beats[0].is_running, true); + assert.equal(api.beats[1].is_running, false); + assert.equal( + api.beats[0].turn_id, + api.beats[1].turn_id, + "the end beat names the REFUSED turn, never the live one", + ); + } finally { + await runner.close(); + await api.close(); + } + }); +}); + +describe("runner admission: an admitted turn proceeds", () => { + it("runs the turn when the first heartbeat admits it", async () => { + const api = await startFakeApi(() => true); + process.env[INTERNAL_ENV] = api.url; + const runCalls: AgentRunRequest[] = []; + const runner = await startRunner(async (request): Promise => { + runCalls.push(request); + return { ok: true, output: "answered", events: [] }; + }); + try { + const { records } = await postRun(runner.url, sessionRequest()); + + assert.equal(runCalls.length, 1, "the admitted turn runs"); + const terminal = records.find((r) => r.kind === "result"); + assert.equal(terminal!.result!.ok, true); + } finally { + await runner.close(); + await api.close(); + } + }); + + it("admits an approval RESUME while the previous turn is parked, not running", async () => { + // The park case is the one a naive "is anything alive on this session?" gate gets wrong. A + // parked turn still holds `alive` (that is what makes the session reattachable) but has + // released `running`. The API's heartbeat distinguishes them: with no `running` owner it + // treats the stale `alive` as a legitimate handover, tombstones the parked turn, and admits + // the resume. This test pins that the runner honours an ADMIT answer for a resume-shaped + // request rather than refusing on the presence of a prior turn. + const api = await startFakeApi(() => true); + process.env[INTERNAL_ENV] = api.url; + const runCalls: AgentRunRequest[] = []; + const runner = await startRunner(async (request): Promise => { + runCalls.push(request); + return { ok: true, output: "resumed", events: [] }; + }); + try { + const resume = sessionRequest({ + messages: [ + { role: "user", content: "edit the file" }, + { + role: "assistant", + content: [{ type: "tool_call", toolCallId: "call-1", toolName: "edit" }], + }, + { + role: "user", + content: [ + { + type: "tool_result", + toolCallId: "call-1", + output: { approved: true }, + }, + ], + }, + ], + } as unknown as Partial); + const { records } = await postRun(runner.url, resume); + + assert.equal(runCalls.length, 1, "the resume runs"); + const terminal = records.find((r) => r.kind === "result"); + assert.equal(terminal!.result!.ok, true); + } finally { + await runner.close(); + await api.close(); + } + }); + + it("a refused second turn cannot replace the admitted turn's Stop handle", async () => { + let releaseSecondAdmission!: () => void; + const secondAdmissionMayFinish = new Promise((resolve) => { + releaseSecondAdmission = resolve; + }); + let markSecondAdmissionWaiting!: () => void; + const secondAdmissionWaiting = new Promise((resolve) => { + markSecondAdmissionWaiting = resolve; + }); + const api = await startFakeApi(async (beat) => { + if (beat.turn_id !== "turn-B") return true; + markSecondAdmissionWaiting(); + await secondAdmissionMayFinish; + return false; + }); + process.env[INTERNAL_ENV] = api.url; + + let markFirstRunning!: () => void; + const firstRunning = new Promise((resolve) => { + markFirstRunning = resolve; + }); + let markFirstAborted!: () => void; + const firstAborted = new Promise((resolve) => { + markFirstAborted = resolve; + }); + let finishFirstForCleanup!: () => void; + const firstMayFinishForCleanup = new Promise((resolve) => { + finishFirstForCleanup = resolve; + }); + const runCalls: string[] = []; + const runner = await startRunner( + async (request, _emit, signal): Promise => { + runCalls.push(request.turnId ?? "missing"); + assert.equal(request.turnId, "turn-A", "the refused turn never reaches run()"); + markFirstRunning(); + await Promise.race([ + new Promise((resolve) => { + if (signal?.aborted) resolve(); + else signal?.addEventListener("abort", () => resolve(), { once: true }); + }), + firstMayFinishForCleanup, + ]); + if (signal?.aborted) markFirstAborted(); + return { + ok: true, + output: "", + events: [], + ...(signal?.aborted ? { stopReason: "cancelled" as const } : {}), + }; + }, + ); + + const firstRequest = postRun( + runner.url, + sessionRequest({ turnId: "turn-A" }), + ); + let secondRequest: ReturnType | undefined; + try { + await firstRunning; + secondRequest = postRun( + runner.url, + sessionRequest({ turnId: "turn-B" }), + ); + await secondAdmissionWaiting; + + const cancel = await fetch(`${runner.url}/cancel`, { + method: "POST", + headers: { "content-type": "application/json", ...AUTH }, + body: JSON.stringify({ + commandId: "command-stop-A", + projectId: "project-1", + sessionId: "session-admission-1", + targetTurnId: "turn-A", + createdAt: new Date().toISOString(), + }), + }); + + assert.equal(cancel.status, 202, "the runner still holds admitted turn A"); + await firstAborted; + releaseSecondAdmission(); + const [first, second] = await Promise.all([firstRequest, secondRequest]); + assert.equal( + first.records.find((record) => record.kind === "result")?.result?.ok, + true, + ); + assert.equal( + second.records.find((record) => record.kind === "result")?.result?.error, + SESSION_TURN_IN_USE_MESSAGE, + ); + assert.deepEqual(runCalls, ["turn-A"]); + } finally { + releaseSecondAdmission(); + finishFirstForCleanup(); + await Promise.allSettled([ + firstRequest, + ...(secondRequest ? [secondRequest] : []), + ]); + await runner.close(); + await api.close(); + } + }); + + it("fails closed when the coordination plane cannot confirm admission", async () => { + process.env[INTERNAL_ENV] = "http://127.0.0.1:1"; + const runCalls: AgentRunRequest[] = []; + const runner = await startRunner(async (request): Promise => { + runCalls.push(request); + return { ok: true, output: "answered", events: [] }; + }); + try { + const { records } = await postRun(runner.url, sessionRequest()); + + assert.equal(runCalls.length, 0, "an unconfirmed turn must never reach run()"); + const terminal = records.find((r) => r.kind === "result"); + assert.equal(terminal!.result!.ok, false); + assert.equal(terminal!.result!.error, SESSION_TURN_IN_USE_MESSAGE); + } finally { + await runner.close(); + } + }); +}); + +describe("runner admission: the admitted turn id reaches the client", () => { + // The runner mints the turn id per execution, and until now it told no one. The client's + // `start` frame is built and sent before the runner replies at all, so it cannot carry a + // runner-minted id — which is why `expected_execution_id` on the public Cancel has never had a + // first-party caller able to fill it. A Stop could only mean "whatever is running now", never + // "the turn I was watching". The `turn` event is the earliest frame that can carry it. + + it("emits a turn event carrying the admitted turn id, before any other event", async () => { + const api = await startFakeApi(() => true); + process.env[INTERNAL_ENV] = api.url; + const runner = await startRunner(async () => ({ + ok: true, + output: "answered", + events: [], + })); + try { + const { records } = await postRun(runner.url, sessionRequest()); + + const events = records.filter((r) => r.kind === "event"); + assert.ok(events.length > 0, "the run streamed at least one event"); + assert.equal( + events[0].event!.type, + "turn", + "the turn id must arrive FIRST, so a Stop that races the turn's own output can name it", + ); + const turnId = events[0].event!.turnId; + assert.ok(turnId, "the turn event carries an id"); + assert.match( + String(turnId), + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, + "the id is the uuid the runner minted", + ); + } finally { + await runner.close(); + await api.close(); + } + }); + + it("emits the SAME id the alive lock was acquired under", async () => { + // The whole point of handing the id out is that a client can name THIS execution to the + // control plane. An id that does not match the one holding the session's locks would name + // nothing, so the two must be the same value, not merely both present. + const api = await startFakeApi(() => true); + process.env[INTERNAL_ENV] = api.url; + const runner = await startRunner(async () => ({ + ok: true, + output: "answered", + events: [], + })); + try { + const { records } = await postRun(runner.url, sessionRequest()); + + const turnEvent = records.find( + (r) => r.kind === "event" && r.event?.type === "turn", + ); + assert.ok(turnEvent, "a turn event was emitted"); + assert.equal( + turnEvent!.event!.turnId, + api.beats[0].turn_id, + "the streamed id must be the id that heartbeat the alive lock", + ); + } finally { + await runner.close(); + await api.close(); + } + }); + + it("emits NO turn event for a refused turn, which owns no execution to name", async () => { + const api = await startFakeApi(() => false); + process.env[INTERNAL_ENV] = api.url; + const runner = await startRunner(async () => ({ + ok: true, + output: "", + events: [], + })); + try { + const { records } = await postRun(runner.url, sessionRequest()); + + assert.ok( + !records.some((r) => r.kind === "event" && r.event?.type === "turn"), + "a refused turn must not hand out an id: it runs nothing and there is nothing to stop", + ); + } finally { + await runner.close(); + await api.close(); + } + }); +}); diff --git a/services/runner/tests/unit/session-alive-interrupt.test.ts b/services/runner/tests/unit/session-alive-interrupt.test.ts index 26e45881b75..e466907d7b2 100644 --- a/services/runner/tests/unit/session-alive-interrupt.test.ts +++ b/services/runner/tests/unit/session-alive-interrupt.test.ts @@ -12,7 +12,9 @@ import assert from "node:assert/strict"; const fetchCalls: Array<{ url: string; body: unknown }> = []; let nextIsCurrentTurn: boolean | undefined = true; -vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { +/** The default heartbeat fake. Re-stubbed per test, because the fail-open cases replace it and + * `vi.restoreAllMocks` does not undo a `vi.stubGlobal`. */ +const recordingFetch = async (url: string, init?: RequestInit) => { const body = init?.body ? JSON.parse(init.body as string) : undefined; fetchCalls.push({ url, body }); const payload: Record = { ok: true }; @@ -20,7 +22,9 @@ vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { payload.is_current_turn = nextIsCurrentTurn; } return new Response(JSON.stringify(payload), { status: 200 }); -}); +}; + +vi.stubGlobal("fetch", recordingFetch); const { startAliveWatchdog } = await import("../../src/sessions/alive.ts"); @@ -31,6 +35,7 @@ function flushMicrotasks(): Promise { beforeEach(() => { fetchCalls.length = 0; nextIsCurrentTurn = true; + vi.stubGlobal("fetch", recordingFetch); }); afterEach(() => { @@ -138,3 +143,45 @@ describe("startAliveWatchdog onInterrupted", () => { await assert.doesNotReject(() => watchdog.release()); }); }); + +describe("startAliveWatchdog admitted (single-turn admission)", () => { + // The first beat is this turn's ADMISSION request: its `nx` acquire of the `alive` lock is the + // platform's single atomic arbiter of who runs a session. `admitted` reports that one answer so + // `server.ts` can stop a losing turn at the edge, before it resolves a session environment. + // Before this, the same answer only armed `onInterrupted`, and the losing turn still walked into + // the keepalive pool and destroyed the winning turn's warm sandbox (#6417, #5539, #5538). + + it("is true when the first beat admits the turn", async () => { + const watchdog = await startAliveWatchdog("sess-a", "turn-a", "proj-1"); + assert.equal(watchdog.admitted, true); + await watchdog.release(); + }); + + it("is false when the first beat reports is_current_turn: false", async () => { + nextIsCurrentTurn = false; + const watchdog = await startAliveWatchdog("sess-b", "turn-b", "proj-1"); + assert.equal(watchdog.admitted, false); + await watchdog.release(); + }); + + it("fails closed when the admission API is unreachable", async () => { + // Without an affirmative first heartbeat, the runner cannot prove it owns this turn. + vi.stubGlobal("fetch", async () => { + throw new Error("network down"); + }); + const watchdog = await startAliveWatchdog("sess-c", "turn-c", "proj-1"); + assert.equal(watchdog.admitted, false); + await watchdog.release(); + }); + + it("reads the FIRST beat only: a later interruption is a cancel, not a failed admission", async () => { + // A mid-turn `is_current_turn: false` is a Stop/steer/kill. That travels the + // `onInterrupted` -> abort path and must never retroactively un-admit a turn that already ran. + const watchdog = await startAliveWatchdog("sess-d", "turn-d", "proj-1"); + assert.equal(watchdog.admitted, true); + nextIsCurrentTurn = false; + await flushMicrotasks(); + assert.equal(watchdog.admitted, true, "admitted is a fact about the start of the turn"); + await watchdog.release(); + }); +}); diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index 2e292f678ac..877cd9e6228 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -30,6 +30,7 @@ import { } from "../../src/server.ts"; import { SessionPool } from "../../src/engines/sandbox_agent/session-pool.ts"; import { + approvalDecisionForToolCall, computeCredentialEpoch, configFingerprint, mountExpiryMs, @@ -139,6 +140,12 @@ function makeApprovalEngine( reply: string; toolCallId: string; }>, + settledBeforePrompts: [] as Array<{ + permissionId: string; + reply: string; + toolCallId: string; + }>, + prompts: [] as string[], acquiredEnvs: [] as DispatchFakeEnv[], /** One control per approvalPause turn: settle the parked prompt promise from the test. */ promptControls: [] as Array<{ @@ -190,6 +197,7 @@ function makeApprovalEngine( const applyScript = async ( env: DispatchFakeEnv, + request: AgentRunRequest, opts: any, ): Promise => { const idx = calls.turns.length; @@ -216,6 +224,19 @@ function makeApprovalEngine( }); } } + if (opts?.settleApprovalsThenPrompt) { + for (const decision of opts.settleApprovalsThenPrompt.decisions) { + calls.settledBeforePrompts.push({ + permissionId: decision.permissionId, + reply: decision.reply, + toolCallId: decision.toolCallId, + }); + } + const tail = request.messages?.[request.messages.length - 1]; + if (tail?.role === "user" && typeof tail.content === "string") { + calls.prompts.push(tail.content); + } + } if (script.hold) { await new Promise((resolve) => holds.set(idx, resolve)); } @@ -277,8 +298,8 @@ function makeApprovalEngine( calls.acquiredEnvs.push(env); return { ok: true, env: env as unknown as SessionEnvironment }; }, - async runTurn(env, _request, _emit, _signal, opts) { - return applyScript(env as unknown as DispatchFakeEnv, opts); + async runTurn(env, request, _emit, _signal, opts) { + return applyScript(env as unknown as DispatchFakeEnv, request, opts); }, async runCold(_request, _emit, _signal, _presigned) { calls.cold += 1; @@ -547,6 +568,83 @@ describe("runWithKeepalive: approval park + resume", () => { ); }); + it("settles a rewritten denial then prompts a trailing fresh user turn on the warm session", async () => { + const { engine, calls } = makeApprovalEngine([ + { + approvalPause: { + permissionId: "perm-1", + toolCallId: "tc-gate", + toolName: "commit", + }, + toolCallIds: ["tc-gate"], + }, + ]); + const ctx = makeCtx(engine); + await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); + + const request: AgentRunRequest = { + ...pauseTurn(), + messages: [ + { role: "user", content: "do X" }, + { + role: "assistant", + content: [ + { type: "tool_call", toolCallId: "tc-gate", toolName: "commit" }, + { + type: "tool_result", + toolCallId: "tc-gate", + output: { approved: false }, + }, + ], + }, + { role: "user", content: "What was the codeword I gave you?" }, + ], + }; + + const result = await runWithKeepalive( + request, + undefined, + undefined, + ctx, + ); + + assert.equal(result.ok, true); + assert.equal(calls.acquire, 1, "the fresh turn kept the warm environment"); + assert.equal(calls.resumes.length, 0, "it did not take approval-resume"); + assert.deepEqual(calls.settledBeforePrompts, [ + { permissionId: "perm-1", reply: "reject", toolCallId: "tc-gate" }, + ]); + assert.deepEqual(calls.prompts, ["What was the codeword I gave you?"]); + assert.equal(calls.turns[1].opts.continuation, true); + assert.equal(calls.turns[1].env, calls.turns[0].env); + }); + + it("ignores a denied tool result older than the last assistant message", () => { + const request: AgentRunRequest = { + messages: [ + { role: "user", content: "first" }, + { + role: "assistant", + content: [ + { + type: "tool_result", + toolCallId: "tc-gate", + output: { approved: false }, + }, + ], + }, + { role: "user", content: "second" }, + { role: "assistant", content: "finished a later turn" }, + { role: "user", content: "fresh question" }, + ], + }; + + assert.equal( + approvalDecisionForToolCall(request, "tc-gate"), + undefined, + ); + }); + it("logs park-approval and resume-approve/reject", async () => { const cap = captureStderr(); try { @@ -1572,6 +1670,7 @@ function pausableHarness( logs: [] as string[], resolvePrompt: undefined as ((value: unknown) => void) | undefined, promptCount: 0, + prompts: [] as any[], /** Ordered marks for the settle-before-terminal-record invariant (see the test at the end). */ journal: [] as string[], }; @@ -1645,8 +1744,9 @@ function pausableHarness( queueMicrotask(emitPiBatchResults); } }, - prompt(_blocks: any) { + prompt(blocks: any) { calls.promptCount += 1; + calls.prompts.push(blocks); // Stays pending (Claude never resolves prompt on an unanswered gate) until the test resolves // it — modelling the ORIGINAL prompt continuing after the parked gate is answered. return new Promise((resolve) => { @@ -2041,6 +2141,153 @@ describe("runTurn: real approval park + respondPermission resume", () => { await env.destroy(); }); + it("settles a parked denial before sending a fresh prompt to session.prompt", async () => { + const { calls, deps, captured } = pausableHarness(); + const acquired = await acquireEnvironment(engineReq, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + const env = acquired.env; + + const firstTurn = runTurn(env, engineReq, undefined, undefined, { + approvalParkMode: true, + }); + await flush(); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tc-gate", + title: "commit", + }), + ); + captured.onPermissionRequest!({ + id: "perm-1", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-gate", name: "commit", rawInput: {} }, + }); + await flush(); + await firstTurn; + + const parked = env.parkedApproval!; + const resolveOriginalPrompt = calls.resolvePrompt!; + env.clearTurn(); + const freshText = "What was the codeword I gave you?"; + const freshRequest: AgentRunRequest = { + ...engineReq, + messages: [{ role: "user", content: freshText }], + }; + const secondTurn = runTurn( + env, + freshRequest, + undefined, + undefined, + { + approvalParkMode: true, + continuation: true, + settleApprovalsThenPrompt: { + decisions: [ + { + permissionId: parked.permissionId, + reply: "reject", + toolCallId: parked.toolCallId, + toolName: parked.toolName, + args: parked.args, + interactionToken: parked.interactionToken, + promptPromise: parked.promptPromise, + }, + ], + }, + }, + ); + for (let i = 0; i < 20 && calls.permissionReplies.length === 0; i += 1) { + await flush(); + } + assert.deepEqual(calls.permissionReplies, [ + { id: "perm-1", reply: "reject" }, + ]); + + resolveOriginalPrompt({ + stopReason: "complete", + usage: { inputTokens: 1, outputTokens: 1 }, + }); + for (let i = 0; i < 20 && calls.promptCount < 2; i += 1) await flush(); + assert.equal(calls.promptCount, 2, "the fresh text became a new prompt"); + assert.deepEqual(calls.prompts[1], [{ type: "text", text: freshText }]); + calls.resolvePrompt!({ + stopReason: "complete", + usage: { inputTokens: 1, outputTokens: 1 }, + }); + + const result = await secondTurn; + assert.equal(result.ok, true); + assert.equal(result.stopReason, "complete"); + await env.destroy(); + }); + + it("pauses when the harness re-gates after a denial instead of hanging on the old prompt", async () => { + const { calls, deps, captured } = pausableHarness(); + const acquired = await acquireEnvironment(engineReq, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + const env = acquired.env; + + const firstTurn = runTurn(env, engineReq, undefined, undefined, { + approvalParkMode: true, + }); + await flush(); + captured.onPermissionRequest!({ + id: "perm-1", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-gate", name: "commit", rawInput: {} }, + }); + await flush(); + await firstTurn; + + const parked = env.parkedApproval!; + env.clearTurn(); + const secondTurn = runTurn( + env, + { ...engineReq, messages: [{ role: "user", content: "try another way" }] }, + undefined, + undefined, + { + approvalParkMode: true, + continuation: true, + settleApprovalsThenPrompt: { + decisions: [ + { + permissionId: parked.permissionId, + reply: "reject", + toolCallId: parked.toolCallId, + toolName: parked.toolName, + args: parked.args, + interactionToken: parked.interactionToken, + promptPromise: parked.promptPromise, + }, + ], + }, + }, + ); + for (let i = 0; i < 20 && calls.permissionReplies.length === 0; i += 1) { + await flush(); + } + captured.onPermissionRequest!({ + id: "perm-2", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-regated", name: "deploy", rawInput: {} }, + }); + + const result = await secondTurn; + assert.equal(result.ok, true); + assert.equal(result.stopReason, "paused"); + assert.equal( + calls.promptCount, + 1, + "the fresh prompt was not sent behind a new gate", + ); + assert.equal(env.parkedApproval?.toolCallId, "tc-regated"); + await env.destroy(); + }, 1_000); + it("creates and resolves a durable gate row without workflow context", async () => { const posted: Array<{ url: string; body: Record }> = []; const fetchSpy = vi @@ -3399,6 +3646,117 @@ describe("runTurn: real approval park + respondPermission resume", () => { } }); + it("parks a FIRST-turn Pi batch whose allowed sibling can never close", async () => { + // The browser pass of 2026-09-04, sessions d66e2920 (17:32Z) and 6d06f624 (17:57Z). The model + // asked for a Read and a Bash in ONE parallel batch. The Read answered `allow`; the Bash + // parked. Pi will not execute any call in a batch while a sibling gate is open, so the + // allowed Read never closed. This is the FIRST turn, and the carry-and-park branch used to + // require a resume, so the turn took the closure wait instead and sat on the 30-minute + // per-tool-call bound. It never parked, never emitted `done`, and its alive watchdog kept + // beating `running=true`, so every durable continuation aimed at the next turn was refused + // with "Continuation could not establish alive ownership". + // + // A healthy gated turn from the same hour shows the Read's `tool_result` BEFORE the Bash + // gate. Sequential calls leave nothing open at pause time, which is why this only bites a + // parallel batch. + const batch: PiBatchCall[] = [ + { + permissionId: "permission-read", + toolCallId: "tool-read", + toolName: "reader", + args: { path: "notes.md" }, + output: "read output", + }, + { + permissionId: "permission-bash", + toolCallId: "tool-bash", + toolName: "runner", + args: { command: "echo one" }, + output: "bash output", + }, + ]; + const { deps } = pausableHarness({ piBatching: batch }); + deps.createOtel = createSandboxAgentOtel as any; + // The real responder, so the plan below actually decides. The fake one pends every gate and + // would never mark an allowed execution, which is the whole precondition here. + delete (deps as { responderFactory?: unknown }).responderFactory; + const closureWaitMs = 271_828; + deps.resolveRunLimits = () => ({ + totalMs: 1_000_000, + idleMs: 500_000, + ttfbMs: 500_000, + toolCallMs: closureWaitMs, + }); + deps.createRunLimits = () => ({ + onTrip() {}, + noteToolCallStart() {}, + noteToolCallEnd() {}, + wrapEmit: (emit: (event: any) => void) => emit, + notePaused() {}, + dispose() {}, + }); + // Count the closure waits by their bound, and let one that IS armed fire at once, so the red + // is an assertion rather than a 30-minute hang. + const realSetTimeout = globalThis.setTimeout; + let closureWaitCount = 0; + const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation((( + handler: (...args: any[]) => void, + timeout?: number, + ...args: any[] + ) => { + if (timeout === closureWaitMs) { + closureWaitCount += 1; + return realSetTimeout(handler, 0, ...args); + } + return realSetTimeout(handler, timeout, ...args); + }) as typeof setTimeout); + let env: SessionEnvironment | undefined; + + try { + const piRequest: AgentRunRequest = { + ...engineReq, + harness: "pi_agenta", + permissions: { default: "ask" }, + customTools: [ + { name: "reader", permission: "allow" }, + { name: "runner", permission: "ask" }, + ], + messages: [{ role: "user", content: "read the file then echo" }], + }; + const acquired = await acquireEnvironment(piRequest, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + env = acquired.env; + + const result = await runTurn(env, piRequest, undefined, undefined, { + approvalParkMode: true, + }); + + assert.equal( + result.stopReason, + "paused", + "the gated first turn must END as paused, not hang in terminalization", + ); + assert.equal( + closureWaitCount, + 0, + "an allowed sibling of a pending Pi gate can never close, so the turn must not wait", + ); + assert.deepEqual( + [...(env.parkedApprovedExecutions?.keys() ?? [])], + ["tool-read"], + "the allowed call is carried so the resume re-announces it", + ); + assert.ok( + env.parkedApprovals.has("tool-bash"), + "the gated call is parked for the human to answer", + ); + } finally { + timeoutSpy.mockRestore(); + if (env) await env.destroy(); + } + }); + it("records the non-retry sentinel when an approved result misses the bound", async () => { const { calls, deps, captured } = pausableHarness(); deps.resolveRunLimits = () => ({ diff --git a/services/runner/tests/unit/session-keepalive-dispatch.test.ts b/services/runner/tests/unit/session-keepalive-dispatch.test.ts index 0ed6ea11381..19fbd344241 100644 --- a/services/runner/tests/unit/session-keepalive-dispatch.test.ts +++ b/services/runner/tests/unit/session-keepalive-dispatch.test.ts @@ -27,6 +27,7 @@ import { type KeepaliveEngine, } from "../../src/server.ts"; import { SessionPool } from "../../src/engines/sandbox_agent/session-pool.ts"; +import { USER_STOP_ABORT_REASON } from "../../src/sessions/stop-signal.ts"; import { configFingerprint, mountExpiryMs, @@ -777,10 +778,83 @@ describe("runWithKeepalive: never-park rules", () => { ); assert.equal(ctx.pool.size(), 0); }); + + it("a durable Stop re-parks the warm session even though the browser dropped its stream", async () => { + // The regression this file exists to prevent, replayed end to end at the dispatch seam. + // + // Increment 6, 2026-09-04: a warm Daytona session was Stopped from the browser and the next + // message came back cold on a NEW sandbox. The runner log read `[control] aborted` -> + // `harness_cancel sent=true settled=true` -> `prompt stopReason=cancelled` -> + // `[keepalive] evict reason=no-park:cancelled`. Every ingredient of a warm park was present + // and the sandbox was deleted anyway, because `handleStop` aborts the chat stream in the same + // tick it sends the durable cancel, and the park predicate read the disconnect first. + // + // So this test asserts BOTH halves land together: the client is gone AND the run signal + // carries the user-Stop label. Drop either one and it stops describing the product. + let gone = false; + const controller = new AbortController(); + const { engine, calls } = makeEngine({ + turnResults: [ + { ok: true, output: "hi", stopReason: "complete" }, + // What `run-turn.ts` returns for a Stop the harness confirmed. + { + ok: true, + output: "partial", + stopReason: "cancelled", + cancelSettled: true, + }, + ], + }); + const ctx = makeCtx(engine, {}, () => gone); + const key = "proj-1:stop-warm"; + + // Turn 1: an ordinary turn, parked warm for the next message. + await runWithKeepalive( + turn1("stop-warm"), + undefined, + controller.signal, + ctx, + ); + await flush(); + assert.equal(ctx.pool.get(key)?.state, "idle", "turn 1 parked warm"); + const warmEnv = calls.acquiredEnvs[0]; + + // Turn 2: continues on the SAME environment, and the user presses Stop mid-turn. + const origRunTurn = engine.runTurn.bind(engine); + engine.runTurn = async (env, request, emit, signal, opts) => { + gone = true; // the browser aborted its own chat stream + controller.abort(USER_STOP_ABORT_REASON); // the durable command reached this run + return origRunTurn(env, request, emit, signal, opts); + }; + const stopped = await runWithKeepalive( + turn2("stop-warm"), + undefined, + controller.signal, + ctx, + ); + await flush(); + + assert.equal(stopped.stopReason, "cancelled"); + assert.equal(calls.acquire, 1, "the Stop ran on the warm environment"); + assert.equal( + warmEnv.destroyed, + 0, + "a settled user Stop never destroys the sandbox", + ); + assert.equal( + ctx.pool.get(key)?.state, + "idle", + "re-parked warm, so the next message resumes instead of replaying cold", + ); + }); }); describe("runWithKeepalive: races and failures", () => { - it("a busy session is superseded (destroyed, awaited) and the new turn cold-starts", async () => { + it("a busy session REFUSES the racing turn: no eviction, no cold acquire", async () => { + // Single-turn admission (#6417, #5539, #5538). This branch used to `evict` the busy entry + // and cold-start ("supersede-busy"), which tore the sandbox out from under the turn that was + // still streaming on it. Both turns then died and the session stayed locked until the lease + // expired. The racing turn is now refused and the live turn's environment is untouched. const { engine, calls } = makeEngine(); const ctx = makeCtx(engine); await runWithKeepalive(turn1(), undefined, undefined, ctx); @@ -790,12 +864,41 @@ describe("runWithKeepalive: races and failures", () => { ctx.pool.checkoutIdle(key); assert.equal(ctx.pool.get(key)!.state, "busy"); - await runWithKeepalive(turn2(), undefined, undefined, ctx); + const refused = await runWithKeepalive(turn2(), undefined, undefined, ctx); + + assert.equal(refused.ok, false, "the racing turn is refused"); + assert.match( + String(refused.error), + /already running a turn/i, + "the refusal says a turn is already running, so the client can keep the text", + ); + assert.equal(env1.destroyed, 0, "the live turn keeps its warm environment"); + assert.equal(calls.acquire, 1, "no rival environment is acquired"); assert.equal( - env1.destroyed, - 1, - "the busy (racing) session is superseded/destroyed (awaited, no flush needed)", + ctx.pool.get(key)!.state, + "busy", + "the live turn still owns the pool entry", ); + }); + + it("a DESTROYED pool entry is still evicted and the new turn cold-starts", async () => { + // The other half of the old `else if (existing)` branch. A destroyed entry (a drain, or a + // teardown that already ran) has nothing in flight on it, so clearing the key and + // cold-starting is correct and costs nothing warm. Only `busy` refuses. + const { engine, calls } = makeEngine(); + const ctx = makeCtx(engine); + await runWithKeepalive(turn1(), undefined, undefined, ctx); + const key = "proj-1:s1"; + // Marked directly, because every public route that destroys a session also removes it from + // the map. A `destroyed` entry SEATED at its key is the residue of a race: `checkoutIdle` + // leaves its entry in the map while the turn runs, a teardown marks it destroyed underneath, + // and `repark` then refuses to resurrect it (`session-pool.ts`, the `destroyed` guard). + // Reproducing that race would test the pool, not this branch. + ctx.pool.get(key)!.state = "destroyed"; + + const r = await runWithKeepalive(turn2(), undefined, undefined, ctx); + + assert.equal(r.ok, true, "the new turn runs"); assert.equal(calls.acquire, 2, "the new turn cold-starts"); }); diff --git a/services/runner/tests/unit/session-ownership-release.test.ts b/services/runner/tests/unit/session-ownership-release.test.ts new file mode 100644 index 00000000000..b24cc7601b9 --- /dev/null +++ b/services/runner/tests/unit/session-ownership-release.test.ts @@ -0,0 +1,200 @@ +/** + * The shutdown release of `owner:session:` affinity claims. + * + * `claim_owner` on the API side never steals from a live owner, and nothing released the key, + * so a runner that exited while holding claims locked each of those sessions out of its own + * replacement for the rest of the 120-second lease. On the local sandbox provider that is a + * two-minute outage after every restart: the new replica refuses with "is not the owner of + * session ... Refusing to cold-start on the wrong host". + * + * These tests pin the two halves of the fix: the runner learns which sessions it owns from the + * beats it already sends, and the shutdown handler hands each one back with an inverse beat. + * + * Run: pnpm exec vitest run tests/unit/session-ownership-release.test.ts + */ +import { describe, it, beforeEach, afterEach, vi } from "vitest"; +import assert from "node:assert/strict"; + +const fetchCalls: Array<{ + url: string; + body: any; + headers?: RequestInit["headers"]; +}> = []; +let fetchImpl: ( + url: string, + init?: RequestInit, +) => Promise = async () => + new Response(JSON.stringify({}), { status: 200 }); + +vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + const body = init?.body ? JSON.parse(init.body as string) : undefined; + fetchCalls.push({ url, body, headers: init?.headers }); + return fetchImpl(url, init); +}); + +const { + claimSessionOwnership, + forgetOwnedSession, + ownedSessionCount, + recordOwnedSession, + releaseOwnedSessions, + releaseSessionOwnership, + REPLICA_ID, +} = await import("../../src/sessions/alive.ts"); +const { OWNER_TTL_SECONDS } = await import("../../src/sessions/contract.ts"); + +/** The API answers a claim beat with the winning replica. */ +const ownedBy = (replica: string) => async () => + new Response(JSON.stringify({ replica_id: replica }), { status: 200 }); + +beforeEach(() => { + fetchCalls.length = 0; + fetchImpl = ownedBy(REPLICA_ID); + process.env.AGENTA_RUNNER_TOKEN = "runner-secret"; +}); + +afterEach(async () => { + // The registry is module state; drop whatever a test left in it. + for (const id of ["sess-1", "sess-2", "sess-other", "sess-fail"]) { + forgetOwnedSession(id); + } + vi.restoreAllMocks(); + delete process.env.AGENTA_RUNNER_TOKEN; +}); + +describe("learning which sessions this replica owns", () => { + it("records a session whose claim this replica won", async () => { + await claimSessionOwnership("sess-1", "Bearer tok-1"); + assert.equal(ownedSessionCount(), 1); + }); + + it("records nothing when another replica owns the session", async () => { + fetchImpl = ownedBy("other-replica"); + await claimSessionOwnership("sess-other", "Bearer tok-1"); + assert.equal( + ownedSessionCount(), + 0, + "a lost claim must never be released later", + ); + }); + + it("records nothing when the claim call itself fails", async () => { + fetchImpl = async () => new Response("nope", { status: 503 }); + await claimSessionOwnership("sess-1", "Bearer tok-1"); + assert.equal(ownedSessionCount(), 0); + }); + + it("forgets a claim older than the affinity lease", async () => { + // Every beat records, so a long-lived runner would otherwise hold one entry (and one + // credential) per session it ever served. A claim older than the lease cannot still be held. + const t0 = 1_000_000; + recordOwnedSession("sess-1", "Bearer tok-1", t0); + assert.equal(ownedSessionCount(t0), 1); + + const expired = t0 + OWNER_TTL_SECONDS * 1000 + 1; + assert.equal(ownedSessionCount(expired), 0); + }); + + it("keeps a claim a later beat refreshed", async () => { + const t0 = 1_000_000; + recordOwnedSession("sess-1", "Bearer tok-1", t0); + const later = t0 + OWNER_TTL_SECONDS * 1000 - 1; + recordOwnedSession("sess-1", "Bearer tok-2", later); + + assert.equal( + ownedSessionCount(later + 10), + 1, + "a refreshed claim must not expire on its FIRST beat's age", + ); + }); +}); + +describe("the shutdown release", () => { + it("sends one inverse beat per owned session", async () => { + await claimSessionOwnership("sess-1", "Bearer tok-1"); + await claimSessionOwnership("sess-2", "Bearer tok-2"); + fetchCalls.length = 0; + + await releaseOwnedSessions(1_000); + + assert.equal(fetchCalls.length, 2); + const sessions = fetchCalls.map((c) => c.body.session_id).sort(); + assert.deepEqual(sessions, ["sess-1", "sess-2"]); + for (const call of fetchCalls) { + assert.ok(call.url.endsWith("/sessions/streams/heartbeat")); + assert.equal(call.body.release_owner, true); + assert.equal(call.body.replica_id, REPLICA_ID); + assert.equal( + (call.headers as Record)["x-agenta-runner-token"], + "runner-secret", + ); + assert.equal( + call.body.turn_id, + undefined, + "a departing runner asserts no turn", + ); + assert.equal( + call.body.is_running, + undefined, + "a departing runner asserts no liveness", + ); + } + }); + + it("forgets a released session, so a repeated shutdown sends nothing", async () => { + await claimSessionOwnership("sess-1", "Bearer tok-1"); + await releaseOwnedSessions(1_000); + assert.equal(ownedSessionCount(), 0); + + fetchCalls.length = 0; + await releaseOwnedSessions(1_000); + assert.deepEqual(fetchCalls, []); + }); + + it("sends nothing at all when this replica owns nothing", async () => { + await releaseOwnedSessions(1_000); + assert.deepEqual(fetchCalls, []); + }); + + it("never throws when the API refuses the release", async () => { + await claimSessionOwnership("sess-fail", "Bearer tok-1"); + fetchImpl = async () => new Response("boom", { status: 500 }); + + await releaseOwnedSessions(1_000); + + // Kept, not dropped: the release did not happen, and the 120-second lease is the fallback. + assert.equal(ownedSessionCount(), 1); + }); + + it("never throws when the API is unreachable", async () => { + await claimSessionOwnership("sess-fail", "Bearer tok-1"); + fetchImpl = async () => { + throw new Error("connect ECONNREFUSED"); + }; + + await releaseOwnedSessions(1_000); + assert.equal(ownedSessionCount(), 1); + }); + + it("returns once the deadline passes even if a release never answers", async () => { + await claimSessionOwnership("sess-1", "Bearer tok-1"); + fetchImpl = () => new Promise(() => {}); + + const started = Date.now(); + await releaseOwnedSessions(50); + + assert.ok( + Date.now() - started < 2_000, + "the shutdown release must never hold the process open", + ); + }); +}); + +describe("releaseSessionOwnership on its own", () => { + it("reports success only when the API accepts the release", async () => { + assert.equal(await releaseSessionOwnership("sess-1", "Bearer t"), true); + + fetchImpl = async () => new Response("no", { status: 404 }); + assert.equal(await releaseSessionOwnership("sess-1", "Bearer t"), false); + }); +}); diff --git a/services/runner/tests/unit/session-pool.test.ts b/services/runner/tests/unit/session-pool.test.ts index 20107d4a809..7dd595b845a 100644 --- a/services/runner/tests/unit/session-pool.test.ts +++ b/services/runner/tests/unit/session-pool.test.ts @@ -165,6 +165,7 @@ describe("readKeepaliveConfig", () => { "AGENTA_RUNNER_SESSION_KEEPALIVE", "AGENTA_RUNNER_SESSION_TTL_MS", "AGENTA_RUNNER_SESSION_APPROVAL_TTL_MS", + "AGENTA_RUNNER_SESSION_STOPPED_TTL_MS", "AGENTA_RUNNER_SESSION_POOL_MAX", "AGENTA_RUNNER_DAYTONA_SESSION_IDLE_TTL_MS", "AGENTA_RUNNER_DAYTONA_SESSION_MAX_WARM", @@ -183,13 +184,14 @@ describe("readKeepaliveConfig", () => { } }); - it("defaults: on, 60s idle, 10m approval, cap 8", () => { - // The approval window is the pending-interaction park: 10 minutes so a phone-latency - // answer warm-resumes instead of cold-replaying (mobile approvals plan §4b-4). + it("defaults: on, 60s idle, 10m approval and stopped, cap 8", () => { + // Both human-response windows last 10 minutes so the next action warm-resumes instead of + // cold-replaying (mobile approvals plan §4b-4 and Mahmoud's 2026-09-05 Stop decision). assert.deepEqual(readKeepaliveConfig("local"), { enabled: true, ttlMs: 60_000, approvalTtlMs: 600_000, + stoppedTtlMs: 600_000, poolMax: 8, }); }); @@ -228,6 +230,8 @@ describe("readKeepaliveConfig", () => { assert.deepEqual(readKeepaliveConfig("daytona"), { enabled: true, ttlMs: 120_000, + // The stopped sandbox remains billed for this ten-minute human-response window. + stoppedTtlMs: 600_000, approvalTtlMs: 120_000, poolMax: 20, }); @@ -238,6 +242,7 @@ describe("readKeepaliveConfig", () => { enabled: false, ttlMs: 0, approvalTtlMs: 0, + stoppedTtlMs: 600_000, poolMax: 20, }); process.env.AGENTA_RUNNER_DAYTONA_SESSION_IDLE_TTL_MS = "45000"; @@ -245,6 +250,7 @@ describe("readKeepaliveConfig", () => { enabled: true, ttlMs: 45_000, approvalTtlMs: 45_000, + stoppedTtlMs: 600_000, poolMax: 20, }); process.env.AGENTA_RUNNER_DAYTONA_SESSION_MAX_WARM = "7"; diff --git a/services/runner/tests/unit/session-reconstruct-history.test.ts b/services/runner/tests/unit/session-reconstruct-history.test.ts index ee8b20f3d6f..0cd9fd5c652 100644 --- a/services/runner/tests/unit/session-reconstruct-history.test.ts +++ b/services/runner/tests/unit/session-reconstruct-history.test.ts @@ -126,6 +126,63 @@ describe("reconstructHistoryIfNeeded", () => { assert.equal(fetchCalls, 0, "no query when the log is already known bad"); }); + it("replays a smart-truncated tool result", async () => { + recordsToReturn = [ + { + record_source: "agent", + attributes: { type: "tool_call", id: "toolu_big", name: "Bash", input: {} }, + }, + { + record_source: "agent", + attributes: { + type: "tool_result", + id: "toolu_big", + output: "partial…[truncated]", + _truncated: { fields: ["output"], original_bytes: 80_000 }, + }, + }, + ]; + const req = { messages: [userTurn] } as never; + const out = await reconstructHistoryIfNeeded(req, "sess-1", auth); + + assert.deepEqual(out?.messages, [ + { + role: "assistant", + content: [ + { + type: "tool_call", + toolCallId: "toolu_big", + toolName: "Bash", + input: {}, + }, + { + type: "tool_result", + toolCallId: "toolu_big", + toolName: "Bash", + output: "partial…[truncated]", + isError: undefined, + }, + ], + }, + userTurn, + ]); + }); + + it("refuses reconstruction from a legacy whole-record truncation", async () => { + recordsToReturn = [ + { + record_source: "agent", + attributes: { _truncated: true, _original_bytes: 80_000 }, + }, + ]; + const req = { messages: [userTurn] } as never; + + await assert.rejects( + () => reconstructHistoryIfNeeded(req, "sess-1", auth), + /truncated durable record/, + ); + }); + it("prepends reconstructed prior turns to the inbound message when enabled", async () => { vi.stubEnv("AGENTA_SESSIONS_RECONSTRUCT", "true"); recordsToReturn = [ diff --git a/services/runner/tests/unit/session-steer-mount-loss.test.ts b/services/runner/tests/unit/session-steer-mount-loss.test.ts index 5626470efca..7546c2399bc 100644 --- a/services/runner/tests/unit/session-steer-mount-loss.test.ts +++ b/services/runner/tests/unit/session-steer-mount-loss.test.ts @@ -307,20 +307,27 @@ function approvalReply(toolCallId: string, toolName: string): AgentRunRequest { // --- The scenario the bug report describes ----------------------------------------------- // describe("steer: a second message while a cold turn is running", () => { + // These pinned the OLD outcome: the second turn superseded the first (destroy its environment, + // cold-start a rival). Single-turn admission (#6417, #5539, #5538) replaces that with a refusal, + // which is a strictly better answer to the SAME hazard the reservation was built for. The + // reservation is still what makes the refusal possible: the running cold turn is seated as + // `busy` at its key, so the second turn finds it instead of logging `miss` and cold-acquiring a + // rival environment onto the shared durable cwd. + // + // NOTE ON THE HOLD: the second turn no longer acquires anything, so the first turn's hold is + // released by the REFUSAL settling, not by `onAcquire(2)`. + it("keeps the session's durable cwd, and the next turn succeeds", async () => { const host = makeHost(); const turn1Running = deferred(); - const steerAcquired = deferred(); + const steerSettled = deferred(); const { engine, calls } = makeEngine(host, { hold: async (envId, continuation) => { if (envId !== 1 || continuation) return; turn1Running.resolve(); - // The long turn runs until the steer's environment exists. - await steerAcquired.promise; - }, - onAcquire: (id) => { - if (id === 2) steerAcquired.resolve(); + // The long turn runs until the second message has been answered. + await steerSettled.promise; }, }); const { ctx } = makeCtx(engine); @@ -332,13 +339,26 @@ describe("steer: a second message while a cold turn is running", () => { () => {}, undefined, ctx, - ); - await Promise.all([first, steer]); + ).then((r) => { + steerSettled.resolve(); + return r; + }); + const [firstResult, steerResult] = await Promise.all([first, steer]); + assert.equal(steerResult.ok, false, "the second message is refused"); + assert.match( + String((steerResult as { error?: string }).error), + /already running a turn/i, + ); + assert.equal( + firstResult.ok, + true, + `the running turn was killed by the second message: ${(firstResult as { error?: string }).error}`, + ); assert.equal( host.dirExists, true, - `the durable cwd was destroyed by the steer: ${host.trace.join(" | ")}`, + `the durable cwd was destroyed by the second message: ${host.trace.join(" | ")}`, ); const third = await runWithKeepalive( @@ -350,30 +370,27 @@ describe("steer: a second message while a cold turn is running", () => { assert.equal( third.ok, true, - `the turn after the steer failed: ${(third as { error?: string }).error}`, + `the turn after the refusal failed: ${(third as { error?: string }).error}`, ); - // The steer superseded the first environment rather than running beside it, so exactly two - // environments existed and the third turn reused one of them warm. - assert.equal(calls.acquired.length, 2); + // The refused turn acquired nothing, so exactly ONE environment ever existed and the third + // turn continued it warm. Before admission this was 2 (supersede plus cold rebuild). + assert.equal(calls.acquired.length, 1); }); - it("supersedes the running turn instead of acquiring a rival environment", async () => { + it("refuses the second turn instead of acquiring a rival environment", async () => { const host = makeHost(); const turn1Running = deferred(); - const steerAcquired = deferred(); - // Ordering is the whole fix: the superseded environment's teardown must COMPLETE before the - // steer's acquire mounts, or the steer adopts a mount that is about to be pulled. + const steerSettled = deferred(); const order: string[] = []; const { engine } = makeEngine(host, { hold: async (envId, continuation) => { if (envId !== 1 || continuation) return; turn1Running.resolve(); - await steerAcquired.promise; + await steerSettled.promise; }, onAcquire: (id) => { order.push(`acquire:env${id}`); - if (id === 2) steerAcquired.resolve(); }, }); const { ctx } = makeCtx(engine); @@ -385,40 +402,36 @@ describe("steer: a second message while a cold turn is running", () => { () => {}, undefined, ctx, - ); + ).then((r) => { + steerSettled.resolve(); + return r; + }); await Promise.all([first, steer]); - // env1's unmount+rmSync happened, then env2 mounted fresh — never "already mounted (adopted)". - assert.deepEqual(order, ["acquire:env1", "acquire:env2"]); + // No second acquire at all: nothing to mount, nothing to unmount, nothing to adopt. + assert.deepEqual(order, ["acquire:env1"]); assert.ok( !host.trace.some((line) => line.includes("adopted")), - `the steer adopted the running turn's mount: ${host.trace.join(" | ")}`, + `the refused turn adopted the running turn's mount: ${host.trace.join(" | ")}`, ); assert.equal(host.mounted, true); assert.equal(host.dirExists, true); }); - it("survives a displaced turn that aborts (the API-side heartbeat fix)", async () => { - // The heartbeat bug means the displaced turn is never told it was superseded, so it runs to - // completion beside the steer. Suppose that is fixed and it aborts promptly instead: an - // aborted turn still routes to `env.destroy({ reason: "aborted" })`, which unmounts and - // deletes the shared cwd. Before the reservation, the abort only narrowed the race window. + it("emits no teardown for the refused turn, so the warm session survives", async () => { + // The old supersede path called `env.destroy` on the LIVE turn's environment, which unmounted + // and `rmSync`ed the shared cwd. That is the destruction half of the double-send bug. A + // refusal must touch no environment at all: the running turn keeps its sandbox and its native + // harness session, which is the warm-session constraint this whole slice is bound by. const host = makeHost(); const turn1Running = deferred(); - const steerAcquired = deferred(); + const steerSettled = deferred(); - const { engine } = makeEngine(host, { + const { engine, calls } = makeEngine(host, { hold: async (envId, continuation) => { if (envId !== 1 || continuation) return; turn1Running.resolve(); - await steerAcquired.promise; - }, - resultFor: (envId, continuation) => - envId === 1 && !continuation - ? { ok: false, error: "aborted", stopReason: "aborted" } - : undefined, - onAcquire: (id) => { - if (id === 2) steerAcquired.resolve(); + await steerSettled.promise; }, }); const { ctx } = makeCtx(engine); @@ -430,20 +443,20 @@ describe("steer: a second message while a cold turn is running", () => { () => {}, undefined, ctx, - ); + ).then((r) => { + steerSettled.resolve(); + return r; + }); await Promise.all([first, steer]); - assert.equal(host.dirExists, true, host.trace.join(" | ")); - const third = await runWithKeepalive( - req("still there?"), - () => {}, - undefined, - ctx, - ); assert.equal( - third.ok, - true, - `the turn after an aborted steer failed: ${(third as { error?: string }).error}`, + calls.acquired[0].destroyed, + 0, + `the running turn's environment was destroyed: ${host.trace.join(" | ")}`, + ); + assert.ok( + !host.trace.some((line) => line.includes("teardown")), + `a teardown ran during the refusal: ${host.trace.join(" | ")}`, ); }); }); diff --git a/services/runner/tests/unit/teardown.test.ts b/services/runner/tests/unit/teardown.test.ts index 9ac8bdf785d..47b266dcabb 100644 --- a/services/runner/tests/unit/teardown.test.ts +++ b/services/runner/tests/unit/teardown.test.ts @@ -13,6 +13,8 @@ describe("sandbox teardown disposition", () => { ["kill", "delete"], ["failed-turn", "delete"], ["aborted", "delete"], + // A settled user Stop keeps the sandbox; an unsettled one stays "aborted". + ["cancelled", "stop"], ["compatibility-mismatch", "delete"], // Lifecycle migration, step 1: the four named layers. Only the two whose daemon is sound // may park. See `teardown.ts`. diff --git a/services/runner/tests/unit/turn-settle.test.ts b/services/runner/tests/unit/turn-settle.test.ts new file mode 100644 index 00000000000..061891162bf --- /dev/null +++ b/services/runner/tests/unit/turn-settle.test.ts @@ -0,0 +1,227 @@ +/** + * A turn must reach exactly one terminal outcome, even when `run()` never returns. + * + * The runner writes its terminal record, and releases the alive watchdog, downstream of + * `await run(...)`. A run that never settles therefore leaves the session announcing + * `running=true` every thirty seconds with no ending ever written — issue #6418, and the shape + * behind #6100 and #5327 too. `awaitTurnOrAbandon` bounds that wait. + * + * The contract these tests hold: the happy path is untouched and leaves no timer armed; giving + * up always tries an abort FIRST, because most hangs unwind from one; and the caller is only + * told to write its own ending when the run is genuinely still pending afterwards. + */ + +import { describe, it, expect, vi, afterEach } from "vitest"; + +import { + ABANDON_GRACE_ENV, + DEFAULT_ABANDON_GRACE_MS, + DEFAULT_HARD_DEADLINE_MS, + HARD_DEADLINE_ENV, + awaitTurnOrAbandon, + resolveTurnSettleLimits, + type Clock, + type TurnSettleLimits, +} from "../../src/sessions/turn-settle.ts"; +import { DEFAULT_TOTAL_DEADLINE_MS } from "../../src/engines/sandbox_agent/run-limits.ts"; + +function fakeClock(): Clock & { fireAll(): Promise; pending(): number } { + let nextId = 1; + const timers = new Map void>(); + return { + setTimeout(fn: () => void) { + const id = nextId++; + timers.set(id, fn); + return id as unknown as NodeJS.Timeout; + }, + clearTimeout(handle: NodeJS.Timeout) { + timers.delete(handle as unknown as number); + }, + pending: () => timers.size, + async fireAll() { + for (const [id, fn] of [...timers.entries()]) { + timers.delete(id); + fn(); + } + await new Promise((resolve) => setTimeout(resolve, 0)); + }, + }; +} + +const limits: TurnSettleLimits = { + hardDeadlineMs: 10_000, + abandonGraceMs: 1_000, +}; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("awaitTurnOrAbandon", () => { + it("returns the run's own result and leaves no timer armed", async () => { + const clock = fakeClock(); + const abort = vi.fn(); + + const outcome = await awaitTurnOrAbandon({ + run: Promise.resolve({ ok: true }), + abort, + limits, + clock, + }); + + expect(outcome).toEqual({ settled: true, value: { ok: true } }); + expect(abort).not.toHaveBeenCalled(); + expect(clock.pending()).toBe(0); + }); + + it("rethrows a run that rejects, so the caller's own catch still owns the error", async () => { + const clock = fakeClock(); + + await expect( + awaitTurnOrAbandon({ + run: Promise.reject(new Error("harness blew up")), + abort: vi.fn(), + limits, + clock, + }), + ).rejects.toThrow("harness blew up"); + expect(clock.pending()).toBe(0); + }); + + it("aborts first when the platform says the turn is no longer current", async () => { + const clock = fakeClock(); + let finishRun: ((value: unknown) => void) | undefined; + const run = new Promise((resolve) => { + finishRun = resolve; + }); + // The real run unwinds from its abort; model that. + const abort = vi.fn(() => finishRun?.({ ok: false, error: "cancelled" })); + + const settling = awaitTurnOrAbandon({ + run, + abort, + interrupted: Promise.resolve("stopped by the user"), + limits, + clock, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(abort).toHaveBeenCalledTimes(1); + await expect(settling).resolves.toEqual({ + settled: true, + value: { ok: false, error: "cancelled" }, + }); + expect(clock.pending()).toBe(0); + }); + + it("gives up and hands the caller a reason when the run will not unwind", async () => { + const clock = fakeClock(); + // The wedged case: aborting changes nothing, because the pending ACP request cannot settle. + const run = new Promise(() => {}); + const abort = vi.fn(); + + const settling = awaitTurnOrAbandon({ + run, + abort, + interrupted: Promise.resolve("declared lost by the platform"), + limits, + clock, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(abort).toHaveBeenCalledTimes(1); + await clock.fireAll(); // the grace window closes + + await expect(settling).resolves.toEqual({ + settled: false, + reason: "declared lost by the platform", + }); + expect(clock.pending()).toBe(0); + }); + + it("leaves an abandoned run alive to execute its own teardown when it later settles", async () => { + const clock = fakeClock(); + const teardown = vi.fn(); + let finishRun: ((value: { ok: boolean }) => void) | undefined; + const run = new Promise<{ ok: boolean }>((resolve) => { + finishRun = resolve; + }).finally(teardown); + + const settling = awaitTurnOrAbandon({ + run, + abort: vi.fn(), + interrupted: Promise.resolve("declared lost by the platform"), + limits, + clock, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + await clock.fireAll(); + + await expect(settling).resolves.toEqual({ + settled: false, + reason: "declared lost by the platform", + }); + expect(teardown).not.toHaveBeenCalled(); + + finishRun?.({ ok: false }); + await run; + expect(teardown).toHaveBeenCalledTimes(1); + }); + + it("gives up on the hard deadline even with no interruption signal at all", async () => { + const clock = fakeClock(); + const settling = awaitTurnOrAbandon({ + run: new Promise(() => {}), + abort: vi.fn(), + limits, + clock, + }); + + await clock.fireAll(); // the hard deadline + await clock.fireAll(); // the grace window + + const outcome = await settling; + expect(outcome.settled).toBe(false); + if (outcome.settled) return; + expect(outcome.reason).toContain("hard turn deadline"); + }); + + it("survives an abort that throws", async () => { + const clock = fakeClock(); + const settling = awaitTurnOrAbandon({ + run: new Promise(() => {}), + abort: () => { + throw new Error("controller already closed"); + }, + interrupted: Promise.resolve("lost"), + limits, + clock, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + await clock.fireAll(); + + await expect(settling).resolves.toEqual({ settled: false, reason: "lost" }); + }); +}); + +describe("turn settle limits", () => { + it("keeps the hard deadline above the longest legitimate run", () => { + // A backstop that fired before the run limits would shorten real runs, which is the + // opposite of what users have asked for (issues #6084, #5356). + expect(DEFAULT_HARD_DEADLINE_MS).toBeGreaterThan(DEFAULT_TOTAL_DEADLINE_MS); + expect(resolveTurnSettleLimits()).toEqual({ + hardDeadlineMs: DEFAULT_HARD_DEADLINE_MS, + abandonGraceMs: DEFAULT_ABANDON_GRACE_MS, + }); + }); + + it("takes an operator override", () => { + vi.stubEnv(HARD_DEADLINE_ENV, "120000"); + vi.stubEnv(ABANDON_GRACE_ENV, "5000"); + + expect(resolveTurnSettleLimits()).toEqual({ + hardDeadlineMs: 120_000, + abandonGraceMs: 5_000, + }); + }); +}); diff --git a/services/uv.lock b/services/uv.lock index 228db1fdc9e..c11323d599d 100644 --- a/services/uv.lock +++ b/services/uv.lock @@ -8,7 +8,7 @@ resolution-markers = [ [[package]] name = "agenta" -version = "0.114.8" +version = "0.115.0" source = { editable = "../sdks/python" } dependencies = [ { name = "agenta-client" }, @@ -72,7 +72,7 @@ dev = [ [[package]] name = "agenta-client" -version = "0.114.8" +version = "0.115.0" source = { editable = "../clients/python" } dependencies = [ { name = "httpx" }, @@ -2356,7 +2356,7 @@ wheels = [ [[package]] name = "services" -version = "0.114.8" +version = "0.115.0" source = { virtual = "." } dependencies = [ { name = "agenta" }, diff --git a/web/ee/package.json b/web/ee/package.json index 7773c94409c..1414d0b926d 100644 --- a/web/ee/package.json +++ b/web/ee/package.json @@ -1,6 +1,6 @@ { "name": "@agenta/ee", - "version": "0.114.8", + "version": "0.115.0", "private": true, "engines": { "node": "24.x" diff --git a/web/ee/tests/playwright/acceptance/members/index.ts b/web/ee/tests/playwright/acceptance/members/index.ts index c0a0d0cf79e..7af43f99df5 100644 --- a/web/ee/tests/playwright/acceptance/members/index.ts +++ b/web/ee/tests/playwright/acceptance/members/index.ts @@ -32,6 +32,23 @@ const lightFastTags = buildAcceptanceTags({ const createInviteEmail = (scope: string) => `${scope}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}@agenta.test` +const waitForResendResponse = async (page: any) => { + const response = await page.waitForResponse( + (res: any) => + res.request().method() === "POST" && + res.url().includes("/workspaces/") && + res.url().includes("/invite/resend") && + ![301, 302, 303, 307, 308].includes(res.status()), + {timeout: 15000}, + ) + + if (!response.ok()) { + throw new Error( + `Resend invitation request failed (${response.status()}): ${await response.text()}`, + ) + } +} + const waitForRemoveResponse = async (page: any) => { const response = await page.waitForResponse( (res: any) => @@ -101,20 +118,9 @@ const submitInviteMembersModal = async (inviteModal: any) => { await expect(inviteModal).not.toBeVisible({timeout: 30000}) } -/** - * Invite a member via the EE flow (email sent) and wait for their row to appear - * in the members table with "Invitation Pending" status. - * Returns the invited email so callers can locate the row. - */ -/** - * A row in the members table. - * - * The table is virtualised: the semantic `` carries only the `` and each - * body row is a `[data-row-key]` node outside it, so `locator("tr")` only ever matches - * the header. - */ +/** A member row rendered by the current semantic table. */ const memberRow = (page: any, email: string) => - page.locator("[data-row-key]").filter({hasText: email}).first() + page.getByRole("row").filter({hasText: email}).first() /** * Closes the "Invited user link" dialog that opens after a successful invite. @@ -128,6 +134,7 @@ const dismissInvitedUserLinkDialog = async (page: any) => { await expect(dialog).toBeHidden({timeout: 10000}) } +/** Invites a member and waits for its Pending row state. */ const invitePendingMember = async (page: any, apiHelpers: any, uiHelpers: any): Promise => { const testEmail = createInviteEmail("test-member") @@ -159,7 +166,9 @@ const invitePendingMember = async (page: any, apiHelpers: any, uiHelpers: any): // refreshed at all — which is why callers then failed to find the row. Close it // first, then wait for the row itself. await dismissInvitedUserLinkDialog(page) - await expect(memberRow(page, testEmail)).toBeVisible({timeout: 15000}) + const row = memberRow(page, testEmail) + await expect(row).toBeVisible({timeout: 15000}) + await expect(row.getByText("Pending", {exact: true})).toBeVisible({timeout: 15000}) return testEmail } @@ -263,14 +272,16 @@ const membersTests = () => { }) await scenarios.and("the user clicks Resend invitation", async () => { - await page - .locator(".ant-dropdown-menu-item") - .filter({hasText: "Resend invitation"}) - .click() + await Promise.all([ + waitForResendResponse(page), + page.getByRole("menuitem", {name: "Resend invitation", exact: true}).click(), + ]) }) await scenarios.then("a success confirmation is shown", async () => { - await expect(page.getByText("Invitation sent!")).toBeVisible({timeout: 10000}) + await expect(page.getByText("Invitation sent!", {exact: true})).toBeVisible({ + timeout: 10000, + }) }) }, ) @@ -301,11 +312,8 @@ const membersTests = () => { }) await scenarios.and("the user clicks Remove and confirms", async () => { - await page.locator(".ant-dropdown-menu-item").filter({hasText: "Remove"}).click() + await page.getByRole("menuitem", {name: "Remove", exact: true}).click() - // `AlertPopup` calls `modal.confirm` from `@agenta/ui/app-message`, which - // renders a Radix `AlertDialog`. Its content carries role="alertdialog", - // a distinct role from "dialog" — so `getByRole("dialog")` never matches. const confirmDialog = page.getByRole("alertdialog", {name: "Remove member"}) await expect(confirmDialog).toBeVisible({timeout: 10000}) await Promise.all([ diff --git a/web/mobile/package.json b/web/mobile/package.json index 97cfe149044..54324ce6c10 100644 --- a/web/mobile/package.json +++ b/web/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@agenta/mobile", - "version": "0.114.8", + "version": "0.115.0", "private": true, "engines": { "node": "24.x" diff --git a/web/mobile/src/features/chat/ChatScreen.tsx b/web/mobile/src/features/chat/ChatScreen.tsx index ca656f4aa8e..a0dcc7ae0e4 100644 --- a/web/mobile/src/features/chat/ChatScreen.tsx +++ b/web/mobile/src/features/chat/ChatScreen.tsx @@ -77,9 +77,8 @@ export const ChatScreen = ({ // Only a FIRST load has nothing to hold — that is the one time a spinner is honest. const showLoading = resolving && !heldEntityId const liveness = useLivenessPoll(projectId) - const running = Boolean( - liveness.data?.find((s) => s.session_id === sessionId)?.flags?.is_running, - ) + const stream = liveness.data?.find((s) => s.session_id === sessionId) + const running = Boolean(stream?.flags?.is_running) // The conversation is ALWAYS mounted — the mode only decides what sits beside it (and, on a // narrow frame, which of the two is on screen). Unmounting it on a mode flip would drop a // streaming turn. @@ -99,6 +98,9 @@ export const ChatScreen = ({ projectId={projectId} workspaceId={workspaceId} running={running} + stopStateLoading={liveness.isLoading} + sessionTurnId={stream?.turn_id} + stoppingTurnId={stream?.stopping_turn_id} agentId={resolvedAgentId} /> ) : ( diff --git a/web/mobile/src/features/chat/Composer.tsx b/web/mobile/src/features/chat/Composer.tsx index f1991107cc3..bf8df8cbadf 100644 --- a/web/mobile/src/features/chat/Composer.tsx +++ b/web/mobile/src/features/chat/Composer.tsx @@ -34,6 +34,7 @@ export const Composer = ({ disabled = false, waitingOnUser = false, streaming = false, + stopping = false, onStop, inputRef, placeholder, @@ -46,6 +47,8 @@ export const Composer = ({ waitingOnUser?: boolean /** A run is streaming from this device — the send button becomes Stop. */ streaming?: boolean + /** The durable Stop request has not settled yet. */ + stopping?: boolean onStop?: () => void /** Lets the host write into the input — a rewind puts the rewound message back to edit. */ inputRef?: MutableRefObject @@ -181,6 +184,7 @@ export const Composer = ({ placeholder={placeholder} waitingOnUser={waitingOnUser} streaming={streaming} + stopping={stopping} onStop={onStop} extraPrefix={ (null) const [pendingTaskError, setPendingTaskError] = useState(null) - const {isHydrating, send} = conversation + const {isHydrating, revalidate, send, stop, voidPendingResume} = conversation useEffect(() => { const decision = pendingTaskDecision({ sessionId, @@ -188,22 +206,209 @@ export const LiveConversation = ({ takePendingTask, ]) - // Push-invalidation: a records change (another device's turn, a steer resume) folds into - // the engine's transcript under its adopt guards. - const watch = useSessionWatch({sessionId, projectId, onRecordsChanged: conversation.revalidate}) - // The watch relay is the primary cross-device signal; when it cannot connect, fall back to a - // slow revalidate poll only while the backend says the session is running elsewhere. + const streamingHere = conversation.status === "submitted" || conversation.status === "streaming" + const streamingHereRef = useRef(streamingHere) + streamingHereRef.current = streamingHere + const hitlPendingRef = useRef(conversation.hitlPending) + hitlPendingRef.current = conversation.hitlPending + const [stoppingHere, setStoppingHere] = useState(false) + const stopWatchdogTimerRef = useRef | null>(null) + const expectedStopExecutionIdRef = useRef(undefined) + const retryStopRef = useRef(false) + const stopSessionIdRef = useRef(sessionId) + stopSessionIdRef.current = sessionId + const stopping = + stoppingHere || + isSessionTurnStopping({ + currentTurnId: sessionTurnId ?? latestTurnId(conversation.messages), + stoppingTurnId, + }) || + (stopStateLoading && conversation.hitlPending) + const settleParkedStop = useCallback(() => { + if (stopWatchdogTimerRef.current) clearTimeout(stopWatchdogTimerRef.current) + stopWatchdogTimerRef.current = null + retryStopRef.current = false + expectedStopExecutionIdRef.current = undefined + // Server acceptance makes the local stop a render-only latch. + stop() + setStoppingHere(false) + }, [stop]) + + useEffect(() => { + if (stopWatchdogTimerRef.current) clearTimeout(stopWatchdogTimerRef.current) + stopWatchdogTimerRef.current = null + retryStopRef.current = false + expectedStopExecutionIdRef.current = undefined + setStoppingHere(false) + }, [sessionId]) + + // Push invalidation folds cross-device changes into the guarded transcript. + const watch = useSessionWatch({ + sessionId, + projectId, + onRecordsChanged: revalidate, + }) + // Poll slowly while a cross-device run cannot be watched live. useEffect(() => { if (watch.connected || !running) return - const timer = setInterval(() => conversation.revalidate(), 7_500) + const timer = setInterval(() => revalidate(), 7_500) return () => clearInterval(timer) - }, [watch.connected, running, conversation.revalidate]) + }, [watch.connected, running, revalidate]) + useEffect(() => { + if (streamingHere || !stopWatchdogTimerRef.current) return + clearTimeout(stopWatchdogTimerRef.current) + stopWatchdogTimerRef.current = null + retryStopRef.current = false + expectedStopExecutionIdRef.current = undefined + setStoppingHere(false) + }, [streamingHere]) + useEffect( + () => () => { + if (stopWatchdogTimerRef.current) clearTimeout(stopWatchdogTimerRef.current) + }, + [], + ) + const stopResolutionRef = useRef(null) + useEffect( + () => () => { + stopResolutionRef.current?.abort() + }, + [sessionId], + ) - // The engine's own dock latches the shown set; the mobile dock renders the raw pending list - // (same source function, same index-0 ordering) and acts through the engine. + // Composer Stop cancels on the server before changing local presentation. + const stopHere = useCallback(() => { + if (stopping) return + // Fence a delayed approval release even when cancellation cannot be requested yet. + voidPendingResume() + if (!projectId || !sessionId) return + setStoppingHere(true) + const wasParked = !streamingHereRef.current && conversation.hitlPending + const isRetry = retryStopRef.current + const expectedExecutionId = isRetry + ? expectedStopExecutionIdRef.current + : getSessionTurnId(sessionId) + retryStopRef.current = false + let resolutionController: AbortController | null = null + const cancel = async () => { + let resolvedExecutionId = expectedExecutionId + if (!isRetry && !resolvedExecutionId && streamingHereRef.current) { + const controller = new AbortController() + resolutionController = controller + stopResolutionRef.current?.abort() + stopResolutionRef.current = controller + const resolution = await resolveStopExecution({ + readExecutionId: () => getSessionTurnId(sessionId), + isRunActive: () => streamingHereRef.current, + signal: controller.signal, + }) + if (stopResolutionRef.current === controller) stopResolutionRef.current = null + if (resolution.status !== "resolved") return {resolution} as const + resolvedExecutionId = resolution.executionId + } + expectedStopExecutionIdRef.current = resolvedExecutionId + const outcome = await cancelSessionExecution({ + sessionId, + projectId, + expectedExecutionId: resolvedExecutionId, + }) + return {outcome} as const + } + void cancel() + .then((result) => { + if ("resolution" in result && result.resolution) { + if (result.resolution.status === "settled") { + setStoppingHere(false) + } else if (result.resolution.status === "timed_out") { + setStoppingHere(false) + message.warning("Could not identify the run to stop. Please try again.") + } + return + } + const {outcome} = result + if (stopSessionIdRef.current !== sessionId) return + if (outcome?.accepted) { + const action = cancelledStopAction({ + parkedAtRequest: wasParked, + parkedAtResponse: !streamingHereRef.current && hitlPendingRef.current, + streaming: streamingHereRef.current, + retry: isRetry, + executionState: outcome.execution.state, + }) + if (action === "settle-parked") { + settleParkedStop() + return + } + if (action === "settle-idle") { + setStoppingHere(false) + expectedStopExecutionIdRef.current = undefined + return + } + if (action === "abort-settled" || action === "abort-retry") { + stop() + setStoppingHere(false) + expectedStopExecutionIdRef.current = undefined + return + } + stopWatchdogTimerRef.current = setTimeout(() => { + retryStopRef.current = true + stopWatchdogTimerRef.current = null + setStoppingHere(false) + }, 30_000) + return + } + setStoppingHere(false) + if (outcome && !outcome.conflict && outcome.execution.state === "idle") { + retryStopRef.current = false + expectedStopExecutionIdRef.current = undefined + return + } + if (outcome?.conflict) { + retryStopRef.current = false + expectedStopExecutionIdRef.current = undefined + } else if (isRetry) { + retryStopRef.current = true + } + message.warning( + outcome?.conflict + ? "That run had already finished. The session is running something else now." + : "Could not stop the run. It may still be running.", + ) + }) + .catch((error: unknown) => { + if (stopResolutionRef.current === resolutionController) { + stopResolutionRef.current = null + } + if (stopSessionIdRef.current !== sessionId) return + if (isRetry) retryStopRef.current = true + setStoppingHere(false) + message.warning( + error instanceof Error + ? error.message + : "Could not stop the run. It may still be running.", + ) + }) + }, [ + projectId, + sessionId, + stop, + stopping, + conversation.hitlPending, + settleParkedStop, + voidPendingResume, + ]) + + const interactionAvailability = getInteractionAvailability({ + stopped: conversation.stopped, + stopping, + streaming: streamingHere, + }) const pendingApprovals = useMemo( - () => getPendingApprovals(conversation.messages), - [conversation.messages], + () => + getLivePendingApprovals(conversation.messages, { + stopped: !interactionAvailability.approvals, + }), + [conversation.messages, interactionAvailability.approvals], ) // Steer keeps the detached resume dispatcher; plain approve/deny go through the engine. const steerActions = useApprovalActions({ @@ -239,20 +444,19 @@ export const LiveConversation = ({ ) const autoScroll = useTranscriptAutoScroll(visibleTurns) - const streamingHere = conversation.status === "submitted" || conversation.status === "streaming" // Parked connect interactions → the dock above the composer owns their actions, so a paused // run can't scroll out of reach. Gated the same way desktop gates it. // Parked question forms → the docked card owns the questions and the answers; the transcript // rows are passive markers. const elicits = useElicitationDock({ messages: conversation.messages, - enabled: !streamingHere && !conversation.stopped, + enabled: interactionAvailability.parkedDocks, approvalsPending: pendingApprovals.length > 0, onOutput: conversation.sendToolOutput, }) const connects = useConnectionDock({ messages: conversation.messages, - enabled: !streamingHere && !conversation.stopped, + enabled: interactionAvailability.parkedDocks, approvalsPending: pendingApprovals.length > 0, elicitationPending: elicits.open, }) @@ -477,6 +681,7 @@ export const LiveConversation = ({ { + setStoppingHere(false) // An open edit rewrites its held message instead of sending. The // input clears on submit, so the displaced draft goes back after. if (!conversation.editingId) { @@ -494,8 +699,12 @@ export const LiveConversation = ({ modelBlocked ? "Connect a model to start chatting…" : undefined } waitingOnUser={conversation.hitlPending} - streaming={streamingHere} - onStop={conversation.stop} + streaming={shouldShowStopControl({ + busy: streamingHere, + hitlPending: conversation.hitlPending, + })} + stopping={stopping} + onStop={stopHere} inputRef={composerRef} /> diff --git a/web/mobile/src/features/chat/StopButton.tsx b/web/mobile/src/features/chat/StopButton.tsx index f1e654b0e3c..6ae7ee26a16 100644 --- a/web/mobile/src/features/chat/StopButton.tsx +++ b/web/mobile/src/features/chat/StopButton.tsx @@ -1,25 +1,29 @@ import {useState} from "react" -import {commandSessionStream} from "@agenta/entities/session" +import {cancelSessionExecution} from "@agenta/entities/session" import {Button} from "@agenta/ui/ui" -/** - * Cooperative Stop for a running turn: the no-inputs/no-force stream command drops the - * running locks and the runner aborts on its next heartbeat (≤30s). The liveness poll - * confirms — the button unmounts when the session stops reading as running. Until - * feat/agent-cancel-steer lands the turn settles as an error record, not a clean - * "cancelled"; the copy says so. - */ +/** Cooperative Stop stays pending until shared liveness removes the control. */ export const StopButton = ({sessionId, projectId}: {sessionId: string; projectId: string}) => { const [state, setState] = useState<"idle" | "stopping" | "failed">("idle") + const [staleMessage, setStaleMessage] = useState(null) const onStop = async () => { setState("stopping") + setStaleMessage(null) try { - const result = await commandSessionStream({sessionId, projectId}) - if (!result) setState("failed") + // Cross-device Stop has no locally observed execution id to guard with. + const outcome = await cancelSessionExecution({sessionId, projectId}) + if (!outcome) setState("failed") + if (outcome && !outcome.conflict && outcome.execution.state === "idle") setState("idle") + // A conflict means another execution replaced the offered turn. + if (outcome?.conflict) { + setState("idle") + setStaleMessage( + "That run had already finished. The session is running something else now.", + ) + } } catch { - // A rejection (offline, 5xx) must land on "failed" like a null result. Without this - // the button sits on "Stopping…" forever and the user has no way to retry. + // Network rejection must leave Stop retryable. setState("failed") } } @@ -42,6 +46,9 @@ export const StopButton = ({sessionId, projectId}: {sessionId: string; projectId {state === "failed" ? ( Stop failed — try again. ) : null} + {staleMessage ? ( + {staleMessage} + ) : null} ) } diff --git a/web/mobile/src/features/chat/stopHereState.ts b/web/mobile/src/features/chat/stopHereState.ts new file mode 100644 index 00000000000..78d9706a3db --- /dev/null +++ b/web/mobile/src/features/chat/stopHereState.ts @@ -0,0 +1,27 @@ +export type CancelledStopAction = + | "settle-parked" + | "settle-idle" + | "abort-settled" + | "abort-retry" + | "await-terminal" + +/** Choose the local follow-up after the server confirms a turn cancellation. */ +export const cancelledStopAction = ({ + parkedAtRequest, + parkedAtResponse, + streaming, + retry, + executionState, +}: { + parkedAtRequest: boolean + parkedAtResponse: boolean + streaming: boolean + retry: boolean + executionState: "stopping" | "idle" +}): CancelledStopAction => { + if (parkedAtRequest || parkedAtResponse) return "settle-parked" + if (!streaming) return "settle-idle" + if (executionState === "idle") return "abort-settled" + if (retry) return "abort-retry" + return "await-terminal" +} diff --git a/web/mobile/src/features/chat/useSessionWatch.ts b/web/mobile/src/features/chat/useSessionWatch.ts index b578ef9df79..d5599d2c28d 100644 --- a/web/mobile/src/features/chat/useSessionWatch.ts +++ b/web/mobile/src/features/chat/useSessionWatch.ts @@ -33,15 +33,19 @@ export const useSessionWatch = ({ sessionId, projectId, onRecordsChanged, + onInteractionChanged, }: { sessionId: string projectId: string onRecordsChanged: () => void + onInteractionChanged?: () => void }): {connected: boolean} => { const [connected, setConnected] = useState(false) const queryClient = useQueryClient() const onRecordsChangedRef = useRef(onRecordsChanged) onRecordsChangedRef.current = onRecordsChanged + const onInteractionChangedRef = useRef(onInteractionChanged) + onInteractionChangedRef.current = onInteractionChanged useEffect(() => { if (!sessionId || !projectId) return @@ -112,7 +116,10 @@ export const useSessionWatch = ({ }) es.addEventListener("records-changed", () => onRecordsChangedRef.current()) es.addEventListener("lifecycle", invalidateBadges) - es.addEventListener("interaction", invalidateBadges) + es.addEventListener("interaction", () => { + invalidateBadges() + onInteractionChangedRef.current?.() + }) es.onerror = () => { setConnected(false) // CONNECTING = built-in auto-reconnect; only a fatal CLOSED needs us. diff --git a/web/mobile/src/features/sessions/useActionableInteractions.ts b/web/mobile/src/features/sessions/useActionableInteractions.ts index 879e983c728..de988c41dd8 100644 --- a/web/mobile/src/features/sessions/useActionableInteractions.ts +++ b/web/mobile/src/features/sessions/useActionableInteractions.ts @@ -1,23 +1,14 @@ -import { - queryInteractions, - type SessionInteraction, - type SessionStream, -} from "@agenta/entities/session" -import {useQuery, useQueryClient} from "@tanstack/react-query" +import {queryInteractions, type SessionInteraction} from "@agenta/entities/session" +import {useQuery} from "@tanstack/react-query" -import {livenessQueryKey} from "./useLivenessPoll" +import {useLivenessPoll} from "./useLivenessPoll" export const actionableInteractionsQueryKey = (projectId: string) => ["mobile", "actionable-interactions", projectId] as const -/** - * Every pending HITL request across the project in ONE query (`session_id` omitted, - * `actionable_only: true`) — the list-badge primitive. Same cadence rules as the liveness poll: - * 15s while anything is pending OR alive (a running turn is what mints new gates), stops when - * idle, re-checks on focus. - */ +/** Poll pending project HITL requests while a gate exists or a turn can create one. */ export const useActionableInteractions = (projectId: string) => { - const queryClient = useQueryClient() + const liveness = useLivenessPoll(projectId) return useQuery({ queryKey: actionableInteractionsQueryKey(projectId), queryFn: ({signal}) => @@ -26,10 +17,8 @@ export const useActionableInteractions = (projectId: string) => { staleTime: 10_000, refetchInterval: (query) => { if ((query.state.data?.length ?? 0) > 0) return 15_000 - const alive = queryClient.getQueryData( - livenessQueryKey(projectId), - ) - return (alive?.length ?? 0) > 0 ? 15_000 : false + // Only running turns can mint new gates. + return (liveness.data ?? []).some((stream) => stream.flags?.is_running) ? 15_000 : false }, refetchOnWindowFocus: true, }) diff --git a/web/mobile/src/features/sessions/useLivenessPoll.ts b/web/mobile/src/features/sessions/useLivenessPoll.ts index bfa5a6b0238..00e9781ae7b 100644 --- a/web/mobile/src/features/sessions/useLivenessPoll.ts +++ b/web/mobile/src/features/sessions/useLivenessPoll.ts @@ -1,15 +1,16 @@ -import {deriveStreamNest, querySessionStreams, type SessionStream} from "@agenta/entities/session" +import { + deriveStreamNest, + livenessPollInterval, + querySessionStreams, + type SessionStream, +} from "@agenta/entities/session" import {useQuery} from "@tanstack/react-query" -/** Shared key so other polls (interactions) can read the alive set from the cache. */ +/** Shared key for the project liveness subscription. */ export const livenessQueryKey = (projectId: string) => ["mobile", "session-liveness", projectId] as const -/** - * Backend liveness for the project's sessions — mirrors the desktop pattern - * (oss AgentChatSlice state/liveness.ts): ONE project-scoped `is_alive=true` query backs every - * badge, low-priority, 15s while anything is alive, stops when idle, re-checks on focus. - */ +/** Poll quickly while work runs, slowly while a session remains warm, and stop when idle. */ export const useLivenessPoll = (projectId: string) => useQuery({ queryKey: livenessQueryKey(projectId), @@ -17,7 +18,7 @@ export const useLivenessPoll = (projectId: string) => querySessionStreams({projectId, isAlive: true, abortSignal: signal, lowPriority: true}), enabled: Boolean(projectId), staleTime: 10_000, - refetchInterval: (query) => ((query.state.data?.length ?? 0) > 0 ? 15_000 : false), + refetchInterval: (query) => livenessPollInterval(query.state.data), refetchOnWindowFocus: true, }) diff --git a/web/mobile/tests/unit/stopHereState.test.ts b/web/mobile/tests/unit/stopHereState.test.ts new file mode 100644 index 00000000000..f7eb37dbdd5 --- /dev/null +++ b/web/mobile/tests/unit/stopHereState.test.ts @@ -0,0 +1,65 @@ +import {describe, expect, it} from "vitest" + +import {cancelledStopAction} from "../../src/features/chat/stopHereState" + +describe("mobile local Stop state", () => { + it("settles a parked approval as soon as the server confirms cancellation", () => { + expect( + cancelledStopAction({ + parkedAtRequest: true, + parkedAtResponse: true, + streaming: false, + retry: false, + executionState: "stopping", + }), + ).toBe("settle-parked") + }) + + it("settles when a streaming run parks before cancellation returns", () => { + expect( + cancelledStopAction({ + parkedAtRequest: false, + parkedAtResponse: true, + streaming: false, + retry: false, + executionState: "stopping", + }), + ).toBe("settle-parked") + }) + + it("waits for terminal stream evidence after cancelling an active stream", () => { + expect( + cancelledStopAction({ + parkedAtRequest: false, + parkedAtResponse: false, + streaming: true, + retry: false, + executionState: "stopping", + }), + ).toBe("await-terminal") + }) + + it("hard-aborts an active stream after the watchdog retry is accepted", () => { + expect( + cancelledStopAction({ + parkedAtRequest: false, + parkedAtResponse: false, + streaming: true, + retry: true, + executionState: "stopping", + }), + ).toBe("abort-retry") + }) + + it("settles an acknowledged legacy Stop without waiting for the client deadline", () => { + expect( + cancelledStopAction({ + parkedAtRequest: false, + parkedAtResponse: false, + streaming: true, + retry: false, + executionState: "idle", + }), + ).toBe("abort-settled") + }) +}) diff --git a/web/oss/package.json b/web/oss/package.json index e21d628aeed..43b7d02d7fd 100644 --- a/web/oss/package.json +++ b/web/oss/package.json @@ -1,6 +1,6 @@ { "name": "@agenta/oss", - "version": "0.114.8", + "version": "0.115.0", "private": true, "engines": { "node": "24.x" diff --git a/web/oss/src/components/AgentChatSlice/AgentConversation.tsx b/web/oss/src/components/AgentChatSlice/AgentConversation.tsx index c4a56fcae08..1a30c244aee 100644 --- a/web/oss/src/components/AgentChatSlice/AgentConversation.tsx +++ b/web/oss/src/components/AgentChatSlice/AgentConversation.tsx @@ -22,8 +22,13 @@ import { useVoiceComposer, } from "@agenta/chat/hooks" import {type SessionRunStatus} from "@agenta/chat/model" -import {ignoreStreamRejection, isEmptyAssistantTurn, isVisiblePart} from "@agenta/chat/model" -import {getPendingApprovals} from "@agenta/chat/model" +import { + ignoreStreamRejection, + isEmptyAssistantTurn, + isSessionBusyRefusal, + isVisiblePart, +} from "@agenta/chat/model" +import {getInteractionAvailability, getLivePendingApprovals} from "@agenta/chat/model" import {hasSessionChat, sessionMessagesAtom, setSessionStatusAtom} from "@agenta/chat/state" import {clearSessionFresh} from "@agenta/chat/state" import { @@ -51,6 +56,7 @@ import {TEMPLATE_STRIP_MODE} from "@/oss/components/pages/agent-home/assets/cons import {isAgentFileUploadsEnabled} from "./assets/constants" import {CONTENT_VISIBILITY_ENABLED} from "./assets/conversationLayout" import {runWithInFlightSubmit} from "./assets/inFlightSubmit" +import {restoreHeldRefusedSend} from "./assets/refusedMessageRecovery" import AgentComposerDock from "./components/AgentComposerDock" import AgentTranscript from "./components/AgentTranscript" import AgentTurn from "./components/AgentTurn" @@ -132,6 +138,7 @@ const AgentConversation = ({ isHydrating, hydratedEmpty, stopped, + stopping, setStopped, handleStop, handleClientToolOutput, @@ -236,6 +243,7 @@ const AgentConversation = ({ attachmentsSettled, isDragging, addFiles, + restoreAttachments, } = attachments // Playground-native onboarding: the hero, Create-agent / Continue-in-IDE, the template strip @@ -339,6 +347,7 @@ const AgentConversation = ({ beginEdit, cancelEdit, commitEdit, + takeLastSent, } = useAgentChatQueue({ status, messages, @@ -374,9 +383,11 @@ const AgentConversation = ({ [answerApproval, markLiveGate, submit], ) - // Pending HITL gates for the paused turn, surfaced in the persistent ApprovalDock above the - // composer (not inline in the transcript, so a paused run can't scroll out of reach). - const pendingApprovals = useMemo(() => getPendingApprovals(messages), [messages]) + const interactionAvailability = getInteractionAvailability({stopped, stopping, streaming: busy}) + const pendingApprovals = useMemo( + () => getLivePendingApprovals(messages, {stopped: !interactionAvailability.approvals}), + [messages, interactionAvailability.approvals], + ) // Parked connect interactions on the paused turn → the connect dock owns their actions (the // inline rows are passive markers). Gated off while busy (`input-streaming` isn't parked yet) // and after a user stop (the run is dead, nothing to settle — matches the queue's stop void). @@ -385,13 +396,13 @@ const AgentConversation = ({ // is already false by the time the dock should open. const elicits = useElicitationDock({ messages, - enabled: !busy && !stopped, + enabled: interactionAvailability.parkedDocks, approvalsPending: pendingApprovals.length > 0, onOutput: handleClientToolOutput, }) const connects = useConnectionDock({ messages, - enabled: !busy && !stopped, + enabled: interactionAvailability.parkedDocks, approvalsPending: pendingApprovals.length > 0, elicitationPending: elicits.open, }) @@ -421,6 +432,28 @@ const AgentConversation = ({ }), [messages], ) + const refusedSendRef = useRef(undefined) + const restoreRefusedSend = useCallback( + () => restoreHeldRefusedSend(refusedSendRef, richInputRef.current, restoreAttachments), + [restoreAttachments], + ) + // Restore a refused send after the editor's synchronous submit clear. + useEffect(() => { + if (!error || !isSessionBusyRefusal(error)) return + if (!refusedSendRef.current) refusedSendRef.current = takeLastSent() + requestAnimationFrame(() => { + restoreRefusedSend() + }) + }, [error, restoreRefusedSend, takeLastSent]) + + const handleComposerChange = useCallback( + (text: string) => { + composer.handleComposerChange(text) + if (!text.trim()) restoreRefusedSend() + }, + [composer.handleComposerChange, restoreRefusedSend], + ) + useEffect(() => { const status: SessionRunStatus = error ? "error" @@ -515,11 +548,12 @@ const AgentConversation = ({ trimmed: string, fileParts: FileUIPart[] | undefined, consumedUids: string[], + stagedFiles: typeof files, ) => { if (editingId) { // A rewrite of a held message: nothing is sent, so the transcript must not move. // The input clears itself on submit, so the displaced draft goes back after that. - const draft = commitEdit({text: trimmed, fileParts}) + const draft = commitEdit({text: trimmed, fileParts, stagedFiles}) if (draft) requestAnimationFrame(() => richInputRef.current?.setMarkdown(draft)) } else { // Glide to the bottom; the min-h-full active turn makes that show the new question at the @@ -528,7 +562,7 @@ const AgentConversation = ({ scrollIntent.armGlide() setStopped(false) // One path: `submit` sends now or queues behind held messages via the shared release gate. - submit({text: trimmed, fileParts}) + submit({text: trimmed, fileParts, stagedFiles}) } // The message left the composer — drop its persisted draft (and any pending capture). composer.clearDraft() @@ -569,7 +603,7 @@ const AgentConversation = ({ } fileParts = parts } - finishSubmit(trimmed, fileParts, stagedUids) + finishSubmit(trimmed, fileParts, stagedUids, files) return } @@ -582,7 +616,7 @@ const AgentConversation = ({ const fileParts = outboundFiles.length ? stagedFilesToParts(outboundFiles, sessionId) : undefined - finishSubmit(trimmed, fileParts, stagedUids) + finishSubmit(trimmed, fileParts, stagedUids, outboundFiles) }) handleSubmitRef.current = handleSubmit @@ -838,8 +872,9 @@ const AgentConversation = ({ onClientToolOutput={handleClientToolOutput} onSubmit={handleSubmit} onStop={handleStop} + stopping={stopping} richInputRef={richInputRef} - composer={composer} + composer={{...composer, handleComposerChange}} attachments={attachments} onboardingChat={onboardingChat} voice={voice} diff --git a/web/oss/src/components/AgentChatSlice/assets/refusedMessageRecovery.test.ts b/web/oss/src/components/AgentChatSlice/assets/refusedMessageRecovery.test.ts new file mode 100644 index 00000000000..9c4b51f9e63 --- /dev/null +++ b/web/oss/src/components/AgentChatSlice/assets/refusedMessageRecovery.test.ts @@ -0,0 +1,86 @@ +import {describe, expect, it, vi} from "vitest" + +import { + canRestoreRefusedSend, + restoreRefusedDraft, + restoreHeldRefusedSend, + restoreRefusedSend, +} from "./refusedMessageRecovery" + +describe("restoreRefusedDraft", () => { + it("restores a refused message only into an empty composer", () => { + const setMarkdown = vi.fn() + const editor = {getMarkdown: () => "", setMarkdown} as never + + expect(restoreRefusedDraft(editor, "try again")).toBe(true) + expect(setMarkdown).toHaveBeenCalledWith("try again") + }) + + it("does not overwrite a newer draft", () => { + const setMarkdown = vi.fn() + const editor = {getMarkdown: () => "new draft", setMarkdown} as never + + expect(restoreRefusedDraft(editor, "old refused message")).toBe(false) + expect(setMarkdown).not.toHaveBeenCalled() + }) + + it("allows attachment recovery only while the composer is still empty", () => { + expect(canRestoreRefusedSend({getMarkdown: () => "", setMarkdown: vi.fn()} as never)).toBe( + true, + ) + expect( + canRestoreRefusedSend({getMarkdown: () => "new draft", setMarkdown: vi.fn()} as never), + ).toBe(false) + }) + + it("leaves a refused send with staged attachments untouched behind a newer draft", () => { + const setMarkdown = vi.fn() + const restoreAttachments = vi.fn() + const stagedFiles = [{uid: "file-1", name: "brief.pdf"}] + const editor = {getMarkdown: () => "newer draft", setMarkdown} as never + + expect( + restoreRefusedSend(editor, {text: "refused message", stagedFiles}, restoreAttachments), + ).toBe(false) + expect(setMarkdown).not.toHaveBeenCalled() + expect(restoreAttachments).not.toHaveBeenCalled() + }) + + it("captures a refusal before deferred placement and restores it once", () => { + let markdown = "newer draft" + const setMarkdown = vi.fn((next: string) => { + markdown = next + }) + const restoreAttachments = vi.fn() + const stagedFiles = [{uid: "file-1", name: "brief.pdf"}] + const refused = {text: "refused message", stagedFiles} + const newer = {text: "newer draft", stagedFiles: []} + let lastSent: typeof refused | undefined = refused + const takeLastSent = () => { + const sent = lastSent + lastSent = undefined + return sent + } + const slot: {current: typeof refused | undefined} = {current: undefined} + const editor = {getMarkdown: () => markdown, setMarkdown} as never + const frames: (() => boolean)[] = [] + + expect(slot.current).toBeUndefined() + if (!slot.current) slot.current = takeLastSent() + frames.push(() => restoreHeldRefusedSend(slot, editor, restoreAttachments)) + + lastSent = newer + markdown = "" + expect(frames.shift()?.()).toBe(true) + + expect(setMarkdown).toHaveBeenCalledTimes(1) + expect(setMarkdown).toHaveBeenCalledWith("refused message") + expect(restoreAttachments).toHaveBeenCalledTimes(1) + expect(restoreAttachments).toHaveBeenCalledWith(stagedFiles) + expect(lastSent).toBe(newer) + + expect(restoreHeldRefusedSend(slot, editor, restoreAttachments)).toBe(false) + expect(setMarkdown).toHaveBeenCalledTimes(1) + expect(restoreAttachments).toHaveBeenCalledTimes(1) + }) +}) diff --git a/web/oss/src/components/AgentChatSlice/assets/refusedMessageRecovery.ts b/web/oss/src/components/AgentChatSlice/assets/refusedMessageRecovery.ts new file mode 100644 index 00000000000..3219d1fca6b --- /dev/null +++ b/web/oss/src/components/AgentChatSlice/assets/refusedMessageRecovery.ts @@ -0,0 +1,43 @@ +import type {RichChatInputHandle} from "@agenta/ui/rich-chat-input" + +export const canRestoreRefusedSend = (editor: RichChatInputHandle | null): boolean => + Boolean(editor && editor.getMarkdown() === "") + +export const restoreRefusedDraft = (editor: RichChatInputHandle | null, text: string): boolean => { + if (!editor || !text || !canRestoreRefusedSend(editor)) return false + editor.setMarkdown(text) + return true +} + +interface RefusedSend { + text: string + stagedFiles?: TAttachment[] +} + +interface RefusedSendSlot { + current: RefusedSend | undefined +} + +export const restoreRefusedSend = ( + editor: RichChatInputHandle | null, + sent: RefusedSend, + restoreAttachments: (files: TAttachment[]) => void, +): boolean => { + if (!canRestoreRefusedSend(editor)) return false + if (sent.text && !restoreRefusedDraft(editor, sent.text)) return false + if (sent.stagedFiles?.length) restoreAttachments(sent.stagedFiles) + return true +} + +export const restoreHeldRefusedSend = ( + slot: RefusedSendSlot, + editor: RichChatInputHandle | null, + restoreAttachments: (files: TAttachment[]) => void, +): boolean => { + const sent = slot.current + if (!sent) return false + slot.current = undefined + if (restoreRefusedSend(editor, sent, restoreAttachments)) return true + slot.current = sent + return false +} diff --git a/web/oss/src/components/AgentChatSlice/assets/stopState.test.ts b/web/oss/src/components/AgentChatSlice/assets/stopState.test.ts new file mode 100644 index 00000000000..31d6e18ea48 --- /dev/null +++ b/web/oss/src/components/AgentChatSlice/assets/stopState.test.ts @@ -0,0 +1,62 @@ +import {describe, expect, it} from "vitest" + +import {isStoppingPhase, reduceStopPhase, type StopPhase} from "./stopState" + +const transition = (events: Parameters[1][]): StopPhase => + events.reduce(reduceStopPhase, "idle" as StopPhase) + +describe("stop state", () => { + it("enters stopping while the request is pending", () => { + const phase = transition([{type: "request"}]) + + expect(phase).toBe("requesting") + expect(isStoppingPhase(phase)).toBe(true) + }) + + it("stays stopping after acceptance until the stream terminates", () => { + const phase = transition([{type: "request"}, {type: "accepted"}]) + + expect(phase).toBe("accepted") + expect(isStoppingPhase(phase)).toBe(true) + expect(reduceStopPhase(phase, {type: "terminal"})).toBe("stopped") + }) + + it("settles immediately when the server cancels a parked turn", () => { + const phase = transition([{type: "request"}, {type: "cancelled", parked: true}]) + + expect(phase).toBe("stopped") + expect(isStoppingPhase(phase)).toBe(false) + }) + + it("keeps waiting for a streaming turn after the server accepts cancellation", () => { + const phase = transition([{type: "request"}, {type: "cancelled", parked: false}]) + + expect(phase).toBe("accepted") + expect(isStoppingPhase(phase)).toBe(true) + }) + + it("remembers a terminal event that beats the response", () => { + expect(transition([{type: "request"}, {type: "terminal"}, {type: "accepted"}])).toBe( + "stopped", + ) + }) + + it("keeps an ordinary terminal event idle", () => { + const phase = transition([{type: "terminal"}]) + + expect(phase).toBe("idle") + expect(isStoppingPhase(phase)).toBe(false) + }) + + it("makes an accepted stop retryable after the watchdog timeout", () => { + const phase = transition([{type: "request"}, {type: "accepted"}, {type: "timeout"}]) + + expect(phase).toBe("retryable") + expect(isStoppingPhase(phase)).toBe(false) + expect(reduceStopPhase(phase, {type: "terminal"})).toBe("stopped") + }) + + it.each(["failed", "already_idle"] as const)("returns to idle on %s", (type) => { + expect(transition([{type: "request"}, {type}])).toBe("idle") + }) +}) diff --git a/web/oss/src/components/AgentChatSlice/assets/stopState.ts b/web/oss/src/components/AgentChatSlice/assets/stopState.ts new file mode 100644 index 00000000000..6c824afd47b --- /dev/null +++ b/web/oss/src/components/AgentChatSlice/assets/stopState.ts @@ -0,0 +1,34 @@ +export type StopPhase = "idle" | "requesting" | "accepted" | "retryable" | "terminal" | "stopped" + +export type StopEvent = + | {type: "request"} + | {type: "accepted"} + | {type: "cancelled"; parked: boolean} + | {type: "terminal"} + | {type: "timeout"} + | {type: "failed" | "already_idle" | "reset"} + +export const reduceStopPhase = (phase: StopPhase, event: StopEvent): StopPhase => { + switch (event.type) { + case "request": + return phase === "terminal" ? "terminal" : "requesting" + case "accepted": + return phase === "terminal" ? "stopped" : "accepted" + case "cancelled": + if (event.parked || phase === "terminal") return "stopped" + return "accepted" + case "timeout": + return phase === "accepted" ? "retryable" : phase + case "terminal": + if (phase === "requesting") return "terminal" + if (phase === "accepted" || phase === "retryable") return "stopped" + return phase + case "failed": + case "already_idle": + case "reset": + return "idle" + } +} + +export const isStoppingPhase = (phase: StopPhase): boolean => + phase === "requesting" || phase === "accepted" || phase === "terminal" diff --git a/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.test.ts b/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.test.ts new file mode 100644 index 00000000000..95c80de9592 --- /dev/null +++ b/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.test.ts @@ -0,0 +1,146 @@ +import {act, createElement, useCallback} from "react" + +import {latestTurnId} from "@agenta/chat/assets" +import { + clearSessionEphemera, + clearSessionTurnId, + getSessionTurnId, + setSessionTurnId, +} from "@agenta/chat/state" +import type {UIMessage} from "ai" +import {createRoot} from "react-dom/client" +import {afterAll, afterEach, beforeAll, describe, expect, it, vi} from "vitest" + +import {stopPinnedExecution} from "./stopWhileResolvingExecution" + +const sessionId = "session-1" + +const deferred = () => { + let resolve!: () => void + const promise = new Promise((done) => { + resolve = done + }) + return {promise, resolve} +} + +beforeAll(() => vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true)) +afterAll(() => vi.unstubAllGlobals()) +afterEach(() => clearSessionEphemera(sessionId)) + +describe("stopPinnedExecution", () => { + it("starts the local abort while cancellation is still pending", async () => { + const held = deferred() + const events: string[] = [] + const stop = vi.fn(() => events.push("stop")) + const cancelExecution = vi.fn(async (executionId: string | undefined) => { + events.push(`cancel:${executionId}`) + await held.promise + }) + + const stopping = stopPinnedExecution({ + stop, + expectedExecutionId: "turn-A", + cancelExecution, + }) + + expect(events).toEqual(["stop", "cancel:turn-A"]) + + held.resolve() + await stopping + expect(cancelExecution).toHaveBeenCalledWith("turn-A") + }) + + it("stops turn B before metadata without restoring turn A's id", async () => { + const stop = vi.fn() + const cancelExecution = vi.fn(async (_executionId: string | undefined) => {}) + setSessionTurnId(sessionId, "turn-A") + + clearSessionTurnId(sessionId) + const messages = [ + {id: "a1", role: "assistant", parts: [], metadata: {turnId: "turn-A"}}, + {id: "u2", role: "user", parts: []}, + ] as UIMessage[] + const turnId = latestTurnId(messages) + if (turnId) setSessionTurnId(sessionId, turnId) + + await stopPinnedExecution({ + stop, + expectedExecutionId: getSessionTurnId(sessionId), + cancelExecution, + }) + + expect(stop).toHaveBeenCalledOnce() + expect(cancelExecution).toHaveBeenCalledWith(undefined) + expect(cancelExecution).not.toHaveBeenCalledWith("turn-A") + }) + + it("keeps turn A pinned when turn B is admitted while cancellation is held", async () => { + const held = deferred() + const cancelled: (string | undefined)[] = [] + setSessionTurnId(sessionId, "turn-A") + + const stopping = stopPinnedExecution({ + stop: vi.fn(), + expectedExecutionId: getSessionTurnId(sessionId), + cancelExecution: async (executionId) => { + await held.promise + cancelled.push(executionId) + }, + }) + + clearSessionTurnId(sessionId) + setSessionTurnId(sessionId, "turn-B") + expect(getSessionTurnId(sessionId)).toBe("turn-B") + + held.resolve() + await stopping + expect(cancelled).toEqual(["turn-A"]) + }) + + it("keeps turn A pinned after the hook remounts and admits turn B", async () => { + const held = deferred() + const cancelled: (string | undefined)[] = [] + const cancelExecution = async (executionId: string | undefined) => { + await held.promise + cancelled.push(executionId) + } + let stopFromMount!: () => Promise + const Harness = () => { + stopFromMount = useCallback( + () => + stopPinnedExecution({ + stop: vi.fn(), + expectedExecutionId: getSessionTurnId(sessionId), + cancelExecution, + }), + [], + ) + return null + } + + const mount = () => { + const host = document.createElement("div") + const root = createRoot(host) + act(() => root.render(createElement(Harness))) + return root + } + + setSessionTurnId(sessionId, "turn-A") + const firstMount = mount() + let stopping!: Promise + act(() => { + stopping = stopFromMount() + }) + act(() => firstMount.unmount()) + + clearSessionTurnId(sessionId) + setSessionTurnId(sessionId, "turn-B") + const secondMount = mount() + expect(getSessionTurnId(sessionId)).toBe("turn-B") + + held.resolve() + await stopping + expect(cancelled).toEqual(["turn-A"]) + act(() => secondMount.unmount()) + }) +}) diff --git a/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.ts b/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.ts new file mode 100644 index 00000000000..18d5b29e915 --- /dev/null +++ b/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.ts @@ -0,0 +1,14 @@ +export interface StopPinnedExecutionParams { + stop: () => void + expectedExecutionId: string | undefined + cancelExecution: (executionId: string | undefined) => Promise +} + +export async function stopPinnedExecution({ + stop, + expectedExecutionId, + cancelExecution, +}: StopPinnedExecutionParams): Promise { + stop() + await cancelExecution(expectedExecutionId) +} diff --git a/web/oss/src/components/AgentChatSlice/components/AgentComposerDock.tsx b/web/oss/src/components/AgentChatSlice/components/AgentComposerDock.tsx index e845c32d05a..89f1ca14eb7 100644 --- a/web/oss/src/components/AgentChatSlice/components/AgentComposerDock.tsx +++ b/web/oss/src/components/AgentChatSlice/components/AgentComposerDock.tsx @@ -1,6 +1,6 @@ import {useCallback, useEffect, useRef, type RefObject} from "react" -import {CHAT_COLUMN} from "@agenta/chat/assets" +import {CHAT_COLUMN, shouldShowStopControl} from "@agenta/chat/assets" import type {ClientToolOutputHandler} from "@agenta/chat/clientTools" import { ChatComposer, @@ -73,6 +73,7 @@ const AgentComposerDock = ({ onClientToolOutput, onSubmit, onStop, + stopping, richInputRef, composer, attachments, @@ -109,6 +110,7 @@ const AgentComposerDock = ({ onClientToolOutput: ClientToolOutputHandler onSubmit: (text: string) => void | Promise onStop: () => void + stopping: boolean richInputRef: RefObject composer: ReturnType attachments: ReturnType @@ -446,7 +448,8 @@ const AgentComposerDock = ({ initialMarkdown={composer.initialDraft} slashCommands={slash.sections} onChange={composer.handleComposerChange} - streaming={busy} + streaming={shouldShowStopControl({busy, hitlPending})} + stopping={stopping} onStop={onStop} attachments={attachments} attachmentsBlocked={attachmentsBlocked} diff --git a/web/oss/src/components/AgentChatSlice/components/AgentMessage.tsx b/web/oss/src/components/AgentChatSlice/components/AgentMessage.tsx index 589b51da79e..443ddb59c53 100644 --- a/web/oss/src/components/AgentChatSlice/components/AgentMessage.tsx +++ b/web/oss/src/components/AgentChatSlice/components/AgentMessage.tsx @@ -20,7 +20,7 @@ import { StartupActivity, TurnFooter, } from "@agenta/chat/components" -import {isToolPart, toolIdentity} from "@agenta/chat/model" +import {isToolPart, SESSION_TURN_IN_USE_CODE, toolIdentity} from "@agenta/chat/model" import { errorKey, expandedValueAtomFamily, @@ -157,8 +157,15 @@ const RETRYABLE_CODES = new Set([ "credential_delivery_failed", "starter_credits_unavailable", "rate_limited", + // The run never produced an outcome of its own and was closed for it — by the runner when + // a turn would not unwind, or by the platform's execution watchdog when the runner itself + // was gone. Nothing is wrong with the request, so sending it again is the whole fix. + "execution_lost", ]) +// An admission refusal means the message was not sent, not that an agent run failed. +const NOT_SENT_CODES = new Set([SESSION_TURN_IN_USE_CODE]) + /** The ONE rule driving both the clamp and the toggle — they can't disagree and hide text (#5350). */ const isBigError = (text: string) => text.length > 240 || text.split("\n").length > 4 @@ -189,13 +196,17 @@ export const RunErrorBody = ({ const expanded = stored ?? false const big = isBigError(text) const offerOwnKey = code ? STARTER_CREDIT_CODES.has(code) : false - const offerRetry = !!onRetry && (!!transport || (!!code && RETRYABLE_CODES.has(code))) + const notSent = !!code && NOT_SENT_CODES.has(code) + const offerRetry = + !notSent && !!onRetry && (!!transport || (!!code && RETRYABLE_CODES.has(code))) return (
- The agent run failed + + {notSent ? "Message not sent" : "The agent run failed"} + {big && expanded ? ( 
                         {text}
diff --git a/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.test.ts b/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.test.ts
new file mode 100644
index 00000000000..657231d394b
--- /dev/null
+++ b/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.test.ts
@@ -0,0 +1,596 @@
+import {act, createElement} from "react"
+
+import type {UIMessage} from "ai"
+import {createRoot} from "react-dom/client"
+import {beforeEach, describe, expect, it, vi} from "vitest"
+;(globalThis as typeof globalThis & {IS_REACT_ACT_ENVIRONMENT: boolean}).IS_REACT_ACT_ENVIRONMENT =
+    true
+
+const state = vi.hoisted(() => ({
+    capturedHooks: undefined as
+        | {
+              prepareRequest: (args: {messages: UIMessage[]; id?: string}) => Promise
+              onError: () => void
+              sendAutomaticallyWhen: (args: {messages: UIMessage[]}) => boolean
+          }
+        | undefined,
+    messages: [] as UIMessage[],
+    projectId: "project-id" as string | null,
+    latestTurnId: undefined as string | undefined,
+    hitlPending: false,
+    sessionTurnId: null as string | null,
+    stoppingTurnId: null as string | null,
+    stopStateLoading: false,
+    cancelSessionExecution: vi.fn(),
+    resolveStopExecution: vi.fn(),
+    regenerate: vi.fn(() => Promise.resolve()),
+    sendMessage: vi.fn(() => Promise.resolve()),
+    turnIds: new Map(),
+    busy: false,
+    stop: vi.fn(),
+}))
+
+vi.mock("@agenta/chat/assets", () => ({
+    buildRequestWithinDeadline: (build: () => Promise) => build(),
+    getMessageTraceId: () => undefined,
+    latestTurnId: () => state.latestTurnId,
+    resolveStopExecution: state.resolveStopExecution,
+    startupLabelFromDataPart: () => undefined,
+}))
+
+vi.mock("@agenta/chat/hooks", () => ({
+    useSessionChat: (args: {hooks: NonNullable}) => {
+        state.capturedHooks = args.hooks
+        return {}
+    },
+}))
+
+vi.mock("@agenta/chat/model", () => ({
+    createUserStoppedState: () => ({stopped: false, turnIdentity: null}),
+    ignoreStreamRejection: () => undefined,
+    isSessionTurnStopping: ({
+        currentTurnId,
+        stoppingTurnId,
+    }: {
+        currentTurnId?: string | null
+        stoppingTurnId?: string | null
+    }) => Boolean(currentTurnId && stoppingTurnId === currentTurnId),
+    parseAgentRunError: () => ({message: "error"}),
+    reduceUserStoppedState: (
+        current: {stopped: boolean; turnIdentity: null},
+        event: {type: string},
+    ) => {
+        if (event.type === "user-stop" && !current.stopped) return {...current, stopped: true}
+        if (event.type === "reset" && current.stopped) return {...current, stopped: false}
+        return current
+    },
+}))
+
+vi.mock("@agenta/chat/state", () => ({
+    clearSessionTurnId: (sessionId: string) => state.turnIds.delete(sessionId),
+    clearTurnClockAtom: "clear-turn-clock",
+    expandedKeysForMessages: () => [],
+    getSessionTurnId: (sessionId: string) => state.turnIds.get(sessionId),
+    isChatBusy: () => state.busy,
+    persistSessionMessagesAtom: "persist-messages",
+    pruneExpandedAtom: "prune-expanded",
+    sessionMessagesAtom: "session-messages",
+    sessionRecordCountsReadAtom: "record-counts",
+    setSessionStatusAtom: "set-session-status",
+    setSessionTurnId: (sessionId: string, turnId: string) => state.turnIds.set(sessionId, turnId),
+    stampMessagesCreatedAtAtom: "stamp-created-at",
+    startTurnClockAtom: "start-turn-clock",
+}))
+
+vi.mock("@agenta/entities/session", () => ({
+    cancelSessionExecution: state.cancelSessionExecution,
+    invalidateSessionListQueries: vi.fn(),
+    killSession: vi.fn(),
+    recordInteractionAnswerAtom: "record-interaction-answer",
+    revalidateSessionMountsAtom: "revalidate-mounts",
+    revalidateSessionRecordsAtom: "revalidate-records",
+}))
+
+vi.mock("@agenta/entities/trace", () => ({markTraceAsFresh: vi.fn()}))
+vi.mock("@agenta/entities/workflow", () => ({
+    invalidateAgentCommittedRevisionCache: vi.fn(),
+    workflowMolecule: {
+        selectors: {configuration: () => "workflow-configuration"},
+    },
+}))
+
+vi.mock("@agenta/playground", () => ({
+    agentShouldResumeAfterApproval: ({liveInteraction}: {liveInteraction?: unknown}) =>
+        liveInteraction !== null,
+    approvalResolution: vi.fn(),
+    buildAgentRequest: vi.fn(async () => ({
+        invocationUrl: "https://agent.test/invoke",
+        headers: {},
+        requestBody: {},
+    })),
+    buildTurnCapture: vi.fn(),
+    isHitlPending: () => state.hitlPending,
+    isResumeSend: () => false,
+    playgroundController: {actions: {switchEntity: "switch-entity"}},
+    recordAnswerThenRelease: vi.fn(),
+}))
+
+vi.mock("@agenta/shared/state", () => ({
+    agentSelfCommitSignalAtom: "commit-signal",
+}))
+vi.mock("@agenta/shared/utils", () => ({generateId: () => "generated-id"}))
+vi.mock("@agenta/ui/app-message", () => ({message: {warning: vi.fn()}}))
+vi.mock("@ai-sdk/react", () => ({
+    useChat: () => ({
+        addToolApprovalResponse: vi.fn(),
+        addToolOutput: vi.fn(),
+        error: undefined,
+        messages: state.messages,
+        regenerate: state.regenerate,
+        sendMessage: state.sendMessage,
+        setMessages: vi.fn(),
+        status: "ready",
+        stop: state.stop,
+    }),
+}))
+vi.mock("@tanstack/react-query", () => ({
+    useQueryClient: () => ({invalidateQueries: vi.fn()}),
+}))
+
+vi.mock("jotai", () => ({
+    useAtomValue: () => state.projectId,
+    useSetAtom: () => vi.fn(),
+    useStore: () => ({
+        get: (atom: string) => {
+            if (atom === "record-counts" || atom === "session-messages") return {}
+            if (atom === "open-sessions") return new Set()
+            return undefined
+        },
+    }),
+}))
+
+vi.mock("@/oss/state/project", () => ({projectIdAtom: "project-id"}))
+vi.mock("../assets/constants", () => ({
+    doesAgentChatStopKillSession: () => false,
+}))
+vi.mock("../components/Inspector/invalidate", () => ({
+    invalidateSessionInspector: vi.fn(),
+}))
+vi.mock("../state/scope", () => ({useChatScopeKey: () => "scope"}))
+vi.mock("../state/sessions", () => ({
+    openSessionIdsAtomFamily: () => "open-sessions",
+}))
+vi.mock("../state/turnCaptures", () => ({
+    captureTurnRequestAtom: "capture-request",
+}))
+vi.mock("./useFileActivityDetector", () => ({
+    useFileActivityDetector: vi.fn(),
+}))
+vi.mock("./useSessionHydration", () => ({
+    useSessionHydration: () => ({
+        hydratedEmpty: false,
+        isHydrating: false,
+        runningElsewhere: false,
+        sessionTurnId: state.sessionTurnId,
+        stoppingTurnId: state.stoppingTurnId,
+        stopStateLoading: state.stopStateLoading,
+    }),
+}))
+vi.mock("./useToolCacheInvalidation", () => ({
+    useToolCacheInvalidation: vi.fn(),
+}))
+
+import {useAgentChatSession} from "./useAgentChatSession"
+
+describe("useAgentChatSession execution guard", () => {
+    beforeEach(() => {
+        state.turnIds.clear()
+        state.sendMessage.mockClear()
+        state.regenerate.mockClear()
+        state.cancelSessionExecution.mockReset()
+        state.resolveStopExecution.mockReset()
+        state.stop.mockReset()
+        state.resolveStopExecution.mockImplementation(async ({readExecutionId}) => {
+            const executionId = readExecutionId()
+            return executionId ? {status: "resolved", executionId} : {status: "settled"}
+        })
+        state.projectId = "project-id"
+        state.latestTurnId = undefined
+        state.hitlPending = false
+        state.sessionTurnId = null
+        state.stoppingTurnId = null
+        state.stopStateLoading = false
+        state.busy = false
+    })
+
+    it("clears the previous turn before sends, regeneration, and SDK automatic requests", async () => {
+        const sessionId = "session-1"
+        let result: ReturnType | undefined
+        const container = document.createElement("div")
+        const root = createRoot(container)
+        const Probe = () => {
+            result = useAgentChatSession({
+                entityId: "revision-1",
+                sessionId,
+                initialMessages: [],
+                intent: {} as never,
+            })
+            return null
+        }
+        act(() => root.render(createElement(Probe)))
+
+        state.turnIds.set(sessionId, "turn-before-send")
+        act(() => void result!.sendMessage({text: "next"}))
+        expect(state.turnIds.get(sessionId)).toBeUndefined()
+
+        state.turnIds.set(sessionId, "turn-before-regenerate")
+        act(() => void result!.regenerate())
+        expect(state.turnIds.get(sessionId)).toBeUndefined()
+
+        state.turnIds.set(sessionId, "turn-before-auto-resume")
+        await act(() => state.capturedHooks!.prepareRequest({messages: [], id: sessionId}))
+        expect(state.turnIds.get(sessionId)).toBeUndefined()
+
+        act(() => root.unmount())
+    })
+
+    it("voids an approval resume before cancellation settles or its stream errors", async () => {
+        const sessionId = "session-1"
+        let resolveCancel: ((value: unknown) => void) | undefined
+        state.cancelSessionExecution.mockReturnValue(
+            new Promise((resolve) => {
+                resolveCancel = resolve
+            }),
+        )
+
+        let result: ReturnType | undefined
+        const container = document.createElement("div")
+        const root = createRoot(container)
+        const Probe = () => {
+            result = useAgentChatSession({
+                entityId: "revision-1",
+                sessionId,
+                initialMessages: [],
+                intent: {} as never,
+            })
+            return null
+        }
+        act(() => root.render(createElement(Probe)))
+
+        act(() => result!.markLiveGate({kind: "approval", id: "approval-1"}))
+        act(() => result!.handleStop())
+        act(() => state.capturedHooks!.onError())
+
+        expect(state.capturedHooks!.sendAutomaticallyWhen({messages: []})).toBe(false)
+
+        await act(async () => {
+            resolveCancel?.({
+                accepted: true,
+                conflict: false,
+                execution: {id: "turn-1", state: "stopping"},
+            })
+            await Promise.resolve()
+        })
+        act(() => root.unmount())
+    })
+
+    it("keeps the approval resume void when Stop cannot load the project", () => {
+        state.projectId = null
+        const sessionId = "session-1"
+        let result: ReturnType | undefined
+        const container = document.createElement("div")
+        const root = createRoot(container)
+        const Probe = () => {
+            result = useAgentChatSession({
+                entityId: "revision-1",
+                sessionId,
+                initialMessages: [],
+                intent: {} as never,
+            })
+            return null
+        }
+        act(() => root.render(createElement(Probe)))
+
+        act(() => result!.markLiveGate({kind: "approval", id: "approval-1"}))
+        act(() => result!.handleStop())
+        act(() => state.capturedHooks!.onError())
+
+        expect(state.cancelSessionExecution).not.toHaveBeenCalled()
+        expect(state.capturedHooks!.sendAutomaticallyWhen({messages: []})).toBe(false)
+
+        act(() => root.unmount())
+    })
+
+    it("keeps remounted interaction actions closed until an accepted paused Stop settles", async () => {
+        const sessionId = "session-1"
+        state.latestTurnId = "turn-1"
+        state.hitlPending = true
+        state.cancelSessionExecution.mockResolvedValue({
+            accepted: true,
+            conflict: false,
+            execution: {id: "turn-1", state: "stopping"},
+        })
+
+        let result: ReturnType | undefined
+        const Probe = () => {
+            result = useAgentChatSession({
+                entityId: "revision-1",
+                sessionId,
+                initialMessages: [],
+                intent: {} as never,
+            })
+            return null
+        }
+
+        const firstContainer = document.createElement("div")
+        const firstRoot = createRoot(firstContainer)
+        act(() => firstRoot.render(createElement(Probe)))
+        await act(async () => {
+            result!.handleStop()
+            await Promise.resolve()
+        })
+        expect(state.cancelSessionExecution).toHaveBeenCalledWith({
+            sessionId,
+            projectId: "project-id",
+            expectedExecutionId: "turn-1",
+        })
+        act(() => firstRoot.unmount())
+
+        state.sessionTurnId = "turn-1"
+        state.stoppingTurnId = "turn-1"
+        const remountContainer = document.createElement("div")
+        const remountRoot = createRoot(remountContainer)
+        act(() => remountRoot.render(createElement(Probe)))
+        expect(result!.stopping).toBe(true)
+
+        state.stoppingTurnId = null
+        act(() => remountRoot.render(createElement(Probe)))
+        expect(result!.stopping).toBe(false)
+
+        act(() => remountRoot.unmount())
+    })
+
+    it("settles an acknowledged legacy Stop without entering the retry deadline", async () => {
+        vi.useFakeTimers()
+        const sessionId = "session-1"
+        state.busy = true
+        state.turnIds.set(sessionId, "turn-1")
+        state.cancelSessionExecution.mockResolvedValue({
+            accepted: true,
+            conflict: false,
+            execution: {id: "turn-1", state: "idle"},
+        })
+
+        let result: ReturnType | undefined
+        const container = document.createElement("div")
+        const root = createRoot(container)
+        const Probe = () => {
+            result = useAgentChatSession({
+                entityId: "revision-1",
+                sessionId,
+                initialMessages: [],
+                intent: {} as never,
+            })
+            return null
+        }
+        act(() => root.render(createElement(Probe)))
+
+        await act(async () => {
+            result!.handleStop()
+            await Promise.resolve()
+        })
+        act(() => vi.advanceTimersByTime(30_000))
+
+        expect(state.stop).toHaveBeenCalledOnce()
+        expect(state.cancelSessionExecution).toHaveBeenCalledOnce()
+        expect(result!.stopping).toBe(false)
+
+        act(() => root.unmount())
+        vi.useRealTimers()
+    })
+
+    it("drops a stale retry fence after conflict so the next Stop targets the observed run", async () => {
+        vi.useFakeTimers()
+        const sessionId = "session-1"
+        state.busy = true
+        state.turnIds.set(sessionId, "turn-original")
+        state.cancelSessionExecution
+            .mockResolvedValueOnce({
+                accepted: true,
+                conflict: false,
+                execution: {id: "turn-original", state: "stopping"},
+            })
+            .mockResolvedValueOnce({
+                accepted: false,
+                conflict: true,
+                execution: {id: null, state: "idle"},
+            })
+            .mockResolvedValueOnce({
+                accepted: true,
+                conflict: false,
+                execution: {id: "turn-replacement", state: "stopping"},
+            })
+
+        let result: ReturnType | undefined
+        const container = document.createElement("div")
+        const root = createRoot(container)
+        const Probe = () => {
+            result = useAgentChatSession({
+                entityId: "revision-1",
+                sessionId,
+                initialMessages: [],
+                intent: {} as never,
+            })
+            return null
+        }
+        act(() => root.render(createElement(Probe)))
+
+        await act(async () => {
+            result!.handleStop()
+            await Promise.resolve()
+        })
+        act(() => vi.advanceTimersByTime(30_000))
+
+        state.turnIds.set(sessionId, "turn-replacement")
+        await act(async () => {
+            result!.handleStop()
+            await Promise.resolve()
+        })
+        await act(async () => {
+            result!.handleStop()
+            await Promise.resolve()
+        })
+
+        expect(state.cancelSessionExecution).toHaveBeenNthCalledWith(2, {
+            sessionId,
+            projectId: "project-id",
+            expectedExecutionId: "turn-original",
+        })
+        expect(state.cancelSessionExecution).toHaveBeenNthCalledWith(3, {
+            sessionId,
+            projectId: "project-id",
+            expectedExecutionId: "turn-replacement",
+        })
+
+        act(() => root.unmount())
+        vi.useRealTimers()
+    })
+
+    it("resets a pending execution lookup when the mounted session changes", async () => {
+        let release!: (value: {status: "aborted"}) => void
+        state.busy = true
+        state.resolveStopExecution.mockImplementation(
+            () =>
+                new Promise((resolve) => {
+                    release = resolve
+                }),
+        )
+
+        let sessionId = "session-1"
+        let result: ReturnType | undefined
+        const container = document.createElement("div")
+        const root = createRoot(container)
+        const Probe = () => {
+            result = useAgentChatSession({
+                entityId: "revision-1",
+                sessionId,
+                initialMessages: [],
+                intent: {} as never,
+            })
+            return null
+        }
+        act(() => root.render(createElement(Probe)))
+
+        act(() => result!.handleStop())
+        expect(result!.stopping).toBe(true)
+
+        sessionId = "session-2"
+        act(() => root.render(createElement(Probe)))
+        expect(result!.stopping).toBe(false)
+
+        await act(async () => {
+            release({status: "aborted"})
+            await Promise.resolve()
+        })
+        expect(result!.stopping).toBe(false)
+
+        act(() => root.unmount())
+    })
+
+    it("ignores a cancellation response from the previously mounted session", async () => {
+        let release!: (value: {
+            accepted: true
+            conflict: false
+            execution: {id: string; state: "stopping"}
+        }) => void
+        state.busy = true
+        state.turnIds.set("session-1", "turn-1")
+        state.cancelSessionExecution.mockImplementation(
+            () =>
+                new Promise((resolve) => {
+                    release = resolve
+                }),
+        )
+
+        let sessionId = "session-1"
+        let result: ReturnType | undefined
+        const container = document.createElement("div")
+        const root = createRoot(container)
+        const Probe = () => {
+            result = useAgentChatSession({
+                entityId: "revision-1",
+                sessionId,
+                initialMessages: [],
+                intent: {} as never,
+            })
+            return null
+        }
+        act(() => root.render(createElement(Probe)))
+
+        act(() => result!.handleStop())
+        expect(result!.stopping).toBe(true)
+
+        sessionId = "session-2"
+        act(() => root.render(createElement(Probe)))
+        expect(result!.stopping).toBe(false)
+
+        await act(async () => {
+            release({
+                accepted: true,
+                conflict: false,
+                execution: {id: "turn-1", state: "stopping"},
+            })
+            await Promise.resolve()
+        })
+        expect(result!.stopping).toBe(false)
+
+        act(() => root.unmount())
+    })
+
+    it("waits for a resumed execution id before sending Stop", async () => {
+        const sessionId = "session-1"
+        let release!: (value: {status: "resolved"; executionId: string}) => void
+        state.busy = true
+        state.resolveStopExecution.mockImplementation(
+            () =>
+                new Promise((resolve) => {
+                    release = resolve
+                }),
+        )
+        state.cancelSessionExecution.mockResolvedValue({
+            accepted: true,
+            conflict: false,
+            execution: {id: "turn-resumed", state: "stopping"},
+        })
+
+        let result: ReturnType | undefined
+        const container = document.createElement("div")
+        const root = createRoot(container)
+        const Probe = () => {
+            result = useAgentChatSession({
+                entityId: "revision-1",
+                sessionId,
+                initialMessages: [],
+                intent: {} as never,
+            })
+            return null
+        }
+        act(() => root.render(createElement(Probe)))
+
+        act(() => result!.handleStop())
+        expect(state.resolveStopExecution).toHaveBeenCalledOnce()
+        expect(state.cancelSessionExecution).not.toHaveBeenCalled()
+
+        await act(async () => {
+            release({status: "resolved", executionId: "turn-resumed"})
+            await Promise.resolve()
+        })
+        expect(state.cancelSessionExecution).toHaveBeenCalledWith({
+            sessionId,
+            projectId: "project-id",
+            expectedExecutionId: "turn-resumed",
+        })
+
+        act(() => root.unmount())
+    })
+})
diff --git a/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts b/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts
index f3cceb6bd2e..fd6594c0a0e 100644
--- a/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts
+++ b/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts
@@ -1,13 +1,21 @@
-import {useCallback, useEffect, useRef, useState} from "react"
+import {useCallback, useEffect, useReducer, useRef} from "react"
 
 import {
     buildRequestWithinDeadline,
     getMessageTraceId,
+    latestTurnId,
+    resolveStopExecution,
     startupLabelFromDataPart,
 } from "@agenta/chat/assets"
 import type {ClientToolOutputHandler} from "@agenta/chat/clientTools"
 import {useSessionChat} from "@agenta/chat/hooks"
-import {ignoreStreamRejection, parseAgentRunError} from "@agenta/chat/model"
+import {
+    ignoreStreamRejection,
+    createUserStoppedState,
+    isSessionTurnStopping,
+    parseAgentRunError,
+    reduceUserStoppedState,
+} from "@agenta/chat/model"
 import {
     clearTurnClockAtom,
     stampMessagesCreatedAtAtom,
@@ -15,15 +23,18 @@ import {
 } from "@agenta/chat/state"
 import {expandedKeysForMessages, pruneExpandedAtom} from "@agenta/chat/state"
 import {
+    clearSessionTurnId,
+    getSessionTurnId,
     isChatBusy,
     persistSessionMessagesAtom,
     sessionMessagesAtom,
     sessionRecordCountsReadAtom,
     setSessionStatusAtom,
+    setSessionTurnId,
     type SessionChatHooks,
 } from "@agenta/chat/state"
 import {
-    commandSessionStream,
+    cancelSessionExecution,
     invalidateSessionListQueries,
     killSession,
     recordInteractionAnswerAtom,
@@ -37,6 +48,7 @@ import {
     approvalResolution,
     buildAgentRequest,
     buildTurnCapture,
+    isHitlPending,
     isResumeSend,
     playgroundController,
     recordAnswerThenRelease,
@@ -44,6 +56,7 @@ import {
 } from "@agenta/playground"
 import {agentSelfCommitSignalAtom} from "@agenta/shared/state"
 import {generateId} from "@agenta/shared/utils"
+import {message} from "@agenta/ui/app-message"
 import {useChat} from "@ai-sdk/react"
 import {useQueryClient} from "@tanstack/react-query"
 import {type UIMessage} from "ai"
@@ -52,6 +65,7 @@ import {useAtomValue, useSetAtom, useStore} from "jotai"
 import {projectIdAtom} from "@/oss/state/project"
 
 import {doesAgentChatStopKillSession} from "../assets/constants"
+import {isStoppingPhase, reduceStopPhase} from "../assets/stopState"
 import {invalidateSessionInspector} from "../components/Inspector/invalidate"
 import {useChatScopeKey} from "../state/scope"
 import {openSessionIdsAtomFamily} from "../state/sessions"
@@ -108,7 +122,17 @@ export const useAgentChatSession = ({
     // so this is a single boolean gated on position at render time — independent of message ids (which
     // can be missing/duplicated in restore/error paths and would otherwise smear the tag onto every
     // turn). Cleared on the next send/resend.
-    const [stopped, setStopped] = useState(false)
+    const [userStoppedState, dispatchStopped] = useReducer(
+        reduceUserStoppedState,
+        initialMessages,
+        createUserStoppedState,
+    )
+    const stopped = userStoppedState.stopped
+    const setStopped = useCallback(
+        (next: boolean) => dispatchStopped({type: next ? "user-stop" : "reset"}),
+        [],
+    )
+    const [stopPhase, dispatchStop] = useReducer(reduceStopPhase, "idle")
 
     const captureTurnRequest = useSetAtom(captureTurnRequestAtom)
     const revalidateSessionMounts = useSetAtom(revalidateSessionMountsAtom)
@@ -123,6 +147,7 @@ export const useAgentChatSession = ({
     // Whether this mount is still on screen. The chat outlives it, so its callbacks need to tell
     // "still mine to report" from "running on in the background".
     const mountedRef = useRef(false)
+    const messagesRef = useRef(initialMessages)
     const setTurnStartupLabel = useSetAtom(startTurnClockAtom)
 
     // Rebuilt every render and bound to the chat on every commit (below), so they always see the live
@@ -130,6 +155,7 @@ export const useAgentChatSession = ({
     // instead of sticking to the revision this session first mounted on.
     const hooks: SessionChatHooks = {
         prepareRequest: async ({messages, id}) => {
+            clearSessionTurnId(sessionId)
             // Bounded: retries while the invocation URL is still loading and rejects if the build
             // hangs, so a failed send surfaces as an error bubble instead of an eternal spinner
             // (#6042). The helper owns the not-ready / timed-out errors.
@@ -166,7 +192,12 @@ export const useAgentChatSession = ({
         // `is_running: true` outlived the answer by up to 15s (#5844). Safe to refetch immediately —
         // the runner awaits its `is_running: false` heartbeat BEFORE closing this stream
         // (services/runner/src/server.ts `aliveWatchdog.release()`), so the flag is already cleared.
-        onFinish: ({message}) => {
+        onFinish: ({message, messages: finishedMessages, finishReason}) => {
+            dispatchStopped({
+                type: "stream-terminal",
+                messages: finishedMessages,
+                finishReason,
+            })
             markTraceAsFresh(getMessageTraceId(message))
             revalidateSessionMounts(sessionId)
             revalidateSessionRecords(sessionId)
@@ -181,13 +212,10 @@ export const useAgentChatSession = ({
             if (!mountedRef.current) setSessionStatus({id: sessionId, status: "idle"})
         },
         onError: () => {
-            // Clear the marker but do NOT void the resume. A gateway approval is answered while the
-            // stream is still open, so the SDK skips its own dispatch and only re-evaluates when the
-            // stream ends — often by erroring, right here. `null` made that last evaluation return
-            // false and stranded the answer; `undefined` lets the tail heuristics decide.
-            // Adoption is unaffected: the hydration guard reads this ref as a boolean.
-            // The registry logs the error for the dev overlay (F-033) before calling this.
-            liveGateInteractionRef.current = undefined
+            // Preserve null after resume/Stop; only a live marker may fall back to tail detection.
+            if (liveGateInteractionRef.current !== null) {
+                liveGateInteractionRef.current = undefined
+            }
         },
     }
 
@@ -206,10 +234,10 @@ export const useAgentChatSession = ({
 
     const {
         messages,
-        sendMessage,
+        sendMessage: sendChatMessage,
         status,
         stop,
-        regenerate,
+        regenerate: regenerateChatMessage,
         setMessages,
         addToolApprovalResponse,
         addToolOutput,
@@ -221,15 +249,32 @@ export const useAgentChatSession = ({
         experimental_throttle: 50,
     })
 
-    const busy = isChatBusy(status)
+    const sendMessageWithFreshGuard: typeof sendChatMessage = useCallback(
+        (...args: Parameters) => {
+            clearSessionTurnId(sessionId)
+            return sendChatMessage(...args)
+        },
+        [sendChatMessage, sessionId],
+    )
+    const regenerateWithFreshGuard: typeof regenerateChatMessage = useCallback(
+        (...args: Parameters) => {
+            clearSessionTurnId(sessionId)
+            return regenerateChatMessage(...args)
+        },
+        [regenerateChatMessage, sessionId],
+    )
 
+    const busy = isChatBusy(status)
     // `messages`/`busy` change every token; consumers that must stay referentially stable
     // (`handleRewind`, the hydration/SWR adoption guards) read them through refs instead.
-    const messagesRef = useRef(messages)
     messagesRef.current = messages
     const busyRef = useRef(busy)
     busyRef.current = busy
 
+    useEffect(() => {
+        dispatchStopped({type: "transcript", messages})
+    }, [messages])
+
     // Mid-stream drive signals: settled write-ish tool calls append file-activity entries (and
     // throttle-revalidate the drives) as the turn streams, not just at onFinish.
     useFileActivityDetector({sessionId, messages})
@@ -237,7 +282,14 @@ export const useAgentChatSession = ({
     // Server-side platform ops (create_schedule, …) stale the client cache with no other signal.
     useToolCacheInvalidation({sessionId, messages})
 
-    const {isHydrating, hydratedEmpty, runningElsewhere} = useSessionHydration({
+    const {
+        isHydrating,
+        hydratedEmpty,
+        runningElsewhere,
+        stopStateLoading,
+        sessionTurnId,
+        stoppingTurnId,
+    } = useSessionHydration({
         sessionId,
         initialMessages,
         messagesRef,
@@ -251,6 +303,13 @@ export const useAgentChatSession = ({
         intent,
         pendingResumeRef: liveGateInteractionRef,
     })
+    const stopping =
+        isStoppingPhase(stopPhase) ||
+        isSessionTurnStopping({
+            currentTurnId: sessionTurnId ?? latestTurnId(messages),
+            stoppingTurnId,
+        }) ||
+        (stopStateLoading && isHitlPending(messages))
 
     // A decision made in THIS mount marks the resume as live — a restored approval-requested tail
     // the user answers after a reload genuinely auto-resumes, so the queue's pre-resume hold applies.
@@ -346,6 +405,12 @@ export const useAgentChatSession = ({
         restoredIdsRef.current.has(lastMessage.id) &&
         agentShouldResumeAfterApproval({messages})
 
+    // Cache only the newest turn id observed by this page for guarded Stop.
+    useEffect(() => {
+        const turnId = latestTurnId(messages)
+        if (turnId) setSessionTurnId(sessionId, turnId)
+    }, [messages, sessionId])
+
     // Surface a stream failure inline: stamp the parsed error onto the failing assistant turn so
     // it renders as a red error bubble with the real reason (and persists with the session via the
     // effect below), instead of a transient top banner + a generic "no response". FE-only — it
@@ -469,41 +534,184 @@ export const useAgentChatSession = ({
         }
     }, [messages, entityId, switchEntity, store, setAgentCommitSignal])
 
-    // ── DT3 cancelled state: wrap stop() to mark the in-flight assistant turn ──
-    const markStopped = useCallback(() => {
-        const last = messages[messages.length - 1]
-        if (last && last.role === "assistant") setStopped(true)
-    }, [messages])
-
     const projectId = useAtomValue(projectIdAtom)
+    const expectedStopExecutionIdRef = useRef(undefined)
+    const retryStopRef = useRef(false)
+    const abortAfterAcceptedRef = useRef(false)
+    const stopResolutionRef = useRef(null)
+    const stopAttemptRef = useRef(0)
+
+    useEffect(() => {
+        stopAttemptRef.current += 1
+        dispatchStop({type: "reset"})
+        retryStopRef.current = false
+        abortAfterAcceptedRef.current = false
+        expectedStopExecutionIdRef.current = undefined
+        return () => {
+            stopResolutionRef.current?.abort()
+        }
+    }, [sessionId])
 
     const handleStop = useCallback(() => {
-        markStopped()
-        // A stop voids the pending gate (same rule the queue applies), so the marker must go too —
-        // otherwise it outlives the abandoned resume and blocks this mount's records adoption.
+        if (stopping) return
+        // Fence delayed approval release even when cancellation cannot be requested yet.
         liveGateInteractionRef.current = null
-        stop() // abort the client stream immediately
-        if (!projectId || !sessionId) return
+        const wasParked = !busyRef.current && isHitlPending(messagesRef.current)
+        const stopAttempt = ++stopAttemptRef.current
+        dispatchStop({type: "request"})
+        if (!projectId || !sessionId) {
+            dispatchStop({type: "failed"})
+            message.warning("Could not stop the run. It may still be running.")
+            return
+        }
         // Opt-in hard kill (NEXT_PUBLIC_AGENT_CHAT_STOP_KILLS_SESSION): tear the whole session down.
         if (doesAgentChatStopKillSession()) {
             killSession({sessionId, projectId})
                 .then((ok) => {
+                    if (stopAttemptRef.current !== stopAttempt) return
                     if (ok) {
+                        dispatchStop(
+                            wasParked ? {type: "cancelled", parked: true} : {type: "accepted"},
+                        )
                         queryClient.invalidateQueries({queryKey: ["session-liveness"]})
-                        // Refresh an open Inspector's Runtime lens so its Lifecycle/State reflect the
-                        // kill immediately (mirrors the panel's own Kill button).
+                        // Refresh an open Inspector so it reflects the kill immediately.
                         void invalidateSessionInspector(queryClient, sessionId)
+                    } else {
+                        dispatchStop({type: "failed"})
+                        message.warning("Could not stop the run. It may still be running.")
                     }
                 })
-                .catch(() => {})
+                .catch((error: unknown) => {
+                    if (stopAttemptRef.current !== stopAttempt) return
+                    dispatchStop({type: "failed"})
+                    message.warning(
+                        error instanceof Error
+                            ? error.message
+                            : "Could not stop the run. It may still be running.",
+                    )
+                })
             return
         }
-        // Default Stop: cooperatively cancel the CURRENT TURN. The control-plane `cancel` command
-        // (no inputs, no force) drops the alive lock; the runner closes the turn as interrupted and
-        // the session STAYS OPEN so a follow-up prompt resumes it — instead of the old behaviour where
-        // the client stream aborted but the runner kept running and billing.
-        commandSessionStream({sessionId, projectId}).catch(() => {})
-    }, [markStopped, stop, projectId, sessionId, queryClient])
+        // Keep the stream attached until a terminal event confirms accepted cancellation.
+        const isRetry = retryStopRef.current
+        const expectedExecutionId = isRetry
+            ? expectedStopExecutionIdRef.current
+            : getSessionTurnId(sessionId)
+        retryStopRef.current = false
+        abortAfterAcceptedRef.current = isRetry
+        const cancel = async () => {
+            let resolvedExecutionId = expectedExecutionId
+            if (!isRetry && !resolvedExecutionId && busyRef.current) {
+                const controller = new AbortController()
+                stopResolutionRef.current?.abort()
+                stopResolutionRef.current = controller
+                const resolution = await resolveStopExecution({
+                    readExecutionId: () => getSessionTurnId(sessionId),
+                    isRunActive: () => busyRef.current,
+                    signal: controller.signal,
+                })
+                if (stopResolutionRef.current === controller) stopResolutionRef.current = null
+                if (resolution.status !== "resolved") return {resolution} as const
+                resolvedExecutionId = resolution.executionId
+            }
+            expectedStopExecutionIdRef.current = resolvedExecutionId
+            const outcome = await cancelSessionExecution({
+                sessionId,
+                projectId,
+                expectedExecutionId: resolvedExecutionId,
+            })
+            return {outcome} as const
+        }
+        void cancel()
+            .then((result) => {
+                if (stopAttemptRef.current !== stopAttempt) return
+                if ("resolution" in result && result.resolution) {
+                    if (result.resolution.status === "settled") {
+                        dispatchStop({type: "terminal"})
+                    } else if (result.resolution.status === "timed_out") {
+                        dispatchStop({type: "failed"})
+                        message.warning("Could not identify the run to stop. Please try again.")
+                    }
+                    return
+                }
+                const {outcome} = result
+                void invalidateSessionInspector(queryClient, sessionId)
+                if (outcome?.accepted) {
+                    dispatchStop({type: "cancelled", parked: wasParked})
+                    const legacyStopSettled = outcome.execution.state === "idle"
+                    if (legacyStopSettled || abortAfterAcceptedRef.current) {
+                        stop()
+                        dispatchStop({type: "terminal"})
+                    }
+                    queryClient.invalidateQueries({queryKey: ["session-liveness"]})
+                    return
+                }
+                if (outcome && !outcome.conflict && outcome.execution.state === "idle") {
+                    abortAfterAcceptedRef.current = false
+                    expectedStopExecutionIdRef.current = undefined
+                    dispatchStop({type: "already_idle"})
+                    queryClient.invalidateQueries({queryKey: ["session-liveness"]})
+                    return
+                }
+                if (outcome?.conflict) {
+                    retryStopRef.current = false
+                    expectedStopExecutionIdRef.current = undefined
+                } else if (abortAfterAcceptedRef.current) {
+                    retryStopRef.current = true
+                }
+                abortAfterAcceptedRef.current = false
+                dispatchStop({type: "failed"})
+                message.warning(
+                    outcome?.conflict
+                        ? "That run had already finished. The session is running something else now."
+                        : "Could not stop the run. It may still be running.",
+                )
+                queryClient.invalidateQueries({queryKey: ["session-liveness"]})
+            })
+            .catch((error: unknown) => {
+                if (stopAttemptRef.current !== stopAttempt) return
+                stopResolutionRef.current = null
+                if (abortAfterAcceptedRef.current) retryStopRef.current = true
+                abortAfterAcceptedRef.current = false
+                dispatchStop({type: "failed"})
+                message.warning(
+                    error instanceof Error
+                        ? error.message
+                        : "Could not stop the run. It may still be running.",
+                )
+            })
+    }, [stopping, projectId, sessionId, queryClient, stop])
+
+    useEffect(() => {
+        if (stopPhase !== "accepted") return
+        const timer = setTimeout(() => {
+            retryStopRef.current = true
+            abortAfterAcceptedRef.current = false
+            dispatchStop({type: "timeout"})
+        }, 30_000)
+        return () => clearTimeout(timer)
+    }, [stopPhase])
+
+    const previousBusyRef = useRef(busy)
+    useEffect(() => {
+        const wasBusy = previousBusyRef.current
+        previousBusyRef.current = busy
+        if (wasBusy && !busy) {
+            retryStopRef.current = false
+            dispatchStop({type: "terminal"})
+        }
+        if (!wasBusy && busy) dispatchStop({type: "reset"})
+    }, [busy])
+
+    useEffect(() => {
+        if (stopPhase !== "stopped") return
+        const last = messagesRef.current[messagesRef.current.length - 1]
+        if (last?.role === "assistant") setStopped(true)
+        retryStopRef.current = false
+        abortAfterAcceptedRef.current = false
+        expectedStopExecutionIdRef.current = undefined
+        dispatchStop({type: "reset"})
+    }, [stopPhase])
 
     // ── D9 teardown: `useSessionChat` releases the claim; this tracks what it does not own ──
     // The startup clock only goes with the session when the session itself is gone — clearing it
@@ -536,8 +744,8 @@ export const useAgentChatSession = ({
         status,
         busy,
         error,
-        sendMessage,
-        regenerate,
+        sendMessage: sendMessageWithFreshGuard,
+        regenerate: regenerateWithFreshGuard,
         setMessages,
         addToolApprovalResponse,
         messagesRef,
@@ -546,6 +754,7 @@ export const useAgentChatSession = ({
         hydratedEmpty,
         runningElsewhere,
         stopped,
+        stopping,
         setStopped,
         handleStop,
         handleClientToolOutput,
diff --git a/web/oss/src/components/AgentChatSlice/hooks/useSessionHydration.ts b/web/oss/src/components/AgentChatSlice/hooks/useSessionHydration.ts
index acf5631f9dc..99d7416e2ad 100644
--- a/web/oss/src/components/AgentChatSlice/hooks/useSessionHydration.ts
+++ b/web/oss/src/components/AgentChatSlice/hooks/useSessionHydration.ts
@@ -501,11 +501,20 @@ export const useSessionHydration = ({
         sessionId,
         projectId,
         // #5919 relay; this surface re-reads records on any interaction change.
-        onInteractionChanged: () => revalidateSessionRecords(sessionId),
+        onInteractionChanged: () => {
+            revalidateSessionRecords(sessionId)
+        },
         enabled: activeSessionId === sessionId,
         onReady: refreshOnReady,
         onRecordsChanged: refreshFromRecords,
     })
 
-    return {isHydrating, hydratedEmpty, runningElsewhere}
+    return {
+        isHydrating,
+        hydratedEmpty,
+        runningElsewhere,
+        stopStateLoading: liveness.isLoading,
+        sessionTurnId: liveness.turnId,
+        stoppingTurnId: liveness.stoppingTurnId,
+    }
 }
diff --git a/web/oss/src/components/AgentChatSlice/hooks/useSessionRecordsWatch.ts b/web/oss/src/components/AgentChatSlice/hooks/useSessionRecordsWatch.ts
index b2487d8f8ef..891b6a3073c 100644
--- a/web/oss/src/components/AgentChatSlice/hooks/useSessionRecordsWatch.ts
+++ b/web/oss/src/components/AgentChatSlice/hooks/useSessionRecordsWatch.ts
@@ -1,4 +1,5 @@
 import {useWatchEventSource} from "@agenta/sessions/watch"
+import {useQueryClient} from "@tanstack/react-query"
 
 import {getAgentaApiUrl} from "@/oss/lib/helpers/api"
 import {refreshSession} from "@/oss/lib/helpers/auth/refreshSession"
@@ -32,6 +33,7 @@ export const useSessionRecordsWatch = ({
     onRecordsChanged: () => void
     onInteractionChanged: () => void
 }): void => {
+    const queryClient = useQueryClient()
     const url = sessionId && projectId ? sessionWatchUrl(sessionId, projectId) : null
     useWatchEventSource({
         url,
@@ -41,6 +43,14 @@ export const useSessionRecordsWatch = ({
             ready: onReady,
             "records-changed": onRecordsChanged,
             interaction: onInteractionChanged,
+            // A session that ends without this tab running it — a Stop from elsewhere, or the
+            // execution watchdog settling a turn whose runner went silent. The records arrive
+            // on their own event; this is the half that stops the session still LOOKING alive,
+            // which otherwise waits out the 15s liveness poll. Mobile already does this
+            // (web/mobile/src/features/chat/useSessionWatch.ts).
+            lifecycle: () => {
+                void queryClient.invalidateQueries({queryKey: ["session-liveness"]})
+            },
         },
     })
 }
diff --git a/web/oss/src/components/AgentChatSlice/state/liveness.ts b/web/oss/src/components/AgentChatSlice/state/liveness.ts
index 90797abe5f0..c4c677bb4bc 100644
--- a/web/oss/src/components/AgentChatSlice/state/liveness.ts
+++ b/web/oss/src/components/AgentChatSlice/state/liveness.ts
@@ -3,6 +3,7 @@ import {sessionLocalSettledAtAtomFamily, sessionStatusAtomFamily} from "@agenta/
 import {
     deriveSessionLifecycle,
     deriveStreamNest,
+    livenessPollInterval,
     querySessionStreams,
     type SessionLifecycle,
     type SessionStream,
@@ -14,18 +15,7 @@ import {atomWithQuery} from "jotai-tanstack-query"
 
 import {projectIdAtom} from "@/oss/state/project"
 
-/**
- * Backend liveness for the project's sessions (cross-device truth). The tab dot reads this to
- * reflect a session still running on the backend even when THIS browser isn't streaming it (a
- * reopened chat, or a run started on another device).
- *
- * ONE project-scoped query (`is_alive=true`) backs every dot rather than one fetch per session, so
- * N idle tabs cost ONE request, not N — important on cold load (see the request-count budget). Only
- * alive streams come back, which is exactly what the dot needs (running/alive vs idle); a session
- * absent from the result is dormant/cold/dead/new and simply reads as idle. Kept out of the live
- * conversation's way: the fetch is LOW-PRIORITY, polls only WHILE something is alive (empty result
- * → stop), and re-checks on tab refocus.
- */
+/** One low-priority project query supplies cross-device liveness for every tab dot. */
 const aliveStreamsQueryAtom = atomWithQuery((get) => {
     const projectId = get(projectIdAtom)
     return {
@@ -39,7 +29,7 @@ const aliveStreamsQueryAtom = atomWithQuery((get) => {
             }),
         enabled: Boolean(projectId),
         staleTime: 10_000,
-        refetchInterval: (query) => ((query.state.data?.length ?? 0) > 0 ? 15_000 : false),
+        refetchInterval: (query) => livenessPollInterval(query.state.data),
         refetchOnWindowFocus: true,
     }
 })
@@ -57,6 +47,9 @@ export interface SessionLiveness {
     lifecycle: SessionLifecycle
     /** The stream nest + derived resumable/reattachable predicates. */
     nest: SessionStreamNest
+    /** Current execution and durable Stop admission marker from the stream row. */
+    turnId: string | null
+    stoppingTurnId: string | null
     isLoading: boolean
 }
 
@@ -70,6 +63,8 @@ export const sessionLivenessAtomFamily = atomFamily((sessionId: string) =>
         return {
             lifecycle: deriveSessionLifecycle(stream),
             nest: deriveStreamNest(stream),
+            turnId: stream?.turn_id ?? null,
+            stoppingTurnId: stream?.stopping_turn_id ?? null,
             isLoading: get(aliveStreamsQueryAtom).isLoading,
         }
     }),
diff --git a/web/oss/src/components/Layout/Layout.tsx b/web/oss/src/components/Layout/Layout.tsx
index bb57341402e..67246f5eb41 100644
--- a/web/oss/src/components/Layout/Layout.tsx
+++ b/web/oss/src/components/Layout/Layout.tsx
@@ -4,7 +4,6 @@ import {NotFoundScreen} from "@agenta/auth-ui"
 import {workflowLatestRevisionQueryAtomFamily} from "@agenta/entities/workflow"
 import {SETTINGS_SIDEBAR_SCOPE_ID} from "@agenta/navigation"
 import {ProjectWatch} from "@agenta/sessions/watch"
-import AppMessageContext from "@agenta/ui/app-message"
 import {useVisualViewportHeight} from "@agenta/ui/hooks"
 import {ConfigProvider, Layout, Modal, theme} from "antd"
 import clsx from "clsx"
@@ -449,7 +448,6 @@ const App: React.FC = ({children}) => {
     return (
         <>
             
-            
             {typeof window === "undefined" ? null : isBareRoute ? (
                 
                     
diff --git a/web/package.json b/web/package.json
index 1ca3e05a4a1..752d7adf626 100644
--- a/web/package.json
+++ b/web/package.json
@@ -1,6 +1,6 @@
 {
     "name": "agenta-web",
-    "version": "0.114.8",
+    "version": "0.115.0",
     "workspaces": [
         "ee",
         "mobile",
diff --git a/web/packages/agenta-api-client/package.json b/web/packages/agenta-api-client/package.json
index d86907c6dba..16ca05545ca 100644
--- a/web/packages/agenta-api-client/package.json
+++ b/web/packages/agenta-api-client/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@agentaai/api-client",
-  "version": "0.114.8",
+  "version": "0.115.0",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",
diff --git a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/Client.ts b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/Client.ts
index 2155960e439..c9c0b2ee1e3 100644
--- a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/Client.ts
+++ b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/Client.ts
@@ -2536,4 +2536,82 @@ export class SessionsClient {
 
         return handleNonStatusCodeError(_response.error, _response.rawResponse, "POST", "/sessions/unarchive");
     }
+
+    /**
+     * @param {AgentaApi.CancelSessionExecutionRequest} request
+     * @param {SessionsClient.RequestOptions} requestOptions - Request-specific configuration.
+     *
+     * @throws {@link AgentaApi.UnprocessableEntityError}
+     *
+     * @example
+     *     await client.sessions.cancelSessionExecution({
+     *         session_id: "session_id",
+     *         body: {}
+     *     })
+     */
+    public cancelSessionExecution(
+        request: AgentaApi.CancelSessionExecutionRequest,
+        requestOptions?: SessionsClient.RequestOptions,
+    ): core.HttpResponsePromise {
+        return core.HttpResponsePromise.fromPromise(this.__cancelSessionExecution(request, requestOptions));
+    }
+
+    private async __cancelSessionExecution(
+        request: AgentaApi.CancelSessionExecutionRequest,
+        requestOptions?: SessionsClient.RequestOptions,
+    ): Promise> {
+        const { session_id: sessionId, body: _body } = request;
+        const _authRequest: core.AuthRequest = await this._options.authProvider.getAuthRequest();
+        const _headers: core.Fetcher.Args["headers"] = mergeHeaders(
+            _authRequest.headers,
+            this._options?.headers,
+            requestOptions?.headers,
+        );
+        const _response = await core.fetcher({
+            url: core.url.join(
+                (await core.Supplier.get(this._options.baseUrl)) ??
+                    (await core.Supplier.get(this._options.environment)) ??
+                    environments.AgentaApiEnvironment.Default,
+                `sessions/${core.url.encodePathParam(sessionId)}/cancel`,
+            ),
+            method: "POST",
+            headers: _headers,
+            contentType: "application/json",
+            queryParameters: requestOptions?.queryParams,
+            requestType: "json",
+            body: _body,
+            timeoutMs: (requestOptions?.timeoutInSeconds ?? this._options?.timeoutInSeconds ?? 30) * 1000,
+            maxRetries: requestOptions?.maxRetries ?? this._options?.maxRetries,
+            withCredentials: true,
+            abortSignal: requestOptions?.abortSignal,
+            fetchFn: this._options?.fetch,
+            logging: this._options.logging,
+        });
+        if (_response.ok) {
+            return { data: _response.body, rawResponse: _response.rawResponse };
+        }
+
+        if (_response.error.reason === "status-code") {
+            switch (_response.error.statusCode) {
+                case 422:
+                    throw new AgentaApi.UnprocessableEntityError(
+                        _response.error.body as AgentaApi.HttpValidationError,
+                        _response.rawResponse,
+                    );
+                default:
+                    throw new errors.AgentaApiError({
+                        statusCode: _response.error.statusCode,
+                        body: _response.error.body,
+                        rawResponse: _response.rawResponse,
+                    });
+            }
+        }
+
+        return handleNonStatusCodeError(
+            _response.error,
+            _response.rawResponse,
+            "POST",
+            "/sessions/{session_id}/cancel",
+        );
+    }
 }
diff --git a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/CancelSessionExecutionRequest.ts b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/CancelSessionExecutionRequest.ts
new file mode 100644
index 00000000000..d3cb6ce31e2
--- /dev/null
+++ b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/CancelSessionExecutionRequest.ts
@@ -0,0 +1,15 @@
+// This file was auto-generated by Fern from our API Definition.
+
+import type * as AgentaApi from "../../../../index.js";
+
+/**
+ * @example
+ *     {
+ *         session_id: "session_id",
+ *         body: {}
+ *     }
+ */
+export interface CancelSessionExecutionRequest {
+    session_id: string;
+    body: AgentaApi.SessionCancelRequest | null;
+}
diff --git a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/SessionStreamCommandRequest.ts b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/SessionStreamCommandRequest.ts
index 5d66eef4a2f..a2f69e304b0 100644
--- a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/SessionStreamCommandRequest.ts
+++ b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/SessionStreamCommandRequest.ts
@@ -13,4 +13,5 @@ export interface SessionStreamCommandRequest {
     data?: AgentaApi.WorkflowRequestData | null;
     force?: boolean;
     detached?: boolean;
+    expected_execution_id?: string | null;
 }
diff --git a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/index.ts b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/index.ts
index 5107e65e7d3..304c3ec79d2 100644
--- a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/index.ts
+++ b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/index.ts
@@ -1,5 +1,6 @@
 export type { ArchiveSessionRequest } from "./ArchiveSessionRequest.js";
 export type { BodyUploadSessionMountFile } from "./BodyUploadSessionMountFile.js";
+export type { CancelSessionExecutionRequest } from "./CancelSessionExecutionRequest.js";
 export type { CreateSessionAttachmentRequest } from "./CreateSessionAttachmentRequest.js";
 export type { DeleteSessionRequest } from "./DeleteSessionRequest.js";
 export type { DeleteSessionStreamRequest } from "./DeleteSessionStreamRequest.js";
diff --git a/web/packages/agenta-api-client/src/generated/api/types/SessionCancelRequest.ts b/web/packages/agenta-api-client/src/generated/api/types/SessionCancelRequest.ts
new file mode 100644
index 00000000000..5010870009e
--- /dev/null
+++ b/web/packages/agenta-api-client/src/generated/api/types/SessionCancelRequest.ts
@@ -0,0 +1,5 @@
+// This file was auto-generated by Fern from our API Definition.
+
+export interface SessionCancelRequest {
+    expected_execution_id?: (string | null) | undefined;
+}
diff --git a/web/packages/agenta-api-client/src/generated/api/types/SessionStream.ts b/web/packages/agenta-api-client/src/generated/api/types/SessionStream.ts
index 90177651fb0..6e663895c08 100644
--- a/web/packages/agenta-api-client/src/generated/api/types/SessionStream.ts
+++ b/web/packages/agenta-api-client/src/generated/api/types/SessionStream.ts
@@ -18,6 +18,8 @@ export interface SessionStream {
     tags?: (Record | null) | undefined;
     meta?: (Record | null) | undefined;
     turn_id?: (string | null) | undefined;
+    turn_started_at?: (string | null) | undefined;
+    stopping_turn_id?: (string | null) | undefined;
     references?: (AgentaApi.SessionReference[] | null) | undefined;
     archived_at?: (string | null) | undefined;
     origin?: (AgentaApi.SessionOrigin | null) | undefined;
diff --git a/web/packages/agenta-api-client/src/generated/api/types/SessionStreamCommandResponse.ts b/web/packages/agenta-api-client/src/generated/api/types/SessionStreamCommandResponse.ts
index 18d1acc5032..2ae62484a8e 100644
--- a/web/packages/agenta-api-client/src/generated/api/types/SessionStreamCommandResponse.ts
+++ b/web/packages/agenta-api-client/src/generated/api/types/SessionStreamCommandResponse.ts
@@ -8,4 +8,5 @@ export interface SessionStreamCommandResponse {
     turn_id?: (string | null) | undefined;
     watcher_id?: (string | null) | undefined;
     detached?: boolean | undefined;
+    cancelled_turn_ids?: string[] | undefined;
 }
diff --git a/web/packages/agenta-api-client/src/generated/api/types/index.ts b/web/packages/agenta-api-client/src/generated/api/types/index.ts
index 42bca07e3be..6df14bd2fb2 100644
--- a/web/packages/agenta-api-client/src/generated/api/types/index.ts
+++ b/web/packages/agenta-api-client/src/generated/api/types/index.ts
@@ -378,6 +378,7 @@ export * from "./Selector.js";
 export * from "./SessionAttachment.js";
 export * from "./SessionAttachmentResponse.js";
 export * from "./SessionAttachmentsResponse.js";
+export * from "./SessionCancelRequest.js";
 export * from "./SessionDelivery.js";
 export * from "./SessionExcludeRequest.js";
 export * from "./SessionExpansion.js";
diff --git a/web/packages/agenta-chat/src/assets/agentTurn.ts b/web/packages/agenta-chat/src/assets/agentTurn.ts
new file mode 100644
index 00000000000..842d49a59e4
--- /dev/null
+++ b/web/packages/agenta-chat/src/assets/agentTurn.ts
@@ -0,0 +1,18 @@
+import type {UIMessage} from "ai"
+
+/** Read the runner-minted turn id from merged stream metadata. */
+export const getMessageTurnId = (message: UIMessage | undefined): string | null => {
+    const turnId = (message?.metadata as {turnId?: unknown} | undefined)?.turnId
+    return typeof turnId === "string" && turnId.trim() ? turnId : null
+}
+
+/** Read only the newest assistant turn id; older ids are unsafe Stop guards. */
+export const latestTurnId = (messages: UIMessage[]): string | null => {
+    for (let index = messages.length - 1; index >= 0; index--) {
+        const message = messages[index]
+        if (message.role === "user") return null
+        if (message.role !== "assistant") continue
+        return getMessageTurnId(message)
+    }
+    return null
+}
diff --git a/web/packages/agenta-chat/src/assets/composerState.ts b/web/packages/agenta-chat/src/assets/composerState.ts
new file mode 100644
index 00000000000..0e02866cd0a
--- /dev/null
+++ b/web/packages/agenta-chat/src/assets/composerState.ts
@@ -0,0 +1,8 @@
+/** A parked approval remains stoppable after streaming pauses. */
+export const shouldShowStopControl = ({
+    busy,
+    hitlPending,
+}: {
+    busy: boolean
+    hitlPending: boolean
+}): boolean => busy || hitlPending
diff --git a/web/packages/agenta-chat/src/assets/index.ts b/web/packages/agenta-chat/src/assets/index.ts
index 3dc5493b5b1..472c119882d 100644
--- a/web/packages/agenta-chat/src/assets/index.ts
+++ b/web/packages/agenta-chat/src/assets/index.ts
@@ -2,6 +2,7 @@ export * from "./toolFormat"
 export * from "./trace"
 export * from "./attachmentRules"
 export * from "./attachmentTransport"
+export * from "./composerState"
 export * from "./files"
 export * from "./rewind"
 export * from "./transcriptToMessages"
@@ -10,3 +11,5 @@ export * from "./conversationLayout"
 export * from "./jumpToLatest"
 export * from "./boundedRequest"
 export {startupLabelFromDataPart} from "./startupPhases"
+export {getMessageTurnId, latestTurnId} from "./agentTurn"
+export * from "./resolveStopExecution"
diff --git a/web/packages/agenta-chat/src/assets/resolveStopExecution.ts b/web/packages/agenta-chat/src/assets/resolveStopExecution.ts
new file mode 100644
index 00000000000..c88ea03ffc8
--- /dev/null
+++ b/web/packages/agenta-chat/src/assets/resolveStopExecution.ts
@@ -0,0 +1,41 @@
+export type StopExecutionResolution =
+    | {status: "resolved"; executionId: string}
+    | {status: "settled"}
+    | {status: "timed_out"}
+    | {status: "aborted"}
+
+const waitForPoll = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms))
+
+/**
+ * Wait for the runner-minted execution id during the short window between sending a turn and
+ * receiving its first live frame. An unnamed Stop in that window can reach the server before the
+ * turn is admitted and incorrectly conclude that the session is idle.
+ */
+export const resolveStopExecution = async ({
+    readExecutionId,
+    isRunActive,
+    signal,
+    timeoutMs = 5_000,
+    pollMs = 25,
+    now = Date.now,
+    wait = waitForPoll,
+}: {
+    readExecutionId: () => string | undefined
+    isRunActive: () => boolean
+    signal?: AbortSignal
+    timeoutMs?: number
+    pollMs?: number
+    now?: () => number
+    wait?: (ms: number) => Promise
+}): Promise => {
+    const deadline = now() + timeoutMs
+    while (true) {
+        if (signal?.aborted) return {status: "aborted"}
+        const executionId = readExecutionId()
+        if (executionId) return {status: "resolved", executionId}
+        if (!isRunActive()) return {status: "settled"}
+        const remaining = deadline - now()
+        if (remaining <= 0) return {status: "timed_out"}
+        await wait(Math.min(pollMs, remaining))
+    }
+}
diff --git a/web/packages/agenta-chat/src/assets/transcriptToMessages.ts b/web/packages/agenta-chat/src/assets/transcriptToMessages.ts
index df6e8131317..d834ac38669 100644
--- a/web/packages/agenta-chat/src/assets/transcriptToMessages.ts
+++ b/web/packages/agenta-chat/src/assets/transcriptToMessages.ts
@@ -71,6 +71,8 @@ interface DraftMessage {
     runError?: string
     /** That error's stable failure class (`error.code`), so a reload keeps the callout's action. */
     runErrorCode?: string
+    /** The terminal `done` carried `stopReason:"cancelled"` — a user Stop, not a failure. */
+    runStopped?: boolean
 }
 
 interface TranscriptIndex {
@@ -579,6 +581,20 @@ export function transcriptToMessages(
                 current.paused = true
                 continue
             }
+            if (p.stopReason === "cancelled") {
+                // Keep a carrier so a content-free cancellation can still render Stopped.
+                if (!current || current.role !== "assistant") {
+                    current = newDraft(row.id, "assistant")
+                    drafts.push(current)
+                }
+                current.runStopped = true
+                current.paused = false
+                for (const part of current.parts) {
+                    if (part.state === "approval-requested") part.state = "output-denied"
+                }
+                current = null
+                continue
+            }
             // A resumed-then-completed turn is no longer paused.
             if (current?.paused) current.resumed = true
             if (current) current.paused = false
@@ -597,13 +613,7 @@ export function transcriptToMessages(
     // Recorded results win; otherwise saved answers, neutral terminal state, then pending.
     applyInteractionRowStates(index, options?.interactionRowStates)
 
-    // A RESUMED turn's gate was answered by definition — the runner only emits post-pause records
-    // once the user responded (a deny settles its own part via `tool_result denied`). The durable
-    // log doesn't always persist the `interaction_response`, so settle whatever is left awaiting:
-    // otherwise a completed turn replays as still parked and the reload keeps the approval dock up.
-    // Runs AFTER the rows on purpose: this sweep knows only THAT a gate was answered, never how, so
-    // ahead of them it consumed the `approval-requested` state the row's verdict is applied to, and
-    // every denied gate replayed as approved.
+    // A resumed turn's remaining approval gate was answered even when its response row is absent.
     for (const d of drafts) {
         if (!d.resumed) continue
         for (const part of d.parts) {
@@ -613,7 +623,7 @@ export function transcriptToMessages(
 
     const messages = drafts
         // A turn whose only content was the failure has no parts — keep it, or the error vanishes.
-        .filter((d) => d.parts.length > 0 || d.runError)
+        .filter((d) => d.parts.length > 0 || d.runError || d.runStopped)
         .map((d) => {
             // `getMessageTraceId`/`getMessageUsage` read exactly these, so the hover trace actions
             // and metrics bar light up on reload. traceId stays absent until the backend stamps one;
@@ -622,7 +632,8 @@ export function transcriptToMessages(
             if (d.traceId) metadata.traceId = d.traceId
             if (d.usage) metadata.usage = d.usage
             if (d.paused) metadata.paused = true
-            if (d.runError)
+            if (d.runStopped) metadata.runStopped = true
+            if (d.runError && !d.runStopped)
                 metadata.runError = {
                     message: d.runError,
                     ...(d.runErrorCode ? {code: d.runErrorCode} : {}),
diff --git a/web/packages/agenta-chat/src/components/ChatComposer.tsx b/web/packages/agenta-chat/src/components/ChatComposer.tsx
index e16698e9b00..79059a47974 100644
--- a/web/packages/agenta-chat/src/components/ChatComposer.tsx
+++ b/web/packages/agenta-chat/src/components/ChatComposer.tsx
@@ -52,6 +52,8 @@ export interface ChatComposerProps {
     onChange?: (markdown: string) => void
     /** A run is streaming — the send button becomes Stop. */
     streaming?: boolean
+    /** The Stop request is pending or accepted, awaiting the stream's terminal event. */
+    stopping?: boolean
     onStop?: () => void
     /** Read at event time — attachments are refused right now (a voice take in flight…). */
     attachmentsBlocked?: () => boolean
@@ -84,6 +86,7 @@ export const ChatComposer = ({
     initialMarkdown,
     onChange,
     streaming,
+    stopping,
     onStop,
     attachmentsBlocked,
     composerDisabled,
@@ -165,6 +168,7 @@ export const ChatComposer = ({
                 sendDisabled={files.length > 0 && !attachmentsSettled}
                 sendDisabledReason={uploadBlockReason}
                 streaming={streaming}
+                stopping={stopping}
                 onStop={onStop}
                 prefix={
                     

diff --git a/web/packages/agenta-chat/src/components/RunningElsewhereStrip.tsx b/web/packages/agenta-chat/src/components/RunningElsewhereStrip.tsx
index f0b9ec3a980..827fb23b184 100644
--- a/web/packages/agenta-chat/src/components/RunningElsewhereStrip.tsx
+++ b/web/packages/agenta-chat/src/components/RunningElsewhereStrip.tsx
@@ -17,10 +17,10 @@ import {cn} from "@agenta/ui/ui"
  *
  * The copy stops short of promising the transcript WILL move. `is_running` says a turn took the
  * lock, not that anything is still serving it: a runner that dies mid-turn leaves the flag set
- * until its shutdown drain completes, or failing that until the orphan sweep clears it
- * (`ORPHAN_THRESHOLD_SECONDS`, 300s). Measured on a dev stack, that window runs from ~20s to a few
- * minutes. Asserting progress through it told people to keep waiting on a run that was over, so
- * the second sentence names that possibility instead. It is deliberately not a call to action:
+ * until its shutdown drain completes, or failing that until the execution watchdog settles it
+ * (`ORPHAN_THRESHOLD_SECONDS`, 120s by default). Measured on a dev stack, that window runs from
+ * ~20s to a couple of minutes. Asserting progress through it told people to keep waiting on a run
+ * that was over, so the second sentence names that possibility instead. It is deliberately not a call to action:
  * only /m passes a Stop here, and the desktop has no control to point at.
  *
  * Matches the `running` dot in the session bar (`bg-colorInfo`, pulsing) so the two read as one
diff --git a/web/packages/agenta-chat/src/hooks/useAgentChatQueue.ts b/web/packages/agenta-chat/src/hooks/useAgentChatQueue.ts
index 285e07a5555..f6572babe03 100644
--- a/web/packages/agenta-chat/src/hooks/useAgentChatQueue.ts
+++ b/web/packages/agenta-chat/src/hooks/useAgentChatQueue.ts
@@ -5,10 +5,15 @@ import {canReleaseQueuedMessage, isHitlPending} from "@agenta/playground/agent-c
 import {generateId} from "@agenta/shared/utils"
 import type {FileUIPart, UIMessage} from "ai"
 
+import {latestTurnId} from "../assets/agentTurn"
+
+import type {ComposerAttachment} from "./useComposerAttachments"
+
 export interface QueuedMessage {
     id: string
     text: string
     fileParts?: FileUIPart[]
+    stagedFiles?: ComposerAttachment[]
 }
 
 interface UseAgentChatQueueArgs {
@@ -83,12 +88,29 @@ export const useAgentChatQueue = ({
         queuedRef.current = queued
     }, [queued])
 
+    // Retained until admission so a refused immediate send can return to the composer.
+    const lastSentRef = useRef(undefined)
+
+    const admittedTurnId = latestTurnId(messages)
+    useEffect(() => {
+        if (admittedTurnId) lastSentRef.current = undefined
+    }, [admittedTurnId])
+
+    /** Take back the last sent message only after an optional placement succeeds. */
+    const takeLastSent = useCallback((place?: (message: QueuedMessage) => boolean) => {
+        const message = lastSentRef.current
+        if (!message || (place && !place(message))) return undefined
+        lastSentRef.current = undefined
+        return message
+    }, [])
+
     // Send now only if idle, unlatched, and the queue is empty; otherwise append (FIFO).
     const submit = useCallback(
-        (item: {text: string; fileParts?: FileUIPart[]}) => {
+        (item: {text: string; fileParts?: FileUIPart[]; stagedFiles?: ComposerAttachment[]}) => {
             const message: QueuedMessage = {...item, id: generateId()}
             if (!releasingRef.current && queuedRef.current.length === 0 && canReleaseNow) {
                 releasingRef.current = true
+                lastSentRef.current = message
                 sendQueued(message)
             } else {
                 setQueued((q) => [...q, message])
@@ -142,7 +164,7 @@ export const useAgentChatQueue = ({
      * so the text the session displaced has to come back here too or it is lost for good.
      */
     const commitEdit = useCallback(
-        (item: {text: string; fileParts?: FileUIPart[]}) => {
+        (item: {text: string; fileParts?: FileUIPart[]; stagedFiles?: ComposerAttachment[]}) => {
             const id = editingId
             setEditingId(null)
             const draft = takeStash()
@@ -152,6 +174,7 @@ export const useAgentChatQueue = ({
                 return draft
             }
             const fileParts = [...(target.fileParts ?? []), ...(item.fileParts ?? [])]
+            const stagedFiles = [...(target.stagedFiles ?? []), ...(item.stagedFiles ?? [])]
             // Edited down to nothing and carrying no files: there is no message left to hold.
             if (!item.text.trim() && fileParts.length === 0) {
                 setQueued((q) => q.filter((m) => m.id !== id))
@@ -164,6 +187,7 @@ export const useAgentChatQueue = ({
                               ...m,
                               text: item.text,
                               fileParts: fileParts.length ? fileParts : undefined,
+                              stagedFiles: stagedFiles.length ? stagedFiles : undefined,
                           }
                         : m,
                 ),
@@ -187,6 +211,8 @@ export const useAgentChatQueue = ({
         releasingRef.current = true
         const [head, ...rest] = queued
         setQueued(rest)
+        // A released head also needs refusal recovery because it has left the queue.
+        lastSentRef.current = head
         sendQueued(head)
     }, [settled, canReleaseNow, queued, sendQueued])
 
@@ -201,5 +227,7 @@ export const useAgentChatQueue = ({
         beginEdit,
         cancelEdit,
         commitEdit,
+        /** Reclaim the last immediately-sent message (e.g. the backend refused it). */
+        takeLastSent,
     }
 }
diff --git a/web/packages/agenta-chat/src/hooks/useAgentConversation.ts b/web/packages/agenta-chat/src/hooks/useAgentConversation.ts
index a53d5f25b5a..9adb1793db3 100644
--- a/web/packages/agenta-chat/src/hooks/useAgentConversation.ts
+++ b/web/packages/agenta-chat/src/hooks/useAgentConversation.ts
@@ -14,7 +14,7 @@
 // Deliberately omitted (desktop-only): first-seen timestamp stamping (display metadata for the desktop rows) — the desktop host keeps its own implementation until the re-plumb.
 // Deliberately omitted (desktop-only): session auto-titling and the first-run seed auto-send — the desktop host keeps its own implementation until the re-plumb.
 // Deliberately omitted (desktop-only): the model-key composer gate — compose `useAgentModelKeyStatus` in the skin instead.
-import {useCallback, useEffect, useMemo, useRef, useState} from "react"
+import {useCallback, useEffect, useMemo, useReducer, useRef, useState} from "react"
 
 import {
     invalidateSessionListQueries,
@@ -39,6 +39,7 @@ import {useChat} from "@ai-sdk/react"
 import type {FileUIPart, UIMessage} from "ai"
 import {useSetAtom, useStore} from "jotai"
 
+import {latestTurnId} from "../assets/agentTurn"
 import {buildRequestWithinDeadline} from "../assets/boundedRequest"
 import {filesToParts} from "../assets/files"
 import {loadSessionMessages, type SessionTranscript} from "../assets/loadSession"
@@ -54,6 +55,7 @@ import {
     type ClientToolPartPredicate,
     type TurnViewModel,
 } from "../model/turnViewModel"
+import {createUserStoppedState, reduceUserStoppedState} from "../model/userStop"
 import {expandedKeysForMessages, pruneExpandedAtom} from "../state/expandState"
 import {stampMessagesCreatedAtAtom} from "../state/messageStamps"
 import {
@@ -62,7 +64,13 @@ import {
     isChatBusy,
     type SessionChatHooks,
 } from "../state/sessionChats"
-import {clearSessionFresh, composerDraftBySession, isSessionFresh} from "../state/sessionEphemera"
+import {
+    clearSessionFresh,
+    clearSessionTurnId,
+    composerDraftBySession,
+    isSessionFresh,
+    setSessionTurnId,
+} from "../state/sessionEphemera"
 import {
     persistSessionMessagesAtom,
     sessionMessagesAtom,
@@ -131,6 +139,8 @@ export interface AgentConversation {
     turns: TurnViewModel[]
     /** Send a user message (routes through the queue: sends now, or holds while busy/paused). */
     send: (input: SendInput) => Promise
+    /** Prevent an approval decision still being recorded from starting its delayed resume. */
+    voidPendingResume: () => void
     /** Abort the in-flight stream and tag the last assistant turn as user-stopped. */
     stop: () => void
     /** Re-run an assistant turn by message id (also the "Resend" action after a stop). */
@@ -194,12 +204,19 @@ export const useAgentConversation = ({
     const setTurnStartupLabel = useSetAtom(startTurnClockAtom)
     const clearTurnClock = useSetAtom(clearTurnClockAtom)
 
-    // Whether the LAST assistant turn was user-stopped. You can only cancel the in-flight (last)
-    // turn, so this is a single boolean gated on position at render time. Cleared on the next
-    // send/resend.
-    const [stopped, setStopped] = useState(false)
     // Seed once from the persisted store (read imperatively so our own writes don't feed back).
     const [initialMessages] = useState(() => store.get(sessionMessagesAtom)[sessionId] ?? [])
+    // Only the last assistant turn can carry the current stopped state.
+    const [userStoppedState, dispatchStopped] = useReducer(
+        reduceUserStoppedState,
+        initialMessages,
+        createUserStoppedState,
+    )
+    const stopped = userStoppedState.stopped
+    const setStopped = useCallback(
+        (next: boolean) => dispatchStopped({type: next ? "user-stop" : "reset"}),
+        [],
+    )
     // Restored (not live-streamed) message ids — the orphaned-resume detection reads this, and a
     // skin can use it to skip entrance animations for restored rows.
     const restoredIdsRef = useRef>(new Set(initialMessages.map((m) => m.id)))
@@ -232,9 +249,11 @@ export const useAgentConversation = ({
 
     // Tracks `busy` for callbacks that outlive a render (the preserve verdict at unmount).
     const busyRef = useRef(false)
+    const messagesRef = useRef(initialMessages)
 
     const hooks: SessionChatHooks = {
         prepareRequest: async ({messages, id}) => {
+            clearSessionTurnId(sessionId)
             // Bounded, not instant. A null build means the workflow entity has not loaded its
             // invocation URL YET — the first send to a freshly created agent races that fetch, and
             // failing on the first null made a new user's first message fail (#6042 on the desktop;
@@ -266,7 +285,12 @@ export const useAgentConversation = ({
             const label = startupLabelFromDataPart(part)
             if (label) setTurnStartupLabel(sessionId, label)
         },
-        onFinish: ({message}) => {
+        onFinish: ({message, messages: finishedMessages, finishReason}) => {
+            dispatchStopped({
+                type: "stream-terminal",
+                messages: finishedMessages,
+                finishReason,
+            })
             markTraceAsFresh(getMessageTraceId(message))
             revalidateSessionMounts(sessionId)
             revalidateSessionRecords(sessionId)
@@ -288,13 +312,10 @@ export const useAgentConversation = ({
             }
         },
         onError: () => {
-            // Clear the marker but do NOT void the resume. A gateway approval is answered while the
-            // stream is still open, so the SDK skips its own dispatch and only re-evaluates when the
-            // stream ends — often by erroring, right here. `null` made that last evaluation return
-            // false and stranded the answer; `undefined` lets the tail heuristics decide.
-            // Adoption is unaffected: the hydration guard reads this ref as a boolean.
-            // The registry logs the error for the dev overlay (F-033) before calling this.
-            liveGateInteractionRef.current = undefined
+            // Preserve null after resume/Stop; only a live marker may fall back to tail detection.
+            if (liveGateInteractionRef.current !== null) {
+                liveGateInteractionRef.current = undefined
+            }
         },
     }
 
@@ -327,13 +348,21 @@ export const useAgentConversation = ({
     })
 
     const busy = isChatBusy(status)
-
     // `messages`/`busy` change every commit; consumers that must stay referentially stable
     // (`rewind`, the hydration/revalidation adoption guards) read them through refs instead.
-    const messagesRef = useRef(messages)
     messagesRef.current = messages
     busyRef.current = busy
 
+    useEffect(() => {
+        dispatchStopped({type: "transcript", messages})
+    }, [messages])
+
+    // Keep only the newest turn id observed from this session's live stream.
+    useEffect(() => {
+        const turnId = latestTurnId(messages)
+        if (turnId) setSessionTurnId(sessionId, turnId)
+    }, [messages, sessionId])
+
     // Hybrid history: localStorage holds the cached conversation; the durable content lives in
     // the backend record log. Cache-first — when this session opens with no locally-cached
     // messages (never ran here, or after a storage clear), hydrate once from the server and seed.
@@ -461,8 +490,8 @@ export const useAgentConversation = ({
             // A real send means this session has run — drop the never-run marker so a later
             // cache-cleared reopen hydrates from the server.
             clearSessionFresh(sessionId)
-            // Any actual send supersedes a prior user-stop, so clear the marker here (covers the
-            // queue-release path; the manual path also clears it in `send`).
+            clearSessionTurnId(sessionId)
+            // Any actual send supersedes a prior user-stop.
             setStopped(false)
             sendMessage(
                 item.fileParts && item.fileParts.length
@@ -693,15 +722,20 @@ export const useAgentConversation = ({
         void loadSessionMessages(sessionId, adoptServerTranscript).then(adoptServerTranscript)
     }, [adoptServerTranscript, sessionId])
 
+    // Fence a delayed approval release before the host's durable cancel request settles.
+    const voidPendingResume = useCallback(() => {
+        liveGateInteractionRef.current = null
+    }, [])
+
     // ── DT3 cancelled state: wrap stop() to mark the in-flight assistant turn ──
     const handleStop = useCallback(() => {
         const last = messagesRef.current[messagesRef.current.length - 1]
         if (last && last.role === "assistant") setStopped(true)
         // A stop voids the pending gate (same rule the queue applies), so the marker must go too —
         // otherwise it outlives the abandoned resume and blocks this mount's records adoption.
-        liveGateInteractionRef.current = null
+        voidPendingResume()
         stop()
-    }, [stop])
+    }, [stop, voidPendingResume])
 
     // ── D9 teardown: `useSessionChat` releases this mount's claim on the session's chat ──
     // No `stop()` here: a streaming run is preserved past the unmount on purpose (#5724), and
@@ -730,7 +764,7 @@ export const useAgentConversation = ({
                     files: encoded.rejections.map((r) => r.name),
                 })
             }
-            // Clear any prior "stopped" marker — it's resolved by asking again.
+            clearSessionTurnId(sessionId)
             setStopped(false)
             // One path: `submit` sends now or queues behind held messages via the release gate.
             submit({text: trimmed, fileParts})
@@ -742,10 +776,11 @@ export const useAgentConversation = ({
 
     const regenerateTurn = useCallback(
         (id: string) => {
+            clearSessionTurnId(sessionId)
             setStopped(false)
             regenerate({messageId: id}).catch(ignoreStreamRejection)
         },
-        [regenerate],
+        [regenerate, sessionId],
     )
 
     // Rewind scan: pure side-effect detection + a deferred `confirm()`. The skin owns the
@@ -769,12 +804,13 @@ export const useAgentConversation = ({
                     if (at < 0) return
                     setMessages(current.slice(0, at))
                 } else {
+                    clearSessionTurnId(sessionId)
                     regenerate({messageId: message.id}).catch(ignoreStreamRejection)
                 }
             }
             return {sideEffects, restoreText: isUser ? messageText(message) : undefined, confirm}
         },
-        [regenerate, setMessages],
+        [regenerate, sessionId, setMessages],
     )
 
     // Per-mount executed-identity cache — the desktop's per-message toolSignature memo,
@@ -800,6 +836,7 @@ export const useAgentConversation = ({
         error: parsedError,
         turns,
         send,
+        voidPendingResume,
         stop: handleStop,
         regenerate: regenerateTurn,
         rewind,
diff --git a/web/packages/agenta-chat/src/hooks/useComposerAttachments.ts b/web/packages/agenta-chat/src/hooks/useComposerAttachments.ts
index ad45639de59..1a722de212f 100644
--- a/web/packages/agenta-chat/src/hooks/useComposerAttachments.ts
+++ b/web/packages/agenta-chat/src/hooks/useComposerAttachments.ts
@@ -14,7 +14,8 @@ import {attachmentsBySession} from "../state/sessionEphemera"
 
 import {removeUploadFile, useAttachmentUploads} from "./useAttachmentUploads"
 
-type StagedFile = UploadFile
+export type ComposerAttachment = UploadFile
+type StagedFile = ComposerAttachment
 
 /** Convert settled upload-tray entries into reference `file` parts via the neutral builder. */
 export const stagedFilesToParts = (files: StagedFile[], sessionId: string) =>
@@ -289,12 +290,12 @@ export const useComposerAttachments = ({
      * than through `addFiles`, which would re-upload them as second attachments. Idempotent:
      * anything already back in the tray is left where it is.
      */
-    const restoreAttachments = (restored: StagedFile[]) => {
+    const restoreAttachments = useCallback((restored: StagedFile[]) => {
         setFiles((prev) => [
             ...restored.filter((file) => !prev.some((row) => row.uid === file.uid)),
             ...prev,
         ])
-    }
+    }, [])
 
     return {
         uploadsEnabled,
diff --git a/web/packages/agenta-chat/src/hooks/useSessionChat.ts b/web/packages/agenta-chat/src/hooks/useSessionChat.ts
index bc72d59cc69..be11818fba3 100644
--- a/web/packages/agenta-chat/src/hooks/useSessionChat.ts
+++ b/web/packages/agenta-chat/src/hooks/useSessionChat.ts
@@ -57,6 +57,7 @@ export const useSessionChat = ({
     // Publish + rebind AFTER commit, not during render. No dep array — the callbacks close over
     // every render's values, so a preserved chat never runs the closures of a stale render.
     useEffect(() => {
+        provisional.hooks = hooks
         const live = commitSessionChat(sessionId, provisional)
         if (live !== chat) rebind((n) => n + 1)
     })
diff --git a/web/packages/agenta-chat/src/model/approvals.ts b/web/packages/agenta-chat/src/model/approvals.ts
index 15b1d1c735d..bf308586c5b 100644
--- a/web/packages/agenta-chat/src/model/approvals.ts
+++ b/web/packages/agenta-chat/src/model/approvals.ts
@@ -64,3 +64,9 @@ export const getPendingApprovals = (messages: UIMessage[]): PendingApproval[] =>
     }
     return out
 }
+
+/** Stopped turns expose no actionable approval gates. */
+export const getLivePendingApprovals = (
+    messages: UIMessage[],
+    options?: {stopped?: boolean},
+): PendingApproval[] => (options?.stopped ? [] : getPendingApprovals(messages))
diff --git a/web/packages/agenta-chat/src/model/error.ts b/web/packages/agenta-chat/src/model/error.ts
index c3ea018aa15..61d4bb6ee1a 100644
--- a/web/packages/agenta-chat/src/model/error.ts
+++ b/web/packages/agenta-chat/src/model/error.ts
@@ -1,6 +1,7 @@
 export interface ParsedRunError {
     message: string
-    code?: number
+    /** An HTTP-ish status from a JSON error envelope, or a stable runner failure class string. */
+    code?: number | string
     /** The request never reached Agenta: no server verdict behind it, and retryable as-is. */
     transport?: boolean
 }
@@ -45,6 +46,17 @@ export const isTransportFailure = (raw: string): boolean => {
     return TRANSPORT_MESSAGES.includes(bare)
 }
 
+// Keep this refusal contract byte-identical to the runner message.
+export const SESSION_TURN_IN_USE_CODE = "session_turn_in_use"
+
+export const SESSION_TURN_IN_USE_MESSAGE =
+    "This session is already running a turn. Your message was not sent. Wait for the reply, or stop the turn, then send again."
+
+/** True when a `useChat` error is the single-turn admission refusal. */
+export const isSessionBusyRefusal = (err: unknown): boolean =>
+    parseAgentRunError(err).message.trim() === SESSION_TURN_IN_USE_MESSAGE
+
+// Keep byte parity with the desktop parser until its duplicate is removed.
 /**
  * Best-effort human reason from a useChat stream error: a plain string or a `{status:{…}}`
  * envelope. An engine's own wording is translated — "Failed to fetch" under "The agent run
@@ -72,8 +84,11 @@ export const parseAgentRunError = (err: unknown): ParsedRunError => {
     } catch {
         // raw isn't JSON — it's already the human message.
     }
-    // After the envelope: a server that reports those words means them, and its code is worth more
-    // than this translation. A bare engine string has no envelope to lose.
+    if (fallback.trim() === SESSION_TURN_IN_USE_MESSAGE) {
+        // Carry the class so the bubble can say "not sent" rather than "the agent run failed".
+        return {message: fallback, code: SESSION_TURN_IN_USE_CODE}
+    }
+    // A server envelope outranks transport-phrase translation.
     if (isTransportFailure(fallback)) return {message: TRANSPORT_ERROR_MESSAGE, transport: true}
     return {message: fallback}
 }
diff --git a/web/packages/agenta-chat/src/model/index.ts b/web/packages/agenta-chat/src/model/index.ts
index 6c521d22298..0aaedd26353 100644
--- a/web/packages/agenta-chat/src/model/index.ts
+++ b/web/packages/agenta-chat/src/model/index.ts
@@ -4,6 +4,7 @@ export * from "./parts"
 export * from "./error"
 export * from "./toolSummary"
 export * from "./approvals"
+export * from "./interactionAvailability"
 export * from "./approvalInputSummary"
 export * from "./approvalPreview"
 export * from "./turnStatus"
@@ -11,3 +12,4 @@ export * from "./renderModel"
 export * from "./grouping"
 export * from "./sessionStatus"
 export * from "./turnViewModel"
+export * from "./userStop"
diff --git a/web/packages/agenta-chat/src/model/interactionAvailability.ts b/web/packages/agenta-chat/src/model/interactionAvailability.ts
new file mode 100644
index 00000000000..9411897974d
--- /dev/null
+++ b/web/packages/agenta-chat/src/model/interactionAvailability.ts
@@ -0,0 +1,12 @@
+export const getInteractionAvailability = ({
+    stopped,
+    stopping,
+    streaming,
+}: {
+    stopped: boolean
+    stopping: boolean
+    streaming: boolean
+}) => {
+    const active = !stopped && !stopping
+    return {approvals: active, parkedDocks: active && !streaming}
+}
diff --git a/web/packages/agenta-chat/src/model/userStop.ts b/web/packages/agenta-chat/src/model/userStop.ts
new file mode 100644
index 00000000000..c5c42410397
--- /dev/null
+++ b/web/packages/agenta-chat/src/model/userStop.ts
@@ -0,0 +1,91 @@
+import type {UIMessage} from "ai"
+
+type MessageWithStopMetadata = UIMessage & {metadata?: {runStopped?: boolean}}
+type InteractionPart = UIMessage["parts"][number] & {state?: string}
+
+const hasPendingInteraction = (messages: UIMessage[]): boolean =>
+    messages.some(
+        (message) =>
+            message.role === "assistant" &&
+            message.parts.some((part) => {
+                const state = (part as InteractionPart).state
+                return (
+                    state === "approval-requested" ||
+                    state === "input-available" ||
+                    state === "input-streaming"
+                )
+            }),
+    )
+
+/** True when the durable Stop marker still owns the session's current turn. */
+export const isSessionTurnStopping = ({
+    currentTurnId,
+    stoppingTurnId,
+}: {
+    currentTurnId?: string | null
+    stoppingTurnId?: string | null
+}): boolean => Boolean(currentTurnId && stoppingTurnId === currentTurnId)
+
+/** True only for the durable marker written on a user-cancelled assistant turn. */
+export const lastTurnWasUserStopped = (messages: UIMessage[]): boolean => {
+    const last = messages[messages.length - 1] as MessageWithStopMetadata | undefined
+    return last?.role === "assistant" && last.metadata?.runStopped === true
+}
+
+export type UserStoppedStateEvent =
+    | {type: "user-stop"}
+    | {type: "reset"}
+    | {type: "transcript"; messages: UIMessage[]}
+    | {
+          type: "stream-terminal"
+          messages: UIMessage[]
+          finishReason?: string
+      }
+
+export interface UserStoppedState {
+    stopped: boolean
+    turnIdentity: string | null
+}
+
+const lastTurnIdentity = (messages: UIMessage[]): string | null => {
+    const last = messages[messages.length - 1]
+    if (!last) return null
+    const turnId = (last.metadata as {turnId?: unknown} | undefined)?.turnId
+    if (typeof turnId === "string" && turnId.trim()) return `turn:${turnId}`
+    return `message:${messages.length}:${last.role}:${last.id}`
+}
+
+export const createUserStoppedState = (messages: UIMessage[]): UserStoppedState => ({
+    stopped: lastTurnWasUserStopped(messages),
+    turnIdentity: lastTurnIdentity(messages),
+})
+
+const adoptTranscript = (state: UserStoppedState, messages: UIMessage[]): UserStoppedState => {
+    const turnIdentity = lastTurnIdentity(messages)
+    if (lastTurnWasUserStopped(messages)) return {stopped: true, turnIdentity}
+    const adoptedNewerTurn =
+        state.stopped && state.turnIdentity !== null && turnIdentity !== state.turnIdentity
+    return {stopped: adoptedNewerTurn ? false : state.stopped, turnIdentity}
+}
+
+/** A pending interaction distinguishes a paused `other` finish from cancellation. */
+export const reduceUserStoppedState = (
+    state: UserStoppedState,
+    event: UserStoppedStateEvent,
+): UserStoppedState => {
+    switch (event.type) {
+        case "user-stop":
+            return {...state, stopped: true}
+        case "reset":
+            return {...state, stopped: false}
+        case "transcript":
+            return adoptTranscript(state, event.messages)
+        case "stream-terminal": {
+            const adopted = adoptTranscript(state, event.messages)
+            if (lastTurnWasUserStopped(event.messages)) return adopted
+            if (event.finishReason === "other" && !hasPendingInteraction(event.messages))
+                return {...adopted, stopped: true}
+            return adopted
+        }
+    }
+}
diff --git a/web/packages/agenta-chat/src/state/sessionEphemera.ts b/web/packages/agenta-chat/src/state/sessionEphemera.ts
index ed97d4add0f..3da821179ef 100644
--- a/web/packages/agenta-chat/src/state/sessionEphemera.ts
+++ b/web/packages/agenta-chat/src/state/sessionEphemera.ts
@@ -12,16 +12,7 @@ import {freshSessionIds} from "@agenta/entities/session"
 
 import type {StagedUpload} from "../model"
 
-/**
- * Per-session in-memory ephemera that must survive pane remounts (route re-entry, tab
- * close/reopen) but NOT a session's deletion. Lives outside React and outside the
- * persisted session atoms:
- * - composer drafts/attachments hold live `File` blobs that can't be serialized.
- *
- * `deleteSessionAtomFamily` / `resetScopeAtomFamily` call `clearSessionEphemera` alongside
- * their `sessionMessagesAtom` cleanup, so deleted sessions don't retain blobs for the rest
- * of the page lifetime.
- */
+/** Per-session memory survives pane remounts but is cleared on permanent deletion. */
 
 /** Unsent composer drafts per session — switching back to a session restores its
  * in-progress message. */
@@ -30,6 +21,29 @@ export const composerDraftBySession = new Map()
 /** Pending (not yet sent) attachments per session — same lifetime as the drafts. */
 export const attachmentsBySession = new Map[]>()
 
+/** In-memory turn guards are never restored across page loads. */
+export const turnIdBySession = new Map()
+const supersededTurnIdsBySession = new Map>()
+
+export const setSessionTurnId = (sessionId: string, turnId: string) => {
+    if (supersededTurnIdsBySession.get(sessionId)?.has(turnId)) return
+    turnIdBySession.set(sessionId, turnId)
+}
+
+export const getSessionTurnId = (sessionId: string): string | undefined =>
+    turnIdBySession.get(sessionId)
+
+/** Clear the old guard before starting a replacement turn. */
+export const clearSessionTurnId = (sessionId: string) => {
+    const current = turnIdBySession.get(sessionId)
+    if (current) {
+        const superseded = supersededTurnIdsBySession.get(sessionId) ?? new Set()
+        superseded.add(current)
+        supersededTurnIdsBySession.set(sessionId, superseded)
+    }
+    turnIdBySession.delete(sessionId)
+}
+
 // The fresh-session registry moved to @agenta/entities/session — the drive needs the same
 // predicate, and this package sits ABOVE entity-ui so it cannot be imported from there.
 export {freshSessionIds}
@@ -39,5 +53,7 @@ export {clearSessionFresh, isSessionFresh, markSessionFresh} from "@agenta/entit
 export const clearSessionEphemera = (sessionId: string) => {
     composerDraftBySession.delete(sessionId)
     attachmentsBySession.delete(sessionId)
+    turnIdBySession.delete(sessionId)
+    supersededTurnIdsBySession.delete(sessionId)
     freshSessionIds.delete(sessionId)
 }
diff --git a/web/packages/agenta-chat/tests/unit/assets/agentTurn.test.ts b/web/packages/agenta-chat/tests/unit/assets/agentTurn.test.ts
new file mode 100644
index 00000000000..c1981e033d1
--- /dev/null
+++ b/web/packages/agenta-chat/tests/unit/assets/agentTurn.test.ts
@@ -0,0 +1,84 @@
+import type {UIMessage} from "ai"
+import {afterEach, describe, expect, it} from "vitest"
+
+import {getMessageTurnId, latestTurnId} from "../../../src/assets/agentTurn"
+import {
+    clearSessionEphemera,
+    clearSessionTurnId,
+    getSessionTurnId,
+    setSessionTurnId,
+} from "../../../src/state/sessionEphemera"
+
+const assistant = (id: string, metadata?: unknown): UIMessage =>
+    ({id, role: "assistant", parts: [], metadata}) as UIMessage
+
+const user = (id: string): UIMessage => ({id, role: "user", parts: []}) as UIMessage
+
+afterEach(() => {
+    clearSessionEphemera("s1")
+    clearSessionEphemera("s2")
+})
+
+describe("getMessageTurnId", () => {
+    it("reads the runner-minted id from message metadata", () => {
+        expect(getMessageTurnId(assistant("a1", {turnId: "turn-1"}))).toBe("turn-1")
+    })
+
+    it("rejects missing and malformed ids", () => {
+        expect(getMessageTurnId(assistant("a1"))).toBeNull()
+        expect(getMessageTurnId(assistant("a2", {turnId: "   "}))).toBeNull()
+        expect(getMessageTurnId(assistant("a3", {turnId: 7}))).toBeNull()
+        expect(getMessageTurnId(undefined)).toBeNull()
+    })
+})
+
+describe("latestTurnId", () => {
+    it("reads only the newest assistant turn", () => {
+        expect(
+            latestTurnId([
+                user("u1"),
+                assistant("a1", {turnId: "turn-1"}),
+                user("u2"),
+                assistant("a2", {turnId: "turn-2"}),
+            ]),
+        ).toBe("turn-2")
+    })
+
+    it("does not fall back when the newest assistant has no id", () => {
+        expect(latestTurnId([assistant("a1", {turnId: "turn-1"}), assistant("a2")])).toBeNull()
+    })
+
+    it("does not cross a trailing user message into an older turn", () => {
+        expect(latestTurnId([assistant("a1", {turnId: "turn-A"}), user("u2")])).toBeNull()
+    })
+})
+
+describe("session turn ids", () => {
+    it("survives a pane remount and is replaced by the next admitted turn", () => {
+        setSessionTurnId("s1", "turn-A")
+        expect(getSessionTurnId("s1")).toBe("turn-A")
+
+        setSessionTurnId("s1", "turn-B")
+        expect(getSessionTurnId("s1")).toBe("turn-B")
+    })
+
+    it("is isolated per session and cleared with session ephemera", () => {
+        setSessionTurnId("s1", "turn-1")
+        setSessionTurnId("s2", "turn-2")
+
+        clearSessionTurnId("s1")
+        expect(getSessionTurnId("s1")).toBeUndefined()
+        expect(getSessionTurnId("s2")).toBe("turn-2")
+
+        clearSessionEphemera("s2")
+        expect(getSessionTurnId("s2")).toBeUndefined()
+    })
+
+    it("can be cleared before a replacement turn starts", () => {
+        setSessionTurnId("s4", "turn-old")
+
+        clearSessionTurnId("s4")
+
+        expect(getSessionTurnId("s4")).toBeUndefined()
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/assets/composerState.test.ts b/web/packages/agenta-chat/tests/unit/assets/composerState.test.ts
new file mode 100644
index 00000000000..52a07fc2c74
--- /dev/null
+++ b/web/packages/agenta-chat/tests/unit/assets/composerState.test.ts
@@ -0,0 +1,14 @@
+import {describe, expect, it} from "vitest"
+
+import {shouldShowStopControl} from "../../../src/assets/composerState"
+
+describe("shouldShowStopControl", () => {
+    it.each([
+        [{busy: true, hitlPending: false}, true],
+        [{busy: false, hitlPending: true}, true],
+        [{busy: true, hitlPending: true}, true],
+        [{busy: false, hitlPending: false}, false],
+    ])("returns %s for %o", (state, expected) => {
+        expect(shouldShowStopControl(state)).toBe(expected)
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/assets/resolveStopExecution.test.ts b/web/packages/agenta-chat/tests/unit/assets/resolveStopExecution.test.ts
new file mode 100644
index 00000000000..8f83dedb407
--- /dev/null
+++ b/web/packages/agenta-chat/tests/unit/assets/resolveStopExecution.test.ts
@@ -0,0 +1,81 @@
+import {describe, expect, it, vi} from "vitest"
+
+import {resolveStopExecution} from "../../../src/assets/resolveStopExecution"
+
+const deferred = () => {
+    let resolve!: () => void
+    const promise = new Promise((done) => {
+        resolve = done
+    })
+    return {promise, resolve}
+}
+
+describe("resolveStopExecution", () => {
+    it("waits for the new execution instead of selecting an unnamed Stop", async () => {
+        let executionId: string | undefined
+        const tick = deferred()
+        const resolving = resolveStopExecution({
+            readExecutionId: () => executionId,
+            isRunActive: () => true,
+            wait: vi.fn(() => tick.promise),
+        })
+
+        executionId = "turn-resumed"
+        tick.resolve()
+
+        await expect(resolving).resolves.toEqual({
+            status: "resolved",
+            executionId: "turn-resumed",
+        })
+    })
+
+    it("does not send a Stop after the run settles while its id is unresolved", async () => {
+        let active = true
+        const tick = deferred()
+        const resolving = resolveStopExecution({
+            readExecutionId: () => undefined,
+            isRunActive: () => active,
+            wait: () => tick.promise,
+        })
+
+        active = false
+        tick.resolve()
+
+        await expect(resolving).resolves.toEqual({status: "settled"})
+    })
+
+    it("can be abandoned when the owning mount leaves", async () => {
+        const controller = new AbortController()
+        const tick = deferred()
+        const resolving = resolveStopExecution({
+            readExecutionId: () => undefined,
+            isRunActive: () => true,
+            signal: controller.signal,
+            wait: () => tick.promise,
+        })
+
+        controller.abort()
+        tick.resolve()
+
+        await expect(resolving).resolves.toEqual({status: "aborted"})
+    })
+
+    it("stops waiting at the deadline while the run remains active", async () => {
+        let elapsed = 0
+        const wait = vi.fn(async (ms: number) => {
+            elapsed += ms
+        })
+
+        await expect(
+            resolveStopExecution({
+                readExecutionId: () => undefined,
+                isRunActive: () => true,
+                timeoutMs: 50,
+                pollMs: 25,
+                now: () => elapsed,
+                wait,
+            }),
+        ).resolves.toEqual({status: "timed_out"})
+        expect(wait).toHaveBeenCalledTimes(2)
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/assets/transcriptToMessages.test.ts b/web/packages/agenta-chat/tests/unit/assets/transcriptToMessages.test.ts
index 18862737da9..28e76ebbc56 100644
--- a/web/packages/agenta-chat/tests/unit/assets/transcriptToMessages.test.ts
+++ b/web/packages/agenta-chat/tests/unit/assets/transcriptToMessages.test.ts
@@ -69,6 +69,61 @@ describe("transcriptToMessages", () => {
         expect(transcriptToMessages([record("r1", {type: "done"})])).toBeNull()
     })
 
+    it("preserves a cancelled terminal as a neutral stopped turn", () => {
+        const messages = transcriptToMessages([
+            record("r1", {type: "message", text: "partial answer"}),
+            record("r2", {type: "done", stopReason: "cancelled"}),
+        ])
+
+        expect(messages).toHaveLength(1)
+        expect(messages?.[0]).toMatchObject({
+            role: "assistant",
+            parts: [{type: "text", text: "partial answer"}],
+            metadata: {runStopped: true},
+        })
+    })
+
+    it("keeps a stopped carrier when cancellation lands before any content", () => {
+        const messages = transcriptToMessages([
+            record("r1", {type: "done", stopReason: "cancelled"}),
+        ])
+
+        expect(messages).toEqual([
+            expect.objectContaining({
+                id: "r1",
+                role: "assistant",
+                parts: [],
+                metadata: {runStopped: true},
+            }),
+        ])
+    })
+
+    it("suppresses an abort error when the same durable turn is explicitly user-stopped", () => {
+        const messages = transcriptToMessages([
+            record("r1", {type: "error", message: "Request was aborted"}),
+            record("r2", {type: "done", stopReason: "cancelled"}),
+        ])
+
+        expect(messages?.[0].metadata).toEqual({runStopped: true})
+    })
+
+    it("settles a paused approval as cancelled without interaction row state", () => {
+        const messages = transcriptToMessages([
+            record("r-call", {type: "tool_call", id: "tool-1", name: "bash", input: {}}),
+            record("r-request", {
+                type: "interaction_request",
+                id: "approval-1",
+                kind: "user_approval",
+                payload: {toolCallId: "tool-1"},
+            }),
+            record("r-paused", {type: "done", stopReason: "paused"}),
+            record("r-cancelled", {type: "done", stopReason: "cancelled"}),
+        ])
+
+        expect(messages?.[0]).toMatchObject({metadata: {runStopped: true}})
+        expect(messages?.[0].parts[0]).toMatchObject({state: "output-denied"})
+    })
+
     it("splits assistant turns on a `done` boundary into separate messages", () => {
         const messages = transcriptToMessages([
             record("r1", {type: "message", text: "first turn"}),
@@ -1056,3 +1111,34 @@ describe("transcriptToMessages run-error code", () => {
         expect(runErrorOf({type: "error", message: "boom"})).toEqual({message: "boom"})
     })
 })
+
+describe("transcriptToMessages user-Stop terminal record", () => {
+    // A cancelled terminal closes its turn without swallowing the next one.
+    it("closes a stopped turn like a completed one", () => {
+        const messages = transcriptToMessages([
+            record("r-user", {type: "message", text: "run something long"}, "user"),
+            record("r-msg", {type: "message", text: "starting"}),
+            record("r-done-cancelled", {type: "done", stopReason: "cancelled"}),
+            record("r-user-2", {type: "message", text: "what was the codeword"}, "user"),
+            record("r-msg-2", {type: "message", text: "MANGO"}),
+            record("r-done", {type: "done"}),
+        ])
+
+        // Four bubbles: a stopped turn must not swallow the next one the way a pause does.
+        expect(messages).toHaveLength(4)
+        expect(messages![1].parts).toMatchObject([{type: "text", text: "starting"}])
+        expect(messages![3].parts).toMatchObject([{type: "text", text: "MANGO"}])
+    })
+
+    it("does not mark a stopped turn as paused", () => {
+        const messages = transcriptToMessages([
+            record("r-user", {type: "message", text: "run something long"}, "user"),
+            record("r-msg", {type: "message", text: "starting"}),
+            record("r-done-cancelled", {type: "done", stopReason: "cancelled"}),
+        ])
+
+        expect(
+            (messages![1] as unknown as {metadata?: {paused?: boolean}}).metadata?.paused,
+        ).toBeFalsy()
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/hooks/useAgentChatQueue.test.ts b/web/packages/agenta-chat/tests/unit/hooks/useAgentChatQueue.test.ts
index a12198bddec..41b8aa11169 100644
--- a/web/packages/agenta-chat/tests/unit/hooks/useAgentChatQueue.test.ts
+++ b/web/packages/agenta-chat/tests/unit/hooks/useAgentChatQueue.test.ts
@@ -362,3 +362,97 @@ describe("useAgentChatQueue", () => {
         expect(result.current.queued.map((m) => m.text)).toEqual(["two"])
     })
 })
+
+describe("useAgentChatQueue: reclaiming a sent message", () => {
+    // The host reclaims an immediate send only until the runner confirms admission.
+
+    it("hands back the message that was sent immediately", () => {
+        const {result} = setup(settledEmpty)
+        act(() => {
+            result.current.submit({text: "the refused message"})
+        })
+        expect(result.current.takeLastSent()).toMatchObject({text: "the refused message"})
+    })
+
+    it("hands it back only ONCE, so a re-render cannot re-fill the composer", () => {
+        const {result} = setup(settledEmpty)
+        act(() => {
+            result.current.submit({text: "once"})
+        })
+        expect(result.current.takeLastSent()?.text).toBe("once")
+        expect(result.current.takeLastSent()).toBeUndefined()
+    })
+
+    it("keeps an attachment-only refused send recoverable", () => {
+        const {result} = setup(settledEmpty)
+        const stagedFiles = [{uid: "file-1", name: "brief.pdf", status: "done"}] as never
+        act(() => {
+            result.current.submit({text: "", stagedFiles})
+        })
+
+        expect(result.current.takeLastSent()).toMatchObject({text: "", stagedFiles})
+    })
+
+    it("retains a refused send when the composer cannot place it", () => {
+        const {result} = setup(settledEmpty)
+        const stagedFiles = [{uid: "file-1", name: "brief.pdf", status: "done"}] as never
+        act(() => {
+            result.current.submit({text: "refused message", stagedFiles})
+        })
+
+        expect(result.current.takeLastSent(() => false)).toBeUndefined()
+        expect(result.current.takeLastSent()).toMatchObject({
+            text: "refused message",
+            stagedFiles,
+        })
+    })
+
+    it("does not clear recovery when dispatch only changes the stream status", () => {
+        const {result, rerender} = setup(settledEmpty)
+        act(() => {
+            result.current.submit({text: "sent"})
+        })
+        rerender({status: "streaming", messages: [userTurn("u1", "sent")], stopped: false})
+        expect(result.current.takeLastSent()?.text).toBe("sent")
+    })
+
+    it("clears recovery after a runner turn id confirms admission", () => {
+        const {result, rerender} = setup(settledEmpty)
+        act(() => {
+            result.current.submit({text: "admitted"})
+        })
+        rerender({
+            status: "streaming",
+            messages: [
+                userTurn("u2", "admitted"),
+                {...assistantText("a2", ""), metadata: {turnId: "turn-2"}},
+            ],
+            stopped: false,
+        })
+        expect(result.current.takeLastSent()).toBeUndefined()
+    })
+
+    it("has nothing to hand back for a message that only QUEUED", () => {
+        // A queued message is already safe: it is rendered by the dock and mirrored per session.
+        const {result, sendQueued} = setup({status: "streaming", messages: [], stopped: false})
+        act(() => {
+            result.current.submit({text: "queued, not sent"})
+        })
+        expect(sendQueued).not.toHaveBeenCalled()
+        expect(result.current.queued).toHaveLength(1)
+        expect(result.current.takeLastSent()).toBeUndefined()
+    })
+
+    it("tracks the released queue head too, which the release removed from the queue", () => {
+        const {result, rerender} = setup({status: "streaming", messages: [], stopped: false})
+        act(() => {
+            result.current.submit({text: "held"})
+        })
+        expect(result.current.queued).toHaveLength(1)
+        act(() => {
+            rerender({status: "ready", messages: [], stopped: false})
+        })
+        expect(result.current.queued).toHaveLength(0)
+        expect(result.current.takeLastSent()).toMatchObject({text: "held"})
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/hooks/useAgentConversation.test.ts b/web/packages/agenta-chat/tests/unit/hooks/useAgentConversation.test.ts
index 40201f450ae..16effd32333 100644
--- a/web/packages/agenta-chat/tests/unit/hooks/useAgentConversation.test.ts
+++ b/web/packages/agenta-chat/tests/unit/hooks/useAgentConversation.test.ts
@@ -14,6 +14,11 @@ import type {UIMessage} from "ai"
 import {createStore, Provider} from "jotai"
 import {beforeEach, describe, expect, it, vi} from "vitest"
 
+const approvalRecord = vi.hoisted(() => ({
+    defer: false,
+    resolve: undefined as (() => void) | undefined,
+}))
+
 vi.mock("@agenta/playground/agent-chat", async (importOriginal) => {
     const actual = await importOriginal()
     return {
@@ -37,6 +42,12 @@ vi.mock("@agenta/entities/session", async (importOriginal) => {
         ...actual,
         revalidateSessionMountsAtom: atom(null, () => {}),
         revalidateSessionRecordsAtom: atom(null, () => {}),
+        recordInteractionAnswerAtom: atom(null, async () => {
+            if (!approvalRecord.defer) return
+            await new Promise((resolve) => {
+                approvalRecord.resolve = resolve
+            })
+        }),
         // The hydration seam's records fetch: "no server history" for these tests.
         fetchSessionRecordsAtom: atom(null, () => ({records: null, refreshed: null})),
         fetchSessionInteractionStatesAtom: atom(null, () => new Map()),
@@ -48,10 +59,14 @@ vi.mock("@agenta/entities/trace", () => ({
 }))
 
 import {useAgentConversation} from "../../../src/hooks/useAgentConversation"
-import {markSessionFresh} from "../../../src/state/sessionEphemera"
+import {
+    getSessionTurnId,
+    markSessionFresh,
+    setSessionTurnId,
+} from "../../../src/state/sessionEphemera"
 import {sessionMessagesAtom, sessionStatusAtomFamily} from "../../../src/state/sessionMessages"
 
-const sseBody = (text: string): string => {
+const sseBody = (text: string, finishReason?: string): string => {
     const chunks = [
         {type: "start", messageId: `assist-${Math.random().toString(36).slice(2)}`},
         {type: "start-step"},
@@ -59,7 +74,7 @@ const sseBody = (text: string): string => {
         {type: "text-delta", id: "t1", delta: text},
         {type: "text-end", id: "t1"},
         {type: "finish-step"},
-        {type: "finish"},
+        {type: "finish", ...(finishReason ? {finishReason} : {})},
     ]
     return chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`).join("") + "data: [DONE]\n\n"
 }
@@ -70,6 +85,22 @@ const streamResponse = (text: string): Response =>
         headers: {"content-type": "text/event-stream"},
     })
 
+const approvalResponse = (): Response => {
+    const chunks = [
+        {type: "start", messageId: "approval-assistant"},
+        {type: "start-step"},
+        {type: "tool-input-start", toolCallId: "call-1", toolName: "shell"},
+        {type: "tool-input-available", toolCallId: "call-1", toolName: "shell", input: {}},
+        {type: "tool-approval-request", approvalId: "approval-1", toolCallId: "call-1"},
+        {type: "finish-step"},
+        {type: "finish", finishReason: "tool-calls"},
+    ]
+    return new Response(
+        chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n",
+        {status: 200, headers: {"content-type": "text/event-stream"}},
+    )
+}
+
 const errorResponse = (): Response =>
     new Response(JSON.stringify({status: {code: 500, message: "boom"}}), {
         status: 500,
@@ -93,6 +124,8 @@ const mount = (store: ReturnType, entityId: string, sessionI
     )
 
 beforeEach(() => {
+    approvalRecord.defer = false
+    approvalRecord.resolve = undefined
     fetchMock.mockReset()
     vi.mocked(buildAgentRequest).mockClear()
     // Restore the ready-workflow build: one test replaces it with a not-yet-loaded one, and
@@ -152,6 +185,74 @@ describe("useAgentConversation", () => {
         expect(result.current.isEmpty).toBe(false)
     })
 
+    it("clears the previous execution guard before a second send", async () => {
+        fetchMock.mockImplementation(async () => streamResponse("answer"))
+        const store = createStore()
+        const sessionId = nextSessionId()
+        markSessionFresh(sessionId)
+        const {result} = mount(store, "rev-1", sessionId)
+
+        await act(async () => {
+            await result.current.send({text: "first"})
+        })
+        await waitFor(() => expect(result.current.status).toBe("ready"), {timeout: 5000})
+        setSessionTurnId(sessionId, "turn-old")
+
+        await act(async () => {
+            await result.current.send({text: "second"})
+        })
+
+        expect(getSessionTurnId(sessionId)).toBeUndefined()
+    })
+
+    it("clears the parked turn guard when an approval automatically resumes", async () => {
+        fetchMock
+            .mockResolvedValueOnce(approvalResponse())
+            .mockResolvedValueOnce(streamResponse("done"))
+        const store = createStore()
+        const sessionId = nextSessionId()
+        markSessionFresh(sessionId)
+        const {result} = mount(store, "rev-1", sessionId)
+
+        await act(async () => {
+            await result.current.send({text: "needs approval"})
+        })
+        await waitFor(() => expect(result.current.approvals.open).toBe(true), {timeout: 5000})
+        setSessionTurnId(sessionId, "parked-turn")
+
+        act(() => result.current.approvals.respond(true))
+
+        await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2), {timeout: 5000})
+        expect(getSessionTurnId(sessionId)).toBeUndefined()
+    })
+
+    it("voids an approval resume before its delayed interaction write releases", async () => {
+        approvalRecord.defer = true
+        fetchMock
+            .mockResolvedValueOnce(approvalResponse())
+            .mockResolvedValueOnce(streamResponse("unexpected resume"))
+        const store = createStore()
+        const sessionId = nextSessionId()
+        markSessionFresh(sessionId)
+        const {result} = mount(store, "rev-1", sessionId)
+
+        await act(async () => {
+            await result.current.send({text: "needs approval"})
+        })
+        await waitFor(() => expect(result.current.approvals.open).toBe(true), {timeout: 5000})
+
+        act(() => result.current.approvals.respond(true))
+        await waitFor(() => expect(approvalRecord.resolve).toBeTypeOf("function"), {timeout: 5000})
+        act(() => result.current.voidPendingResume())
+        await act(async () => {
+            approvalRecord.resolve?.()
+            await Promise.resolve()
+        })
+        await new Promise((resolve) => setTimeout(resolve, 100))
+
+        expect(fetchMock).toHaveBeenCalledTimes(1)
+    })
+
     it("survives a revision switch mid-stream instead of aborting the turn", async () => {
         // Auto-commit (#6126) mints a new revision while the agent is running, and the surface
         // follows it. If that arrives as a REMOUNT the unmount teardown calls stop() and kills the
@@ -352,4 +453,26 @@ describe("useAgentConversation", () => {
             expect(last.status.isError).toBe(true)
         })
     })
+
+    it("maps a stream-delivered user Stop to the neutral stopped state", async () => {
+        fetchMock.mockResolvedValue(
+            new Response(sseBody("partial answer", "other"), {
+                status: 200,
+                headers: {"content-type": "text/event-stream"},
+            }),
+        )
+        const store = createStore()
+        const sessionId = nextSessionId()
+        markSessionFresh(sessionId)
+        const {result} = mount(store, "rev-1", sessionId)
+
+        await act(async () => {
+            await result.current.send({text: "start"})
+        })
+        await waitFor(() => expect(result.current.status).toBe("ready"), {timeout: 5000})
+
+        expect(result.current.stopped).toBe(true)
+        expect(result.current.error).toBeUndefined()
+        expect(result.current.runStatus).toBe("idle")
+    })
 })
diff --git a/web/packages/agenta-chat/tests/unit/hooks/useSessionChat.test.ts b/web/packages/agenta-chat/tests/unit/hooks/useSessionChat.test.ts
new file mode 100644
index 00000000000..304912f586d
--- /dev/null
+++ b/web/packages/agenta-chat/tests/unit/hooks/useSessionChat.test.ts
@@ -0,0 +1,94 @@
+import {act, renderHook} from "@testing-library/react"
+import {beforeEach, describe, expect, it, vi} from "vitest"
+
+import {useSessionChat} from "../../../src/hooks/useSessionChat"
+import {
+    __resetSessionChatsForTest,
+    peekSessionChat,
+    type SessionChatHooks,
+} from "../../../src/state/sessionChats"
+
+vi.mock("../../../src/transport/AgentChatTransport", () => ({
+    AgentChatTransport: class {
+        constructor(
+            public init: {
+                prepareSendMessagesRequest: (args: unknown) => Promise
+            },
+        ) {}
+    },
+}))
+
+vi.mock("@ai-sdk/react", () => ({
+    Chat: class {
+        status = "ready"
+        stop = vi.fn().mockResolvedValue(undefined)
+        constructor(
+            public init: {
+                transport: {
+                    init: {
+                        prepareSendMessagesRequest: (args: unknown) => Promise
+                    }
+                }
+                onFinish: (event: unknown) => void
+                onData: (part: unknown) => void
+            },
+        ) {}
+    },
+}))
+
+const hooks = (label: string): SessionChatHooks => ({
+    prepareRequest: vi.fn().mockResolvedValue({label}),
+    sendAutomaticallyWhen: vi.fn(() => false),
+    onFinish: vi.fn(),
+    onError: vi.fn(),
+    onData: vi.fn(),
+})
+
+interface FakeChat {
+    init: {
+        transport: {
+            init: {
+                prepareSendMessagesRequest: (args: unknown) => Promise
+            }
+        }
+        onFinish: (event: unknown) => void
+        onData: (part: unknown) => void
+    }
+}
+
+beforeEach(() => {
+    __resetSessionChatsForTest()
+})
+
+describe("useSessionChat", () => {
+    it("keeps the chat instance while rebinding every callback to the latest render", async () => {
+        const initialHooks = hooks("ephemeral")
+        const currentHooks = hooks("committed")
+        const view = renderHook(
+            ({sessionHooks}) =>
+                useSessionChat({
+                    sessionId: "session-1",
+                    initialMessages: [],
+                    hooks: sessionHooks,
+                    shouldPreserve: () => true,
+                }),
+            {initialProps: {sessionHooks: initialHooks}},
+        )
+        const chat = peekSessionChat("session-1") as unknown as FakeChat
+
+        view.rerender({sessionHooks: currentHooks})
+
+        expect(peekSessionChat("session-1")).toBe(chat)
+        await act(async () => {
+            await chat.init.transport.init.prepareSendMessagesRequest({messages: []})
+            chat.init.onData({type: "data-status"})
+            chat.init.onFinish({message: {id: "message-1"}})
+        })
+        expect(currentHooks.prepareRequest).toHaveBeenCalledOnce()
+        expect(currentHooks.onData).toHaveBeenCalledOnce()
+        expect(currentHooks.onFinish).toHaveBeenCalledOnce()
+        expect(initialHooks.prepareRequest).not.toHaveBeenCalled()
+        expect(initialHooks.onData).not.toHaveBeenCalled()
+        expect(initialHooks.onFinish).not.toHaveBeenCalled()
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/model/error.test.ts b/web/packages/agenta-chat/tests/unit/model/error.test.ts
index 71f2c0e1237..d1df74f894b 100644
--- a/web/packages/agenta-chat/tests/unit/model/error.test.ts
+++ b/web/packages/agenta-chat/tests/unit/model/error.test.ts
@@ -2,7 +2,10 @@ import {describe, expect, it} from "vitest"
 
 import {
     isTransportFailure,
+    isSessionBusyRefusal,
     parseAgentRunError,
+    SESSION_TURN_IN_USE_CODE,
+    SESSION_TURN_IN_USE_MESSAGE,
     TRANSPORT_ERROR_MESSAGE,
 } from "../../../src/model/error"
 
@@ -79,3 +82,29 @@ describe("parseAgentRunError", () => {
         expect(isTransportFailure("The agent run failed.")).toBe(false)
     })
 })
+
+describe("single-turn admission refusal", () => {
+    // The runner refusal message is the browser recovery contract.
+
+    it("recognises the refusal and carries its stable class", () => {
+        expect(parseAgentRunError(new Error(SESSION_TURN_IN_USE_MESSAGE))).toEqual({
+            message: SESSION_TURN_IN_USE_MESSAGE,
+            code: SESSION_TURN_IN_USE_CODE,
+        })
+        expect(isSessionBusyRefusal(new Error(SESSION_TURN_IN_USE_MESSAGE))).toBe(true)
+    })
+
+    it("recognises it through surrounding whitespace, as the wire may add", () => {
+        expect(isSessionBusyRefusal(`  ${SESSION_TURN_IN_USE_MESSAGE}\n`)).toBe(true)
+    })
+
+    it("does NOT claim an ordinary run failure, which must keep the failure bubble", () => {
+        expect(isSessionBusyRefusal(new Error("The model provider timed out."))).toBe(false)
+        expect(isSessionBusyRefusal(undefined)).toBe(false)
+        expect(parseAgentRunError("The model provider timed out.").code).toBeUndefined()
+    })
+
+    it("keeps the message one line, or the SDK truncates it at the first newline", () => {
+        expect(SESSION_TURN_IN_USE_MESSAGE).not.toContain("\n")
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/model/interactionAvailability.test.ts b/web/packages/agenta-chat/tests/unit/model/interactionAvailability.test.ts
new file mode 100644
index 00000000000..85ed95d5277
--- /dev/null
+++ b/web/packages/agenta-chat/tests/unit/model/interactionAvailability.test.ts
@@ -0,0 +1,23 @@
+import {describe, expect, it} from "vitest"
+
+import {getInteractionAvailability} from "../../../src/model/interactionAvailability"
+
+describe("interaction availability during Stop", () => {
+    it("disables approval and parked interaction actions as soon as Stop starts", () => {
+        expect(
+            getInteractionAvailability({stopped: false, stopping: true, streaming: false}),
+        ).toEqual({approvals: false, parkedDocks: false})
+    })
+
+    it("restores parked interaction actions after a failed Stop", () => {
+        expect(
+            getInteractionAvailability({stopped: false, stopping: false, streaming: false}),
+        ).toEqual({approvals: true, parkedDocks: true})
+    })
+
+    it("keeps parked docks closed while a turn streams", () => {
+        expect(
+            getInteractionAvailability({stopped: false, stopping: false, streaming: true}),
+        ).toEqual({approvals: true, parkedDocks: false})
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/model/liveApprovals.test.ts b/web/packages/agenta-chat/tests/unit/model/liveApprovals.test.ts
new file mode 100644
index 00000000000..039326d7ee8
--- /dev/null
+++ b/web/packages/agenta-chat/tests/unit/model/liveApprovals.test.ts
@@ -0,0 +1,26 @@
+import type {UIMessage} from "ai"
+import {describe, expect, it} from "vitest"
+
+import {getLivePendingApprovals, getPendingApprovals} from "../../../src/model/approvals"
+import approvalTurnFixture from "../fixtures/approvalTurn.json"
+
+const messages = approvalTurnFixture as UIMessage[]
+
+describe("getLivePendingApprovals", () => {
+    it("returns the pending gates while the turn is live", () => {
+        expect(getLivePendingApprovals(messages)).toEqual(getPendingApprovals(messages))
+        expect(getLivePendingApprovals(messages, {stopped: false})).toEqual(
+            getPendingApprovals(messages),
+        )
+        expect(getLivePendingApprovals(messages).length).toBeGreaterThan(0)
+    })
+
+    it("returns nothing once the user stopped the turn", () => {
+        expect(getLivePendingApprovals(messages, {stopped: true})).toEqual([])
+    })
+
+    it("is empty for an empty transcript either way", () => {
+        expect(getLivePendingApprovals([], {stopped: false})).toEqual([])
+        expect(getLivePendingApprovals([], {stopped: true})).toEqual([])
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/model/userStop.test.ts b/web/packages/agenta-chat/tests/unit/model/userStop.test.ts
new file mode 100644
index 00000000000..5af63de7662
--- /dev/null
+++ b/web/packages/agenta-chat/tests/unit/model/userStop.test.ts
@@ -0,0 +1,130 @@
+import type {UIMessage} from "ai"
+import {describe, expect, it} from "vitest"
+
+import {
+    createUserStoppedState,
+    isSessionTurnStopping,
+    lastTurnWasUserStopped,
+    reduceUserStoppedState,
+    type UserStoppedState,
+    type UserStoppedStateEvent,
+} from "../../../src/model/userStop"
+
+const assistant = (metadata?: Record): UIMessage =>
+    ({id: "a1", role: "assistant", parts: [], metadata}) as UIMessage
+
+const approval = {
+    id: "a1",
+    role: "assistant" as const,
+    parts: [
+        {
+            type: "tool-shell",
+            toolCallId: "call-1",
+            state: "approval-requested",
+            approval: {id: "approval-1"},
+            input: {},
+        },
+    ],
+} as UIMessage
+
+const clientInteraction = {
+    id: "a2",
+    role: "assistant" as const,
+    parts: [
+        {
+            type: "tool-request_input",
+            toolCallId: "call-2",
+            state: "input-available",
+            input: {},
+        },
+    ],
+} as UIMessage
+
+const reduce = (
+    event: UserStoppedStateEvent,
+    state: UserStoppedState = createUserStoppedState([]),
+) => reduceUserStoppedState(state, event)
+
+describe("user stopped state", () => {
+    it("keeps a remounted turn guarded until its durable Stop settles", () => {
+        expect(isSessionTurnStopping({currentTurnId: "turn-1", stoppingTurnId: "turn-1"})).toBe(
+            true,
+        )
+        expect(isSessionTurnStopping({currentTurnId: "turn-1", stoppingTurnId: null})).toBe(false)
+    })
+
+    it("does not apply a stale Stop marker to a newer turn", () => {
+        expect(isSessionTurnStopping({currentTurnId: "turn-2", stoppingTurnId: "turn-1"})).toBe(
+            false,
+        )
+    })
+
+    it("maps a stream-delivered cancelled ending to the neutral state", () => {
+        expect(
+            reduce({
+                type: "stream-terminal",
+                finishReason: "other",
+                messages: [assistant()],
+            }).stopped,
+        ).toBe(true)
+    })
+
+    it("does not mistake a paused approval for a cancellation", () => {
+        expect(
+            reduce({
+                type: "stream-terminal",
+                finishReason: "other",
+                messages: [approval],
+            }).stopped,
+        ).toBe(false)
+    })
+
+    it("does not mistake a parked client interaction for a cancellation", () => {
+        expect(
+            reduce({
+                type: "stream-terminal",
+                finishReason: "other",
+                messages: [clientInteraction],
+            }).stopped,
+        ).toBe(false)
+    })
+
+    it("maps a replayed cancelled turn to the neutral state", () => {
+        const messages = [assistant({runStopped: true})]
+
+        expect(lastTurnWasUserStopped(messages)).toBe(true)
+        expect(reduce({type: "transcript", messages}).stopped).toBe(true)
+    })
+
+    it("keeps genuine stream failures non-neutral", () => {
+        expect(
+            reduce({
+                type: "stream-terminal",
+                finishReason: "error",
+                messages: [assistant()],
+            }).stopped,
+        ).toBe(false)
+    })
+
+    it("clears the marker when a new turn starts", () => {
+        const state = reduce({type: "user-stop"}, createUserStoppedState([assistant()]))
+        expect(reduce({type: "reset"}, state).stopped).toBe(false)
+    })
+
+    it("keeps the local latch while the same stopped turn changes in place", () => {
+        const stoppedTurn = assistant({turnId: "turn-1"})
+        const state = reduce({type: "user-stop"}, createUserStoppedState([stoppedTurn]))
+
+        expect(reduce({type: "transcript", messages: [stoppedTurn]}, state).stopped).toBe(true)
+    })
+
+    it("clears the latch when revalidation adopts a newer resumed turn", () => {
+        const state = reduce(
+            {type: "user-stop"},
+            createUserStoppedState([assistant({turnId: "turn-1"})]),
+        )
+        const resumedTurn = assistant({turnId: "turn-2"})
+
+        expect(reduce({type: "transcript", messages: [resumedTurn]}, state).stopped).toBe(false)
+    })
+})
diff --git a/web/packages/agenta-chat/tests/unit/state/sessionEphemera.test.ts b/web/packages/agenta-chat/tests/unit/state/sessionEphemera.test.ts
index 2cffe9b85ff..bd04581fe35 100644
--- a/web/packages/agenta-chat/tests/unit/state/sessionEphemera.test.ts
+++ b/web/packages/agenta-chat/tests/unit/state/sessionEphemera.test.ts
@@ -5,10 +5,13 @@ import {
     attachmentsBySession,
     clearSessionEphemera,
     clearSessionFresh,
+    clearSessionTurnId,
     composerDraftBySession,
     freshSessionIds,
+    getSessionTurnId,
     isSessionFresh,
     markSessionFresh,
+    setSessionTurnId,
 } from "../../../src/state/sessionEphemera"
 
 const attachment = (uid: string): PendingAttachment => ({
@@ -18,6 +21,8 @@ const attachment = (uid: string): PendingAttachment => ({
 })
 
 beforeEach(() => {
+    clearSessionEphemera("s1")
+    clearSessionEphemera("s2")
     composerDraftBySession.clear()
     attachmentsBySession.clear()
     freshSessionIds.clear()
@@ -49,6 +54,26 @@ describe("fresh-session marker", () => {
     })
 })
 
+describe("turn id freshness", () => {
+    it("does not resurrect the superseded turn while a resumed execution is starting", () => {
+        setSessionTurnId("s1", "turn-parked")
+        clearSessionTurnId("s1")
+        clearSessionTurnId("s1")
+
+        // An approval rerender must not restore old metadata before the new runner frame arrives.
+        setSessionTurnId("s1", "turn-parked")
+        expect(getSessionTurnId("s1")).toBeUndefined()
+
+        setSessionTurnId("s1", "turn-resumed")
+        expect(getSessionTurnId("s1")).toBe("turn-resumed")
+
+        clearSessionTurnId("s1")
+        setSessionTurnId("s1", "turn-latest")
+        setSessionTurnId("s1", "turn-parked")
+        expect(getSessionTurnId("s1")).toBe("turn-latest")
+    })
+})
+
 describe("clearSessionEphemera", () => {
     it("clears the draft, attachments, and fresh marker for one session", () => {
         composerDraftBySession.set("s1", "draft")
diff --git a/web/packages/agenta-entities/src/session/api/api.ts b/web/packages/agenta-entities/src/session/api/api.ts
index 62862067d05..1085d2abbcd 100644
--- a/web/packages/agenta-entities/src/session/api/api.ts
+++ b/web/packages/agenta-entities/src/session/api/api.ts
@@ -17,6 +17,7 @@ import {
     sessionInteractionResponseSchema,
     sessionInteractionsResponseSchema,
     sessionRecordsQueryResponseSchema,
+    sessionCancelExecutionResponseSchema,
     sessionsQueryResponseSchema,
     sessionStreamCommandResponseSchema,
     sessionStreamSchema,
@@ -43,6 +44,7 @@ import {
     getLowPrioritySessionsClient,
     getMountsClient,
     getSessionsClient,
+    isAbortError,
     projectScopedRequest,
 } from "./client"
 
@@ -620,6 +622,81 @@ export async function killSession({
     return data !== null
 }
 
+/** Stop keeps accepted, idle, stale, and failed outcomes distinct. */
+export interface CancelSessionStreamParams extends SessionScopedParams {
+    /** The server cancels this observed execution or nothing. */
+    expectedExecutionId?: string
+}
+
+export type CancelSessionOutcome =
+    | {status: "cancelled"; response: SessionStreamCommandResponse | null}
+    | {status: "idle"}
+    /** The server refused: another turn holds the session, or the Stop arrived too late. */
+    | {status: "stale"; message: string}
+    | {status: "failed"; message: string}
+
+const STALE_CANCEL_FALLBACK =
+    "That run had already finished. The session is running something else now."
+const FAILED_CANCEL_FALLBACK = "Could not stop the run. It may still be running."
+
+/** The response envelope's error message, when it is there. */
+const cancelErrorMessage = (error: unknown, fallback: string): string => {
+    const detail = (error as {body?: {detail?: unknown}} | null)?.body?.detail
+    if (typeof detail === "string") return detail
+    const message = (detail as {message?: unknown} | null)?.message
+    if (typeof message === "string") return message
+    return error instanceof Error && error.message ? error.message : fallback
+}
+
+/** Stop the current turn and preserve the server outcome for the caller. */
+export async function cancelSessionStream({
+    sessionId,
+    projectId,
+    appId,
+    abortSignal,
+    expectedExecutionId,
+}: CancelSessionStreamParams): Promise {
+    if (!projectId || !sessionId) return {status: "failed", message: FAILED_CANCEL_FALLBACK}
+
+    try {
+        const data = await getSessionsClient().setSessionStream(
+            {
+                session_id: sessionId,
+                // Omission selects the server's arrival-time guard.
+                ...(expectedExecutionId ? {expected_execution_id: expectedExecutionId} : {}),
+            },
+            projectScopedRequest(projectId, appId, abortSignal),
+        )
+        const response =
+            safeParseWithLogging(
+                sessionStreamCommandResponseSchema,
+                data,
+                "[cancelSessionStream]",
+            ) ?? null
+        if (!response || !response.cancelled_turn_ids) {
+            return {status: "failed", message: FAILED_CANCEL_FALLBACK}
+        }
+        if (response.cancelled_turn_ids.length === 0) return {status: "idle"}
+        return {
+            status: "cancelled",
+            response,
+        }
+    } catch (error) {
+        if (isAbortError(error)) throw error
+        if (isInteractionConflict(error)) {
+            return {
+                status: "stale",
+                message: cancelErrorMessage(error, STALE_CANCEL_FALLBACK),
+            }
+        }
+        console.error(
+            "[cancelSessionStream] failed:",
+            error instanceof Error ? error.message : String(error),
+        )
+        return {status: "failed", message: cancelErrorMessage(error, FAILED_CANCEL_FALLBACK)}
+    }
+}
+
 /**
  * DELETE — permanently remove a session (root hard-delete fan-out across turns/streams/
  * interactions/mounts). Distinct from `killSession` (a soft end that stays resumable). Propagates
@@ -937,3 +1014,84 @@ export async function readMountFile({
     const validated = safeParseWithLogging(mountFileContentResponseSchema, data, "[readMountFile]")
     return validated?.content ?? null
 }
+
+export interface CancelSessionExecutionParams extends SessionScopedParams {
+    /** Fence Stop to the execution the caller observed. */
+    expectedExecutionId?: string
+    /** Retry identity for this request. Two sends of the same key are one command. */
+    idempotencyKey?: string
+}
+
+export interface CancelSessionExecutionResult {
+    /** The durable command's id and DELIVERY state — never the execution's state. */
+    command: {id: string; state: string}
+    /** What to render: the execution being stopped, or nothing. */
+    execution: {id: string | null; state: "stopping" | "idle"}
+    /** True when the active API path accepted or completed the Stop. */
+    accepted: boolean
+    /** True when the API refused because another execution is running (409). */
+    conflict: boolean
+}
+
+/** Cancel current work through Fern while keeping the session warm. */
+export async function cancelSessionExecution({
+    sessionId,
+    projectId,
+    appId,
+    abortSignal,
+    expectedExecutionId,
+    idempotencyKey,
+}: CancelSessionExecutionParams): Promise {
+    if (!projectId || !sessionId) return null
+
+    try {
+        const requestOptions = {
+            ...projectScopedRequest(projectId, appId, abortSignal),
+            ...(idempotencyKey ? {headers: {"Idempotency-Key": idempotencyKey}} : {}),
+        }
+        const {data, rawResponse} = await getSessionsClient()
+            .cancelSessionExecution(
+                {
+                    session_id: sessionId,
+                    body: expectedExecutionId ? {expected_execution_id: expectedExecutionId} : null,
+                },
+                requestOptions,
+            )
+            .withRawResponse()
+        const validated = safeParseWithLogging(
+            sessionCancelExecutionResponseSchema,
+            data,
+            "[cancelSessionExecution]",
+        )
+        if (!validated) return null
+        if (!("command" in validated)) {
+            return {
+                command: {id: "", state: "applied"},
+                execution: {id: validated.turn_id ?? null, state: "idle"},
+                accepted: true,
+                conflict: false,
+            }
+        }
+        return {
+            command: validated.command,
+            execution: {...validated.execution, id: validated.execution.id ?? null},
+            accepted: rawResponse.status === 202,
+            conflict: false,
+        }
+    } catch (error) {
+        if (isAbortError(error)) throw error
+        if ((error as {statusCode?: number} | null)?.statusCode === 409) {
+            return {
+                command: {id: "", state: "obsolete"},
+                execution: {id: null, state: "idle"},
+                accepted: false,
+                conflict: true,
+            }
+        }
+        console.error(
+            "[cancelSessionExecution] failed:",
+            error instanceof Error ? error.message : String(error),
+        )
+        return null
+    }
+}
diff --git a/web/packages/agenta-entities/src/session/core/liveness.ts b/web/packages/agenta-entities/src/session/core/liveness.ts
index 32871468f0e..33a6a539ad0 100644
--- a/web/packages/agenta-entities/src/session/core/liveness.ts
+++ b/web/packages/agenta-entities/src/session/core/liveness.ts
@@ -88,3 +88,22 @@ export function refineLifecycleWithSandbox(
     if (sandbox.alive === true) return sandbox.warm ? "warm" : "cold"
     return lifecycle
 }
+
+/** What a liveness-driven `refetchInterval` may return: a period in ms, or `false` to stop. */
+export type LivenessPollInterval = number | false
+
+/** Fast cadence: something is executing right now, so the view changes on its own. */
+const RUNNING_POLL_MS = 15_000
+/** Slow cadence: nothing runs, but a warm session can be resumed from another device. */
+const RESUMABLE_POLL_MS = 60_000
+
+/** Poll `is_running` quickly; `is_alive` alone means warm and uses the slow cadence. */
+export function livenessPollInterval(
+    rows: readonly (SessionStream | null | undefined)[] | null | undefined,
+    options?: {idle?: LivenessPollInterval},
+): LivenessPollInterval {
+    const list = rows ?? []
+    if (list.some((row) => row?.flags?.is_running)) return RUNNING_POLL_MS
+    if (list.some((row) => row?.flags?.is_alive)) return RESUMABLE_POLL_MS
+    return options?.idle ?? false
+}
diff --git a/web/packages/agenta-entities/src/session/core/schema.ts b/web/packages/agenta-entities/src/session/core/schema.ts
index ca945bb0124..f15855544c7 100644
--- a/web/packages/agenta-entities/src/session/core/schema.ts
+++ b/web/packages/agenta-entities/src/session/core/schema.ts
@@ -152,6 +152,7 @@ export const sessionStreamSchema = z.object({
     name: z.string().nullish(),
     description: z.string().nullish(),
     turn_id: z.string().nullish(),
+    stopping_turn_id: z.string().nullish(),
     // User-visible tags; attribution has dedicated typed fields below.
     tags: z.record(z.string(), z.unknown()).nullish(),
     status: z.object({code: z.string().nullish(), message: z.string().nullish()}).nullish(),
@@ -206,8 +207,20 @@ export const sessionStreamCommandResponseSchema = z.object({
     turn_id: z.string().nullish(),
     watcher_id: z.string().nullish(),
     detached: z.boolean().nullish(),
+    cancelled_turn_ids: z.array(z.string()).nullish(),
 })
 
+export const sessionCancelExecutionResponseSchema = z.union([
+    z.object({
+        command: z.object({id: z.string(), state: z.string()}),
+        execution: z.object({
+            id: z.string().nullish(),
+            state: z.enum(["stopping", "idle"]),
+        }),
+    }),
+    sessionStreamCommandResponseSchema,
+])
+
 export type SessionStream = z.infer
 export type SessionReference = z.infer
 export type SessionOrigin = z.infer
diff --git a/web/packages/agenta-entities/src/session/index.ts b/web/packages/agenta-entities/src/session/index.ts
index ba7fd03a94d..823aa229466 100644
--- a/web/packages/agenta-entities/src/session/index.ts
+++ b/web/packages/agenta-entities/src/session/index.ts
@@ -18,6 +18,10 @@ export {
     setSessionHeader,
     fetchSessionStream,
     commandSessionStream,
+    cancelSessionExecution,
+    cancelSessionStream,
+    type CancelSessionOutcome,
+    type CancelSessionStreamParams,
     killSession,
     deleteSession as deleteSessionRemote,
     archiveSession as archiveSessionRemote,
@@ -38,6 +42,8 @@ export {
     type RespondInteractionParams,
     type TransitionInteractionParams,
     type CommandSessionStreamParams,
+    type CancelSessionExecutionParams,
+    type CancelSessionExecutionResult,
 } from "./api/api"
 export {
     getSessionsClient,
@@ -80,6 +86,8 @@ export {
     deriveStreamNest,
     deriveSessionLifecycle,
     refineLifecycleWithSandbox,
+    livenessPollInterval,
+    type LivenessPollInterval,
     type SessionLifecycle,
     type SessionStreamNest,
     type SandboxLiveness,
diff --git a/web/packages/agenta-entities/tests/unit/session-cancel-api.test.ts b/web/packages/agenta-entities/tests/unit/session-cancel-api.test.ts
new file mode 100644
index 00000000000..f96d2d6230f
--- /dev/null
+++ b/web/packages/agenta-entities/tests/unit/session-cancel-api.test.ts
@@ -0,0 +1,113 @@
+import {beforeEach, describe, expect, it, vi} from "vitest"
+
+const fernCancelSessionExecution = vi.fn()
+
+vi.mock("@agenta/sdk/resources", () => ({
+    getSessionsClient: () => ({cancelSessionExecution: fernCancelSessionExecution}),
+    getLowPrioritySessionsClient: vi.fn(),
+    getMountsClient: vi.fn(),
+    getLowPriorityMountsClient: vi.fn(),
+}))
+
+import {cancelSessionExecution} from "../../src/session/api/api"
+
+const response = {
+    command: {id: "command-1", state: "pending"},
+    execution: {id: "turn-1", state: "stopping"},
+}
+
+beforeEach(() => {
+    fernCancelSessionExecution.mockReset()
+    fernCancelSessionExecution.mockReturnValue({
+        withRawResponse: () => Promise.resolve({data: response, rawResponse: {status: 202}}),
+    })
+})
+
+describe("cancelSessionExecution", () => {
+    it("uses the Fern route with project query scope and the observed turn", async () => {
+        const result = await cancelSessionExecution({
+            projectId: "project-1",
+            appId: "app-1",
+            sessionId: "session-1",
+            expectedExecutionId: "turn-1",
+            idempotencyKey: "stop-1",
+        })
+
+        expect(fernCancelSessionExecution).toHaveBeenCalledWith(
+            {
+                session_id: "session-1",
+                body: {expected_execution_id: "turn-1"},
+            },
+            {
+                queryParams: {project_id: "project-1", application_id: "app-1"},
+                abortSignal: undefined,
+                headers: {"Idempotency-Key": "stop-1"},
+            },
+        )
+        expect(result).toEqual({...response, accepted: true, conflict: false})
+    })
+
+    it("maps Fern 409 to the stale-execution conflict result", async () => {
+        fernCancelSessionExecution.mockReturnValue({
+            withRawResponse: () => Promise.reject({statusCode: 409}),
+        })
+
+        const result = await cancelSessionExecution({
+            projectId: "project-1",
+            sessionId: "session-1",
+        })
+
+        expect(result).toEqual({
+            command: {id: "", state: "obsolete"},
+            execution: {id: null, state: "idle"},
+            accepted: false,
+            conflict: true,
+        })
+    })
+
+    it("accepts and normalizes the API flag-off legacy cancel payload", async () => {
+        const consoleError = vi.spyOn(console, "error").mockImplementation(() => {})
+        fernCancelSessionExecution.mockReturnValue({
+            withRawResponse: () =>
+                Promise.resolve({
+                    data: {
+                        mode: "cancel",
+                        session_id: "session-1",
+                        turn_id: "turn-1",
+                        watcher_id: null,
+                        detached: true,
+                        cancelled_turn_ids: [],
+                    },
+                    rawResponse: {status: 200},
+                }),
+        })
+
+        const result = await cancelSessionExecution({
+            projectId: "project-1",
+            sessionId: "session-1",
+            expectedExecutionId: "turn-1",
+        })
+
+        expect(result).toEqual({
+            command: {id: "", state: "applied"},
+            execution: {id: "turn-1", state: "idle"},
+            accepted: true,
+            conflict: false,
+        })
+        expect(consoleError).not.toHaveBeenCalled()
+        consoleError.mockRestore()
+    })
+
+    it("rejects malformed successful payloads at the Zod boundary", async () => {
+        const consoleError = vi.spyOn(console, "error").mockImplementation(() => {})
+        fernCancelSessionExecution.mockReturnValue({
+            withRawResponse: () =>
+                Promise.resolve({data: {command: null}, rawResponse: {status: 202}}),
+        })
+
+        await expect(
+            cancelSessionExecution({projectId: "project-1", sessionId: "session-1"}),
+        ).resolves.toBeNull()
+        expect(consoleError).toHaveBeenCalled()
+    })
+})
diff --git a/web/packages/agenta-entities/tests/unit/session-cancel-stream.test.ts b/web/packages/agenta-entities/tests/unit/session-cancel-stream.test.ts
new file mode 100644
index 00000000000..44eb51fbfc8
--- /dev/null
+++ b/web/packages/agenta-entities/tests/unit/session-cancel-stream.test.ts
@@ -0,0 +1,138 @@
+import {beforeEach, describe, expect, it, vi} from "vitest"
+
+const setSessionStream = vi.fn()
+
+vi.mock("@agenta/sdk/resources", () => ({
+    getSessionsClient: () => ({setSessionStream}),
+    getLowPrioritySessionsClient: () => ({setSessionStream}),
+    getMountsClient: vi.fn(),
+    getLowPriorityMountsClient: vi.fn(),
+}))
+
+const {cancelSessionStream} = await import("../../src/session/api/api")
+
+const params = {sessionId: "s1", projectId: "p1"}
+
+const apiError = (statusCode: number, body?: unknown) =>
+    Object.assign(new Error("AgentaApiError"), {name: "AgentaApiError", statusCode, body})
+
+beforeEach(() => {
+    setSessionStream.mockReset()
+})
+
+describe("cancelSessionStream", () => {
+    it("reports the cancelled turns when the server accepts", async () => {
+        setSessionStream.mockResolvedValue({
+            mode: "cancel",
+            session_id: "s1",
+            turn_id: "turn-1",
+            cancelled_turn_ids: ["turn-1"],
+            detached: true,
+        })
+
+        const outcome = await cancelSessionStream(params)
+
+        expect(outcome.status).toBe("cancelled")
+        expect(outcome.status === "cancelled" && outcome.response?.turn_id).toBe("turn-1")
+        expect(setSessionStream).toHaveBeenCalledWith({session_id: "s1"}, expect.anything())
+    })
+
+    it("reports idle when the server accepted but found no running turn", async () => {
+        setSessionStream.mockResolvedValue({
+            mode: "cancel",
+            session_id: "s1",
+            cancelled_turn_ids: [],
+        })
+
+        expect(await cancelSessionStream(params)).toEqual({status: "idle"})
+    })
+
+    it("reports failure when the response omits cancellation evidence", async () => {
+        setSessionStream.mockResolvedValue({mode: "cancel", session_id: "s1"})
+
+        expect(await cancelSessionStream(params)).toEqual({
+            status: "failed",
+            message: "Could not stop the run. It may still be running.",
+        })
+    })
+
+    it("reports failure when the response cannot be parsed", async () => {
+        setSessionStream.mockResolvedValue({unexpected: true})
+
+        expect(await cancelSessionStream(params)).toEqual({
+            status: "failed",
+            message: "Could not stop the run. It may still be running.",
+        })
+    })
+
+    it("sends the turn id as expected_execution_id when the client knows it", async () => {
+        setSessionStream.mockResolvedValue({mode: "cancel", session_id: "s1"})
+
+        await cancelSessionStream({...params, expectedExecutionId: "turn-7"})
+
+        expect(setSessionStream).toHaveBeenCalledWith(
+            {session_id: "s1", expected_execution_id: "turn-7"},
+            expect.anything(),
+        )
+    })
+
+    it("omits the field entirely when the client never learned the turn id", async () => {
+        setSessionStream.mockResolvedValue({mode: "cancel", session_id: "s1"})
+
+        await cancelSessionStream({...params, expectedExecutionId: undefined})
+
+        expect(setSessionStream).toHaveBeenCalledWith({session_id: "s1"}, expect.anything())
+    })
+
+    it("reports a 409 as stale, carrying the server's own message", async () => {
+        setSessionStream.mockRejectedValue(
+            apiError(409, {
+                detail: {
+                    message:
+                        "Session 's1' is running turn 'turn-2', not the expected turn 'turn-1'.",
+                    expected_execution_id: "turn-1",
+                    actual_execution_id: "turn-2",
+                },
+            }),
+        )
+
+        const outcome = await cancelSessionStream(params)
+
+        expect(outcome.status).toBe("stale")
+        expect(outcome.status === "stale" && outcome.message).toContain("turn-2")
+    })
+
+    it("falls back to plain wording when a 409 carries no readable message", async () => {
+        setSessionStream.mockRejectedValue(apiError(409, {detail: {}}))
+
+        const outcome = await cancelSessionStream(params)
+
+        expect(outcome.status).toBe("stale")
+        expect(outcome.status === "stale" && outcome.message.length).toBeGreaterThan(0)
+    })
+
+    it("reports any other error as failed, never as stale", async () => {
+        setSessionStream.mockRejectedValue(apiError(500, {detail: {message: "Runner unavailable"}}))
+
+        expect(await cancelSessionStream(params)).toEqual({
+            status: "failed",
+            message: "Runner unavailable",
+        })
+    })
+
+    it("rethrows an abort so a cancelled query settles as cancelled", async () => {
+        setSessionStream.mockRejectedValue(
+            Object.assign(new Error("AgentaApiError"), {
+                name: "AgentaApiError",
+                message: "The user aborted a request",
+            }),
+        )
+
+        await expect(cancelSessionStream(params)).rejects.toThrow()
+    })
+
+    it("is a no-op without a project or a session", async () => {
+        expect((await cancelSessionStream({sessionId: "s1", projectId: ""})).status).toBe("failed")
+        expect(setSessionStream).not.toHaveBeenCalled()
+    })
+})
diff --git a/web/packages/agenta-entities/tests/unit/session-liveness.test.ts b/web/packages/agenta-entities/tests/unit/session-liveness.test.ts
index e5db3f25ea3..f1d943a563c 100644
--- a/web/packages/agenta-entities/tests/unit/session-liveness.test.ts
+++ b/web/packages/agenta-entities/tests/unit/session-liveness.test.ts
@@ -9,6 +9,7 @@ import {describe, expect, it} from "vitest"
 import {
     deriveSessionLifecycle,
     deriveStreamNest,
+    livenessPollInterval,
     refineLifecycleWithSandbox,
 } from "../../src/session/core/liveness"
 import type {SessionStream} from "../../src/session/core/schema"
@@ -92,3 +93,31 @@ describe("refineLifecycleWithSandbox", () => {
         expect(refineLifecycleWithSandbox("cold", {alive: true, warm: false})).toBe("cold")
     })
 })
+
+// Every liveness poll shares the running-versus-warm cadence rule.
+describe("livenessPollInterval", () => {
+    const rows = (...flags: Partial>[]) => flags.map(streamWith)
+
+    it("polls fast while any row is running", () => {
+        expect(livenessPollInterval(rows({is_alive: true, is_running: true}))).toBe(15_000)
+        expect(livenessPollInterval(rows({is_alive: true}, {is_running: true}))).toBe(15_000)
+    })
+
+    // A stopped session stays alive for warm resume but no longer polls quickly.
+    it("drops to the slow cadence for a session that is alive but not running", () => {
+        expect(livenessPollInterval(rows({is_alive: true}))).toBe(60_000)
+    })
+
+    it("stops by default when nothing is alive", () => {
+        expect(livenessPollInterval(rows({}))).toBe(false)
+        expect(livenessPollInterval([])).toBe(false)
+        expect(livenessPollInterval(null)).toBe(false)
+        expect(livenessPollInterval(undefined)).toBe(false)
+    })
+
+    // The rail must still DISCOVER a run it did not start, so it names a floor instead of false.
+    it("uses the caller's idle floor when one is given", () => {
+        expect(livenessPollInterval([], {idle: 60_000})).toBe(60_000)
+        expect(livenessPollInterval(rows({}), {idle: 60_000})).toBe(60_000)
+    })
+})
diff --git a/web/packages/agenta-entities/tests/unit/session-query-schema.test.ts b/web/packages/agenta-entities/tests/unit/session-query-schema.test.ts
index bca0ea658f7..369a5fdde9c 100644
--- a/web/packages/agenta-entities/tests/unit/session-query-schema.test.ts
+++ b/web/packages/agenta-entities/tests/unit/session-query-schema.test.ts
@@ -27,6 +27,7 @@ const wireRow = {
     tags: {priority: "high"},
     meta: {source: "web"},
     turn_id: "turn-7",
+    stopping_turn_id: "turn-7",
     created_at: "2026-07-20T10:00:00Z",
     updated_at: "2026-07-24T09:30:00Z",
     references: [
@@ -71,6 +72,7 @@ describe("sessionStreamSchema (/sessions/query rows)", () => {
         expect(out.name).toBe("Refactor the auth flow")
         expect(out.flags).toEqual({is_alive: true, is_running: false, is_attached: false})
         expect(out.updated_at).toBe("2026-07-24T09:30:00Z")
+        expect(out.stopping_turn_id).toBe("turn-7")
         expect(out.references?.[0]?.id).toBe("33333333-3333-3333-3333-333333333333")
         expect(out.references?.[0]?.slug).toBe("support-router")
         expect(out.references?.[0]?.version).toBe("v3")
@@ -90,6 +92,7 @@ describe("sessionStreamSchema (/sessions/query rows)", () => {
         expect(out.description).toBeUndefined()
         expect(out.flags).toBeUndefined()
         expect(out.turn_id).toBeUndefined()
+        expect(out.stopping_turn_id).toBeUndefined()
         expect(out.created_at).toBeUndefined()
         expect(out.updated_at).toBeUndefined()
         expect(out.deleted_at).toBeUndefined()
diff --git a/web/packages/agenta-navigation/src/dynamic/sessionsSource.ts b/web/packages/agenta-navigation/src/dynamic/sessionsSource.ts
index c453ffa5482..d65afdf6d6a 100644
--- a/web/packages/agenta-navigation/src/dynamic/sessionsSource.ts
+++ b/web/packages/agenta-navigation/src/dynamic/sessionsSource.ts
@@ -1,4 +1,9 @@
-import {queryInteractions, querySessions, type SessionStream} from "@agenta/entities/session"
+import {
+    livenessPollInterval,
+    queryInteractions,
+    querySessions,
+    type SessionStream,
+} from "@agenta/entities/session"
 import {
     agentWorkflowsListQueryStateAtom,
     appWorkflowsListQueryAtom,
@@ -115,28 +120,12 @@ const requestFilters = (filters: SidebarSessionFilters) => {
     }
 }
 
-/** Fast enough that a dot clears about when the stream does. */
-const LIVE_POLL_MS = 15_000
-
 /** Slow enough to be background noise, quick enough to notice a run you did not start. */
 const IDLE_POLL_MS = 60_000
 
-/**
- * Poll fast while something can still change, slowly the rest of the time.
- *
- * A row's dot is driven by `is_alive`/`is_running`, which the server flips when the stream ends —
- * with no request, the dot stays filled until you reload. The BASELINE matters just as much: a
- * turn started under another agent (a trigger, another browser) is invisible to this client, so a
- * rail that stopped polling when it looked quiet could never discover it, and only the session you
- * were driving yourself ever appeared to run.
- *
- * Both intervals are gated: the source only subscribes while the Sessions group is open and the
- * rail is expanded, and React Query holds the timer while the window is unfocused.
- */
+/** Poll fast for running work and keep a slow baseline for cross-client discovery. */
 export const livePollInterval = (rows: SessionStream[] | null | undefined) =>
-    (rows ?? []).some((row) => row.flags?.is_alive || row.flags?.is_running)
-        ? LIVE_POLL_MS
-        : IDLE_POLL_MS
+    livenessPollInterval(rows, {idle: IDLE_POLL_MS})
 
 /**
  * One request per selected agent, merged — see `requestFilters` on why they cannot be one.
diff --git a/web/packages/agenta-navigation/tests/unit/sidebarChildren.test.ts b/web/packages/agenta-navigation/tests/unit/sidebarChildren.test.ts
index 0929ec8800a..680aee3fa21 100644
--- a/web/packages/agenta-navigation/tests/unit/sidebarChildren.test.ts
+++ b/web/packages/agenta-navigation/tests/unit/sidebarChildren.test.ts
@@ -537,17 +537,21 @@ describe("localSessionRefsMatching", () => {
     })
 })
 
-// The baseline is the half that is easy to lose: without it the rail can only ever show the run
-// you started yourself, because a turn under another agent reaches this client through the poll.
+// The baseline discovers runs started by another client.
 describe("livePollInterval", () => {
     // Only `flags` is read; the rest of a SessionStream is irrelevant here.
     const rows = (...flags: {is_alive?: boolean; is_running?: boolean}[]) =>
         flags.map((f) => ({session_id: "s1", flags: f})) as Parameters[0] &
             object[]
 
-    it("polls fast while a session is alive or running", () => {
-        expect(livePollInterval(rows({is_alive: true}))).toBe(15_000)
-        expect(livePollInterval(rows({is_running: true}))).toBe(15_000)
+    it("polls fast only while a session is RUNNING", () => {
+        expect(livePollInterval(rows({is_alive: true, is_running: true}))).toBe(15_000)
+        expect(livePollInterval(rows({}, {is_running: true}))).toBe(15_000)
+    })
+
+    // Warm but idle sessions use the slow cadence.
+    it("drops to the slow baseline for a session that is alive but not running", () => {
+        expect(livePollInterval(rows({is_alive: true}))).toBe(60_000)
     })
 
     it("keeps a slow baseline when every row looks idle", () => {
diff --git a/web/packages/agenta-ui/src/RichChatInput/RichChatInput.tsx b/web/packages/agenta-ui/src/RichChatInput/RichChatInput.tsx
index ed54dd01757..75fd4a1b35b 100644
--- a/web/packages/agenta-ui/src/RichChatInput/RichChatInput.tsx
+++ b/web/packages/agenta-ui/src/RichChatInput/RichChatInput.tsx
@@ -100,7 +100,9 @@ export interface RichChatInputProps {
     hideSendButton?: boolean
     /** A stream is in flight — the send button becomes a Stop button. */
     streaming?: boolean
-    /** Abort the in-flight stream (used while `streaming`). */
+    /** Disable the Stop control while its durable request is settling. */
+    stopping?: boolean
+    /** Request a durable stop (used while `streaming`). */
     onStop?: () => void
     /** Min-height class for the editor area (default `min-h-[72px]`). */
     minHeightClassName?: string
@@ -163,6 +165,7 @@ export const RichChatInput = forwardRef
             sendDisabled,
             hideSendButton,
             streaming,
+            stopping,
             onStop,
             minHeightClassName = "min-h-[72px]",
             size = "compact",
@@ -367,6 +370,7 @@ export const RichChatInput = forwardRef
                                     disabledReason={sendDisabledReason}
                                     streaming={streaming}
                                     onStop={onStop}
+                                    stopping={stopping}
                                 />
                             )}
                             {trailing}
diff --git a/web/packages/agenta-ui/src/RichChatInput/plugins/SendButton.tsx b/web/packages/agenta-ui/src/RichChatInput/plugins/SendButton.tsx
index cc09b49f65f..322df735bbb 100644
--- a/web/packages/agenta-ui/src/RichChatInput/plugins/SendButton.tsx
+++ b/web/packages/agenta-ui/src/RichChatInput/plugins/SendButton.tsx
@@ -16,9 +16,10 @@ interface SendButtonProps {
     disabled?: boolean
     /** Tooltip shown when a caller blocks submit. */
     disabledReason?: ReactNode
-    /** When true, the button becomes a Stop button that aborts the in-flight stream. */
+    /** When true, the button becomes a Stop button for the in-flight stream. */
     streaming?: boolean
-    /** Abort the in-flight stream — required for the `streaming` state. */
+    stopping?: boolean
+    /** Request a durable stop — required for the `streaming` state. */
     onStop?: () => void
 }
 
@@ -31,6 +32,7 @@ export function SendButton({
     disabled,
     disabledReason,
     streaming,
+    stopping,
     onStop,
 }: SendButtonProps) {
     const [editor] = useLexicalComposerContext()
@@ -73,9 +75,10 @@ export function SendButton({
                     size="icon"
                     variant="ghost"
                     className="rounded-control-round"
-                    aria-label="Stop"
+                    aria-label={stopping ? "Stopping" : "Stop"}
                     aria-keyshortcuts={shortcutAria("run.stop")}
                     onClick={onStop}
+                    disabled={stopping}
                 >
                     
                 
diff --git a/web/tests/README.md b/web/tests/README.md
index de4a3a87246..204c0d28556 100644
--- a/web/tests/README.md
+++ b/web/tests/README.md
@@ -33,7 +33,7 @@ Auth behavior in global setup:
 - If the frontend renders password auth, a password must be available (see below).
 - If the frontend renders OTP auth, Testmail envs must be available.
 
-Teardown cleans up the ephemeral project and model hub secrets created by the run.
+Teardown deletes the ephemeral project created by the run.
 
 ---
 
diff --git a/web/tests/playwright/global-setup.ts b/web/tests/playwright/global-setup.ts
index 7150ecdabf5..3e2ecb6be80 100644
--- a/web/tests/playwright/global-setup.ts
+++ b/web/tests/playwright/global-setup.ts
@@ -73,9 +73,9 @@ function getConfiguredTestEmail(): string | null {
 }
 
 async function fillOTPDigits(page: Page, otp: string, delay: number): Promise {
-    // Ant Design 5.x Input.OTP renders: 
...

+    // Target the OTP autofill field independently of the component library.
     // Click the first cell to ensure focus (autoFocus may have been lost), then type sequentially.
-    const firstInput = page.locator(".ant-otp input").first()
+    const firstInput = page.locator('input[autocomplete="one-time-code"]').first()
     await firstInput.waitFor({state: "visible", timeout: 10000})
     await firstInput.click()
     await page.keyboard.type(otp, {delay})
@@ -1006,7 +1006,7 @@ async function maybeCreateEphemeralProject(page: Page, baseURL: string): Promise
             console.log(
                 "[global-setup] Ephemeral project disabled (AGENTA_TEST_EPHEMERAL_PROJECT=false)",
             )
-            writeProjectMetadata(projectMetadataPath, defaultProject, page, null)
+            writeProjectMetadata(projectMetadataPath, defaultProject, page, null, false)
             return
         }
 
@@ -1022,7 +1022,7 @@ async function maybeCreateEphemeralProject(page: Page, baseURL: string): Promise
             console.warn(
                 `[global-setup] Failed to create ephemeral project (${response.status()}): ${text}`,
             )
-            writeProjectMetadata(projectMetadataPath, defaultProject, page, null)
+            writeProjectMetadata(projectMetadataPath, defaultProject, page, null, false)
             return
         }
 
@@ -1031,12 +1031,12 @@ async function maybeCreateEphemeralProject(page: Page, baseURL: string): Promise
             `[global-setup] Created ephemeral project: ${projectName} (${project.project_id})`,
         )
 
-        writeProjectMetadata(projectMetadataPath, project, page, originalDefaultProjectId)
+        writeProjectMetadata(projectMetadataPath, project, page, originalDefaultProjectId, true)
     } catch (error) {
         console.warn("[global-setup] Failed to create ephemeral project, using default:", error)
         try {
             const projectMetadataPath = getProjectMetadataPath()
-            writeProjectMetadata(projectMetadataPath, null, page, null)
+            writeProjectMetadata(projectMetadataPath, null, page, null, false)
         } catch (writeError) {
             console.warn("[global-setup] Could not write fallback project metadata:", writeError)
         }
@@ -1048,6 +1048,7 @@ function writeProjectMetadata(
     project: any,
     page: Page,
     originalDefaultProjectId: string | null,
+    ephemeral: boolean,
 ): void {
     let metadata: Record | null = null
 
@@ -1056,6 +1057,7 @@ function writeProjectMetadata(
             project_id: project.project_id,
             project_name: project.project_name ?? null,
             workspace_id: project.workspace_id,
+            ephemeral,
             ...(originalDefaultProjectId !== null
                 ? {original_default_project_id: originalDefaultProjectId}
                 : {}),
@@ -1070,6 +1072,7 @@ function writeProjectMetadata(
                 metadata = {
                     workspace_id: match[1],
                     project_id: match[2],
+                    ephemeral,
                     created_at: new Date().toISOString(),
                 }
                 console.log("[global-setup] Derived project metadata from page URL")
diff --git a/web/tests/playwright/global-teardown.test.ts b/web/tests/playwright/global-teardown.test.ts
new file mode 100644
index 00000000000..ad149fe9076
--- /dev/null
+++ b/web/tests/playwright/global-teardown.test.ts
@@ -0,0 +1,92 @@
+import assert from "node:assert/strict"
+import {existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from "node:fs"
+import {tmpdir} from "node:os"
+import {join} from "node:path"
+import {afterEach, describe, it} from "node:test"
+
+import {deleteEphemeralProject} from "./global-teardown.ts"
+
+const roots: string[] = []
+
+function fixture(metadata: Record) {
+    const root = mkdtempSync(join(tmpdir(), "agenta-global-teardown-"))
+    roots.push(root)
+    const projectPath = join(root, "test-project.json")
+    const statePath = join(root, "state.json")
+    writeFileSync(projectPath, JSON.stringify(metadata))
+    writeFileSync(
+        statePath,
+        JSON.stringify({cookies: [{name: "sAccessToken", value: "test-session"}]}),
+    )
+    return {projectPath, statePath}
+}
+
+afterEach(() => {
+    for (const root of roots.splice(0)) rmSync(root, {recursive: true, force: true})
+})
+
+describe("deleteEphemeralProject", () => {
+    it("never deletes a fallback or default project", async () => {
+        const paths = fixture({
+            project_id: "persistent-project",
+            workspace_id: "workspace",
+            ephemeral: false,
+        })
+        let requestCount = 0
+
+        await deleteEphemeralProject("https://example.test/api", {
+            ...paths,
+            fetchFn: async () => {
+                requestCount += 1
+                return new Response(null, {status: 204})
+            },
+        })
+
+        assert.equal(requestCount, 0)
+        assert.equal(existsSync(paths.projectPath), false)
+    })
+
+    it("deletes an owned ephemeral project and removes its metadata", async () => {
+        const paths = fixture({
+            project_id: "ephemeral-project",
+            project_name: "e2e-test",
+            workspace_id: "workspace",
+            ephemeral: true,
+        })
+        const requests: Array<{url: string; method?: string}> = []
+
+        await deleteEphemeralProject("https://example.test/api", {
+            ...paths,
+            fetchFn: async (input, init) => {
+                requests.push({url: String(input), method: init?.method})
+                return new Response(null, {status: 204})
+            },
+        })
+
+        assert.deepEqual(requests, [
+            {
+                url: "https://example.test/api/projects/ephemeral-project",
+                method: "DELETE",
+            },
+        ])
+        assert.equal(existsSync(paths.projectPath), false)
+    })
+
+    it("retains owned-project metadata when deletion fails", async () => {
+        const metadata = {
+            project_id: "ephemeral-project",
+            project_name: "e2e-test",
+            workspace_id: "workspace",
+            ephemeral: true,
+        }
+        const paths = fixture(metadata)
+
+        await deleteEphemeralProject("https://example.test/api", {
+            ...paths,
+            fetchFn: async () => new Response("temporary failure", {status: 503}),
+        })
+
+        assert.equal(existsSync(paths.projectPath), true)
+        assert.deepEqual(JSON.parse(readFileSync(paths.projectPath, "utf8")), metadata)
+    })
+})
diff --git a/web/tests/playwright/global-teardown.ts b/web/tests/playwright/global-teardown.ts
index 1a1f2e3754c..34de2ca3352 100644
--- a/web/tests/playwright/global-teardown.ts
+++ b/web/tests/playwright/global-teardown.ts
@@ -1,10 +1,4 @@
-/**
- * This script cleans up after Playwright tests.
- * Deletes the ephemeral project created during global-setup (if any),
- * then cleans up model hub secrets.
- */
-
-import {StandardSecretDTO} from "../../oss/src/lib/Types"
+/** Deletes the ephemeral project created during global setup. */
 
 import {existsSync, readFileSync, unlinkSync} from "fs"
 
@@ -22,11 +16,6 @@ function getSessionToken(statePath: string): string | null {
     return state.cookies?.find((c: any) => c.name === "sAccessToken")?.value ?? null
 }
 
-/**
- * Runs after tests complete.
- * 1. Deletes the ephemeral project created during setup (if any).
- * 2. Cleans up model hub secrets (OpenAI keys added during tests).
- */
 /**
  * Derives the API base URL from AGENTA_WEB_URL.
  * The web app may live at a subpath (e.g. /w) but the API is always at /api on the origin.
@@ -49,28 +38,43 @@ async function globalTeardown() {
     const apiURL = getApiURL(baseURL)
     console.log(`[global-teardown] Using api-url: ${apiURL}`)
 
-    // --- Phase 1: Delete ephemeral project ---
     await deleteEphemeralProject(apiURL)
-
-    // --- Phase 2: Clean up model hub secrets ---
-    await cleanupModelHubSecrets(apiURL)
 }
 
 /**
- * Deletes the ephemeral project created during global-setup.
- * Reads project metadata from the runtime metadata file, calls DELETE /api/projects/{id},
- * then removes the metadata file.
+ * Deletes a project only when setup explicitly marked it as ephemeral.
+ * Keeps the metadata after a failed deletion so a later teardown can retry.
  */
-async function deleteEphemeralProject(apiURL: string): Promise {
-    const projectPath = getProjectMetadataPath()
+interface DeleteEphemeralProjectOptions {
+    projectPath?: string
+    statePath?: string
+    fetchFn?: typeof fetch
+}
+
+export async function deleteEphemeralProject(
+    apiURL: string,
+    options: DeleteEphemeralProjectOptions = {},
+): Promise {
+    const projectPath = options.projectPath ?? getProjectMetadataPath()
+    const statePath = options.statePath ?? getStorageStatePath()
+    const fetchFn = options.fetchFn ?? fetch
 
     if (!existsSync(projectPath)) {
         console.log("[global-teardown] No test project metadata found, skipping project cleanup")
         return
     }
 
+    let removeMetadata = false
+
     try {
         const projectData = JSON.parse(readFileSync(projectPath, "utf8"))
+
+        if (projectData.ephemeral !== true) {
+            console.log("[global-teardown] Project is not marked ephemeral, skipping cleanup")
+            removeMetadata = true
+            return
+        }
+
         const projectId = projectData.project_id
         const projectName = projectData.project_name
 
@@ -81,7 +85,6 @@ async function deleteEphemeralProject(apiURL: string): Promise {
 
         console.log(`[global-teardown] Deleting ephemeral project: ${projectName} (${projectId})`)
 
-        const statePath = getStorageStatePath()
         const sessionToken = getSessionToken(statePath)
 
         if (!sessionToken) {
@@ -102,7 +105,7 @@ async function deleteEphemeralProject(apiURL: string): Promise {
             console.log(
                 `[global-teardown] Restoring original default project: ${originalDefaultId}`,
             )
-            const patchResponse = await fetch(`${apiURL}/projects/${originalDefaultId}`, {
+            const patchResponse = await fetchFn(`${apiURL}/projects/${originalDefaultId}`, {
                 method: "PATCH",
                 headers: authHeaders,
                 body: JSON.stringify({make_default: true}),
@@ -113,17 +116,19 @@ async function deleteEphemeralProject(apiURL: string): Promise {
                 console.warn(
                     `[global-teardown] Failed to restore default project (${patchResponse.status})`,
                 )
+                return
             }
         }
 
         // Now delete the ephemeral project
-        const response = await fetch(`${apiURL}/projects/${projectId}`, {
+        const response = await fetchFn(`${apiURL}/projects/${projectId}`, {
             method: "DELETE",
             headers: authHeaders,
         })
 
-        if (response.ok) {
+        if (response.ok || response.status === 404) {
             console.log(`[global-teardown] Deleted ephemeral project: ${projectName}`)
+            removeMetadata = true
         } else {
             const text = await response.text()
             console.warn(
@@ -133,64 +138,18 @@ async function deleteEphemeralProject(apiURL: string): Promise {
     } catch (error) {
         console.warn("[global-teardown] Error deleting ephemeral project:", error)
     } finally {
-        // Always clean up the metadata file
-        try {
-            unlinkSync(projectPath)
-            console.log("[global-teardown] Removed test project metadata")
-        } catch {
-            // Ignore if already deleted
-        }
-    }
-}
-
-/**
- * Cleans up OpenAI model hub secrets that were added during test runs.
- */
-async function cleanupModelHubSecrets(apiURL: string): Promise {
-    try {
-        console.log("[global-teardown] Deleting model hub secrets...")
-        const statePath = getStorageStatePath()
-        const sessionToken = getSessionToken(statePath)
-
-        if (!sessionToken) {
-            console.log(
-                "[global-teardown] No session token in storage state, skipping model hub cleanup",
-            )
-            return
-        }
-
-        console.log(
-            `[teardown] Extracted session token from storage state: ${sessionToken ? "present" : "absent"}`,
-        )
-
-        const secretsResp = await fetch(`${apiURL}/secrets/`, {
-            headers: {Authorization: `Bearer ${sessionToken}`},
-        })
-
-        if (!secretsResp.ok) {
-            console.error("[global-teardown] Failed to fetch secrets", await secretsResp.text())
-            return
-        }
-
-        const secrets = (await secretsResp.json()) as StandardSecretDTO[]
-
-        const openaiSecrets = secrets.filter((s) =>
-            s?.header?.name?.toLowerCase().includes("openai"),
-        )
-
-        for (const secret of openaiSecrets) {
+        if (removeMetadata) {
             try {
-                await fetch(`${apiURL}/secrets/${secret.id}`, {
-                    method: "DELETE",
-                    headers: {Authorization: `Bearer ${sessionToken}`},
-                })
-                console.log(`[global-teardown] Deleted model hub secret ${secret.id}`)
-            } catch (err) {
-                console.error(`[global-teardown] Failed to delete secret ${secret.id}`, err)
+                unlinkSync(projectPath)
+                console.log("[global-teardown] Removed test project metadata")
+            } catch {
+                // Ignore if already deleted
             }
+        } else {
+            console.warn(
+                `[global-teardown] Retained test project metadata for a later cleanup attempt: ${projectPath}`,
+            )
         }
-    } catch (err) {
-        console.error("[global-teardown] Error cleaning up model hub key", err)
     }
 }
 
diff --git a/web/tests/tests/fixtures/base.fixture/providerHelpers/index.ts b/web/tests/tests/fixtures/base.fixture/providerHelpers/index.ts
index b7a7675a188..5c62361d340 100644
--- a/web/tests/tests/fixtures/base.fixture/providerHelpers/index.ts
+++ b/web/tests/tests/fixtures/base.fixture/providerHelpers/index.ts
@@ -139,8 +139,6 @@ function readTestProjectMetadata(): TestProjectMetadata | null {
 async function waitForModelsPageReady(page: Page): Promise {
     const providersSection = getProvidersSection(page)
 
-    await page.waitForLoadState("networkidle", {timeout: 10000}).catch(() => {})
-
     await expect
         .poll(
             async () => {
diff --git a/web/tests/tests/fixtures/user.fixture/authHelpers/index.ts b/web/tests/tests/fixtures/user.fixture/authHelpers/index.ts
index 45cfd52b8a6..7ea5fd7737b 100644
--- a/web/tests/tests/fixtures/user.fixture/authHelpers/index.ts
+++ b/web/tests/tests/fixtures/user.fixture/authHelpers/index.ts
@@ -81,9 +81,9 @@ export const authHelpers = () => {
                 logAuthEmail("Login flow start", email)
 
                 async function fillOTPDigits(otp: string, delay: number): Promise {
-                    // Ant Design 5.x Input.OTP: 
...

+                    // Target the OTP autofill field independently of the component library.
                     // Click the first cell to ensure focus, then type sequentially.
-                    const firstInput = page.locator(".ant-otp input").first()
+                    const firstInput = page.locator('input[autocomplete="one-time-code"]').first()
                     await firstInput.waitFor({state: "visible", timeout: 10000})
                     await firstInput.click()
                     await page.keyboard.type(otp, {delay})