diff --git a/CHANGELOG.md b/CHANGELOG.md index cb4e205..005dc42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,34 @@ All notable changes to `codesema` (the npm package in `packages/cli`) are documented here. Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Versioning: [SemVer](https://semver.org). +## [0.21.0] - unreleased + +### Added + +- **The pilot grid is now the default workspace shell**, with the previous interface reachable behind a toggle and a mobile layout below 760px. +- **Decision D17: a turn's visual proof of its own effect is now part of the review.** A `microvm` task can declare and capture a screenshot or a Playwright journey each turn (`proof` in `.codesema/config.json`, a `PROOF: ` line in the agent's reply); the reviewer checks the declaration against the diff and raises a finding only on an unproven visible change or an unexplained failed proof. +- **`codesema runbook validate`** validates a hand-edited `.codesema/runbook.json` as-is, without asking an agent for a new proposal. +- **The runbook can declare background services**, started in the verification VM ahead of its healthchecks and tests. +- **A task now carries its own activity**: the phase a turn is currently in is broadcast live and shown on its card and in the evidence block. +- **A turn now emits its recap the moment it ends, and a finished task without one gets it computed on read**, with the microvm verification surfaced alongside it. + +### Changed + +- **A task card stacks its four blocks in one column** (evidence, recap, checks, criteria) instead of a two-by-two grid. +- **Every font size in the web UI is now one of seven `--fs-*` tokens declared once in `style.css`**, in rem, replacing 500 hard-coded px values; the dense sizes moved up (nothing below 11px, read text at 14px), and the tests refuse any new px font size in a component. +- **Every screenshot and video in the evidence block opens a full-screen viewer** with wheel, button and keyboard zoom, click-to-zoom and drag-to-pan, closed by Escape. +- **A widened lane, the full view and the mobile pane now share one chat-shaped thread** (`PilotThread`): the journal reads as a conversation, with the criteria, checks, evidence and recap blocks anchored where the run produced them instead of stacked at the end. +- **Task events hydrate both the card and the expanded view**, instead of only one of the two. +- **Checks hydrate the same way, and task cards gained their own actions.** + +### Fixed + +- **A task thread now opens scrolled to its latest message**, and keeps following new ones until the reader scrolls up. +- **A task card that is not the expanded lane now scrolls** between its header and its footer instead of clipping its blocks. +- **The lens now frames the zoomed block in an opaque panel** with a bounded width and its own scroll, instead of floating it bare over the veil. +- **Claude credentials are now seeded into microvm turns and reviews**, closing the gap where either ran without them. +- **Evidence file URLs are now scoped to their project**, so one project can no longer read another's captures. + ## [0.20.0] - 2026-08-28 ### Added diff --git a/README.md b/README.md index db4f072..effe86f 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,37 @@ From the page you can also: **Merging.** The workspace does not merge on its own unless you ask it to: `mergePolicy` defaults to `human`. Task state lives under `.codesema/tasks//` in the repository it belongs to. +## Visual proof + +With `isolation: "microvm"` and a validated runbook whose services include one that serves the app under test, a task's turn can prove its own effect: the verification VM replays a Playwright journey against the running app and stores the resulting screenshot or video as evidence. The runbook's image needs Playwright available, for example `mcr.microsoft.com/playwright:v1.62.0-noble`. + +Turn it on by adding a `proof` block to `.codesema/config.json`: + +```json +{ + "proof": { + "url": "http://localhost:3000", + "journey": "e2e/checkout.spec.ts" + } +} +``` + +`url` is required; `journey` is optional and names the default Playwright spec a turn replays when it declares `journey` proof without naming its own. + +A hand-edited `.codesema/runbook.json` can be checked without asking an agent for a new proposal: `codesema runbook validate` replays install, services, healthchecks and tests as-is, and only persists the validation once all of them pass. + +Each turn declares its own proof in its final message, one line of the form `PROOF: [pages or spec] | `: + +``` +PROOF: screenshot /dashboard /settings | the new toggle changed the settings layout +PROOF: journey e2e/checkout.spec.ts | the fix touches three screens in sequence +PROOF: none | only the API handler changed, nothing rendered differently +``` + +The task's evidence panel shows the capture (screenshot or video), the agent's stated intent for it, and the reviewer's verdict on whether the declaration matched the diff. Only the last 5 captures of a task are kept; older ones are purged as new ones arrive. + +Known limitation: evidence files are served whole, with no HTTP Range support, so scrubbing through a stored video is degraded to reloading it from the start each time. + ## Runner mode A runner is a background process that connects the workspace to the codesema hub (codesema.com, or your own instance) and works hands-off through its backlog of tickets for a repository: the hub publishes tickets, the runner codes them, ships them, reviews them and reports every transition back. diff --git a/bun.lock b/bun.lock index 664c471..c028b5e 100644 --- a/bun.lock +++ b/bun.lock @@ -528,7 +528,7 @@ "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], - "fast-uri": ["fast-uri@3.1.5", "", {}, "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw=="], + "fast-uri": ["fast-uri@3.1.7", "", {}, "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg=="], "fd-package-json": ["fd-package-json@2.0.0", "", { "dependencies": { "walk-up-path": "^4.0.0" } }, "sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ=="], diff --git a/docs/internals.md b/docs/internals.md index ee6d84d..203c34b 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -110,3 +110,26 @@ What this never does: - **It never blocks a task.** A forge that cannot be reached, an absent `gh`/`glab`, a command that fails: the transition completes exactly as it would with the labels off — same status, same record — and the degradation is stated instead, as a journal line carrying `forge_unreachable`. The task record itself is never modified by this channel. **One asymmetry, documented rather than smoothed over (D8):** the prefix uses a **simple** colon, not GitLab's scoped-label form (`scope::value`). A scoped label would give GitLab forge-side mutual exclusion for free, but GitHub has no such notion, so half the users would get an exclusion the other half would not. The exclusion is therefore computed by codesema, identically on both forges; the consequence on GitLab is that `codesema:` labels are ordinary labels there — no scoped behaviour, no forge-side exclusion, no scoped rendering in its UI. + +## The checks chapter of a review (D16) + +**Decision D16 is settled: a task's checks run once, and that single result becomes a mandatory chapter of the review and fix prompts and the evidence for the mechanical `command` criteria, read once from disk and never re-derived from the diff.** + +## Visual proof (D17) + +**Decision D17 is settled: the visual proof is proportionate, declared by the agent each turn, executed mechanically, judged by the reviewer.** + +A screenshot or a recorded journey costs a turn its own time to capture and a reviewer real attention to read, so it is not owed for every turn: only a turn whose effect is visible earns one, and whether an effect is visible is closer to a judgment call than something a fixed rule could settle. + +**Declaration.** Every turn opens its final message with one line, `PROOF: [pages or spec] | `, placed after the `BRANCH:`/`CRITERION:` lines when those apply. + +**The decision grid the agent follows:** + +- the interface changed and the change is visible: `screenshot` naming the pages, or `journey` naming the spec when the change is a sequence rather than one screen; +- a UI file changed with no visible effect (a refactor with no rendered difference, a prop threaded through with no new output): `none`, with the reason stated; +- nothing outside the interface was touched: `none`; +- doubt: proof, not `none`. The grid resolves a tie toward capturing rather than skipping. + +**Judgment.** The reviewer is handed one mechanical fact, whether UI files were touched, read straight from the diff with no model call, and renders one verdict, `proof_review { expected, coherent, reason }`. Only two shapes of that verdict are a blocking incoherence: a visible change with no proof and no acceptable reason, or a proof that came back failed with nothing said about it. Both are raised as a `design`/`major` finding, anchored on the diff line that made the change visible. A proof supplied when none was owed is never blocking, since over-proving costs nothing the review needs to act on. + +The mechanical fact and the judgment stay on separate sides of the same verdict: what changed is read off the diff, whether the response to it was reasonable is decided by the model. A project with no `proof.url` configured turns every declaration into `skipped`, reason `no_target`, and that is never blocking: there is nothing to replay the proof against. An `undeclared` turn is not the same thing as a declared `none`: skipping the line entirely falls back to the project's own configured default, and that substitution is itself recorded as a finding, non-blocking, rather than silently read as if the agent had chosen `none` itself. diff --git a/docs/pilot-agent.md b/docs/pilot-agent.md new file mode 100644 index 0000000..aaa9e76 --- /dev/null +++ b/docs/pilot-agent.md @@ -0,0 +1,95 @@ +# The pilot agent (D18, open) + +**Status: design note, not implemented, not scheduled.** Written on 2026-09-03 after a +reflection session. Nothing here is settled; the code does not know this agent exists. + +## The idea + +Today the human is the scheduler: they read the backlog, decide an issue is ready, start +a task, come back to see whether it landed in `waiting_for_you`. The runner mode already +removes that loop for hub tickets, but the local workspace still waits for a click. + +The pilot is a single conversational agent that becomes the entry point of the product: +the home page of the local web UI first, later the same interlocutor on Telegram or Slack. +You ask it where things stand, you tell it what to start, it reports back. It never codes +and never replaces the deterministic machinery underneath (queue, concurrency cap, claim, +transitions, cycle labels, fix rounds). It sits on top of it as a remote control. + +## Constraints that make the idea defensible + +- **MCP tools only.** The pilot has no shell, no file access and no network. Its whole + surface is a set of tools exposed by a codesema MCP server, which is the only holder of + the local API token, the hub token and the forge credentials. Removing a power means + removing a tool. +- **Reads are free, writes are confirmed.** Listing projects, tasks, recaps, reviews and + issues needs no approval. Starting a task, replying to one, resuming, stopping, + shipping and merging come back as a proposal the human confirms (a button on the web, + an inline keyboard on Telegram), except for an explicit allowlist in the config such as + "start a task on an issue carrying the ready label". +- **Confirmation is enforced by the host, never by the model and never by MCP tool + annotations.** `readOnlyHint` and `destructiveHint` are untrusted hints in the MCP + specification (2026-07-28). The documented mechanism on the Claude Agent SDK side is + `permissionMode: "default"`, an allowlist of the read tools, and every other call + falling into `canUseTool`. Write tools should also carry + `_meta["anthropic/requiresUserInteraction"]` so a future allowlist cannot skip the + prompt. `dontAsk` is out: it refuses instead of asking. +- **Trust boundary.** The pilot acts on two inputs only: a sentence from the human in the + chat, or a signal a human put on the forge (label, assignee). The body of an issue is + data, never an instruction. Issue content reaches the model only inside `tool_result` + blocks, labelled with its source, and the MCP server sanitizes what it returns. +- **Short runs, no permanent session.** One message is one short agent run fed with a + compact snapshot of the state; the conversation itself is persisted server side as one + thread per workspace so the web and a chat channel continue the same conversation. The + pilot keeps an append-only journal of every action it triggered and why. +- **Isolated like any other agent.** It holds no forge or API secret, only the provider + key, sealed the same way as for the coding agents, and runs as an ephemeral turn. + +## What the home page must keep + +The state grid stays deterministic and instant next to the chat. The chat is for +questions, summaries and orders, not the only way to read the state: otherwise every +"where are we" costs tokens and seconds, and a chat channel would have nothing else. + +## First increment worth building + +The morning triage. It reads every task in `waiting_for_you`, sorts them into three +piles (restartable with a precise instruction, blocked on a question, to abandon), posts +one summary with the unblocking question for each, and acts only after a yes. Success +metric: the share of waiting tasks that restart with no intervention beyond one click. +Then "start whatever is tagged ready" within the existing concurrency cap. + +Not first: a meta-agent that prioritizes the backlog on its own, a hierarchy of agents, +a new piloting UI, a wider auto-merge. + +## What was verified on 2026-09-03 + +- The Claude Agent SDK runs an agent with MCP tools only through `tools: []` + (`disallowedTools: ["*"]` would remove the MCP tools too). Session resume is + documented via `resume` and `sessionStore`. + https://code.claude.com/docs/en/agent-sdk/custom-tools + https://code.claude.com/docs/en/agent-sdk/permissions + https://code.claude.com/docs/en/agent-sdk/session-storage +- The MCP specification forbids token passthrough, requires servers to sanitize outputs, + and says clients should keep a human able to refuse an invocation. Elicitation lets a + server ask the user for input mid call but has no confirmation mode as such. + https://modelcontextprotocol.io/specification/2026-07-28/server/tools + https://modelcontextprotocol.io/docs/2026-07-28/tutorials/security/security_best_practices +- Anthropic's guidance on prompt injection: third party content only in `tool_result`, + source labelled, least privilege, tool output treated as data. + https://platform.claude.com/docs/en/test-and-evaluate/strengthen-guardrails/mitigate-jailbreaks + https://www.anthropic.com/engineering/building-effective-agents +- Market: GitHub Copilot (issue assignment, MCP tools used without approval, review moved + to the PR), Cursor (Slack mention), Claude in Slack, Docker MCP Gateway (one proxy + holding the credentials). None interposes a non coding pilot agent that confirms before + each write. + https://docs.github.com/en/copilot/how-tos/use-copilot-agents/coding-agent/extend-coding-agent-with-mcp + https://docs.docker.com/ai/mcp-gateway/ + +## Open questions + +- The MCP only restriction is verified for the Claude Agent SDK. Whether the pilot can + also run on `opencode` with the same restriction is not verified. +- One shared MCP server with the hub side toolbox, or two implementations of one + contract. +- Telegram identity: a single authorized chat id, everything else ignored; long polling + needs no inbound network. diff --git a/package.json b/package.json index 2a10d4c..7b17e50 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codesema-tools", - "version": "0.20.0", + "version": "0.21.0", "private": true, "type": "module", "workspaces": [ diff --git a/packages/cli/package.json b/packages/cli/package.json index a3a6724..23c5d8d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "codesema", - "version": "0.20.0", + "version": "0.21.0", "description": "Local merge request review, step by step. Your AI agent reviews, codesema displays.", "license": "MIT", "author": "Hasan TASKIN", diff --git a/packages/cli/src/microvm-bootstrap.test.ts b/packages/cli/src/microvm-bootstrap.test.ts new file mode 100644 index 0000000..729608e --- /dev/null +++ b/packages/cli/src/microvm-bootstrap.test.ts @@ -0,0 +1,195 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, test } from 'bun:test' +import type { SandboxExecOptions, SandboxExecResult, SandboxHandle } from './microsandbox-driver.js' +import { + assertValidGuestUser, + ensureAgentCredentials, + ensureAgentInstalled, + ensureGuestUser, +} from './microvm-bootstrap.js' +import { CAGE_HOME_DIR } from './task-isolation.js' + +const cleanups: string[] = [] + +afterEach(() => { + for (const dir of cleanups.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +function makeCredentialsFile(content = '{"token":"secret-token-value"}'): string { + const dir = mkdtempSync(join(tmpdir(), 'codesema-microvm-bootstrap-')) + cleanups.push(dir) + const path = join(dir, 'credentials.json') + writeFileSync(path, content) + return path +} + +type ShellCall = { script: string; opts: SandboxExecOptions } + +function fakeHandle(shellResponder?: (script: string) => Partial | undefined): { + handle: SandboxHandle + shellCalls: ShellCall[] + writeFileCalls: Array<[string, string]> +} { + const shellCalls: ShellCall[] = [] + const writeFileCalls: Array<[string, string]> = [] + const handle: SandboxHandle = { + name: 'fake', + exec: () => { + throw new Error('exec should never be used for the credentials bootstrap') + }, + shell: (script, opts) => { + shellCalls.push({ script, opts }) + return Promise.resolve({ + code: 0, + stdout: '', + stderr: '', + timedOut: false, + ...shellResponder?.(script), + }) + }, + copyFromHost: () => Promise.resolve(), + copyToHost: () => Promise.resolve(), + writeFile: (guestPath, content) => { + writeFileCalls.push([guestPath, content]) + return Promise.resolve() + }, + readFile: () => Promise.resolve(''), + metrics: () => + Promise.resolve({ memoryHostResidentBytes: null, memoryBytes: null, cpuPercent: null }), + stop: () => Promise.resolve(), + } + return { handle, shellCalls, writeFileCalls } +} + +describe('assertValidGuestUser', () => { + test('accepts a useradd-shaped name', () => { + expect(() => assertValidGuestUser('agent')).not.toThrow() + }) + + test('rejects anything that could break out of a spliced shell script', () => { + expect(() => assertValidGuestUser('agent; rm -rf /')).toThrow() + expect(() => assertValidGuestUser('')).toThrow() + expect(() => assertValidGuestUser('Agent')).toThrow() + }) +}) + +describe('ensureGuestUser', () => { + test('creates the user as root, idempotently', async () => { + const { handle, shellCalls } = fakeHandle() + await ensureGuestUser(handle, 'agent') + expect(shellCalls).toHaveLength(1) + expect(shellCalls[0]?.opts.user).toBe('root') + expect(shellCalls[0]?.script).toContain('useradd -m -s /bin/bash agent') + }) + + test('refuses an invalid user before any shell call', async () => { + const { handle, shellCalls } = fakeHandle() + await expect(ensureGuestUser(handle, 'not a user')).rejects.toThrow() + expect(shellCalls).toHaveLength(0) + }) +}) + +describe('ensureAgentInstalled', () => { + test('a found agent never installs', async () => { + const { handle, shellCalls } = fakeHandle(() => ({ code: 0 })) + await ensureAgentInstalled(handle, 'claude', { install: true }) + expect(shellCalls).toHaveLength(1) + expect(shellCalls[0]?.script).toBe('command -v claude') + }) + + test('a missing agent installs on a cold boot', async () => { + const { handle, shellCalls } = fakeHandle((script) => + script === 'command -v claude' ? { code: 1, stderr: 'not found' } : { code: 0 }, + ) + await ensureAgentInstalled(handle, 'claude', { install: true }) + expect(shellCalls).toHaveLength(2) + expect(shellCalls[1]?.opts.user).toBe('root') + }) + + test('a missing agent on a hot boot (snapshot) refuses instead of installing', async () => { + const { handle, shellCalls } = fakeHandle(() => ({ code: 1, stderr: 'not found' })) + await expect(ensureAgentInstalled(handle, 'claude', { install: false })).rejects.toThrow( + /not installed in this microVM/, + ) + expect(shellCalls).toHaveLength(1) + }) +}) + +describe('ensureAgentCredentials', () => { + test('claude, no oauth token, a readable credentials file: written through writeFile, chmod 600, chowned to the guest user, never in a shell command', async () => { + const credentialsPath = makeCredentialsFile('{"token":"secret-token-value"}') + const { handle, shellCalls, writeFileCalls } = fakeHandle() + + await ensureAgentCredentials(handle, 'agent', 'claude', { env: {}, credentialsPath }) + + expect(writeFileCalls).toEqual([ + [`${CAGE_HOME_DIR}/.claude/.credentials.json`, '{"token":"secret-token-value"}'], + ]) + expect(shellCalls.some((c) => c.script.includes(`mkdir -p ${CAGE_HOME_DIR}/.claude`))).toBe( + true, + ) + const chmodChown = shellCalls.find((c) => c.script.includes('chmod 600')) + expect(chmodChown?.script).toBe( + `chmod 600 ${CAGE_HOME_DIR}/.claude/.credentials.json && chown -R agent:agent ${CAGE_HOME_DIR}/.claude`, + ) + expect(chmodChown?.opts.user).toBe('root') + for (const call of shellCalls) { + expect(call.script).not.toContain('secret-token-value') + } + }) + + test('CLAUDE_CODE_OAUTH_TOKEN set: nothing is read from disk, nothing is written', async () => { + const credentialsPath = makeCredentialsFile() + const { handle, shellCalls, writeFileCalls } = fakeHandle() + + await ensureAgentCredentials(handle, 'agent', 'claude', { + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, + credentialsPath, + }) + + expect(writeFileCalls).toHaveLength(0) + expect(shellCalls).toHaveLength(0) + }) + + test('an unreadable or absent credentials file: no write, no error', async () => { + const missingPath = join( + mkdtempSync(join(tmpdir(), 'codesema-microvm-bootstrap-')), + 'nope.json', + ) + cleanups.push(missingPath) + const { handle, shellCalls, writeFileCalls } = fakeHandle() + + await expect( + ensureAgentCredentials(handle, 'agent', 'claude', { env: {}, credentialsPath: missingPath }), + ).resolves.toBeUndefined() + + expect(writeFileCalls).toHaveLength(0) + expect(shellCalls).toHaveLength(0) + }) + + test('a non-claude agent (e.g. opencode) is never touched: its own auth.json is a separate concern', async () => { + const credentialsPath = makeCredentialsFile() + const { handle, shellCalls, writeFileCalls } = fakeHandle() + + await ensureAgentCredentials(handle, 'agent', 'opencode', { env: {}, credentialsPath }) + + expect(writeFileCalls).toHaveLength(0) + expect(shellCalls).toHaveLength(0) + }) + + test('refuses an invalid guest user before touching the file or the sandbox', async () => { + const credentialsPath = makeCredentialsFile() + const { handle, shellCalls, writeFileCalls } = fakeHandle() + + await expect( + ensureAgentCredentials(handle, 'not a user', 'claude', { env: {}, credentialsPath }), + ).rejects.toThrow() + + expect(writeFileCalls).toHaveLength(0) + expect(shellCalls).toHaveLength(0) + }) +}) diff --git a/packages/cli/src/microvm-bootstrap.ts b/packages/cli/src/microvm-bootstrap.ts index a42eb06..4dd1b47 100644 --- a/packages/cli/src/microvm-bootstrap.ts +++ b/packages/cli/src/microvm-bootstrap.ts @@ -6,9 +6,17 @@ * three used to duplicate (or, for the review, simply omit) the same two * steps: create the non-root guest user, make sure the agent binary is on * its PATH. + * + * `ensureAgentCredentials` below is the microVM counterpart of container + * mode's `bootstrapAgentHome` (task-isolation.ts): a container-mode cage + * always gets the host's `~/.claude/.credentials.json` copied in when no + * `CLAUDE_CODE_OAUTH_TOKEN` is forwarded, and now every microVM entry point + * does too: `runMicrovmTurn` (dev turn, and the scan proposal that calls it) + * and `runMicrovmReview` both call it right after `ensureAgentInstalled`. */ +import { readFileSync } from 'node:fs' import type { SandboxHandle } from './microsandbox-driver.js' -import { installCommandFor } from './task-isolation.js' +import { agentCredentialsPath, CAGE_HOME_DIR, installCommandFor } from './task-isolation.js' /** * The only domain a microVM ever needs to install the agent CLI: npm only, @@ -92,3 +100,51 @@ export async function ensureAgentInstalled( throw new Error(`could not install ${agentId} in the microVM: ${tail}`) } } + +export type EnsureAgentCredentialsOptions = { + env?: NodeJS.ProcessEnv + /** Host credentials file; defaults to agentCredentialsPath(agentId, env). Test seam, same doctrine as BootstrapAgentHomeOptions.credentialsPath. */ + credentialsPath?: string +} + +/** + * Seeds the guest's Claude credentials, the microVM counterpart of + * `bootstrapAgentHome` (task-isolation.ts). A no-op for every agent but + * `claude` (opencode's own auth.json is a separate concern), and whenever + * `CLAUDE_CODE_OAUTH_TOKEN` is set (forwarded per run instead) or the host + * credentials file is unreadable, the VM then simply boots unauthenticated, + * same fallback doctrine as the container flow, never a throw. + * + * The content travels through the driver's own guest file writer, never + * through an argv or an interpolated shell command: `ps` on the host, or a + * command logged anywhere along the way, must never carry the secret. + */ +export async function ensureAgentCredentials( + handle: SandboxHandle, + user: string, + agentId: string, + opts: EnsureAgentCredentialsOptions = {}, +): Promise { + if (agentId !== 'claude') { + return + } + const env = opts.env ?? process.env + if (env.CLAUDE_CODE_OAUTH_TOKEN) { + return + } + assertValidGuestUser(user) + let content: string + try { + content = readFileSync(opts.credentialsPath ?? agentCredentialsPath(agentId, env), 'utf8') + } catch { + return + } + const destDir = `${CAGE_HOME_DIR}/.claude` + const destFile = `${destDir}/.credentials.json` + await handle.shell(`mkdir -p ${destDir}`, { timeoutMs: BOOTSTRAP_TIMEOUT_MS, user: 'root' }) + await handle.writeFile(destFile, content) + await handle.shell(`chmod 600 ${destFile} && chown -R ${user}:${user} ${destDir}`, { + timeoutMs: BOOTSTRAP_TIMEOUT_MS, + user: 'root', + }) +} diff --git a/packages/cli/src/microvm-turn.test.ts b/packages/cli/src/microvm-turn.test.ts index 188ba51..5b7878b 100644 --- a/packages/cli/src/microvm-turn.test.ts +++ b/packages/cli/src/microvm-turn.test.ts @@ -339,6 +339,79 @@ describe('runMicrovmTurn: sandbox spec', () => { }) }) + describe('agent credentials', () => { + function makeCredentialsFile(content = '{"token":"secret-token-value"}'): string { + const dir = makeDir('codesema-microvm-creds-') + const path = join(dir, 'credentials.json') + writeFileSync(path, content) + return path + } + + test('no oauth token, credentials file present: written through writeFile with the right permissions, never in a shell command', async () => { + const credentialsPath = makeCredentialsFile('{"token":"secret-token-value"}') + const { opts, rig } = baseOptions({ env: {}, credentialsPath }) + const promise = runMicrovmTurn(opts) + await flush() + + expect(rig.writeFileCalls).toHaveLength(1) + expect(rig.writeFileCalls[0]?.[0]).toBe('/home/agent/.claude/.credentials.json') + expect(rig.writeFileCalls[0]?.[1]).toBe('{"token":"secret-token-value"}') + + const chmodChown = rig.shellCalls.find((c) => c.script.includes('chmod 600')) + expect(chmodChown?.script).toBe( + 'chmod 600 /home/agent/.claude/.credentials.json && chown -R agent:agent /home/agent/.claude', + ) + expect(chmodChown?.opts.user).toBe('root') + expect(rig.shellCalls.some((c) => c.script.includes('mkdir -p /home/agent/.claude'))).toBe( + true, + ) + + for (const call of rig.shellCalls) { + expect(call.script).not.toContain('secret-token-value') + } + + rig.resolveTurn({ stdout: 'ok' }) + await promise + }) + + test('CLAUDE_CODE_OAUTH_TOKEN present: nothing is copied', async () => { + const credentialsPath = makeCredentialsFile() + const { opts, rig } = baseOptions({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, + credentialsPath, + }) + const promise = runMicrovmTurn(opts) + await flush() + expect(rig.writeFileCalls).toHaveLength(0) + rig.resolveTurn({ stdout: 'ok' }) + await promise + }) + + test('no credentials file on the host: no write, no error, the turn still runs', async () => { + const missingPath = join(makeDir('codesema-microvm-creds-'), 'nope.json') + const { opts, rig } = baseOptions({ env: {}, credentialsPath: missingPath }) + const promise = runMicrovmTurn(opts) + await flush() + expect(rig.writeFileCalls).toHaveLength(0) + rig.resolveTurn({ stdout: 'ok' }) + await expect(promise).resolves.toBe('ok') + }) + + test('a non-claude agent never gets its credentials copied', async () => { + const credentialsPath = makeCredentialsFile() + const { opts, rig } = baseOptions({ + command: 'opencode run --model x', + env: {}, + credentialsPath, + }) + const promise = runMicrovmTurn(opts) + await flush() + expect(rig.writeFileCalls).toHaveLength(0) + rig.resolveTurn({ stdout: 'ok' }) + await promise + }) + }) + test('a validated runbook joins its egress to the allowlist for the whole turn', async () => { const runbook: RunbookConfig = { version: 1, diff --git a/packages/cli/src/microvm-turn.ts b/packages/cli/src/microvm-turn.ts index 82fd0bc..0de3c85 100644 --- a/packages/cli/src/microvm-turn.ts +++ b/packages/cli/src/microvm-turn.ts @@ -47,6 +47,7 @@ import { import { AGENT_INSTALL_DOMAINS, assertValidGuestUser, + ensureAgentCredentials, ensureAgentInstalled, ensureGuestUser, } from './microvm-bootstrap.js' @@ -74,6 +75,8 @@ export type RunMicrovmTurnOptions = Omit< secrets: readonly SandboxSecret[] /** Guest user the agent runs as (never root: `--dangerously-skip-permissions` refuses root). */ user?: string + /** Test seam: overrides ensureAgentCredentials' host credentials file. */ + credentialsPath?: string } export const MICROVM_TURN_DEFAULTS = { @@ -375,6 +378,10 @@ export async function runMicrovmTurn(opts: RunMicrovmTurnOptions): Promise { expect(() => writeChecksConfig(repoDir, checks)).toThrow(/not a JSON object/) }) }) + +describe('readProofConfig', () => { + let repoDir: string + + beforeEach(() => { + repoDir = mkdtempSync(join(tmpdir(), 'codesema-proofcfg-')) + }) + + afterEach(() => { + rmSync(repoDir, { recursive: true, force: true }) + }) + + const writeRepoConfig = (content: string) => { + mkdirSync(join(repoDir, '.codesema'), { recursive: true }) + writeFileSync(join(repoDir, '.codesema', 'config.json'), content) + } + + test('missing file, invalid json or absent proof key: null', () => { + expect(readProofConfig(repoDir)).toBeNull() + writeRepoConfig('{ not json') + expect(readProofConfig(repoDir)).toBeNull() + writeRepoConfig(JSON.stringify({ agent: 'claude -p' })) + expect(readProofConfig(repoDir)).toBeNull() + writeRepoConfig(JSON.stringify({ proof: 'yes' })) + expect(readProofConfig(repoDir)).toBeNull() + writeRepoConfig(JSON.stringify({ proof: ['npm test'] })) + expect(readProofConfig(repoDir)).toBeNull() + }) + + test('journey missing: journey is null, the record is still valid', () => { + writeRepoConfig(JSON.stringify({ proof: { url: 'http://localhost:3000' } })) + expect(readProofConfig(repoDir)).toEqual({ + journey: null, + url: 'http://localhost:3000', + timeoutSeconds: null, + keep: null, + }) + }) + + test('url missing: the whole record is null', () => { + writeRepoConfig(JSON.stringify({ proof: { journey: 'checkout' } })) + expect(readProofConfig(repoDir)).toBeNull() + }) + + test('out-of-range timeoutSeconds and keep fall back to null, not the record', () => { + writeRepoConfig( + JSON.stringify({ + proof: { + journey: 'checkout', + url: 'http://localhost:3000', + timeoutSeconds: -5, + keep: 0, + }, + }), + ) + expect(readProofConfig(repoDir)).toEqual({ + journey: 'checkout', + url: 'http://localhost:3000', + timeoutSeconds: null, + keep: null, + }) + }) + + test('a full proof block is read back field by field', () => { + writeRepoConfig( + JSON.stringify({ + proof: { + journey: 'checkout', + url: 'http://localhost:3000', + timeoutSeconds: 120, + keep: 5, + }, + }), + ) + expect(readProofConfig(repoDir)).toEqual({ + journey: 'checkout', + url: 'http://localhost:3000', + timeoutSeconds: 120, + keep: 5, + }) + }) + + test('keep is capped at 20', () => { + writeRepoConfig( + JSON.stringify({ + proof: { journey: 'checkout', url: 'http://localhost:3000', keep: 999 }, + }), + ) + expect(readProofConfig(repoDir)?.keep).toBe(20) + }) +}) diff --git a/packages/cli/src/repo-config.ts b/packages/cli/src/repo-config.ts index 9863f4c..ef71e21 100644 --- a/packages/cli/src/repo-config.ts +++ b/packages/cli/src/repo-config.ts @@ -95,6 +95,56 @@ export function readChecksConfig(repoRoot: string): ChecksConfig | null { } } +export type ProofConfig = { + /** The repo's default replay spec (D17); null when the project has none configured yet. */ + journey: string | null + url: string + timeoutSeconds: number | null + keep: number | null +} + +const PROOF_STRING_MAX = 500 +const PROOF_KEEP_MAX = 20 + +/** + * Reads the repo's `proof` key. Same raw re-read pattern as readChecksConfig + * (config.ts's parseConfig whitelists its own fields). Only `url` is + * mandatory (D17: the agent declares and names its own proof per turn, so a + * standing journey spec is a convenience, not a requirement): a missing or + * invalid `url` degrades the whole record to null, since there is nothing to + * replay a proof against without one. + */ +export function readProofConfig(repoRoot: string): ProofConfig | null { + const path = repoConfigPath(repoRoot) + let raw: unknown + try { + raw = JSON.parse(readFileSync(path, 'utf8')) + } catch { + return null + } + const proof = (raw as { proof?: unknown } | null)?.proof + if (!proof || typeof proof !== 'object' || Array.isArray(proof)) { + return null + } + const p = proof as Record + const str = (v: unknown): string | undefined => + typeof v === 'string' && v.trim() ? v.trim().slice(0, PROOF_STRING_MAX) : undefined + const journey = str(p.journey) + const url = str(p.url) + if (url === undefined) { + return null + } + const timeoutSeconds = + Number.isInteger(p.timeoutSeconds) && (p.timeoutSeconds as number) > 0 + ? (p.timeoutSeconds as number) + : null + const keep = + Number.isInteger(p.keep) && (p.keep as number) > 0 + ? Math.min(p.keep as number, PROOF_KEEP_MAX) + : null + return { journey: journey ?? null, url, timeoutSeconds, keep } +} + /** * Writes ONLY the `checks` key of .codesema/config.json, keeping every other * key (agent, port, language...) byte-for-byte in place — this is a diff --git a/packages/cli/src/review.test.ts b/packages/cli/src/review.test.ts index f333617..07de341 100644 --- a/packages/cli/src/review.test.ts +++ b/packages/cli/src/review.test.ts @@ -583,6 +583,68 @@ describe('runDualFlow', () => { expect(outcome.record.review.criteria).toEqual([{ criterion_id: CRITERION, status: 'unmet' }]) }, 30000) + // D17: proof_review is arbitrated the same pessimistic way as criteria, + // right beside them, and would otherwise be dropped on the floor by + // `assembleDualReview` (findings only). + const laneReviewWithProof = (proofReview: string): string => + `{"verdict":"approve","summary":"ok","findings":[],"proof_review":${proofReview}}` + + test('proof_review fuses pessimistically: an incoherent lane wins over a coherent one', async () => { + const fixture = setupDualRepo(REVIEW) + twoLaneAgent( + fixture, + laneReviewWithProof('{"expected":"none","coherent":true,"reason":"nothing visible changed"}'), + laneReviewWithProof( + '{"expected":"screenshot","coherent":false,"reason":"a visible change has no proof"}', + ), + ) + + const outcome = await runDualFlow(flowOpts(fixture)) + + expect(outcome.ok).toBe(true) + if (!outcome.ok) { + return + } + expect(outcome.record.review.proof_review).toEqual({ + expected: 'screenshot', + coherent: false, + reason: 'a visible change has no proof', + }) + }, 30000) + + test('proof_review fuses to lane A when both lanes agree it is coherent', async () => { + const fixture = setupDualRepo(REVIEW) + twoLaneAgent( + fixture, + laneReviewWithProof('{"expected":"none","coherent":true,"reason":"lane a reason"}'), + laneReviewWithProof('{"expected":"none","coherent":true,"reason":"lane b reason"}'), + ) + + const outcome = await runDualFlow(flowOpts(fixture)) + + expect(outcome.ok).toBe(true) + if (!outcome.ok) { + return + } + expect(outcome.record.review.proof_review).toEqual({ + expected: 'none', + coherent: true, + reason: 'lane a reason', + }) + }, 30000) + + test('neither lane declaring a proof_review leaves the record with no such key', async () => { + const fixture = setupDualRepo(REVIEW) + + const outcome = await runDualFlow(flowOpts(fixture)) + + expect(outcome.ok).toBe(true) + if (!outcome.ok) { + return + } + expect(outcome.record.review.proof_review).toBeUndefined() + }, 20000) + test('an aborted signal cuts the dual LANES, not just the simple flow', async () => { const fixture = setupDualRepo(REVIEW) // Lanes that would hold the review for a full minute if left alone. diff --git a/packages/cli/src/review.ts b/packages/cli/src/review.ts index d3b9883..3c12d12 100644 --- a/packages/cli/src/review.ts +++ b/packages/cli/src/review.ts @@ -20,6 +20,7 @@ import { sanitizeReview, type FindingSeverity, type GroundingReport, + type ProofReview, type ReviewedFile, type ReviewRecord, type SanitizedReview, @@ -572,6 +573,27 @@ export async function runSimpleFlow(opts: { } } +/** + * D17, dual mode: the two lanes' `proof_review` folded into one, pessimistic + * the same way `mergeCriterionVerdicts` is: a lane that found the + * declaration incoherent is never overridden by a lane that shrugged or + * approved it, whichever lane it was. Neither lane carrying one (a task with + * no proof chapter) returns undefined, so a dual review without one keeps + * writing a record with no `proof_review` key at all. + */ +function mergeProofReview( + a: ProofReview | undefined, + b: ProofReview | undefined, +): ProofReview | undefined { + if (a && !a.coherent) { + return a + } + if (b && !b.coherent) { + return b + } + return a ?? b +} + export type DualOutcome = | { ok: true; record: ReviewRecord; reportLines: string[] } | { ok: false; failure: 'run' | 'output'; message: string; rawOutput?: string } @@ -748,8 +770,20 @@ export async function runDualFlow(opts: { groundedA.review.criteria, groundedB.review.criteria, ) + // D17: `proof_review` is arbitrated the same way, pessimistically, right + // beside the criteria it is merged alongside: `assembleDualReview` only + // ever arbitrates findings, so this too would be dropped on the floor + // otherwise. + const mergedProofReview = mergeProofReview( + groundedA.review.proof_review, + groundedB.review.proof_review, + ) const final = groundReview( - mergedCriteria.length > 0 ? { ...assembly.review, criteria: mergedCriteria } : assembly.review, + { + ...assembly.review, + ...(mergedCriteria.length > 0 ? { criteria: mergedCriteria } : {}), + ...(mergedProofReview ? { proof_review: mergedProofReview } : {}), + }, input.diff, ) const resolved = buildRecord(final.review) diff --git a/packages/cli/src/runbook-runner.test.ts b/packages/cli/src/runbook-runner.test.ts index 1e40fde..6d4f6df 100644 --- a/packages/cli/src/runbook-runner.test.ts +++ b/packages/cli/src/runbook-runner.test.ts @@ -15,10 +15,12 @@ import { RUNBOOK_SCAN_MAX_ATTEMPTS, runOneRunbookScan, runRunbookScan, + runRunbookValidate, type RunbookScanOutcome, type RunRunbookScanOptions, + type RunRunbookValidateOptions, } from './runbook-runner.js' -import { sanitizeRunbookProposal, type RunbookProposalInput } from './runbook-setup.js' +import { runbookSha, sanitizeRunbookProposal, type RunbookProposalInput } from './runbook-setup.js' // --------------------------------------------------------------------------- // Fakes: a scripted SandboxDriver/SandboxHandle. NEVER touches a real VM. @@ -256,6 +258,102 @@ describe('runRunbookScan — happy path', () => { }) }) +describe('runRunbookValidate', () => { + function baseValidateOptions( + overrides: Partial = {}, + ): RunRunbookValidateOptions { + const { driver } = fakeDriver() + return { + worktree: '/repo', + projectId: 'proj1', + headSha: 'a'.repeat(40), + driver, + timeoutMs: 5_000, + agentId: 'claude', + readRunbookConfigFn: () => sampleRunbook(), + writeRunbookValidationFn: () => {}, + buildProjectSnapshotFn: async () => + ({ kind: 'ready', name: 'snap1', hash: 'h1' }) as ProjectSnapshot, + sleepFn: async () => {}, + ...overrides, + } + } + + test('no runbook.json on disk: failed immediately, attempts 0, the driver is never called', async () => { + let createCalled = false + const { driver } = fakeDriver() + const guardedDriver: SandboxDriver = { + ...driver, + create: async (spec) => { + createCalled = true + return driver.create(spec) + }, + } + const outcome = await runRunbookValidate( + baseValidateOptions({ driver: guardedDriver, readRunbookConfigFn: () => null }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status !== 'failed') { + throw new Error('unreachable') + } + expect(outcome.attempts).toBe(0) + expect(outcome.error).toContain('runbook scan') + expect(createCalled).toBe(false) + }) + + test('a green execution validates the runbook as read from disk: runbook_sha matches runbookSha(runbook)', async () => { + const { driver, calls } = fakeDriver({ + respond: scriptedRespond({ + 'bun install': okResult('installed'), + 'bun test': okResult('1 pass'), + }), + }) + const runbook = sampleRunbook() + const writeValidationCalls: { worktree: string; validation: unknown }[] = [] + const outcome = await runRunbookValidate( + baseValidateOptions({ + driver, + readRunbookConfigFn: () => runbook, + writeRunbookValidationFn: (worktree, validation) => { + writeValidationCalls.push({ worktree, validation }) + }, + }), + ) + expect(outcome.status).toBe('completed') + if (outcome.status !== 'completed') { + throw new Error('unreachable') + } + expect(outcome.attempts).toBe(1) + expect(outcome.runbook).toEqual(runbook) + expect(outcome.validation.runbook_sha).toBe(runbookSha(runbook)) + expect(outcome.validation.validated_sha).toBe('a'.repeat(40)) + expect(writeValidationCalls).toEqual([{ worktree: '/repo', validation: outcome.validation }]) + expect(calls.map((c) => c.command)).toContain('bun install') + expect(calls.map((c) => c.command)).toContain('bun test') + }) + + test('a failing execution reports failed with the tail, and never writes a validation', async () => { + const { driver } = fakeDriver({ + respond: scriptedRespond({ 'bun install': failResult(1, '', 'boom') }), + }) + let validationWritten = false + const outcome = await runRunbookValidate( + baseValidateOptions({ + driver, + writeRunbookValidationFn: () => { + validationWritten = true + }, + }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') { + expect(outcome.attempts).toBe(1) + expect(outcome.lastTail).toContain('boom') + } + expect(validationWritten).toBe(false) + }) +}) + describe('runRunbookScan — real sanitizeRunbookProposal (regression)', () => { // `sanitizeRunbookProposal` extracts JSON from raw agent TEXT itself // (runbook-setup.ts); passing it an already-parsed value makes it reject diff --git a/packages/cli/src/runbook-runner.ts b/packages/cli/src/runbook-runner.ts index c972d98..ab49b96 100644 --- a/packages/cli/src/runbook-runner.ts +++ b/packages/cli/src/runbook-runner.ts @@ -30,8 +30,12 @@ import { } from './microsandbox-driver.js' import { buildProjectSnapshot, type ProjectSnapshot } from './microvm-snapshot.js' import { MICROVM_TURN_DEFAULTS, runMicrovmTurn } from './microvm-turn.js' +import { SERVICE_LAUNCH_TIMEOUT_MS, serviceLaunchScript } from './runbook-services.js' import { buildRunbookSetupPrompt, + runbookSha as computeRunbookSha, + readRunbookConfig, + RUNBOOK_FILE, sanitizeRunbookProposal, writeRunbookConfig, writeRunbookValidation, @@ -59,9 +63,6 @@ export const RUNBOOK_SCAN_LEASE_SECONDS = 900 /** How often a healthcheck is retried before the attempt's shared deadline. */ export const RUNBOOK_HEALTHCHECK_RETRY_MS = 2_000 -/** Wall-clock budget for launching one background service (the launcher itself, not the service). */ -const SERVICE_LAUNCH_TIMEOUT_MS = 15_000 - /** Absolute ceiling on a scan's own VM lease, whatever the runbook's command count computes to. */ const EXECUTION_MAX_DURATION_SECONDS_CAP = 6 * 3600 @@ -101,11 +102,6 @@ function defaultRenewSleep(ms: number, signal: AbortSignal): Promise { }) } -/** Single-quotes a value for `sh -c '...'`, escaping any embedded single quote. */ -function shellSingleQuote(value: string): string { - return `'${value.replace(/'/g, `'\\''`)}'` -} - function toCheckResult( command: string, result: SandboxExecResult, @@ -197,6 +193,21 @@ export type RunRunbookScanOptions = { sleepFn?: (ms: number) => Promise } +export type RunRunbookValidateOptions = Omit< + RunRunbookScanOptions, + | 'command' + | 'secrets' + | 'allowedDomains' + | 'buildPromptFn' + | 'sanitizeProposalFn' + | 'runProposalFn' +> & { + /** Fingerprint the snapshot is built and installed under; matches the scan's own `commandBin(command) || 'claude'`. */ + agentId: string + /** Test seam: never a real file read in a test. */ + readRunbookConfigFn?: (worktree: string) => RunbookConfig | null +} + type ExecuteRunbookResult = { ok: true; checks: TaskCheckResult[] } | { ok: false; tail: string; checks: TaskCheckResult[] } @@ -296,7 +307,7 @@ async function executeRunbook(input: ExecuteRunbookInput): Promise /tmp/codesema-service-${i}.log 2>&1 &` + const script = serviceLaunchScript(command, i) const startedAt = Date.now() const result = await handle.shell(script, { cwd: MICROVM_TURN_DEFAULTS.workDir, @@ -348,6 +359,79 @@ async function executeRunbook(input: ExecuteRunbookInput): Promise void + buildSnapshot: typeof buildProjectSnapshot + writeValidation: typeof writeRunbookValidation +} + +/** + * Shared tail of a green execution, for both the scan (after a fresh proposal + * passes) and validate (after the existing runbook passes as-is): builds the + * project snapshot, writes the local validation record, returns the outcome. + */ +async function finalizeValidatedRunbook( + input: FinalizeValidatedRunbookInput, +): Promise { + const { + driver, + projectId, + worktree, + headSha, + runbook, + runbookSha, + agentId, + timeoutMs, + checks, + attempts, + onProgress, + buildSnapshot, + writeValidation, + } = input + let snapshotName: string | null = null + try { + const snapshot = await buildSnapshot({ + driver, + projectId, + worktree, + runbook, + agentId, + timeoutMs, + ...(onProgress ? { onProgress } : {}), + }) + snapshotName = snapshot.kind === 'cold' ? null : snapshot.name + } catch (err) { + onProgress?.( + `snapshot build failed (keeping the validated runbook, cold boots only): ${errorMessage(err)}`, + ) + snapshotName = null + } + + const validation: RunbookValidation = { + runbook_sha: runbookSha, + validated_sha: headSha, + validated_at: new Date().toISOString(), + status: 'valid', + } + // Written locally at the project root, alongside RUNBOOK_FILE: the + // mechanical verification (task-server.ts's verifyAfterCommit) reads it + // back to find the sha this runbook was validated against, rather than + // walking git history for a commit that touched RUNBOOK_FILE: that file + // is gitignored and never committed, so no such commit ever exists. + writeValidation(worktree, validation) + return { status: 'completed', runbook, validation, snapshotName, checks, attempts } +} + export async function runRunbookScan(opts: RunRunbookScanOptions): Promise { const collectFiles = opts.collectSetupFilesFn ?? collectSetupFiles const buildPrompt = opts.buildPromptFn ?? buildRunbookSetupPrompt @@ -437,45 +521,21 @@ export async function runRunbookScan(opts: RunRunbookScanOptions): Promise { + const readConfig = opts.readRunbookConfigFn ?? readRunbookConfig + const writeValidation = opts.writeRunbookValidationFn ?? writeRunbookValidation + const buildSnapshot = opts.buildProjectSnapshotFn ?? buildProjectSnapshot + const sleepFn = opts.sleepFn ?? defaultSleep + + const runbook = readConfig(opts.worktree) + if (!runbook) { + return { + status: 'failed', + error: `no runbook found at ${RUNBOOK_FILE}, run \`codesema runbook scan\` first`, + attempts: 0, + lastTail: null, + } + } + + opts.onProgress?.('runbook validate: executing') + let executed: ExecuteRunbookResult + try { + executed = await executeRunbook({ + driver: opts.driver, + runbook, + worktree: opts.worktree, + projectId: opts.projectId, + attempt: 1, + timeoutMs: opts.timeoutMs, + sleepFn, + ...(opts.onProgress ? { onProgress: opts.onProgress } : {}), + ...(opts.signal ? { signal: opts.signal } : {}), + }) + } catch (err) { + executed = { ok: false, tail: tailOf(errorMessage(err)), checks: [] } + } + if (!executed.ok) { + return { status: 'failed', error: executed.tail, attempts: 1, lastTail: executed.tail } + } + + opts.onProgress?.('runbook validate: green') + return finalizeValidatedRunbook({ + driver: opts.driver, + projectId: opts.projectId, + worktree: opts.worktree, + headSha: opts.headSha, + runbook, + runbookSha: computeRunbookSha(runbook), + agentId: opts.agentId, + timeoutMs: opts.timeoutMs, + checks: executed.checks, + attempts: 1, + ...(opts.onProgress ? { onProgress: opts.onProgress } : {}), + buildSnapshot, + writeValidation, + }) +} + export type RunbookScanRunnerOptions = { creds: SyncCredentials driver: SandboxDriver diff --git a/packages/cli/src/runbook-services.test.ts b/packages/cli/src/runbook-services.test.ts new file mode 100644 index 0000000..c365ff3 --- /dev/null +++ b/packages/cli/src/runbook-services.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, test } from 'bun:test' +import { serviceLaunchScript, shellSingleQuote } from './runbook-services.js' + +describe('shellSingleQuote', () => { + test('wraps a plain value in single quotes', () => { + expect(shellSingleQuote('dockerd')).toBe("'dockerd'") + }) + + test('escapes an embedded single quote', () => { + expect(shellSingleQuote("it's")).toBe("'it'\\''s'") + }) +}) + +describe('serviceLaunchScript', () => { + test('index 0 backgrounds the command into /tmp/codesema-service-0.log', () => { + expect(serviceLaunchScript('dockerd', 0)).toBe( + "nohup sh -c 'dockerd' > /tmp/codesema-service-0.log 2>&1 &", + ) + }) + + test('index 3 backgrounds the command into /tmp/codesema-service-3.log', () => { + expect(serviceLaunchScript('sleep 1', 3)).toBe( + "nohup sh -c 'sleep 1' > /tmp/codesema-service-3.log 2>&1 &", + ) + }) +}) diff --git a/packages/cli/src/runbook-services.ts b/packages/cli/src/runbook-services.ts new file mode 100644 index 0000000..898b9c5 --- /dev/null +++ b/packages/cli/src/runbook-services.ts @@ -0,0 +1,20 @@ +/** + * The background service launcher shared by the runbook scan + * (runbook-runner.ts) and the mechanical verification (task-verification.ts): + * a `nohup ... &` script so `services.host_up` commands never block the + * caller's shell call the way a foreground `handle.shell` would for a + * long-running server. + */ + +/** Wall-clock budget for launching one background service (the launcher itself, not the service). */ +export const SERVICE_LAUNCH_TIMEOUT_MS = 15_000 + +/** Single-quotes a value for `sh -c '...'`, escaping any embedded single quote. */ +export function shellSingleQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'` +} + +/** The `nohup sh -c '' > /tmp/codesema-service-.log 2>&1 &` script for one `services.host_up` entry. */ +export function serviceLaunchScript(command: string, index: number): string { + return `nohup sh -c ${shellSingleQuote(command)} > /tmp/codesema-service-${index}.log 2>&1 &` +} diff --git a/packages/cli/src/runner-commands.test.ts b/packages/cli/src/runner-commands.test.ts index 020b1d0..f1af02a 100644 --- a/packages/cli/src/runner-commands.test.ts +++ b/packages/cli/src/runner-commands.test.ts @@ -746,8 +746,13 @@ describe('runnerCommand', () => { ).rejects.toThrow() }) - describe('connected, non-interactive (no TTY in this test environment)', () => { + describe('connected, non-interactive (stdin and stdout forced off the TTY)', () => { + const previousStdinIsTTY = process.stdin.isTTY + const previousStdoutIsTTY = process.stdout.isTTY + beforeEach(() => { + process.stdin.isTTY = false + process.stdout.isTTY = false saveGlobalConfig({ ...loadGlobalConfig(), syncUrl: 'https://hub.example', @@ -756,6 +761,11 @@ describe('runnerCommand', () => { }) }) + afterEach(() => { + process.stdin.isTTY = previousStdinIsTTY + process.stdout.isTTY = previousStdoutIsTTY + }) + test('without --fingerprint or a token flag, refuses immediately and lists what is missing', async () => { await expect( runnerCommand({ action: 'autoconfig', cwd, runInheritedFn: () => {} }), @@ -1483,4 +1493,112 @@ describe('runbookCommand', () => { }), ).resolves.toBeUndefined() }) + + test('usage lists both scan and validate', async () => { + const lines = await captureLog(() => runbookCommand({ cwd })) + expect(lines.some((l) => l.includes('runbook scan|validate'))).toBe(true) + }) + + test('validate outside a git repository throws', async () => { + await expect( + runbookCommand({ + action: 'validate', + cwd, + agent: 'claude -p', + runRunbookValidateFn: async () => completedOutcome(), + }), + ).rejects.toThrow(/not a git repository/) + }) + + test('validate routes to runRunbookValidateFn with the resolved head sha, a stable projectId, timeoutMs and an agentId', async () => { + initRepo(cwd) + const seen: { + headSha: string + projectId: string + timeoutMs: number + agentId: string + } = { headSha: '', projectId: '', timeoutMs: 0, agentId: '' } + await runbookCommand({ + action: 'validate', + cwd, + agent: 'claude -p', + timeoutSeconds: 42, + runRunbookValidateFn: async (opts) => { + seen.headSha = opts.headSha + seen.projectId = opts.projectId + seen.timeoutMs = opts.timeoutMs + seen.agentId = opts.agentId + return completedOutcome() + }, + }) + expect(seen.headSha).toMatch(/^[0-9a-f]{40}$/) + expect(seen.projectId).toMatch(/^[0-9a-f]{16}$/) + expect(seen.timeoutMs).toBe(42_000) + expect(seen.agentId).toBe('claude') + }) + + test('validate and scan resolve to the same projectId for the same worktree', async () => { + initRepo(cwd) + const seen: string[] = [] + await runbookCommand({ + action: 'scan', + cwd, + agent: 'claude -p', + runRunbookScanFn: async (opts) => { + seen.push(opts.projectId) + return completedOutcome() + }, + }) + await runbookCommand({ + action: 'validate', + cwd, + agent: 'claude -p', + runRunbookValidateFn: async (opts) => { + seen.push(opts.projectId) + return completedOutcome() + }, + }) + expect(seen).toHaveLength(2) + expect(seen[0]).toBe(seen[1]) + }) + + test('a completed validate prints the runbook summary', async () => { + initRepo(cwd) + const lines = await captureLog(() => + runbookCommand({ + action: 'validate', + cwd, + agent: 'claude -p', + runRunbookValidateFn: async () => completedOutcome(), + }), + ) + const joined = lines.join('\n') + expect(joined).toContain('Runbook validated') + expect(joined).toContain(RUNBOOK_FILE) + }) + + test('a failed validate prints the error and sets a non-zero exit code', async () => { + initRepo(cwd) + const previousExitCode = process.exitCode + process.exitCode = undefined + try { + const lines = await captureLog(() => + runbookCommand({ + action: 'validate', + cwd, + agent: 'claude -p', + runRunbookValidateFn: async () => ({ + status: 'failed', + error: 'no runbook found', + attempts: 0, + lastTail: null, + }), + }), + ) + expect(lines.join('\n')).toContain('no runbook found') + expect(process.exitCode as unknown).toBe(1) + } finally { + process.exitCode = previousExitCode ?? 0 + } + }) }) diff --git a/packages/cli/src/runner-commands.ts b/packages/cli/src/runner-commands.ts index 5a6a97f..ee7e20d 100644 --- a/packages/cli/src/runner-commands.ts +++ b/packages/cli/src/runner-commands.ts @@ -38,6 +38,7 @@ import { createMicrosandboxDriver, type SandboxDriver } from './microsandbox-dri import { DEFAULT_RUNBOOK_SCAN_TIMEOUT_MS, runRunbookScan, + runRunbookValidate, type RunbookScanOutcome, } from './runbook-runner.js' import { RUNBOOK_FILE } from './runbook-setup.js' @@ -56,7 +57,7 @@ import { } from './runner-service.js' import { formatFingerprint, runnerKeyFingerprint, seal, unseal } from './sealed-box.js' import { loadSyncCredentials } from './sync.js' -import { microvmSecretsFromEnv } from './task-isolation.js' +import { commandBin, microvmSecretsFromEnv } from './task-isolation.js' import { draftAndPublishTicket } from './ticket-draft.js' import { confirm, isInteractive, select, textInput, type SelectOption } from './tui.js' import { ACCENT, AMBER, dim, GREEN, paint, RED, renderFieldRows, type FieldRow } from './ui.js' @@ -1003,6 +1004,8 @@ export type RunbookCommandOptions = { driver?: SandboxDriver | undefined /** Test seam. */ runRunbookScanFn?: typeof runRunbookScan | undefined + /** Test seam. */ + runRunbookValidateFn?: typeof runRunbookValidate | undefined } /** sha256 (16 hex) of the worktree's absolute path: a stable local id when there is no hub project to name one. */ @@ -1081,13 +1084,42 @@ async function runbookScanCommand(opts: RunbookCommandOptions): Promise { printRunbookScanOutcome(outcome) } +async function runbookValidateCommand(opts: RunbookCommandOptions): Promise { + const worktree = opts.cwd + const headSha = tryGit(['rev-parse', 'HEAD'], worktree) + if (!headSha) { + throw new Error('not a git repository (or no commits yet)') + } + const command = await resolveRunbookScanCommand(worktree, opts.agent) + const driver = opts.driver ?? createMicrosandboxDriver() + const run = opts.runRunbookValidateFn ?? runRunbookValidate + const timeoutMs = + (opts.timeoutSeconds ?? Math.round(DEFAULT_RUNBOOK_SCAN_TIMEOUT_MS / 1000)) * 1000 + + console.log('') + console.log(` ${dim('validating the runbook…')}`) + const outcome = await run({ + worktree, + projectId: localProjectId(worktree), + headSha, + driver, + agentId: commandBin(command) || 'claude', + timeoutMs, + onProgress: (line) => console.log(` ${dim(line)}`), + }) + printRunbookScanOutcome(outcome) +} + export async function runbookCommand(opts: RunbookCommandOptions): Promise { switch (opts.action) { case 'scan': await runbookScanCommand(opts) return + case 'validate': + await runbookValidateCommand(opts) + return case undefined: - console.log('usage: codesema runbook scan [--timeout ]') + console.log('usage: codesema runbook scan|validate [--timeout ]') return default: throw new Error(`unknown runbook action: ${opts.action}`) diff --git a/packages/cli/src/serve.test.ts b/packages/cli/src/serve.test.ts index 28dd4fc..7f1f89a 100644 --- a/packages/cli/src/serve.test.ts +++ b/packages/cli/src/serve.test.ts @@ -1,5 +1,6 @@ import { execFileSync } from 'node:child_process' import { + mkdirSync, mkdtempSync, readdirSync, readFileSync, @@ -12,7 +13,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { afterAll, beforeAll, describe, expect, test } from 'bun:test' -import { sanitizeRecord, type ReviewRecord } from './contract.js' +import { sanitizeRecord, type RecapRecord, type ReviewRecord } from './contract.js' import type { ForgeMrsResult } from './forge-mrs.js' import type { MrReviewMode, @@ -36,6 +37,9 @@ import { type LiveSession, type SessionEvent, } from './serve.js' +import { evidenceDir, writeTaskEvidence } from './task-evidence.js' +import { readTaskRecap, writeTaskRecap } from './task-recap.js' +import { createTask, saveTask } from './tasks-store.js' describe('isLoopbackHost', () => { test('accepts loopback hosts, with and without a port', () => { @@ -1127,6 +1131,219 @@ describe('project-scoped repo routes (?project=)', () => { }) }) +describe('task recap and evidence routes (?project=)', () => { + let configDir: string + let repoDir: string + let projectId: string + let projectPath: string + let port: number + let stop: () => Promise + const previousConfigDir = process.env.CODESEMA_CONFIG_DIR + + const TASK_ID = 'abcdef123456' + const OTHER_TASK_ID = '0123456789ab' + + function minimalRecap(): RecapRecord { + return { + version: 1, + summary: 'did the thing', + changes: [], + decisions: [], + files: [], + tests: [], + branch: 'codesema/task-x', + } + } + + beforeAll(async () => { + configDir = mkdtempSync(join(tmpdir(), 'codesema-evidence-config-')) + process.env.CODESEMA_CONFIG_DIR = configDir + repoDir = mkdtempSync(join(tmpdir(), 'codesema-evidence-repo-')) + execFileSync('git', ['init', '-b', 'main'], { cwd: repoDir, stdio: 'ignore' }) + const added = addProject(repoDir) + if (!added.ok) { + throw new Error('failed to register test repo') + } + projectId = added.project.id + projectPath = added.project.path + + writeTaskRecap(projectPath, TASK_ID, minimalRecap()) + writeTaskEvidence(projectPath, TASK_ID, { + version: 1, + status: 'passed', + reason: null, + head_sha: null, + items: [], + }) + + const dir = evidenceDir(projectPath, TASK_ID) + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, 'shot.png'), 'a-small-png') + writeFileSync(join(dir, 'clip.webm'), 'a-small-webm') + writeFileSync(join(dir, 'huge.png'), Buffer.alloc(64 * 1024 * 1024 + 1)) + + const started = await startServer(createSession(), { cwd: repoDir, port: 4990 }) + port = started.port + stop = started.stop + }) + + afterAll(async () => { + await stop() + if (previousConfigDir === undefined) { + delete process.env.CODESEMA_CONFIG_DIR + } else { + process.env.CODESEMA_CONFIG_DIR = previousConfigDir + } + rmSync(configDir, { recursive: true, force: true }) + rmSync(repoDir, { recursive: true, force: true }) + }) + + test('200s the recap of a task that has one', async () => { + const res = await rawRequest(port, `/api/tasks/${TASK_ID}/recap?project=${projectId}`) + expect(res.status).toBe(200) + expect(res.contentType).toBe('application/json; charset=utf-8') + expect(JSON.parse(res.body)).toMatchObject({ summary: 'did the thing' }) + }) + + test('404s a task with no recap', async () => { + const res = await rawRequest(port, `/api/tasks/${OTHER_TASK_ID}/recap?project=${projectId}`) + expect(res.status).toBe(404) + }) + + test('404s an invalid task id on recap', async () => { + const res = await rawRequest(port, `/api/tasks/not-a-task-id/recap?project=${projectId}`) + expect(res.status).toBe(404) + }) + + function seedTaskRecord(branch: string): ReturnType { + const record = createTask(projectPath, { + title: branch, + prompt: 'do it', + autoShip: false, + base: 'main', + branch, + worktree: join(projectPath, '.codesema', 'worktrees', branch), + isolation: 'policy', + }) + return record + } + + test('computes a fallback recap for a review_ok task with no recap.json, without writing it to disk', async () => { + const record = seedTaskRecord('codesema/task-fallback') + record.status = 'review_ok' + record.turns = [ + { + prompt: 'do it', + response: 'Rewired the worktree cleanup.', + question: null, + started_at: '2026-01-01T00:00:00.000Z', + ended_at: '2026-01-01T00:01:00.000Z', + }, + ] + saveTask(projectPath, record) + + const res = await rawRequest(port, `/api/tasks/${record.id}/recap?project=${projectId}`) + expect(res.status).toBe(200) + const body = JSON.parse(res.body) + expect(body.summary).toBe('Rewired the worktree cleanup.') + expect(body.mr_url).toBeUndefined() + expect(body.branch).toBe('codesema/task-fallback') + + expect(readTaskRecap(projectPath, record.id)).toBeNull() + }) + + test('recap.json on disk wins over the fallback computation', async () => { + const record = seedTaskRecord('codesema/task-has-recap') + record.status = 'review_ok' + saveTask(projectPath, record) + writeTaskRecap(projectPath, record.id, { + version: 1, + summary: 'the persisted one', + changes: [], + decisions: [], + files: [], + tests: [], + branch: record.branch, + }) + + const res = await rawRequest(port, `/api/tasks/${record.id}/recap?project=${projectId}`) + expect(res.status).toBe(200) + expect(JSON.parse(res.body)).toMatchObject({ summary: 'the persisted one' }) + }) + + test('a failed task with no recap.json still 404s: no fallback for a status the generator was never meant to cover', async () => { + const record = seedTaskRecord('codesema/task-failed') + record.status = 'failed' + saveTask(projectPath, record) + + const res = await rawRequest(port, `/api/tasks/${record.id}/recap?project=${projectId}`) + expect(res.status).toBe(404) + }) + + test('200s the evidence record of a task that has one', async () => { + const res = await rawRequest(port, `/api/tasks/${TASK_ID}/evidence?project=${projectId}`) + expect(res.status).toBe(200) + expect(res.contentType).toBe('application/json; charset=utf-8') + expect(JSON.parse(res.body)).toMatchObject({ status: 'passed', items: [] }) + }) + + test('404s a task with no evidence record', async () => { + const res = await rawRequest(port, `/api/tasks/${OTHER_TASK_ID}/evidence?project=${projectId}`) + expect(res.status).toBe(404) + }) + + test('404s an invalid task id on evidence', async () => { + const res = await rawRequest(port, `/api/tasks/not-a-task-id/evidence?project=${projectId}`) + expect(res.status).toBe(404) + }) + + test('serves a png evidence file with the right content type', async () => { + const res = await rawRequest( + port, + `/api/tasks/${TASK_ID}/evidence/shot.png?project=${projectId}`, + ) + expect(res.status).toBe(200) + expect(res.contentType).toBe('image/png') + expect(res.nosniff).toBe('nosniff') + }) + + test('serves a webm evidence file with the right content type', async () => { + const res = await rawRequest( + port, + `/api/tasks/${TASK_ID}/evidence/clip.webm?project=${projectId}`, + ) + expect(res.status).toBe(200) + expect(res.contentType).toBe('video/webm') + expect(res.nosniff).toBe('nosniff') + }) + + test('413s an evidence file over the size limit', async () => { + const res = await rawRequest( + port, + `/api/tasks/${TASK_ID}/evidence/huge.png?project=${projectId}`, + ) + expect(res.status).toBe(413) + }) + + test('404s an absent evidence file', async () => { + const res = await rawRequest( + port, + `/api/tasks/${TASK_ID}/evidence/nope.png?project=${projectId}`, + ) + expect(res.status).toBe(404) + }) + + test('404s a traversing evidence file name', async () => { + const encoded = await rawRequest( + port, + `/api/tasks/${TASK_ID}/evidence/..%2Fpackage.json?project=${projectId}`, + ) + expect(encoded.status).toBe(404) + const nested = await rawRequest(port, `/api/tasks/${TASK_ID}/evidence/a/b?project=${projectId}`) + expect(nested.status).toBe(404) + }) +}) + // GET /api/mrs beyond the default open state (D2 states beyond open). Kept in // its own describe/repo rather than folded into the scoped describe above: // this one asserts on the `state` argument itself, which the scoped describe's diff --git a/packages/cli/src/serve.ts b/packages/cli/src/serve.ts index 2562039..17f1f7d 100644 --- a/packages/cli/src/serve.ts +++ b/packages/cli/src/serve.ts @@ -1,5 +1,5 @@ import { randomBytes } from 'node:crypto' -import { existsSync, readFileSync } from 'node:fs' +import { existsSync, readFileSync, statSync } from 'node:fs' import { readFile } from 'node:fs/promises' import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' import { extname, join, resolve, sep } from 'node:path' @@ -20,6 +20,7 @@ import { sanitizeRecord, TASK_AGENT_MAX, type ArmTicket, + type RecapRecord, type ReviewRecord, } from './contract.js' import type { JudgeDecision } from './dual.js' @@ -60,8 +61,11 @@ import { import { startRunnerDaemon, type RunnerDaemonHandle } from './runner-daemon.js' import { loadSyncCredentials } from './sync.js' import { applyTaskCriteria } from './task-criteria.js' +import { EVIDENCE_MAX_BYTES, evidenceDir, readTaskEvidence } from './task-evidence.js' +import { generateRecap, readTaskRecap, recapOptionsFor } from './task-recap.js' import type { TaskActionResult } from './task-runner.js' import type { CreateTaskManagerInput, TaskEnvelope, TaskManager } from './task-server.js' +import { loadTask } from './tasks-store.js' import { AGENT_DEFS, composeCommand, @@ -232,6 +236,7 @@ const MIME_BY_EXTENSION: Record = { '.png': 'image/png', '.ico': 'image/x-icon', '.woff2': 'font/woff2', + '.webm': 'video/webm', } /** @@ -1460,11 +1465,79 @@ async function serveStaticFile(res: ServerResponse, pathname: string): Promise { + const projectId = requiredProjectParam(params) + if (!projectId) { + return sendText(res, 400, 'bad request') + } + if (!isTaskId(taskId) || !EVIDENCE_FILENAME_RE.test(filename)) { + return sendText(res, 404, 'not found') + } + const project = getProject(projectId) + if (!project) { + return sendText(res, 404, 'not found') + } + const filePath = resolveStaticPath(evidenceDir(project.path, taskId), `/${filename}`) + if (!filePath) { + return sendText(res, 404, 'not found') + } + let size: number + try { + size = statSync(filePath).size + } catch { + return sendText(res, 404, 'not found') + } + if (size > EVIDENCE_MAX_BYTES) { + return sendText(res, 413, 'payload too large') + } + let content: Buffer + try { + content = await readFile(filePath) + } catch { + return sendText(res, 404, 'not found') + } + const evidenceMime = + MIME_BY_EXTENSION[extname(filePath).toLowerCase()] ?? 'application/octet-stream' + res.writeHead(200, { 'content-type': evidenceMime, 'x-content-type-options': 'nosniff' }) + res.end(content) +} + +/** + * Read-time fallback for a task whose turn ended before the end-of-turn + * recap write existed (task-server.ts's `onTurnDone`): every entry the + * generator needs is already on disk, so this computes the SAME recap that + * write would have produced, using the SAME entries (`recapOptionsFor`, + * task-recap.ts). Pure read: never calls `writeTaskRecap`, so a GET never + * creates the file `onTurnDone` and the ship are the only writers of: the + * disk stays the single source of truth for what actually happened at + * turn-end or ship time. `'review_ok'`/`'shipped'` only: any other status + * means either the review never landed clean or nothing to summarize yet, + * and the route answers 404 exactly as it did before this fallback existed. + */ +function computeFallbackRecap(cwd: string, id: string): RecapRecord | null { + const task = loadTask(cwd, id) + if (!task || (task.status !== 'review_ok' && task.status !== 'shipped')) { + return null + } + return generateRecap(recapOptionsFor(cwd, task)).recap +} + const TASK_ACTION_RE = /^\/api\/tasks\/([^/]+)\/(reply|ship|interrupt|abandon|checks|resume|criteria|attach)$/ const TASK_GET_RE = /^\/api\/tasks\/([^/]+)$/ const TASK_CHECKS_RE = /^\/api\/tasks\/([^/]+)\/checks$/ +const TASK_VERIFICATION_RE = /^\/api\/tasks\/([^/]+)\/verification$/ const TASK_REVIEW_RE = /^\/api\/tasks\/([^/]+)\/review$/ +const TASK_RECAP_RE = /^\/api\/tasks\/([^/]+)\/recap$/ +const TASK_EVIDENCE_RE = /^\/api\/tasks\/([^/]+)\/evidence$/ +const TASK_EVIDENCE_FILE_RE = /^\/api\/tasks\/([^/]+)\/evidence\/(.+)$/ const PROJECT_DELETE_RE = /^\/api\/projects\/([^/]+)$/ const PROJECT_CHECKS_SETUP_RE = /^\/api\/projects\/([^/]+)\/checks-setup$/ const PROJECT_CHECKS_APPLY_RE = /^\/api\/projects\/([^/]+)\/checks-apply$/ @@ -1767,6 +1840,26 @@ function createRequestHandler(handlerOpts: { } return sendJson(res, 200, checks) } + const taskVerificationGet = TASK_VERIFICATION_RE.exec(pathname) + if (taskVerificationGet?.[1]) { + if (!tasks) { + return sendJson(res, 501, { error: 'task manager unavailable' }) + } + const projectId = requiredProjectParam(searchParams) + if (!projectId) { + return sendText(res, 400, 'bad request') + } + // 404 covers unknown project, unknown/malformed task id AND a task + // whose mechanical verification never ran: the file simply is not + // there yet, same doctrine as the checks route above. + const verification = isTaskId(taskVerificationGet[1]) + ? tasks.manager.getVerification(projectId, taskVerificationGet[1]) + : null + if (!verification) { + return sendText(res, 404, 'not found') + } + return sendJson(res, 200, verification) + } const taskReviewGet = TASK_REVIEW_RE.exec(pathname) if (taskReviewGet?.[1]) { if (!tasks) { @@ -1788,6 +1881,48 @@ function createRequestHandler(handlerOpts: { } return sendJson(res, 200, review) } + const taskRecapGet = TASK_RECAP_RE.exec(pathname) + if (taskRecapGet?.[1]) { + const projectId = requiredProjectParam(searchParams) + if (!projectId) { + return sendText(res, 400, 'bad request') + } + const project = getProject(projectId) + const recap = + project && isTaskId(taskRecapGet[1]) + ? (readTaskRecap(project.path, taskRecapGet[1]) ?? + computeFallbackRecap(project.path, taskRecapGet[1])) + : null + if (!recap) { + return sendText(res, 404, 'not found') + } + return sendJson(res, 200, recap) + } + const taskEvidenceGet = TASK_EVIDENCE_RE.exec(pathname) + if (taskEvidenceGet?.[1]) { + const projectId = requiredProjectParam(searchParams) + if (!projectId) { + return sendText(res, 400, 'bad request') + } + const project = getProject(projectId) + const evidence = + project && isTaskId(taskEvidenceGet[1]) + ? readTaskEvidence(project.path, taskEvidenceGet[1]) + : null + if (!evidence) { + return sendText(res, 404, 'not found') + } + return sendJson(res, 200, evidence) + } + const taskEvidenceFileGet = TASK_EVIDENCE_FILE_RE.exec(pathname) + if (taskEvidenceFileGet?.[1] && taskEvidenceFileGet[2]) { + return void serveTaskEvidenceFile( + res, + taskEvidenceFileGet[1], + taskEvidenceFileGet[2], + searchParams, + ) + } const taskGet = TASK_GET_RE.exec(pathname) if (taskGet?.[1]) { if (!tasks) { diff --git a/packages/cli/src/task-evidence.test.ts b/packages/cli/src/task-evidence.test.ts new file mode 100644 index 0000000..4d2a427 --- /dev/null +++ b/packages/cli/src/task-evidence.test.ts @@ -0,0 +1,219 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, test } from 'bun:test' +import type { EvidenceItem, EvidenceRecord } from './contract.js' +import { + EVIDENCE_MAX_BYTES, + evidenceDir, + ingestEvidenceFiles, + readTaskEvidence, + writeTaskEvidence, +} from './task-evidence.js' +import { taskDir } from './tasks-store.js' + +const cleanups: string[] = [] + +afterEach(() => { + for (const dir of cleanups.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +function makeDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'codesema-task-evidence-')) + cleanups.push(dir) + return dir +} + +const TASK_ID = 'abcdef123456' + +function baseRecord(overrides: Partial = {}): EvidenceRecord { + return { + version: 1, + status: 'passed', + reason: null, + head_sha: null, + items: [], + ...overrides, + } +} + +describe('readTaskEvidence / writeTaskEvidence', () => { + test('a task with no evidence.json reads as null', () => { + const repo = makeDir() + expect(readTaskEvidence(repo, TASK_ID)).toBeNull() + }) + + test('write then read round-trips the sanitized record', () => { + const repo = makeDir() + const item: EvidenceItem = { + kind: 'screenshot', + path: 'shot.png', + bytes: 12, + turn: 1, + created_at: '2026-08-01T00:00:00.000Z', + } + const written = writeTaskEvidence(repo, TASK_ID, baseRecord({ items: [item] })) + expect(written).toEqual(baseRecord({ items: [item] })) + expect(readTaskEvidence(repo, TASK_ID)).toEqual(baseRecord({ items: [item] })) + }) + + test('an unknown task id reads as null and refuses to write', () => { + const repo = makeDir() + expect(readTaskEvidence(repo, 'not-a-task-id')).toBeNull() + expect(() => writeTaskEvidence(repo, 'not-a-task-id', baseRecord())).toThrow() + }) + + test('a malformed file on disk reads back as null rather than throwing', () => { + const repo = makeDir() + mkdirSync(taskDir(repo, TASK_ID), { recursive: true }) + writeFileSync(join(taskDir(repo, TASK_ID), 'evidence.json'), 'not json') + expect(readTaskEvidence(repo, TASK_ID)).toBeNull() + }) +}) + +describe('ingestEvidenceFiles', () => { + function makeIncoming(): string { + const dir = mkdtempSync(join(tmpdir(), 'codesema-evidence-incoming-')) + return dir + } + + test('png and webm are ingested, including from a subfolder, other extensions are ignored', () => { + const repo = makeDir() + const incoming = makeIncoming() + writeFileSync(join(incoming, 'shot.png'), 'png-bytes') + writeFileSync(join(incoming, 'notes.txt'), 'not evidence') + mkdirSync(join(incoming, 'videos'), { recursive: true }) + writeFileSync(join(incoming, 'videos', 'clip.webm'), 'webm-bytes') + + const record = ingestEvidenceFiles(repo, TASK_ID, incoming, { + turn: 3, + status: 'passed', + reason: null, + head_sha: 'deadbeef', + keep: null, + }) + + expect(record.status).toBe('passed') + expect(record.head_sha).toBe('deadbeef') + expect(record.items).toHaveLength(2) + const kinds = record.items.map((item) => item.kind).toSorted() + expect(kinds).toEqual(['screenshot', 'video']) + for (const item of record.items) { + expect(item.turn).toBe(3) + expect(existsSync(join(evidenceDir(repo, TASK_ID), item.path))).toBe(true) + } + }) + + test('a file over the size limit is ignored', () => { + const repo = makeDir() + const incoming = makeIncoming() + writeFileSync(join(incoming, 'huge.png'), Buffer.alloc(EVIDENCE_MAX_BYTES + 1)) + writeFileSync(join(incoming, 'small.png'), 'ok') + + const record = ingestEvidenceFiles(repo, TASK_ID, incoming, { + turn: 1, + status: 'passed', + reason: null, + head_sha: null, + keep: null, + }) + + expect(record.items).toHaveLength(1) + expect(record.items[0]?.path.endsWith('.png')).toBe(true) + }) + + test('empty incoming yields zero items while status and reason are still applied', () => { + const repo = makeDir() + const incoming = makeIncoming() + + const record = ingestEvidenceFiles(repo, TASK_ID, incoming, { + turn: 1, + status: 'failed', + reason: 'the checkout journey timed out', + head_sha: null, + keep: null, + }) + + expect(record.items).toEqual([]) + expect(record.status).toBe('failed') + expect(record.reason).toBe('the checkout journey timed out') + }) + + test('incomingDir is removed once ingestion completes', () => { + const repo = makeDir() + const incoming = makeIncoming() + writeFileSync(join(incoming, 'shot.png'), 'png-bytes') + + ingestEvidenceFiles(repo, TASK_ID, incoming, { + turn: 1, + status: 'passed', + reason: null, + head_sha: null, + keep: null, + }) + + expect(existsSync(incoming)).toBe(false) + }) + + test('keep-N purges the oldest items and drops them from disk and from the record', () => { + const repo = makeDir() + const dir = evidenceDir(repo, TASK_ID) + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, 'old-0.png'), 'a') + writeFileSync(join(dir, 'old-1.png'), 'b') + writeFileSync(join(dir, 'old-2.png'), 'c') + writeTaskEvidence( + repo, + TASK_ID, + baseRecord({ + items: [ + { + kind: 'screenshot', + path: 'old-0.png', + bytes: 1, + turn: 1, + created_at: '2020-01-01T00:00:00.000Z', + }, + { + kind: 'screenshot', + path: 'old-1.png', + bytes: 1, + turn: 1, + created_at: '2020-01-01T00:00:01.000Z', + }, + { + kind: 'screenshot', + path: 'old-2.png', + bytes: 1, + turn: 1, + created_at: '2020-01-01T00:00:02.000Z', + }, + ], + }), + ) + + const incoming = makeIncoming() + writeFileSync(join(incoming, 'new-0.png'), 'x') + writeFileSync(join(incoming, 'new-1.png'), 'y') + writeFileSync(join(incoming, 'new-2.webm'), 'z') + + const record = ingestEvidenceFiles(repo, TASK_ID, incoming, { + turn: 2, + status: 'passed', + reason: null, + head_sha: null, + keep: 4, + }) + + expect(record.items).toHaveLength(4) + const paths = record.items.map((item) => item.path) + expect(paths).toContain('old-2.png') + expect(paths).not.toContain('old-0.png') + expect(paths).not.toContain('old-1.png') + expect(existsSync(join(dir, 'old-0.png'))).toBe(false) + expect(existsSync(join(dir, 'old-1.png'))).toBe(false) + expect(existsSync(join(dir, 'old-2.png'))).toBe(true) + }) +}) diff --git a/packages/cli/src/task-evidence.ts b/packages/cli/src/task-evidence.ts new file mode 100644 index 0000000..4339703 --- /dev/null +++ b/packages/cli/src/task-evidence.ts @@ -0,0 +1,182 @@ +import { + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + unlinkSync, + type Dirent, +} from 'node:fs' +import { extname, join, relative } from 'node:path' +import { writeJsonAtomic } from './atomic-write.js' +import { + isTaskId, + sanitizeEvidence, + type EvidenceItem, + type EvidenceKind, + type EvidenceRecord, + type EvidenceStatus, + type ProofIntent, +} from './contract.js' +import { taskDir } from './tasks-store.js' + +export const EVIDENCE_MAX_BYTES = 64 * 1024 * 1024 +const EVIDENCE_DEFAULT_KEEP = 5 + +const EVIDENCE_EXTENSION_KIND: Readonly> = { + '.png': 'screenshot', + '.webm': 'video', +} + +export type IngestEvidenceMeta = { + turn: number + status: EvidenceStatus + reason: string | null + head_sha: string | null + keep: number | null + intent?: ProofIntent +} + +export function evidenceDir(cwd: string, id: string): string { + return join(taskDir(cwd, id), 'evidence') +} + +/** + * Latest evidence record of a task (.codesema/tasks//evidence.json), + * calque of readTaskChecks/readTaskVerification. Null on unknown id, + * unreadable file or unusable content, never a throw. + */ +export function readTaskEvidence(cwd: string, id: string): EvidenceRecord | null { + if (!isTaskId(id)) { + return null + } + const path = join(taskDir(cwd, id), 'evidence.json') + let raw: unknown + try { + raw = JSON.parse(readFileSync(path, 'utf8')) + } catch { + return null + } + return sanitizeEvidence(raw) +} + +/** + * Atomic rewrite of evidence.json, calque of writeTaskChecks/writeTaskVerification. + * Sanitized before writing so the file on disk is always bounded; the + * sanitized copy is returned so the caller broadcasts exactly what was + * persisted. + */ +export function writeTaskEvidence(cwd: string, id: string, record: EvidenceRecord): EvidenceRecord { + if (!isTaskId(id)) { + throw new Error(`invalid task id: ${id}`) + } + const clean = sanitizeEvidence(record) + if (!clean) { + throw new Error('invalid task evidence') + } + writeJsonAtomic(join(taskDir(cwd, id), 'evidence.json'), clean) + return clean +} + +function collectIncomingFiles(dir: string, base = dir): string[] { + let entries: Dirent[] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return [] + } + const files: string[] = [] + for (const entry of entries) { + const full = join(dir, entry.name) + if (entry.isDirectory()) { + files.push(...collectIncomingFiles(full, base)) + } else if (entry.isFile()) { + files.push(full) + } + } + return files.toSorted((a, b) => relative(base, a).localeCompare(relative(base, b))) +} + +/** + * Ingests every .png/.webm found recursively under incomingDir (Playwright + * writes videos into a subfolder) into /evidence, merges the result + * with the previous record's surviving items, purges down to meta.keep + * (default 5, most recent created_at first), and persists the merged record. + * incomingDir is removed once ingestion completes, whether or not it held + * any usable file. + */ +export function ingestEvidenceFiles( + cwd: string, + id: string, + incomingDir: string, + meta: IngestEvidenceMeta, +): EvidenceRecord { + if (!isTaskId(id)) { + throw new Error(`invalid task id: ${id}`) + } + const targetDir = evidenceDir(cwd, id) + mkdirSync(targetDir, { recursive: true }) + + const candidates = collectIncomingFiles(incomingDir) + const epochMs = Date.now() + const createdAt = new Date(epochMs).toISOString() + const newItems: EvidenceItem[] = [] + let index = 0 + for (const file of candidates) { + const ext = extname(file).toLowerCase() + const kind = EVIDENCE_EXTENSION_KIND[ext] + if (!kind) { + continue + } + if (statSync(file).size > EVIDENCE_MAX_BYTES) { + continue + } + const filename = `t${meta.turn}-${epochMs}-${index}${ext}` + const target = join(targetDir, filename) + renameSync(file, target) + newItems.push({ + kind, + path: filename, + bytes: statSync(target).size, + turn: meta.turn, + created_at: createdAt, + }) + index += 1 + } + + const previousItems = readTaskEvidence(cwd, id)?.items ?? [] + const combined = [...previousItems, ...newItems] + const keepN = meta.keep ?? EVIDENCE_DEFAULT_KEEP + const sorted = combined.toSorted((a, b) => b.created_at.localeCompare(a.created_at)) + const toKeep = sorted.slice(0, keepN) + const toDrop = sorted.slice(keepN) + + for (const item of toDrop) { + try { + unlinkSync(join(targetDir, item.path)) + } catch { + continue + } + } + + const survivors = toKeep.filter((item) => { + try { + statSync(join(targetDir, item.path)) + return true + } catch { + return false + } + }) + + rmSync(incomingDir, { recursive: true, force: true }) + + return writeTaskEvidence(cwd, id, { + version: 1, + status: meta.status, + reason: meta.reason, + head_sha: meta.head_sha, + items: survivors, + ...(meta.intent !== undefined ? { intent: meta.intent } : {}), + }) +} diff --git a/packages/cli/src/task-proof-chapter.test.ts b/packages/cli/src/task-proof-chapter.test.ts new file mode 100644 index 0000000..54a6f89 --- /dev/null +++ b/packages/cli/src/task-proof-chapter.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test } from 'bun:test' +import type { EvidenceRecord, ProofIntent } from './contract.js' +import { buildProofChapter } from './task-proof-chapter.js' + +function baseInput() { + return { + uiFiles: [] as string[], + otherCount: 0, + intent: null as ProofIntent | null, + evidence: null as EvidenceRecord | null, + declared: false, + } +} + +describe('buildProofChapter', () => { + test('the mandatory heading, grid and output requirement are always present', () => { + const chapter = buildProofChapter(baseInput()) + expect(chapter).toContain('Visual proof (MANDATORY chapter)') + expect(chapter).toContain('"screenshot" naming the pages') + expect(chapter).toContain('"journey" naming the spec') + expect(chapter).toContain('"none", with the reason stated') + expect(chapter).toContain('doubt: proof, not "none"') + expect(chapter).toContain('"proof_review"') + expect(chapter).toContain('"kind": "design"') + expect(chapter).toContain('"severity": "major"') + }) + + test('the first AND the last line state the answer is incomplete without proof_review', () => { + const chapter = buildProofChapter(baseInput()) + const lines = chapter.split('\n') + expect(lines[0]).toContain('INCOMPLETE without a "proof_review" field') + expect(lines[0]).toContain('logged and treated as a MISSING verdict') + const last = lines.at(-1) + expect(last).toContain('INCOMPLETE without a "proof_review" field') + expect(last).toContain('logged and treated as a MISSING verdict') + }) + + test('states it extends the output shape, names the exact insertion point, and gives a literal example', () => { + const chapter = buildProofChapter(baseInput()) + expect(chapter).toContain('EXTENDS the output shape') + expect(chapter).toContain('Output JSON shape (exactly these fields)') + expect(chapter).toContain('after "files_reviewed" and before the closing brace') + expect(chapter).toContain( + 'Example: "proof_review": { "expected": "screenshot", "coherent": false, "reason":', + ) + }) + + test('lists the touched UI files, or "none" when there are none', () => { + expect(buildProofChapter(baseInput())).toContain('UI files touched by this diff: none') + expect( + buildProofChapter({ ...baseInput(), uiFiles: ['src/App.vue', 'src/Foo.tsx'] }), + ).toContain('UI files touched by this diff: src/App.vue, src/Foo.tsx') + }) + + test('reports the count of other, non-UI files touched', () => { + expect(buildProofChapter({ ...baseInput(), otherCount: 3 })).toContain('other files touched: 3') + }) + + test('undeclared: states plainly that the turn never declared a proof', () => { + const chapter = buildProofChapter({ ...baseInput(), declared: false, intent: null }) + expect(chapter).toContain('declaration: the agent did not declare a proof this turn') + }) + + test('declared none: states the kind and the reason', () => { + const chapter = buildProofChapter({ + ...baseInput(), + declared: true, + intent: { kind: 'none', reason: 'pure refactor, no rendered difference' }, + }) + expect(chapter).toContain( + 'declaration: kind=none, reason: "pure refactor, no rendered difference"', + ) + }) + + test('declared screenshot: states the pages', () => { + const chapter = buildProofChapter({ + ...baseInput(), + declared: true, + intent: { kind: 'screenshot', reason: 'new settings row', pages: ['/settings', '/profile'] }, + }) + expect(chapter).toContain( + 'declaration: kind=screenshot pages: /settings, /profile, reason: "new settings row"', + ) + }) + + test('declared journey: states the spec', () => { + const chapter = buildProofChapter({ + ...baseInput(), + declared: true, + intent: { kind: 'journey', reason: 'multi-step checkout', journey: 'tests/checkout.spec.ts' }, + }) + expect(chapter).toContain( + 'declaration: kind=journey spec: tests/checkout.spec.ts, reason: "multi-step checkout"', + ) + }) + + test('no evidence for this commit: states it plainly', () => { + const chapter = buildProofChapter({ ...baseInput(), evidence: null }) + expect(chapter).toContain('proof produced: no proof for this commit') + }) + + test('evidence present: states the status and every item', () => { + const evidence: EvidenceRecord = { + version: 1, + status: 'passed', + reason: null, + head_sha: 'a'.repeat(40), + items: [ + { + kind: 'screenshot', + path: 'p0.png', + bytes: 1234, + turn: 2, + created_at: '2026-08-01T00:00:00.000Z', + }, + { + kind: 'video', + path: 'v0.webm', + bytes: 5678, + turn: 2, + created_at: '2026-08-01T00:00:00.000Z', + }, + ], + } + const chapter = buildProofChapter({ ...baseInput(), evidence }) + expect(chapter).toContain('proof produced: status=passed') + expect(chapter).toContain('kind=screenshot bytes=1234 turn=2') + expect(chapter).toContain('kind=video bytes=5678 turn=2') + }) + + test('a failed or declined proof carries its reason', () => { + const evidence: EvidenceRecord = { + version: 1, + status: 'failed', + reason: 'replay timed out after 120000ms', + head_sha: 'a'.repeat(40), + items: [], + } + const chapter = buildProofChapter({ ...baseInput(), evidence }) + expect(chapter).toContain('status=failed') + expect(chapter).toContain('reason: replay timed out after 120000ms') + }) +}) diff --git a/packages/cli/src/task-proof-chapter.ts b/packages/cli/src/task-proof-chapter.ts new file mode 100644 index 0000000..e0d1240 --- /dev/null +++ b/packages/cli/src/task-proof-chapter.ts @@ -0,0 +1,85 @@ +import type { EvidenceRecord, ProofIntent } from './contract.js' + +function describeIntent(intent: ProofIntent): string { + const detail = + intent.kind === 'screenshot' && intent.pages + ? ` pages: ${intent.pages.join(', ')}` + : intent.kind === 'journey' && intent.journey + ? ` spec: ${intent.journey}` + : '' + return `kind=${intent.kind}${detail}, reason: "${intent.reason}"` +} + +function describeEvidence(evidence: EvidenceRecord): string { + const parts = [`status=${evidence.status}`] + if (evidence.items.length > 0) { + const items = evidence.items + .map((item) => `kind=${item.kind} bytes=${item.bytes} turn=${item.turn}`) + .join('; ') + parts.push(`items: ${items}`) + } + if (evidence.reason) { + parts.push(`reason: ${evidence.reason}`) + } + return parts.join(', ') +} + +export type BuildProofChapterInput = { + /** UI-classified paths touched by this diff (ui-surface.ts's classifyUiPaths). */ + uiFiles: string[] + /** How many other, non-UI paths this diff also touches. */ + otherCount: number + /** This turn's own PROOF declaration, or null when it never stated one. */ + intent: ProofIntent | null + /** This task's evidence.json, already checked by the caller to match the reviewed head_sha; null otherwise. */ + evidence: EvidenceRecord | null + /** Whether the turn declared a PROOF line at all, independent of what it declared. */ + declared: boolean +} + +/** + * D17: the mandatory chapter that hands the reviewer the mechanical facts + * (which files are UI, what the turn declared, what capture actually ran) and + * asks for exactly one judgment call — was the declaration a reasonable + * response to those facts. Same split as buildCriteriaChapter: the fact is + * read off the diff/disk by this file, never by the model; only the + * reasonableness of the response is asked of it. + */ +export function buildProofChapter(input: BuildProofChapterInput): string { + const uiLine = input.uiFiles.length > 0 ? input.uiFiles.join(', ') : 'none' + const declarationLine = + input.declared && input.intent + ? describeIntent(input.intent) + : 'the agent did not declare a proof this turn' + const evidenceLine = input.evidence + ? describeEvidence(input.evidence) + : 'no proof for this commit' + + const incompleteWithoutVerdict = + 'Your JSON answer is INCOMPLETE without a "proof_review" field: an output that omits it is logged and treated as a MISSING verdict, never read as "nothing to add" or as this chapter waived.' + + return [ + `Visual proof (MANDATORY chapter): ${incompleteWithoutVerdict}`, + 'FACTS:', + `- UI files touched by this diff: ${uiLine}`, + `- other files touched: ${input.otherCount}`, + `- declaration: ${declarationLine}`, + `- proof produced: ${evidenceLine}`, + '', + 'The decision grid the agent was asked to follow when it wrote its PROOF line:', + '- the interface changed and the change is visible: "screenshot" naming the pages, or "journey" naming the spec when the change is a sequence rather than one screen;', + '- a UI file changed with no visible effect (a refactor with no rendered difference, a prop threaded through with no new output): "none", with the reason stated;', + '- nothing outside the interface was touched: "none";', + '- doubt: proof, not "none". The grid resolves a tie toward capturing rather than skipping.', + '', + 'Judge whether the declaration above was a reasonable response to the FACTS above, using the grid as your standard. A proof supplied when none was owed is never blocking. An absence justified by a real lack of visible effect is coherent. A project with no proof target configured turns every declaration into "skipped", reason "no_target": that is always coherent, since there is nothing to replay a proof against.', + '', + 'This chapter EXTENDS the output shape given above ("Output JSON shape (exactly these fields)"): "proof_review" is REQUIRED in addition to those fields, one more top-level property, placed after "files_reviewed" and before the closing brace of the JSON object:', + '"proof_review": { "expected": "none" | "screenshot" | "journey", "coherent": true | false, "reason": "" }', + 'Example: "proof_review": { "expected": "screenshot", "coherent": false, "reason": "the diff adds a visible settings row with no captured proof" }', + '', + 'Rule for this chapter: if "coherent" is false AND the UI files listed above are non-empty, you MUST ALSO emit a finding with "kind": "design" and "severity": "major", anchored on a line of one of those UI files, whose message names the proof that was expected. Never emit this finding when the UI files list above is empty, and never emit it merely because a proof was supplied that was not owed.', + '', + incompleteWithoutVerdict, + ].join('\n') +} diff --git a/packages/cli/src/task-proof.test.ts b/packages/cli/src/task-proof.test.ts new file mode 100644 index 0000000..f85a8ef --- /dev/null +++ b/packages/cli/src/task-proof.test.ts @@ -0,0 +1,286 @@ +import { describe, expect, test } from 'bun:test' +import type { + SandboxExecOptions, + SandboxExecResult, + SandboxHandle, + SandboxMetrics, +} from './microsandbox-driver.js' +import { + captureProof, + captureScreenshots, + type CaptureProofOptions, + type CaptureScreenshotsOptions, +} from './task-proof.js' + +type Call = { method: string; args: unknown[] } + +const ok = (over: Partial = {}): SandboxExecResult => ({ + code: 0, + stdout: '', + stderr: '', + timedOut: false, + ...over, +}) + +function fakeHandle(opts: { + script?: (command: string, execOpts: SandboxExecOptions) => SandboxExecResult + copyToHostFails?: boolean +}): { handle: SandboxHandle; calls: Call[] } { + const calls: Call[] = [] + const script = opts.script ?? (() => ok()) + const handle: SandboxHandle = { + name: 'fake', + exec: (command, args, execOpts) => { + calls.push({ method: 'exec', args: [command, args] }) + return Promise.resolve(script(command, execOpts)) + }, + shell: (command, execOpts) => { + calls.push({ method: 'shell', args: [command] }) + return Promise.resolve(script(command, execOpts)) + }, + copyFromHost: () => Promise.resolve(), + copyToHost: (guestPath, hostPath) => { + calls.push({ method: 'copyToHost', args: [guestPath, hostPath] }) + if (opts.copyToHostFails) { + return Promise.reject(new Error('copy failed')) + } + return Promise.resolve() + }, + writeFile: () => Promise.resolve(), + readFile: () => Promise.resolve(''), + metrics: (): Promise => + Promise.resolve({ memoryHostResidentBytes: null, memoryBytes: null, cpuPercent: null }), + stop: () => Promise.resolve(), + } + return { handle, calls } +} + +function baseOpts(overrides: Partial = {}): CaptureProofOptions { + return { + journey: 'journeys/login.spec.ts', + url: 'http://localhost:3000', + timeoutMs: 5000, + guestWorkDir: '/work', + guestProofDir: '/work/.proof', + hostIncomingDir: '/host/incoming', + ...overrides, + } +} + +describe('captureProof', () => { + test('a passing replay copies the proof dir, in mkdir -> replay -> copyToHost order', async () => { + const { handle, calls } = fakeHandle({}) + const result = await captureProof(handle, baseOpts()) + expect(result).toEqual({ status: 'passed', reason: null }) + expect(calls.map((c) => c.method)).toEqual(['shell', 'shell', 'copyToHost']) + expect(calls[0]?.args[0]).toBe('mkdir -p /work/.proof') + const replay = calls[1]?.args[0] as string + expect(replay).toContain("CODESEMA_BASE_URL='http://localhost:3000'") + expect(replay).toContain('CODESEMA_PROOF_DIR=/work/.proof') + expect(replay).toContain("npx playwright test 'journeys/login.spec.ts'") + expect(replay).toContain('--output=/work/.proof') + expect(calls[2]).toEqual({ + method: 'copyToHost', + args: ['/work/.proof', '/host/incoming'], + }) + }) + + test('a failing replay attempts a fallback screenshot and reports the tail as reason', async () => { + const { handle, calls } = fakeHandle({ + script: (command) => + command.includes('npx playwright test') + ? ok({ code: 1, stderr: 'assertion failed' }) + : ok(), + }) + const result = await captureProof(handle, baseOpts()) + expect(result.status).toBe('failed') + expect(result.reason).toContain('assertion failed') + const shellCommands = calls.filter((c) => c.method === 'shell').map((c) => c.args[0] as string) + expect(shellCommands[2]).toContain('npx playwright screenshot --full-page') + expect(shellCommands[2]).toContain("'http://localhost:3000'") + expect(shellCommands[2]).toContain('/work/.proof/fallback.png') + expect(calls.some((c) => c.method === 'copyToHost')).toBe(true) + }) + + test('a timed out replay reports an explicit timeout reason and still attempts the fallback', async () => { + const { handle, calls } = fakeHandle({ + script: (command) => + command.includes('npx playwright test') ? ok({ timedOut: true, code: null }) : ok(), + }) + const result = await captureProof(handle, baseOpts({ timeoutMs: 1234 })) + expect(result.status).toBe('failed') + expect(result.reason).toContain('timed out') + expect(result.reason).toContain('1234') + const shellCommands = calls.filter((c) => c.method === 'shell').map((c) => c.args[0] as string) + expect(shellCommands[2]).toContain('npx playwright screenshot') + }) + + test('an apostrophe in the url is refused without ever running the replay shell', async () => { + const { handle, calls } = fakeHandle({}) + const result = await captureProof(handle, baseOpts({ url: "http://localhost:3000/it's" })) + expect(result.status).toBe('failed') + expect(result.reason).toContain('url') + expect(result.reason).toContain('single quote') + const shellCommands = calls.filter((c) => c.method === 'shell').map((c) => c.args[0] as string) + expect(shellCommands).toEqual(['mkdir -p /work/.proof']) + expect(calls.some((c) => c.method === 'copyToHost')).toBe(true) + }) + + test('an apostrophe in the journey is refused the same way', async () => { + const { handle, calls } = fakeHandle({}) + const result = await captureProof(handle, baseOpts({ journey: "journeys/it's-broken.spec.ts" })) + expect(result.status).toBe('failed') + expect(result.reason).toContain('journey') + const shellCommands = calls.filter((c) => c.method === 'shell').map((c) => c.args[0] as string) + expect(shellCommands).toEqual(['mkdir -p /work/.proof']) + }) + + test('a copy failure after a passing replay degrades the verdict to failed', async () => { + const { handle } = fakeHandle({ copyToHostFails: true }) + const result = await captureProof(handle, baseOpts()) + expect(result.status).toBe('failed') + expect(result.reason).toContain('copy') + }) + + test('a copy failure after an already-failing replay keeps the original failure reason', async () => { + const { handle } = fakeHandle({ + script: (command) => + command.includes('npx playwright test') ? ok({ code: 1, stderr: 'boom' }) : ok(), + copyToHostFails: true, + }) + const result = await captureProof(handle, baseOpts()) + expect(result.status).toBe('failed') + expect(result.reason).toContain('boom') + }) + + test('the reason tail is bounded to 2000 characters', async () => { + const { handle } = fakeHandle({ + script: (command) => + command.includes('npx playwright test') ? ok({ code: 1, stderr: 'x'.repeat(3000) }) : ok(), + }) + const result = await captureProof(handle, baseOpts()) + expect(result.reason).toHaveLength(2000) + }) + + test('no exception ever escapes captureProof, even when the sandbox rejects', async () => { + const handle: SandboxHandle = { + name: 'broken', + exec: () => Promise.reject(new Error('exec unavailable')), + shell: () => Promise.reject(new Error('sandbox is gone')), + copyFromHost: () => Promise.resolve(), + copyToHost: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + readFile: () => Promise.resolve(''), + metrics: (): Promise => + Promise.resolve({ memoryHostResidentBytes: null, memoryBytes: null, cpuPercent: null }), + stop: () => Promise.resolve(), + } + const result = await captureProof(handle, baseOpts()) + expect(result.status).toBe('failed') + expect(result.reason).toContain('sandbox is gone') + }) +}) + +function baseScreenshotOpts( + overrides: Partial = {}, +): CaptureScreenshotsOptions { + return { + pages: ['/dashboard'], + url: 'http://localhost:3000', + timeoutMs: 5000, + guestWorkDir: '/work', + guestProofDir: '/work/.proof', + hostIncomingDir: '/host/incoming', + ...overrides, + } +} + +describe('captureScreenshots', () => { + test('one command per page, resolved against the base url, in mkdir -> screenshots -> copyToHost order', async () => { + const { handle, calls } = fakeHandle({}) + const result = await captureScreenshots( + handle, + baseScreenshotOpts({ pages: ['/dashboard', '/settings'] }), + ) + expect(result).toEqual({ status: 'passed', reason: null }) + expect(calls.map((c) => c.method)).toEqual(['shell', 'shell', 'shell', 'copyToHost']) + expect(calls[0]?.args[0]).toBe('mkdir -p /work/.proof') + const shellCommands = calls.filter((c) => c.method === 'shell').map((c) => c.args[0] as string) + expect(shellCommands[1]).toBe( + "npx playwright screenshot --full-page 'http://localhost:3000/dashboard' /work/.proof/p0.png", + ) + expect(shellCommands[2]).toBe( + "npx playwright screenshot --full-page 'http://localhost:3000/settings' /work/.proof/p1.png", + ) + expect(calls.at(-1)).toEqual({ method: 'copyToHost', args: ['/work/.proof', '/host/incoming'] }) + }) + + test('one page failing out of two still counts as an overall pass', async () => { + const { handle } = fakeHandle({ + script: (command) => (command.includes('p0.png') ? ok({ code: 1, stderr: 'boom' }) : ok()), + }) + const result = await captureScreenshots( + handle, + baseScreenshotOpts({ pages: ['/red', '/green'] }), + ) + expect(result).toEqual({ status: 'passed', reason: null }) + }) + + test('every page failing reports the concatenated tails', async () => { + const { handle } = fakeHandle({ + script: (command) => + command.includes('playwright screenshot') ? ok({ code: 1, stderr: 'boom' }) : ok(), + }) + const result = await captureScreenshots( + handle, + baseScreenshotOpts({ pages: ['/red', '/also-red'] }), + ) + expect(result.status).toBe('failed') + expect(result.reason).toContain('boom') + }) + + test('an apostrophe in a resolved page url is refused without ever running its screenshot shell', async () => { + const { handle, calls } = fakeHandle({}) + const result = await captureScreenshots(handle, baseScreenshotOpts({ pages: ["/it's-broken"] })) + expect(result.status).toBe('failed') + expect(result.reason).toContain('single quote') + const shellCommands = calls.filter((c) => c.method === 'shell').map((c) => c.args[0] as string) + expect(shellCommands.some((c) => c.includes('playwright screenshot'))).toBe(false) + }) + + test('a copy failure after a passing capture degrades the verdict to failed', async () => { + const { handle } = fakeHandle({ copyToHostFails: true }) + const result = await captureScreenshots(handle, baseScreenshotOpts()) + expect(result.status).toBe('failed') + expect(result.reason).toContain('copy') + }) + + test('the reason tail is bounded to 2000 characters', async () => { + const { handle } = fakeHandle({ + script: (command) => + command.includes('playwright screenshot') + ? ok({ code: 1, stderr: 'x'.repeat(3000) }) + : ok(), + }) + const result = await captureScreenshots(handle, baseScreenshotOpts()) + expect(result.reason).toHaveLength(2000) + }) + + test('no exception ever escapes captureScreenshots, even when the sandbox rejects', async () => { + const handle: SandboxHandle = { + name: 'broken', + exec: () => Promise.reject(new Error('exec unavailable')), + shell: () => Promise.reject(new Error('sandbox is gone')), + copyFromHost: () => Promise.resolve(), + copyToHost: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + readFile: () => Promise.resolve(''), + metrics: (): Promise => + Promise.resolve({ memoryHostResidentBytes: null, memoryBytes: null, cpuPercent: null }), + stop: () => Promise.resolve(), + } + const result = await captureScreenshots(handle, baseScreenshotOpts()) + expect(result.status).toBe('failed') + expect(result.reason).toContain('sandbox is gone') + }) +}) diff --git a/packages/cli/src/task-proof.ts b/packages/cli/src/task-proof.ts new file mode 100644 index 0000000..71ee4e3 --- /dev/null +++ b/packages/cli/src/task-proof.ts @@ -0,0 +1,175 @@ +import type { SandboxExecOptions, SandboxHandle } from './microsandbox-driver.js' + +export type ProofCaptureResult = { + status: 'passed' | 'failed' + reason: string | null +} + +export type CaptureProofOptions = { + journey: string + url: string + timeoutMs: number + guestWorkDir: string + guestProofDir: string + hostIncomingDir: string +} + +export type CaptureScreenshotsOptions = { + pages: string[] + url: string + timeoutMs: number + guestWorkDir: string + guestProofDir: string + hostIncomingDir: string +} + +const PROOF_REASON_TAIL_MAX = 2_000 + +function shellQuote(value: string): string | null { + if (value.includes("'")) { + return null + } + return `'${value}'` +} + +function tail(text: string): string { + return text.slice(-PROOF_REASON_TAIL_MAX) +} + +async function ensureProofDir( + handle: SandboxHandle, + guestProofDir: string, + timeoutMs: number, +): Promise { + await handle.shell(`mkdir -p ${guestProofDir}`, { timeoutMs }) +} + +async function finalizeCapture( + handle: SandboxHandle, + guestProofDir: string, + hostIncomingDir: string, + result: ProofCaptureResult, +): Promise { + try { + await handle.copyToHost(guestProofDir, hostIncomingDir) + } catch { + if (result.status === 'passed') { + return { + status: 'failed', + reason: 'proof capture passed but copying the evidence to the host failed', + } + } + } + return result +} + +async function runReplay( + handle: SandboxHandle, + opts: CaptureProofOptions, + quotedJourney: string, + quotedUrl: string, +): Promise { + const execOpts: SandboxExecOptions = { timeoutMs: opts.timeoutMs, cwd: opts.guestWorkDir } + const replayCommand = `CODESEMA_BASE_URL=${quotedUrl} CODESEMA_PROOF_DIR=${opts.guestProofDir} npx playwright test ${quotedJourney} --output=${opts.guestProofDir}` + const replay = await handle.shell(replayCommand, execOpts) + if (!replay.timedOut && replay.code === 0) { + return { status: 'passed', reason: null } + } + const reason = replay.timedOut + ? `replay timed out after ${opts.timeoutMs}ms` + : tail(replay.stdout + replay.stderr) + await handle + .shell( + `npx playwright screenshot --full-page ${quotedUrl} ${opts.guestProofDir}/fallback.png`, + execOpts, + ) + .catch(() => undefined) + return { status: 'failed', reason } +} + +async function captureProofUnsafe( + handle: SandboxHandle, + opts: CaptureProofOptions, +): Promise { + await ensureProofDir(handle, opts.guestProofDir, opts.timeoutMs) + + const quotedJourney = shellQuote(opts.journey) + const quotedUrl = shellQuote(opts.url) + + let result: ProofCaptureResult + if (quotedJourney === null || quotedUrl === null) { + const badField = quotedJourney === null ? 'journey' : 'url' + result = { + status: 'failed', + reason: `refusing to run the replay: ${badField} contains a single quote, which cannot be safely quoted for the shell`, + } + } else { + result = await runReplay(handle, opts, quotedJourney, quotedUrl) + } + + return finalizeCapture(handle, opts.guestProofDir, opts.hostIncomingDir, result) +} + +export async function captureProof( + handle: SandboxHandle, + opts: CaptureProofOptions, +): Promise { + try { + return await captureProofUnsafe(handle, opts) + } catch (err) { + return { status: 'failed', reason: err instanceof Error ? err.message : String(err) } + } +} + +async function captureScreenshotsUnsafe( + handle: SandboxHandle, + opts: CaptureScreenshotsOptions, +): Promise { + await ensureProofDir(handle, opts.guestProofDir, opts.timeoutMs) + + const execOpts: SandboxExecOptions = { timeoutMs: opts.timeoutMs, cwd: opts.guestWorkDir } + let passedCount = 0 + const tails: string[] = [] + for (const [index, page] of opts.pages.entries()) { + const resolvedUrl = new URL(page, opts.url).toString() + const quotedUrl = shellQuote(resolvedUrl) + if (quotedUrl === null) { + tails.push( + `refusing to screenshot ${page}: resolved URL contains a single quote, which cannot be safely quoted for the shell`, + ) + continue + } + const target = `${opts.guestProofDir}/p${index}.png` + const shot = await handle.shell( + `npx playwright screenshot --full-page ${quotedUrl} ${target}`, + execOpts, + ) + if (!shot.timedOut && shot.code === 0) { + passedCount += 1 + } else { + tails.push( + shot.timedOut + ? `${page}: screenshot timed out after ${opts.timeoutMs}ms` + : `${page}: ${shot.stdout}${shot.stderr}`, + ) + } + } + + const result: ProofCaptureResult = + passedCount > 0 + ? { status: 'passed', reason: null } + : { status: 'failed', reason: tail(tails.join('\n')) } + + return finalizeCapture(handle, opts.guestProofDir, opts.hostIncomingDir, result) +} + +export async function captureScreenshots( + handle: SandboxHandle, + opts: CaptureScreenshotsOptions, +): Promise { + try { + return await captureScreenshotsUnsafe(handle, opts) + } catch (err) { + return { status: 'failed', reason: err instanceof Error ? err.message : String(err) } + } +} diff --git a/packages/cli/src/task-recap.test.ts b/packages/cli/src/task-recap.test.ts index b5500f9..6ba5856 100644 --- a/packages/cli/src/task-recap.test.ts +++ b/packages/cli/src/task-recap.test.ts @@ -8,13 +8,16 @@ import { type AcceptanceCriterion, type CriterionVerdict, type RecapRecord, + type ReviewRecord, type TaskChecks, type TaskEvent, type TaskRecord, } from './contract.js' import { generateRecap, + lastTurnResponse, readTaskRecap, + recapOptionsFor, renderRecapMarkdown, writeTaskRecap, type DiffFilesFn, @@ -571,6 +574,147 @@ describe('generateRecap: the model contributes only summary/changes/decisions (i }) }) +// --- lastTurnResponse: the prose source for a recap generated before any ship + +function fakeTurn(over: Partial = {}): TaskRecord['turns'][number] { + return { + prompt: 'do it', + response: null, + question: null, + started_at: '2026-01-01T00:00:00.000Z', + ended_at: '2026-01-01T00:01:00.000Z', + ...over, + } +} + +describe('lastTurnResponse', () => { + test('the last turn carrying a response, not necessarily the last turn overall', () => { + const task = fakeTask({ + turns: [ + fakeTurn({ response: 'first turn done' }), + fakeTurn({ response: null, question: 'need more context' }), + ], + }) + expect(lastTurnResponse(task)).toEqual({ summary: 'first turn done' }) + }) + + test('no turn has a response yet: null, not an empty string', () => { + const task = fakeTask({ turns: [fakeTurn({ response: null })] }) + expect(lastTurnResponse(task)).toBeNull() + }) + + test('no turns at all: null', () => { + const task = fakeTask({ turns: [] }) + expect(lastTurnResponse(task)).toBeNull() + }) + + test('a hand-edited task.json without turns degrades to null instead of throwing', () => { + const task = fakeTask({ turns: undefined as unknown as TaskRecord['turns'] }) + expect(() => lastTurnResponse(task)).not.toThrow() + expect(lastTurnResponse(task)).toBeNull() + }) + + test('the result feeds generateRecap as modelOutput.summary, changes/decisions stay empty', () => { + const task = fakeTask({ turns: [fakeTurn({ response: 'Rewired the worktree cleanup.' })] }) + const contribution = lastTurnResponse(task) + const { recap } = generate( + baseOptions({ task, ...(contribution ? { modelOutput: contribution } : {}) }), + ) + expect(recap.summary).toBe('Rewired the worktree cleanup.') + expect(recap.changes).toEqual([]) + expect(recap.decisions).toEqual([]) + }) +}) + +// --- recapOptionsFor: the shared entries for a recap built outside the ship - + +function fakeReviewRecord(criteria: CriterionVerdict[]): ReviewRecord { + return { + version: 1, + meta: { + title: 'x', + branch: 'codesema/task-x', + target: 'main', + merge_base: 'deadbeef', + repo_root: '/unused', + created_at: '2026-01-01T00:00:00.000Z', + }, + commits: [], + diff: '', + review: { + verdict: 'approve', + summary: 'ok', + findings: [], + criteria, + }, + } as unknown as ReviewRecord +} + +describe('recapOptionsFor', () => { + test('sources criteria from taskCriteria + the injected review reader, and modelOutput from the last turn', () => { + const cwd = tmpCwd() + const criterion: AcceptanceCriterion = { + id: acceptanceCriterionId('WHEN a ticket is launched THE SYSTEM SHALL lint its body'), + text: 'WHEN a ticket is launched THE SYSTEM SHALL lint its body', + } + const task = createTask(cwd, { + title: 'x', + prompt: 'x', + autoShip: false, + base: '', + branch: 'codesema/task-x', + worktree: '', + isolation: 'policy', + }) + task.criteria = [criterion] + task.turns = [fakeTurn({ response: 'done' })] + const verdicts: CriterionVerdict[] = [ + { criterion_id: criterion.id, status: 'met', evidence: 'ok' }, + ] + const readTaskReviewFn = () => fakeReviewRecord(verdicts) + + const opts = recapOptionsFor(cwd, task, readTaskReviewFn) + + expect(opts.acceptanceCriteria).toEqual([criterion]) + expect(opts.criteriaVerdicts).toEqual(verdicts) + expect(opts.modelOutput).toEqual({ summary: 'done' }) + }) + + test('a task with no criteria at all: neither field is set (DP12)', () => { + const cwd = tmpCwd() + const task = createTask(cwd, { + title: 'x', + prompt: 'x', + autoShip: false, + base: '', + branch: 'codesema/task-y', + worktree: '', + isolation: 'policy', + }) + const opts = recapOptionsFor(cwd, task, () => null) + expect(opts.acceptanceCriteria).toBeUndefined() + expect(opts.criteriaVerdicts).toBeUndefined() + expect(opts.modelOutput).toBeUndefined() + }) + + test('by default (no readTaskReviewFn override) it reads the real review archive', () => { + const cwd = tmpCwd() + const task = createTask(cwd, { + title: 'x', + prompt: 'x', + autoShip: false, + base: '', + branch: 'codesema/task-z', + worktree: '', + isolation: 'policy', + }) + const opts = recapOptionsFor(cwd, task) + expect(opts.criteriaVerdicts).toBeUndefined() + expect(opts.cwd).toBe(cwd) + expect(opts.task).toBe(task) + }) +}) + // --- real git, default diffFilesFn: the un-injected seam actually works ------ function makeRepo(): string { diff --git a/packages/cli/src/task-recap.ts b/packages/cli/src/task-recap.ts index 21e8cfc..599746b 100644 --- a/packages/cli/src/task-recap.ts +++ b/packages/cli/src/task-recap.ts @@ -34,6 +34,8 @@ import { type TaskRecord, } from './contract.js' import { tryGit } from './git.js' +import { readTaskReview } from './task-review.js' +import { taskCriteria } from './task-runner.js' import { readTaskChecks, readTaskEvents, taskDir } from './tasks-store.js' /** Model's own draft of the recap: read for exactly these three keys, structurally nothing else (see module doc). */ @@ -270,6 +272,51 @@ function buildModelFields(opts: GenerateRecapOptions): { return { summary, changes, decisions } } +/** + * The agent's own last turn response, as `modelOutput.summary`: the same + * read task-ship.ts's `lastTurnContribution` does for the ship path, + * duplicated here rather than shared across that module boundary. The + * caller this ticket adds (task-server.ts's end-of-turn recap) has no other + * prose source and does not spend an extra model turn getting one: a + * deterministic recap without `changes[]`/`decisions[]` is what it produces + * instead. `Array.isArray`, not a bare `.findLast`: a hand-edited or + * truncated task.json can carry a record without `turns`, and this must + * degrade, not throw. + */ +export function lastTurnResponse(task: TaskRecord): RecapModelContribution | null { + const turns = Array.isArray(task.turns) ? task.turns : [] + const summary = turns.findLast((turn) => turn?.response)?.response + return summary ? { summary } : null +} + +/** + * `GenerateRecapOptions` for a recap built OUTSIDE the ship: task-server.ts's + * end-of-turn write, and the recap route's read-time fallback for a task + * that finished its turn before either existed. Both need the SAME + * criteria/model inputs, so this is the one place that builds them, + * `criteriaVerdicts`/`acceptanceCriteria` read `taskCriteria` and the + * review archive the same way task-ship.ts's own `criteriaForRecap` does for + * the ship path (duplicated there rather than shared across that module + * boundary, unchanged by this ticket). + */ +export function recapOptionsFor( + cwd: string, + task: TaskRecord, + readTaskReviewFn: typeof readTaskReview = readTaskReview, +): GenerateRecapOptions { + const criteria = taskCriteria(task) + const criteriaVerdicts = readTaskReviewFn(cwd, task.id)?.review.criteria + const modelOutput = lastTurnResponse(task) + return { + cwd, + task, + ...(modelOutput ? { modelOutput } : {}), + ...(criteria.length > 0 && criteriaVerdicts + ? { criteriaVerdicts, acceptanceCriteria: criteria } + : {}), + } +} + /** * Builds a `RecapRecord` for one task. Every factual field is read from its * ONE source (see module doc); the model contributes only prose. NEVER diff --git a/packages/cli/src/task-review.test.ts b/packages/cli/src/task-review.test.ts index 9d5d370..e44b5f3 100644 --- a/packages/cli/src/task-review.test.ts +++ b/packages/cli/src/task-review.test.ts @@ -1,5 +1,5 @@ import { execFileSync } from 'node:child_process' -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, test } from 'bun:test' @@ -9,6 +9,7 @@ import { type AcceptanceCriterion, type CriterionVerdict, type Finding, + type ProofReview, type ReviewRecord, type RunbookConfig, type TaskChecks, @@ -35,6 +36,7 @@ import { type runSimpleFlow, type SimpleOutcome, } from './review.js' +import { readTaskEvidence, writeTaskEvidence } from './task-evidence.js' import { DEFAULT_ISOLATION_ALLOWED_DOMAINS } from './task-isolation.js' import { actionableFindingIds, @@ -223,6 +225,15 @@ function verificationOf(over: Partial = {}): TaskVerification } } +/** Writes the repo's `.codesema/config.json` with a `proof.url`, the D17 target readProofConfig needs to return non-null. */ +function writeProofConfig(repo: string): void { + mkdirSync(join(repo, '.codesema'), { recursive: true }) + writeFileSync( + join(repo, '.codesema', 'config.json'), + JSON.stringify({ proof: { url: 'http://localhost:3000' } }), + ) +} + /** * Lot C1 (a parallel lot) has not implemented `FakeSandboxDriver` yet: a * minimal fake of the same `SandboxDriver` interface, local to this test @@ -1619,6 +1630,213 @@ describe('createTaskReviewer: the checks chapter (D16)', () => { }) }) +// --- D17: the visual proof chapter ----------------------------------------- + +function fakeReviewWithProof( + verdict: Verdict, + findings: Finding[], + proofReview: ProofReview, +): ReviewRecord { + const base = fakeReview(verdict, findings) + return { ...base, review: { ...base.review, proof_review: proofReview } } +} + +describe('createTaskReviewer: the visual proof chapter (D17)', () => { + test('no proof configured: no chapter, and no evidence.json is ever touched', async () => { + const repo = makeRepo() + const record = await makeTaskWithWorktree(repo, 'unconfigured proof task') + record.isolation = 'microvm' + saveTask(repo, record) + commitChange(record.worktree, 'App.vue') + const rig = fakeIo(record) + const flow = fakeSimpleFlow({ ok: true, record: fakeReview('approve'), reportLines: [] }) + + await reviewer(repo, { runSimpleFlowFn: flow.fn })(record, rig.io) + + expect(flow.calls[0]?.prompt ?? '').not.toContain('Visual proof') + expect(readTaskEvidence(repo, record.id)).toBeNull() + }) + + test('a non-microvm task never gets the chapter, even with proof configured', async () => { + const repo = makeRepo() + writeProofConfig(repo) + const record = await makeTaskWithWorktree(repo, 'policy-isolated task') + commitChange(record.worktree, 'App.vue') + const rig = fakeIo(record) + const flow = fakeSimpleFlow({ ok: true, record: fakeReview('approve'), reportLines: [] }) + + await reviewer(repo, { runSimpleFlowFn: flow.fn })(record, rig.io) + + expect(flow.calls[0]?.prompt ?? '').not.toContain('Visual proof') + }) + + test('a microvm task with proof configured gets the chapter, naming the UI files and the grid', async () => { + const repo = makeRepo() + writeProofConfig(repo) + const record = await makeTaskWithWorktree(repo, 'proof-eligible task') + record.isolation = 'microvm' + saveTask(repo, record) + commitChange(record.worktree, 'App.vue') + const rig = fakeIo(record) + const flow = fakeSimpleFlow({ ok: true, record: fakeReview('approve'), reportLines: [] }) + + await reviewer(repo, { runSimpleFlowFn: flow.fn })(record, rig.io) + + const prompt = flow.calls[0]?.prompt ?? '' + expect(prompt).toContain('Visual proof (MANDATORY chapter)') + expect(prompt).toContain('UI files touched by this diff: App.vue') + expect(prompt).toContain('"proof_review"') + expect(prompt).toContain('declaration: the agent did not declare a proof this turn') + expect(prompt).toContain('proof produced: no proof for this commit') + }) + + test('evidence from a DIFFERENT head_sha than the reviewed record is never read', async () => { + const repo = makeRepo() + writeProofConfig(repo) + const record = await makeTaskWithWorktree(repo, 'stale evidence task') + record.isolation = 'microvm' + record.head_sha = 'a'.repeat(40) + saveTask(repo, record) + commitChange(record.worktree, 'App.vue') + writeTaskEvidence(repo, record.id, { + version: 1, + status: 'passed', + reason: null, + head_sha: 'b'.repeat(40), + items: [ + { + kind: 'screenshot', + path: 'x.png', + bytes: 10, + turn: 1, + created_at: new Date().toISOString(), + }, + ], + }) + const rig = fakeIo(record) + const flow = fakeSimpleFlow({ ok: true, record: fakeReview('approve'), reportLines: [] }) + + await reviewer(repo, { runSimpleFlowFn: flow.fn })(record, rig.io) + + const prompt = flow.calls[0]?.prompt ?? '' + expect(prompt).toContain('proof produced: no proof for this commit') + expect(prompt).not.toContain('x.png') + }) + + test('an incoherent proof_review with a design/major finding blocks the task via hasBlockingFindings, and evidence.json records the verdict for the matching head_sha', async () => { + const repo = makeRepo() + writeProofConfig(repo) + const record = await makeTaskWithWorktree(repo, 'incoherent proof task') + record.isolation = 'microvm' + record.head_sha = 'c'.repeat(40) + saveTask(repo, record) + commitChange(record.worktree, 'App.vue') + writeTaskEvidence(repo, record.id, { + version: 1, + status: 'skipped', + reason: 'undeclared, defaulted to none', + head_sha: 'c'.repeat(40), + items: [], + }) + const rig = fakeIo(record) + const finding: Finding = { + file: 'App.vue', + line: 1, + severity: 'major', + kind: 'design', + message: 'the interface changed but no proof was captured for it', + } + const proofReview: ProofReview = { + expected: 'screenshot', + coherent: false, + reason: 'the diff shows a visible UI change with no captured proof', + } + const flow = fakeSimpleFlow({ + ok: true, + record: fakeReviewWithProof('approve', [finding], proofReview), + reportLines: [], + }) + + await reviewer(repo, { runSimpleFlowFn: flow.fn })(record, rig.io) + + expect(record.status).toBe('review_ko') + const evidence = readTaskEvidence(repo, record.id) + expect(evidence?.review).toEqual(proofReview) + }) + + test('a coherent proof_review with no finding never blocks, and still records the verdict', async () => { + const repo = makeRepo() + writeProofConfig(repo) + const record = await makeTaskWithWorktree(repo, 'coherent proof task') + record.isolation = 'microvm' + record.head_sha = 'd'.repeat(40) + saveTask(repo, record) + commitChange(record.worktree, 'App.vue') + writeTaskEvidence(repo, record.id, { + version: 1, + status: 'skipped', + reason: 'no visible effect', + head_sha: 'd'.repeat(40), + items: [], + }) + const rig = fakeIo(record) + const proofReview: ProofReview = { + expected: 'none', + coherent: true, + reason: 'a pure refactor with no rendered difference', + } + const flow = fakeSimpleFlow({ + ok: true, + record: fakeReviewWithProof('approve', [], proofReview), + reportLines: [], + }) + + await reviewer(repo, { runSimpleFlowFn: flow.fn })(record, rig.io) + + expect(record.status).toBe('review_ok') + const evidence = readTaskEvidence(repo, record.id) + expect(evidence?.review).toEqual(proofReview) + }) + + test('the chapter was injected but the reviewer JSON carries no proof_review: journaled, never invented', async () => { + const repo = makeRepo() + writeProofConfig(repo) + const record = await makeTaskWithWorktree(repo, 'silent proof task') + record.isolation = 'microvm' + record.head_sha = 'e'.repeat(40) + saveTask(repo, record) + commitChange(record.worktree, 'App.vue') + writeTaskEvidence(repo, record.id, { + version: 1, + status: 'skipped', + reason: 'no visible effect', + head_sha: 'e'.repeat(40), + items: [], + }) + const rig = fakeIo(record) + const flow = fakeSimpleFlow({ ok: true, record: fakeReview('approve'), reportLines: [] }) + + await reviewer(repo, { runSimpleFlowFn: flow.fn })(record, rig.io) + + const proofEvents = rig.events.filter((event) => event.type === 'proof') + expect(proofEvents).toHaveLength(1) + expect(proofEvents[0]?.data).toMatchObject({ name: 'review_missing' }) + expect(readTaskEvidence(repo, record.id)?.review).toBeUndefined() + }) + + test('no chapter was injected: a missing proof_review is never journaled', async () => { + const repo = makeRepo() + const record = await makeTaskWithWorktree(repo, 'no chapter, no proof event task') + commitChange(record.worktree, 'App.vue') + const rig = fakeIo(record) + const flow = fakeSimpleFlow({ ok: true, record: fakeReview('approve'), reportLines: [] }) + + await reviewer(repo, { runSimpleFlowFn: flow.fn })(record, rig.io) + + expect(rig.events.some((event) => event.type === 'proof')).toBe(false) + }) +}) + // --- D17: mechanical criteria decided without the reviewer ------------------ describe('createTaskReviewer: mechanical criteria (D17)', () => { @@ -2586,6 +2804,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver() const stdout = await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2614,6 +2833,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver() await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2633,6 +2853,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver() await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2653,6 +2874,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver() await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2672,6 +2894,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver() await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2693,6 +2916,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver() await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2721,6 +2945,7 @@ describe('runMicrovmReview', () => { const fake2 = fakeMicrovmDriver() await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake1.driver, worktree: repo, projectId: 'proj-1', @@ -2731,6 +2956,7 @@ describe('runMicrovmReview', () => { timeoutMs: 5000, }) await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake2.driver, worktree: repo, projectId: 'proj-1', @@ -2753,6 +2979,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver() await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2789,6 +3016,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver() await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2818,6 +3046,7 @@ describe('runMicrovmReview', () => { await expect( runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2847,6 +3076,7 @@ describe('runMicrovmReview', () => { }) const stdout = await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2871,6 +3101,7 @@ describe('runMicrovmReview', () => { await Promise.all([ runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2882,6 +3113,7 @@ describe('runMicrovmReview', () => { taskId: 'task-shared', }), runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2909,6 +3141,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver() await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2928,6 +3161,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver({ destroyError: new Error('sandbox already gone') }) const stdout = await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2951,6 +3185,7 @@ describe('runMicrovmReview', () => { await expect( runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2970,6 +3205,7 @@ describe('runMicrovmReview', () => { const fake = fakeMicrovmDriver({ probeMissingFor: 'opencode' }) await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -2994,6 +3230,7 @@ describe('runMicrovmReview', () => { await expect( runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, driver: fake.driver, worktree: repo, projectId: 'proj-1', @@ -3009,6 +3246,100 @@ describe('runMicrovmReview', () => { expect(fake.calls.some((c) => String(c.args[0]).includes('npm install -g'))).toBe(false) }) }) + + describe('agent credentials', () => { + function makeCredentialsFile(content = '{"token":"secret-token-value"}'): string { + const dir = mkdtempSync(join(tmpdir(), 'codesema-review-creds-')) + cleanups.push(dir) + writeFileSync(join(dir, 'credentials.json'), content) + return join(dir, 'credentials.json') + } + + test('no oauth token, credentials file present: written into the review VM, chmod 600, chowned, never in a shell command', async () => { + const repo = makeRepo() + const fake = fakeMicrovmDriver() + const credentialsPath = makeCredentialsFile('{"token":"secret-token-value"}') + + await runMicrovmReview({ + env: {}, + credentialsPath, + driver: fake.driver, + worktree: repo, + projectId: 'proj-1', + snapshotName: null, + image: 'node:26', + command: 'claude -p', + prompt: 'p', + timeoutMs: 5000, + taskId: 'task-abc', + }) + + const write = fake.calls.find((c) => c.method === 'writeFile') + expect(write?.args).toEqual([ + '/home/agent/.claude/.credentials.json', + '{"token":"secret-token-value"}', + ]) + const chmodChown = fake.calls.find( + (c) => c.method === 'shell' && String(c.args[0]).includes('chmod 600'), + ) + expect(chmodChown?.args[0]).toBe( + 'chmod 600 /home/agent/.claude/.credentials.json && chown -R agent:agent /home/agent/.claude', + ) + expect((chmodChown?.args[1] as { user?: string } | undefined)?.user).toBe('root') + for (const call of fake.calls) { + if (call.method === 'shell') { + expect(String(call.args[0])).not.toContain('secret-token-value') + } + } + }) + + test('CLAUDE_CODE_OAUTH_TOKEN present: nothing is copied into the review VM', async () => { + const repo = makeRepo() + const fake = fakeMicrovmDriver() + const credentialsPath = makeCredentialsFile() + + await runMicrovmReview({ + env: { CLAUDE_CODE_OAUTH_TOKEN: 'tok-secret' }, + credentialsPath, + driver: fake.driver, + worktree: repo, + projectId: 'proj-1', + snapshotName: null, + image: 'node:26', + command: 'claude -p', + prompt: 'p', + timeoutMs: 5000, + taskId: 'task-abc', + }) + + expect(fake.calls.some((c) => c.method === 'writeFile')).toBe(false) + }) + + test('no credentials file on the host: no write, no error', async () => { + const missingDir = mkdtempSync(join(tmpdir(), 'codesema-review-creds-')) + cleanups.push(missingDir) + const missingPath = join(missingDir, 'nope.json') + const repo = makeRepo() + const fake = fakeMicrovmDriver() + + const stdout = await runMicrovmReview({ + env: {}, + credentialsPath: missingPath, + driver: fake.driver, + worktree: repo, + projectId: 'proj-1', + snapshotName: null, + image: 'node:26', + command: 'claude -p', + prompt: 'p', + timeoutMs: 5000, + taskId: 'task-abc', + }) + + expect(stdout).toBe('{"verdict":"approve","summary":"ok","findings":[]}') + expect(fake.calls.some((c) => c.method === 'writeFile')).toBe(false) + }) + }) }) // --- createTaskReviewer: microvm wiring (lot C8) --------------------------- @@ -3050,7 +3381,21 @@ describe('createTaskReviewer: microvm wiring', () => { // Calling it drives the fake driver exactly as `runMicrovmReview` alone // does — the wiring built by `createTaskReviewer`, exercised end to end. - const raw = await runAgentInVm?.('hand-built prompt') + // This call site (task-review.ts's own runAgentInVm closure) has no env + // seam, so it falls through to the real process.env: pinned here so + // ensureAgentCredentials never touches this machine's real credentials. + const previousToken = process.env.CLAUDE_CODE_OAUTH_TOKEN + process.env.CLAUDE_CODE_OAUTH_TOKEN = 'tok-secret' + let raw: string | undefined + try { + raw = await runAgentInVm?.('hand-built prompt') + } finally { + if (previousToken === undefined) { + delete process.env.CLAUDE_CODE_OAUTH_TOKEN + } else { + process.env.CLAUDE_CODE_OAUTH_TOKEN = previousToken + } + } expect(raw).toBe('{"verdict":"approve","summary":"ok","findings":[]}') expect(fake.specs[0]?.name).toMatch(new RegExp(`^codesema-review-${record.id}-[0-9a-f]{8}$`)) expect(fake.calls.map((c) => c.method)).toEqual([ diff --git a/packages/cli/src/task-review.ts b/packages/cli/src/task-review.ts index a8d50ef..fe69c09 100644 --- a/packages/cli/src/task-review.ts +++ b/packages/cli/src/task-review.ts @@ -13,6 +13,7 @@ import { randomUUID } from 'node:crypto' import { ensureWorkDir, type ReviewMode } from './config.js' import { sanitizeRecord, + type EvidenceRecord, type Finding, type ReviewRecord, type RunbookConfig, @@ -34,11 +35,13 @@ import { } from './microsandbox-driver.js' import { AGENT_INSTALL_DOMAINS, + ensureAgentCredentials, ensureAgentInstalled, ensureGuestUser, } from './microvm-bootstrap.js' import { prep } from './prep.js' import { archiveRecord, findPreviousReview, readJson, resolveArchivePath } from './record.js' +import { readProofConfig } from './repo-config.js' import { buildFullReviewPrompt, buildIncrementalPrompt, @@ -62,6 +65,7 @@ import { unmetCriteriaFixChapter, type CriteriaOutcome, } from './task-criteria-gate.js' +import { readTaskEvidence, writeTaskEvidence } from './task-evidence.js' import { reportHubTransition, type ArmTransitionDraft } from './task-hub.js' import { commandBin, @@ -69,6 +73,7 @@ import { DEFAULT_BASE_IMAGE, DEFAULT_ISOLATION_ALLOWED_DOMAINS, } from './task-isolation.js' +import { buildProofChapter } from './task-proof-chapter.js' import { REVIEW_CUT_DETAIL, taskCriteria, @@ -76,6 +81,7 @@ import { type TaskTurnReviewFn, } from './task-runner.js' import { loadTask, readTaskChecks, taskReason } from './tasks-store.js' +import { classifyUiPaths } from './ui-surface.js' import { progressLabel } from './ui.js' /** @@ -831,8 +837,36 @@ export function createTaskReviewer(opts: CreateTaskReviewerOptions): TaskTurnRev reviewCtx.runbook, reviewCtx.verification, ) + // D17: only a 'microvm' task whose project actually configured a proof + // target earns this chapter: a task with no target has nothing to + // replay a proof against, and a non-'microvm' task never captures one + // at all (task-server.ts's verifyAfterCommit gates capture the same + // way). `proofEvidence` is read once, here, and reused after the + // review to decide whether this turn's verdict may be folded back into + // evidence.json (see below): it is null whenever the file on disk does + // not match THIS record's own head_sha, which is the honest "no proof + // for this commit" rather than a stale one from an earlier turn. + let proofEvidence: EvidenceRecord | null = null + let proofChapter: string | null = null + if (record.isolation === 'microvm') { + const proofConfig = readProofConfig(opts.cwd) + if (proofConfig) { + const { ui, other } = classifyUiPaths(changed ? changed.split('\n').filter(Boolean) : []) + const lastTurn = record.turns.at(-1) + const intent = lastTurn?.proof_intent ?? null + const onDisk = readTaskEvidence(opts.cwd, record.id) + proofEvidence = onDisk && onDisk.head_sha === record.head_sha ? onDisk : null + proofChapter = buildProofChapter({ + uiFiles: ui, + otherCount: other.length, + intent, + evidence: proofEvidence, + declared: intent !== null, + }) + } + } const chapter = - [criteriaChapter, checksChapter, runbookChapter] + [criteriaChapter, checksChapter, runbookChapter, proofChapter] .filter((c): c is string => Boolean(c)) .join('\n\n') || null @@ -975,6 +1009,34 @@ export function createTaskReviewer(opts: CreateTaskReviewerOptions): TaskTurnRev }) outcome.record.review.findings = reproOutcome.findings + // D17: the reviewer's proof_review verdict is folded back into + // evidence.json right beside the finding-repro pass above, same file + // task-server.ts's onTurnDone re-reads and re-emits once this hook + // returns. Only when the evidence this review actually judged (the one + // matching THIS record's own head_sha, resolved once above) is still + // on disk: a review with nothing to judge (no proof chapter, an + // unconfigured project) never writes one. + if (outcome.record.review.proof_review) { + if (proofEvidence) { + writeTaskEvidence(opts.cwd, record.id, { + ...proofEvidence, + review: outcome.record.review.proof_review, + }) + } + } else if (proofChapter) { + // Observed in the field: the chapter was mandatory, the model wrote a + // valid review anyway, and "proof_review" simply never came back. + // Never invented here: evidence.json's `review` stays absent, since + // fabricating a verdict the model never gave would be worse than + // saying nothing. Journaled instead, on the same 'proof' channel as + // 'declared'/'undeclared'/'unparsed' (task-runner.ts), so a run + // missing the verdict is as visible as one missing the declaration. + const message = + "the reviewer's JSON carried no proof_review despite the mandatory visual-proof chapter: the PROOF declaration above was never judged" + io.emit({ type: 'proof', data: { name: 'review_missing', message } }) + console.warn(`${record.id}: ${message}`) + } + // T3.2, and BEFORE the archive on purpose: the normalized per-criterion // statuses are what T3.6 reads back, possibly at a later boot, so they // have to be part of the record that lands on disk, not a structure that @@ -1138,6 +1200,9 @@ export type RunMicrovmReviewOptions = { secrets?: readonly SandboxSecret[] onText?: (text: string) => void signal?: AbortSignal + env?: NodeJS.ProcessEnv + /** Test seam: overrides ensureAgentCredentials' host credentials file. */ + credentialsPath?: string } const MICROVM_REVIEW_DEFAULTS = { @@ -1191,6 +1256,10 @@ export async function runMicrovmReview(opts: RunMicrovmReviewOptions): Promise { }) }) +function sampleProof(overrides: Partial = {}): ProofConfig { + return { + journey: 'tests/e2e/main-flow.spec.ts', + url: 'http://localhost:3000', + timeoutSeconds: 30, + keep: 5, + ...overrides, + } +} + +describe('buildTaskPrompt with a proof config', () => { + test('microvm isolation with a proof config adds the D17 PROOF bullet', () => { + const task = { title: 'Add rate limiting', isolation: 'microvm' } as TaskRecord + const prompt = buildTaskPrompt(task, { proof: sampleProof() }) + expect(prompt).toContain('PROOF:') + expect(prompt).toContain('CODESEMA_BASE_URL') + expect(prompt).toContain('tests/e2e/main-flow.spec.ts') + }) + + test('url alone (no default journey configured) still adds the bullet, with no default-journey mention', () => { + const task = { title: 'Add rate limiting', isolation: 'microvm' } as TaskRecord + const prompt = buildTaskPrompt(task, { proof: sampleProof({ journey: null }) }) + expect(prompt).toContain('PROOF:') + expect(prompt).not.toContain('default journey spec') + }) + + test('non-microvm isolation never adds the bullet, even with a proof config', () => { + const containerTask = { title: 'Add rate limiting', isolation: 'container' } as TaskRecord + expect(buildTaskPrompt(containerTask, { proof: sampleProof() })).not.toContain('PROOF:') + const policyTask = { title: 'Add rate limiting', isolation: 'policy' } as TaskRecord + expect(buildTaskPrompt(policyTask, { proof: sampleProof() })).not.toContain('PROOF:') + }) + + test('microvm isolation without a proof config adds nothing', () => { + const task = { title: 'Add rate limiting', isolation: 'microvm' } as TaskRecord + expect(buildTaskPrompt(task)).not.toContain('PROOF:') + expect(buildTaskPrompt(task, { proof: null })).not.toContain('PROOF:') + }) +}) + describe('parseTaskBranchProposal', () => { test('the first line names the branch and leaves the reply', () => { expect(parseTaskBranchProposal('BRANCH: fix-preview-rename\n\nDid the thing.')).toEqual({ @@ -339,6 +381,73 @@ describe('parseCriteriaProposal', () => { }) }) +describe('parseProofDeclaration (D17)', () => { + test('none: kind and reason, the line is stripped', () => { + const parsed = parseProofDeclaration( + 'PROOF: none | only server-side logging changed\nDid the thing.', + ) + expect(parsed.intent).toEqual({ kind: 'none', reason: 'only server-side logging changed' }) + expect(parsed.rest).toBe('Did the thing.') + }) + + test('screenshot: pages are collected in declaration order', () => { + const parsed = parseProofDeclaration( + 'PROOF: screenshot /dashboard /dashboard/settings | the settings page now shows the new toggle\nDone.', + ) + expect(parsed.intent).toEqual({ + kind: 'screenshot', + reason: 'the settings page now shows the new toggle', + pages: ['/dashboard', '/dashboard/settings'], + }) + expect(parsed.rest).toBe('Done.') + }) + + test('journey: the spec path is read', () => { + const parsed = parseProofDeclaration( + 'PROOF: journey tests/e2e/checkout.spec.ts | the checkout flow gained a confirmation step\nDone.', + ) + expect(parsed.intent).toEqual({ + kind: 'journey', + reason: 'the checkout flow gained a confirmation step', + journey: 'tests/e2e/checkout.spec.ts', + }) + expect(parsed.rest).toBe('Done.') + }) + + test('a quoted kind is not the protocol shape: intent null, rest unchanged', () => { + const text = "PROOF: 'none' | reason\nDone." + expect(parseProofDeclaration(text)).toEqual({ intent: null, rest: text }) + }) + + test('no PROOF line at all is the normal absent case', () => { + const text = 'all done, tests pass' + expect(parseProofDeclaration(text)).toEqual({ intent: null, rest: text }) + }) + + test('a PROOF: line mid-text is prose, not protocol', () => { + const text = 'did stuff\nPROOF: none | too late' + expect(parseProofDeclaration(text)).toEqual({ intent: null, rest: text }) + }) + + test('the shape matches but the reason is blank: unusable, still stripped as protocol', () => { + const parsed = parseProofDeclaration('PROOF: none | \nDone.') + expect(parsed.intent).toBeNull() + expect(parsed.rest).toBe('Done.') + }) + + test('parses after a CRITERION block once the criteria are already stripped', () => { + const full = [ + 'CRITERION: WHEN a THE SYSTEM SHALL b', + 'PROOF: none | no interface touched', + 'Done.', + ].join('\n') + const criteria = parseCriteriaProposal(full) + const parsed = parseProofDeclaration(criteria?.rest ?? '') + expect(parsed.intent).toEqual({ kind: 'none', reason: 'no interface touched' }) + expect(parsed.rest).toBe('Done.') + }) +}) + // --- test rig: real git repo + real store + injected agent --- const cleanups: string[] = [] @@ -739,6 +848,94 @@ describe('runTaskTurn', () => { }) }) +describe('runTaskTurn: PROOF declaration wiring (D17)', () => { + test('proofAsked + a declared PROOF line: journaled and carried on the outcome, line stripped from the response', async () => { + const repo = makeRepo() + const task = makeTask(repo, 'demo', 'do the thing') + const events: { type: string; data: Record }[] = [] + const outcome = await runTaskTurn({ + cwd: repo, + task, + prompt: 'do the thing', + command: 'claude -p', + timeoutMs: 1000, + proofAsked: true, + onEvent: (e) => events.push(e), + runAgentFn: fakeClaude(() => 'PROOF: none | only server-side logging changed\nAll done.').run, + }) + expect(outcome.response).toBe('All done.') + expect(outcome.kind).toBe('done') + expect(outcome.kind === 'done' ? outcome.proofIntent : undefined).toEqual({ + kind: 'none', + reason: 'only server-side logging changed', + }) + const proofEvents = events.filter((e) => e.type === 'proof') + expect(proofEvents).toEqual([{ type: 'proof', data: { name: 'declared', kind: 'none' } }]) + }) + + test('proofAsked + a present but blank-reason line: journaled as unparsed, no proofIntent, still stripped', async () => { + const repo = makeRepo() + const task = makeTask(repo, 'demo', 'do the thing') + const events: { type: string; data: Record }[] = [] + const outcome = await runTaskTurn({ + cwd: repo, + task, + prompt: 'do the thing', + command: 'claude -p', + timeoutMs: 1000, + proofAsked: true, + onEvent: (e) => events.push(e), + runAgentFn: fakeClaude(() => 'PROOF: none | \nAll done.').run, + }) + expect(outcome.response).toBe('All done.') + expect(outcome.kind === 'done' ? outcome.proofIntent : undefined).toBeUndefined() + const unparsed = events.find((e) => e.type === 'proof' && e.data.name === 'unparsed') + expect(unparsed).toBeDefined() + expect(typeof unparsed?.data.message).toBe('string') + expect(String(unparsed?.data.message).length).toBeGreaterThan(0) + }) + + test('proofAsked + no PROOF line at all: journaled as undeclared', async () => { + const repo = makeRepo() + const task = makeTask(repo, 'demo', 'do the thing') + const events: { type: string; data: Record }[] = [] + const outcome = await runTaskTurn({ + cwd: repo, + task, + prompt: 'do the thing', + command: 'claude -p', + timeoutMs: 1000, + proofAsked: true, + onEvent: (e) => events.push(e), + runAgentFn: fakeClaude(() => 'All done, no protocol here.').run, + }) + expect(outcome.response).toBe('All done, no protocol here.') + expect(outcome.kind === 'done' ? outcome.proofIntent : undefined).toBeUndefined() + expect(events.filter((e) => e.type === 'proof')).toEqual([ + { type: 'proof', data: { name: 'undeclared' } }, + ]) + }) + + test('proofAsked not set (the common non-microvm case): no proof event at all, even with a PROOF-shaped line', async () => { + const repo = makeRepo() + const task = makeTask(repo, 'demo', 'do the thing') + const events: { type: string; data: Record }[] = [] + const outcome = await runTaskTurn({ + cwd: repo, + task, + prompt: 'do the thing', + command: 'claude -p', + timeoutMs: 1000, + onEvent: (e) => events.push(e), + runAgentFn: fakeClaude(() => 'PROOF: none | unrelated to this turn\nAll done.').run, + }) + // Never parsed: the line stays exactly where the agent put it. + expect(outcome.response).toBe('PROOF: none | unrelated to this turn\nAll done.') + expect(outcome.kind === 'done' ? outcome.proofIntent : undefined).toBeUndefined() + expect(events.some((e) => e.type === 'proof')).toBe(false) + }) +}) + // --- createTaskRunner --- describe('createTaskRunner', () => { @@ -4093,6 +4290,28 @@ describe('microvm isolation branch', () => { ) expect(calls[0]?.worktree).toBe(record?.worktree) }) + + test("a declared PROOF line persists to the turn's proof_intent (D17)", async () => { + const repo = makeRepo() + const task = makeTask(repo, 'vmed', 'do it', 'microvm') + writeFileSync( + join(repo, '.codesema', 'config.json'), + JSON.stringify({ proof: { url: 'http://localhost:3000' } }), + ) + const runner = createTaskRunner({ + cwd: repo, + command: 'claude -p', + timeoutMs: 1000, + resolveMicrovmFn: () => Promise.resolve(microvmOptions()), + runMicrovmTurnFn: () => + Promise.resolve(claudeStream('PROOF: none | only server-side logging changed\nDone.')), + }) + expect(runner.start(task)).toEqual({ ok: true }) + await until(() => status(repo, task.id) === 'waiting_for_you') + const turn = loadTask(repo, task.id)?.turns[0] + expect(turn?.response).toBe('Done.') + expect(turn?.proof_intent).toEqual({ kind: 'none', reason: 'only server-side logging changed' }) + }) }) // --- T1.7: the DEFAULT configuration, end to end --------------------------- diff --git a/packages/cli/src/task-runner.ts b/packages/cli/src/task-runner.ts index e537757..0aad968 100644 --- a/packages/cli/src/task-runner.ts +++ b/packages/cli/src/task-runner.ts @@ -36,9 +36,12 @@ import { EARS_TRIGGER, isTerminalReason, reasonCodeOf, + sanitizeProofIntent, TASK_ATTACHMENTS_MAX, TICKET_CRITERIA_MIN, type AcceptanceCriterion, + type ProofIntent, + type ProofIntentKind, type ReasonCode, type RunbookConfig, type TaskEvent, @@ -65,7 +68,7 @@ import { import type { SandboxDriver, SandboxSecret } from './microsandbox-driver.js' import { runMicrovmTurn, type RunMicrovmTurnOptions } from './microvm-turn.js' import { projectIdFor } from './projects.js' -import type { ChecksConfig } from './repo-config.js' +import { readProofConfig, type ChecksConfig, type ProofConfig } from './repo-config.js' import { RUNNER_FALLBACK_GIT_IDENTITY } from './runner-secrets.js' import { bootstrapWorktreeInstall, @@ -249,8 +252,15 @@ function criteriaDraftInstruction(): string { * `askBranchName` adds the one-line BRANCH protocol, asked on the FIRST turn * of a forked task only (see parseTaskBranchProposal): once the branch has the * agent's name, re-asking would only invite a rename mid-conversation. + * + * `proof` adds the D17 PROOF declaration bullet, only when the task's + * isolation is 'microvm': the standing rules never ask a caged or + * policy-isolated turn for a proof it has no way to capture. */ -export function buildTaskPrompt(task: TaskRecord, opts: { askBranchName?: boolean } = {}): string { +export function buildTaskPrompt( + task: TaskRecord, + opts: { askBranchName?: boolean; proof?: ProofConfig | null } = {}, +): string { const lines = [ 'You are an autonomous coding agent working on a task in a dedicated git worktree of this repository (your current directory).', '', @@ -270,6 +280,16 @@ export function buildTaskPrompt(task: TaskRecord, opts: { askBranchName?: boolea : []), '- Follow the existing code style and conventions of the repository.', '- If the repo has cheap checks (typecheck, unit tests, lint), run them and fix what YOUR changes broke before finishing.', + ...(task.isolation === 'microvm' && opts.proof + ? [ + "- Before your final message, decide whether this turn's changes are visible in the interface, and open your final message with a line of the exact form 'PROOF: [pages or spec path] | ' (first lines of your final message, right after any BRANCH:/CRITERION: lines). If the interface changed and the change is visible, use 'screenshot' naming the changed pages (e.g. 'PROOF: screenshot /dashboard /dashboard/settings | the settings page now shows the new toggle'), or 'journey' naming a Playwright spec path when the change is a sequence across screens rather than one (e.g. 'PROOF: journey tests/e2e/checkout.spec.ts | the checkout flow gained a new confirmation step'), reading the base URL from the CODESEMA_BASE_URL environment variable. If a UI file changed with no visible effect, or nothing outside the interface was touched, use 'none' with a precise reason (e.g. 'PROOF: none | only server-side logging changed, nothing rendered differs'). When in doubt, capture proof rather than skip it.", + ...(opts.proof.journey + ? [ + `- This repository's default journey spec is at ${opts.proof.journey}: update it if it covers the flow this task touches, rather than creating a new one.`, + ] + : []), + ] + : []), `- Language: ${taskLanguageRule()}. This covers your summary and any 'QUESTION: ' line; code identifiers, file paths and commit messages stay as they are.`, "- If you cannot proceed without a human decision, end your reply with a single final line of the exact form 'QUESTION: ' (nothing after that line). Ask only when truly blocked.", '- Otherwise end your reply with a short plain-text summary of what you did and how you verified it (no code fences).', @@ -366,6 +386,52 @@ export function parseCriteriaProposal(response: string): TaskCriteriaProposal | return { texts, rest: lines.slice(i).join('\n').trim() } } +export type TaskProofDeclaration = { + /** Parsed intent, or null when the line was absent or unusable. */ + intent: ProofIntent | null + /** The reply with the protocol line removed; unchanged when none was found. */ + rest: string +} + +/** Group 1: kind. Group 2: pages/journey tokens. Group 3: reason. */ +const PROOF_DECLARATION_RE = /^PROOF:\s*(none|screenshot|journey)\b([^|]*)\|\s*(.+)$/i + +/** + * The D17 prompt asks the agent to OPEN its final message (after any + * BRANCH:/CRITERION: lines already consumed) with a dedicated + * `PROOF: [pages or spec] | ` line. Like `parseTaskBranchProposal`, + * only the FIRST line of the text is recognized as protocol: a PROOF: + * mention further down is prose, not a declaration, so `rest` comes back + * byte-for-byte unchanged and `intent` null. A first line that matches the + * shape but whose content `sanitizeProofIntent` refuses (e.g. an empty + * reason) is still protocol and still stripped: `intent` is null but `rest` + * loses the line, exactly like an unreadable CRITERION list. + */ +export function parseProofDeclaration(text: string): TaskProofDeclaration { + const trimmed = text.trimStart() + const breakAt = trimmed.indexOf('\n') + // NOT trimmed at the end, unlike parseTaskBranchProposal's `first`: a line + // whose reason is pure trailing whitespace (the agent forgot to state one) + // must still match the shape and fail sanitizeProofIntent's empty-reason + // check, landing on 'unparsed' rather than silently reading as no line at + // all: trimming here first would strip that whitespace before the regex + // ever saw it and turn a forgotten reason into a falsely undeclared turn. + const first = breakAt === -1 ? trimmed : trimmed.slice(0, breakAt) + const match = PROOF_DECLARATION_RE.exec(first) + if (!match) { + return { intent: null, rest: text } + } + const rest = (breakAt === -1 ? '' : trimmed.slice(breakAt + 1)).trim() + const kind = (match[1] ?? '').toLowerCase() as ProofIntentKind + const tokens = (match[2] ?? '').trim().split(/\s+/).filter(Boolean) + // pages/journey are handed over unconditionally: sanitizeProofIntent already + // reads only the one that matches `kind` and drops an empty pages array or + // an undefined journey on its own, so gating them here first would only + // repeat that same decision. + const raw = { kind, reason: (match[3] ?? '').trim(), pages: tokens, journey: tokens[0] } + return { intent: sanitizeProofIntent(raw), rest } +} + /** * The task prompt asks the agent to END its reply with 'QUESTION: ' * when it needs a human decision: only the last non-empty line counts, a @@ -553,6 +619,8 @@ export type TaskTurnOutcome = cost: TurnCost | null /** First turn only: the branch name the agent proposed for the task. */ branchProposal?: string + /** D17: the agent's declared visual-proof intent for this turn, when it declared one. */ + proofIntent?: ProofIntent } | { kind: 'question' @@ -567,6 +635,8 @@ export type TaskTurnOutcome = cost: TurnCost | null /** First turn only: the branch name the agent proposed for the task. */ branchProposal?: string + /** D17: the agent's declared visual-proof intent for this turn, when it declared one. */ + proofIntent?: ProofIntent } export type RunTaskTurnMicrovmOptions = { @@ -589,6 +659,14 @@ export type RunTaskTurnOptions = { prompt: string /** Raw configured agent command; per-turn flags are added here. */ command: string + /** + * True when this turn's prompt carried the D17 PROOF protocol bullet, or, + * for a resumed session that skipped resending the standing rules, when it + * carried the bullet earlier in that same session. Gates whether a missing + * `PROOF:` line is journaled as `undeclared` rather than silently ignored: + * a turn never asked for a declaration cannot be faulted for not sending one. + */ + proofAsked?: boolean /** Last-resort absolute ceiling of the turn; the watchdog is what detects a dead one. */ timeoutMs: number /** Watchdog budgets (D3), applied to the host path AND to the caged one; D3 defaults when absent. */ @@ -866,13 +944,39 @@ export async function runTaskTurn(opts: RunTaskTurnOptions): Promise repositories ? `${repositories}\n\n${text}` : text + const proof = record.isolation === 'microvm' ? readProofConfig(repoRoot) : null + const proofAsked = record.isolation === 'microvm' && proof !== null if (record.turns.length <= 1) { // A work-on conversation is not asked to name anything: it works on the // user's own pre-existing branch, which is never renamed. - const standing = buildTaskPrompt(record, { askBranchName: !record.work_on }) + const standing = buildTaskPrompt(record, { askBranchName: !record.work_on, proof }) const draft = taskCriteria(record).length === 0 ? `\n\n${criteriaDraftInstruction()}` : '' - return withRepositories(`${standing}${draft}\n\n${message}`) + return { prompt: withRepositories(`${standing}${draft}\n\n${message}`), proofAsked } } if (supportsSessionResume(command) && record.agent_session_id) { - return withRepositories(message) + return { prompt: withRepositories(message), proofAsked } + } + return { + prompt: withRepositories( + [ + buildTaskPrompt(record, { proof }), + '', + transcript(record), + '', + `New instruction: ${message}`, + ].join('\n'), + ), + proofAsked, } - return withRepositories( - [buildTaskPrompt(record), '', transcript(record), '', `New instruction: ${message}`].join('\n'), - ) } /** @@ -1824,6 +1955,9 @@ export function createTaskRunner(opts: TaskRunnerOptions): TaskRunner { if (outcome.tokens > 0) { turn.tokens = outcome.tokens } + if (outcome.proofIntent) { + turn.proof_intent = outcome.proofIntent + } } // `attempt.cost` and `outcome.cost` are the same last publication of the // same meter; the attempt carries it so the failure path can fold the same @@ -2526,10 +2660,12 @@ export function createTaskRunner(opts: TaskRunnerOptions): TaskRunner { record.isolation === 'microvm' && opts.resolveMicrovmFn ? await opts.resolveMicrovmFn(record) : undefined + const { prompt, proofAsked } = composeTurnPrompt(record, taskCommand, opts.cwd) return runTaskTurn({ cwd: record.worktree, task: record, - prompt: composeTurnPrompt(record, taskCommand), + prompt, + proofAsked, command: opts.command, timeoutMs: opts.timeoutMs, ...(opts.watchdog ? { watchdog: opts.watchdog } : {}), diff --git a/packages/cli/src/task-server.test.ts b/packages/cli/src/task-server.test.ts index 44b060d..d5abd21 100644 --- a/packages/cli/src/task-server.test.ts +++ b/packages/cli/src/task-server.test.ts @@ -28,6 +28,7 @@ import { type ReviewRecord, type RunbookConfig, type RunbookValidation, + type TaskActivityPhase, type TaskChecks, type TaskEvent, type TaskIssueRef, @@ -40,7 +41,7 @@ import { import type { ForgeCli, ForgeCliOutcome, ForgeIssuesExecFn } from './forge-issues.js' import { t as translate } from './i18n.js' import { createLoadCap } from './load-cap.js' -import type { SandboxDriver, SandboxSweepOutcome } from './microsandbox-driver.js' +import type { SandboxDriver, SandboxHandle, SandboxSweepOutcome } from './microsandbox-driver.js' import type { ProjectSnapshot } from './microvm-snapshot.js' import type { RunMicrovmTurnOptions } from './microvm-turn.js' import { addProject, listProjects, projectsPath, scratchProject, type Project } from './projects.js' @@ -49,6 +50,7 @@ import { readChecksConfig } from './repo-config.js' import { runbookSha as computeRunbookSha } from './runbook-setup.js' import { createSession, startServer } from './serve.js' import type { MicrovmStepExecutorOptions, RunChecksOptions, StepExecutor } from './task-checks.js' +import { evidenceDir, readTaskEvidence } from './task-evidence.js' import { AUTO_FIX_EXHAUSTED_NAME, AUTO_FIX_JOURNAL_DAMAGED_NAME, @@ -74,7 +76,7 @@ import { resetQueueDegradedReports, } from './task-queue.js' import { RECAP_MARKER_PREFIX } from './task-recap-publish.js' -import { writeTaskRecap } from './task-recap.js' +import { readTaskRecap, writeTaskRecap } from './task-recap.js' import type { TaskRetentionOutcome } from './task-retention.js' import { readTaskReview, type CreateTaskReviewerOptions } from './task-review.js' import { @@ -2358,6 +2360,78 @@ describe('manager.ship', () => { expect(loadTask(cwd, record.id)?.status).toBe('review_ok') expect(loadTask(cwd, record.id)?.cycle_step).toBeUndefined() }) + + describe('task_recap frame', () => { + function minimalRecap(branch: string) { + return { + version: 1 as const, + summary: 'Rewired the worktree cleanup.', + changes: ['worktree: prune before delete'], + decisions: [], + files: ['src/task-worktree.ts'], + tests: [{ command: 'bun test', status: 'passed' as const }], + branch, + } + } + + test('a recap present on disk after a successful push emits task_recap', async () => { + const project = register(makeRepo()) + const cwd = project.path + const record = seedShippable(cwd) + const stub = shipStub({ + pushed: true, + mrUrl: 'https://github.com/o/r/pull/9', + note: null, + }) + const manager = createTaskManager({ ...managerOpts, shipTaskFn: stub.fn, ...fakeRunner() }) + const written = writeTaskRecap(cwd, record.id, minimalRecap(record.branch)) + const envelopes: TaskEnvelope[] = [] + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(await manager.ship(project.id, record.id)).toEqual({ ok: true }) + + const recapEnvelope = envelopes.find((e) => e.event.name === 'task_recap') + expect(recapEnvelope?.event.data).toEqual(written) + }) + + test('no recap on disk after a successful push: no task_recap frame', async () => { + const project = register(makeRepo()) + const cwd = project.path + const record = seedShippable(cwd) + const stub = shipStub({ pushed: true, mrUrl: 'https://github.com/o/r/pull/9', note: null }) + const manager = createTaskManager({ ...managerOpts, shipTaskFn: stub.fn, ...fakeRunner() }) + const envelopes: TaskEnvelope[] = [] + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(await manager.ship(project.id, record.id)).toEqual({ ok: true }) + + expect(envelopes.some((e) => e.event.name === 'task_recap')).toBe(false) + }) + + test('a recap withheld from the MR description for carrying a secret never rides the SSE frame either', async () => { + const project = register(makeRepo()) + const cwd = project.path + const record = seedShippable(cwd) + const stub = shipStub({ + pushed: true, + mrUrl: 'https://github.com/o/r/pull/9', + note: 'recap withheld: looked like a secret', + recapState: 'recap_blocked_secrets', + }) + const manager = createTaskManager({ ...managerOpts, shipTaskFn: stub.fn, ...fakeRunner() }) + writeTaskRecap(cwd, record.id, minimalRecap(record.branch)) + const envelopes: TaskEnvelope[] = [] + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(await manager.ship(project.id, record.id)).toEqual({ ok: true }) + + // The recap DOES exist on disk (generateAndPersist runs before the + // secret scan) — proving the frame's silence comes from `recapState`, + // not from an absent file. + expect(readTaskRecap(cwd, record.id)).not.toBeNull() + expect(envelopes.some((e) => e.event.name === 'task_recap')).toBe(false) + }) + }) }) // --- D20: cycle_step ship/merge -------------------------------------------- @@ -4514,7 +4588,8 @@ describe('task routes with a stub manager', () => { }, getChecks: (projectId, id) => known(projectId) && id === record.id ? readTaskChecks(project.path, id) : null, - getVerification: () => null, + getVerification: (projectId, id) => + known(projectId) && id === record.id ? readTaskVerification(project.path, id) : null, getReview: (projectId, id, ref) => known(projectId) ? readTaskReview(project.path, id, ref) : null, checksSetup: (projectId) => { @@ -5197,6 +5272,52 @@ describe('task routes with a stub manager', () => { } }) + test('verification route: 404 before any run, 400 with no project, 404 unknown project/id, then the file', async () => { + const project = register(makeRepo()) + const { manager, record } = stubManager(project) + const started = await startServer(createSession(), { + cwd: project.path, + port: 5174, + taskManager: manager, + }) + const path = `/api/tasks/${record.id}/verification?project=${project.id}` + try { + // Never run: 404, same doctrine as the checks route above. + expect((await rawRequest(started.port, path)).status).toBe(404) + expect((await rawRequest(started.port, `/api/tasks/${record.id}/verification`)).status).toBe( + 400, + ) + expect( + (await rawRequest(started.port, `/api/tasks/${record.id}/verification?project=ffffffff`)) + .status, + ).toBe(404) + expect( + (await rawRequest(started.port, `/api/tasks/not-an-id/verification?project=${project.id}`)) + .status, + ).toBe(404) + + writeTaskVerification(project.path, record.id, { + head_sha: 'abc', + runbook_sha: '0123456789abcdef', + started_at: '2026-08-14T10:00:00.000Z', + finished_at: '2026-08-14T10:05:00.000Z', + status: 'passed', + checks: [{ command: 'npm test', status: 'passed', exit_code: 0, duration_ms: 5, tail: '' }], + integrity_ok: true, + changed_dependency_files: [], + error: null, + }) + const got = await rawRequest(started.port, path) + expect(got.status).toBe(200) + expect(JSON.parse(got.body)).toMatchObject({ + status: 'passed', + checks: [{ command: 'npm test', status: 'passed' }], + }) + } finally { + await started.stop() + } + }) + test('review route: 404 before any review, then the archive, ref-scoped and traversal-proof', async () => { const project = register(makeRepo()) const { manager, record } = stubManager(project) @@ -9744,8 +9865,13 @@ describe('automatic fix loop (T3.3)', () => { ) // The reviewer's own settle() and the hook's belt-and-braces write both // land on the FINAL status: the loop's decision is folded INTO the - // transition, never applied as a second write after it. - expect([...new Set(loop.written)]).toEqual(['waiting_for_you']) + // transition, never applied as a second write after it. 'reviewing' is + // filtered out: it is the activity markers' own persists (checks, + // verification, review posed/cleared), which carry the turn's status + // unchanged and are expected alongside the single status transition. + expect([...new Set(loop.written.filter((status) => status !== 'reviewing'))]).toEqual([ + 'waiting_for_you', + ]) }) test('the bound is configurable: 1 allows one round, 3 allows three', async () => { @@ -10576,6 +10702,11 @@ describe('cycle labels and the recap, wired onto a real run', () => { tests: [{ command: 'bun test', status: 'passed' }], branch: options.task.branch, }) + } else { + // The end-of-turn recap (onTurnDone) already wrote one before this + // stub ever runs: erased here to keep simulating a ship whose own + // recap never made it onto disk. + rmSync(join(taskDir(options.cwd, options.task.id), 'recap.json'), { force: true }) } return Promise.resolve({ pushed: true, @@ -11212,6 +11343,146 @@ describe('cycle labels and the recap, wired onto a real run', () => { }) }) +describe('end-of-turn recap (onTurnDone)', () => { + const jsonl = (events: unknown[]) => `${events.map((e) => JSON.stringify(e)).join('\n')}\n` + const claudeStream = (response: string) => + jsonl([ + { type: 'system', subtype: 'init', session_id: 'sess-recap' }, + { type: 'result', result: response }, + ]) + + test('a green review generates and persists a recap right after the turn, mr_url absent, and emits task_recap', async () => { + const project = register(makeRepo()) + const manager = createTaskManager({ + ...managerOpts, + runAgentFn: (options: AgentRunOptions) => { + writeFileSync(join(options.cwd, 'feature.txt'), 'done\n') + const raw = claudeStream('Rewired the worktree cleanup.') + options.onText?.(raw) + return Promise.resolve(raw) + }, + reviewTurnFn: (record, io) => { + record.status = 'review_ok' + io.persist() + return Promise.resolve() + }, + }) + const envelopes: TaskEnvelope[] = [] + manager.subscribe((envelope) => envelopes.push(envelope)) + + const created = await manager.create(project.id, { + autoShip: false, + title: 'no ship yet', + prompt: 'do it', + }) + if (!created.ok) { + throw new Error(`create refused: ${created.error}`) + } + await until(() => loadTask(project.path, created.record.id)?.status === 'review_ok') + + const recap = readTaskRecap(project.path, created.record.id) + expect(recap?.summary).toBe('Rewired the worktree cleanup.') + expect(recap?.mr_url).toBeUndefined() + expect(recap?.branch).toBe(loadTask(project.path, created.record.id)?.branch) + + const recapEnvelope = envelopes.find((e) => e.event.name === 'task_recap') + expect(recapEnvelope?.event.data).toEqual(recap) + }) + + test('a failed turn (review_ko) generates no recap at all', async () => { + const project = register(makeRepo()) + const manager = createTaskManager({ + ...managerOpts, + runAgentFn: (options: AgentRunOptions) => { + const raw = claudeStream('did something wrong') + options.onText?.(raw) + return Promise.resolve(raw) + }, + reviewTurnFn: (record, io) => { + record.status = 'review_ko' + io.persist() + return Promise.resolve() + }, + }) + const envelopes: TaskEnvelope[] = [] + manager.subscribe((envelope) => envelopes.push(envelope)) + + const created = await manager.create(project.id, { + autoShip: false, + title: 'stays ko', + prompt: 'do it', + }) + if (!created.ok) { + throw new Error(`create refused: ${created.error}`) + } + await until(() => loadTask(project.path, created.record.id)?.status === 'review_ko') + + expect(readTaskRecap(project.path, created.record.id)).toBeNull() + expect(envelopes.some((e) => e.event.name === 'task_recap')).toBe(false) + }) + + test('the ship regenerates and re-emits the recap afterwards, now carrying mr_url', async () => { + const project = register(makeRepo()) + const manager = createTaskManager({ + ...managerOpts, + runAgentFn: (options: AgentRunOptions) => { + writeFileSync(join(options.cwd, 'feature.txt'), 'done\n') + const raw = claudeStream('Rewired the worktree cleanup.') + options.onText?.(raw) + return Promise.resolve(raw) + }, + reviewTurnFn: (record, io) => { + record.status = 'review_ok' + io.persist() + return Promise.resolve() + }, + shipTaskFn: (options: ShipTaskOptions) => { + // What the real ship's own generateAndPersist (task-ship.ts) leaves + // behind: a regenerated recap, this time with mr_url. shipTaskFn is + // stubbed here (no real push), so that regeneration is simulated + // rather than exercised: task-ship.ts owns and already tests it. + writeTaskRecap(options.cwd, options.task.id, { + version: 1, + summary: 'Rewired the worktree cleanup.', + changes: [], + decisions: [], + files: ['feature.txt'], + tests: [], + branch: options.task.branch, + mr_url: 'https://github.com/acme/repo/pull/9', + }) + return Promise.resolve({ + pushed: true, + mrUrl: 'https://github.com/acme/repo/pull/9', + note: null, + }) + }, + }) + const envelopes: TaskEnvelope[] = [] + manager.subscribe((envelope) => envelopes.push(envelope)) + + const created = await manager.create(project.id, { + autoShip: false, + title: 'ships later', + prompt: 'do it', + }) + if (!created.ok) { + throw new Error(`create refused: ${created.error}`) + } + await until(() => loadTask(project.path, created.record.id)?.status === 'review_ok') + const beforeShip = readTaskRecap(project.path, created.record.id) + expect(beforeShip?.mr_url).toBeUndefined() + + expect(await manager.ship(project.id, created.record.id)).toEqual({ ok: true }) + + const afterShip = readTaskRecap(project.path, created.record.id) + expect(afterShip?.mr_url).toBe('https://github.com/acme/repo/pull/9') + const recapEnvelopes = envelopes.filter((e) => e.event.name === 'task_recap') + expect(recapEnvelopes.length).toBeGreaterThanOrEqual(2) + expect(recapEnvelopes.at(-1)?.event.data).toEqual(afterShip) + }) +}) + // ── Conversations with no repository (the scratch project) ───────────────── describe('scratch conversations over HTTP', () => { @@ -11975,6 +12246,8 @@ describe('microvm wiring (lot C7)', () => { e.event.data.data.status === 'passed', ), ).toBe(true) + const verificationEnvelope = envelopes.find((e) => e.event.name === 'task_verification') + expect(verificationEnvelope?.event.data).toEqual(verification) }) test('a refused verification (runbook integrity drifted) sends the task back with checks_failed', async () => { @@ -12369,6 +12642,747 @@ describe('microvm wiring (lot C7)', () => { expect(verifyCalls[0]?.validatedSha).toBe('abc1234abc1234ab') void worktree }) + + describe('proof capture wiring', () => { + function fakeSandboxHandle(): SandboxHandle { + return { + name: 'fake-proof-handle', + exec: () => Promise.resolve({ code: 0, stdout: '', stderr: '', timedOut: false }), + shell: () => Promise.resolve({ code: 0, stdout: '', stderr: '', timedOut: false }), + copyFromHost: () => Promise.resolve(), + copyToHost: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + readFile: () => Promise.resolve(''), + metrics: () => + Promise.resolve({ memoryHostResidentBytes: null, memoryBytes: null, cpuPercent: null }), + stop: () => Promise.resolve(), + } + } + + function writeProofConfig(cwd: string): void { + mkdirSync(join(cwd, '.codesema'), { recursive: true }) + writeFileSync( + join(cwd, '.codesema', 'config.json'), + JSON.stringify({ + proof: { + journey: 'proof/checkout.spec.ts', + url: 'http://localhost:3000', + timeoutSeconds: 30, + keep: 3, + }, + }), + ) + } + + function writeProofConfigUrlOnly(cwd: string): void { + mkdirSync(join(cwd, '.codesema'), { recursive: true }) + writeFileSync( + join(cwd, '.codesema', 'config.json'), + JSON.stringify({ proof: { url: 'http://localhost:3000' } }), + ) + } + + test('proof configured and the spec is present: ingest runs, evidence.json and a task_evidence frame land', async () => { + const project = register(makeRepo()) + const { record, worktree } = seedInterruptedMicrovmTask(project.path) + writeProofConfig(project.path) + mkdirSync(join(worktree, 'proof'), { recursive: true }) + writeFileSync(join(worktree, 'proof', 'checkout.spec.ts'), 'test()\n') + + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + const verification: TaskVerification = { + head_sha: 'proofsha1', + runbook_sha: computeRunbookSha(runbook), + started_at: '2026-01-01T00:00:00.000Z', + finished_at: '2026-01-01T00:05:00.000Z', + status: 'passed', + checks: [ + { command: 'npm test', status: 'passed', exit_code: 0, duration_ms: 5, tail: '' }, + ], + integrity_ok: true, + changed_dependency_files: [], + error: null, + } + const verifyCalls: VerifyTaskOptions[] = [] + const incomingDir = join(evidenceDir(project.path, record.id), '.incoming') + const envelopes: TaskEnvelope[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: async (opts) => { + verifyCalls.push(opts) + mkdirSync(incomingDir, { recursive: true }) + writeFileSync(join(incomingDir, 'shot.png'), 'fake-png') + await opts.captureProof?.(fakeSandboxHandle()) + return verification + }, + captureProofFn: () => Promise.resolve({ status: 'passed', reason: null }), + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ok' + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream('all done') + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ok') + + expect(verifyCalls).toHaveLength(1) + expect(typeof verifyCalls[0]?.captureProof).toBe('function') + + const evidence = readTaskEvidence(project.path, record.id) + expect(evidence?.status).toBe('passed') + expect(evidence?.reason).toBeNull() + expect(evidence?.head_sha).toBe('proofsha1') + expect(evidence?.items).toHaveLength(1) + expect(evidence?.items[0]?.kind).toBe('screenshot') + + expect( + envelopes.some( + (e) => e.event.name === 'task_evidence' && e.event.data.status === 'passed', + ), + ).toBe(true) + }) + + test('proof configured but the spec is missing from the worktree: a skipped evidence record is written and a frame emitted', async () => { + const project = register(makeRepo()) + const { record } = seedInterruptedMicrovmTask(project.path) + writeProofConfig(project.path) + + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + const verification: TaskVerification = { + head_sha: 'proofsha2', + runbook_sha: computeRunbookSha(runbook), + started_at: '2026-01-01T00:00:00.000Z', + finished_at: '2026-01-01T00:05:00.000Z', + status: 'passed', + checks: [], + integrity_ok: true, + changed_dependency_files: [], + error: null, + } + const verifyCalls: VerifyTaskOptions[] = [] + const envelopes: TaskEnvelope[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: (opts) => { + verifyCalls.push(opts) + return Promise.resolve(verification) + }, + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ok' + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream('all done') + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ok') + + expect(verifyCalls[0]?.captureProof).toBeUndefined() + + const evidence = readTaskEvidence(project.path, record.id) + expect(evidence?.status).toBe('skipped') + expect(evidence?.reason).toContain('proof/checkout.spec.ts') + expect(evidence?.items).toEqual([]) + + expect( + envelopes.some( + (e) => e.event.name === 'task_evidence' && e.event.data.status === 'skipped', + ), + ).toBe(true) + }) + + test('proof not configured: no evidence record, no frame', async () => { + const project = register(makeRepo()) + const { record } = seedInterruptedMicrovmTask(project.path) + + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + const verification: TaskVerification = { + head_sha: 'proofsha3', + runbook_sha: computeRunbookSha(runbook), + started_at: '2026-01-01T00:00:00.000Z', + finished_at: '2026-01-01T00:05:00.000Z', + status: 'passed', + checks: [], + integrity_ok: true, + changed_dependency_files: [], + error: null, + } + const envelopes: TaskEnvelope[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: () => Promise.resolve(verification), + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ok' + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream('all done') + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ok') + + expect(readTaskEvidence(project.path, record.id)).toBeNull() + expect(envelopes.some((e) => e.event.name === 'task_evidence')).toBe(false) + }) + + test('an ingest error is swallowed: the turn still settles, no frame goes out', async () => { + const project = register(makeRepo()) + const { record, worktree } = seedInterruptedMicrovmTask(project.path) + writeProofConfig(project.path) + mkdirSync(join(worktree, 'proof'), { recursive: true }) + writeFileSync(join(worktree, 'proof', 'checkout.spec.ts'), 'test()\n') + // A plain FILE sits where ingestEvidenceFiles needs to mkdir a + // directory: its `mkdirSync(targetDir, { recursive: true })` throws. + mkdirSync(taskDir(project.path, record.id), { recursive: true }) + writeFileSync(join(taskDir(project.path, record.id), 'evidence'), 'not a directory') + + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + const verification: TaskVerification = { + head_sha: 'proofsha4', + runbook_sha: computeRunbookSha(runbook), + started_at: '2026-01-01T00:00:00.000Z', + finished_at: '2026-01-01T00:05:00.000Z', + status: 'passed', + checks: [], + integrity_ok: true, + changed_dependency_files: [], + error: null, + } + const envelopes: TaskEnvelope[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: async (opts) => { + await opts.captureProof?.(fakeSandboxHandle()) + return verification + }, + captureProofFn: () => Promise.resolve({ status: 'passed', reason: null }), + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ok' + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream('all done') + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ok') + + expect(envelopes.some((e) => e.event.name === 'task_evidence')).toBe(false) + }) + + test('PROOF: none is declined: captureProofFn never runs, evidence.json records the reason and the intent', async () => { + const project = register(makeRepo()) + const { record } = seedInterruptedMicrovmTask(project.path) + writeProofConfig(project.path) + + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + const verification: TaskVerification = { + head_sha: 'proofsha5', + runbook_sha: computeRunbookSha(runbook), + started_at: '2026-01-01T00:00:00.000Z', + finished_at: '2026-01-01T00:05:00.000Z', + status: 'passed', + checks: [], + integrity_ok: true, + changed_dependency_files: [], + error: null, + } + let captureProofCalled = false + const envelopes: TaskEnvelope[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: async (opts) => { + await opts.captureProof?.(fakeSandboxHandle()) + return verification + }, + captureProofFn: () => { + captureProofCalled = true + return Promise.resolve({ status: 'passed', reason: null }) + }, + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ok' + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream( + 'PROOF: none | refactor only, nothing rendered changed\n\nall done', + ) + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ok') + + expect(captureProofCalled).toBe(false) + const evidence = readTaskEvidence(project.path, record.id) + expect(evidence?.status).toBe('skipped') + expect(evidence?.reason).toBe('refactor only, nothing rendered changed') + expect(evidence?.intent?.kind).toBe('none') + + expect( + envelopes.some( + (e) => e.event.name === 'task_evidence' && e.event.data.status === 'skipped', + ), + ).toBe(true) + }) + + test('PROOF: screenshot replays the declared pages through captureScreenshotsFn', async () => { + const project = register(makeRepo()) + const { record } = seedInterruptedMicrovmTask(project.path) + writeProofConfig(project.path) + + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + const verification: TaskVerification = { + head_sha: 'proofsha6', + runbook_sha: computeRunbookSha(runbook), + started_at: '2026-01-01T00:00:00.000Z', + finished_at: '2026-01-01T00:05:00.000Z', + status: 'passed', + checks: [], + integrity_ok: true, + changed_dependency_files: [], + error: null, + } + const screenshotCalls: { pages: string[] }[] = [] + const envelopes: TaskEnvelope[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: async (opts) => { + await opts.captureProof?.(fakeSandboxHandle()) + return verification + }, + captureScreenshotsFn: (_handle, screenshotOpts) => { + screenshotCalls.push({ pages: screenshotOpts.pages }) + return Promise.resolve({ status: 'passed', reason: null }) + }, + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ok' + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream( + 'PROOF: screenshot /dashboard /settings | new settings panel\n\nall done', + ) + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ok') + + expect(screenshotCalls).toEqual([{ pages: ['/dashboard', '/settings'] }]) + const evidence = readTaskEvidence(project.path, record.id) + expect(evidence?.status).toBe('passed') + expect(evidence?.intent?.kind).toBe('screenshot') + + expect( + envelopes.some( + (e) => e.event.name === 'task_evidence' && e.event.data.status === 'passed', + ), + ).toBe(true) + }) + + test('PROOF: journey with an explicit spec replays it even when the project has no default journey', async () => { + const project = register(makeRepo()) + const { record, worktree } = seedInterruptedMicrovmTask(project.path) + writeProofConfigUrlOnly(project.path) + mkdirSync(join(worktree, 'flows'), { recursive: true }) + writeFileSync(join(worktree, 'flows', 'checkout.spec.ts'), 'test()\n') + + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + const verification: TaskVerification = { + head_sha: 'proofsha7', + runbook_sha: computeRunbookSha(runbook), + started_at: '2026-01-01T00:00:00.000Z', + finished_at: '2026-01-01T00:05:00.000Z', + status: 'passed', + checks: [], + integrity_ok: true, + changed_dependency_files: [], + error: null, + } + const journeyCalls: { journey: string }[] = [] + const envelopes: TaskEnvelope[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: async (opts) => { + await opts.captureProof?.(fakeSandboxHandle()) + return verification + }, + captureProofFn: (_handle, proofOpts) => { + journeyCalls.push({ journey: proofOpts.journey }) + return Promise.resolve({ status: 'passed', reason: null }) + }, + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ok' + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream( + 'PROOF: journey flows/checkout.spec.ts | multi-step checkout flow changed\n\nall done', + ) + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ok') + + expect(journeyCalls).toEqual([{ journey: 'flows/checkout.spec.ts' }]) + const evidence = readTaskEvidence(project.path, record.id) + expect(evidence?.status).toBe('passed') + expect(evidence?.intent?.kind).toBe('journey') + void worktree + }) + + test('proof configured with only a url and no declared intent: not_attempted, no evidence, no frame', async () => { + const project = register(makeRepo()) + const { record } = seedInterruptedMicrovmTask(project.path) + writeProofConfigUrlOnly(project.path) + + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + const verification: TaskVerification = { + head_sha: 'proofsha8', + runbook_sha: computeRunbookSha(runbook), + started_at: '2026-01-01T00:00:00.000Z', + finished_at: '2026-01-01T00:05:00.000Z', + status: 'passed', + checks: [], + integrity_ok: true, + changed_dependency_files: [], + error: null, + } + const envelopes: TaskEnvelope[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: () => Promise.resolve(verification), + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ok' + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream('all done') + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => envelopes.push(envelope)) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ok') + + expect(readTaskEvidence(project.path, record.id)).toBeNull() + expect(envelopes.some((e) => e.event.name === 'task_evidence')).toBe(false) + }) + }) + }) + + describe('activity narration (D-activity, onTurnDone + boot)', () => { + // Local calques of the same-named helpers in 'mechanical verification + // (onTurnDone)' above (private to that describe, out of reach here). + const jsonl = (events: unknown[]) => `${events.map((e) => JSON.stringify(e)).join('\n')}\n` + const claudeStream = (response: string) => + jsonl([ + { type: 'system', subtype: 'init', session_id: 'sess-activity' }, + { type: 'result', result: response }, + ]) + + function seedInterruptedMicrovmTask(cwd: string): { record: TaskRecord; worktree: string } { + const worktree = makeRepo() + const record = createTask(cwd, { + title: 'vm task', + prompt: 'do it', + autoShip: false, + base: '', + branch: '', + worktree, + isolation: 'microvm', + }) + record.worktree = worktree + record.status = 'interrupted' + saveTask(cwd, record) + return { record, worktree } + } + + function validRunbookValidation(runbook: RunbookConfig): RunbookValidation { + return { + runbook_sha: computeRunbookSha(runbook), + validated_sha: 'deadbeefdeadbeef', + validated_at: '2026-01-01T00:00:00.000Z', + status: 'valid', + } + } + + function fakeSandboxHandle(): SandboxHandle { + return { + name: 'fake-activity-handle', + exec: () => Promise.resolve({ code: 0, stdout: '', stderr: '', timedOut: false }), + shell: () => Promise.resolve({ code: 0, stdout: '', stderr: '', timedOut: false }), + copyFromHost: () => Promise.resolve(), + copyToHost: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + readFile: () => Promise.resolve(''), + metrics: () => + Promise.resolve({ memoryHostResidentBytes: null, memoryBytes: null, cpuPercent: null }), + stop: () => Promise.resolve(), + } + } + + function writeProofConfig(cwd: string): void { + mkdirSync(join(cwd, '.codesema'), { recursive: true }) + writeFileSync( + join(cwd, '.codesema', 'config.json'), + JSON.stringify({ + proof: { + journey: 'proof/checkout.spec.ts', + url: 'http://localhost:3000', + timeoutSeconds: 30, + keep: 3, + }, + }), + ) + } + + test('a full turn narrates checks, verification, proof, review, recap in order, with no activity on the verdict frame nor the final frame', async () => { + const project = register(makeRepo()) + const { record, worktree } = seedInterruptedMicrovmTask(project.path) + writeProofConfig(project.path) + mkdirSync(join(worktree, 'proof'), { recursive: true }) + writeFileSync(join(worktree, 'proof', 'checkout.spec.ts'), 'test()\n') + + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + const verification: TaskVerification = { + head_sha: 'activitysha1', + runbook_sha: computeRunbookSha(runbook), + started_at: '2026-01-01T00:00:00.000Z', + finished_at: '2026-01-01T00:05:00.000Z', + status: 'passed', + checks: [{ command: 'npm test', status: 'passed', exit_code: 0, duration_ms: 5, tail: '' }], + integrity_ok: true, + changed_dependency_files: [], + error: null, + } + // `envelope.event.data` for a 'task' frame is the manager's own + // mutable `record`, broadcast BY REFERENCE: reading it back after the + // turn settles would show every entry as its FINAL state. The phase + // (and status) actually observed at each broadcast is read out + // synchronously, right here, the same way the queue-position tests + // above do for the same reason. + const taskFrames: { status: TaskStatus; activityPhase: TaskActivityPhase | null }[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: async (opts) => { + await opts.captureProof?.(fakeSandboxHandle()) + return verification + }, + captureProofFn: () => Promise.resolve({ status: 'passed', reason: null }), + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ok' + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream('all done') + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => { + if (envelope.event.name === 'task') { + taskFrames.push({ + status: envelope.event.data.status, + activityPhase: envelope.event.data.activity?.phase ?? null, + }) + } + }) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ok') + + expect( + taskFrames + .map((f) => f.activityPhase) + .filter((phase): phase is TaskActivityPhase => phase !== null), + ).toEqual(['checks', 'verification', 'proof', 'review', 'recap']) + const verdictFrame = taskFrames.find((f) => f.status === 'review_ok') + expect(verdictFrame?.activityPhase).toBeNull() + expect(taskFrames.at(-1)?.activityPhase).toBeNull() + }) + + test('a rejecting verifyTaskFn is swallowed by verifyAfterCommit: the final frame still carries no activity', async () => { + const project = register(makeRepo()) + const { record } = seedInterruptedMicrovmTask(project.path) + const runbook = baseRunbook() + const validation = validRunbookValidation(runbook) + // Same reasoning as the test above: read the phase at broadcast time, + // never off the shared `record` object after the fact. + const taskFrames: { status: TaskStatus; activityPhase: TaskActivityPhase | null }[] = [] + const manager = createTaskManager({ + ...managerOpts, + sandboxDriverFn: () => fakeDriver, + readRunbookConfigFn: () => runbook, + readRunbookValidationFn: () => validation, + resolveProjectSnapshotFn: () => + Promise.resolve({ kind: 'cold', reason: 'test' } as ProjectSnapshot), + verifyTaskFn: () => Promise.reject(new Error('sandbox exploded')), + runChecksFn: () => Promise.resolve(finishedChecks()), + reviewTurnFn: async (r, io) => { + r.status = 'review_ko' + r.reason = taskReason('review_blocked', 'review failed: the review agent died') + io.persist() + }, + runMicrovmTurnFn: (options: RunMicrovmTurnOptions) => { + writeFileSync(join(options.worktree, 'feature.txt'), 'from the vm\n') + const raw = claudeStream('all done') + options.onText?.(raw) + return Promise.resolve(raw) + }, + }) + manager.subscribe((envelope) => { + if (envelope.event.name === 'task') { + taskFrames.push({ + status: envelope.event.data.status, + activityPhase: envelope.event.data.activity?.phase ?? null, + }) + } + }) + + expect(manager.resume(project.id, record.id)).toEqual({ ok: true }) + await until(() => loadTask(project.path, record.id)?.status === 'review_ko') + + // No 'proof': verifyAfterCommit's own try/catch swallows the rejection + // before captureProof is ever reached. No 'recap' either: the turn + // settled on 'review_ko', which the recap block only runs past on + // 'review_ok'. + expect( + taskFrames + .map((f) => f.activityPhase) + .filter((phase): phase is TaskActivityPhase => phase !== null), + ).toEqual(['checks', 'verification', 'review']) + expect(taskFrames.at(-1)?.activityPhase).toBeNull() + expect(readTaskVerification(project.path, record.id)).toBeNull() + }) + + test('boot never lets a phase survive a restart, whether or not the record itself is rewritten', () => { + const repo = makeRepo() + register(repo) + const idle = seedTask(repo, 'idle with a stale phase') + idle.status = 'waiting_for_you' + idle.activity = { phase: 'review', since: '2026-01-01T00:00:00.000Z' } + saveTask(repo, idle) + const orphaned = seedTask(repo, 'orphaned with a stale phase') + orphaned.status = 'reviewing' + orphaned.activity = { phase: 'checks', since: '2026-01-01T00:00:00.000Z' } + saveTask(repo, orphaned) + + createTaskManager({ ...managerOpts, ...fakeRunner() }) + + const tasks = listTasks(repo) + const idleAfter = tasks.find((t) => t.id === idle.id) + const orphanedAfter = tasks.find((t) => t.id === orphaned.id) + expect(idleAfter?.activity).toBeUndefined() + expect(idleAfter?.status).toBe('waiting_for_you') + expect(orphanedAfter?.activity).toBeUndefined() + expect(orphanedAfter?.status).toBe('interrupted') + }) }) describe('D8: the validated runbook is read from the PROJECT root, never the task worktree', () => { diff --git a/packages/cli/src/task-server.ts b/packages/cli/src/task-server.ts index ecf83fe..fea918f 100644 --- a/packages/cli/src/task-server.ts +++ b/packages/cli/src/task-server.ts @@ -11,6 +11,7 @@ // across N projects ride one EventSource. import { existsSync } from 'node:fs' +import { join } from 'node:path' import { knownAgent, type AgentRunOptions, type WatchdogBudgets } from './agent.js' import { createChecksSetupRunner, @@ -36,9 +37,12 @@ import { TASK_TURN_TEXT_MAX, type AcceptanceCriterion, type ArmTicket, + type EvidenceRecord, type ReasonCode, + type RecapRecord, type ReviewRecord, type RunbookConfig, + type TaskActivityPhase, type TaskChecks, type TaskEvent, type TaskIsolation, @@ -69,6 +73,7 @@ import { createMicrosandboxDriver, sweepOrphanedSandboxes as sweepOrphanedSandboxesImpl, type SandboxDriver, + type SandboxHandle, type SandboxSweepOutcome, } from './microsandbox-driver.js' import { buildProjectSnapshot, resolveProjectSnapshot } from './microvm-snapshot.js' @@ -79,7 +84,7 @@ import { scratchProject, type Project, } from './projects.js' -import { readChecksConfig } from './repo-config.js' +import { readChecksConfig, readProofConfig } from './repo-config.js' import { runbookSha as computeRunbookSha, readRunbookConfig, @@ -88,6 +93,12 @@ import { import { loadSyncCredentials } from './sync.js' import { microvmStepExecutor, runChecks } from './task-checks.js' import { criteriaBlockKind } from './task-criteria-gate.js' +import { + evidenceDir, + ingestEvidenceFiles, + readTaskEvidence, + writeTaskEvidence, +} from './task-evidence.js' import { applyFixLoopDecision, AUTO_FIX_EXHAUSTED_NAME, @@ -138,8 +149,10 @@ import { import { effectiveMergePolicyIsAuto, mergeTask, type MergeOutcome } from './task-merge.js' import { resolveTaskPlan, type TaskPlanDeps, type TaskPreviewResult } from './task-plan.js' import { replayChecksOnDefaultBranch } from './task-post-merge-checks.js' +import { captureProof, captureScreenshots, type ProofCaptureResult } from './task-proof.js' import { createTaskQueue, type TaskQueue } from './task-queue.js' import { publishTaskRecap } from './task-recap-publish.js' +import { generateRecap, readTaskRecap, recapOptionsFor, writeTaskRecap } from './task-recap.js' import { applyTaskRetention, DEFAULT_TASK_RETENTION, @@ -246,6 +259,13 @@ export type TaskEnvelope = // PROJECT-scoped, hence no task_id: the checks setup agent proposes a // configuration for the whole repo, not for one conversation. | { project_id: string; event: { name: 'checks_proposal'; data: ChecksSetupState } } + | { project_id: string; task_id: string; event: { name: 'task_evidence'; data: EvidenceRecord } } + | { project_id: string; task_id: string; event: { name: 'task_recap'; data: RecapRecord } } + | { + project_id: string + task_id: string + event: { name: 'task_verification'; data: TaskVerification } + } /** * T2.4: the raw issue reference as it arrives from the wire — everything @@ -657,6 +677,10 @@ export type CreateTaskManagerOptions = { buildProjectSnapshotFn?: typeof buildProjectSnapshot /** Test seam: the default replays `runbook.tests` in a fresh sandbox (lot C7). */ verifyTaskFn?: typeof verifyTask + /** Test seam: the default replays the configured `proof.journey` and screenshots the fallback. */ + captureProofFn?: typeof captureProof + /** Test seam: the default screenshots the turn's own `PROOF: screenshot` pages (D17). */ + captureScreenshotsFn?: typeof captureScreenshots /** Test seam: the default builds task-checks.ts's own microvm executor (lot C7). */ microvmStepExecutorFn?: typeof microvmStepExecutor headShaFn?: typeof resolveHeadSha @@ -854,8 +878,21 @@ function reconcileTasks(cwd: string, projectId: string): ReconcileOutcome { appendTaskEvent(cwd, record.id, { ...event, reason_code: reason.code }) } for (const record of records) { + // A phase never survives a restart: whatever agent was 'checks', + // 'verification', 'proof', 'review' or 'recap' at the moment this + // process died is gone with it, so a stale phase is worse than none. + // `rewrite` below saves the SAME record object, so deleting it here + // (without a save) is enough to fold it into that write; a record that + // reaches neither branch is saved here directly. + const hadActivity = record.activity !== undefined + if (hadActivity) { + delete record.activity + } const status = reconciledStatus(cwd, record) if (status === null) { + if (hadActivity) { + saveTask(cwd, record) + } continue } if (status === 'failed') { @@ -2139,6 +2176,19 @@ export function createTaskManager(opts: CreateTaskManagerOptions): TaskManager { record.updated_at = new Date().toISOString() saveTask(cwd, record) emit({ project_id: projectId, task_id: record.id, event: { name: 'task', data: record } }) + // `recap.json` is written to disk whether or not it rides in the MR + // description (generateAndPersist runs before the secret scan) — + // `outcome.recapState` is what actually says "withheld", so silence on + // it is required here too: emitting on `recap` alone would leak a + // secret-blocked recap over SSE that the description itself refused. + const recap = readTaskRecap(cwd, id) + if (recap && !outcome.recapState) { + emit({ + project_id: projectId, + task_id: record.id, + event: { name: 'task_recap', data: recap }, + }) + } // T3.7, AFTER the write and the frame: this transition never reaches // `onTask` (ship persists and broadcasts on its own), so it is mirrored // here or nowhere. Not awaited — the ship's own answer must not wait on @@ -2738,64 +2788,110 @@ export function createTaskManager(opts: CreateTaskManagerOptions): TaskManager { } } + /** + * The guest path `verifyTask` copies the worktree into before any command + * runs (task-verification.ts's own unexported `VERIFY_WORK_DIR`) — kept in + * sync by hand since a proof replay runs on the same handle, in the same + * worktree, after the same copy. + */ + const PROOF_VERIFY_GUEST_WORK_DIR = '/work' + + /** + * What a turn's proof capture attempt came to, for `onTurnDone` to fold + * into evidence.json without re-deriving `verifyAfterCommit`'s own checks: + * `'not_attempted'` covers both "proof isn't configured" and "verification + * itself never reached the capture seam" (no commit, no runbook, stale + * runbook, isolation not `'microvm'`) — in every one of those cases nothing + * was captured, so nothing is recorded and no frame goes out. `'declined'` + * is the turn's own `PROOF: none` (D17): the agent judged nothing visible + * changed, so evidence.json still gets a `'skipped'` record carrying the + * stated reason, but no capture ever runs. + */ + type ProofCaptureOutcome = + | { kind: 'not_attempted' } + | { kind: 'spec_missing'; journey: string } + | { kind: 'declined'; reason: string } + | { + kind: 'attempted' + result: ProofCaptureResult + hostIncomingDir: string + keep: number | null + } + /** * The mechanical verification (lot C7): a `'microvm'` task whose worktree * carries a validated runbook gets `runbook.tests` replayed in a FRESH VM * restored from the project snapshot, right after the same commit checks - * would verify. Null when there is nothing to verify — no commit from - * THIS turn, no runbook, no local validation record, or the task is not - * `'microvm'` — never a signal of failure by itself. A runbook whose sha - * no longer matches its own local validation record (edited by hand, or - * simply re-scanned since) is REFUSED outright rather than silently - * verified against expectations it no longer meets. + * would verify. `verification` is null when there is nothing to verify — + * no commit from THIS turn, no runbook, no local validation record, or the + * task is not `'microvm'` — never a signal of failure by itself. A runbook + * whose sha no longer matches its own local validation record (edited by + * hand, or simply re-scanned since) is REFUSED outright rather than + * silently verified against expectations it no longer meets. * * `validatedSha`: read from `.codesema/runbook.validation.json` at the * PROJECT root — written by the scan (runbook-runner.ts) right alongside * `.codesema/runbook.json` itself. Never derived from git history: that * file is gitignored and never committed, so no commit ever touches it. + * + * `proof`: what the turn itself declared via `PROOF:` (D17) decides the + * capture: `none` is `'declined'`, no closure at all; `screenshot` + * replays the turn's own named pages; `journey` (or an undeclared turn, + * which falls back to the project's own default) replays a journey spec, + * guarded by `existsSync` as before. Every replaying case rides a closure + * along on `verifyTask`'s own seam: it only fires once the healthchecks + * are green (verifyTask's own contract), so `'attempted'` here means the + * app was proven alive first. `hostIncomingDir` is FORCED under + * `evidenceDir` (never a scratch dir elsewhere): `ingestEvidenceFiles`' + * merge relies on a same-filesystem `renameSync`. */ const verifyAfterCommit = async ( ctx: ProjectContext, record: TaskRecord, timeoutMs: number, - ): Promise => { + /** Called the instant `captureProof` actually starts, never on a skip. */ + onProofStart?: () => void, + ): Promise<{ verification: TaskVerification | null; proof: ProofCaptureOutcome }> => { if (record.isolation !== 'microvm') { - return null + return { verification: null, proof: { kind: 'not_attempted' } } } try { const commits = readTaskEvents(ctx.project.path, record.id).filter( (event) => event.type === 'commit', ) if (commits.at(-1)?.data.turn !== record.turns.length) { - return null + return { verification: null, proof: { kind: 'not_attempted' } } } const runbook = resolveTaskRunbook(ctx.project.path) if (!runbook) { - return null + return { verification: null, proof: { kind: 'not_attempted' } } } const readValidation = opts.readRunbookValidationFn ?? readRunbookValidation const validation = readValidation(ctx.project.path) if (!validation) { - return null + return { verification: null, proof: { kind: 'not_attempted' } } } const getHeadSha = opts.headShaFn ?? resolveHeadSha const headSha = getHeadSha(record.worktree) if (!headSha) { - return null + return { verification: null, proof: { kind: 'not_attempted' } } } const runbookSha = computeRunbookSha(runbook) if (validation.runbook_sha !== runbookSha) { const startedAt = new Date().toISOString() return { - head_sha: headSha, - runbook_sha: runbookSha, - started_at: startedAt, - finished_at: new Date().toISOString(), - status: 'refused', - checks: [], - integrity_ok: false, - changed_dependency_files: [], - error: 'runbook changed since its validation, rerun codesema runbook scan', + verification: { + head_sha: headSha, + runbook_sha: runbookSha, + started_at: startedAt, + finished_at: new Date().toISOString(), + status: 'refused', + checks: [], + integrity_ok: false, + changed_dependency_files: [], + error: 'runbook changed since its validation, rerun codesema runbook scan', + }, + proof: { kind: 'not_attempted' }, } } const build = await resolveMicrovmBuild(record, { @@ -2804,8 +2900,68 @@ export function createTaskManager(opts: CreateTaskManagerOptions): TaskManager { timeoutMs, command: ctx.command, }) + const proofConfig = readProofConfig(ctx.project.path) + let proofOutcome: ProofCaptureOutcome = { kind: 'not_attempted' } + let captureProofOption: ((handle: SandboxHandle) => Promise) | undefined + if (proofConfig) { + const intent = record.turns.at(-1)?.proof_intent + const hostIncomingDir = join(evidenceDir(ctx.project.path, record.id), '.incoming') + const guestProofDir = `${PROOF_VERIFY_GUEST_WORK_DIR}/.codesema-proof` + const proofTimeoutMs = (proofConfig.timeoutSeconds ?? 120) * 1000 + const buildJourneyCapture = + (journey: string) => + async (handle: SandboxHandle): Promise => { + onProofStart?.() + const result = await (opts.captureProofFn ?? captureProof)(handle, { + journey, + url: proofConfig.url, + timeoutMs: proofTimeoutMs, + guestWorkDir: PROOF_VERIFY_GUEST_WORK_DIR, + guestProofDir, + hostIncomingDir, + }) + proofOutcome = { kind: 'attempted', result, hostIncomingDir, keep: proofConfig.keep } + } + const buildScreenshotCapture = + (pages: string[]) => + async (handle: SandboxHandle): Promise => { + onProofStart?.() + const result = await (opts.captureScreenshotsFn ?? captureScreenshots)(handle, { + pages, + url: proofConfig.url, + timeoutMs: proofTimeoutMs, + guestWorkDir: PROOF_VERIFY_GUEST_WORK_DIR, + guestProofDir, + hostIncomingDir, + }) + proofOutcome = { kind: 'attempted', result, hostIncomingDir, keep: proofConfig.keep } + } + + if (intent === undefined) { + if (proofConfig.journey !== null) { + if (!existsSync(join(record.worktree, proofConfig.journey))) { + proofOutcome = { kind: 'spec_missing', journey: proofConfig.journey } + } else { + captureProofOption = buildJourneyCapture(proofConfig.journey) + } + } + } else if (intent.kind === 'none') { + proofOutcome = { kind: 'declined', reason: intent.reason } + } else if (intent.kind === 'screenshot') { + captureProofOption = buildScreenshotCapture(intent.pages ?? []) + } else { + const journey = intent.journey ?? proofConfig.journey + if (journey === null) { + proofOutcome = { kind: 'spec_missing', journey: '' } + } else if (!existsSync(join(record.worktree, journey))) { + proofOutcome = { kind: 'spec_missing', journey } + } else { + captureProofOption = buildJourneyCapture(journey) + } + } + } const run = opts.verifyTaskFn ?? verifyTask - return await run({ + const verification = await run({ driver: build.driver, worktree: record.worktree, projectId: ctx.project.id, @@ -2816,9 +2972,12 @@ export function createTaskManager(opts: CreateTaskManagerOptions): TaskManager { validatedSha: validation.validated_sha, snapshotName: build.snapshotName, timeoutMs, + onProgress: (line) => notice(`${record.id}: ${line}`), + ...(captureProofOption ? { captureProof: captureProofOption } : {}), }) + return { verification, proof: proofOutcome } } catch { - return null + return { verification: null, proof: { kind: 'not_attempted' } } } } @@ -2976,21 +3135,50 @@ export function createTaskManager(opts: CreateTaskManagerOptions): TaskManager { // rejects, so a failed auto-push cannot trip the runner's review_ko // fallback. `ctx` is assigned right below, before any turn can end. const onTurnDone: TaskTurnReviewFn = async (record, io) => { + // What this task's agent is doing right now (D-activity): narration + // only, nothing downstream branches on it. `setActivity` persists on + // the RAW `io`, never `gatedIo` below: the gate's own `applyGates` + // must not run every time a phase merely starts or ends. + const setActivity = (phase: TaskActivityPhase): void => { + record.activity = { phase, since: new Date().toISOString() } + io.persist() + } + const clearActivity = (): void => { + delete record.activity + io.persist() + } + const runPhase = async (phase: TaskActivityPhase, fn: () => Promise): Promise => { + setActivity(phase) + try { + return await fn() + } finally { + clearActivity() + } + } // T3.1: wait for THIS turn's checks (if it committed) BEFORE the // review. The checks slot is acquired and released inside the job, so // it is never held while the reviewer asks for its own — the T1.3 // deadlock the design forbids. A 409 (already running / no commit) // returns null immediately and does not stall the turn. - const thisTurnChecks = await startChecksAfterCommit(ctx, record) + const thisTurnChecks = await runPhase('checks', () => startChecksAfterCommit(ctx, record)) const gateChecks = terminalChecksResult(thisTurnChecks) ?? terminalChecksResult(readTaskChecks(cwd, record.id)) // Lot C7: the mechanical verification, right after checks and BEFORE // the review — a `'microvm'` task with a validated runbook gets // `runbook.tests` replayed in a fresh VM. Null (no runbook, no commit // from this turn, or not a 'microvm' task) means nothing to fold in. - const verification = await verifyAfterCommit(ctx, record, timeoutMs) + // `onProofStart` turns the phase from 'verification' into 'proof' the + // instant the capture actually starts, without a clear in between. + const { verification, proof } = await runPhase('verification', () => + verifyAfterCommit(ctx, record, timeoutMs, () => setActivity('proof')), + ) if (verification) { const cleanVerification = writeTaskVerification(cwd, record.id, verification) + emit({ + project_id: projectId, + task_id: record.id, + event: { name: 'task_verification', data: cleanVerification }, + }) const verificationBlocking = cleanVerification.status === 'refused' || cleanVerification.status === 'failed' const verificationEvent = appendTaskEvent(cwd, record.id, { @@ -3022,6 +3210,44 @@ export function createTaskManager(opts: CreateTaskManagerOptions): TaskManager { } } } + // Folds a turn's proof capture attempt into evidence.json, right + // beside the verification block above: 'not_attempted' covers both + // "proof isn't configured" and "verification never reached the + // capture seam", so nothing is written and no frame goes out. + if (proof.kind !== 'not_attempted') { + try { + const intent = record.turns.at(-1)?.proof_intent + const evidence = + proof.kind === 'attempted' + ? ingestEvidenceFiles(cwd, record.id, proof.hostIncomingDir, { + turn: record.turns.length, + status: proof.result.status, + reason: proof.result.reason, + head_sha: verification?.head_sha ?? null, + keep: proof.keep, + ...(intent ? { intent } : {}), + }) + : writeTaskEvidence(cwd, record.id, { + version: 1, + status: 'skipped', + reason: + proof.kind === 'declined' + ? proof.reason + : `proof journey spec not found in the worktree: ${proof.journey}`, + head_sha: verification?.head_sha ?? null, + items: readTaskEvidence(cwd, record.id)?.items ?? [], + ...(intent ? { intent } : {}), + }) + emit({ + project_id: projectId, + task_id: record.id, + event: { name: 'task_evidence', data: evidence }, + }) + } catch { + // Best-effort, same doctrine as the mechanical verification above: + // a failed ingest or write must never fail the turn. + } + } // T3.3: whether THIS review got as far as archiving a verdict. A review // that crashed leaves `record.review_ref` pointing at a PREVIOUS turn's // archive, and a fix turn built from it would ask the agent to re-fix @@ -3107,6 +3333,10 @@ export function createTaskManager(opts: CreateTaskManagerOptions): TaskManager { io.emit(input) }, persist: () => { + // Cleared BEFORE the gates, so the verdict `applyGates` is about to + // write and the end of the 'review' phase land in the SAME write, + // never a separate persist a reader could catch between the two. + delete record.activity applyGates() io.persist() }, @@ -3145,11 +3375,62 @@ export function createTaskManager(opts: CreateTaskManagerOptions): TaskManager { return } } - await reviewTurn(record, gatedIo) + setActivity('review') + try { + await reviewTurn(record, gatedIo) + } finally { + // Safety net: `gatedIo.persist` already clears this on the ordinary + // path, so this is a no-op there: it only matters for a reviewer + // that returns (or throws) without ever calling it, which would + // otherwise leave 'review' stuck on the record past this turn. + delete record.activity + } + // D17: the reviewer may have just folded its proof_review verdict into + // evidence.json (task-review.ts's createTaskReviewer, right after + // finding-repro verification): re-read and re-emit so the verdict + // reaches the card without a client re-fetch. Silent when nothing + // changed (no proof chapter, no evidence for this commit). + const postReviewEvidence = readTaskEvidence(cwd, record.id) + if (postReviewEvidence) { + emit({ + project_id: projectId, + task_id: record.id, + event: { name: 'task_evidence', data: postReviewEvidence }, + }) + } // Stubs that set status without persist, and the no-review path, still // fold the gates in: idempotent if the wrapped persist already did. applyGates() io.persist() + // Product decision: the recap no longer waits for the ship. Generated + // here, right after the settle, with the SAME `generateRecap` the ship + // still calls (task-ship.ts): `mr_url` is read off the LAST 'shipped' + // journal event (buildMrUrl), which does not exist yet on a task that + // has never shipped, so it comes back honestly absent rather than a + // value this hook would have to invent. The ship's own regeneration + // (with mr_url once pushed) overwrites this record and re-emits under + // its own `recapState` guard, unchanged by this addition. No extra + // model call: `changes[]`/`decisions[]` stay empty here, same + // `lastTurnResponse` read task-ship.ts's `lastTurnContribution` does + // for `summary`. Best-effort: never blocks or fails the turn's settle. + if (record.status === 'review_ok') { + await runPhase('recap', async () => { + try { + const result = generateRecap(recapOptionsFor(cwd, record)) + if (result.recap) { + const cleanRecap = writeTaskRecap(cwd, record.id, result.recap) + emit({ + project_id: projectId, + task_id: record.id, + event: { name: 'task_recap', data: cleanRecap }, + }) + } + } catch { + // ignored: best-effort, same doctrine as the verification/evidence + // blocks above + } + }) + } if (record.auto_ship && record.status === 'review_ok') { await ship(ctx, record.id) // T3.6, AWAITED and not fired off: the spec promises that a missing diff --git a/packages/cli/src/task-verification.test.ts b/packages/cli/src/task-verification.test.ts index fe9108c..89cd136 100644 --- a/packages/cli/src/task-verification.test.ts +++ b/packages/cli/src/task-verification.test.ts @@ -375,7 +375,7 @@ describe('verifyTask', () => { const repo = makeRepo() const sha = commitSha(repo) const { driver, calls } = fakeDriver((command) => - command === 'docker compose up -d' ? ok({ code: 1 }) : ok(), + command.includes('nohup') ? ok({ code: 1 }) : ok(), ) const result = await verifyTask({ driver, @@ -397,6 +397,54 @@ describe('verifyTask', () => { expect(calls.filter((c) => c.method === 'shell')).toHaveLength(1) }) + test('a service is launched in the background with nohup, one shell call per service', async () => { + const repo = makeRepo() + const sha = commitSha(repo) + const { driver, calls } = fakeDriver(() => ok()) + const result = await verifyTask({ + driver, + worktree: repo, + projectId: 'p1', + taskId: 't1', + headSha: 'headsha1', + runbook: baseRunbook({ + services: { host_up: ['npm start'], compose_file: null }, + }), + runbookSha: '0123456789abcdef', + validatedSha: sha, + snapshotName: 'codesema-p1-hash', + timeoutMs: 5000, + }) + expect(result.status).toBe('passed') + const shellCommands = calls.filter((c) => c.method === 'shell').map((c) => c.args[0]) + expect(shellCommands[0]).toContain('nohup') + expect(shellCommands[0]).toContain('/tmp/codesema-service-0.log') + }) + + test('a single quote in the service command is escaped in the background script', async () => { + const repo = makeRepo() + const sha = commitSha(repo) + const { driver, calls } = fakeDriver(() => ok()) + await verifyTask({ + driver, + worktree: repo, + projectId: 'p1', + taskId: 't1', + headSha: 'headsha1', + runbook: baseRunbook({ + services: { host_up: ["echo it's up"], compose_file: null }, + }), + runbookSha: '0123456789abcdef', + validatedSha: sha, + snapshotName: 'codesema-p1-hash', + timeoutMs: 5000, + }) + const shellCommands = calls.filter((c) => c.method === 'shell').map((c) => c.args[0]) + expect(shellCommands[0]).toBe( + "nohup sh -c 'echo it'\\''s up' > /tmp/codesema-service-0.log 2>&1 &", + ) + }) + test('the sandbox is created with the exact sandboxName for the task, never a generic one', async () => { const repo = makeRepo() const sha = commitSha(repo) @@ -478,6 +526,81 @@ describe('verifyTask', () => { expect(destroy?.args[0]).toBe('codesema-verify-t1') }) + test('captureProof runs after healthchecks pass and before runbook.tests', async () => { + const repo = makeRepo() + const sha = commitSha(repo) + const { driver, calls } = fakeDriver(() => ok()) + const result = await verifyTask({ + driver, + worktree: repo, + projectId: 'p1', + taskId: 't1', + headSha: 'headsha1', + runbook: baseRunbook({ healthchecks: ['curl -f http://localhost:3000'] }), + runbookSha: '0123456789abcdef', + validatedSha: sha, + snapshotName: 'codesema-p1-hash', + timeoutMs: 5000, + captureProof: async (handle) => { + await handle.shell('proof-marker', { timeoutMs: 1000, cwd: '/work' }) + }, + }) + expect(result.status).toBe('passed') + const shellCommands = calls.filter((c) => c.method === 'shell').map((c) => c.args[0]) + expect(shellCommands).toEqual(['curl -f http://localhost:3000', 'proof-marker', 'npm test']) + }) + + test('captureProof is not called when healthchecks never pass', async () => { + const repo = makeRepo() + const sha = commitSha(repo) + const { driver } = fakeDriver((command) => + command === 'curl -f http://localhost:3000' ? ok({ code: 1, stderr: 'not up' }) : ok(), + ) + let called = false + const result = await verifyTask({ + driver, + worktree: repo, + projectId: 'p1', + taskId: 't1', + headSha: 'headsha1', + runbook: baseRunbook({ healthchecks: ['curl -f http://localhost:3000'] }), + runbookSha: '0123456789abcdef', + validatedSha: sha, + snapshotName: 'codesema-p1-hash', + timeoutMs: 5000, + healthcheckDeadlineMs: 10, + healthcheckRetryDelayMs: 1, + captureProof: async () => { + called = true + }, + }) + expect(result.status).toBe('error') + expect(called).toBe(false) + }) + + test('an exception from captureProof is swallowed and never changes the verdict', async () => { + const repo = makeRepo() + const sha = commitSha(repo) + const { driver } = fakeDriver(() => ok()) + const result = await verifyTask({ + driver, + worktree: repo, + projectId: 'p1', + taskId: 't1', + headSha: 'headsha1', + runbook: baseRunbook(), + runbookSha: '0123456789abcdef', + validatedSha: sha, + snapshotName: 'codesema-p1-hash', + timeoutMs: 5000, + captureProof: async () => { + throw new Error('proof capture boom') + }, + }) + expect(result.status).toBe('passed') + expect(result.checks).toHaveLength(1) + }) + test('carries head_sha and runbook_sha through, on every status', async () => { const repo = makeRepo() const sha = commitSha(repo) diff --git a/packages/cli/src/task-verification.ts b/packages/cli/src/task-verification.ts index 1165ffc..5cbb639 100644 --- a/packages/cli/src/task-verification.ts +++ b/packages/cli/src/task-verification.ts @@ -22,6 +22,7 @@ import { type SandboxHandle, type SandboxSpec, } from './microsandbox-driver.js' +import { SERVICE_LAUNCH_TIMEOUT_MS, serviceLaunchScript } from './runbook-services.js' import { taskDir } from './tasks-store.js' export type VerifyTaskOptions = { @@ -45,6 +46,8 @@ export type VerifyTaskOptions = { healthcheckDeadlineMs?: number /** Delay between healthcheck retries; default 1s. */ healthcheckRetryDelayMs?: number + /** Best-effort browser proof capture, run after healthchecks pass and before `runbook.tests`; never affects the verdict. */ + captureProof?: (handle: SandboxHandle) => Promise } const VERIFY_WORK_DIR = '/work' @@ -160,11 +163,11 @@ export async function verifyTask(opts: VerifyTaskOptions): Promise undefined) + } + const checks: TaskCheckResult[] = [] for (const command of opts.runbook.tests) { const testStartedAt = Date.now() diff --git a/packages/cli/src/ui-surface.test.ts b/packages/cli/src/ui-surface.test.ts new file mode 100644 index 0000000..954163d --- /dev/null +++ b/packages/cli/src/ui-surface.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, test } from 'bun:test' +import { classifyUiPaths } from './ui-surface.js' + +describe('classifyUiPaths', () => { + test('recognizes UI extensions', () => { + expect(classifyUiPaths(['Button.vue', 'App.tsx', 'index.html', 'main.scss']).ui).toEqual([ + 'Button.vue', + 'App.tsx', + 'index.html', + 'main.scss', + ]) + }) + + test('classifies non-UI extensions as other', () => { + expect(classifyUiPaths(['server.ts', 'README.md', 'package.json']).other).toEqual([ + 'server.ts', + 'README.md', + 'package.json', + ]) + }) + + test('recognizes UI path hint segment', () => { + expect(classifyUiPaths(['src/components/x.ts']).ui).toEqual(['src/components/x.ts']) + }) + + test('recognizes whole segment app as UI hint, not a prefix match', () => { + expect(classifyUiPaths(['app/server.ts']).ui).toEqual(['app/server.ts']) + expect(classifyUiPaths(['application/x.ts']).other).toEqual(['application/x.ts']) + }) + + test('recognizes styles path hint segment', () => { + expect(classifyUiPaths(['styles/tokens.ts']).ui).toEqual(['styles/tokens.ts']) + }) + + test('extension match is case-insensitive', () => { + expect(classifyUiPaths(['Legacy.VUE']).ui).toEqual(['Legacy.VUE']) + }) + + test('normalizes leading ./ and backslash separators before classifying', () => { + const result = classifyUiPaths(['./src/pages/a.ts', 'src\\views\\b.ts']) + expect(result.ui).toEqual(['./src/pages/a.ts', 'src\\views\\b.ts']) + expect(result.other).toEqual([]) + }) + + test('preserves input order and keeps duplicates', () => { + const result = classifyUiPaths(['server.ts', 'App.tsx', 'server.ts', 'README.md', 'App.tsx']) + expect(result.ui).toEqual(['App.tsx', 'App.tsx']) + expect(result.other).toEqual(['server.ts', 'server.ts', 'README.md']) + }) + + test('empty input returns two empty arrays', () => { + expect(classifyUiPaths([])).toEqual({ ui: [], other: [] }) + }) + + test('ignores empty path entries', () => { + expect(classifyUiPaths(['', 'server.ts', ''])).toEqual({ ui: [], other: ['server.ts'] }) + }) +}) diff --git a/packages/cli/src/ui-surface.ts b/packages/cli/src/ui-surface.ts new file mode 100644 index 0000000..49015a9 --- /dev/null +++ b/packages/cli/src/ui-surface.ts @@ -0,0 +1,60 @@ +export const UI_EXTENSIONS: ReadonlySet = new Set([ + 'vue', + 'svelte', + 'tsx', + 'jsx', + 'html', + 'htm', + 'css', + 'scss', + 'less', +]) + +export const UI_PATH_HINTS: readonly string[] = [ + 'components', + 'pages', + 'views', + 'public', + 'ui', + 'app', + 'layouts', + 'styles', +] + +function normalizePath(raw: string): string { + const slashed = raw.replaceAll('\\', '/') + return slashed.startsWith('./') ? slashed.slice(2) : slashed +} + +function extensionOf(path: string): string | null { + const base = path.slice(path.lastIndexOf('/') + 1) + const dot = base.lastIndexOf('.') + if (dot <= 0) { + return null + } + return base.slice(dot + 1).toLowerCase() +} + +function hasUiPathHint(path: string): boolean { + const segments = path.split('/') + return segments.some((segment) => UI_PATH_HINTS.includes(segment.toLowerCase())) +} + +export function classifyUiPaths(paths: readonly string[]): { ui: string[]; other: string[] } { + const ui: string[] = [] + const other: string[] = [] + for (const raw of paths) { + if (raw === '') { + continue + } + const path = normalizePath(raw) + const extension = extensionOf(path) + const isUi = (extension !== null && UI_EXTENSIONS.has(extension)) || hasUiPathHint(path) + if (isUi) { + ui.push(raw) + } else { + other.push(raw) + } + } + return { ui, other } +} diff --git a/packages/contract/package.json b/packages/contract/package.json index a9902e5..5c5b99d 100644 --- a/packages/contract/package.json +++ b/packages/contract/package.json @@ -1,6 +1,6 @@ { "name": "@codesema/contract", - "version": "0.11.0", + "version": "0.12.0", "description": "Shared review contract (types + sanitizers) between the codesema CLI and codesema.com.", "license": "MIT", "author": "Hasan TASKIN", diff --git a/packages/contract/src/evidence.test.ts b/packages/contract/src/evidence.test.ts new file mode 100644 index 0000000..cf7f3e5 --- /dev/null +++ b/packages/contract/src/evidence.test.ts @@ -0,0 +1,294 @@ +import { describe, expect, test } from 'bun:test' +import { + EVIDENCE_ITEMS_MAX, + EVIDENCE_PATH_MAX, + EVIDENCE_REASON_MAX, + sanitizeEvidence, + type EvidenceRecord, +} from './evidence.js' + +const FULL_RECORD: EvidenceRecord = { + version: 1, + status: 'passed', + reason: null, + head_sha: 'a1b2c3d4', + items: [ + { + kind: 'screenshot', + path: 'step-1.png', + bytes: 1_024, + turn: 1, + created_at: '2026-08-30T10:00:00Z', + }, + { kind: 'video', path: 'run.mp4', bytes: 2_048, turn: 2, created_at: '2026-08-30T10:05:00Z' }, + ], +} + +test('published bounds are locked to their literal values', () => { + expect(EVIDENCE_ITEMS_MAX).toBe(40) + expect(EVIDENCE_PATH_MAX).toBe(200) + expect(EVIDENCE_REASON_MAX).toBe(2_000) +}) + +describe('sanitizeEvidence', () => { + test('a valid, full record round-trips unchanged', () => { + expect(sanitizeEvidence(structuredClone(FULL_RECORD))).toEqual(FULL_RECORD) + }) + + test('a minimal record (no reason, no head_sha, no items) is honest about the rest', () => { + const out = sanitizeEvidence({ version: 1, status: 'skipped', items: [] }) + expect(out).toEqual({ version: 1, status: 'skipped', reason: null, head_sha: null, items: [] }) + }) + + test('non-object input never throws and returns null', () => { + for (const raw of [null, undefined, 42, 'x', [], Symbol('x'), true]) { + expect(() => sanitizeEvidence(raw)).not.toThrow() + expect(sanitizeEvidence(raw)).toBeNull() + } + }) + + test('an unknown or missing version is refused: the record identity is version 1', () => { + for (const raw of [ + { version: 2, status: 'passed', items: [] }, + { version: '1', status: 'passed', items: [] }, + { status: 'passed', items: [] }, + ]) { + expect(sanitizeEvidence(raw)).toBeNull() + } + }) + + test('a status outside the union is refused', () => { + expect(sanitizeEvidence({ version: 1, status: 'bogus', items: [] })).toBeNull() + }) + + test('reason: over-long is truncated to its published bound, not dropped', () => { + const out = sanitizeEvidence({ + version: 1, + status: 'failed', + reason: 'x'.repeat(EVIDENCE_REASON_MAX + 500), + items: [], + }) + expect(out?.reason).toHaveLength(EVIDENCE_REASON_MAX) + }) + + test('reason: absent or non-string is null, never a placeholder', () => { + for (const bad of [undefined, 42, null, {}, []]) { + const out = sanitizeEvidence({ version: 1, status: 'passed', reason: bad, items: [] }) + expect(out?.reason).toBeNull() + } + }) + + test('head_sha: non-string or empty is null, never a placeholder', () => { + for (const bad of [undefined, 42, null, '', {}, []]) { + const out = sanitizeEvidence({ version: 1, status: 'passed', head_sha: bad, items: [] }) + expect(out?.head_sha).toBeNull() + } + }) + + test('unknown top-level fields are dropped (whitelist)', () => { + const withExtra = { ...structuredClone(FULL_RECORD), evil: 'payload', __proto__: { x: 1 } } + expect(Object.keys(sanitizeEvidence(withExtra) ?? {})).not.toContain('evil') + }) + + test('unknown item fields are dropped (whitelist)', () => { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [ + { + kind: 'screenshot', + path: 'ok.png', + bytes: 1, + turn: 1, + created_at: 'x', + evil: 'payload', + }, + ], + }) + expect(out?.items).toEqual([ + { kind: 'screenshot', path: 'ok.png', bytes: 1, turn: 1, created_at: 'x' }, + ]) + }) + + test('items[]: an entry with an unrecognized kind is dropped, not the whole list', () => { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [ + { kind: 'gif', path: 'a.gif', bytes: 1, turn: 1, created_at: 'x' }, + { kind: 'screenshot', path: 'ok.png', bytes: 1, turn: 1, created_at: 'x' }, + ], + }) + expect(out?.items).toEqual([ + { kind: 'screenshot', path: 'ok.png', bytes: 1, turn: 1, created_at: 'x' }, + ]) + }) + + test('items[].path: a traversal segment ("../x") is rejected outright', () => { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [{ kind: 'screenshot', path: '../x', bytes: 1, turn: 1, created_at: 'x' }], + }) + expect(out?.items).toEqual([]) + }) + + test('items[].path: any path containing a slash is rejected', () => { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [{ kind: 'screenshot', path: 'dir/file.png', bytes: 1, turn: 1, created_at: 'x' }], + }) + expect(out?.items).toEqual([]) + }) + + test('items[].path: over the max length is rejected outright, never truncated', () => { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [ + { + kind: 'screenshot', + path: `${'a'.repeat(EVIDENCE_PATH_MAX + 1)}.png`, + bytes: 1, + turn: 1, + created_at: 'x', + }, + ], + }) + expect(out?.items).toEqual([]) + }) + + test('items[].path: a path at exactly the max length, matching the whitelist, is kept', () => { + const path = 'a'.repeat(EVIDENCE_PATH_MAX) + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [{ kind: 'screenshot', path, bytes: 1, turn: 1, created_at: 'x' }], + }) + expect(out?.items).toEqual([{ kind: 'screenshot', path, bytes: 1, turn: 1, created_at: 'x' }]) + }) + + test('items[].bytes: a negative, non-integer or non-numeric value is rejected, one entry at a time', () => { + for (const bad of [-1, 1.5, 'x', null, undefined, Number.NaN, 2 ** 53]) { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [{ kind: 'screenshot', path: 'ok.png', bytes: bad, turn: 1, created_at: 'x' }], + }) + expect(out?.items).toEqual([]) + } + }) + + test('items[].turn: a negative, non-integer or non-numeric value is rejected, one entry at a time', () => { + for (const bad of [-1, 1.5, 'x', null, undefined, Number.NaN]) { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [{ kind: 'screenshot', path: 'ok.png', bytes: 1, turn: bad, created_at: 'x' }], + }) + expect(out?.items).toEqual([]) + } + }) + + test('items[].created_at: a non-string or empty value is rejected, one entry at a time', () => { + for (const bad of ['', 42, null, undefined]) { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [{ kind: 'screenshot', path: 'ok.png', bytes: 1, turn: 1, created_at: bad }], + }) + expect(out?.items).toEqual([]) + } + }) + + test('items[]: a non-object entry (null, string, array) is dropped, never thrown on', () => { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [ + null, + 'x', + [], + { kind: 'screenshot', path: 'ok.png', bytes: 1, turn: 1, created_at: 'x' }, + ], + }) + expect(out?.items).toEqual([ + { kind: 'screenshot', path: 'ok.png', bytes: 1, turn: 1, created_at: 'x' }, + ]) + }) + + test('items[] is capped at EVIDENCE_ITEMS_MAX; the excess is discarded, not the whole list', () => { + const many = Array.from({ length: EVIDENCE_ITEMS_MAX + 1 }, (_, i) => ({ + kind: 'screenshot' as const, + path: `s${i}.png`, + bytes: 1, + turn: i, + created_at: 'x', + })) + const out = sanitizeEvidence({ version: 1, status: 'passed', items: many }) + expect(out?.items).toHaveLength(EVIDENCE_ITEMS_MAX) + }) + + test('items that is not an array yields an empty list, never a throw', () => { + for (const bad of ['not-an-array', 42, null]) { + const out = sanitizeEvidence({ version: 1, status: 'passed', items: bad }) + expect(out?.items).toEqual([]) + } + }) + + test('intent and review: absent when not supplied', () => { + const out = sanitizeEvidence({ version: 1, status: 'passed', items: [] }) + expect(out?.intent).toBeUndefined() + expect(out?.review).toBeUndefined() + }) + + test('intent and review: round-trip when valid', () => { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [], + intent: { kind: 'screenshot', reason: 'proves the flow rendered', pages: ['/checkout'] }, + review: { expected: 'screenshot', coherent: true, reason: 'matches the declared intent' }, + }) + expect(out?.intent).toEqual({ + kind: 'screenshot', + reason: 'proves the flow rendered', + pages: ['/checkout'], + }) + expect(out?.review).toEqual({ + expected: 'screenshot', + coherent: true, + reason: 'matches the declared intent', + }) + }) + + test('intent and review: an unreadable value drops the key rather than storing junk', () => { + const out = sanitizeEvidence({ + version: 1, + status: 'passed', + items: [], + intent: { kind: 'bogus', reason: 'r' }, + review: { expected: 'none', coherent: 'yes', reason: '' }, + }) + expect(out?.intent).toBeUndefined() + expect(out?.review).toBeUndefined() + }) + + test('hostile entry never throws: wrong types everywhere, null and undefined', () => { + const hostile = { + version: '1', + status: 'BOGUS', + reason: { nested: true }, + head_sha: 42, + items: [ + { kind: 'gif', path: 'a.gif', bytes: 1, turn: 1, created_at: 'x' }, + { kind: 'screenshot', path: '../etc/passwd', bytes: -1, turn: 'z', created_at: '' }, + null, + 'x', + ], + } + expect(() => sanitizeEvidence(hostile)).not.toThrow() + expect(sanitizeEvidence(hostile)).toBeNull() + }) +}) diff --git a/packages/contract/src/evidence.ts b/packages/contract/src/evidence.ts new file mode 100644 index 0000000..7705119 --- /dev/null +++ b/packages/contract/src/evidence.ts @@ -0,0 +1,114 @@ +import { + sanitizeProofIntent, + sanitizeProofReview, + type ProofIntent, + type ProofReview, +} from './proof-intent.js' +import { cutCodePoints } from './ticket.js' + +export type EvidenceKind = 'screenshot' | 'video' + +export type EvidenceItem = { + kind: EvidenceKind + path: string + bytes: number + turn: number + created_at: string +} + +export type EvidenceStatus = 'passed' | 'failed' | 'skipped' + +export type EvidenceRecord = { + version: 1 + status: EvidenceStatus + reason: string | null + head_sha: string | null + items: EvidenceItem[] + intent?: ProofIntent + review?: ProofReview +} + +export const EVIDENCE_ITEMS_MAX = 40 +export const EVIDENCE_PATH_MAX = 200 +export const EVIDENCE_REASON_MAX = 2_000 + +const EVIDENCE_KINDS: ReadonlySet = new Set(['screenshot', 'video']) +const EVIDENCE_STATUSES: ReadonlySet = new Set(['passed', 'failed', 'skipped']) +const EVIDENCE_PATH_PATTERN = /^[A-Za-z0-9._-]+$/ + +function isNonNegativeInt(v: unknown): v is number { + return typeof v === 'number' && Number.isSafeInteger(v) && v >= 0 +} + +function sanitizeEvidenceItem(raw: unknown): EvidenceItem | null { + if (!raw || typeof raw !== 'object') { + return null + } + const r = raw as Record + if (!EVIDENCE_KINDS.has(r.kind as EvidenceKind)) { + return null + } + if ( + typeof r.path !== 'string' || + r.path.length > EVIDENCE_PATH_MAX || + !EVIDENCE_PATH_PATTERN.test(r.path) + ) { + return null + } + if (!isNonNegativeInt(r.bytes) || !isNonNegativeInt(r.turn)) { + return null + } + if (typeof r.created_at !== 'string' || r.created_at.length === 0) { + return null + } + return { + kind: r.kind as EvidenceKind, + path: r.path, + bytes: r.bytes, + turn: r.turn, + created_at: r.created_at, + } +} + +function sanitizeEvidenceItems(raw: unknown): EvidenceItem[] { + if (!Array.isArray(raw)) { + return [] + } + const out: EvidenceItem[] = [] + for (const item of raw) { + if (out.length >= EVIDENCE_ITEMS_MAX) { + break + } + const sanitized = sanitizeEvidenceItem(item) + if (sanitized) { + out.push(sanitized) + } + } + return out +} + +export function sanitizeEvidence(raw: unknown): EvidenceRecord | null { + if (!raw || typeof raw !== 'object') { + return null + } + const r = raw as Record + if (r.version !== 1) { + return null + } + if (!EVIDENCE_STATUSES.has(r.status as EvidenceStatus)) { + return null + } + const reason = typeof r.reason === 'string' ? cutCodePoints(r.reason, EVIDENCE_REASON_MAX) : null + const headSha = typeof r.head_sha === 'string' && r.head_sha.length > 0 ? r.head_sha : null + const intent = sanitizeProofIntent(r.intent) + const review = sanitizeProofReview(r.review) + return { + version: 1, + status: r.status as EvidenceStatus, + reason, + head_sha: headSha, + items: sanitizeEvidenceItems(r.items), + ...(intent !== null ? { intent } : {}), + ...(review !== null ? { review } : {}), + } +} diff --git a/packages/contract/src/index.test.ts b/packages/contract/src/index.test.ts index dc08a06..b09eac9 100644 --- a/packages/contract/src/index.test.ts +++ b/packages/contract/src/index.test.ts @@ -706,6 +706,7 @@ describe('reviewRecordSchema', () => { narrative: { intent: 'i', steps: [], review_first: [] }, files_reviewed: ['a.ts'], criteria: [{ criterion_id: 'ac-000000000001', status: 'met', evidence: 'a.ts:1 x' }], + proof_review: { expected: 'none', coherent: true, reason: '' }, }) const declared = new Set(Object.keys(reviewRecordSchema.$defs.review.properties)) expect(Object.keys(produced).length).toBeGreaterThan(4) @@ -777,6 +778,41 @@ describe('sanitizeReview criteria (DP12)', () => { }) }) +describe('sanitizeReview proof_review (D17)', () => { + test('a valid proof_review round-trips', () => { + const review = sanitizeReview({ + proof_review: { expected: 'journey', coherent: true, reason: 'matches what was declared' }, + }) + expect(review.proof_review).toEqual({ + expected: 'journey', + coherent: true, + reason: 'matches what was declared', + }) + }) + + test('an unreadable proof_review omits the key', () => { + expect(sanitizeReview({}).proof_review).toBeUndefined() + expect( + sanitizeReview({ proof_review: { expected: 'bogus', coherent: true, reason: '' } }) + .proof_review, + ).toBeUndefined() + }) + + test('sanitizeRecord carries proof_review back off disk (the whitelist keeps it)', () => { + const record = sanitizeRecord({ + version: 1, + meta: {}, + commits: [], + diff: '', + review: { + verdict: 'approve', + proof_review: { expected: 'none', coherent: true, reason: '' }, + }, + }) + expect(record?.review.proof_review).toEqual({ expected: 'none', coherent: true, reason: '' }) + }) +}) + describe('parseEvidenceAnchor', () => { test('reads the path:line an evidence opens with, prose and all', () => { expect(parseEvidenceAnchor('src/auth.ts:11 — the guard is added here')).toEqual({ @@ -1367,10 +1403,20 @@ describe('cross test: sanitizeRecord output validates against reviewRecordSchema { criterion_id: AC_A, status: 'met', evidence: 'src/auth.ts:11 — added here' }, { criterion_id: AC_B, status: 'unclear', question: 'does this cover offline mode too?' }, ], + proof_review: { + expected: 'journey', + coherent: true, + reason: 'matches the declared intent', + }, }, }) expect(schemaErrors(record)).toEqual([]) expect(record?.review.findings[0]?.repro).toEqual({ command: 'npm test', expected: 'exit 0' }) + expect(record?.review.proof_review).toEqual({ + expected: 'journey', + coherent: true, + reason: 'matches the declared intent', + }) }) test('the minimal record — everything the sanitizer defaults — validates', () => { diff --git a/packages/contract/src/index.ts b/packages/contract/src/index.ts index e607781..eebd8f9 100644 --- a/packages/contract/src/index.ts +++ b/packages/contract/src/index.ts @@ -3,6 +3,12 @@ // `ReviewRecord`) and by `RecapRecord` (T3.4). Importing it rather than // restating it is what keeps the three consumers from drifting into two // spellings of the same fact. ticket.ts imports nothing, so this cannot cycle. +import { + PROOF_INTENT_KINDS, + PROOF_INTENT_REASON_MAX, + sanitizeProofReview, + type ProofReview, +} from './proof-intent.js' import { CRITERION_VERDICT_EVIDENCE_MAX, CRITERION_VERDICT_QUESTION_MAX, @@ -16,6 +22,8 @@ import { // All agent input passes through here: whitelist and truncate, never throw. export * from './arm.js' +export * from './evidence.js' +export * from './proof-intent.js' export * from './reasons.js' export * from './recap.js' export * from './runbook.js' @@ -134,6 +142,7 @@ export type SanitizedReview = { * deterministic; nothing here is a verdict the model produced. */ criteria?: CriterionVerdict[] + proof_review?: ProofReview } export type DualStats = { @@ -453,6 +462,7 @@ export function sanitizeReview(raw: unknown): SanitizedReview { // Nothing survives means the key is OMITTED, not emptied: see the field's // own doc on `SanitizedReview`. const criteria = sanitizeCriterionVerdicts(r.criteria) + const proofReview = sanitizeProofReview(r.proof_review) return { verdict, summary, @@ -462,6 +472,7 @@ export function sanitizeReview(raw: unknown): SanitizedReview { ? { files_reviewed: reviewedFilesFrom(reviewedPaths, findings) } : {}), ...(criteria.length > 0 ? { criteria } : {}), + ...(proofReview !== null ? { proof_review: proofReview } : {}), } } @@ -1152,6 +1163,17 @@ export const reviewRecordSchema = { uniqueItems: true, items: { $ref: '#/$defs/criterionVerdict' }, }, + proof_review: { $ref: '#/$defs/proofReview' }, + }, + }, + proofReview: { + type: 'object', + additionalProperties: false, + required: ['expected', 'coherent', 'reason'], + properties: { + expected: { enum: [...PROOF_INTENT_KINDS] }, + coherent: { type: 'boolean' }, + reason: { type: 'string', maxLength: PROOF_INTENT_REASON_MAX }, }, }, criterionVerdict: { diff --git a/packages/contract/src/proof-intent.test.ts b/packages/contract/src/proof-intent.test.ts new file mode 100644 index 0000000..b0ed656 --- /dev/null +++ b/packages/contract/src/proof-intent.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, test } from 'bun:test' +import { + PROOF_INTENT_PAGES_MAX, + PROOF_INTENT_PATH_MAX, + PROOF_INTENT_REASON_MAX, + sanitizeProofIntent, + sanitizeProofReview, +} from './proof-intent.js' + +test('published bounds are locked to their literal values', () => { + expect(PROOF_INTENT_REASON_MAX).toBe(500) + expect(PROOF_INTENT_PAGES_MAX).toBe(8) + expect(PROOF_INTENT_PATH_MAX).toBe(200) +}) + +describe('sanitizeProofIntent', () => { + test('non-object input never throws and returns null', () => { + for (const raw of [null, undefined, 42, 'x', [], Symbol('x'), true]) { + expect(() => sanitizeProofIntent(raw)).not.toThrow() + expect(sanitizeProofIntent(raw)).toBeNull() + } + }) + + test('a kind outside the closed enum is refused', () => { + expect(sanitizeProofIntent({ kind: 'bogus', reason: 'r' })).toBeNull() + }) + + test('a missing, empty or non-string reason is refused', () => { + for (const bad of [undefined, '', ' ', 42, null, {}]) { + expect(sanitizeProofIntent({ kind: 'none', reason: bad })).toBeNull() + } + }) + + test('reason: over-long is truncated to the published bound', () => { + const out = sanitizeProofIntent({ + kind: 'none', + reason: 'x'.repeat(PROOF_INTENT_REASON_MAX + 500), + }) + expect(out?.reason).toHaveLength(PROOF_INTENT_REASON_MAX) + }) + + test('kind: none carries no pages or journey even when supplied', () => { + const out = sanitizeProofIntent({ kind: 'none', reason: 'r', pages: ['/a'], journey: 'b' }) + expect(out).toEqual({ kind: 'none', reason: 'r' }) + }) + + test('kind: screenshot with valid pages keeps them', () => { + const out = sanitizeProofIntent({ kind: 'screenshot', reason: 'r', pages: ['/a', '/b/c'] }) + expect(out).toEqual({ kind: 'screenshot', reason: 'r', pages: ['/a', '/b/c'] }) + }) + + test('kind: screenshot ignores journey — not relevant to this kind', () => { + const out = sanitizeProofIntent({ kind: 'screenshot', reason: 'r', journey: 'checkout' }) + expect(out).toEqual({ kind: 'screenshot', reason: 'r' }) + }) + + test('kind: journey with a valid path keeps it', () => { + const out = sanitizeProofIntent({ kind: 'journey', reason: 'r', journey: 'checkout/pay' }) + expect(out).toEqual({ kind: 'journey', reason: 'r', journey: 'checkout/pay' }) + }) + + test('kind: journey ignores pages — not relevant to this kind', () => { + const out = sanitizeProofIntent({ kind: 'journey', reason: 'r', pages: ['/a'] }) + expect(out).toEqual({ kind: 'journey', reason: 'r' }) + }) + + test('pages: a protocol-relative entry ("//evil.com") is rejected outright', () => { + const out = sanitizeProofIntent({ + kind: 'screenshot', + reason: 'r', + pages: ['//evil.com', '/ok'], + }) + expect(out?.pages).toEqual(['/ok']) + }) + + test('pages: an entry without a leading slash is rejected', () => { + const out = sanitizeProofIntent({ kind: 'screenshot', reason: 'r', pages: ['a', '/ok'] }) + expect(out?.pages).toEqual(['/ok']) + }) + + test('pages: an entry containing ".." is rejected', () => { + const out = sanitizeProofIntent({ + kind: 'screenshot', + reason: 'r', + pages: ['/a/../b', '/ok'], + }) + expect(out?.pages).toEqual(['/ok']) + }) + + test('pages: an entry containing an apostrophe is rejected', () => { + const out = sanitizeProofIntent({ kind: 'screenshot', reason: 'r', pages: ["/it's", '/ok'] }) + expect(out?.pages).toEqual(['/ok']) + }) + + test('pages: an entry containing a space is rejected', () => { + const out = sanitizeProofIntent({ kind: 'screenshot', reason: 'r', pages: ['/a b', '/ok'] }) + expect(out?.pages).toEqual(['/ok']) + }) + + test('pages: an entry over the max path length is rejected outright, never truncated', () => { + const tooLong = `/${'a'.repeat(PROOF_INTENT_PATH_MAX)}` + const out = sanitizeProofIntent({ kind: 'screenshot', reason: 'r', pages: [tooLong, '/ok'] }) + expect(out?.pages).toEqual(['/ok']) + }) + + test('pages: only the first 8 valid entries are kept, the excess dropped', () => { + const pages = Array.from({ length: PROOF_INTENT_PAGES_MAX + 1 }, (_, i) => `/p${i}`) + const out = sanitizeProofIntent({ kind: 'screenshot', reason: 'r', pages }) + expect(out?.pages).toHaveLength(PROOF_INTENT_PAGES_MAX) + }) + + test('pages: absent when the input carries none valid — never an empty array', () => { + const out = sanitizeProofIntent({ kind: 'screenshot', reason: 'r', pages: ['nope', '//x'] }) + expect(out?.pages).toBeUndefined() + }) + + test('journey: an absolute path (leading slash) is rejected', () => { + const out = sanitizeProofIntent({ kind: 'journey', reason: 'r', journey: '/checkout' }) + expect(out?.journey).toBeUndefined() + }) + + test('journey: a path containing ".." is rejected', () => { + const out = sanitizeProofIntent({ kind: 'journey', reason: 'r', journey: 'a/../b' }) + expect(out?.journey).toBeUndefined() + }) + + test('journey: a path containing an apostrophe or a space is rejected', () => { + expect( + sanitizeProofIntent({ kind: 'journey', reason: 'r', journey: "it's" })?.journey, + ).toBeUndefined() + expect( + sanitizeProofIntent({ kind: 'journey', reason: 'r', journey: 'a b' })?.journey, + ).toBeUndefined() + }) + + test('journey: over the max path length is rejected outright', () => { + const tooLong = 'a'.repeat(PROOF_INTENT_PATH_MAX + 1) + expect( + sanitizeProofIntent({ kind: 'journey', reason: 'r', journey: tooLong })?.journey, + ).toBeUndefined() + }) + + test('hostile input never throws', () => { + const hostile = { + kind: 'screenshot', + reason: { nested: true }, + pages: ['//evil.com', '../x', "it's", null, 42], + journey: 42, + } + expect(() => sanitizeProofIntent(hostile)).not.toThrow() + expect(sanitizeProofIntent(hostile)).toBeNull() + }) +}) + +describe('sanitizeProofReview', () => { + test('non-object input never throws and returns null', () => { + for (const raw of [null, undefined, 42, 'x', [], true]) { + expect(() => sanitizeProofReview(raw)).not.toThrow() + expect(sanitizeProofReview(raw)).toBeNull() + } + }) + + test('an expected kind outside the closed enum is refused', () => { + expect(sanitizeProofReview({ expected: 'bogus', coherent: true, reason: '' })).toBeNull() + }) + + test('a non-boolean coherent is refused', () => { + for (const bad of ['true', 1, null, undefined]) { + expect(sanitizeProofReview({ expected: 'none', coherent: bad, reason: '' })).toBeNull() + } + }) + + test('an empty reason is allowed', () => { + const out = sanitizeProofReview({ expected: 'none', coherent: true, reason: '' }) + expect(out).toEqual({ expected: 'none', coherent: true, reason: '' }) + }) + + test('reason: over-long is truncated to the published bound', () => { + const out = sanitizeProofReview({ + expected: 'journey', + coherent: false, + reason: 'x'.repeat(PROOF_INTENT_REASON_MAX + 10), + }) + expect(out?.reason).toHaveLength(PROOF_INTENT_REASON_MAX) + }) + + test('a full, valid review round-trips', () => { + const out = sanitizeProofReview({ + expected: 'screenshot', + coherent: false, + reason: 'no proof was actually taken', + }) + expect(out).toEqual({ + expected: 'screenshot', + coherent: false, + reason: 'no proof was actually taken', + }) + }) +}) diff --git a/packages/contract/src/proof-intent.ts b/packages/contract/src/proof-intent.ts new file mode 100644 index 0000000..c9384a4 --- /dev/null +++ b/packages/contract/src/proof-intent.ts @@ -0,0 +1,114 @@ +import { cutCodePoints } from './ticket.js' + +export const PROOF_INTENT_KINDS = ['none', 'screenshot', 'journey'] as const + +export type ProofIntentKind = (typeof PROOF_INTENT_KINDS)[number] + +export type ProofIntent = { + kind: ProofIntentKind + reason: string + pages?: string[] + journey?: string +} + +export type ProofReview = { + expected: ProofIntentKind + coherent: boolean + reason: string +} + +export const PROOF_INTENT_REASON_MAX = 500 +export const PROOF_INTENT_PAGES_MAX = 8 +export const PROOF_INTENT_PATH_MAX = 200 + +const PROOF_INTENT_KIND_SET: ReadonlySet = new Set(PROOF_INTENT_KINDS) + +function isValidPagePath(raw: unknown): raw is string { + if (typeof raw !== 'string') { + return false + } + const length = [...raw].length + if (length === 0 || length > PROOF_INTENT_PATH_MAX) { + return false + } + if (!raw.startsWith('/') || raw.startsWith('//')) { + return false + } + return !raw.includes('..') && !raw.includes("'") && !raw.includes(' ') +} + +function isValidJourneyPath(raw: unknown): raw is string { + if (typeof raw !== 'string') { + return false + } + const length = [...raw].length + if (length === 0 || length > PROOF_INTENT_PATH_MAX) { + return false + } + if (raw.startsWith('/')) { + return false + } + return !raw.includes('..') && !raw.includes("'") && !raw.includes(' ') +} + +function sanitizeProofIntentReason(raw: unknown): string { + const trimmed = typeof raw === 'string' ? raw.trim() : '' + return trimmed ? cutCodePoints(trimmed, PROOF_INTENT_REASON_MAX).trim() : '' +} + +function sanitizeProofIntentPages(raw: unknown): string[] | undefined { + if (!Array.isArray(raw)) { + return undefined + } + const pages: string[] = [] + for (const item of raw) { + if (pages.length >= PROOF_INTENT_PAGES_MAX) { + break + } + if (isValidPagePath(item)) { + pages.push(item) + } + } + return pages.length > 0 ? pages : undefined +} + +export function sanitizeProofIntent(raw: unknown): ProofIntent | null { + if (!raw || typeof raw !== 'object') { + return null + } + const r = raw as Record + if (!PROOF_INTENT_KIND_SET.has(r.kind as ProofIntentKind)) { + return null + } + const kind = r.kind as ProofIntentKind + const reason = sanitizeProofIntentReason(r.reason) + if (!reason) { + return null + } + const pages = kind === 'screenshot' ? sanitizeProofIntentPages(r.pages) : undefined + const journey = kind === 'journey' && isValidJourneyPath(r.journey) ? r.journey : undefined + return { + kind, + reason, + ...(pages !== undefined ? { pages } : {}), + ...(journey !== undefined ? { journey } : {}), + } +} + +export function sanitizeProofReview(raw: unknown): ProofReview | null { + if (!raw || typeof raw !== 'object') { + return null + } + const r = raw as Record + if (!PROOF_INTENT_KIND_SET.has(r.expected as ProofIntentKind)) { + return null + } + if (typeof r.coherent !== 'boolean') { + return null + } + return { + expected: r.expected as ProofIntentKind, + coherent: r.coherent, + reason: sanitizeProofIntentReason(r.reason), + } +} diff --git a/packages/contract/src/tasks.test.ts b/packages/contract/src/tasks.test.ts index fa05d93..dbe9357 100644 --- a/packages/contract/src/tasks.test.ts +++ b/packages/contract/src/tasks.test.ts @@ -11,6 +11,7 @@ import { sanitizeTaskEvent, sanitizeTaskRecord, sanitizeTaskVerification, + TASK_ACTIVITY_PHASES, TASK_AGENT_MAX, TASK_CHECK_COMMAND_MAX, TASK_CHECK_TAIL_MAX, @@ -351,6 +352,38 @@ describe('sanitizeTaskRecord', () => { ).toBe(false) }) + test('activity: optional, whitelisted phase + bounded since, unusable dropped', () => { + expect(sanitizeTaskRecord(validRecord) && 'activity' in sanitizeTaskRecord(validRecord)!).toBe( + false, + ) + for (const phase of TASK_ACTIVITY_PHASES) { + const activity = { phase, since: '2026-08-14T09:00:00.000Z' } + expect(sanitizeTaskRecord({ ...validRecord, activity })?.activity).toEqual(activity) + } + const unknownPhase = { activity: { phase: 'unknown', since: '2026-08-14T09:00:00.000Z' } } + expect( + sanitizeTaskRecord({ ...validRecord, ...unknownPhase }) && + 'activity' in sanitizeTaskRecord({ ...validRecord, ...unknownPhase })!, + ).toBe(false) + const missingSince = { activity: { phase: 'checks' } } + expect( + sanitizeTaskRecord({ ...validRecord, ...missingSince }) && + 'activity' in sanitizeTaskRecord({ ...validRecord, ...missingSince })!, + ).toBe(false) + const blankSince = { activity: { phase: 'checks', since: '' } } + expect( + sanitizeTaskRecord({ ...validRecord, ...blankSince }) && + 'activity' in sanitizeTaskRecord({ ...validRecord, ...blankSince })!, + ).toBe(false) + const overlongSince = { + activity: { phase: 'checks', since: '2'.repeat(TASK_TIMESTAMP_MAX + 1) }, + } + expect( + sanitizeTaskRecord({ ...validRecord, ...overlongSince }) && + 'activity' in sanitizeTaskRecord({ ...validRecord, ...overlongSince })!, + ).toBe(false) + }) + test('cost_ticks: a 0.12 record has none, on the record and on its turns', () => { // FROZEN fixture of a record as codesema 0.12 wrote it: no `cost_ticks` // key anywhere, because the cost unit did not exist yet. @@ -755,6 +788,37 @@ describe('sanitizeTaskRecord', () => { expect(r?.turns[0]?.ended_at).toBeNull() }) + test('turns: proof_intent round-trips when valid, is absent when not supplied', () => { + const withIntent = sanitizeTaskRecord({ + ...validRecord, + turns: [ + { + ...validRecord.turns[0], + proof_intent: { + kind: 'journey', + reason: 'exercises the checkout flow', + journey: 'checkout', + }, + }, + ], + }) + expect(withIntent?.turns[0]?.proof_intent).toEqual({ + kind: 'journey', + reason: 'exercises the checkout flow', + journey: 'checkout', + }) + const withoutIntent = sanitizeTaskRecord(structuredClone(validRecord)) + expect(withoutIntent?.turns[0] && 'proof_intent' in withoutIntent.turns[0]).toBe(false) + }) + + test('turns: an unreadable proof_intent drops the key rather than storing junk', () => { + const r = sanitizeTaskRecord({ + ...validRecord, + turns: [{ ...validRecord.turns[0], proof_intent: { kind: 'bogus', reason: 'r' } }], + }) + expect(r?.turns[0] && 'proof_intent' in r.turns[0]).toBe(false) + }) + test('turns are capped', () => { const turns = Array.from({ length: TASK_TURNS_MAX + 10 }, () => ({ prompt: 'go' })) expect(sanitizeTaskRecord({ ...validRecord, turns })?.turns.length).toBe(TASK_TURNS_MAX) @@ -1212,6 +1276,7 @@ describe('sanitizeTaskEvent', () => { 'issue', 'criteria', 'post_merge_checks', + 'proof', ] as const for (const type of types) { expect(sanitizeTaskEvent({ ...validEvent, type })?.type).toBe(type) diff --git a/packages/contract/src/tasks.ts b/packages/contract/src/tasks.ts index 9e97324..34a29de 100644 --- a/packages/contract/src/tasks.ts +++ b/packages/contract/src/tasks.ts @@ -8,6 +8,7 @@ // counterpart is the locally spelled TASK_HUB_TICKET_STATUSES below, locked // to arm.ts's own set by a cross-module test. import type { ArmTicketStatus } from './arm.js' +import { sanitizeProofIntent, type ProofIntent } from './proof-intent.js' import { sanitizeReasonCode, sanitizeTaskReason, @@ -116,6 +117,7 @@ export type TaskTurn = { * describes nothing. */ cost_basis?: CostBasis + proof_intent?: ProofIntent } export type TaskEventType = @@ -250,6 +252,7 @@ export type TaskEventType = * fire-and-forget, after the merge step itself is already settled. */ | 'post_merge_checks' + | 'proof' /** * How a task's agent turns are contained. @@ -395,6 +398,26 @@ export function isActiveTaskStatus(status: TaskStatus): boolean { */ export type CycleStep = 'ship' | 'merge' +/** + * The closed set of `TaskActivity.phase` values: what a running task's agent + * is doing right now, one level more granular than `TaskStatus` alone can + * say. Published as an array (not just a type) so a consumer outside this + * module can validate a phase without hand-copying it. + */ +export const TASK_ACTIVITY_PHASES = ['checks', 'verification', 'proof', 'review', 'recap'] as const + +export type TaskActivityPhase = (typeof TASK_ACTIVITY_PHASES)[number] + +/** + * What a task's agent is doing right now, and since when (ISO-8601). Purely + * informational: nothing downstream branches on it the way `TaskStatus` or + * `checks_status` do, it only narrates the current status for a reader. + */ +export type TaskActivity = { + phase: TaskActivityPhase + since: string +} + export type TaskRecord = { version: 1 /** 12 lowercase hex chars, doubles as the on-disk directory name. */ @@ -529,6 +552,13 @@ export type TaskRecord = { * rather than treated as a verdict. */ checks_status?: Exclude + /** + * What this task's agent is doing right now, when it is running, and since + * when. OPTIONAL, and absence is the honest default: a record written + * before this field existed, and a task between phases (or not running), + * report nothing rather than a stale or guessed phase. + */ + activity?: TaskActivity /** * Last liveness beat of this task's agent (ISO-8601), written by the * semantic watchdog's heartbeat. It is what lets a reader tell a task that @@ -738,6 +768,7 @@ const TASK_EVENT_TYPES: ReadonlySet = new Set([ 'criteria', 'merge', 'post_merge_checks', + 'proof', ]) const TASK_ISOLATIONS: ReadonlySet = new Set(['container', 'policy', 'microvm']) @@ -1024,6 +1055,7 @@ function sanitizeTaskTurn(raw: unknown): TaskTurn | null { return null } const cost = costPair(t.cost_ticks, t.cost_basis) + const proofIntent = sanitizeProofIntent(t.proof_intent) return { prompt, response: @@ -1040,6 +1072,7 @@ function sanitizeTaskTurn(raw: unknown): TaskTurn | null { // are one fact and travel as one (see costPair) — never one without the // other, in either direction. ...cost, + ...(proofIntent !== null ? { proof_intent: proofIntent } : {}), } } @@ -1085,6 +1118,28 @@ function sanitizeHubTicket(raw: unknown): { id: string; title: string; url?: str } } +const TASK_ACTIVITY_PHASE_SET: ReadonlySet = new Set(TASK_ACTIVITY_PHASES) + +/** + * Whitelist, never throw: an unknown phase, or a `since` that is not a + * non-blank string within the same bound as every other stored timestamp, + * drops the WHOLE field, same doctrine as `sanitizeHubTicket` above, a + * phase with no honest timestamp is worse than reporting no activity at all. + */ +function sanitizeTaskActivity(raw: unknown): TaskActivity | undefined { + if (!raw || typeof raw !== 'object') { + return undefined + } + const r = raw as Record + if (!TASK_ACTIVITY_PHASE_SET.has(r.phase as TaskActivityPhase)) { + return undefined + } + if (typeof r.since !== 'string' || !r.since || r.since.length > TASK_TIMESTAMP_MAX) { + return undefined + } + return { phase: r.phase as TaskActivityPhase, since: r.since } +} + /** * Revalidates a TaskRecord read back from disk. Returns null when the input * has no usable identity (missing or malformed id); every other field is @@ -1130,6 +1185,7 @@ export function sanitizeTaskRecord(raw: unknown): TaskRecord | null { const criteria = sanitizeAcceptanceCriteria(r.criteria) // Legacy on-disk tasks.json may still carry `brain_ticket`; `hub_ticket` wins when both are present. const hubTicket = sanitizeHubTicket(r.hub_ticket !== undefined ? r.hub_ticket : r.brain_ticket) + const activity = sanitizeTaskActivity(r.activity) return { version: 1, id, @@ -1194,6 +1250,10 @@ export function sanitizeTaskRecord(raw: unknown): TaskRecord | null { TASK_RECORD_CHECKS_STATUSES.has(r.checks_status as Exclude) ? { checks_status: r.checks_status as Exclude } : {}), + // Optional and whitelisted, same doctrine as `checks_status`: a record + // written before this field existed keeps none, and an unknown phase or + // an unusable `since` drops the whole field rather than inventing one. + ...(activity ? { activity } : {}), // Same doctrine: optional, whitelisted to a plain bounded string, dropped // entirely when it is not one. A missing beat means "we know nothing", // never "the agent is dead". diff --git a/packages/web/src/App.vue b/packages/web/src/App.vue index 637b93e..5233cc2 100644 --- a/packages/web/src/App.vue +++ b/packages/web/src/App.vue @@ -1,9 +1,11 @@