From a3c0dd4ba3c82e43630d9170a048d93ee49823df Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 29 Aug 2026 02:21:01 -0700 Subject: [PATCH] feat(matrix): keep everything a run measured, not only its headline A row answered "how did it do". Every autopsy in this work has needed "what happened", and the data to answer it was already collected and then discarded at the row boundary: `latencyMs` was a full per-decision array, the evidence watcher held every channel at every decision, and driver health was a structured record being flattened into one sentence of prose. THE RAW TRACE. Every decision, in order, carrying the turn, the word played, that decision's latency, and every evidence channel as it stood afterwards. On ALE Breakout that is the ball and paddle RAM addresses, so a reader can watch the ball spawn at the exact decision that pressed FIRE. It is written beside the study as JSONL, one decision per line, because a 20,000-decision episode inside the row would make the study file unopenable. DISTRIBUTIONS, NOT MEANS. `spreadOf` reports n, min, p50, p90, p99, max, mean and a twelve-bucket histogram. Measured on one cell: 125 decisions at about 0 ms and one at 329 ms, a mean of 3.20 that describes neither. A mean has hidden every interesting result here, including a profile that looked consistent to 0.2% at a censored horizon and varied 5.3x uncensored. WHAT ELSE THE ROW NOW CARRIES. The scored channel as a series, every channel's last value, input totals against distinct inputs, per-milestone verification, the transport's health as data, budget exhaustion, and the build phase's input and output tokens and turns apart. That ratio is diagnostic: 1.8M in against 72 out is an agent reading, not one writing. ARTIFACTS SURVIVE. The authoring sandbox is a temp directory that is reclaimed, so the program an agent wrote and the transcript of it writing were lost when the run ended. Both are copied beside the study now. The trace found a defect immediately. The bundled masher emitted a hardcoded lowercase word, so on Breakout every input was illegal and the first decision spent ten seconds reaching a driver timeout. It reads its vocabulary from the request now, as the cycler already did. --- drivers/stream-sandbox.ts | 27 +++- examples/arena/airstriker.matrix | 17 +++ examples/arena/breakout.matrix | 17 +++ examples/arena/cartpole.matrix | 17 +++ examples/arena/frozenlake.matrix | 17 +++ examples/arena/puzzle.matrix | 17 +++ examples/board/airstriker.matrix | 11 ++ examples/board/breakout.matrix | 11 ++ examples/board/cartpole.matrix | 11 ++ examples/board/frozenlake.matrix | 11 ++ examples/board/puzzle.matrix | 11 ++ examples/policies/masher.mjs | 19 ++- examples/transfer-agents-2048.matrix | 12 ++ examples/transfer-agents-breakout.matrix | 10 ++ matrix-run.ts | 163 +++++++++++++++++++++++ matrix.mts | 39 +++++- 16 files changed, 402 insertions(+), 8 deletions(-) create mode 100644 examples/arena/airstriker.matrix create mode 100644 examples/arena/breakout.matrix create mode 100644 examples/arena/cartpole.matrix create mode 100644 examples/arena/frozenlake.matrix create mode 100644 examples/arena/puzzle.matrix create mode 100644 examples/board/airstriker.matrix create mode 100644 examples/board/breakout.matrix create mode 100644 examples/board/cartpole.matrix create mode 100644 examples/board/frozenlake.matrix create mode 100644 examples/board/puzzle.matrix create mode 100644 examples/transfer-agents-2048.matrix create mode 100644 examples/transfer-agents-breakout.matrix diff --git a/drivers/stream-sandbox.ts b/drivers/stream-sandbox.ts index 31ce654..b62942a 100644 --- a/drivers/stream-sandbox.ts +++ b/drivers/stream-sandbox.ts @@ -134,6 +134,11 @@ export interface StreamHealth { costUsd: number | null /** Tokens the agent says it has spent, or null when it did not say. */ tokens: number | null + /** Input and output tokens apart, and turns taken. A ratio is diagnostic: + * 1.8M in against 72 out is an agent reading, not one writing. */ + inputTokens: number | null + outputTokens: number | null + turns: number | null /** * Which credential path the agent used, as its launcher reported it. * @@ -408,16 +413,30 @@ export function createStreamSandboxDriver(options: StreamSandboxDriverOptions): * Written when the agent exits, so it is absent for a cell read while the * agent still runs. Absence is reported as absence. */ - function reportedMeter(): { costUsd: number | null; tokens: number | null; authMode: 'api-key' | 'oauth' | null } { + function reportedMeter(): { + costUsd: number | null + tokens: number | null + inputTokens: number | null + outputTokens: number | null + turns: number | null + authMode: 'api-key' | 'oauth' | null + } { const nonNegative = (value: unknown): number | null => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null try { const parsed: unknown = JSON.parse(readFileSync(join(dir, 'agent-cost.json'), 'utf8')) - const meter = parsed as { usd?: unknown; tokens?: unknown; authMode?: unknown } + const meter = parsed as Record const mode = meter.authMode === 'api-key' || meter.authMode === 'oauth' ? meter.authMode : null - return { costUsd: nonNegative(meter.usd), tokens: nonNegative(meter.tokens), authMode: mode } + return { + costUsd: nonNegative(meter.usd), + tokens: nonNegative(meter.tokens), + inputTokens: nonNegative(meter.inputTokens), + outputTokens: nonNegative(meter.outputTokens), + turns: nonNegative(meter.turns), + authMode: mode, + } } catch { - return { costUsd: null, tokens: null, authMode: null } + return { costUsd: null, tokens: null, inputTokens: null, outputTokens: null, turns: null, authMode: null } } } diff --git a/examples/arena/airstriker.matrix b/examples/arena/airstriker.matrix new file mode 100644 index 0000000..f3f6a65 --- /dev/null +++ b/examples/arena/airstriker.matrix @@ -0,0 +1,17 @@ +# airstriker on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.sky adapter=stable-retro target=Airstriker-Genesis +objective.score goal=maximize:score horizon=2000 budgetUsd=4 +protocol.det frameskip=4 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/arena/breakout.matrix b/examples/arena/breakout.matrix new file mode 100644 index 0000000..57fc716 --- /dev/null +++ b/examples/arena/breakout.matrix @@ -0,0 +1,17 @@ +# breakout on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.brick adapter=ale target=breakout +objective.score goal=maximize:score horizon=3000 budgetUsd=4 +protocol.det frameskip=4 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/arena/cartpole.matrix b/examples/arena/cartpole.matrix new file mode 100644 index 0000000..10c44f4 --- /dev/null +++ b/examples/arena/cartpole.matrix @@ -0,0 +1,17 @@ +# cartpole on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.pole adapter=gymnasium target=CartPole-v1 +objective.steps goal=maximize:steps horizon=500 budgetUsd=4 +protocol.det frameskip=1 sticky=0 seeds=1 seed0=7 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/arena/frozenlake.matrix b/examples/arena/frozenlake.matrix new file mode 100644 index 0000000..5b2365d --- /dev/null +++ b/examples/arena/frozenlake.matrix @@ -0,0 +1,17 @@ +# frozenlake on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.ice adapter=gymnasium target=FrozenLake-v1 +objective.goal goal=maximize:steps horizon=200 budgetUsd=4 +protocol.det frameskip=1 sticky=0 seeds=1 seed0=3 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/arena/puzzle.matrix b/examples/arena/puzzle.matrix new file mode 100644 index 0000000..de285be --- /dev/null +++ b/examples/arena/puzzle.matrix @@ -0,0 +1,17 @@ +# puzzle on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.puzzle adapter=native-2048 target=2048 +objective.score goal=maximize:score horizon=20000 budgetUsd=4 +protocol.det frameskip=1 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/airstriker.matrix b/examples/board/airstriker.matrix new file mode 100644 index 0000000..b3782d9 --- /dev/null +++ b/examples/board/airstriker.matrix @@ -0,0 +1,11 @@ +# airstriker: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.sky adapter=stable-retro target=Airstriker-Genesis +objective.score goal=maximize:score horizon=300 budgetUsd=1 +protocol.det frameskip=4 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/breakout.matrix b/examples/board/breakout.matrix new file mode 100644 index 0000000..d7e4bb2 --- /dev/null +++ b/examples/board/breakout.matrix @@ -0,0 +1,11 @@ +# breakout: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.brick adapter=ale target=breakout +objective.score goal=maximize:score horizon=300 budgetUsd=1 +protocol.det frameskip=4 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/cartpole.matrix b/examples/board/cartpole.matrix new file mode 100644 index 0000000..a1714b7 --- /dev/null +++ b/examples/board/cartpole.matrix @@ -0,0 +1,11 @@ +# cartpole: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.pole adapter=gymnasium target=CartPole-v1 +objective.steps goal=maximize:steps horizon=200 budgetUsd=1 +protocol.det frameskip=1 sticky=0 seeds=1 seed0=7 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/frozenlake.matrix b/examples/board/frozenlake.matrix new file mode 100644 index 0000000..fb01bb2 --- /dev/null +++ b/examples/board/frozenlake.matrix @@ -0,0 +1,11 @@ +# frozenlake: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.ice adapter=gymnasium target=FrozenLake-v1 +objective.goal goal=maximize:steps horizon=100 budgetUsd=1 +protocol.det frameskip=1 sticky=0 seeds=1 seed0=3 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/puzzle.matrix b/examples/board/puzzle.matrix new file mode 100644 index 0000000..3bf000a --- /dev/null +++ b/examples/board/puzzle.matrix @@ -0,0 +1,11 @@ +# puzzle: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.puzzle adapter=native-2048 target=2048 +objective.score goal=maximize:score horizon=300 budgetUsd=1 +protocol.det frameskip=1 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/policies/masher.mjs b/examples/policies/masher.mjs index d379ccc..8b6093d 100755 --- a/examples/policies/masher.mjs +++ b/examples/policies/masher.mjs @@ -1,5 +1,11 @@ #!/usr/bin/env node -// A trivial baseline: one word, always. It is what a real arm has to beat. +// Plays the first word the game states, forever: the weakest honest baseline. +// +// It used to emit a hardcoded lowercase word. On ALE Breakout, whose words are +// NOOP, FIRE, RIGHT and LEFT, every one was illegal and substituted, and the +// first decision spent ten seconds reaching a driver timeout. A control that +// cannot name a legal move measures the harness, not the game. +const FALLBACK = 'up' let buffer = '' process.stdin.setEncoding('utf8') process.stdin.on('data', (chunk) => { @@ -7,7 +13,16 @@ process.stdin.on('data', (chunk) => { for (;;) { const at = buffer.indexOf('\n') if (at < 0) break + const line = buffer.slice(0, at) buffer = buffer.slice(at + 1) - process.stdout.write('up\n') + let word = FALLBACK + try { + const request = JSON.parse(line) + if (Array.isArray(request.commands) && request.commands.length > 0) word = request.commands[0] + } catch { + // An unreadable request is still a request. Answering keeps the episode + // gradeable instead of stalling it at a timeout. + } + process.stdout.write(`${word}\n`) } }) diff --git a/examples/transfer-agents-2048.matrix b/examples/transfer-agents-2048.matrix new file mode 100644 index 0000000..214d92a --- /dev/null +++ b/examples/transfer-agents-2048.matrix @@ -0,0 +1,12 @@ +# Half of a two-game transfer study, played by real profiles. +# +# The cheap controls tie at 0 on Breakout, and two tied players have no order, +# so a rank correlation has nothing to correlate. These build a player instead. +profile.opus harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=6 transport=persistent +profile.sonnet harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=6 transport=persistent + +game.puzzle adapter=native-2048 target=2048 +objective.score goal=maximize:score horizon=20000 budgetUsd=4 +protocol.det frameskip=1 sticky=0 seeds=1 queue=8 empty=noop pace=0 +sensor.ascii pixels=off channels=- +reps 2 diff --git a/examples/transfer-agents-breakout.matrix b/examples/transfer-agents-breakout.matrix new file mode 100644 index 0000000..17bea58 --- /dev/null +++ b/examples/transfer-agents-breakout.matrix @@ -0,0 +1,10 @@ +# The other half. frameskip 4 is the clock Breakout's packaged reference was +# recorded at; any other value makes its contract underivable. +profile.opus harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=6 transport=persistent +profile.sonnet harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=6 transport=persistent + +game.brick adapter=ale target=breakout +objective.score goal=maximize:score horizon=3000 budgetUsd=4 +protocol.det frameskip=4 sticky=0 seeds=1 queue=8 empty=noop pace=0 +sensor.ascii pixels=off channels=- +reps 2 diff --git a/matrix-run.ts b/matrix-run.ts index c62b6a6..dd69d5b 100644 --- a/matrix-run.ts +++ b/matrix-run.ts @@ -156,6 +156,17 @@ export interface CellResult { */ authMode: 'api-key' | 'oauth' | null } | null + /** + * Everything the run measured that a single number cannot carry. + * + * A row answers "how did it do". This answers "what happened", which is the + * question every autopsy in this work has actually needed. The data was + * always collected and then discarded at this boundary: `latencyMs` is a full + * per-decision array, the evidence watcher holds every channel at every + * decision, and driver health is a structured record that was being flattened + * into one sentence of prose. + */ + telemetry: CellTelemetry | null scoreField: string | null /** Which way is better on that channel. A minimize goal ranks inverted. */ scoreDirection: 'maximize' | 'minimize' | null @@ -423,6 +434,9 @@ export function blockedResult(cell: MatrixCell, reason: BlockedReason, detail: s tokens: null, usd: 0, build: null, + // A cell that never played measured nothing. Null, not an empty spread: an + // absent distribution and one with no spread are different facts. + telemetry: null, // A blocked cell still states what it WOULD have scored on, so a reader can // see that an excluded row and a played row were aimed at the same channel. scoreField: null, @@ -498,6 +512,8 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P // practice game nobody scores, and the program it leaves behind is what the // scored episode actually runs. The agent is not running by then. let authored: CellResult['build'] = null + /** Build-phase token split, kept apart from the row's headline totals. */ + let authoredDetail: { inputTokens: number | null; outputTokens: number | null; turns: number | null } | null = null let policyCommand: string | null = null if (cell.profile.author !== undefined) { const attempt = await authorPolicy(cell, options, now) @@ -508,6 +524,7 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P policy: attempt.policy, authMode: attempt.authMode, } + authoredDetail = attempt.detailTokens if (attempt.policy === null) { // Blocked, never scored zero. A profile that built nothing did not play // badly; it produced no player, and those are different findings. @@ -553,6 +570,8 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P const watched = watchEvidence(built.game) let record: EpisodeRecord let actionsHash: string | null + /** Every word the driver played, in order, for the raw trace. */ + let actions: readonly string[] = [] try { const played = await playEpisode( watched.game, @@ -566,6 +585,7 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P ) record = played.record actionsHash = played.log.head() + actions = played.log.inputs() } catch (error) { return blockedResult(cell, 'episode-failed', (error as Error).message, now() - started) } @@ -590,6 +610,33 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P usd: meter.metered && meter.costUsd === null ? null : (meter.costUsd ?? 0) + record.spentUsd, authMode: meter.authMode, build: authored, + telemetry: { + latencyMs: spreadOf(record.latencyMs), + scoreSeries: spreadOf(channel(watched.snapshots, goal.field)), + channelsLast: watched.snapshots[watched.snapshots.length - 1] ?? {}, + inputs: { + total: record.inputStats.inputs, + distinct: Math.round(record.inputStats.uniqueRatio * record.inputStats.inputs), + uniqueRatio: record.inputStats.uniqueRatio, + }, + milestones: built.contract.milestones.map((milestone) => ({ + id: milestone.id, + verified: record.verified.includes(milestone.id), + })), + health: healthOf(driver), + build: authoredDetail, + // The evidence watcher records the state BEFORE the first input as + // snapshot 0, so decision n is snapshot n+1. Off by one here would + // attribute every outcome to the input before the one that caused it. + trace: actions.map((input: string, i: number) => ({ + turn: i + 1, + input, + latencyMs: record.latencyMs[i] ?? null, + channels: watched.snapshots[i + 1] ?? {}, + })), + budgetExhausted: record.budgetExhausted, + spentUsd: record.spentUsd, + }, cleared: record.gameOver, verified: record.verified, milestones: record.score, @@ -692,6 +739,7 @@ async function authorPolicy( minutes: number policy: string | null authMode: 'api-key' | 'oauth' | null + detailTokens: { inputTokens: number | null; outputTokens: number | null; turns: number | null } detail: string | null }> { const profile = cell.profile @@ -717,6 +765,7 @@ async function authorPolicy( minutes: 0, policy: null, authMode: null, + detailTokens: { inputTokens: null, outputTokens: null, turns: null }, detail: `no practice game: ${(error as Error).message}`, } } @@ -787,16 +836,130 @@ async function authorPolicy( ? 'the agent left a policy that is not executable' : 'the agent left no policy' } + const full = healthOf(driver) ?? {} + const num = (v: unknown): number | null => (typeof v === 'number' && Number.isFinite(v) ? v : null) return { usd: meter.costUsd, tokens: meter.tokens, authMode: meter.authMode, + detailTokens: { + inputTokens: num(full.inputTokens), + outputTokens: num(full.outputTokens), + turns: num(full.turns), + }, minutes: (now() - started) / 60_000, policy, detail, } } +/** One decision, as it happened. */ +export interface DecisionTrace { + turn: number + /** The word the driver emitted and the game accepted. */ + input: string + /** Wall clock this decision took, in milliseconds. */ + latencyMs: number | null + /** Every evidence channel after the input was applied. */ + channels: Record +} + +/** A distribution, reported so a reader can see shape rather than a mean. */ +export interface Spread { + n: number + min: number + p50: number + p90: number + p99: number + max: number + mean: number + /** Equal-width buckets over [min, max], for a histogram. Empty when n < 2. */ + histogram: { from: number; to: number; count: number }[] +} + +/** What one cell measured beyond its headline number. */ +export interface CellTelemetry { + /** Wall clock per decision, in milliseconds. */ + latencyMs: Spread | null + /** The scored channel at every decision, so a trajectory is visible. */ + scoreSeries: Spread | null + /** Every evidence channel this game published, and its last value. */ + channelsLast: Record + /** Inputs the driver emitted, and how varied they were. */ + inputs: { total: number; distinct: number; uniqueRatio: number } + /** Per-milestone cost, as the contract recorded it. */ + milestones: { id: string; verified: boolean }[] + /** The transport's own record, structured rather than prose. */ + health: Record | null + /** Build-phase detail when the profile authored its player. */ + build: { inputTokens: number | null; outputTokens: number | null; turns: number | null } | null + /** + * Every decision, raw, in the order it happened. + * + * A summary answers "what shape"; this answers "what happened at decision + * 811", which is the question every autopsy in this work has ended up + * asking. Each entry carries the turn, the word played, that decision's + * latency, and every evidence channel as it stood afterwards. + * + * It is the whole episode, so a caller writing it to disk should put it + * beside the row rather than inside it. + */ + trace: DecisionTrace[] + /** + * Where the trace was written, when a runner moved it out of the row. + * + * Null while it is still inline. A reader that finds `trace` empty and this + * set knows the decisions exist rather than that there were none. + */ + tracePath?: string + budgetExhausted: boolean + spentUsd: number +} + +/** + * Summarise a series without hiding its shape. + * + * A mean alone has hidden every interesting result in this work: opus looked + * consistent to 0.2% at a censored horizon and varied 5.3x uncensored. Quantiles + * and a histogram make that visible on the row instead of in a later autopsy. + */ +export function spreadOf(values: readonly number[], buckets = 12): Spread | null { + if (values.length === 0) return null + const sorted = [...values].sort((a, b) => a - b) + const at = (q: number): number => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]! + const min = sorted[0]! + const max = sorted[sorted.length - 1]! + const histogram: { from: number; to: number; count: number }[] = [] + if (sorted.length > 1 && max > min) { + const width = (max - min) / buckets + for (let i = 0; i < buckets; i += 1) { + const from = min + i * width + const to = i === buckets - 1 ? max : from + width + histogram.push({ + from, + to, + count: sorted.filter((v) => v >= from && (i === buckets - 1 ? v <= to : v < to)).length, + }) + } + } + return { + n: sorted.length, + min, + p50: at(0.5), + p90: at(0.9), + p99: at(0.99), + max, + mean: sorted.reduce((a, b) => a + b, 0) / sorted.length, + histogram, + } +} + +/** Read a driver's own health record, whatever transport it is. */ +function healthOf(driver: AgentDriver): Record | null { + const reporter = driver as { health?: () => Record } + return typeof reporter.health === 'function' ? reporter.health() : null +} + function meterOf(driver: AgentDriver): { metered: boolean costUsd: number | null diff --git a/matrix.mts b/matrix.mts index 589a25d..3036615 100644 --- a/matrix.mts +++ b/matrix.mts @@ -11,8 +11,8 @@ * Env knobs: * PLAYPROOF_PYTHON interpreter that runs an emulator worker (default python3) */ -import { mkdir, readFile, writeFile } from 'node:fs/promises' -import { dirname } from 'node:path' +import { copyFile, mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' import { cellName, enumerateCells, parseMatrix } from './matrix' import { assertJoinable, blockedResult, effectiveArms, generalization, runCell, type CellResult } from './matrix-run' @@ -116,6 +116,41 @@ for (const [index, cell] of cells.entries()) { } finally { clearInterval(heartbeat) } + // Keep what the cell produced. The authoring sandbox is a temp directory + // that is reclaimed, so the program an agent wrote and the transcript of how + // it wrote it are lost the moment the run ends. Those are the two artifacts + // an autopsy of a build actually needs. + if (row.build?.policy != null && outPath !== undefined) { + const keep = join(dirname(outPath), 'artifacts', cellName(cell).replace(/[^\w.-]+/gu, '_')) + try { + await mkdir(keep, { recursive: true }) + const from = dirname(row.build.policy) + for (const name of await readdir(from)) { + // Observation files are the game talking, not the agent working, and a + // practice run writes thousands of them. + if (name === 'observations' || name === 'latest.json') continue + const source = join(from, name) + if ((await stat(source)).isDirectory()) continue + await copyFile(source, join(keep, name)) + } + } catch (error) { + console.error(`[${index + 1}/${cells.length}] could not keep artifacts: ${(error as Error).message}`) + } + } + // The raw trace is the whole episode. It goes beside the artifact as JSONL, + // one decision per line, so the row stays readable and nothing is lost: a + // 20,000-decision run would otherwise make the study file unopenable. + if (row.telemetry !== null && row.telemetry.trace.length > 0 && outPath !== undefined) { + const slug = cellName(cell).replace(/[^\w.-]+/gu, '_') + const tracePath = join(dirname(outPath), 'traces', `${slug}.jsonl`) + try { + await mkdir(dirname(tracePath), { recursive: true }) + await writeFile(tracePath, row.telemetry.trace.map((d) => JSON.stringify(d)).join('\n') + '\n') + row.telemetry = { ...row.telemetry, trace: [], tracePath } + } catch (error) { + console.error(`[${index + 1}/${cells.length}] could not write the trace: ${(error as Error).message}`) + } + } rows.push(row) const headline = row.status === 'played' ? `score=${row.score ?? '-'} deaths=${row.deaths ?? '-'} decisions=${row.decisions}`