diff --git a/drivers/stream-sandbox.ts b/drivers/stream-sandbox.ts index 31ce654..b62942a 100644 --- a/drivers/stream-sandbox.ts +++ b/drivers/stream-sandbox.ts @@ -134,6 +134,11 @@ export interface StreamHealth { costUsd: number | null /** Tokens the agent says it has spent, or null when it did not say. */ tokens: number | null + /** Input and output tokens apart, and turns taken. A ratio is diagnostic: + * 1.8M in against 72 out is an agent reading, not one writing. */ + inputTokens: number | null + outputTokens: number | null + turns: number | null /** * Which credential path the agent used, as its launcher reported it. * @@ -408,16 +413,30 @@ export function createStreamSandboxDriver(options: StreamSandboxDriverOptions): * Written when the agent exits, so it is absent for a cell read while the * agent still runs. Absence is reported as absence. */ - function reportedMeter(): { costUsd: number | null; tokens: number | null; authMode: 'api-key' | 'oauth' | null } { + function reportedMeter(): { + costUsd: number | null + tokens: number | null + inputTokens: number | null + outputTokens: number | null + turns: number | null + authMode: 'api-key' | 'oauth' | null + } { const nonNegative = (value: unknown): number | null => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null try { const parsed: unknown = JSON.parse(readFileSync(join(dir, 'agent-cost.json'), 'utf8')) - const meter = parsed as { usd?: unknown; tokens?: unknown; authMode?: unknown } + const meter = parsed as Record const mode = meter.authMode === 'api-key' || meter.authMode === 'oauth' ? meter.authMode : null - return { costUsd: nonNegative(meter.usd), tokens: nonNegative(meter.tokens), authMode: mode } + return { + costUsd: nonNegative(meter.usd), + tokens: nonNegative(meter.tokens), + inputTokens: nonNegative(meter.inputTokens), + outputTokens: nonNegative(meter.outputTokens), + turns: nonNegative(meter.turns), + authMode: mode, + } } catch { - return { costUsd: null, tokens: null, authMode: null } + return { costUsd: null, tokens: null, inputTokens: null, outputTokens: null, turns: null, authMode: null } } } diff --git a/examples/arena/airstriker.matrix b/examples/arena/airstriker.matrix new file mode 100644 index 0000000..f3f6a65 --- /dev/null +++ b/examples/arena/airstriker.matrix @@ -0,0 +1,17 @@ +# airstriker on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.sky adapter=stable-retro target=Airstriker-Genesis +objective.score goal=maximize:score horizon=2000 budgetUsd=4 +protocol.det frameskip=4 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/arena/breakout.matrix b/examples/arena/breakout.matrix new file mode 100644 index 0000000..57fc716 --- /dev/null +++ b/examples/arena/breakout.matrix @@ -0,0 +1,17 @@ +# breakout on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.brick adapter=ale target=breakout +objective.score goal=maximize:score horizon=3000 budgetUsd=4 +protocol.det frameskip=4 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/arena/cartpole.matrix b/examples/arena/cartpole.matrix new file mode 100644 index 0000000..10c44f4 --- /dev/null +++ b/examples/arena/cartpole.matrix @@ -0,0 +1,17 @@ +# cartpole on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.pole adapter=gymnasium target=CartPole-v1 +objective.steps goal=maximize:steps horizon=500 budgetUsd=4 +protocol.det frameskip=1 sticky=0 seeds=1 seed0=7 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/arena/frozenlake.matrix b/examples/arena/frozenlake.matrix new file mode 100644 index 0000000..5b2365d --- /dev/null +++ b/examples/arena/frozenlake.matrix @@ -0,0 +1,17 @@ +# frozenlake on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.ice adapter=gymnasium target=FrozenLake-v1 +objective.goal goal=maximize:steps horizon=200 budgetUsd=4 +protocol.det frameskip=1 sticky=0 seeds=1 seed0=3 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/arena/puzzle.matrix b/examples/arena/puzzle.matrix new file mode 100644 index 0000000..de285be --- /dev/null +++ b/examples/arena/puzzle.matrix @@ -0,0 +1,17 @@ +# puzzle on the arena board. The clock and seed are the ones this game's packaged +# reference was recorded at; any other value makes its contract underivable. +# +# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a +# better player, which is a different question from which model is stronger and +# the one an efficiency study actually needs. +profile.opus-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=3 transport=persistent +profile.opus-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=8 transport=persistent +profile.sonnet-3 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent +profile.sonnet-8 harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent + +game.puzzle adapter=native-2048 target=2048 +objective.score goal=maximize:score horizon=20000 budgetUsd=4 +protocol.det frameskip=1 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/airstriker.matrix b/examples/board/airstriker.matrix new file mode 100644 index 0000000..b3782d9 --- /dev/null +++ b/examples/board/airstriker.matrix @@ -0,0 +1,11 @@ +# airstriker: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.sky adapter=stable-retro target=Airstriker-Genesis +objective.score goal=maximize:score horizon=300 budgetUsd=1 +protocol.det frameskip=4 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/breakout.matrix b/examples/board/breakout.matrix new file mode 100644 index 0000000..d7e4bb2 --- /dev/null +++ b/examples/board/breakout.matrix @@ -0,0 +1,11 @@ +# breakout: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.brick adapter=ale target=breakout +objective.score goal=maximize:score horizon=300 budgetUsd=1 +protocol.det frameskip=4 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/cartpole.matrix b/examples/board/cartpole.matrix new file mode 100644 index 0000000..a1714b7 --- /dev/null +++ b/examples/board/cartpole.matrix @@ -0,0 +1,11 @@ +# cartpole: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.pole adapter=gymnasium target=CartPole-v1 +objective.steps goal=maximize:steps horizon=200 budgetUsd=1 +protocol.det frameskip=1 sticky=0 seeds=1 seed0=7 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/frozenlake.matrix b/examples/board/frozenlake.matrix new file mode 100644 index 0000000..fb01bb2 --- /dev/null +++ b/examples/board/frozenlake.matrix @@ -0,0 +1,11 @@ +# frozenlake: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.ice adapter=gymnasium target=FrozenLake-v1 +objective.goal goal=maximize:steps horizon=100 budgetUsd=1 +protocol.det frameskip=1 sticky=0 seeds=1 seed0=3 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/board/puzzle.matrix b/examples/board/puzzle.matrix new file mode 100644 index 0000000..3bf000a --- /dev/null +++ b/examples/board/puzzle.matrix @@ -0,0 +1,11 @@ +# puzzle: one game of the board. Its clock and seed are the ones its packaged +# reference was recorded at; any other value makes the contract underivable. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.puzzle adapter=native-2048 target=2048 +objective.score goal=maximize:score horizon=300 budgetUsd=1 +protocol.det frameskip=1 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- + +reps 1 diff --git a/examples/policies/masher.mjs b/examples/policies/masher.mjs index d379ccc..8b6093d 100755 --- a/examples/policies/masher.mjs +++ b/examples/policies/masher.mjs @@ -1,5 +1,11 @@ #!/usr/bin/env node -// A trivial baseline: one word, always. It is what a real arm has to beat. +// Plays the first word the game states, forever: the weakest honest baseline. +// +// It used to emit a hardcoded lowercase word. On ALE Breakout, whose words are +// NOOP, FIRE, RIGHT and LEFT, every one was illegal and substituted, and the +// first decision spent ten seconds reaching a driver timeout. A control that +// cannot name a legal move measures the harness, not the game. +const FALLBACK = 'up' let buffer = '' process.stdin.setEncoding('utf8') process.stdin.on('data', (chunk) => { @@ -7,7 +13,16 @@ process.stdin.on('data', (chunk) => { for (;;) { const at = buffer.indexOf('\n') if (at < 0) break + const line = buffer.slice(0, at) buffer = buffer.slice(at + 1) - process.stdout.write('up\n') + let word = FALLBACK + try { + const request = JSON.parse(line) + if (Array.isArray(request.commands) && request.commands.length > 0) word = request.commands[0] + } catch { + // An unreadable request is still a request. Answering keeps the episode + // gradeable instead of stalling it at a timeout. + } + process.stdout.write(`${word}\n`) } }) diff --git a/examples/transfer-agents-2048.matrix b/examples/transfer-agents-2048.matrix new file mode 100644 index 0000000..214d92a --- /dev/null +++ b/examples/transfer-agents-2048.matrix @@ -0,0 +1,12 @@ +# Half of a two-game transfer study, played by real profiles. +# +# The cheap controls tie at 0 on Breakout, and two tied players have no order, +# so a rank correlation has nothing to correlate. These build a player instead. +profile.opus harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=6 transport=persistent +profile.sonnet harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=6 transport=persistent + +game.puzzle adapter=native-2048 target=2048 +objective.score goal=maximize:score horizon=20000 budgetUsd=4 +protocol.det frameskip=1 sticky=0 seeds=1 queue=8 empty=noop pace=0 +sensor.ascii pixels=off channels=- +reps 2 diff --git a/examples/transfer-agents-breakout.matrix b/examples/transfer-agents-breakout.matrix new file mode 100644 index 0000000..17bea58 --- /dev/null +++ b/examples/transfer-agents-breakout.matrix @@ -0,0 +1,10 @@ +# The other half. frameskip 4 is the clock Breakout's packaged reference was +# recorded at; any other value makes its contract underivable. +profile.opus harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5 buildMin=6 transport=persistent +profile.sonnet harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=6 transport=persistent + +game.brick adapter=ale target=breakout +objective.score goal=maximize:score horizon=3000 budgetUsd=4 +protocol.det frameskip=4 sticky=0 seeds=1 queue=8 empty=noop pace=0 +sensor.ascii pixels=off channels=- +reps 2 diff --git a/matrix-run.ts b/matrix-run.ts index c62b6a6..dd69d5b 100644 --- a/matrix-run.ts +++ b/matrix-run.ts @@ -156,6 +156,17 @@ export interface CellResult { */ authMode: 'api-key' | 'oauth' | null } | null + /** + * Everything the run measured that a single number cannot carry. + * + * A row answers "how did it do". This answers "what happened", which is the + * question every autopsy in this work has actually needed. The data was + * always collected and then discarded at this boundary: `latencyMs` is a full + * per-decision array, the evidence watcher holds every channel at every + * decision, and driver health is a structured record that was being flattened + * into one sentence of prose. + */ + telemetry: CellTelemetry | null scoreField: string | null /** Which way is better on that channel. A minimize goal ranks inverted. */ scoreDirection: 'maximize' | 'minimize' | null @@ -423,6 +434,9 @@ export function blockedResult(cell: MatrixCell, reason: BlockedReason, detail: s tokens: null, usd: 0, build: null, + // A cell that never played measured nothing. Null, not an empty spread: an + // absent distribution and one with no spread are different facts. + telemetry: null, // A blocked cell still states what it WOULD have scored on, so a reader can // see that an excluded row and a played row were aimed at the same channel. scoreField: null, @@ -498,6 +512,8 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P // practice game nobody scores, and the program it leaves behind is what the // scored episode actually runs. The agent is not running by then. let authored: CellResult['build'] = null + /** Build-phase token split, kept apart from the row's headline totals. */ + let authoredDetail: { inputTokens: number | null; outputTokens: number | null; turns: number | null } | null = null let policyCommand: string | null = null if (cell.profile.author !== undefined) { const attempt = await authorPolicy(cell, options, now) @@ -508,6 +524,7 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P policy: attempt.policy, authMode: attempt.authMode, } + authoredDetail = attempt.detailTokens if (attempt.policy === null) { // Blocked, never scored zero. A profile that built nothing did not play // badly; it produced no player, and those are different findings. @@ -553,6 +570,8 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P const watched = watchEvidence(built.game) let record: EpisodeRecord let actionsHash: string | null + /** Every word the driver played, in order, for the raw trace. */ + let actions: readonly string[] = [] try { const played = await playEpisode( watched.game, @@ -566,6 +585,7 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P ) record = played.record actionsHash = played.log.head() + actions = played.log.inputs() } catch (error) { return blockedResult(cell, 'episode-failed', (error as Error).message, now() - started) } @@ -590,6 +610,33 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P usd: meter.metered && meter.costUsd === null ? null : (meter.costUsd ?? 0) + record.spentUsd, authMode: meter.authMode, build: authored, + telemetry: { + latencyMs: spreadOf(record.latencyMs), + scoreSeries: spreadOf(channel(watched.snapshots, goal.field)), + channelsLast: watched.snapshots[watched.snapshots.length - 1] ?? {}, + inputs: { + total: record.inputStats.inputs, + distinct: Math.round(record.inputStats.uniqueRatio * record.inputStats.inputs), + uniqueRatio: record.inputStats.uniqueRatio, + }, + milestones: built.contract.milestones.map((milestone) => ({ + id: milestone.id, + verified: record.verified.includes(milestone.id), + })), + health: healthOf(driver), + build: authoredDetail, + // The evidence watcher records the state BEFORE the first input as + // snapshot 0, so decision n is snapshot n+1. Off by one here would + // attribute every outcome to the input before the one that caused it. + trace: actions.map((input: string, i: number) => ({ + turn: i + 1, + input, + latencyMs: record.latencyMs[i] ?? null, + channels: watched.snapshots[i + 1] ?? {}, + })), + budgetExhausted: record.budgetExhausted, + spentUsd: record.spentUsd, + }, cleared: record.gameOver, verified: record.verified, milestones: record.score, @@ -692,6 +739,7 @@ async function authorPolicy( minutes: number policy: string | null authMode: 'api-key' | 'oauth' | null + detailTokens: { inputTokens: number | null; outputTokens: number | null; turns: number | null } detail: string | null }> { const profile = cell.profile @@ -717,6 +765,7 @@ async function authorPolicy( minutes: 0, policy: null, authMode: null, + detailTokens: { inputTokens: null, outputTokens: null, turns: null }, detail: `no practice game: ${(error as Error).message}`, } } @@ -787,16 +836,130 @@ async function authorPolicy( ? 'the agent left a policy that is not executable' : 'the agent left no policy' } + const full = healthOf(driver) ?? {} + const num = (v: unknown): number | null => (typeof v === 'number' && Number.isFinite(v) ? v : null) return { usd: meter.costUsd, tokens: meter.tokens, authMode: meter.authMode, + detailTokens: { + inputTokens: num(full.inputTokens), + outputTokens: num(full.outputTokens), + turns: num(full.turns), + }, minutes: (now() - started) / 60_000, policy, detail, } } +/** One decision, as it happened. */ +export interface DecisionTrace { + turn: number + /** The word the driver emitted and the game accepted. */ + input: string + /** Wall clock this decision took, in milliseconds. */ + latencyMs: number | null + /** Every evidence channel after the input was applied. */ + channels: Record +} + +/** A distribution, reported so a reader can see shape rather than a mean. */ +export interface Spread { + n: number + min: number + p50: number + p90: number + p99: number + max: number + mean: number + /** Equal-width buckets over [min, max], for a histogram. Empty when n < 2. */ + histogram: { from: number; to: number; count: number }[] +} + +/** What one cell measured beyond its headline number. */ +export interface CellTelemetry { + /** Wall clock per decision, in milliseconds. */ + latencyMs: Spread | null + /** The scored channel at every decision, so a trajectory is visible. */ + scoreSeries: Spread | null + /** Every evidence channel this game published, and its last value. */ + channelsLast: Record + /** Inputs the driver emitted, and how varied they were. */ + inputs: { total: number; distinct: number; uniqueRatio: number } + /** Per-milestone cost, as the contract recorded it. */ + milestones: { id: string; verified: boolean }[] + /** The transport's own record, structured rather than prose. */ + health: Record | null + /** Build-phase detail when the profile authored its player. */ + build: { inputTokens: number | null; outputTokens: number | null; turns: number | null } | null + /** + * Every decision, raw, in the order it happened. + * + * A summary answers "what shape"; this answers "what happened at decision + * 811", which is the question every autopsy in this work has ended up + * asking. Each entry carries the turn, the word played, that decision's + * latency, and every evidence channel as it stood afterwards. + * + * It is the whole episode, so a caller writing it to disk should put it + * beside the row rather than inside it. + */ + trace: DecisionTrace[] + /** + * Where the trace was written, when a runner moved it out of the row. + * + * Null while it is still inline. A reader that finds `trace` empty and this + * set knows the decisions exist rather than that there were none. + */ + tracePath?: string + budgetExhausted: boolean + spentUsd: number +} + +/** + * Summarise a series without hiding its shape. + * + * A mean alone has hidden every interesting result in this work: opus looked + * consistent to 0.2% at a censored horizon and varied 5.3x uncensored. Quantiles + * and a histogram make that visible on the row instead of in a later autopsy. + */ +export function spreadOf(values: readonly number[], buckets = 12): Spread | null { + if (values.length === 0) return null + const sorted = [...values].sort((a, b) => a - b) + const at = (q: number): number => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]! + const min = sorted[0]! + const max = sorted[sorted.length - 1]! + const histogram: { from: number; to: number; count: number }[] = [] + if (sorted.length > 1 && max > min) { + const width = (max - min) / buckets + for (let i = 0; i < buckets; i += 1) { + const from = min + i * width + const to = i === buckets - 1 ? max : from + width + histogram.push({ + from, + to, + count: sorted.filter((v) => v >= from && (i === buckets - 1 ? v <= to : v < to)).length, + }) + } + } + return { + n: sorted.length, + min, + p50: at(0.5), + p90: at(0.9), + p99: at(0.99), + max, + mean: sorted.reduce((a, b) => a + b, 0) / sorted.length, + histogram, + } +} + +/** Read a driver's own health record, whatever transport it is. */ +function healthOf(driver: AgentDriver): Record | null { + const reporter = driver as { health?: () => Record } + return typeof reporter.health === 'function' ? reporter.health() : null +} + function meterOf(driver: AgentDriver): { metered: boolean costUsd: number | null diff --git a/matrix.mts b/matrix.mts index 589a25d..3036615 100644 --- a/matrix.mts +++ b/matrix.mts @@ -11,8 +11,8 @@ * Env knobs: * PLAYPROOF_PYTHON interpreter that runs an emulator worker (default python3) */ -import { mkdir, readFile, writeFile } from 'node:fs/promises' -import { dirname } from 'node:path' +import { copyFile, mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' import { cellName, enumerateCells, parseMatrix } from './matrix' import { assertJoinable, blockedResult, effectiveArms, generalization, runCell, type CellResult } from './matrix-run' @@ -116,6 +116,41 @@ for (const [index, cell] of cells.entries()) { } finally { clearInterval(heartbeat) } + // Keep what the cell produced. The authoring sandbox is a temp directory + // that is reclaimed, so the program an agent wrote and the transcript of how + // it wrote it are lost the moment the run ends. Those are the two artifacts + // an autopsy of a build actually needs. + if (row.build?.policy != null && outPath !== undefined) { + const keep = join(dirname(outPath), 'artifacts', cellName(cell).replace(/[^\w.-]+/gu, '_')) + try { + await mkdir(keep, { recursive: true }) + const from = dirname(row.build.policy) + for (const name of await readdir(from)) { + // Observation files are the game talking, not the agent working, and a + // practice run writes thousands of them. + if (name === 'observations' || name === 'latest.json') continue + const source = join(from, name) + if ((await stat(source)).isDirectory()) continue + await copyFile(source, join(keep, name)) + } + } catch (error) { + console.error(`[${index + 1}/${cells.length}] could not keep artifacts: ${(error as Error).message}`) + } + } + // The raw trace is the whole episode. It goes beside the artifact as JSONL, + // one decision per line, so the row stays readable and nothing is lost: a + // 20,000-decision run would otherwise make the study file unopenable. + if (row.telemetry !== null && row.telemetry.trace.length > 0 && outPath !== undefined) { + const slug = cellName(cell).replace(/[^\w.-]+/gu, '_') + const tracePath = join(dirname(outPath), 'traces', `${slug}.jsonl`) + try { + await mkdir(dirname(tracePath), { recursive: true }) + await writeFile(tracePath, row.telemetry.trace.map((d) => JSON.stringify(d)).join('\n') + '\n') + row.telemetry = { ...row.telemetry, trace: [], tracePath } + } catch (error) { + console.error(`[${index + 1}/${cells.length}] could not write the trace: ${(error as Error).message}`) + } + } rows.push(row) const headline = row.status === 'played' ? `score=${row.score ?? '-'} deaths=${row.deaths ?? '-'} decisions=${row.decisions}`