From 41647d9da7df5160e14fb9e3262d840d67c4cf16 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 28 Aug 2026 17:46:51 -0700 Subject: [PATCH] feat(matrix): state the vocabulary, and let two games have two clocks Three faults, all found by running the first two-game study this repo has completed. A POLICY WAS NEVER TOLD WHAT WORDS THE GAME TAKES. The streaming transport states them in `brief.json`, exactly so an agent does not have to guess. The persistent and per-decision transports sent frame, history and context and never said. Measured: a control that cycles 2048's four words scored 1948 there and 0 on ALE Breakout across three replicates, with `distinctInputs=1` and no lives lost, because Breakout takes NOOP, FIRE, RIGHT and LEFT and none of the four it emitted was legal. It read as a player that did nothing rather than one that was never told the rules. Both transports now carry `commands` in the request, and the bundled control reads them instead of assuming. With the words stated it launches the ball and loses five lives in 126 decisions. `assertJoinable` REFUSED THE STUDY IT EXISTS TO SERVE. It rejected two protocols across the whole pool, and two games legitimately have two clocks: a packaged reference pins the clock it was recorded at, so Breakout is derivable only at frameskip 4 while native-2048 runs at 1. A transfer statistic ranks profiles inside each game and correlates the orders, so a raw score never crosses a game boundary. The check is now scoped per game. Two clocks inside ONE game is still refused, because those rows are averaged and the dominant term disappears into the mean. The bundled controls lived under `runs/`, which is gitignored, so a definition naming them was broken for everyone but its author. They are in `examples/` now, with the two halves of the study that found all of this. Measured end to end: 12 cells, 0 blocked, and the first cross-game statistic this repository has produced. It reports `tau=0.000` and says why that is not a finding: both controls score 0 on Breakout, so there is no order to correlate, and one game pair is a single observation. --- drivers/cli.ts | 19 ++++++++++ drivers/persistent-cli.ts | 5 +++ examples/policies/cycler.mjs | 30 +++++++++++++++ examples/policies/masher.mjs | 13 +++++++ examples/transfer-2048.matrix | 13 +++++++ examples/transfer-breakout.matrix | 11 ++++++ matrix-run.ts | 62 +++++++++++++++++++++---------- 7 files changed, 133 insertions(+), 20 deletions(-) create mode 100755 examples/policies/cycler.mjs create mode 100755 examples/policies/masher.mjs create mode 100644 examples/transfer-2048.matrix create mode 100644 examples/transfer-breakout.matrix diff --git a/drivers/cli.ts b/drivers/cli.ts index 74163b6..e6409cf 100644 --- a/drivers/cli.ts +++ b/drivers/cli.ts @@ -18,6 +18,20 @@ export interface CliAgentRequest { * a CLI that ignores this key behaves exactly as before. */ images?: readonly ObservationImage[] + /** + * The words this game accepts, when the driver was told them. + * + * A policy that guesses its vocabulary writes illegal words, every one is + * substituted, and the run reads as a player that did nothing rather than one + * that was never told the rules. The streaming transport already states this + * in `brief.json`; this is the same fact on the other transports. + * + * MEASURED: a control that cycles 2048's four words scored 1948 there and 0 + * on ALE Breakout across three replicates, with `distinctInputs=1` and no + * lives lost, because Breakout's words are NOOP, FIRE, RIGHT and LEFT and + * none of the four it emitted was legal. + */ + commands?: readonly string[] /** JSON-safe decision context; cancellation remains process-local. */ context: Readonly> } @@ -96,6 +110,11 @@ export function createCliAgentDriver(options: CliAgentDriverOptions): AgentDrive frame, history: history.map((entry) => ({ ...entry })), ...(images.length === 0 ? {} : { images: images.map((image) => ({ ...image })) }), + // Stated, not guessed. Omitted when the caller named no vocabulary, so + // a policy can still tell "any word goes" from "these words go". + ...(options.commands === undefined || options.commands.length === 0 + ? {} + : { commands: [...options.commands] }), context: { turn: context.turn, maxTurns: context.maxTurns, diff --git a/drivers/persistent-cli.ts b/drivers/persistent-cli.ts index 6bf3675..5b7985c 100644 --- a/drivers/persistent-cli.ts +++ b/drivers/persistent-cli.ts @@ -414,6 +414,11 @@ export function createPersistentCliDriver(options: PersistentCliDriverOptions): frame, history: history.map((entry) => ({ ...entry })), ...(images.length === 0 ? {} : { images: images.map((image) => ({ ...image })) }), + // Stated, not guessed. Omitted when the caller named no vocabulary, so + // a policy can still tell "any word goes" from "these words go". + ...(options.commands === undefined || options.commands.length === 0 + ? {} + : { commands: [...options.commands] }), context: { turn: context.turn, maxTurns: context.maxTurns, diff --git a/examples/policies/cycler.mjs b/examples/policies/cycler.mjs new file mode 100755 index 0000000..aa41806 --- /dev/null +++ b/examples/policies/cycler.mjs @@ -0,0 +1,30 @@ +#!/usr/bin/env node +// Cycles whatever vocabulary the game states, in order. +// +// It used to hardcode 2048's four words, which made it a 2048-only control +// wearing a general name: on ALE Breakout it emitted four illegal words, every +// one was substituted, and three replicates scored 0 with distinctInputs=1 and +// no lives lost. That reads as a player that did nothing rather than one that +// was never told the rules. +const FALLBACK = ['up', 'right', 'down', 'left'] +let buffer = '' +let n = 0 +process.stdin.setEncoding('utf8') +process.stdin.on('data', (chunk) => { + buffer += chunk + for (;;) { + const at = buffer.indexOf('\n') + if (at < 0) break + const line = buffer.slice(0, at) + buffer = buffer.slice(at + 1) + let words = FALLBACK + try { + const request = JSON.parse(line) + if (Array.isArray(request.commands) && request.commands.length > 0) words = request.commands + } catch { + // A request this policy cannot read is still a request. Answering from + // the fallback keeps the episode gradeable instead of stalling it. + } + process.stdout.write(`${words[n++ % words.length]}\n`) + } +}) diff --git a/examples/policies/masher.mjs b/examples/policies/masher.mjs new file mode 100755 index 0000000..d379ccc --- /dev/null +++ b/examples/policies/masher.mjs @@ -0,0 +1,13 @@ +#!/usr/bin/env node +// A trivial baseline: one word, always. It is what a real arm has to beat. +let buffer = '' +process.stdin.setEncoding('utf8') +process.stdin.on('data', (chunk) => { + buffer += chunk + for (;;) { + const at = buffer.indexOf('\n') + if (at < 0) break + buffer = buffer.slice(at + 1) + process.stdout.write('up\n') + } +}) diff --git a/examples/transfer-2048.matrix b/examples/transfer-2048.matrix new file mode 100644 index 0000000..9dfe643 --- /dev/null +++ b/examples/transfer-2048.matrix @@ -0,0 +1,13 @@ +# Half of the first two-game study. Pool it with transfer-breakout.matrix. +# +# Two cheap controls, so the transfer statistic itself is what is being tested +# rather than any agent. A rank correlation needs the same profiles ranked in +# both games; these cost nothing and finish in minutes. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.puzzle adapter=native-2048 target=2048 +objective.score goal=maximize:score horizon=300 budgetUsd=1 +protocol.det frameskip=1 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- +reps 3 diff --git a/examples/transfer-breakout.matrix b/examples/transfer-breakout.matrix new file mode 100644 index 0000000..608ba0b --- /dev/null +++ b/examples/transfer-breakout.matrix @@ -0,0 +1,11 @@ +# The other half. Breakout runs at frameskip 4 because that is the clock its +# packaged reference was recorded at; any other value makes the contract +# underivable and the cell is blocked rather than scored zero. +profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-vocabulary +profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline + +game.brick adapter=ale target=breakout +objective.score goal=maximize:score horizon=300 budgetUsd=1 +protocol.det frameskip=4 sticky=0 seeds=1 +sensor.ascii pixels=off channels=- +reps 3 diff --git a/matrix-run.ts b/matrix-run.ts index 77df2d8..c62b6a6 100644 --- a/matrix-run.ts +++ b/matrix-run.ts @@ -863,26 +863,48 @@ export async function runMatrix(definition: MatrixDefinition, options: RunCellOp * different protocols or horizons are not. */ export function assertJoinable(rows: readonly CellResult[]): void { - const protocols = new Set(rows.map((r) => `${r.protocol}:frameskip=${r.frameskip},sticky=${r.sticky}`)) - if (protocols.size > 1) { - throw new Error( - `these rows were measured under ${protocols.size} protocols and cannot be pooled: ${[...protocols].sort().join(' | ')}`, - ) - } - // The sensor is a 43x variable on one measured game. Pooling across it is - // pooling across the dominant term. - const sensors = new Set(rows.map((r) => `${r.sensor}:${r.sensorDetail}`)) - if (sensors.size > 1) { - throw new Error( - `these rows were measured through ${sensors.size} sensors and cannot be pooled: ${[...sensors].sort().join(' | ')}`, - ) - } - const objectives = new Set(rows.map((r) => r.objective)) - if (objectives.size > 1) { - throw new Error( - `these rows were measured at ${objectives.size} objectives, so their clocks differ and they cannot be pooled:` - + ` ${[...objectives].sort().join(', ')}`, - ) + // Scoped PER GAME, because that is the comparison the transfer statistic + // actually makes. It ranks profiles inside each game and then correlates the + // orders; a raw score never crosses a game boundary. Two games therefore + // SHOULD differ in clock, sensor and objective, and usually must: a packaged + // reference pins the clock it was recorded at, so ALE Breakout is only + // derivable at frameskip 4 while native-2048 runs at 1. + // + // Checked across the whole pool, this refused exactly the study it exists to + // serve. Measured: a two-game run of 2048 and Breakout completed all twelve + // cells and was then refused for having "2 protocols", which were simply the + // two games' own clocks. + // + // What is still refused is two clocks INSIDE one game, which is a genuine + // pooling error: those rows are averaged together and the dominant term + // disappears into the mean. + const games = new Set(rows.map((row) => row.game)) + for (const game of games) { + const inGame = rows.filter((row) => row.game === game) + const where = games.size === 1 ? 'these rows' : `the rows for game "${game}"` + const protocols = new Set(inGame.map((r) => `${r.protocol}:frameskip=${r.frameskip},sticky=${r.sticky}`)) + if (protocols.size > 1) { + throw new Error( + `${where} were measured under ${protocols.size} protocols and cannot be pooled:` + + ` ${[...protocols].sort().join(' | ')}`, + ) + } + // The sensor is a 43x variable on one measured game. Pooling across it is + // pooling across the dominant term. + const sensors = new Set(inGame.map((r) => `${r.sensor}:${r.sensorDetail}`)) + if (sensors.size > 1) { + throw new Error( + `${where} were measured through ${sensors.size} sensors and cannot be pooled:` + + ` ${[...sensors].sort().join(' | ')}`, + ) + } + const objectives = new Set(inGame.map((r) => r.objective)) + if (objectives.size > 1) { + throw new Error( + `${where} were measured at ${objectives.size} objectives, so their clocks differ and they cannot be` + + ` pooled: ${[...objectives].sort().join(', ')}`, + ) + } } }