feat(matrix): state the vocabulary, and let two games have two clocks

drivers/cli.ts

@@ -18,6 +18,20 @@ export interface CliAgentRequest {
    * a CLI that ignores this key behaves exactly as before.
    */
   images?: readonly ObservationImage[]
+  /**
+   * The words this game accepts, when the driver was told them.
+   *
+   * A policy that guesses its vocabulary writes illegal words, every one is
+   * substituted, and the run reads as a player that did nothing rather than one
+   * that was never told the rules. The streaming transport already states this
+   * in `brief.json`; this is the same fact on the other transports.
+   *
+   * MEASURED: a control that cycles 2048's four words scored 1948 there and 0
+   * on ALE Breakout across three replicates, with `distinctInputs=1` and no
+   * lives lost, because Breakout's words are NOOP, FIRE, RIGHT and LEFT and
+   * none of the four it emitted was legal.
+   */
+  commands?: readonly string[]
   /** JSON-safe decision context; cancellation remains process-local. */
   context: Readonly<Omit<AgentDecisionContext, 'signal' | 'observation'>>
 }
@@ -96,6 +110,11 @@ export function createCliAgentDriver(options: CliAgentDriverOptions): AgentDrive
         frame,
         history: history.map((entry) => ({ ...entry })),
         ...(images.length === 0 ? {} : { images: images.map((image) => ({ ...image })) }),
+        // Stated, not guessed. Omitted when the caller named no vocabulary, so
+        // a policy can still tell "any word goes" from "these words go".
+        ...(options.commands === undefined || options.commands.length === 0
+          ? {}
+          : { commands: [...options.commands] }),
         context: {
           turn: context.turn,
           maxTurns: context.maxTurns,

drivers/persistent-cli.ts

@@ -414,6 +414,11 @@ export function createPersistentCliDriver(options: PersistentCliDriverOptions):
         frame,
         history: history.map((entry) => ({ ...entry })),
         ...(images.length === 0 ? {} : { images: images.map((image) => ({ ...image })) }),
+        // Stated, not guessed. Omitted when the caller named no vocabulary, so
+        // a policy can still tell "any word goes" from "these words go".
+        ...(options.commands === undefined || options.commands.length === 0
+          ? {}
+          : { commands: [...options.commands] }),
         context: {
           turn: context.turn,
           maxTurns: context.maxTurns,

examples/policies/cycler.mjs

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+// Cycles whatever vocabulary the game states, in order.
+//
+// It used to hardcode 2048's four words, which made it a 2048-only control
+// wearing a general name: on ALE Breakout it emitted four illegal words, every
+// one was substituted, and three replicates scored 0 with distinctInputs=1 and
+// no lives lost. That reads as a player that did nothing rather than one that
+// was never told the rules.
+const FALLBACK = ['up', 'right', 'down', 'left']
+let buffer = ''
+let n = 0
+process.stdin.setEncoding('utf8')
+process.stdin.on('data', (chunk) => {
+  buffer += chunk
+  for (;;) {
+    const at = buffer.indexOf('\n')
+    if (at < 0) break
+    const line = buffer.slice(0, at)
+    buffer = buffer.slice(at + 1)
+    let words = FALLBACK
+    try {
+      const request = JSON.parse(line)
+      if (Array.isArray(request.commands) && request.commands.length > 0) words = request.commands
+    } catch {
+      // A request this policy cannot read is still a request. Answering from
+      // the fallback keeps the episode gradeable instead of stalling it.
+    }
+    process.stdout.write(`${words[n++ % words.length]}\n`)
+  }
+})

examples/policies/masher.mjs

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+// A trivial baseline: one word, always. It is what a real arm has to beat.
+let buffer = ''
+process.stdin.setEncoding('utf8')
+process.stdin.on('data', (chunk) => {
+  buffer += chunk
+  for (;;) {
+    const at = buffer.indexOf('\n')
+    if (at < 0) break
+    buffer = buffer.slice(at + 1)
+    process.stdout.write('up\n')
+  }
+})

examples/transfer-2048.matrix

@@ -0,0 +1,13 @@
+# Half of the first two-game study. Pool it with transfer-breakout.matrix.
+#
+# Two cheap controls, so the transfer statistic itself is what is being tested
+# rather than any agent. A rank correlation needs the same profiles ranked in
+# both games; these cost nothing and finish in minutes.
+profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-vocabulary
+profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline
+
+game.puzzle adapter=native-2048 target=2048
+objective.score goal=maximize:score horizon=300 budgetUsd=1
+protocol.det frameskip=1 sticky=0 seeds=1
+sensor.ascii pixels=off channels=-
+reps 3

examples/transfer-breakout.matrix

@@ -0,0 +1,11 @@
+# The other half. Breakout runs at frameskip 4 because that is the clock its
+# packaged reference was recorded at; any other value makes the contract
+# underivable and the cell is blocked rather than scored zero.
+profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-vocabulary
+profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline
+
+game.brick adapter=ale target=breakout
+objective.score goal=maximize:score horizon=300 budgetUsd=1
+protocol.det frameskip=4 sticky=0 seeds=1
+sensor.ascii pixels=off channels=-
+reps 3

matrix-run.ts

@@ -863,26 +863,48 @@ export async function runMatrix(definition: MatrixDefinition, options: RunCellOp
  * different protocols or horizons are not.
  */
 export function assertJoinable(rows: readonly CellResult[]): void {
-  const protocols = new Set(rows.map((r) => `${r.protocol}:frameskip=${r.frameskip},sticky=${r.sticky}`))
-  if (protocols.size > 1) {
-    throw new Error(
-      `these rows were measured under ${protocols.size} protocols and cannot be pooled: ${[...protocols].sort().join(' | ')}`,
-    )
-  }
-  // The sensor is a 43x variable on one measured game. Pooling across it is
-  // pooling across the dominant term.
-  const sensors = new Set(rows.map((r) => `${r.sensor}:${r.sensorDetail}`))
-  if (sensors.size > 1) {
-    throw new Error(
-      `these rows were measured through ${sensors.size} sensors and cannot be pooled: ${[...sensors].sort().join(' | ')}`,
-    )
-  }
-  const objectives = new Set(rows.map((r) => r.objective))
-  if (objectives.size > 1) {
-    throw new Error(
-      `these rows were measured at ${objectives.size} objectives, so their clocks differ and they cannot be pooled:`
-      + ` ${[...objectives].sort().join(', ')}`,
-    )
+  // Scoped PER GAME, because that is the comparison the transfer statistic
+  // actually makes. It ranks profiles inside each game and then correlates the
+  // orders; a raw score never crosses a game boundary. Two games therefore
+  // SHOULD differ in clock, sensor and objective, and usually must: a packaged
+  // reference pins the clock it was recorded at, so ALE Breakout is only
+  // derivable at frameskip 4 while native-2048 runs at 1.
+  //
+  // Checked across the whole pool, this refused exactly the study it exists to
+  // serve. Measured: a two-game run of 2048 and Breakout completed all twelve
+  // cells and was then refused for having "2 protocols", which were simply the
+  // two games' own clocks.
+  //
+  // What is still refused is two clocks INSIDE one game, which is a genuine
+  // pooling error: those rows are averaged together and the dominant term
+  // disappears into the mean.
+  const games = new Set(rows.map((row) => row.game))
+  for (const game of games) {
+    const inGame = rows.filter((row) => row.game === game)
+    const where = games.size === 1 ? 'these rows' : `the rows for game "${game}"`
+    const protocols = new Set(inGame.map((r) => `${r.protocol}:frameskip=${r.frameskip},sticky=${r.sticky}`))
+    if (protocols.size > 1) {
+      throw new Error(
+        `${where} were measured under ${protocols.size} protocols and cannot be pooled:`
+        + ` ${[...protocols].sort().join(' | ')}`,
+      )
+    }
+    // The sensor is a 43x variable on one measured game. Pooling across it is
+    // pooling across the dominant term.
+    const sensors = new Set(inGame.map((r) => `${r.sensor}:${r.sensorDetail}`))
+    if (sensors.size > 1) {
+      throw new Error(
+        `${where} were measured through ${sensors.size} sensors and cannot be pooled:`
+        + ` ${[...sensors].sort().join(' | ')}`,
+      )
+    }
+    const objectives = new Set(inGame.map((r) => r.objective))
+    if (objectives.size > 1) {
+      throw new Error(
+        `${where} were measured at ${objectives.size} objectives, so their clocks differ and they cannot be`
+        + ` pooled: ${[...objectives].sort().join(', ')}`,
+      )
+    }
   }
 }