feat(matrix): keep everything a run measured, not only its headline

drivers/stream-sandbox.ts

@@ -134,6 +134,11 @@ export interface StreamHealth {
   costUsd: number | null
   /** Tokens the agent says it has spent, or null when it did not say. */
   tokens: number | null
+  /** Input and output tokens apart, and turns taken. A ratio is diagnostic:
+   *  1.8M in against 72 out is an agent reading, not one writing. */
+  inputTokens: number | null
+  outputTokens: number | null
+  turns: number | null
   /**
    * Which credential path the agent used, as its launcher reported it.
    *
@@ -408,16 +413,30 @@ export function createStreamSandboxDriver(options: StreamSandboxDriverOptions):
    * Written when the agent exits, so it is absent for a cell read while the
    * agent still runs. Absence is reported as absence.
    */
-  function reportedMeter(): { costUsd: number | null; tokens: number | null; authMode: 'api-key' | 'oauth' | null } {
+  function reportedMeter(): {
+    costUsd: number | null
+    tokens: number | null
+    inputTokens: number | null
+    outputTokens: number | null
+    turns: number | null
+    authMode: 'api-key' | 'oauth' | null
+  } {
     const nonNegative = (value: unknown): number | null =>
       typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null
     try {
       const parsed: unknown = JSON.parse(readFileSync(join(dir, 'agent-cost.json'), 'utf8'))
-      const meter = parsed as { usd?: unknown; tokens?: unknown; authMode?: unknown }
+      const meter = parsed as Record<string, unknown>
       const mode = meter.authMode === 'api-key' || meter.authMode === 'oauth' ? meter.authMode : null
-      return { costUsd: nonNegative(meter.usd), tokens: nonNegative(meter.tokens), authMode: mode }
+      return {
+        costUsd: nonNegative(meter.usd),
+        tokens: nonNegative(meter.tokens),
+        inputTokens: nonNegative(meter.inputTokens),
+        outputTokens: nonNegative(meter.outputTokens),
+        turns: nonNegative(meter.turns),
+        authMode: mode,
+      }
     } catch {
-      return { costUsd: null, tokens: null, authMode: null }
+      return { costUsd: null, tokens: null, inputTokens: null, outputTokens: null, turns: null, authMode: null }
     }
   }
 

examples/arena/airstriker.matrix

@@ -0,0 +1,17 @@
+# airstriker on the arena board. The clock and seed are the ones this game's packaged
+# reference was recorded at; any other value makes its contract underivable.
+#
+# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a
+# better player, which is a different question from which model is stronger and
+# the one an efficiency study actually needs.
+profile.opus-3    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=3 transport=persistent
+profile.opus-8    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=8 transport=persistent
+profile.sonnet-3  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent
+profile.sonnet-8  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent
+
+game.sky adapter=stable-retro target=Airstriker-Genesis
+objective.score goal=maximize:score horizon=2000 budgetUsd=4
+protocol.det frameskip=4 sticky=0 seeds=1
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/arena/breakout.matrix

@@ -0,0 +1,17 @@
+# breakout on the arena board. The clock and seed are the ones this game's packaged
+# reference was recorded at; any other value makes its contract underivable.
+#
+# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a
+# better player, which is a different question from which model is stronger and
+# the one an efficiency study actually needs.
+profile.opus-3    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=3 transport=persistent
+profile.opus-8    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=8 transport=persistent
+profile.sonnet-3  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent
+profile.sonnet-8  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent
+
+game.brick adapter=ale target=breakout
+objective.score goal=maximize:score horizon=3000 budgetUsd=4
+protocol.det frameskip=4 sticky=0 seeds=1
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/arena/cartpole.matrix

@@ -0,0 +1,17 @@
+# cartpole on the arena board. The clock and seed are the ones this game's packaged
+# reference was recorded at; any other value makes its contract underivable.
+#
+# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a
+# better player, which is a different question from which model is stronger and
+# the one an efficiency study actually needs.
+profile.opus-3    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=3 transport=persistent
+profile.opus-8    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=8 transport=persistent
+profile.sonnet-3  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent
+profile.sonnet-8  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent
+
+game.pole adapter=gymnasium target=CartPole-v1
+objective.steps goal=maximize:steps horizon=500 budgetUsd=4
+protocol.det frameskip=1 sticky=0 seeds=1 seed0=7
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/arena/frozenlake.matrix

@@ -0,0 +1,17 @@
+# frozenlake on the arena board. The clock and seed are the ones this game's packaged
+# reference was recorded at; any other value makes its contract underivable.
+#
+# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a
+# better player, which is a different question from which model is stronger and
+# the one an efficiency study actually needs.
+profile.opus-3    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=3 transport=persistent
+profile.opus-8    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=8 transport=persistent
+profile.sonnet-3  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent
+profile.sonnet-8  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent
+
+game.ice adapter=gymnasium target=FrozenLake-v1
+objective.goal goal=maximize:steps horizon=200 budgetUsd=4
+protocol.det frameskip=1 sticky=0 seeds=1 seed0=3
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/arena/puzzle.matrix

@@ -0,0 +1,17 @@
+# puzzle on the arena board. The clock and seed are the ones this game's packaged
+# reference was recorded at; any other value makes its contract underivable.
+#
+# Two axes vary, not one. `buildMin` asks whether more time to BUILD produces a
+# better player, which is a different question from which model is stronger and
+# the one an efficiency study actually needs.
+profile.opus-3    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=3 transport=persistent
+profile.opus-8    harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=8 transport=persistent
+profile.sonnet-3  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=3 transport=persistent
+profile.sonnet-8  harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=8 transport=persistent
+
+game.puzzle adapter=native-2048 target=2048
+objective.score goal=maximize:score horizon=20000 budgetUsd=4
+protocol.det frameskip=1 sticky=0 seeds=1
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/board/airstriker.matrix

@@ -0,0 +1,11 @@
+# airstriker: one game of the board. Its clock and seed are the ones its packaged
+# reference was recorded at; any other value makes the contract underivable.
+profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary
+profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline
+
+game.sky adapter=stable-retro target=Airstriker-Genesis
+objective.score goal=maximize:score horizon=300 budgetUsd=1
+protocol.det frameskip=4 sticky=0 seeds=1
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/board/breakout.matrix

@@ -0,0 +1,11 @@
+# breakout: one game of the board. Its clock and seed are the ones its packaged
+# reference was recorded at; any other value makes the contract underivable.
+profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary
+profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline
+
+game.brick adapter=ale target=breakout
+objective.score goal=maximize:score horizon=300 budgetUsd=1
+protocol.det frameskip=4 sticky=0 seeds=1
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/board/cartpole.matrix

@@ -0,0 +1,11 @@
+# cartpole: one game of the board. Its clock and seed are the ones its packaged
+# reference was recorded at; any other value makes the contract underivable.
+profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary
+profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline
+
+game.pole adapter=gymnasium target=CartPole-v1
+objective.steps goal=maximize:steps horizon=200 budgetUsd=1
+protocol.det frameskip=1 sticky=0 seeds=1 seed0=7
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/board/frozenlake.matrix

@@ -0,0 +1,11 @@
+# frozenlake: one game of the board. Its clock and seed are the ones its packaged
+# reference was recorded at; any other value makes the contract underivable.
+profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary
+profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline
+
+game.ice adapter=gymnasium target=FrozenLake-v1
+objective.goal goal=maximize:steps horizon=100 budgetUsd=1
+protocol.det frameskip=1 sticky=0 seeds=1 seed0=3
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/board/puzzle.matrix

@@ -0,0 +1,11 @@
+# puzzle: one game of the board. Its clock and seed are the ones its packaged
+# reference was recorded at; any other value makes the contract underivable.
+profile.cycler harness=none policy=./examples/policies/cycler.mjs note=cycles-the-stated-vocabulary
+profile.masher harness=none policy=./examples/policies/masher.mjs note=one-word-baseline
+
+game.puzzle adapter=native-2048 target=2048
+objective.score goal=maximize:score horizon=300 budgetUsd=1
+protocol.det frameskip=1 sticky=0 seeds=1
+sensor.ascii pixels=off channels=-
+
+reps 1

examples/policies/masher.mjs

@@ -1,13 +1,28 @@
 #!/usr/bin/env node
-// A trivial baseline: one word, always. It is what a real arm has to beat.
+// Plays the first word the game states, forever: the weakest honest baseline.
+//
+// It used to emit a hardcoded lowercase word. On ALE Breakout, whose words are
+// NOOP, FIRE, RIGHT and LEFT, every one was illegal and substituted, and the
+// first decision spent ten seconds reaching a driver timeout. A control that
+// cannot name a legal move measures the harness, not the game.
+const FALLBACK = 'up'
 let buffer = ''
 process.stdin.setEncoding('utf8')
 process.stdin.on('data', (chunk) => {
   buffer += chunk
   for (;;) {
     const at = buffer.indexOf('\n')
     if (at < 0) break
+    const line = buffer.slice(0, at)
     buffer = buffer.slice(at + 1)
-    process.stdout.write('up\n')
+    let word = FALLBACK
+    try {
+      const request = JSON.parse(line)
+      if (Array.isArray(request.commands) && request.commands.length > 0) word = request.commands[0]
+    } catch {
+      // An unreadable request is still a request. Answering keeps the episode
+      // gradeable instead of stalling it at a timeout.
+    }
+    process.stdout.write(`${word}\n`)
   }
 })

examples/transfer-agents-2048.matrix

@@ -0,0 +1,12 @@
+# Half of a two-game transfer study, played by real profiles.
+#
+# The cheap controls tie at 0 on Breakout, and two tied players have no order,
+# so a rank correlation has nothing to correlate. These build a player instead.
+profile.opus   harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=6 transport=persistent
+profile.sonnet harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=6 transport=persistent
+
+game.puzzle adapter=native-2048 target=2048
+objective.score goal=maximize:score horizon=20000 budgetUsd=4
+protocol.det frameskip=1 sticky=0 seeds=1 queue=8 empty=noop pace=0
+sensor.ascii pixels=off channels=-
+reps 2

examples/transfer-agents-breakout.matrix

@@ -0,0 +1,10 @@
+# The other half. frameskip 4 is the clock Breakout's packaged reference was
+# recorded at; any other value makes its contract underivable.
+profile.opus   harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-opus-5   buildMin=6 transport=persistent
+profile.sonnet harness=./harnesses/claude-code author=./harnesses/author-policy model=claude-sonnet-5 buildMin=6 transport=persistent
+
+game.brick adapter=ale target=breakout
+objective.score goal=maximize:score horizon=3000 budgetUsd=4
+protocol.det frameskip=4 sticky=0 seeds=1 queue=8 empty=noop pace=0
+sensor.ascii pixels=off channels=-
+reps 2