Bind benchmark evidence to native usage

bench/README.md

@@ -420,14 +420,19 @@ case index before the report is written, so the output is byte-for-byte
 deterministic in ordering regardless of completion order. Set `--concurrency 1`
 to fall back to fully sequential execution.
 
-Each case is retried **once** (after a short backoff) when its first attempt
-fails with a transient provider error: a non-zero exit whose stderr carries an
-HTTP 5xx/429, rate-limit, timeout, or connection signature, or a run that
-produced no valid v1 envelope at all (empty/garbled output, typically a dropped
-response). A valid envelope is always treated as a normal result and is never
-retried, including a gate-failing exit (exit 1 with a scored envelope) or one
-that merely reports findings unrelated to the authored target. A case that fails on both
-attempts is recorded as an operational error and excluded from scoring.
+Exploratory live screens retry each case **once** by default (after a short
+backoff) when its first attempt fails with a transient provider error:
+a non-zero exit whose stderr carries an HTTP 5xx/429, rate-limit, timeout, or
+connection signature, or a run that produced no valid v1 envelope at all
+(empty/garbled output, typically a dropped response). `--retries <n>` changes
+that outer retry count. A valid envelope is always treated as a normal result
+and is never retried, including a gate-failing exit (exit 1 with a scored
+envelope) or one that merely reports findings unrelated to the authored target.
+
+Formal calibration and release cohort manifests pin the outer retry count to
+zero. The CLI under test retains its own provider retries, while every accepted
+provider generation remains represented in the attested benchmark report and
+cost evidence.
 
 ### What live mode scores
 
@@ -469,7 +474,7 @@ then attests its completed report and receipt together. Every accepted provider
 response contributes its OpenRouter generation ID to the report. The fan-in job
 verifies globally distinct generation IDs against OpenRouter's authenticated
 generation API, including the exact canonical provider model pinned for each
-logical profile model, provider, token totals, and cost. It also verifies every
+logical profile model, provider, native token totals, and cost. It also verifies every
 subject against the exact repository, release workflow, source commit, tag ref,
 OIDC issuer, and GitHub-hosted runner before parsing it.
 Only the unique first workflow run for

bench/baseline.json

@@ -2,7 +2,7 @@
   "schemaVersion": 2,
   "corpus": {
     "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065",
-    "evaluatorSha256": "b3c6d4085f173ef3abf280b38d4e4221bd39b7752cfbe210e5c19706b700b7e2"
+    "evaluatorSha256": "b0129ecd04903d4cd8ef2ca405c272d7362b9bc867da9270d940795cbc7d1c74"
   },
   "profiles": {
     "z-ai/glm-5.2": {

bench/src/cohort-run.ts

@@ -30,6 +30,31 @@ interface SlotContext {
   paths: ReturnType<typeof cohortSlotPaths>;
 }
 
+export function cohortBenchmarkArguments(options: {
+  screeningProfilePath: string;
+  runId: string;
+  reportPath: string;
+  caseRetries: number;
+}): string[] {
+  if (options.caseRetries !== 0) {
+    throw new Error("formal cohort case retries must be zero");
+  }
+  return [
+    process.execPath,
+    "run",
+    "src/run.ts",
+    "--live",
+    "--screen-profile",
+    options.screeningProfilePath,
+    "--run-id",
+    options.runId,
+    "--retries",
+    String(options.caseRetries),
+    "--json-out",
+    options.reportPath,
+  ];
+}
+
 async function slotContext(options: SlotOptions): Promise<SlotContext> {
   const { manifest, rawSha256: manifestSha256 } = await readCohortManifest(options.manifestPath);
   const cohortSlot = manifest.slots.find((candidate) => candidate.slot === options.slot);
@@ -123,7 +148,11 @@ export async function reserveCohortSlot(options: SlotOptions): Promise<CohortRec
 }
 
 export async function executeReservedCohortSlot(options: SlotOptions & {
-  executeBenchmark?: (options: { reportPath: string; runId: string }) => Promise<number>;
+  executeBenchmark?: (options: {
+    reportPath: string;
+    runId: string;
+    caseRetries: number;
+  }) => Promise<number>;
 }): Promise<number> {
   const context = await slotContext(options);
   const receiptRaw = await readFile(context.paths.receiptPath).catch((error) => {
@@ -146,20 +175,15 @@ export async function executeReservedCohortSlot(options: SlotOptions & {
       exitCode = await options.executeBenchmark({
         reportPath: context.paths.reportPath,
         runId: context.cohortSlot.runId,
+        caseRetries: context.manifest.caseRetries,
       });
     } else {
-      const child = Bun.spawn([
-        process.execPath,
-        "run",
-        "src/run.ts",
-        "--live",
-        "--screen-profile",
-        options.screeningProfilePath,
-        "--run-id",
-        context.cohortSlot.runId,
-        "--json-out",
-        context.paths.reportPath,
-      ], {
+      const child = Bun.spawn(cohortBenchmarkArguments({
+        screeningProfilePath: options.screeningProfilePath,
+        runId: context.cohortSlot.runId,
+        reportPath: context.paths.reportPath,
+        caseRetries: context.manifest.caseRetries,
+      }), {
         cwd: resolve(import.meta.dir, ".."),
         env: { ...process.env, POSTIL_BIN: options.binaryPath },
         stdin: "inherit",
@@ -234,7 +258,11 @@ export async function executeReservedCohortSlot(options: SlotOptions & {
 }
 
 export async function runCohortSlot(options: SlotOptions & {
-  executeBenchmark?: (options: { reportPath: string; runId: string }) => Promise<number>;
+  executeBenchmark?: (options: {
+    reportPath: string;
+    runId: string;
+    caseRetries: number;
+  }) => Promise<number>;
 }): Promise<number> {
   await reserveCohortSlot(options);
   return executeReservedCohortSlot(options);

bench/src/cohort.test.ts

@@ -11,7 +11,11 @@ import {
   reportSemanticSha256,
   type CohortManifest,
 } from "./cohort";
-import { executeReservedCohortSlot, reserveCohortSlot } from "./cohort-run";
+import {
+  cohortBenchmarkArguments,
+  executeReservedCohortSlot,
+  reserveCohortSlot,
+} from "./cohort-run";
 
 const temporaryDirectories: string[] = [];
 
@@ -63,6 +67,7 @@ describe("cohort manifests", () => {
       uuid: () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`,
     });
     expect(manifest.reportCount).toBe(10);
+    expect(manifest.caseRetries).toBe(0);
     expect(manifest.slots.map((slot) => slot.slot)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
     expect(manifest.slots.map((slot) => slot.runId)).toEqual([
       "calibration-e-01", "calibration-e-02", "calibration-e-03", "calibration-e-04",
@@ -109,6 +114,14 @@ describe("cohort manifests", () => {
       ...calibration,
       slots: [calibration.slots[1], calibration.slots[0], ...calibration.slots.slice(2)],
     })).toThrow("ordered and contiguous");
+    expect(() => cohortManifestSchema.parse({
+      ...calibration,
+      caseRetries: 1,
+    })).toThrow();
+    expect(() => cohortManifestSchema.parse({
+      ...calibration,
+      schemaVersion: 2,
+    })).toThrow();
   });
 
   test("binds GitHub release execution to the first run attempt", async () => {
@@ -156,6 +169,22 @@ describe("cohort manifests", () => {
   });
 });
 
+test("formal cohorts pass their immutable zero-retry contract to live screening", () => {
+  const arguments_ = cohortBenchmarkArguments({
+    screeningProfilePath: "/profiles/luna.json",
+    runId: "calibration-01",
+    reportPath: "/reports/01.json",
+    caseRetries: 0,
+  });
+  expect(arguments_[arguments_.indexOf("--retries") + 1]).toBe("0");
+  expect(() => cohortBenchmarkArguments({
+    screeningProfilePath: "/profiles/luna.json",
+    runId: "calibration-01",
+    reportPath: "/reports/01.json",
+    caseRetries: 1,
+  })).toThrow("must be zero");
+});
+
 test("semantic digest excludes execution noise", () => {
   const original = {
     summary: { runId: "one", ranAt: "2026-08-26T00:00:00.000Z", durationMs: 10 },
@@ -234,7 +263,8 @@ test("an authenticated reservation is required before slot execution", async ()
     binaryPath: process.execPath,
     screeningProfilePath,
     environment,
-    executeBenchmark: async ({ reportPath, runId }) => {
+    executeBenchmark: async ({ reportPath, runId, caseRetries }) => {
+      expect(caseRetries).toBe(0);
       await writeFile(reportPath, JSON.stringify({
         summary: { runId, ranAt: new Date().toISOString() },
       }));

bench/src/cohort.ts

@@ -33,11 +33,12 @@ const executionBindingSchema = z.object({
 }).strict();
 
 export const cohortManifestSchema = z.object({
-  schemaVersion: z.literal(2),
+  schemaVersion: z.literal(3),
   purpose: z.enum(["calibration", "release"]),
   cohortId: z.string().uuid(),
   createdAt: z.string().datetime({ offset: true }),
   reportCount: z.union([z.literal(5), z.literal(10)]),
+  caseRetries: z.literal(0),
   binarySha256: sha256Schema,
   evaluatorSha256: sha256Schema,
   fixtureCorpusSha256: sha256Schema,
@@ -307,11 +308,12 @@ export async function createCohortManifest(options: {
     runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? "",
   } as CohortManifest["execution"];
   return cohortManifestSchema.parse({
-    schemaVersion: 2,
+    schemaVersion: 3,
     purpose: options.purpose,
     cohortId,
     createdAt: (options.now ?? new Date()).toISOString(),
     reportCount: count,
+    caseRetries: 0,
     ...bindings,
     execution,
     slots: Array.from({ length: count }, (_, index) => ({

bench/src/compare-baseline.test.ts

@@ -445,11 +445,12 @@ function fakeReleaseCohort(reports: readonly LiveReportForComparison[]): {
     nonce: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`,
   }));
   const manifest: CohortManifest = {
-    schemaVersion: 2,
+    schemaVersion: 3,
     purpose: reports.length === 10 ? "calibration" : "release",
     cohortId: "00000000-0000-4000-8000-000000000099",
     createdAt,
     reportCount: reports.length as 5 | 10,
+    caseRetries: 0,
     binarySha256: HASHES.binary,
     evaluatorSha256: HASHES.evaluator,
     fixtureCorpusSha256: HASHES.corpus,

bench/src/generation-evidence.test.ts

@@ -60,17 +60,21 @@ const records = {
     created_at: "2026-08-26T12:00:10.000Z",
     model: "openai/gpt-5.6-luna-20260709",
     provider_name: "Azure",
-    tokens_prompt: 10,
-    tokens_completion: 5,
+    tokens_prompt: 12,
+    tokens_completion: 4,
+    native_tokens_prompt: 10,
+    native_tokens_completion: 5,
     total_cost: 0.0015,
   },
   "gen-two": {
     id: "gen-two",
     created_at: "2026-08-26T12:00:20.000Z",
     model: "openai/gpt-5.6-luna-20260709",
     provider_name: "Azure",
-    tokens_prompt: 20,
-    tokens_completion: 7,
+    tokens_prompt: 21,
+    tokens_completion: 6,
+    native_tokens_prompt: 20,
+    native_tokens_completion: 7,
     total_cost: 0.0027,
   },
 };
@@ -83,7 +87,7 @@ const profile = {
 };
 
 describe("provider generation evidence", () => {
-  test("verifies distinct generation identity, route, tokens, and cost", async () => {
+  test("verifies distinct generation identity, route, native tokens, and cost", async () => {
     await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], {
       apiKey: "fixture",
       profile,

bench/src/generation-evidence.ts

@@ -92,6 +92,8 @@ const generationSchema = z.object({
     provider_name: z.string().trim().min(1),
     tokens_prompt: z.number().int().nonnegative(),
     tokens_completion: z.number().int().nonnegative(),
+    native_tokens_prompt: z.number().int().nonnegative(),
+    native_tokens_completion: z.number().int().nonnegative(),
     total_cost: z.number().finite().nonnegative(),
   }),
 });
@@ -269,9 +271,14 @@ export async function verifyGenerationEvidence(
     )) {
       throw new Error(`benchmark report ${reportIndex + 1} contains a generation from another provider`);
     }
-    const promptTokens = reportGenerations.reduce((sum, generation) => sum + generation.tokens_prompt, 0);
+    // OpenRouter response usage uses the routed model's native tokenizer.
+    // Generation-level normalized token fields can differ for the same calls.
+    const promptTokens = reportGenerations.reduce(
+      (sum, generation) => sum + generation.native_tokens_prompt,
+      0,
+    );
     const completionTokens = reportGenerations.reduce(
-      (sum, generation) => sum + generation.tokens_completion,
+      (sum, generation) => sum + generation.native_tokens_completion,
       0,
     );
     if (promptTokens !== report.summary.totalTokens.prompt ||

bench/src/run.test.ts

@@ -17,6 +17,7 @@ import {
   createLiveModelsFailureReport,
   generatedLiveScreenRunId,
   invalidateExplicitOutputs,
+  parseLiveRetries,
   parseLiveModelsFailureReport,
   prepareExplicitOutputs,
   qualificationProviderInputs,
@@ -71,7 +72,7 @@ describe("diff-file live screening selection", () => {
   });
 
   test("keeps screen-only flags outside formal admission", () => {
-    for (const flag of ["--case", "--scorer-model", "--screen-profile", "--run-id"]) {
+    for (const flag of ["--case", "--scorer-model", "--screen-profile", "--run-id", "--retries"]) {
       expect(() => validateModeSpecificFlags([flag, "value"], "live-admission"))
         .toThrow("non-admission");
       expect(() => validateModeSpecificFlags([flag, "value"], "mock"))
@@ -81,6 +82,19 @@ describe("diff-file live screening selection", () => {
     }
   });
 
+  test("accepts an explicit zero outer-retry count and rejects ambiguous values", () => {
+    expect(parseLiveRetries([])).toBeUndefined();
+    expect(parseLiveRetries(["--retries", "0"])).toBe(0);
+    expect(parseLiveRetries(["--retries", "2"])).toBe(2);
+    expect(() => parseLiveRetries(["--retries"])).toThrow("requires a value");
+    expect(() => parseLiveRetries(["--retries", "-1"])).toThrow("nonnegative integer");
+    expect(() => parseLiveRetries(["--retries", "1.5"])).toThrow("nonnegative integer");
+    expect(() => parseLiveRetries(["--retries", "9007199254740992"]))
+      .toThrow("safe integer");
+    expect(() => parseLiveRetries(["--retries", "1", "--retries", "2"]))
+      .toThrow("only once");
+  });
+
   test("generates path-safe unique screen identities and scopes the environment override", () => {
     expect(generatedLiveScreenRunId(
       new Date("2026-07-20T12:34:56.789Z"),