diff --git a/bench/README.md b/bench/README.md index 01f76cf..77cf236 100644 --- a/bench/README.md +++ b/bench/README.md @@ -420,14 +420,19 @@ case index before the report is written, so the output is byte-for-byte deterministic in ordering regardless of completion order. Set `--concurrency 1` to fall back to fully sequential execution. -Each case is retried **once** (after a short backoff) when its first attempt -fails with a transient provider error: a non-zero exit whose stderr carries an -HTTP 5xx/429, rate-limit, timeout, or connection signature, or a run that -produced no valid v1 envelope at all (empty/garbled output, typically a dropped -response). A valid envelope is always treated as a normal result and is never -retried, including a gate-failing exit (exit 1 with a scored envelope) or one -that merely reports findings unrelated to the authored target. A case that fails on both -attempts is recorded as an operational error and excluded from scoring. +Exploratory live screens retry each case **once** by default (after a short +backoff) when its first attempt fails with a transient provider error: +a non-zero exit whose stderr carries an HTTP 5xx/429, rate-limit, timeout, or +connection signature, or a run that produced no valid v1 envelope at all +(empty/garbled output, typically a dropped response). `--retries ` changes +that outer retry count. A valid envelope is always treated as a normal result +and is never retried, including a gate-failing exit (exit 1 with a scored +envelope) or one that merely reports findings unrelated to the authored target. + +Formal calibration and release cohort manifests pin the outer retry count to +zero. The CLI under test retains its own provider retries, while every accepted +provider generation remains represented in the attested benchmark report and +cost evidence. ### What live mode scores @@ -469,7 +474,7 @@ then attests its completed report and receipt together. Every accepted provider response contributes its OpenRouter generation ID to the report. The fan-in job verifies globally distinct generation IDs against OpenRouter's authenticated generation API, including the exact canonical provider model pinned for each -logical profile model, provider, token totals, and cost. It also verifies every +logical profile model, provider, native token totals, and cost. It also verifies every subject against the exact repository, release workflow, source commit, tag ref, OIDC issuer, and GitHub-hosted runner before parsing it. Only the unique first workflow run for diff --git a/bench/baseline.json b/bench/baseline.json index a160900..144ab0f 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -2,7 +2,7 @@ "schemaVersion": 2, "corpus": { "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065", - "evaluatorSha256": "b3c6d4085f173ef3abf280b38d4e4221bd39b7752cfbe210e5c19706b700b7e2" + "evaluatorSha256": "b0129ecd04903d4cd8ef2ca405c272d7362b9bc867da9270d940795cbc7d1c74" }, "profiles": { "z-ai/glm-5.2": { diff --git a/bench/src/cohort-run.ts b/bench/src/cohort-run.ts index c04f0e7..37ae64c 100644 --- a/bench/src/cohort-run.ts +++ b/bench/src/cohort-run.ts @@ -30,6 +30,31 @@ interface SlotContext { paths: ReturnType; } +export function cohortBenchmarkArguments(options: { + screeningProfilePath: string; + runId: string; + reportPath: string; + caseRetries: number; +}): string[] { + if (options.caseRetries !== 0) { + throw new Error("formal cohort case retries must be zero"); + } + return [ + process.execPath, + "run", + "src/run.ts", + "--live", + "--screen-profile", + options.screeningProfilePath, + "--run-id", + options.runId, + "--retries", + String(options.caseRetries), + "--json-out", + options.reportPath, + ]; +} + async function slotContext(options: SlotOptions): Promise { const { manifest, rawSha256: manifestSha256 } = await readCohortManifest(options.manifestPath); const cohortSlot = manifest.slots.find((candidate) => candidate.slot === options.slot); @@ -123,7 +148,11 @@ export async function reserveCohortSlot(options: SlotOptions): Promise Promise; + executeBenchmark?: (options: { + reportPath: string; + runId: string; + caseRetries: number; + }) => Promise; }): Promise { const context = await slotContext(options); const receiptRaw = await readFile(context.paths.receiptPath).catch((error) => { @@ -146,20 +175,15 @@ export async function executeReservedCohortSlot(options: SlotOptions & { exitCode = await options.executeBenchmark({ reportPath: context.paths.reportPath, runId: context.cohortSlot.runId, + caseRetries: context.manifest.caseRetries, }); } else { - const child = Bun.spawn([ - process.execPath, - "run", - "src/run.ts", - "--live", - "--screen-profile", - options.screeningProfilePath, - "--run-id", - context.cohortSlot.runId, - "--json-out", - context.paths.reportPath, - ], { + const child = Bun.spawn(cohortBenchmarkArguments({ + screeningProfilePath: options.screeningProfilePath, + runId: context.cohortSlot.runId, + reportPath: context.paths.reportPath, + caseRetries: context.manifest.caseRetries, + }), { cwd: resolve(import.meta.dir, ".."), env: { ...process.env, POSTIL_BIN: options.binaryPath }, stdin: "inherit", @@ -234,7 +258,11 @@ export async function executeReservedCohortSlot(options: SlotOptions & { } export async function runCohortSlot(options: SlotOptions & { - executeBenchmark?: (options: { reportPath: string; runId: string }) => Promise; + executeBenchmark?: (options: { + reportPath: string; + runId: string; + caseRetries: number; + }) => Promise; }): Promise { await reserveCohortSlot(options); return executeReservedCohortSlot(options); diff --git a/bench/src/cohort.test.ts b/bench/src/cohort.test.ts index f8ea2d5..38bcec6 100644 --- a/bench/src/cohort.test.ts +++ b/bench/src/cohort.test.ts @@ -11,7 +11,11 @@ import { reportSemanticSha256, type CohortManifest, } from "./cohort"; -import { executeReservedCohortSlot, reserveCohortSlot } from "./cohort-run"; +import { + cohortBenchmarkArguments, + executeReservedCohortSlot, + reserveCohortSlot, +} from "./cohort-run"; const temporaryDirectories: string[] = []; @@ -63,6 +67,7 @@ describe("cohort manifests", () => { uuid: () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`, }); expect(manifest.reportCount).toBe(10); + expect(manifest.caseRetries).toBe(0); expect(manifest.slots.map((slot) => slot.slot)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); expect(manifest.slots.map((slot) => slot.runId)).toEqual([ "calibration-e-01", "calibration-e-02", "calibration-e-03", "calibration-e-04", @@ -109,6 +114,14 @@ describe("cohort manifests", () => { ...calibration, slots: [calibration.slots[1], calibration.slots[0], ...calibration.slots.slice(2)], })).toThrow("ordered and contiguous"); + expect(() => cohortManifestSchema.parse({ + ...calibration, + caseRetries: 1, + })).toThrow(); + expect(() => cohortManifestSchema.parse({ + ...calibration, + schemaVersion: 2, + })).toThrow(); }); test("binds GitHub release execution to the first run attempt", async () => { @@ -156,6 +169,22 @@ describe("cohort manifests", () => { }); }); +test("formal cohorts pass their immutable zero-retry contract to live screening", () => { + const arguments_ = cohortBenchmarkArguments({ + screeningProfilePath: "/profiles/luna.json", + runId: "calibration-01", + reportPath: "/reports/01.json", + caseRetries: 0, + }); + expect(arguments_[arguments_.indexOf("--retries") + 1]).toBe("0"); + expect(() => cohortBenchmarkArguments({ + screeningProfilePath: "/profiles/luna.json", + runId: "calibration-01", + reportPath: "/reports/01.json", + caseRetries: 1, + })).toThrow("must be zero"); +}); + test("semantic digest excludes execution noise", () => { const original = { summary: { runId: "one", ranAt: "2026-08-26T00:00:00.000Z", durationMs: 10 }, @@ -234,7 +263,8 @@ test("an authenticated reservation is required before slot execution", async () binaryPath: process.execPath, screeningProfilePath, environment, - executeBenchmark: async ({ reportPath, runId }) => { + executeBenchmark: async ({ reportPath, runId, caseRetries }) => { + expect(caseRetries).toBe(0); await writeFile(reportPath, JSON.stringify({ summary: { runId, ranAt: new Date().toISOString() }, })); diff --git a/bench/src/cohort.ts b/bench/src/cohort.ts index b00c2cd..a0c9c68 100644 --- a/bench/src/cohort.ts +++ b/bench/src/cohort.ts @@ -33,11 +33,12 @@ const executionBindingSchema = z.object({ }).strict(); export const cohortManifestSchema = z.object({ - schemaVersion: z.literal(2), + schemaVersion: z.literal(3), purpose: z.enum(["calibration", "release"]), cohortId: z.string().uuid(), createdAt: z.string().datetime({ offset: true }), reportCount: z.union([z.literal(5), z.literal(10)]), + caseRetries: z.literal(0), binarySha256: sha256Schema, evaluatorSha256: sha256Schema, fixtureCorpusSha256: sha256Schema, @@ -307,11 +308,12 @@ export async function createCohortManifest(options: { runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? "", } as CohortManifest["execution"]; return cohortManifestSchema.parse({ - schemaVersion: 2, + schemaVersion: 3, purpose: options.purpose, cohortId, createdAt: (options.now ?? new Date()).toISOString(), reportCount: count, + caseRetries: 0, ...bindings, execution, slots: Array.from({ length: count }, (_, index) => ({ diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index 6cc77b3..615ddf0 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -445,11 +445,12 @@ function fakeReleaseCohort(reports: readonly LiveReportForComparison[]): { nonce: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, })); const manifest: CohortManifest = { - schemaVersion: 2, + schemaVersion: 3, purpose: reports.length === 10 ? "calibration" : "release", cohortId: "00000000-0000-4000-8000-000000000099", createdAt, reportCount: reports.length as 5 | 10, + caseRetries: 0, binarySha256: HASHES.binary, evaluatorSha256: HASHES.evaluator, fixtureCorpusSha256: HASHES.corpus, diff --git a/bench/src/generation-evidence.test.ts b/bench/src/generation-evidence.test.ts index 1bab800..a06a5a3 100644 --- a/bench/src/generation-evidence.test.ts +++ b/bench/src/generation-evidence.test.ts @@ -60,8 +60,10 @@ const records = { created_at: "2026-08-26T12:00:10.000Z", model: "openai/gpt-5.6-luna-20260709", provider_name: "Azure", - tokens_prompt: 10, - tokens_completion: 5, + tokens_prompt: 12, + tokens_completion: 4, + native_tokens_prompt: 10, + native_tokens_completion: 5, total_cost: 0.0015, }, "gen-two": { @@ -69,8 +71,10 @@ const records = { created_at: "2026-08-26T12:00:20.000Z", model: "openai/gpt-5.6-luna-20260709", provider_name: "Azure", - tokens_prompt: 20, - tokens_completion: 7, + tokens_prompt: 21, + tokens_completion: 6, + native_tokens_prompt: 20, + native_tokens_completion: 7, total_cost: 0.0027, }, }; @@ -83,7 +87,7 @@ const profile = { }; describe("provider generation evidence", () => { - test("verifies distinct generation identity, route, tokens, and cost", async () => { + test("verifies distinct generation identity, route, native tokens, and cost", async () => { await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { apiKey: "fixture", profile, diff --git a/bench/src/generation-evidence.ts b/bench/src/generation-evidence.ts index 5577d73..c7a5c7b 100644 --- a/bench/src/generation-evidence.ts +++ b/bench/src/generation-evidence.ts @@ -92,6 +92,8 @@ const generationSchema = z.object({ provider_name: z.string().trim().min(1), tokens_prompt: z.number().int().nonnegative(), tokens_completion: z.number().int().nonnegative(), + native_tokens_prompt: z.number().int().nonnegative(), + native_tokens_completion: z.number().int().nonnegative(), total_cost: z.number().finite().nonnegative(), }), }); @@ -269,9 +271,14 @@ export async function verifyGenerationEvidence( )) { throw new Error(`benchmark report ${reportIndex + 1} contains a generation from another provider`); } - const promptTokens = reportGenerations.reduce((sum, generation) => sum + generation.tokens_prompt, 0); + // OpenRouter response usage uses the routed model's native tokenizer. + // Generation-level normalized token fields can differ for the same calls. + const promptTokens = reportGenerations.reduce( + (sum, generation) => sum + generation.native_tokens_prompt, + 0, + ); const completionTokens = reportGenerations.reduce( - (sum, generation) => sum + generation.tokens_completion, + (sum, generation) => sum + generation.native_tokens_completion, 0, ); if (promptTokens !== report.summary.totalTokens.prompt || diff --git a/bench/src/run.test.ts b/bench/src/run.test.ts index db8db9a..c02e5ee 100644 --- a/bench/src/run.test.ts +++ b/bench/src/run.test.ts @@ -17,6 +17,7 @@ import { createLiveModelsFailureReport, generatedLiveScreenRunId, invalidateExplicitOutputs, + parseLiveRetries, parseLiveModelsFailureReport, prepareExplicitOutputs, qualificationProviderInputs, @@ -71,7 +72,7 @@ describe("diff-file live screening selection", () => { }); test("keeps screen-only flags outside formal admission", () => { - for (const flag of ["--case", "--scorer-model", "--screen-profile", "--run-id"]) { + for (const flag of ["--case", "--scorer-model", "--screen-profile", "--run-id", "--retries"]) { expect(() => validateModeSpecificFlags([flag, "value"], "live-admission")) .toThrow("non-admission"); expect(() => validateModeSpecificFlags([flag, "value"], "mock")) @@ -81,6 +82,19 @@ describe("diff-file live screening selection", () => { } }); + test("accepts an explicit zero outer-retry count and rejects ambiguous values", () => { + expect(parseLiveRetries([])).toBeUndefined(); + expect(parseLiveRetries(["--retries", "0"])).toBe(0); + expect(parseLiveRetries(["--retries", "2"])).toBe(2); + expect(() => parseLiveRetries(["--retries"])).toThrow("requires a value"); + expect(() => parseLiveRetries(["--retries", "-1"])).toThrow("nonnegative integer"); + expect(() => parseLiveRetries(["--retries", "1.5"])).toThrow("nonnegative integer"); + expect(() => parseLiveRetries(["--retries", "9007199254740992"])) + .toThrow("safe integer"); + expect(() => parseLiveRetries(["--retries", "1", "--retries", "2"])) + .toThrow("only once"); + }); + test("generates path-safe unique screen identities and scopes the environment override", () => { expect(generatedLiveScreenRunId( new Date("2026-07-20T12:34:56.789Z"), diff --git a/bench/src/run.ts b/bench/src/run.ts index ef70e79..ba370d4 100644 --- a/bench/src/run.ts +++ b/bench/src/run.ts @@ -19,7 +19,7 @@ // no mock GitHub at all. Selected by --live / BENCH_LIVE. // // bun run bench:live # or: BENCH_LIVE=1 bun run src/run.ts -// bun run bench --live [--json] [--json-out ] [--model ] [--concurrency ] +// bun run bench --live [--json] [--json-out ] [--model ] [--concurrency ] [--retries ] // // Environment: // POSTIL_BIN path to the postil binary (default ../target/release/postil) @@ -48,6 +48,7 @@ // --screen-profile exact provider and price contract for selected cases // --scorer-model optional scorer for non-admission diff-file screening // --run-id optional live-screen artifact namespace +// --retries outer case retries after the first attempt (default 1) import { createHash, randomUUID } from "node:crypto"; import { readFileSync } from "node:fs"; @@ -188,7 +189,7 @@ export function validateModeSpecificFlags( args: readonly string[], mode: "mock" | "live-screen" | "live-admission", ): void { - for (const flag of ["--case", "--scorer-model", "--screen-profile", "--run-id"]) { + for (const flag of ["--case", "--scorer-model", "--screen-profile", "--run-id", "--retries"]) { if (!args.includes(flag)) continue; if (mode === "live-admission") { throw new Error(`${flag} is a non-admission diff-file screen option and is unavailable in live-models admission mode`); @@ -199,6 +200,29 @@ export function validateModeSpecificFlags( } } +export function parseLiveRetries(args: readonly string[]): number | undefined { + let raw: string | undefined; + for (let index = 0; index < args.length; index += 1) { + if (args[index] !== "--retries") continue; + if (raw !== undefined) throw new Error("--retries may be specified only once"); + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) { + throw new Error("--retries requires a value"); + } + raw = value; + index += 1; + } + if (raw === undefined) return undefined; + if (!/^(?:0|[1-9][0-9]*)$/u.test(raw)) { + throw new Error("--retries must be a nonnegative integer"); + } + const retries = Number(raw); + if (!Number.isSafeInteger(retries)) { + throw new Error("--retries must be a nonnegative safe integer"); + } + return retries; +} + export function validateRunIdentityEnvironment( runId: string | undefined, mode: "mock" | "live-screen" | "live-admission", @@ -351,6 +375,7 @@ async function main() { throw new Error("live benchmark needs an explicit model: set REVIEW_MODEL or --model"); } const concurrency = liveConcurrency(args); + const retries = parseLiveRetries(args); const scorerModel = flagValue(args, "--scorer-model"); if (args.includes("--scorer-model") && scorerModel === undefined) { throw new Error("--scorer-model requires a value"); @@ -375,6 +400,7 @@ async function main() { scorerModel, screenProfilePath, concurrency, + retries, bounded, selectedCaseIds, runId,