From bffb724f2ba9c82fc7c922b29e4c0aedd466ba26 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 05:36:54 +0000 Subject: [PATCH] Preserve formal retry evidence --- bench/README.md | 16 ++-- bench/baseline.json | 2 +- bench/src/cohort-run.ts | 4 +- bench/src/cohort.test.ts | 16 ++-- bench/src/cohort.ts | 8 +- bench/src/compare-baseline.test.ts | 64 +++++++++++++- bench/src/compare-baseline.ts | 32 +++++++ bench/src/live.test.ts | 133 +++++++++++++++++++++++++++++ bench/src/live.ts | 88 +++++++++++++++---- 9 files changed, 321 insertions(+), 42 deletions(-) diff --git a/bench/README.md b/bench/README.md index 77cf236..af62797 100644 --- a/bench/README.md +++ b/bench/README.md @@ -421,18 +421,18 @@ deterministic in ordering regardless of completion order. Set `--concurrency 1` to fall back to fully sequential execution. Exploratory live screens retry each case **once** by default (after a short -backoff) when its first attempt fails with a transient provider error: +backoff) when its first attempt fails with a retryable operational error: a non-zero exit whose stderr carries an HTTP 5xx/429, rate-limit, timeout, or connection signature, or a run that produced no valid v1 envelope at all -(empty/garbled output, typically a dropped response). `--retries ` changes -that outer retry count. A valid envelope is always treated as a normal result -and is never retried, including a gate-failing exit (exit 1 with a scored -envelope) or one that merely reports findings unrelated to the authored target. +(empty/garbled output, typically a dropped response). An envelope with an +unrecovered `review/invalidOutput` incident is also retried. `--retries ` +changes that outer retry count. A scored envelope is always a final result, +including a gate-failing exit or findings unrelated to the authored target. Formal calibration and release cohort manifests pin the outer retry count to -zero. The CLI under test retains its own provider retries, while every accepted -provider generation remains represented in the attested benchmark report and -cost evidence. +one. Reports preserve each case's attempt count, recovered failure categories, +and aggregate duration, token, and exact provider-cost accounting. Every +accepted provider generation remains represented in the attested evidence. ### What live mode scores diff --git a/bench/baseline.json b/bench/baseline.json index 144ab0f..e169857 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -2,7 +2,7 @@ "schemaVersion": 2, "corpus": { "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065", - "evaluatorSha256": "b0129ecd04903d4cd8ef2ca405c272d7362b9bc867da9270d940795cbc7d1c74" + "evaluatorSha256": "c1df592f5042b05705a9c7280b9cfb0860d11ae0efba860f719e3b83653fe97b" }, "profiles": { "z-ai/glm-5.2": { diff --git a/bench/src/cohort-run.ts b/bench/src/cohort-run.ts index 37ae64c..5b2db12 100644 --- a/bench/src/cohort-run.ts +++ b/bench/src/cohort-run.ts @@ -36,8 +36,8 @@ export function cohortBenchmarkArguments(options: { reportPath: string; caseRetries: number; }): string[] { - if (options.caseRetries !== 0) { - throw new Error("formal cohort case retries must be zero"); + if (options.caseRetries !== 1) { + throw new Error("formal cohort case retries must be exactly one"); } return [ process.execPath, diff --git a/bench/src/cohort.test.ts b/bench/src/cohort.test.ts index 38bcec6..bd9bc84 100644 --- a/bench/src/cohort.test.ts +++ b/bench/src/cohort.test.ts @@ -67,7 +67,7 @@ describe("cohort manifests", () => { uuid: () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`, }); expect(manifest.reportCount).toBe(10); - expect(manifest.caseRetries).toBe(0); + expect(manifest.caseRetries).toBe(1); expect(manifest.slots.map((slot) => slot.slot)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); expect(manifest.slots.map((slot) => slot.runId)).toEqual([ "calibration-e-01", "calibration-e-02", "calibration-e-03", "calibration-e-04", @@ -116,7 +116,7 @@ describe("cohort manifests", () => { })).toThrow("ordered and contiguous"); expect(() => cohortManifestSchema.parse({ ...calibration, - caseRetries: 1, + caseRetries: 0, })).toThrow(); expect(() => cohortManifestSchema.parse({ ...calibration, @@ -169,20 +169,20 @@ describe("cohort manifests", () => { }); }); -test("formal cohorts pass their immutable zero-retry contract to live screening", () => { +test("formal cohorts pass their immutable one-retry contract to live screening", () => { const arguments_ = cohortBenchmarkArguments({ screeningProfilePath: "/profiles/luna.json", runId: "calibration-01", reportPath: "/reports/01.json", - caseRetries: 0, + caseRetries: 1, }); - expect(arguments_[arguments_.indexOf("--retries") + 1]).toBe("0"); + expect(arguments_[arguments_.indexOf("--retries") + 1]).toBe("1"); expect(() => cohortBenchmarkArguments({ screeningProfilePath: "/profiles/luna.json", runId: "calibration-01", reportPath: "/reports/01.json", - caseRetries: 1, - })).toThrow("must be zero"); + caseRetries: 0, + })).toThrow("exactly one"); }); test("semantic digest excludes execution noise", () => { @@ -264,7 +264,7 @@ test("an authenticated reservation is required before slot execution", async () screeningProfilePath, environment, executeBenchmark: async ({ reportPath, runId, caseRetries }) => { - expect(caseRetries).toBe(0); + expect(caseRetries).toBe(1); await writeFile(reportPath, JSON.stringify({ summary: { runId, ranAt: new Date().toISOString() }, })); diff --git a/bench/src/cohort.ts b/bench/src/cohort.ts index a0c9c68..75515ad 100644 --- a/bench/src/cohort.ts +++ b/bench/src/cohort.ts @@ -33,12 +33,12 @@ const executionBindingSchema = z.object({ }).strict(); export const cohortManifestSchema = z.object({ - schemaVersion: z.literal(3), + schemaVersion: z.literal(4), purpose: z.enum(["calibration", "release"]), cohortId: z.string().uuid(), createdAt: z.string().datetime({ offset: true }), reportCount: z.union([z.literal(5), z.literal(10)]), - caseRetries: z.literal(0), + caseRetries: z.literal(1), binarySha256: sha256Schema, evaluatorSha256: sha256Schema, fixtureCorpusSha256: sha256Schema, @@ -308,12 +308,12 @@ export async function createCohortManifest(options: { runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? "", } as CohortManifest["execution"]; return cohortManifestSchema.parse({ - schemaVersion: 3, + schemaVersion: 4, purpose: options.purpose, cohortId, createdAt: (options.now ?? new Date()).toISOString(), reportCount: count, - caseRetries: 0, + caseRetries: 1, ...bindings, execution, slots: Array.from({ length: count }, (_, index) => ({ diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index 615ddf0..ac1d052 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -138,6 +138,8 @@ function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison { durationMs: (index + 1) * durationMultiplier, observedProviderCostUsdDecimal: costPerCase, costAccountingComplete: true, + attemptCount: 1, + recoveredErrors: [], exitCode: truthSeverity === "error" && caseDetected === true ? 1 : 0, }; }); @@ -181,6 +183,7 @@ function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison { observedProviderCostUsdDecimal, costAccountingComplete: true, providerGenerationIds: [`gen-fixture-${fakeRunSequence}`], + retryAccounting: { totalAttempts: 70, retriedCases: 0, recoveredErrors: [] }, errors: 0, ranAt: options.ranAt ?? new Date(Date.UTC(2026, 7, 25, 0, 0, fakeRunSequence)).toISOString(), }, @@ -211,6 +214,8 @@ async function inputBoundReport(): Promise { durationMs: (index + 1) * 100, observedProviderCostUsdDecimal: "0.001", costAccountingComplete: true, + attemptCount: 1, + recoveredErrors: [], exitCode: caseDetected && finding?.severity === "error" ? 1 : 0, }; }); @@ -445,12 +450,12 @@ function fakeReleaseCohort(reports: readonly LiveReportForComparison[]): { nonce: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, })); const manifest: CohortManifest = { - schemaVersion: 3, + schemaVersion: 4, purpose: reports.length === 10 ? "calibration" : "release", cohortId: "00000000-0000-4000-8000-000000000099", createdAt, reportCount: reports.length as 5 | 10, - caseRetries: 0, + caseRetries: 1, binarySha256: HASHES.binary, evaluatorSha256: HASHES.evaluator, fixtureCorpusSha256: HASHES.corpus, @@ -537,6 +542,29 @@ describe("predeclared cohort evidence", () => { })).not.toThrow(); }); + test("rejects attempts beyond the formal manifest retry allowance", () => { + const reports = Array.from({ length: 5 }, (_, index) => fakeReport({ + durationMultiplier: index + 1, + })); + reports[0]!.results[12]!.attemptCount = 3; + reports[0]!.results[12]!.recoveredErrors = [ + "operational envelope: review/invalidOutput", + "operational envelope: review/invalidOutput", + ]; + reports[0]!.summary.retryAccounting = { + totalAttempts: 72, + retriedCases: 1, + recoveredErrors: [{ error: "operational envelope: review/invalidOutput", count: 2 }], + }; + const cohort = fakeReleaseCohort(reports); + expect(() => assertCompleteCohortEvidence({ + ...cohort, + manifestSha256: "9".repeat(64), + reports, + record: false, + })).toThrow("exceeds its manifest outer retry allowance"); + }); + test("rejects missing, failed, and mismatched slots", () => { const reports = Array.from({ length: 5 }, (_, index) => fakeReport({ durationMultiplier: index + 1, @@ -650,6 +678,38 @@ describe("release report validation", () => { expect(() => assertValidReleaseReport(report)).toThrow("errors must be 0"); }); + test("accepts exposed recovered retries and rejects inconsistent retry accounting", () => { + const report = cloneReport(fakeReport()); + report.results[12]!.attemptCount = 2; + report.results[12]!.recoveredErrors = ["operational envelope: review/invalidOutput"]; + report.summary.retryAccounting = { + totalAttempts: 71, + retriedCases: 1, + recoveredErrors: [{ error: "operational envelope: review/invalidOutput", count: 1 }], + }; + expect(() => assertValidReleaseReport(report)).not.toThrow(); + + report.summary.retryAccounting.totalAttempts = 70; + expect(() => assertValidReleaseReport(report)).toThrow( + "summary retry accounting does not match result attempts", + ); + }); + + test("allows exploratory reports to expose more than one outer retry", () => { + const report = cloneReport(fakeReport()); + report.results[12]!.attemptCount = 3; + report.results[12]!.recoveredErrors = [ + "operational envelope: review/invalidOutput", + "operational envelope: review/invalidOutput", + ]; + report.summary.retryAccounting = { + totalAttempts: 72, + retriedCases: 1, + recoveredErrors: [{ error: "operational envelope: review/invalidOutput", count: 2 }], + }; + expect(() => assertValidReleaseReport(report)).not.toThrow(); + }); + test("rejects a missing scored-case exit code", () => { const report = cloneReport(fakeReport()); (report.results[12] as { exitCode?: number }).exitCode = undefined; diff --git a/bench/src/compare-baseline.ts b/bench/src/compare-baseline.ts index df071a0..8d7506c 100644 --- a/bench/src/compare-baseline.ts +++ b/bench/src/compare-baseline.ts @@ -149,6 +149,8 @@ const liveCaseResultSchema = z.object({ durationMs: z.number().finite().nonnegative().nullable(), observedProviderCostUsdDecimal: canonicalCostSchema.nullable(), costAccountingComplete: z.boolean(), + attemptCount: z.number().int().positive(), + recoveredErrors: z.array(nonemptyStringSchema), exitCode: z.union([z.literal(0), z.literal(1)]), }); @@ -188,6 +190,14 @@ const liveReportSchema = z.object({ observedProviderCostUsdDecimal: canonicalCostSchema, costAccountingComplete: z.boolean(), providerGenerationIds: z.array(z.string().regex(/^gen-[A-Za-z0-9_-]+$/u)).min(1), + retryAccounting: z.object({ + totalAttempts: z.number().int().positive(), + retriedCases: z.number().int().nonnegative(), + recoveredErrors: z.array(z.object({ + error: nonemptyStringSchema, + count: z.number().int().positive(), + })), + }), errors: z.number().int().nonnegative(), ranAt: nonemptyStringSchema, }), @@ -468,6 +478,23 @@ export function assertValidReleaseReport(report: LiveReportForComparison): void if (report.results.some((result) => result.observedProviderCostUsdDecimal === null)) { invalidReport("every result must have canonical observed provider cost"); } + if (report.results.some((result) => result.attemptCount !== result.recoveredErrors.length + 1)) { + invalidReport("every result must expose one recovered error for each prior outer attempt"); + } + const totalAttempts = report.results.reduce((sum, result) => sum + result.attemptCount, 0); + const retriedCases = report.results.filter((result) => result.attemptCount > 1).length; + const recoveredErrorCounts = new Map(); + for (const error of report.results.flatMap((result) => result.recoveredErrors)) { + recoveredErrorCounts.set(error, (recoveredErrorCounts.get(error) ?? 0) + 1); + } + const recoveredErrors = Array.from(recoveredErrorCounts, ([error, count]) => ({ error, count })); + if ( + s.retryAccounting.totalAttempts !== totalAttempts || + s.retryAccounting.retriedCases !== retriedCases || + !isDeepStrictEqual(s.retryAccounting.recoveredErrors, recoveredErrors) + ) { + invalidReport("summary retry accounting does not match result attempts"); + } const ids = report.results.map((result) => result.id); if (new Set(ids).size !== ids.length) invalidReport("result IDs must be unique"); @@ -1339,6 +1366,11 @@ export function assertCompleteCohortEvidence(options: { if (report.summary.runId !== slot.runId) { throw new Error(`benchmark report slot ${slot.slot} does not match its predeclared run ID`); } + if (report.results.some((result) => result.attemptCount > manifest.caseRetries + 1)) { + throw new Error( + `benchmark report slot ${slot.slot} exceeds its manifest outer retry allowance`, + ); + } if (receipt.reportRawSha256 !== options.rawReportSha256[index]) { throw new Error(`benchmark report slot ${slot.slot} raw digest does not match its receipt`); } diff --git a/bench/src/live.test.ts b/bench/src/live.test.ts index b5dbbb2..b3a3703 100644 --- a/bench/src/live.test.ts +++ b/bench/src/live.test.ts @@ -6,6 +6,7 @@ import { join } from "node:path"; import { cases } from "../fixtures/cases"; import type { Envelope } from "./harness"; import { + aggregateAttemptAccounting, boundedCoverageFailure, envelopeOperationalFailure, exactProviderCost, @@ -428,6 +429,138 @@ describe("live benchmark operational envelopes", () => { })).toEqual({ costUsdDecimal: null, complete: false }); }); + test("retains every billed attempt when invalid model output is retried", async () => { + const root = await mkdtemp(join(tmpdir(), "postil-live-retry-accounting-")); + const marker = join(root, "attempt-count"); + const previousKey = process.env.MODEL_API_KEY; + process.env.MODEL_API_KEY = "test-key-not-sent-anywhere"; + try { + const envelope = ( + promptTokens: number, + completionTokens: number, + cost: string, + durationMs: number, + ): Envelope => ({ + version: 1, + summary: "No findings.", + silent: true, + findings: [], + suppressedFindings: [], + resolved: [], + counts: { info: 0, warn: 0, error: 0, suppressed: 0, ungrounded: 0 }, + confidenceBuckets: [0, 0, 0, 0, 0], + gate: { failOn: "error", failing: false, blockOnKinds: [] }, + modelUsed: "test/model", + usage: { promptTokens, completionTokens }, + modelUsage: [{ + model: "test/model", + role: "reviewGenerator", + promptTokens, + completionTokens, + costProviderDecimal: cost, + costSource: "providerReported", + accountingComplete: true, + }], + modelIncidents: [], + reviewCoverage: { + mode: "exhaustive", + selectedBatches: 1, + totalBatches: 1, + plannerFallback: false, + }, + usageAccountingComplete: true, + durationMs, + baseSha: null, + headSha: null, + sinceSha: null, + }); + const first: Envelope = { + ...envelope(10, 2, "0.00012", 110), + modelIncidents: [{ phase: "review", category: "invalidOutput", recovered: false }], + }; + const second = envelope(20, 3, "0.00018", 90); + const binary = join(root, "fake-postil"); + await writeFile(binary, `#!/bin/sh +if [ -f '${marker}' ]; then + printf 2 > '${marker}' + printf '%s\\n' '${JSON.stringify(second)}' +else + printf 1 > '${marker}' + printf '%s\\n' '${JSON.stringify(first)}' +fi +`, { mode: 0o700 }); + await chmod(binary, 0o700); + + const report = await runLive([cases[0]!], { + binary, + model: "test/model", + rootDir: root, + runId: "invalid-output-retry", + concurrency: 1, + retries: 1, + }); + + expect(await readFile(marker, "utf8")).toBe("2"); + expect(report.results[0]).toMatchObject({ + scored: true, + durationMs: 200, + promptTokens: 30, + completionTokens: 5, + observedProviderCostUsdDecimal: "0.0003", + costAccountingComplete: true, + attemptCount: 2, + recoveredErrors: ["operational envelope: review/invalidOutput"], + }); + expect(report.summary.totalTokens).toEqual({ prompt: 30, completion: 5, total: 35 }); + expect(report.summary.observedProviderCostUsdDecimal).toBe("0.0003"); + expect(report.summary.costAccountingComplete).toBe(true); + expect(report.summary.retryAccounting).toEqual({ + totalAttempts: 2, + retriedCases: 1, + recoveredErrors: [{ error: "operational envelope: review/invalidOutput", count: 1 }], + }); + } finally { + if (previousKey === undefined) delete process.env.MODEL_API_KEY; + else process.env.MODEL_API_KEY = previousKey; + await rm(root, { recursive: true, force: true }); + } + }, 30_000); + + test("keeps unknown attempt cost incomplete after a retry", () => { + const finalAttempt = { + scored: true, + durationMs: 90, + promptTokens: 20, + completionTokens: 3, + observedProviderCostUsdDecimal: "0.00018", + costAccountingComplete: true, + attemptCount: 1, + recoveredErrors: [], + } as Parameters[0]; + const unknownAttempt = { + ...finalAttempt, + scored: false, + durationMs: null, + promptTokens: 0, + completionTokens: 0, + observedProviderCostUsdDecimal: null, + costAccountingComplete: false, + attemptCount: 1, + recoveredErrors: [], + error: "unknown accounting", + }; + + expect(aggregateAttemptAccounting(finalAttempt, [unknownAttempt, finalAttempt])).toMatchObject({ + durationMs: null, + promptTokens: 20, + completionTokens: 3, + observedProviderCostUsdDecimal: null, + costAccountingComplete: false, + attemptCount: 2, + recoveredErrors: ["unknown accounting"], + }); + }); + test("requires an exercised exact scorer identity when screening a scorer", () => { const scored = { ...valid, diff --git a/bench/src/live.ts b/bench/src/live.ts index 9b79eb2..7fa6d21 100644 --- a/bench/src/live.ts +++ b/bench/src/live.ts @@ -54,11 +54,11 @@ const DEFAULT_CASE_TIMEOUT_MS = 180_000; * Override with --concurrency or BENCH_CONCURRENCY. */ export const DEFAULT_LIVE_CONCURRENCY = 6; -/** Each case that fails with a transient/provider error is retried this many - * extra times (so one retry total). A second failure is recorded as an error. */ +/** Each case that fails with a transient/provider error or unrecovered invalid + * output is retried this many extra times. A second failure is recorded. */ const DEFAULT_LIVE_RETRIES = 1; -/** Backoff before the single retry of a transiently-failed case. */ +/** Backoff before the single retry of a retryable failed case. */ const RETRY_BACKOFF_MS = 2_000; /** Severity tiers, ordered low to high (mirrors src/envelope.rs: info < warn < @@ -101,7 +101,7 @@ export interface LiveOptions { timeoutMs?: number; /** Cases run concurrently (default DEFAULT_LIVE_CONCURRENCY). */ concurrency?: number; - /** Extra attempts on a transient/provider failure (default 1 = one retry). */ + /** Extra attempts on a retryable operational failure (default 1). */ retries?: number; /** Exercise deterministic risk selection and synthesis for large reviews. */ bounded?: boolean; @@ -152,6 +152,10 @@ export interface LiveCaseResult { completionTokens: number; observedProviderCostUsdDecimal: string | null; costAccountingComplete: boolean; + /** Total outer benchmark attempts used for this final result. */ + attemptCount: number; + /** Failed outer attempts recovered before the final result, in order. */ + recoveredErrors: string[]; reviewCoverage: Envelope["reviewCoverage"] | null; /** Suppression reasons for findings that overlapped the authored target. * Diagnostic only: suppressed findings never count as detections. */ @@ -210,6 +214,11 @@ export interface LiveSummary { observedProviderCostUsdDecimal: string; costAccountingComplete: boolean; providerGenerationIds: string[]; + retryAccounting: { + totalAttempts: number; + retriedCases: number; + recoveredErrors: Array<{ error: string; count: number }>; + }; errors: number; } @@ -484,11 +493,10 @@ export async function evaluatorSourceSha256(): Promise { /** * Runs a case, retrying once (by default) after a short backoff if the first - * attempt fails with a transient/provider error: a non-zero exit whose stderr - * carries an HTTP-5xx/429/timeout/connection signature, or no valid v1 envelope - * at all (empty/garbled output). A valid envelope with findings is a normal - * result and is never retried. A case that fails every attempt is returned as - * the last attempt's error result, which summarize() already counts as an error. + * attempt fails with a transient/provider error or an unrecovered invalid model + * output. A scored envelope is a final result and is never retried. Tokens, + * provider cost, and duration from every attempt are retained in the returned + * result so a successful retry cannot hide billed generations. */ async function runLiveCaseWithRetry( c: ReturnType, @@ -501,6 +509,7 @@ async function runLiveCaseWithRetry( ): Promise { const maxRetries = options.retries ?? DEFAULT_LIVE_RETRIES; let last: LiveCaseResult | undefined; + const attempts: LiveCaseResult[] = []; for (let attempt = 0; attempt <= maxRetries; attempt++) { last = await runLiveCase( c, @@ -512,16 +521,51 @@ async function runLiveCaseWithRetry( provider, captureApiBase, ); + attempts.push(last); // Scored => a valid envelope was produced; that is a normal result (even if // it has findings or false positives), so never retry it. - if (last.scored) return last; + if (last.scored) return aggregateAttemptAccounting(last, attempts); if (attempt < maxRetries && isTransientFailure(last)) { await sleep(RETRY_BACKOFF_MS); continue; } - return last; + return aggregateAttemptAccounting(last, attempts); } - return last!; + return aggregateAttemptAccounting(last!, attempts); +} + +/** Keeps the final attempt's review verdict while accounting for all work used + * to obtain it. Incomplete accounting on any attempt remains incomplete after + * a retry, which prevents formal evidence from silently dropping unknown cost. */ +export function aggregateAttemptAccounting( + finalAttempt: LiveCaseResult, + attempts: readonly LiveCaseResult[], +): LiveCaseResult { + if (attempts.length === 0 || attempts[attempts.length - 1] !== finalAttempt) { + throw new Error("attempt accounting requires the final attempt last"); + } + const costAccountingComplete = attempts.every((attempt) => + attempt.costAccountingComplete && attempt.observedProviderCostUsdDecimal !== null + ); + const durationComplete = attempts.every((attempt) => attempt.durationMs !== null); + return { + ...finalAttempt, + durationMs: durationComplete + ? attempts.reduce((sum, attempt) => sum + attempt.durationMs!, 0) + : null, + promptTokens: attempts.reduce((sum, attempt) => sum + attempt.promptTokens, 0), + completionTokens: attempts.reduce((sum, attempt) => sum + attempt.completionTokens, 0), + observedProviderCostUsdDecimal: costAccountingComplete + ? formatCanonicalDecimal(sumCanonicalDecimals(attempts.map((attempt) => + parseCanonicalDecimal(attempt.observedProviderCostUsdDecimal!) + ))) + : null, + costAccountingComplete, + attemptCount: attempts.length, + recoveredErrors: attempts.slice(0, -1).flatMap((attempt) => + attempt.error === undefined ? [] : [attempt.error] + ), + }; } /** Signatures of a transient provider/transport failure in stderr: HTTP 5xx and @@ -549,14 +593,13 @@ const TRANSIENT_STDERR = new RegExp( "i", ); -/** True when an unscored case looks like a transient provider failure: either - * the binary emitted a recognizable transient signature on stderr, or it - * produced no valid envelope at all (empty/garbled output, e.g. a dropped - * response). Both are worth one retry; a deterministic parse-shaped failure that - * is not provider-side will just fail again and be recorded as an error. */ +/** True when an unscored case is worth one outer retry: the binary emitted a + * transient signature, produced no valid envelope, or explicitly reported an + * unrecovered review/invalidOutput incident. */ function isTransientFailure(result: LiveCaseResult): boolean { if (result.scored) return false; if (result.stderr && TRANSIENT_STDERR.test(result.stderr)) return true; + if (result.error === "operational envelope: review/invalidOutput") return true; // No valid envelope (empty/invalid output), typically a dropped or truncated // provider response; retry once. return result.error?.startsWith("no valid v1 envelope") ?? false; @@ -638,6 +681,8 @@ async function runLiveCase( completionTokens: 0, observedProviderCostUsdDecimal: null, costAccountingComplete: false, + attemptCount: 1, + recoveredErrors: [], reviewCoverage: null, suppressedTargetReasons: [], exitCode: undefined, @@ -1024,6 +1069,10 @@ function summarize( result.observedProviderCostUsdDecimal === null ? [] : [parseCanonicalDecimal(result.observedProviderCostUsdDecimal)])); + const recoveredErrorCounts = new Map(); + for (const error of results.flatMap((result) => result.recoveredErrors)) { + recoveredErrorCounts.set(error, (recoveredErrorCounts.get(error) ?? 0) + 1); + } return { runId: options.runId, @@ -1078,6 +1127,11 @@ function summarize( observedProviderCostUsdDecimal: formatCanonicalDecimal(observedProviderCost), costAccountingComplete: liveCostAccountingComplete(results), providerGenerationIds, + retryAccounting: { + totalAttempts: results.reduce((sum, result) => sum + result.attemptCount, 0), + retriedCases: results.filter((result) => result.attemptCount > 1).length, + recoveredErrors: Array.from(recoveredErrorCounts, ([error, count]) => ({ error, count })), + }, errors: results.filter((r) => r.error !== undefined).length, }; }