Preserve formal retry evidence

bench/README.md

@@ -421,18 +421,18 @@ deterministic in ordering regardless of completion order. Set `--concurrency 1`
 to fall back to fully sequential execution.
 
 Exploratory live screens retry each case **once** by default (after a short
-backoff) when its first attempt fails with a transient provider error:
+backoff) when its first attempt fails with a retryable operational error:
 a non-zero exit whose stderr carries an HTTP 5xx/429, rate-limit, timeout, or
 connection signature, or a run that produced no valid v1 envelope at all
-(empty/garbled output, typically a dropped response). `--retries <n>` changes
-that outer retry count. A valid envelope is always treated as a normal result
-and is never retried, including a gate-failing exit (exit 1 with a scored
-envelope) or one that merely reports findings unrelated to the authored target.
+(empty/garbled output, typically a dropped response). An envelope with an
+unrecovered `review/invalidOutput` incident is also retried. `--retries <n>`
+changes that outer retry count. A scored envelope is always a final result,
+including a gate-failing exit or findings unrelated to the authored target.
 
 Formal calibration and release cohort manifests pin the outer retry count to
-zero. The CLI under test retains its own provider retries, while every accepted
-provider generation remains represented in the attested benchmark report and
-cost evidence.
+one. Reports preserve each case's attempt count, recovered failure categories,
+and aggregate duration, token, and exact provider-cost accounting. Every
+accepted provider generation remains represented in the attested evidence.
 
 ### What live mode scores
 

bench/baseline.json

@@ -2,7 +2,7 @@
   "schemaVersion": 2,
   "corpus": {
     "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065",
-    "evaluatorSha256": "b0129ecd04903d4cd8ef2ca405c272d7362b9bc867da9270d940795cbc7d1c74"
+    "evaluatorSha256": "c1df592f5042b05705a9c7280b9cfb0860d11ae0efba860f719e3b83653fe97b"
   },
   "profiles": {
     "z-ai/glm-5.2": {

bench/src/cohort-run.ts

@@ -36,8 +36,8 @@ export function cohortBenchmarkArguments(options: {
   reportPath: string;
   caseRetries: number;
 }): string[] {
-  if (options.caseRetries !== 0) {
-    throw new Error("formal cohort case retries must be zero");
+  if (options.caseRetries !== 1) {
+    throw new Error("formal cohort case retries must be exactly one");
   }
   return [
     process.execPath,

bench/src/cohort.test.ts

@@ -67,7 +67,7 @@ describe("cohort manifests", () => {
       uuid: () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`,
     });
     expect(manifest.reportCount).toBe(10);
-    expect(manifest.caseRetries).toBe(0);
+    expect(manifest.caseRetries).toBe(1);
     expect(manifest.slots.map((slot) => slot.slot)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
     expect(manifest.slots.map((slot) => slot.runId)).toEqual([
       "calibration-e-01", "calibration-e-02", "calibration-e-03", "calibration-e-04",
@@ -116,7 +116,7 @@ describe("cohort manifests", () => {
     })).toThrow("ordered and contiguous");
     expect(() => cohortManifestSchema.parse({
       ...calibration,
-      caseRetries: 1,
+      caseRetries: 0,
     })).toThrow();
     expect(() => cohortManifestSchema.parse({
       ...calibration,
@@ -169,20 +169,20 @@ describe("cohort manifests", () => {
   });
 });
 
-test("formal cohorts pass their immutable zero-retry contract to live screening", () => {
+test("formal cohorts pass their immutable one-retry contract to live screening", () => {
   const arguments_ = cohortBenchmarkArguments({
     screeningProfilePath: "/profiles/luna.json",
     runId: "calibration-01",
     reportPath: "/reports/01.json",
-    caseRetries: 0,
+    caseRetries: 1,
   });
-  expect(arguments_[arguments_.indexOf("--retries") + 1]).toBe("0");
+  expect(arguments_[arguments_.indexOf("--retries") + 1]).toBe("1");
   expect(() => cohortBenchmarkArguments({
     screeningProfilePath: "/profiles/luna.json",
     runId: "calibration-01",
     reportPath: "/reports/01.json",
-    caseRetries: 1,
-  })).toThrow("must be zero");
+    caseRetries: 0,
+  })).toThrow("exactly one");
 });
 
 test("semantic digest excludes execution noise", () => {
@@ -264,7 +264,7 @@ test("an authenticated reservation is required before slot execution", async ()
     screeningProfilePath,
     environment,
     executeBenchmark: async ({ reportPath, runId, caseRetries }) => {
-      expect(caseRetries).toBe(0);
+      expect(caseRetries).toBe(1);
       await writeFile(reportPath, JSON.stringify({
         summary: { runId, ranAt: new Date().toISOString() },
       }));

bench/src/cohort.ts

@@ -33,12 +33,12 @@ const executionBindingSchema = z.object({
 }).strict();
 
 export const cohortManifestSchema = z.object({
-  schemaVersion: z.literal(3),
+  schemaVersion: z.literal(4),
   purpose: z.enum(["calibration", "release"]),
   cohortId: z.string().uuid(),
   createdAt: z.string().datetime({ offset: true }),
   reportCount: z.union([z.literal(5), z.literal(10)]),
-  caseRetries: z.literal(0),
+  caseRetries: z.literal(1),
   binarySha256: sha256Schema,
   evaluatorSha256: sha256Schema,
   fixtureCorpusSha256: sha256Schema,
@@ -308,12 +308,12 @@ export async function createCohortManifest(options: {
     runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? "",
   } as CohortManifest["execution"];
   return cohortManifestSchema.parse({
-    schemaVersion: 3,
+    schemaVersion: 4,
     purpose: options.purpose,
     cohortId,
     createdAt: (options.now ?? new Date()).toISOString(),
     reportCount: count,
-    caseRetries: 0,
+    caseRetries: 1,
     ...bindings,
     execution,
     slots: Array.from({ length: count }, (_, index) => ({

bench/src/compare-baseline.test.ts

@@ -138,6 +138,8 @@ function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison {
       durationMs: (index + 1) * durationMultiplier,
       observedProviderCostUsdDecimal: costPerCase,
       costAccountingComplete: true,
+      attemptCount: 1,
+      recoveredErrors: [],
       exitCode: truthSeverity === "error" && caseDetected === true ? 1 : 0,
     };
   });
@@ -181,6 +183,7 @@ function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison {
       observedProviderCostUsdDecimal,
       costAccountingComplete: true,
       providerGenerationIds: [`gen-fixture-${fakeRunSequence}`],
+      retryAccounting: { totalAttempts: 70, retriedCases: 0, recoveredErrors: [] },
       errors: 0,
       ranAt: options.ranAt ?? new Date(Date.UTC(2026, 7, 25, 0, 0, fakeRunSequence)).toISOString(),
     },
@@ -211,6 +214,8 @@ async function inputBoundReport(): Promise<LiveReportForComparison> {
       durationMs: (index + 1) * 100,
       observedProviderCostUsdDecimal: "0.001",
       costAccountingComplete: true,
+      attemptCount: 1,
+      recoveredErrors: [],
       exitCode: caseDetected && finding?.severity === "error" ? 1 : 0,
     };
   });
@@ -445,12 +450,12 @@ function fakeReleaseCohort(reports: readonly LiveReportForComparison[]): {
     nonce: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`,
   }));
   const manifest: CohortManifest = {
-    schemaVersion: 3,
+    schemaVersion: 4,
     purpose: reports.length === 10 ? "calibration" : "release",
     cohortId: "00000000-0000-4000-8000-000000000099",
     createdAt,
     reportCount: reports.length as 5 | 10,
-    caseRetries: 0,
+    caseRetries: 1,
     binarySha256: HASHES.binary,
     evaluatorSha256: HASHES.evaluator,
     fixtureCorpusSha256: HASHES.corpus,
@@ -537,6 +542,29 @@ describe("predeclared cohort evidence", () => {
     })).not.toThrow();
   });
 
+  test("rejects attempts beyond the formal manifest retry allowance", () => {
+    const reports = Array.from({ length: 5 }, (_, index) => fakeReport({
+      durationMultiplier: index + 1,
+    }));
+    reports[0]!.results[12]!.attemptCount = 3;
+    reports[0]!.results[12]!.recoveredErrors = [
+      "operational envelope: review/invalidOutput",
+      "operational envelope: review/invalidOutput",
+    ];
+    reports[0]!.summary.retryAccounting = {
+      totalAttempts: 72,
+      retriedCases: 1,
+      recoveredErrors: [{ error: "operational envelope: review/invalidOutput", count: 2 }],
+    };
+    const cohort = fakeReleaseCohort(reports);
+    expect(() => assertCompleteCohortEvidence({
+      ...cohort,
+      manifestSha256: "9".repeat(64),
+      reports,
+      record: false,
+    })).toThrow("exceeds its manifest outer retry allowance");
+  });
+
   test("rejects missing, failed, and mismatched slots", () => {
     const reports = Array.from({ length: 5 }, (_, index) => fakeReport({
       durationMultiplier: index + 1,
@@ -650,6 +678,38 @@ describe("release report validation", () => {
     expect(() => assertValidReleaseReport(report)).toThrow("errors must be 0");
   });
 
+  test("accepts exposed recovered retries and rejects inconsistent retry accounting", () => {
+    const report = cloneReport(fakeReport());
+    report.results[12]!.attemptCount = 2;
+    report.results[12]!.recoveredErrors = ["operational envelope: review/invalidOutput"];
+    report.summary.retryAccounting = {
+      totalAttempts: 71,
+      retriedCases: 1,
+      recoveredErrors: [{ error: "operational envelope: review/invalidOutput", count: 1 }],
+    };
+    expect(() => assertValidReleaseReport(report)).not.toThrow();
+
+    report.summary.retryAccounting.totalAttempts = 70;
+    expect(() => assertValidReleaseReport(report)).toThrow(
+      "summary retry accounting does not match result attempts",
+    );
+  });
+
+  test("allows exploratory reports to expose more than one outer retry", () => {
+    const report = cloneReport(fakeReport());
+    report.results[12]!.attemptCount = 3;
+    report.results[12]!.recoveredErrors = [
+      "operational envelope: review/invalidOutput",
+      "operational envelope: review/invalidOutput",
+    ];
+    report.summary.retryAccounting = {
+      totalAttempts: 72,
+      retriedCases: 1,
+      recoveredErrors: [{ error: "operational envelope: review/invalidOutput", count: 2 }],
+    };
+    expect(() => assertValidReleaseReport(report)).not.toThrow();
+  });
+
   test("rejects a missing scored-case exit code", () => {
     const report = cloneReport(fakeReport());
     (report.results[12] as { exitCode?: number }).exitCode = undefined;

bench/src/compare-baseline.ts

@@ -149,6 +149,8 @@ const liveCaseResultSchema = z.object({
   durationMs: z.number().finite().nonnegative().nullable(),
   observedProviderCostUsdDecimal: canonicalCostSchema.nullable(),
   costAccountingComplete: z.boolean(),
+  attemptCount: z.number().int().positive(),
+  recoveredErrors: z.array(nonemptyStringSchema),
   exitCode: z.union([z.literal(0), z.literal(1)]),
 });
 
@@ -188,6 +190,14 @@ const liveReportSchema = z.object({
     observedProviderCostUsdDecimal: canonicalCostSchema,
     costAccountingComplete: z.boolean(),
     providerGenerationIds: z.array(z.string().regex(/^gen-[A-Za-z0-9_-]+$/u)).min(1),
+    retryAccounting: z.object({
+      totalAttempts: z.number().int().positive(),
+      retriedCases: z.number().int().nonnegative(),
+      recoveredErrors: z.array(z.object({
+        error: nonemptyStringSchema,
+        count: z.number().int().positive(),
+      })),
+    }),
     errors: z.number().int().nonnegative(),
     ranAt: nonemptyStringSchema,
   }),
@@ -468,6 +478,23 @@ export function assertValidReleaseReport(report: LiveReportForComparison): void
   if (report.results.some((result) => result.observedProviderCostUsdDecimal === null)) {
     invalidReport("every result must have canonical observed provider cost");
   }
+  if (report.results.some((result) => result.attemptCount !== result.recoveredErrors.length + 1)) {
+    invalidReport("every result must expose one recovered error for each prior outer attempt");
+  }
+  const totalAttempts = report.results.reduce((sum, result) => sum + result.attemptCount, 0);
+  const retriedCases = report.results.filter((result) => result.attemptCount > 1).length;
+  const recoveredErrorCounts = new Map<string, number>();
+  for (const error of report.results.flatMap((result) => result.recoveredErrors)) {
+    recoveredErrorCounts.set(error, (recoveredErrorCounts.get(error) ?? 0) + 1);
+  }
+  const recoveredErrors = Array.from(recoveredErrorCounts, ([error, count]) => ({ error, count }));
+  if (
+    s.retryAccounting.totalAttempts !== totalAttempts ||
+    s.retryAccounting.retriedCases !== retriedCases ||
+    !isDeepStrictEqual(s.retryAccounting.recoveredErrors, recoveredErrors)
+  ) {
+    invalidReport("summary retry accounting does not match result attempts");
+  }
 
   const ids = report.results.map((result) => result.id);
   if (new Set(ids).size !== ids.length) invalidReport("result IDs must be unique");
@@ -1339,6 +1366,11 @@ export function assertCompleteCohortEvidence(options: {
     if (report.summary.runId !== slot.runId) {
       throw new Error(`benchmark report slot ${slot.slot} does not match its predeclared run ID`);
     }
+    if (report.results.some((result) => result.attemptCount > manifest.caseRetries + 1)) {
+      throw new Error(
+        `benchmark report slot ${slot.slot} exceeds its manifest outer retry allowance`,
+      );
+    }
     if (receipt.reportRawSha256 !== options.rawReportSha256[index]) {
       throw new Error(`benchmark report slot ${slot.slot} raw digest does not match its receipt`);
     }