From 5a95d956f70d33079f9527ecf0ed07f26982c60c Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 27 Aug 2026 01:57:35 +0000 Subject: [PATCH 1/2] Pin provider generation identities --- .github/workflows/benchmark-calibration.yml | 1 + .github/workflows/release.yml | 1 + ARCHITECTURE.md | 9 +- bench/README.md | 7 +- bench/baseline.json | 2 +- bench/src/generation-evidence.test.ts | 115 ++++++++++++++- bench/src/generation-evidence.ts | 147 +++++++++++++++++--- bench/src/livemodels.test.ts | 22 ++- bench/src/verify-admission.test.ts | 44 ++++++ bench/src/verify-admission.ts | 16 +++ provisional-models.json | 3 + src/config.rs | 86 ++++++++++++ 12 files changed, 422 insertions(+), 31 deletions(-) diff --git a/.github/workflows/benchmark-calibration.yml b/.github/workflows/benchmark-calibration.yml index ef358bb..451533c 100644 --- a/.github/workflows/benchmark-calibration.yml +++ b/.github/workflows/benchmark-calibration.yml @@ -347,6 +347,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | bun run bench:verify-generations -- \ + --screen-profile ../provisional-models.json \ --result "${RUNNER_TEMP}/verified-slots/01/report.json" \ --receipt "${RUNNER_TEMP}/verified-slots/01/receipt.json" \ --result "${RUNNER_TEMP}/verified-slots/02/report.json" \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31812c8..7a788a0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -511,6 +511,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | bun run bench:verify-generations -- \ + --screen-profile ../provisional-models.json \ --result "${RUNNER_TEMP}/bench-live-reports/slots/01/report.json" \ --receipt "${RUNNER_TEMP}/bench-live-reports/slots/01/receipt.json" \ --result "${RUNNER_TEMP}/bench-live-reports/slots/02/report.json" \ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1de095d..aba863b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -210,10 +210,11 @@ acquire diff --> parse supported lockfiles --> parse + index --> bounded evidenc An operator can activate the embedded provisional roster with `POSTIL_PROVISIONAL_HOSTED_ROSTER=1` while the formal admission manifest is empty. `provisional-models.json` fixes the managed endpoint, upstream provider, - model chains, consensus, and price ceilings. The release verifier requires that - profile to match `config.toml`, and the runtime applies the same provider pin, - privacy policy, response-identity checks, and operation cost cap used by an - admitted roster. Removing the flag restores the attested-profile requirement. + model chains, canonical provider generation identities, consensus, and price + ceilings. The release verifier requires that profile to match `config.toml`, + and the runtime applies the same provider pin, privacy policy, + response-identity checks, and operation cost cap used by an admitted roster. + Removing the flag restores the attested-profile requirement. ## Prompt-injected policy sources diff --git a/bench/README.md b/bench/README.md index 20bd231..01f76cf 100644 --- a/bench/README.md +++ b/bench/README.md @@ -468,9 +468,10 @@ canonical slot directory and attests the running receipt before inference, then attests its completed report and receipt together. Every accepted provider response contributes its OpenRouter generation ID to the report. The fan-in job verifies globally distinct generation IDs against OpenRouter's authenticated -generation API, including the exact model, provider, token totals, and cost, -then verifies every subject against the exact repository, release workflow, -source commit, tag ref, OIDC issuer, and GitHub-hosted runner before parsing it. +generation API, including the exact canonical provider model pinned for each +logical profile model, provider, token totals, and cost. It also verifies every +subject against the exact repository, release workflow, source commit, tag ref, +OIDC issuer, and GitHub-hosted runner before parsing it. Only the unique first workflow run for the version tag is authoritative. Tag-scoped concurrency, an existing-release check, and duplicate-run rejection prevent a second publisher path. A failed diff --git a/bench/baseline.json b/bench/baseline.json index 7178f26..a160900 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -2,7 +2,7 @@ "schemaVersion": 2, "corpus": { "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065", - "evaluatorSha256": "00b424fcc4fb13a1c4bd4010ef79a992d0692d59ceb70eae0669ea9823e6645e" + "evaluatorSha256": "b3c6d4085f173ef3abf280b38d4e4221bd39b7752cfbe210e5c19706b700b7e2" }, "profiles": { "z-ai/glm-5.2": { diff --git a/bench/src/generation-evidence.test.ts b/bench/src/generation-evidence.test.ts index eab008f..1bab800 100644 --- a/bench/src/generation-evidence.test.ts +++ b/bench/src/generation-evidence.test.ts @@ -12,6 +12,9 @@ function sample( runId: "fixture-run", ranAt: "2026-08-26T12:00:30.000Z", model: "openai/gpt-5.6-luna", + scorerMode: "disabled", + scorerModel: null, + screeningProfileSha256: "c".repeat(64), upstreamProviderIdentity: "Azure", totalTokens: { prompt: 30, completion: 12, total: 42 }, observedProviderCostUsdDecimal: "0.0042", @@ -55,7 +58,7 @@ const records = { "gen-one": { id: "gen-one", created_at: "2026-08-26T12:00:10.000Z", - model: "openai/gpt-5.6-luna", + model: "openai/gpt-5.6-luna-20260709", provider_name: "Azure", tokens_prompt: 10, tokens_completion: 5, @@ -64,7 +67,7 @@ const records = { "gen-two": { id: "gen-two", created_at: "2026-08-26T12:00:20.000Z", - model: "openai/gpt-5.6-luna", + model: "openai/gpt-5.6-luna-20260709", provider_name: "Azure", tokens_prompt: 20, tokens_completion: 7, @@ -72,10 +75,18 @@ const records = { }, }; +const profile = { + sha256: "c".repeat(64), + providerGenerationModels: { + "openai/gpt-5.6-luna": "openai/gpt-5.6-luna-20260709", + }, +}; + describe("provider generation evidence", () => { test("verifies distinct generation identity, route, tokens, and cost", async () => { await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { apiKey: "fixture", + profile, fetchImpl: generationFetch(records), })).resolves.toBe(2); }); @@ -86,6 +97,7 @@ describe("provider generation evidence", () => { sample(["gen-one", "gen-two"]), ], { apiKey: "fixture", + profile, fetchImpl: generationFetch(records), })).rejects.toThrow("duplicate provider generation IDs"); }); @@ -95,10 +107,12 @@ describe("provider generation evidence", () => { sample(["gen-one", "gen-two"], { totalTokens: { prompt: 31, completion: 12, total: 43 } }), ], { apiKey: "fixture", + profile, fetchImpl: generationFetch(records), })).rejects.toThrow("token totals do not match provider generations"); await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { apiKey: "fixture", + profile, fetchImpl: generationFetch({ ...records, "gen-two": { ...records["gen-two"], provider_name: "Other" }, @@ -106,9 +120,103 @@ describe("provider generation evidence", () => { })).rejects.toThrow("generation from another provider"); }); + test("binds the logical alias to the exact provider generation model", async () => { + await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { + apiKey: "fixture", + profile, + fetchImpl: generationFetch({ + ...records, + "gen-two": { ...records["gen-two"], model: "openai/gpt-5.6-luna-20260801" }, + }), + })).rejects.toThrow("generation for another model"); + + await expect(verifyGenerationEvidence([ + sample(["gen-one", "gen-two"], { screeningProfileSha256: "d".repeat(64) }), + ], { + apiKey: "fixture", + profile, + fetchImpl: generationFetch(records), + })).rejects.toThrow("does not match its screening profile"); + }); + + test("binds each report to only its generator and scorer identities", async () => { + const multipleModels = { + sha256: "c".repeat(64), + providerGenerationModels: { + "logical/generator": "provider/generator-20260801", + "logical/other": "provider/other-20260801", + "logical/scorer": "provider/scorer-20260801", + }, + }; + const generatorRecord = { + ...records["gen-one"], + model: "provider/generator-20260801", + }; + const otherRecord = { + ...records["gen-two"], + model: "provider/other-20260801", + }; + const generatorReport = sample(["gen-one", "gen-two"], { + model: "logical/generator", + }); + await expect(verifyGenerationEvidence([generatorReport], { + apiKey: "fixture", + profile: multipleModels, + fetchImpl: generationFetch({ + "gen-one": generatorRecord, + "gen-two": otherRecord, + }), + })).rejects.toThrow("generation for another model"); + + const scorerRecord = { + ...records["gen-two"], + model: "provider/scorer-20260801", + }; + await expect(verifyGenerationEvidence([ + sample(["gen-one", "gen-two"], { + model: "logical/generator", + scorerMode: "enabled", + scorerModel: "logical/scorer", + }), + ], { + apiKey: "fixture", + profile: multipleModels, + fetchImpl: generationFetch({ + "gen-one": generatorRecord, + "gen-two": scorerRecord, + }), + })).resolves.toBe(2); + }); + + test("rejects ambiguous provider identity maps", async () => { + await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { + apiKey: "fixture", + profile: { + sha256: "c".repeat(64), + providerGenerationModels: { + "logical/one": "provider/shared-20260801", + "logical/two": "provider/shared-20260801", + }, + }, + fetchImpl: generationFetch(records), + })).rejects.toThrow("must not repeat canonical models"); + + await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { + apiKey: "fixture", + profile: { + sha256: "c".repeat(64), + providerGenerationModels: { + "openai/gpt-5.6-luna": "openai/gpt-5.6-luna", + }, + }, + fetchImpl: generationFetch(records), + })).rejects.toThrow("must be distinct from logical model IDs"); + }); + test("rejects a lookup whose returned generation identity differs", async () => { await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { apiKey: "fixture", + profile, fetchImpl: generationFetch({ ...records, "gen-two": { ...records["gen-two"], id: "gen-one" }, @@ -119,6 +227,7 @@ describe("provider generation evidence", () => { test("binds every generation to the attested receipt interval", async () => { await expect(verifyGenerationEvidence([sample(["gen-one", "gen-two"])], { apiKey: "fixture", + profile, fetchImpl: generationFetch({ ...records, "gen-two": { ...records["gen-two"], created_at: "2024-01-01T00:00:00.000Z" }, @@ -131,6 +240,7 @@ describe("provider generation evidence", () => { sample(["gen-one", "gen-two"], {}, { reportRawSha256: "b".repeat(64) }), ], { apiKey: "fixture", + profile, fetchImpl: generationFetch(records), })).rejects.toThrow("does not match its receipt digest"); @@ -138,6 +248,7 @@ describe("provider generation evidence", () => { sample(["gen-one", "gen-two"], {}, { runId: "another-run" }), ], { apiKey: "fixture", + profile, fetchImpl: generationFetch(records), })).rejects.toThrow("does not match its receipt run identity"); }); diff --git a/bench/src/generation-evidence.ts b/bench/src/generation-evidence.ts index 50e5906..5577d73 100644 --- a/bench/src/generation-evidence.ts +++ b/bench/src/generation-evidence.ts @@ -11,12 +11,20 @@ const GENERATION_API = "https://openrouter.ai/api/v1/generation"; const MAX_ATTEMPTS = 5; const MAX_RETRY_MS = 5_000; const generationIdSchema = z.string().regex(/^gen-[A-Za-z0-9_-]+$/u); +const sha256Schema = z.string().regex(/^[0-9a-f]{64}$/u); +const modelIdSchema = z.string().trim().min(1).refine( + (value) => !/[\r\n]/u.test(value), + "model ID must not contain line breaks", +); const reportSchema = z.object({ summary: z.object({ runId: z.string().trim().min(1), ranAt: z.string().datetime({ offset: true }), model: z.string().trim().min(1), + scorerMode: z.enum(["disabled", "enabled"]), + scorerModel: z.string().trim().min(1).nullable(), + screeningProfileSha256: sha256Schema, upstreamProviderIdentity: z.string().trim().min(1), totalTokens: z.object({ prompt: z.number().int().nonnegative(), @@ -25,9 +33,57 @@ const reportSchema = z.object({ }), observedProviderCostUsdDecimal: z.string().regex(/^(?:0|[1-9][0-9]*|(?:0|[1-9][0-9]*)\.[0-9]*[1-9])$/u), providerGenerationIds: z.array(generationIdSchema).min(1), + }).superRefine((summary, context) => { + if ( + (summary.scorerMode === "disabled" && summary.scorerModel !== null) || + (summary.scorerMode === "enabled" && summary.scorerModel === null) + ) { + context.addIssue({ + code: "custom", + message: "scorerMode and scorerModel must agree", + path: ["scorerModel"], + }); + } }), }); +const screeningProfileSchema = z.object({ + generatorChain: z.array(modelIdSchema).min(1), + scorerChain: z.array(modelIdSchema), + providerGenerationModels: z.record(modelIdSchema, modelIdSchema).refine( + (models) => Object.keys(models).length > 0, + "providerGenerationModels must not be empty", + ), +}).passthrough().superRefine((profile, context) => { + const chainModels = [...new Set([...profile.generatorChain, ...profile.scorerChain])].sort(); + const mappedModels = Object.keys(profile.providerGenerationModels).sort(); + if ( + chainModels.length !== mappedModels.length || + chainModels.some((model, index) => model !== mappedModels[index]) + ) { + context.addIssue({ + code: "custom", + message: "providerGenerationModels must exactly cover the screening model chains", + path: ["providerGenerationModels"], + }); + } + const providerModels = Object.values(profile.providerGenerationModels); + if (new Set(providerModels).size !== providerModels.length) { + context.addIssue({ + code: "custom", + message: "providerGenerationModels must not repeat canonical models", + path: ["providerGenerationModels"], + }); + } + if (providerModels.some((model) => chainModels.includes(model))) { + context.addIssue({ + code: "custom", + message: "providerGenerationModels must be distinct from logical screening model IDs", + path: ["providerGenerationModels"], + }); + } +}); + const generationSchema = z.object({ data: z.object({ id: generationIdSchema, @@ -49,6 +105,20 @@ export interface GenerationEvidenceSample { reportRawSha256: string; } +export interface GenerationEvidenceProfile { + sha256: string; + providerGenerationModels: Readonly>; +} + +async function readGenerationEvidenceProfile(path: string): Promise { + const raw = await readFile(resolve(path)); + const profile = screeningProfileSchema.parse(JSON.parse(raw.toString("utf8"))); + return { + sha256: sha256(raw), + providerGenerationModels: profile.providerGenerationModels, + }; +} + function retryDelay(response: Response, attempt: number): number { const retryAfter = response.headers.get("retry-after")?.trim(); if (retryAfter !== undefined && /^\d+$/u.test(retryAfter)) { @@ -98,8 +168,21 @@ async function fetchGeneration( export async function verifyGenerationEvidence( samples: readonly GenerationEvidenceSample[], - options: { apiKey: string; fetchImpl?: typeof fetch; concurrency?: number }, + options: { + apiKey: string; + profile: GenerationEvidenceProfile; + fetchImpl?: typeof fetch; + concurrency?: number; + }, ): Promise { + const logicalModels = Object.keys(options.profile.providerGenerationModels); + const providerModels = Object.values(options.profile.providerGenerationModels); + if (new Set(providerModels).size !== providerModels.length) { + throw new Error("provider generation identities must not repeat canonical models"); + } + if (providerModels.some((model) => logicalModels.includes(model))) { + throw new Error("provider generation identities must be distinct from logical model IDs"); + } const parsed = samples.map((sample, sampleIndex) => { const report = reportSchema.parse(sample.report); const receipt = cohortReceiptSchema.parse(sample.receipt); @@ -112,6 +195,18 @@ export async function verifyGenerationEvidence( if (report.summary.runId !== receipt.runId) { throw new Error(`benchmark report ${sampleIndex + 1} does not match its receipt run identity`); } + if (report.summary.screeningProfileSha256 !== options.profile.sha256) { + throw new Error(`benchmark report ${sampleIndex + 1} does not match its screening profile`); + } + if (options.profile.providerGenerationModels[report.summary.model] === undefined) { + throw new Error(`benchmark report ${sampleIndex + 1} model has no pinned provider generation identity`); + } + if ( + report.summary.scorerModel !== null && + options.profile.providerGenerationModels[report.summary.scorerModel] === undefined + ) { + throw new Error(`benchmark report ${sampleIndex + 1} scorer has no pinned provider generation identity`); + } const startedAt = Date.parse(receipt.startedAt); const finishedAt = Date.parse(receipt.finishedAt); const ranAt = Date.parse(report.summary.ranAt); @@ -160,7 +255,13 @@ export async function verifyGenerationEvidence( const reportGenerations = generations.filter((_, generationIndex) => expected[generationIndex]!.reportIndex === reportIndex ); - if (reportGenerations.some((generation) => generation.model !== report.summary.model)) { + const expectedGenerationModels = new Set([ + options.profile.providerGenerationModels[report.summary.model]!, + ...(report.summary.scorerModel === null + ? [] + : [options.profile.providerGenerationModels[report.summary.scorerModel]!]), + ]); + if (reportGenerations.some((generation) => !expectedGenerationModels.has(generation.model))) { throw new Error(`benchmark report ${reportIndex + 1} contains a generation for another model`); } if (reportGenerations.some((generation) => @@ -195,10 +296,15 @@ function requiredValue(args: readonly string[], index: number, flag: string): st async function main(): Promise { const args = process.argv.slice(2); + if (args[0] !== "--screen-profile") { + throw new Error("generation-evidence verification requires --screen-profile first"); + } + const profilePath = requiredValue(args, 0, "--screen-profile"); + const sampleArgs = args.slice(2); const samplePaths: Array<{ reportPath: string; receiptPath: string }> = []; - for (let index = 0; index < args.length; index += 4) { - const resultFlag = args[index]; - const receiptFlag = args[index + 2]; + for (let index = 0; index < sampleArgs.length; index += 4) { + const resultFlag = sampleArgs[index]; + const receiptFlag = sampleArgs[index + 2]; if (resultFlag !== "--result") { throw new Error(`expected --result, received ${resultFlag ?? "end of arguments"}`); } @@ -206,8 +312,8 @@ async function main(): Promise { throw new Error(`each --result must be followed by --receipt`); } samplePaths.push({ - reportPath: resolve(requiredValue(args, index, resultFlag)), - receiptPath: resolve(requiredValue(args, index + 2, receiptFlag)), + reportPath: resolve(requiredValue(sampleArgs, index, resultFlag)), + receiptPath: resolve(requiredValue(sampleArgs, index + 2, receiptFlag)), }); } if (samplePaths.length === 0) { @@ -218,18 +324,21 @@ async function main(): Promise { throw new Error(`generation-evidence verification requires ${API_KEY_ENV_NAMES_TEXT}`); } const apiKey = process.env[keyName]!; - const samples = await Promise.all(samplePaths.map(async ({ reportPath, receiptPath }) => { - const [reportRaw, receiptRaw] = await Promise.all([ - readFile(reportPath), - readFile(receiptPath, "utf8"), - ]); - return { - report: JSON.parse(reportRaw.toString("utf8")) as unknown, - receipt: JSON.parse(receiptRaw) as unknown, - reportRawSha256: sha256(reportRaw), - }; - })); - const count = await verifyGenerationEvidence(samples, { apiKey }); + const [profile, samples] = await Promise.all([ + readGenerationEvidenceProfile(profilePath), + Promise.all(samplePaths.map(async ({ reportPath, receiptPath }) => { + const [reportRaw, receiptRaw] = await Promise.all([ + readFile(reportPath), + readFile(receiptPath, "utf8"), + ]); + return { + report: JSON.parse(reportRaw.toString("utf8")) as unknown, + receipt: JSON.parse(receiptRaw) as unknown, + reportRawSha256: sha256(reportRaw), + }; + })), + ]); + const count = await verifyGenerationEvidence(samples, { apiKey, profile }); console.log(`Verified ${count} distinct OpenRouter generations.`); } diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index aa90fcc..06b446d 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -1377,7 +1377,21 @@ describe("managed admission workflow", () => { expect(calibration).toContain("name: Record the attested baseline"); expect(calibration).toContain("name: Attest the populated baseline"); expect(calibration).toContain("name: Verify independent calibration generations"); - expect(calibration).toContain("bun run bench:verify-generations --"); + const generationVerificationCommand = (workflow: string): string => { + const commandStart = workflow.indexOf("bun run bench:verify-generations --"); + expect(commandStart).toBeGreaterThanOrEqual(0); + const commandLines: string[] = []; + for (const line of workflow.slice(commandStart).split("\n")) { + commandLines.push(line); + if (!line.trimEnd().endsWith("\\")) break; + } + return commandLines.join("\n"); + }; + const calibrationGenerationVerification = generationVerificationCommand(calibration); + expect(calibrationGenerationVerification).toMatch( + /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models\.json \\\n\s+--result /u, + ); + expect([...calibrationGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models\.json/gu)]).toHaveLength(1); expect(calibration).toContain("--record"); expect(release).not.toContain("workflow_dispatch"); expect(release).toContain("name: Require the unique first release run for this tag"); @@ -1484,7 +1498,11 @@ describe("managed admission workflow", () => { expect(final).toContain("merge-multiple: false"); expect(final).toContain("name: Verify signed release benchmark evidence"); expect(final).toContain("name: Verify independent release generations"); - expect(final).toContain("bun run bench:verify-generations --"); + const releaseGenerationVerification = generationVerificationCommand(final); + expect(releaseGenerationVerification).toMatch( + /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models\.json \\\n\s+--result /u, + ); + expect([...releaseGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models\.json/gu)]).toHaveLength(1); expect(final).toContain("gh attestation verify"); expect(final).toContain("--deny-self-hosted-runners"); expect(final).toContain("name: bench-live-cohort-${{ github.run_attempt }}"); diff --git a/bench/src/verify-admission.test.ts b/bench/src/verify-admission.test.ts index 66f4da8..c97436f 100644 --- a/bench/src/verify-admission.test.ts +++ b/bench/src/verify-admission.test.ts @@ -49,10 +49,14 @@ describe("admission attestation verification", () => { const profile = JSON.parse(await readFile(join(repositoryRoot, "provisional-models.json"), "utf8")) as { generatorChain: string[]; scorerChain: string[]; + providerGenerationModels: Record; modelPriceBounds: Array<{ model: string }>; }; expect(profile.generatorChain).toEqual(["openai/gpt-5.6-luna"]); expect(profile.scorerChain).toEqual(["openai/gpt-5.6-luna"]); + expect(profile.providerGenerationModels).toEqual({ + "openai/gpt-5.6-luna": "openai/gpt-5.6-luna-20260709", + }); expect(profile.modelPriceBounds.map((bound) => bound.model)).toEqual(["openai/gpt-5.6-luna"]); expect(await verifyProvisionalRelease( join(repositoryRoot, "qualified-models.json"), @@ -110,6 +114,44 @@ describe("admission attestation verification", () => { } }); + test("requires one pinned provider generation identity per provisional model", async () => { + const repositoryRoot = join(import.meta.dir, "..", ".."); + const directory = await temporaryDirectory(); + const manifest = join(directory, "qualified-models.json"); + const config = join(directory, "config.toml"); + const profilePath = join(directory, "provisional-models.json"); + await writeFile(manifest, await readFile(join(repositoryRoot, "qualified-models.json"))); + await writeFile(config, await readFile(join(repositoryRoot, "config.toml"))); + const profile = JSON.parse( + await readFile(join(repositoryRoot, "provisional-models.json"), "utf8"), + ) as { providerGenerationModels: Record }; + + profile.providerGenerationModels = {}; + await writeFile(profilePath, JSON.stringify(profile)); + await expect(verifyProvisionalRelease(manifest, config, profilePath)).rejects.toThrow( + "generation identities must exactly cover sorted model chains", + ); + }); + + test("rejects logical aliases as provider generation identities", async () => { + const repositoryRoot = join(import.meta.dir, "..", ".."); + const directory = await temporaryDirectory(); + const manifest = join(directory, "qualified-models.json"); + const config = join(directory, "config.toml"); + const profilePath = join(directory, "provisional-models.json"); + await writeFile(manifest, await readFile(join(repositoryRoot, "qualified-models.json"))); + await writeFile(config, await readFile(join(repositoryRoot, "config.toml"))); + const profile = JSON.parse( + await readFile(join(repositoryRoot, "provisional-models.json"), "utf8"), + ) as { providerGenerationModels: Record }; + + profile.providerGenerationModels["openai/gpt-5.6-luna"] = "openai/gpt-5.6-luna"; + await writeFile(profilePath, JSON.stringify(profile)); + await expect(verifyProvisionalRelease(manifest, config, profilePath)).rejects.toThrow( + "generation identities must be distinct from logical model IDs", + ); + }); + test("keeps attestation verification active after a profile is formally admitted", async () => { const directory = await temporaryDirectory(); const manifest = join(directory, "qualified-models.json"); @@ -150,6 +192,7 @@ describe("admission attestation verification", () => { await readFile(join(repositoryRoot, "provisional-models.json"), "utf8"), ) as { generatorChain: string[]; + providerGenerationModels: Record; modelPriceBounds: Array<{ model: string; inputMicrosPerMillionTokens: number; @@ -157,6 +200,7 @@ describe("admission attestation verification", () => { }>; }; altered.generatorChain = ["other/model"]; + altered.providerGenerationModels["other/model"] = "other/model-20260801"; // Keep both role models fully priced so the assertion reaches the exact // embedded-default comparison instead of failing an earlier bound check. altered.modelPriceBounds.push({ ...altered.modelPriceBounds[0]!, model: "other/model" }); diff --git a/bench/src/verify-admission.ts b/bench/src/verify-admission.ts index dd264a1..512b8c3 100644 --- a/bench/src/verify-admission.ts +++ b/bench/src/verify-admission.ts @@ -72,6 +72,7 @@ const provisionalProfileSchema = z.object({ generatorChain: z.array(boundedIdentifierSchema).min(1), consensus: z.number().int().positive(), scorerChain: z.array(boundedIdentifierSchema).max(2), + providerGenerationModels: z.record(boundedIdentifierSchema, boundedIdentifierSchema), modelPriceBounds: z.array(z.object({ model: boundedIdentifierSchema, inputMicrosPerMillionTokens: z.number().int().positive().safe(), @@ -235,6 +236,21 @@ export async function verifyProvisionalRelease( throw new Error("provisional hosted scorer chain must not repeat models"); } const expectedModels = [...new Set([...profile.generatorChain, ...profile.scorerChain])].sort(); + const generationModels = Object.keys(profile.providerGenerationModels).sort(); + if ( + generationModels.length !== expectedModels.length || + generationModels.some((model, index) => model !== expectedModels[index]) + ) { + throw new Error("provisional hosted generation identities must exactly cover sorted model chains"); + } + if (new Set(Object.values(profile.providerGenerationModels)).size !== generationModels.length) { + throw new Error("provisional hosted generation identities must not repeat canonical models"); + } + if (Object.values(profile.providerGenerationModels).some((model) => expectedModels.includes(model))) { + throw new Error( + "provisional hosted generation identities must be distinct from logical model IDs", + ); + } const boundedModels = profile.modelPriceBounds.map((bound) => bound.model); if ( boundedModels.length !== expectedModels.length || diff --git a/provisional-models.json b/provisional-models.json index 7782159..f1fa9fc 100644 --- a/provisional-models.json +++ b/provisional-models.json @@ -7,6 +7,9 @@ "generatorChain": ["openai/gpt-5.6-luna"], "consensus": 1, "scorerChain": ["openai/gpt-5.6-luna"], + "providerGenerationModels": { + "openai/gpt-5.6-luna": "openai/gpt-5.6-luna-20260709" + }, "modelPriceBounds": [ { "model": "openai/gpt-5.6-luna", diff --git a/src/config.rs b/src/config.rs index a040d00..0644db6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -260,6 +260,8 @@ pub(crate) struct QualificationCandidateProfile { pub generator_chain: Vec, pub consensus: usize, pub scorer_chain: Vec, + #[serde(default)] + pub provider_generation_models: std::collections::BTreeMap, pub model_price_bounds: Vec, } @@ -617,6 +619,40 @@ fn validate_profile_model_price_bounds( Ok(()) } +fn validate_profile_generation_models( + label: &str, + generator_chain: &[String], + scorer_chain: &[String], + provider_generation_models: &std::collections::BTreeMap, +) -> Result<()> { + let expected_models = generator_chain + .iter() + .chain(scorer_chain) + .cloned() + .collect::>(); + let mapped_models = provider_generation_models + .keys() + .cloned() + .collect::>(); + anyhow::ensure!( + mapped_models == expected_models, + "{label} provider generation identities must exactly cover the generator and scorer models" + ); + let mut canonical_models = std::collections::BTreeSet::new(); + for model in provider_generation_models.values() { + validate_model_id(&format!("{label} provider generation identity"), model)?; + anyhow::ensure!( + !expected_models.contains(model), + "{label} provider generation identities must be distinct from logical model IDs" + ); + anyhow::ensure!( + canonical_models.insert(model), + "{label} provider generation identities must not repeat canonical models" + ); + } + Ok(()) +} + fn parse_qualification_manifest(raw: &str) -> Result { let manifest: QualificationManifest = serde_json::from_str(raw)?; anyhow::ensure!( @@ -1901,6 +1937,12 @@ fn benchmark_screening_profile() -> Result profile.scorer_chain.len() <= 2, "benchmark screening scorer chain supports at most two models" ); + validate_profile_generation_models( + "benchmark screening profile", + &profile.generator_chain, + &profile.scorer_chain, + &profile.provider_generation_models, + )?; validate_profile_model_price_bounds( "benchmark screening profile", &profile.generator_chain, @@ -2171,6 +2213,12 @@ fn parse_provisional_hosted_profile(raw: &str) -> Result Date: Thu, 27 Aug 2026 02:09:02 +0000 Subject: [PATCH 2/2] Preserve portable OpenRouter output limits --- src/llm.rs | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/llm.rs b/src/llm.rs index 4a488ad..2993b37 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -5332,7 +5332,10 @@ fn apply_openrouter_scorer_contract( // Strict scorer routing requires every request parameter to be supported // by the selected endpoint. Reasoning endpoints expose their output limit // through `reasoning.effort`. Azure names the output limit - // `max_completion_tokens`; OpenAI names the same limit `max_tokens`. + // `max_completion_tokens`; portable OpenRouter routes and OpenAI use + // `max_tokens`. Only rewrite the field for an explicitly pinned Azure + // route, because `require_parameters` otherwise removes endpoints that + // correctly advertise only the portable parameter. // A redundant temperature would disqualify both endpoint families. apply_openrouter_strict_output_limit(body, "scorer", pinned_provider); let provider = body @@ -5426,7 +5429,7 @@ fn apply_openrouter_strict_output_limit( .as_object_mut() .expect("model request body is an object"); request.remove("temperature"); - if pinned_provider != Some("OpenAI") { + if pinned_provider == Some("Azure") { let max_tokens = request.remove("max_tokens").unwrap_or_else(|| { panic!("OpenRouter {phase} request has a bounded output token limit") }); @@ -9821,8 +9824,8 @@ mod tests { json!({"effort": "low", "exclude": true}) ); assert!(scorer.get("temperature").is_none()); - assert!(scorer.get("max_tokens").is_none()); - assert_eq!(scorer["max_completion_tokens"], 400); + assert_eq!(scorer["max_tokens"], 400); + assert!(scorer.get("max_completion_tokens").is_none()); assert!(scorer["reasoning"].get("enabled").is_none()); assert_eq!(scorer["provider"]["require_parameters"], true); assert_eq!(scorer["response_format"]["type"], "json_schema"); @@ -9861,8 +9864,8 @@ mod tests { LlmPhase::Adjudication, ); assert!(adjudication.get("temperature").is_none()); - assert!(adjudication.get("max_tokens").is_none()); - assert_eq!(adjudication["max_completion_tokens"], 8_000); + assert_eq!(adjudication["max_tokens"], 8_000); + assert!(adjudication.get("max_completion_tokens").is_none()); assert_eq!(adjudication["provider"]["require_parameters"], true); assert_eq!( adjudication["reasoning"], @@ -9870,6 +9873,29 @@ mod tests { ); assert!(adjudication.get("response_format").is_none()); + let mut azure_client = client.clone(); + azure_client.request_decorations.pinned_upstream_provider = Some("Azure".into()); + let azure_scorer = azure_client.request_body( + "provider/scorer", + "system", + "user", + 400, + 0.0, + LlmPhase::Scorer { expected_len: 1 }, + ); + assert!(azure_scorer.get("max_tokens").is_none()); + assert_eq!(azure_scorer["max_completion_tokens"], 400); + let azure_adjudication = azure_client.request_body( + "provider/scorer", + "system", + "user", + 8_000, + 0.0, + LlmPhase::Adjudication, + ); + assert!(azure_adjudication.get("max_tokens").is_none()); + assert_eq!(azure_adjudication["max_completion_tokens"], 8_000); + let mut openai_client = client.clone(); openai_client.request_decorations.pinned_upstream_provider = Some("OpenAI".into()); let openai_scorer = openai_client.request_body(