From 7f542b604d7ab708b965f24b10028aa99cd05105 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 3 Sep 2026 02:38:49 -0700 Subject: [PATCH 1/2] fix(judge-families): Anthropic's bare aliases sonnet, opus, haiku classify as anthropic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `claude --model sonnet` is Anthropic's documented alias for the current Sonnet, and the answer is claude-sonnet-5 by definition of the alias. judgeFamily gave the bare alias no family, so assertServedModel refused `sonnet → claude-sonnet-5` as a cross-family substitution. Measured 2026-09-03 09:21Z in discovery-lab: the first two claude-code leads requested `sonnet` and failed to grade on ModelSubstitutionError. A name pattern for the three bare aliases, anchored at the start and bounded so `sonnetful-model` stays unknown. Tests: audit-primitives 19/19. --- src/judge-families.ts | 4 ++++ tests/audit-primitives.test.ts | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/judge-families.ts b/src/judge-families.ts index dd312cd1..4eb8c5cc 100644 --- a/src/judge-families.ts +++ b/src/judge-families.ts @@ -56,6 +56,10 @@ const PROVIDER_PREFIX: Record = { /** Fallback model-name patterns when there's no recognised provider prefix. */ const NAME_PATTERNS: Array<[RegExp, JudgeFamily]> = [ [/claude/i, 'anthropic'], + // Anthropic's own bare aliases (`claude --model sonnet|opus|haiku`). A run that requested `sonnet` + // and was served claude-sonnet-5 is an alias resolution, not a cross-family substitution: without + // this row the alias classified as `unknown` and assertServedModel refused the pair. + [/^(?:sonnet|opus|haiku)(?:[-@\s]|$)/i, 'anthropic'], [/\b(gpt|davinci|babbage)\b|^o[134]\b|[-/]o[134]\b|gpt-/i, 'openai'], [/gemini|palm|gemma|bison/i, 'google'], [/llama/i, 'meta'], diff --git a/tests/audit-primitives.test.ts b/tests/audit-primitives.test.ts index 1a8ec035..db56703b 100644 --- a/tests/audit-primitives.test.ts +++ b/tests/audit-primitives.test.ts @@ -62,6 +62,11 @@ describe('judgeFamily + assertCrossFamily', () => { expect(judgeFamily('anthropic/claude-opus-4-5-20251101')).toBe('anthropic') expect(judgeFamily('openai/gpt-4o')).toBe('openai') expect(judgeFamily('claude-sonnet-4-6@2026-05-08')).toBe('anthropic') + // Anthropic's bare CLI aliases: an alias resolution is within-family, never a cross-family substitution. + expect(judgeFamily('sonnet')).toBe('anthropic') + expect(judgeFamily('opus')).toBe('anthropic') + expect(judgeFamily('haiku@2026-09-03')).toBe('anthropic') + expect(judgeFamily('sonnetful-model')).toBe('unknown') expect(judgeFamily('gemini-2.5-pro')).toBe('google') expect(judgeFamily('meta-llama/llama-3.3-70b')).toBe('meta') expect(judgeFamily('some-internal-model')).toBe('unknown') From 0d5316a4ee3036bf79911f93431163bae01015cb Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 3 Sep 2026 02:44:18 -0700 Subject: [PATCH 2/2] chore(analyst): re-pin the benchmark implementation digest after the judge-families change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit judge-families.ts is a bound source of the public analyst benchmark implementation; the alias pattern changes its bytes, so the pin moves from fffa4858… to 320fe06f… by intent. --- src/analyst/benchmark-implementation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index a66c1805..1be6e472 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -139,7 +139,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - 'fffa485899aa8f8e95213f56fb2aaefd2a244874165cee4e7dc09b81b872b94e' + '320fe06f563021515f9b1ed73a4d1fd555266f29b3c32a8cecd2fc13679e6655' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256