diff --git a/src/checks.ts b/src/checks.ts index 7b2b79b..de77a6d 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -31,8 +31,19 @@ import { resourceContentOf, externalUrlOf, isMcpUiMime, + SUITE_ROLE_PREFIX, type ServerJsonClaims, } from './discovery.js' +// The card-declared test-suite gates and the shared expectation engine — the +// SAME code the observe side ran, so what was fetched and what it means cannot +// drift. `judgeExpect` in particular is imported, never re-implemented: a +// second expectation judge would make conformance depend on which door asked. +import { + MAX_SUITE_REQUIREMENTS, + gateTestSuiteCard, + gateTestSuiteDocument, +} from './test-suite.js' +import { captureInto, judgeExpect, resolveEndpoint, type Bindings } from './expect.js' import { isPubliclyRoutableSameOrigin, isPublicHttpsOffOriginAllowed } from './http.js' import { validateSchema } from './schema.js' import { @@ -876,6 +887,104 @@ export function runChecks(bundle: EvidenceBundle): CheckResult[] { })) } + // ── published-test-suite (OPTIONAL, DECLARATION-ARMED) ──────────────────── + // The second optional interface, and the owner's motivating case: a + // service COULD publish tests for more complex workflows, but a simple + // CRUD or lookup API does not need that complexity and must not be + // penalised for omitting them. So the check is armed by the card's OWN key + // (`interfaces.testSuite`), NOT by the AXP opt-in signal, and a card that + // omits the key SKIPs — omission is conformance. + // + // WHAT IS VERIFIED — deliberately tier 3, "run it". A check that only + // confirmed a file exists and hashes correctly would verify nothing about + // the workflows this interface exists for, and a mechanism tested only on + // the happy path is a mechanism nobody tested. The suite was EXECUTED + // during observe (see observeTestSuite) under a tightened boundary; this + // judge re-derives the identical requirement list and binding scope from + // the bundle and re-judges the recorded exchanges with the SHARED + // `judgeExpect` — so a replay reaches the same verdict without refetching. + // + // axItem is undefined — an additive readiness dimension that moves no AX + // point, exactly like digital-link-resolver. + { + const claim = agents.testSuite + const evidence: string[] = [ROLE.agentsJson, ROLE.testSuite] + let result: { verdict: Verdict; detail: string } + + if (!claim) { + result = { + verdict: 'skip', + detail: + 'no published test suite interface declared (agents.json `interfaces.testSuite` absent) — the interface is OPTIONAL and this card does not claim it, so nothing was fetched and nothing is judged; under a pinned must:pass this fails closed', + } + } else { + const cardGate = gateTestSuiteCard(claim, bundle.target) + if (!cardGate.ok) { + result = { verdict: 'fail', detail: cardGate.problem } + } else { + const docEv = findEvidence(bundle, ROLE.testSuite) + if (!ok(docEv) || docEv?.body == null) { + result = { + verdict: 'fail', + detail: + `GET ${cardGate.url} did not answer 2xx with a body — ${!docEv ? 'not fetched' : docEv.status === null ? `fetch failed (${docEv.error ?? 'unknown'})` : `status ${docEv.status}`}. ` + + 'The card DECLARES a published test suite, so the suite document must be served.', + } + } else { + const plan = gateTestSuiteDocument(claim, docEv.body, cardGate.digest) + if (!plan.ok) { + result = { verdict: 'fail', detail: plan.problems.slice(0, 6).join('; ') } + } else { + // Re-derive the run PURELY from the bundle: same env vars, same + // requirement order, same capture-on-pass gate the observe side + // used, so the two scopes are identical by construction. + const bindings: Bindings = { ...plan.vars } + const problems: string[] = [] + const pathnames = new Set() + for (const req of plan.requirements) { + const resolved = resolveEndpoint(req, bundle.target, bindings) + if (!resolved.ok) { + problems.push(`"${req.id}": ${resolved.detail}`) + continue + } + const role = `${SUITE_ROLE_PREFIX}pinned:${req.id}` + evidence.push(role) + const ev = bundle.items.find((e) => e.role === role) + const ps = judgeExpect(ev, resolved.expect) + if (ps.length === 0) { + try { pathnames.add(new URL(resolved.url).pathname) } catch { /* resolved urls parse */ } + if (req.capture) captureInto(bindings, req.capture, ev) + } else { + problems.push(`"${req.id}" ${resolved.method} ${resolved.url}: ${ps.join('; ')}`) + } + } + result = + problems.length === 0 + ? pass( + `interfaces.testSuite declared; ${cardGate.url} → ${docEv!.status}, ${cardGate.digest.slice(0, 19)}… matches the card pin; ` + + `suite "${plan.suite.name}"@${plan.suite.version}, environment "${claim.environment}", ` + + `${plan.requirements.length} requirement(s) over ${pathnames.size} distinct pathname(s), all passed. ` + + `Run GET/HEAD-only with writes disabled, budget <=${MAX_SUITE_REQUIREMENTS}, target pinned to the card origin. ` + + 'NOT judged: whether the suite is ambitious — api.qa verifies the surface keeps its OWN published promise, ' + + 'not that the promise is demanding.', + ) + : { + verdict: 'fail', + detail: + `the surface violated its OWN published suite "${plan.suite.name}"@${plan.suite.version} ` + + `(${cardGate.digest.slice(0, 19)}…, environment "${claim.environment}"): ` + + problems.slice(0, 6).join('; '), + } + } + } + } + } + + checks.push(check('published-test-suite', + 'a DECLARED test-suite interface publishes a digest-pinned suite the surface actually passes', undefined, + evidence, result)) + } + // ── AXP structural checks (Clause 3 conneg + Clause 6 cross-linking) ────── // Discriminating checks the AXP pinned spec binds via kind:'check': // machine-legible-home, conneg-accept, conneg-client-class, diff --git a/src/discovery.ts b/src/discovery.ts index 7d6471a..e3e3a31 100644 --- a/src/discovery.ts +++ b/src/discovery.ts @@ -11,6 +11,16 @@ import { Observer, isPubliclyRoutableSameOrigin, isPublicHttpsOffOriginAllowed } from './http.js' import { GS1_RESOLVER_WELL_KNOWN_PATH } from './gs1-resolver.js' import { canonicalJson, sha256Hex, sampleSeeded } from './digest.js' +// Card-declared test-suite gates + the shared expectation engine. Both are the +// SAME code the judge (checks.ts) runs, so what is fetched and what it means +// can never drift apart. +import { + MAX_SUITE_REQUIREMENTS, + SUITE_DEADLINE_MS, + gateTestSuiteCard, + gateTestSuiteDocument, +} from './test-suite.js' +import { captureInto, judgeExpect, resolveEndpoint, type Bindings } from './expect.js' import type { ClaimedEndpoint, DiscoveryReport, @@ -157,8 +167,32 @@ export const ROLE = { * and the digital-link-resolver check SKIPs. */ gs1Resolver: 'surface:gs1resolver', + // ── Published test suite (OPTIONAL, target-DECLARED) ───────────────────── + /** + * GET of the card-declared Suite DOCUMENT (`interfaces.testSuite.url`). + * Same-origin SSRF-gated. Recorded ONLY when the card DECLARES the interface: + * a card that omits `interfaces.testSuite` records nothing, spends no budget, + * and the published-test-suite check SKIPs. + */ + testSuite: 'surface:testsuite', } as const +/** + * ⚠ ROLE-COLLISION GUARD for a card-declared suite run. + * + * A suite sub-run naturally records its requirements under `pinned:` and + * `pinned::` — which are the PARENT's role keys for the admission spec's + * own requirements. Merged unprefixed, `items.find(e => e.role === 'pinned:x')` + * would resolve the WRONG evidence item: exactly the class of bug + * `validateRequirements`' derived-role-collision guard exists to prevent, but + * arriving through a door that guard does not watch (it validates ONE + * requirement list, and these are two). + * + * So every merged role is prefixed, yielding `suite:pinned::`, and the + * merge ASSERTS that no prefixed role already exists in the parent. + */ +export const SUITE_ROLE_PREFIX = 'suite:' + export function findEvidence(bundle: EvidenceBundle, role: string): Evidence | undefined { return bundle.items.find((e) => e.role === role) } @@ -298,6 +332,19 @@ export interface AgentsClaims { * the verification. */ digitalLink?: DigitalLinkClaim + /** + * The target-declared PUBLISHED TEST SUITE interface (`interfaces.testSuite`) + * — an OPTIONAL capability, armed by its own card key exactly like + * `interfaces.digitalLink`. PRESENT ⇒ the card CLAIMS it publishes a + * digest-pinned api.qa Suite describing workflows it holds itself to, and the + * published-test-suite check is ARMED; ABSENT ⇒ the field is undefined, + * nothing is fetched, nothing is run, and that check SKIPs. + * + * A simple CRUD or lookup API that publishes no suite is FULLY CONFORMANT — + * that is the entire point of the interface being optional. Declaring it is + * what invites the verification. + */ + testSuite?: TestSuiteClaim } /** @@ -331,6 +378,63 @@ export interface DigitalLinkClaim { resolverRoot?: string } +/** + * A card's `interfaces.testSuite` declaration, as parsed off the wire. + * + * Mirrors `DigitalLinkClaim` deliberately, `malformed` included: the key being + * PRESENT but not a plain object is a DEFECTIVE claim, never an absence. A card + * meaning "I publish no test suite" OMITS the key — that is fully conforming. + * Collapsing a present-but-defective value to "not declared" would hand a + * target a free skip for declaring the interface in a shape no verifier can + * check, which is the evasion this whole mechanism exists to prevent. + * + * WHAT IT POINTS AT: a published, digest-pinned api.qa **Suite** document — the + * artifact that already exists (`types.ts` `Suite`, `parseSuite`, + * `verifySuite`) — NOT an npm tarball of TypeScript/vitest tests. That + * rejection is load-bearing: api.qa would have to EXECUTE a stranger's code to + * reach a verdict, which would end the determinism contract (the verdict must + * be a pure function of an EvidenceBundle), make replay impossible, and let the + * judged party supply the judging code. A declarative Suite already expresses + * multi-step workflows — ordered requirements, `capture` chaining, `{{var}}` + * interpolation, named environments — and is already vitest-runnable BY THE + * TARGET in the target's own CI via `autonomous-qa/vitest`. The declarative + * artifact is the interoperable one; vitest is a runner for it, not a wire + * format. + */ +export interface TestSuiteClaim { + /** The `interfaces.testSuite` key was present on the card. Always true. */ + declared: true + /** Present but not a plain JSON object — a defective declaration. */ + malformed?: boolean + /** `typeof`-style name of the malformed value, for the failure message. */ + malformedAs?: string + /** Suite location exactly as the card wrote it (relative or absolute). */ + urlRaw?: string + /** + * Suite location to fetch, absolutized against the target origin. An ABSOLUTE + * off-origin value is PRESERVED verbatim (never rewritten to the target) so + * the same-origin gate DROPS it and the check FAILS the card for publishing + * another origin's suite. + */ + url: string + /** + * `"sha256:<64 lowercase hex>"` over the suite document's exact bytes. + * REQUIRED — see the check for why a suite pin is not optional the way a + * Digital Link description file needs none. + */ + digest?: string + /** Environment name to select. Defaults to `"public"`. */ + environment: string + /** Suite dialect. Only `"api.qa/suite@1"` is defined. */ + runner: string +} + +/** The only suite dialect this verifier implements. */ +export const TEST_SUITE_RUNNER = 'api.qa/suite@1' + +/** Default environment selected when the card names none. */ +export const TEST_SUITE_DEFAULT_ENVIRONMENT = 'public' + export function parseAgentsJson(doc: unknown, origin: string): AgentsClaims { const out: AgentsClaims = { endpoints: [] } if (!doc || typeof doc !== 'object') return out @@ -417,6 +521,46 @@ export function parseAgentsJson(doc: unknown, origin: string): AgentsClaims { } } + // Published test-suite face (OPTIONAL, declaration-armed): + // interfaces.testSuite.{url,digest,environment,runner}. PRESENCE of the key + // is the whole declaration signal — `'testSuite' in interfaces`, not + // truthiness — so a present-but-defective value is recorded as MALFORMED and + // failed, never silently treated as absent. `url` is card-derived and + // therefore adversarial: absolutize it (a relative "/.well-known/axp/ + // suite.json" resolves same-origin; an absolute foreign url is PRESERVED so + // the same-origin gate downstream drops it and the check fails the card for + // publishing another origin's suite), exactly like interfaces.digitalLink. + if (Object.prototype.hasOwnProperty.call(interfaces, 'testSuite')) { + const ts = interfaces.testSuite + if (ts !== null && typeof ts === 'object' && !Array.isArray(ts)) { + const t = ts as Record + const urlRaw = str(t.url) + const digest = str(t.digest) + const environment = str(t.environment) + const runner = str(t.runner) + out.testSuite = { + declared: true, + ...(urlRaw !== undefined && { urlRaw }), + // A card that declares the interface with NO url has declared nothing + // fetchable. Absolutizing '' yields the origin root, which the check + // reports as a missing url rather than silently GETting `/`. + url: urlRaw !== undefined ? absolutize(urlRaw, origin) : '', + ...(digest !== undefined && { digest }), + environment: environment ?? TEST_SUITE_DEFAULT_ENVIRONMENT, + runner: runner ?? TEST_SUITE_RUNNER, + } + } else { + out.testSuite = { + declared: true, + malformed: true, + malformedAs: ts === null ? 'null' : Array.isArray(ts) ? 'array' : typeof ts, + url: '', + environment: TEST_SUITE_DEFAULT_ENVIRONMENT, + runner: TEST_SUITE_RUNNER, + } + } + } + // Top-level server.json pointer (card-derived; absolutized, SSRF-gated at fetch). const declaredServerJson = str(d.serverJson) ?? str(d.server_json) if (out.serverJsonUrl === undefined && declaredServerJson !== undefined) { @@ -1277,6 +1421,96 @@ async function observeDigitalLink(origin: string, observer: Observer): Promise { + const items = observer.items + const bundleView: EvidenceBundle = { target: origin, fetchedAt: '', seed: 0, items } + const agents = parseAgentsJson(parseJsonBody(findEvidence(bundleView, ROLE.agentsJson)), origin) + const claim = agents.testSuite + if (!claim) return // interface not declared — optional, nothing to verify + + // Card-only gate. A malformed declaration, a missing/bad digest, an unknown + // runner or an off-origin url is judged from the card alone and never + // fetched; the check reports it from the same shared gate. + const gate = gateTestSuiteCard(claim, origin) + if (!gate.ok) return + + const docEv = await observer.observe(ROLE.testSuite, gate.url, { accept: 'application/json' }) + if (docEv.status === null || docEv.status < 200 || docEv.status >= 300 || docEv.body == null) return + + const plan = gateTestSuiteDocument(claim, docEv.body, gate.digest) + if (!plan.ok) return // digest mismatch / ineligible shape — judged from the bundle, never run + + // A PRIVATE observer for the sub-run. Three tightenings over the parent: + // read-only (a stranger's document may not direct a write), its own small + // budget (a long suite cannot drain the parent's politeness budget out from + // under the fixed probes), and `allowPrivate` inherited so a consented local + // target still works in dev without ever loosening the deployed posture. + const sub = observer.child({ allowWrites: false, budget: MAX_SUITE_REQUIREMENTS }) + + // The binding scope starts from the SELECTED ENVIRONMENT's vars, exactly as a + // `verifySuite` run does. A `baseUrl` among them cannot steer the run: the + // target is `origin`, and `resolveEndpoint` re-gates every resolved URL + // same-origin, so an off-origin interpolation fails closed WITHOUT a fetch. + const bindings: Bindings = { ...plan.vars } + const deadline = Date.now() + SUITE_DEADLINE_MS + for (const req of plan.requirements) { + if (Date.now() > deadline) break // the judge fails the run for the missing evidence + const resolved = resolveEndpoint(req, origin, bindings) + if (!resolved.ok) continue // judge re-derives the identical resolution failure + if (!isPubliclyRoutableSameOrigin(resolved.url, origin)) continue + // Belt-and-suspenders over the Observer's own read-only refusal and the + // document gate's declared-method check: never issue a non-safe method. + if (resolved.method !== 'GET' && resolved.method !== 'HEAD') continue + const ev = await sub.observe(`pinned:${req.id}`, resolved.url, { + method: resolved.method, + accept: 'application/json', + }) + if (req.capture && judgeExpect(ev, resolved.expect).length === 0) { + captureInto(bindings, req.capture, ev) + } + } + + // ⚠ MERGE UNDER A PREFIX. The sub-run recorded `pinned:` — which is the + // PARENT's role-key space for the admission spec's own requirements. Merged + // raw, `find(role === 'pinned:x')` would resolve the wrong evidence item. + // Prefix every merged role, and REFUSE (loudly) if a prefixed role somehow + // already exists in the parent rather than silently shadowing it. + for (const ev of sub.items) { + const role = `${SUITE_ROLE_PREFIX}${ev.role}` + if (items.some((e) => e.role === role)) { + throw new Error( + `card-declared suite evidence role "${role}" already exists in the parent bundle — refusing to shadow it. ` + + 'Suite evidence is namespaced under "suite:" precisely so it cannot collide with the admission run.', + ) + } + items.push({ ...ev, role }) + } +} + export async function observeTarget(origin: string, observer: Observer, seed: number): Promise { // 1. The fixed surface plan — identical for every target (no fingerprint). const rootAgentEv = await observer.observe(ROLE.rootAgent, `${origin}/`, { accept: '*/*' }) @@ -1561,6 +1795,17 @@ export async function observeTarget(origin: string, observer: Observer, seed: nu // by an endpoint-rich target — the same priority rule as 4b/4c/4d. await observeDigitalLink(origin, observer) + // 4f. Published test suite (OPTIONAL, target-declared): when the card + // declares `interfaces.testSuite`, GET the digest-pinned Suite document + // and RUN it — GET/HEAD only, on a private budget, against this origin — + // so the published-test-suite judge can hold the surface to the workflows + // it published about itself. Zero-overhead and zero-budget for the common + // case (no declaration ⇒ no probe): a CRUD API that publishes no suite is + // fully conforming. Ordered with the other fixed high-value probes, + // BEFORE the unbounded contract-diff enumeration, and on its OWN observer + // so a long suite can never starve them. + await observeTestSuite(origin, observer) + // 5. Contract-diff probing (ax-e6b.28.4): for a FULL OpenAPI<->live diff, // fetch EVERY GET-safe candidate path once — not just the seeded keyless // sample above — so the diff enumerates every declared operation, not a diff --git a/src/expect.ts b/src/expect.ts new file mode 100644 index 0000000..e081df7 --- /dev/null +++ b/src/expect.ts @@ -0,0 +1,248 @@ +/** + * The expectation engine — interpolation, endpoint resolution, capture and the + * expectation judge. SHARED, deliberately, by every caller that has to decide + * whether one observed exchange conforms to one pinned expectation. + * + * WHY THIS IS ITS OWN MODULE. These functions began life private to + * `pinned.ts`, where `verifyPinnedSpec` was their only caller. They now have a + * second one: the `published-test-suite` check judges a CARD-DECLARED suite, + * and it must reach the SAME verdict `verifyPinnedSpec` would reach over the + * same evidence. A second copy of the expectation judge — even a faithful one — + * is precisely the drift this estate spends its effort preventing: the moment + * two judges disagree, a target's conformance depends on which door asked. + * So the judge is extracted, not forked, and both callers import this module. + * + * Everything here is PURE and deterministic in (evidence, expectation, + * bindings). Nothing fetches. `resolveEndpoint` re-gates every resolved URL + * through the shared same-origin + publicly-routable check, so a + * TARGET-CONTROLLED captured value cannot steer a request off-origin no matter + * which caller resolved it. + */ + +import { isPubliclyRoutableSameOrigin } from './http.js' +import { validateSchema, readPath } from './schema.js' +import type { EndpointExpect, Evidence, PinnedRequirement } from './types.js' + +// --------------------------------------------------------------------------- +// Variable-capture + chaining (endpoint requirements) +// --------------------------------------------------------------------------- + +/** Per-run capture scope: `varName -> value` extracted from a response body. */ +export type Bindings = Record + +export type EndpointReq = Extract + +/** `{{var}}` token — dot/word chars only (matches a capture var name). */ +const VAR_TOKEN = /\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g + +/** A string that is EXACTLY one `{{var}}` token, edge to edge (no surrounding text). */ +const WHOLE_VALUE_TOKEN = /^\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}$/ + +/** + * Interpolate every `{{var}}` in a string from the binding scope. A reference + * to an unbound var is an ERROR (fail-closed) — the literal token is never + * emitted onto the wire. Bound values render as their primitive text; a bound + * object/array renders as compact JSON. + * + * This is the STRING-coercing path, used unconditionally for the URL path and + * method (both are always strings on the wire) and for any partial/embedded + * token (a token surrounded by other text, e.g. `/things/{{id}}`). + */ +export function interpolateString(s: string, bindings: Bindings): { value: string } | { error: string } { + let undef: string | undefined + const value = s.replace(VAR_TOKEN, (_m, name: string) => { + if (!Object.hasOwn(bindings, name)) { + undef ??= name + return '' + } + const v = bindings[name] + if (v === null || v === undefined) return '' + return typeof v === 'object' ? JSON.stringify(v) : String(v) + }) + if (undef !== undefined) return { error: `undefined capture var {{${undef}}}` } + return { value } +} + +/** + * Interpolate a string leaf in a TYPED context (a JSON value inside `body` or + * `expect` — e.g. `expect.paths[].equals`, an expected scalar, a body field). + * When the ENTIRE string is a single whole-value `{{var}}` token, the RAW bound + * value is substituted PRESERVING ITS TYPE (number / boolean / object / null), + * so a captured numeric/boolean id chained into a typed compare or a JSON body + * value is judged/serialized as the value it is — not falsely stringified to + * `"1"` where `judgeExpect` would then mismatch `1`. Any other string (a + * partial/embedded token, or plain text) falls through to string coercion. + * + * Surrounding whitespace is incidental ONLY for a NON-STRING binding: `'{{n}} '` + * or `' {{n}} '` bound to a number/boolean/object/null is still a lone + * whole-value token meant AS that value — whitespace cannot be part of the + * intended literal — so it is TRIMMED before classification and the RAW typed + * value is substituted (otherwise the trailing space would push it onto the + * string-coercing path and silently false-FAIL a compliant numeric target, + * `"1 "` vs `1`). But for a STRING binding the surrounding whitespace MAY be an + * intended literal (`' {{tid}} '` with tid = 'hello' meaning the literal + * ' hello '), so a string value keeps the string-coercing in-place path, which + * substitutes the token where it sits and PRESERVES the surrounding whitespace. + * (An edge-to-edge string token `'{{tid}}'` coerces to the identical raw string, + * so it is unaffected either way.) A token adjacent to NON-whitespace text + * (`'v{{n}}'`, `'{{a}}{{b}}'`) is genuine embedded interpolation and coerces. + */ +function interpolateTypedString(s: string, bindings: Bindings): { value: unknown } | { error: string } { + const whole = WHOLE_VALUE_TOKEN.exec(s.trim()) + if (whole) { + const name = whole[1]! + if (!Object.hasOwn(bindings, name)) return { error: `undefined capture var {{${name}}}` } + const v = bindings[name] + // Preserve TYPE (trimming incidental whitespace) only when whitespace cannot + // be part of an intended literal — i.e. the bound value is NON-STRING. A + // STRING binding falls through to the string-coercing path below, which + // preserves any surrounding whitespace in `s`. + if (typeof v !== 'string') return { value: v } + } + return interpolateString(s, bindings) +} + +/** Deep-interpolate strings inside an arbitrary JSON value (body / expect). */ +export function interpolateDeep(value: unknown, bindings: Bindings): { value: unknown } | { error: string } { + if (typeof value === 'string') return interpolateTypedString(value, bindings) + if (Array.isArray(value)) { + const out: unknown[] = [] + for (const item of value) { + const r = interpolateDeep(item, bindings) + if ('error' in r) return r + out.push(r.value) + } + return { value: out } + } + if (value !== null && typeof value === 'object') { + const out: Record = {} + for (const [k, v] of Object.entries(value)) { + const r = interpolateDeep(v, bindings) + if ('error' in r) return r + out[k] = r.value + } + return { value: out } + } + return { value } +} + +export type ResolvedEndpoint = + | { ok: true; method: string; url: string; body: unknown; expect: EndpointExpect } + | { ok: false; detail: string } + +/** + * Resolve an endpoint requirement against the current binding scope: interpolate + * method/path/body/expect, build the concrete URL, and RE-GATE it through the + * SAME same-origin + publicly-routable + non-private check every pinned fetch + * uses. Because a captured value is target-controlled, this gate is what stops a + * malicious target from steering an interpolated path off-origin or at a + * private/metadata address — such a resolution fails closed and is never + * fetched. Deterministic in `bindings`, so observe and judge agree. + */ +export function resolveEndpoint(req: EndpointReq, origin: string, bindings: Bindings): ResolvedEndpoint { + const under = (detail: string): ResolvedEndpoint => ({ + ok: false, + detail: `requirement ${req.id} references ${detail}`, + }) + const m = interpolateString(req.method, bindings) + if ('error' in m) return under(m.error) + const p = interpolateString(req.path, bindings) + if ('error' in p) return under(p.error) + const b = interpolateDeep(req.body, bindings) + if ('error' in b) return under(b.error) + const e = interpolateDeep(req.expect, bindings) + if ('error' in e) return under(e.error) + + let url: URL + try { + url = new URL(p.value, `${origin}/`) + } catch { + return { ok: false, detail: `requirement ${req.id} resolved to an unparseable url from path "${p.value}"` } + } + const resolvedUrl = url.toString() + if (!isPubliclyRoutableSameOrigin(resolvedUrl, origin)) { + return { + ok: false, + detail: + `requirement ${req.id} resolved to off-origin/private url ${resolvedUrl} ` + + '(a captured value must not steer the request off-origin) — refused, fail closed', + } + } + return { ok: true, method: m.value.toUpperCase(), url: resolvedUrl, body: b.value, expect: e.value as EndpointExpect } +} + +/** + * Extract each `capture` dot-path from an observed response body and bind it. + * A path that does not resolve (or a non-JSON body) leaves the var UNBOUND, so a + * downstream `{{var}}` reference fails closed rather than silently skipping. + */ +export function captureInto(bindings: Bindings, capture: Record, ev: Evidence | undefined): void { + let body: unknown + try { + body = JSON.parse(ev?.body ?? '') + } catch { + return + } + for (const [varName, path] of Object.entries(capture)) { + const r = readPath(body, path) + if (r.found) bindings[varName] = r.value + } +} + +/** + * Judge one observed exchange against an expectation block. Pure; returns the + * list of problems (empty = conforms). Shared by `endpoint` and `probe` + * requirement kinds. + */ +export function judgeExpect(ev: Evidence | undefined, expect: EndpointExpect): string[] { + const problems: string[] = [] + if (!ev || ev.status === null) { + problems.push(`fetch failed (${ev?.error ?? 'not observed'})`) + return problems + } + const wanted = expect.status === undefined ? [200] : Array.isArray(expect.status) ? expect.status : [expect.status] + if (!wanted.includes(ev.status)) problems.push(`status ${ev.status}, wanted ${wanted.join('|')}`) + if (expect.contentTypeIncludes && !(ev.contentType ?? '').includes(expect.contentTypeIncludes)) { + problems.push(`content-type ${ev.contentType}, wanted *${expect.contentTypeIncludes}*`) + } + if (expect.schema || expect.paths) { + let body: unknown + try { body = JSON.parse(ev.body ?? '') } catch { problems.push('body is not JSON') } + if (body !== undefined) { + if (expect.schema) { + for (const v of validateSchema(body, expect.schema)) problems.push(`${v.path} ${v.message}`) + } + for (const p of expect.paths ?? []) { + const r = readPath(body, p.path) + if (p.exists !== undefined && r.found !== p.exists) problems.push(`path ${p.path} ${p.exists ? 'missing' : 'unexpectedly present'}`) + if (p.equals !== undefined && (!r.found || JSON.stringify(r.value) !== JSON.stringify(p.equals))) { + problems.push(`path ${p.path} = ${JSON.stringify(r.found ? r.value : undefined)}, wanted ${JSON.stringify(p.equals)}`) + } + // Closed-vocabulary membership (e.g. AXP pricing model ∈ [free, metered]). + if (p.oneOf !== undefined && + (!r.found || !p.oneOf.some((v) => JSON.stringify(v) === JSON.stringify(r.value)))) { + problems.push(`path ${p.path} = ${JSON.stringify(r.found ? r.value : undefined)}, wanted one of ${JSON.stringify(p.oneOf)}`) + } + // Numeric comparators — the pinned floor/ceiling. A comparator on a + // path that is absent or non-numeric is itself a failure (the target + // did not report the number the contract measures). + const comparators: Array<[keyof typeof p, string, (a: number, b: number) => boolean]> = [ + ['gte', '>=', (a, b) => a >= b], + ['lte', '<=', (a, b) => a <= b], + ['gt', '>', (a, b) => a > b], + ['lt', '<', (a, b) => a < b], + ] + for (const [key, sym, cmp] of comparators) { + const bound = p[key] as number | undefined + if (bound === undefined) continue + if (!r.found || typeof r.value !== 'number') { + problems.push(`path ${p.path} = ${JSON.stringify(r.found ? r.value : undefined)}, wanted a number ${sym} ${bound}`) + } else if (!cmp(r.value, bound)) { + problems.push(`path ${p.path} = ${r.value}, wanted ${sym} ${bound}`) + } + } + } + } + } + return problems +} diff --git a/src/http.ts b/src/http.ts index e79aca7..bb83e2a 100644 --- a/src/http.ts +++ b/src/http.ts @@ -82,6 +82,32 @@ export class Observer { return this.opts.budget - this.used } + /** + * A SIBLING observer sharing this one's TRANSPORT configuration (fetcher, + * inter-request delay, per-request timeout, body byte cap, and the consented + * private-target escape hatch) but carrying its OWN budget and its own write + * posture. + * + * This exists for the card-declared test-suite sub-run, which must not + * inherit the parent's allowances. A pinned run sets `allowWrites: true` + * because "the target is yours"; a suite named by a stranger's card carries + * no such consent, and a long suite must not drain the politeness budget out + * from under the parent's fixed high-value probes. Sharing the FETCHER is the + * point — a mock/injected transport, and the deployed Worker's, must apply to + * the sub-run identically — while the budget and write posture are isolated. + */ + child(overrides: Pick): Observer { + return new Observer({ + fetcher: this.opts.fetcher, + delayMs: this.opts.delayMs, + timeoutMs: this.opts.timeoutMs, + maxBodyBytes: this.opts.maxBodyBytes, + allowPrivate: this.opts.allowPrivate, + budget: overrides.budget ?? this.opts.budget, + allowWrites: overrides.allowWrites ?? false, + }) + } + /** * Fetch once, record Evidence, return it. Never throws. * diff --git a/src/pinned.ts b/src/pinned.ts index 1f65550..2988b2e 100644 --- a/src/pinned.ts +++ b/src/pinned.ts @@ -29,13 +29,31 @@ import { observeTarget, ROLE, parseAgentsJson, parseJsonBody, parseOpenapi } fro import { runChecks } from './checks.js' import { axScoreOf } from './grade.js' import { sha256Hex } from './digest.js' -import { validateSchema, readPath } from './schema.js' +import { readPath } from './schema.js' import { VERIFIER_VERSION } from './verify.js' +// The expectation engine (interpolation, endpoint resolution, capture, the +// expectation judge) lives in ./expect.js so the `published-test-suite` check +// judges a card-declared suite with the IDENTICAL code path this module uses. +// Extracted, never forked — two expectation judges that can drift is exactly +// the failure that would make conformance depend on which door asked. import { - OPTIONAL_DECLARED_INTERFACES, - OPTIONAL_INTERFACE_PATH_RE, - eligibleOptionalChecks, -} from './optional-interfaces.js' + captureInto, + interpolateDeep, + judgeExpect, + resolveEndpoint, + type Bindings, +} from './expect.js' +// The pure document layer (requirement-list validation + Suite parsing) lives +// in ./suite-doc.js: the observe and judge sides of the `published-test-suite` +// check both need it, and both live in modules THIS one imports, so keeping it +// here would close an import cycle. Re-exported so this module's public API is +// unchanged for `dataset.ts`, `worker.ts`, `index.ts` and the tests. +// The EVASION GUARD (`validateAppliesWhen`) travels WITH `validateRequirements` +// into that module: it is a document-layer rule, and it must run for a +// card-declared Suite exactly as it runs for a ratified PinnedSpec. Splitting +// them would leave the suite door unguarded. +import { parseSuite, validateRequirements } from './suite-doc.js' +export { parseSuite, validateRequirements } import type { AppliesWhen, CheckResult, @@ -104,319 +122,6 @@ export function parsePinnedSpec(text: string): PinnedSpec { return doc } -/** - * Validate an ordered requirement list — the id-uniqueness, derived-role-key - * collision-freeness, and colon-in-id guards. Extracted so BOTH a PinnedSpec - * and a reusable Suite (which is a PinnedSpec parameterized by an environment) - * run the SAME checks over the SAME requirement shape — the suite format does - * not fork the requirement contract, it reuses it. - */ -export function validateRequirements(requirements: PinnedRequirement[]): void { - const doc = { requirements } - // VACUOUS-PASS GUARD. `passed: results.every(r => r.verdict === 'pass')` is - // `true` for an EMPTY array — an all() over nothing is vacuously true. A - // PinnedSpec (or Suite) with zero requirements would therefore ALWAYS report - // `passed: true` regardless of what the target does, including a totally - // broken worker: `expect(anyWorker).toConform({spec: emptySpec})` would pass - // every time. That is the exact class of silent-faked-success this verifier - // exists to catch, so refuse it categorically, LOUDLY, at parse — before any - // probe fires — rather than let an empty spec verify nothing while looking - // like a green report. - if (doc.requirements.length === 0) { - throw new Error( - 'a PinnedSpec with no requirements verifies nothing; refusing to vacuously pass. ' + - 'Add at least one requirement (or delete this spec/suite rather than pin an empty one).', - ) - } - // Every requirement id MUST be a UNIQUE, NON-EMPTY STRING. The role key - // (`pinned:`) is what observe records evidence under and what the judge - // looks up by `find(role === 'pinned:')` (FIRST match). A PinnedSpec is - // EXTERNAL JSON parsed at runtime, so the `id: string` TS type is a - // compile-time fiction: a runtime id can be a number, boolean, null, missing, - // or the empty string. Any of those, or a duplicate, would let two - // requirements share one role — observe records under it by loop POSITION, - // the judge resolves BOTH to the first match — a self-contradictory report - // that re-opens the observe/judge divergence. Two numeric `1`s collapse to - // `pinned:1`; two missing ids to `pinned:undefined`. So reject any id that is - // not a unique non-empty string LOUDLY at parse, naming the offender — never - // `continue`-skip it. (Numeric `1` and string `"1"` both become the same - // role, so rejecting every non-string id also stops that cross-type - // collision.) - const seen = new Set() - for (const req of doc.requirements) { - const id = (req as { id?: unknown }).id - if (typeof id !== 'string' || id.length === 0) { - throw new Error( - `invalid requirement id ${JSON.stringify(id)} in PinnedSpec — every requirement id must be ` + - 'a unique NON-EMPTY STRING. This spec is external JSON: a numeric/boolean/null/missing/empty ' + - 'id collapses to a shared role (pinned:), making observe and the judge resolve different ' + - 'requirements — refusing to verify something incoherent', - ) - } - if (seen.has(id)) { - throw new Error( - `duplicate requirement id "${id}" in PinnedSpec — requirement ids must be unique ` + - '(observe indexes evidence by position, the judge by id; a repeat makes them disagree)', - ) - } - seen.add(id) - } - - // DERIVED-ROLE COLLISION GUARD. Raw-id uniqueness (above) is NOT enough: the - // role key a requirement records/is-judged under is DERIVED, not the raw id, - // and it is NON-INJECTIVE ACROSS KINDS: - // endpoint id X → the single role key `pinned:X` (observe/judge - // both use `pinned:${id}`) - // probe id Y → the role-key NAMESPACE `pinned:Y:` (one per manifest - // entry i), modeled here as the PREFIX `pinned:Y:` - // surface / ax-floor / check → record NO `pinned:` role at all, so they can - // never collide on a derived role key. - // So endpoint "x:0" derives `pinned:x:0`, which is ALSO probe "x"'s entry-0 - // role: both raw ids are distinct strings, the dup guard accepts the spec, - // then the judge's find(role === 'pinned:x:0') resolves BOTH requirements to - // the FIRST-recorded item — a probe judged against an endpoint's body (a - // false-FAIL, or a vacuous false-PASS: a conformance requirement that never - // judges the thing it names). Reject at parse if any two requirements' derived - // role keys can collide — an endpoint's point key falling inside a probe's - // namespace, or one probe namespace nested inside another. - const reservations: RoleReservation[] = [] - for (const req of doc.requirements) { - const kind = (req as { kind?: unknown }).kind - const id = (req as { id: string }).id - if (kind === 'endpoint') reservations.push({ id, kind: 'endpoint', point: `pinned:${id}` }) - else if (kind === 'probe') reservations.push({ id, kind: 'probe', prefix: `pinned:${id}:` }) - } - for (let i = 0; i < reservations.length; i++) { - for (let j = i + 1; j < reservations.length; j++) { - const a = reservations[i]! - const b = reservations[j]! - const shared = roleKeysCollide(a, b) - if (shared !== undefined) { - throw new Error( - `derived role-key collision in PinnedSpec: requirement "${a.id}" (${a.kind}) and ` + - `requirement "${b.id}" (${b.kind}) both derive role key(s) under "${shared}". The role ` + - 'key is DERIVED (endpoint → pinned:, probe → pinned::), not the raw id, so ' + - 'two distinct raw ids can still share a role and make observe and the judge resolve ' + - 'different requirements — refusing to verify something incoherent', - ) - } - } - } - - // Belt-and-suspenders: ':' is the role-key separator (`pinned:[:]`), so - // a colon INSIDE a raw id is the only way a derived role key can ever be - // ambiguous. The collision guard above already rejects the concrete colliding - // cases; this closes the whole class categorically — including ids like "a:b" - // that happen to collide with nothing yet still muddy role parsing. - for (const req of doc.requirements) { - const id = (req as { id: string }).id - if (id.includes(':')) { - throw new Error( - `requirement id "${id}" in PinnedSpec contains the ':' role-key separator — a requirement ` + - 'id must not contain ":" (the derived role key is pinned:[:]; a colon in the raw ' + - 'id makes that key ambiguous). Rename the requirement.', - ) - } - } - - // THE EVASION GUARD. See optional-interfaces.ts for why it exists. - for (const req of doc.requirements) validateAppliesWhen(req) -} - -/** - * THE EVASION GUARD — five rules, all THROWN at parse. - * - * `appliesWhen` is the one place in a PinnedSpec where a requirement can decide - * NOT to judge the target. The `cardDeclares` arm makes that decision from a - * key the TARGET writes. So it is only safe if the set of requirements that can - * reach it is fixed by the VERIFIER, not by the spec — otherwise any MUST - * clause becomes optional by omission and the standard quietly stops being one. - * - * This runs inside `validateRequirements`, which runs inside BOTH - * `parsePinnedSpec` and `parseSuite` — i.e. before `verifyPinnedSpec` fires a - * single probe. A spec that tries to gate an always-required check does not get - * a lenient verdict; it gets NO verdict, loudly, with the offending requirement - * named. There is no reviewer in the loop, which is what makes this ENFORCED - * rather than documented. - * - * The rules also give FORWARD protection the pre-union verifier could not have: - * an `appliesWhen` in a shape this verifier does not understand throws instead - * of silently degrading into "unobservable → applies → armed check skips → - * requirement fails", which would fail every conforming target for the wrong - * reason. - */ -function validateAppliesWhen(req: PinnedRequirement): void { - const raw = (req as { appliesWhen?: unknown }).appliesWhen - if (raw === undefined) return - const id = (req as { id: string }).id - const kind = (req as { kind?: unknown }).kind - const where = `requirement "${id}"` - - // Rule 2a: only `probe` and `check` requirements have ever consulted - // `appliesWhen`. On `surface` / `ax-floor` / `endpoint` it was silently - // ignored — a conditional-looking clause that conditions nothing is a - // false statement in a contract document. Make it explicit and throw. - if (kind !== 'probe' && kind !== 'check') { - throw new Error( - `${where} (kind:'${String(kind)}') carries an \`appliesWhen\`, which only kind:'probe' and ` + - "kind:'check' requirements evaluate. On this kind it would be silently ignored — a " + - 'conditional-looking clause that conditions nothing. Remove it, or change the kind.', - ) - } - - if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) { - throw new Error( - `${where} carries an \`appliesWhen\` that is not a JSON object (got ` + - `${raw === null ? 'null' : Array.isArray(raw) ? 'array' : typeof raw}). It must be exactly ` + - 'one of { fromProbe, path, equals } or { cardDeclares }.', - ) - } - const aw = raw as Record - const has = (k: string) => Object.prototype.hasOwnProperty.call(aw, k) - - // Rule 1: SHAPE TOTALITY. Exactly one arm. Both, neither, or a mixed shape is - // a spec this verifier cannot evaluate — and the union discriminates on key - // PRESENCE (no `source:` tag, so the standard's verbatim probe block stays - // byte-identical), which is only total because this rule rejects everything - // else at parse. - const fromProbeArm = has('fromProbe') - const cardArm = has('cardDeclares') - if (fromProbeArm === cardArm) { - throw new Error( - `${where} carries an \`appliesWhen\` with ${fromProbeArm ? 'BOTH' : 'NEITHER'} \`fromProbe\` ` + - 'and `cardDeclares`. Exactly one arm is legal: { fromProbe, path, equals } judges an ' + - 'OBSERVED VALUE and fails closed when it cannot be observed; { cardDeclares } judges a ' + - 'CARD DECLARATION and is not applicable when the key is absent. They are different in kind, ' + - 'so a requirement must say which one it means.', - ) - } - - if (cardArm) { - // Rule 2b: KIND RESTRICTION. `cardDeclares` is legal only on kind:'check'. - // This is what makes behavioural probe requirements — the ones that pin - // wire behaviour for always-required clauses — categorically un-gatable by - // any card key, with no registry lookup involved at all. - if (kind !== 'check') { - throw new Error( - `${where} is a kind:'probe' requirement carrying \`appliesWhen.cardDeclares\`. The ` + - "card-declaration arm is legal ONLY on kind:'check'. A behavioural probe requirement " + - 'pins what the wire must do for an always-required clause; letting a card key switch one ' + - 'off would let a target opt out of that clause by omission. An OPTIONAL capability that ' + - 'needs behavioural probing gets a CHECK that does the probing.', - ) - } - if (has('path') || has('equals')) { - throw new Error( - `${where} mixes \`cardDeclares\` with \`${has('path') ? 'path' : 'equals'}\`. The ` + - 'card-declaration arm tests PRESENCE only — there is deliberately no value test, because ' + - 'a present-but-unexpected value would have to mean either "not applicable" or "malformed" ' + - 'and two independent implementations would resolve that differently.', - ) - } - const cardDeclares = aw.cardDeclares - // Rule 5: PATH GRAMMAR. Deliberately redundant with rule 4 — it holds even - // if the registry is later mis-edited, and it forbids `cardDeclares: - // 'probes'`, which would gate an optional check on the AXP opt-in signal - // itself rather than on its own interface key. - if (typeof cardDeclares !== 'string' || !OPTIONAL_INTERFACE_PATH_RE.test(cardDeclares)) { - throw new Error( - `${where} carries \`appliesWhen.cardDeclares\` = ${JSON.stringify(cardDeclares)}, which is ` + - `not a legal optional-interface card path. It must match ${String(OPTIONAL_INTERFACE_PATH_RE)} ` + - '— exactly two segments, the first literally "interfaces", e.g. "interfaces.digitalLink". ' + - 'An optional interface is declared as a member of `interfaces`, nowhere else.', - ) - } - const check = (req as { check?: unknown }).check - // Rule 3: ALLOWLIST MEMBERSHIP. The registry is keyed by CHECK id — a - // string api.qa owns and a spec author cannot mint — not by requirement id, - // which the author chooses freely. - if (typeof check !== 'string' || !Object.prototype.hasOwnProperty.call(OPTIONAL_DECLARED_INTERFACES, check)) { - throw new Error( - `${where} tries to make check ${JSON.stringify(check)} conditional on the card declaration ` + - `${JSON.stringify(cardDeclares)}, but that check is NOT an api.qa optional-declared ` + - 'interface. A requirement can only be skipped by omission when the capability it verifies ' + - 'is ADDITIVE — otherwise the clause it binds stops being a MUST the moment a target leaves ' + - `a key out. Eligible checks: ${eligibleOptionalChecks().map((c) => `"${c}"`).join(', ')}. ` + - `Pin ${JSON.stringify(check)} WITHOUT \`appliesWhen\` if you mean to demand it of everyone.`, - ) - } - // Rule 4: PATH BINDING. Blocks cross-wiring — arming one optional check - // with a DIFFERENT optional interface's key, which would let a card skip a - // check by declaring something unrelated. - const bound = OPTIONAL_DECLARED_INTERFACES[check]! - if (cardDeclares !== bound) { - throw new Error( - `${where} arms check "${check}" with \`cardDeclares\` = ${JSON.stringify(cardDeclares)}, but ` + - `api.qa binds that check to ${JSON.stringify(bound)}. A check is armed by ITS OWN ` + - 'interface declaration; cross-wiring would let a card skip one capability by declaring ' + - 'another.', - ) - } - return - } - - // The OBSERVED-VALUE arm. Behaviour is unchanged; this only rejects shapes - // the evaluator could not have judged coherently anyway (a non-string source - // or path silently resolves to "unobservable → applies", which reads as a - // target failure when it is really a spec defect). - if (typeof aw.fromProbe !== 'string' || aw.fromProbe.length === 0) { - throw new Error( - `${where} carries \`appliesWhen.fromProbe\` = ${JSON.stringify(aw.fromProbe)} — it must be a ` + - 'non-empty string naming a probe channel this spec also declares a requirement for.', - ) - } - if (typeof aw.path !== 'string' || aw.path.length === 0) { - throw new Error( - `${where} carries \`appliesWhen.path\` = ${JSON.stringify(aw.path)} — the observed-value arm ` + - 'needs a non-empty dot-path into the source probe body.', - ) - } - if (!has('equals')) { - throw new Error( - `${where} carries \`appliesWhen.fromProbe\`/\`path\` with no \`equals\`. The observed-value arm ` + - 'applies the requirement only when the observed value deep-equals a PINNED value; without ' + - 'one there is nothing to compare against.', - ) - } -} - -/** - * One requirement's reservation in the DERIVED role-key space. An `endpoint` - * reserves a single POINT (`pinned:`); a `probe` reserves a whole NAMESPACE - * (`pinned::` for every manifest entry i), modeled as the PREFIX - * `pinned::`. - */ -interface RoleReservation { - id: string - kind: 'endpoint' | 'probe' - point?: string - prefix?: string -} - -/** - * Return the shared role key (a descriptive string) if two reservations' derived - * role-key spaces intersect, else undefined. A point falls inside a namespace - * when it starts with the namespace prefix; two namespaces collide when one - * prefix is a prefix of the other (nested). Two points can only match on an - * identical raw id, which the dup guard already rejects. - */ -function roleKeysCollide(a: RoleReservation, b: RoleReservation): string | undefined { - if (a.point !== undefined && b.point !== undefined) { - return a.point === b.point ? a.point : undefined - } - if (a.point !== undefined && b.prefix !== undefined) { - return a.point.startsWith(b.prefix) ? a.point : undefined - } - if (b.point !== undefined && a.prefix !== undefined) { - return b.point.startsWith(a.prefix) ? b.point : undefined - } - if (a.prefix !== undefined && b.prefix !== undefined) { - if (a.prefix.startsWith(b.prefix)) return `${a.prefix}` - if (b.prefix.startsWith(a.prefix)) return `${b.prefix}` - } - return undefined -} - export async function verifyPinnedSpec( target: string, specText: string, @@ -910,32 +615,6 @@ export interface VerifySuiteOpts extends ObserverOpts { rowBindings?: Record } -/** - * Parse + validate a reusable Suite. Reuses `validateRequirements` (the SAME - * id-uniqueness / derived-role-collision / colon guards a PinnedSpec runs) so - * the suite format does not fork the requirement contract. Additionally checks - * the `environments` map shape: each entry must be `{ vars: { ... } }`. - */ -export function parseSuite(text: string): Suite { - const doc = JSON.parse(text) as Suite - if (doc.$type !== 'Suite' || !Array.isArray(doc.requirements)) { - throw new Error('not a Suite: expected {"$type":"Suite","environments":{...},"requirements":[...]}') - } - const envs = doc.environments as unknown - if (envs === null || typeof envs !== 'object' || Array.isArray(envs)) { - throw new Error('Suite.environments must be an object mapping env name -> { vars: { : } }') - } - for (const [name, env] of Object.entries(envs as Record)) { - const vars = (env as { vars?: unknown } | null)?.vars - if (env === null || typeof env !== 'object' || Array.isArray(env) || - vars === null || typeof vars !== 'object' || Array.isArray(vars)) { - throw new Error(`Suite environment "${name}" must be an object of the form { "vars": { : } }`) - } - } - validateRequirements(doc.requirements) - return doc -} - /** * Run a reusable Suite against a selected ENVIRONMENT. A Suite is a PinnedSpec * parameterized by the environment's vars, so this DELEGATES to @@ -1313,226 +992,3 @@ function okPathnamesOf(bundle: EvidenceBundle): Set { return out } -// --------------------------------------------------------------------------- -// Variable-capture + chaining (endpoint requirements) -// --------------------------------------------------------------------------- - -/** Per-run capture scope: `varName -> value` extracted from a response body. */ -type Bindings = Record - -type EndpointReq = Extract - -/** `{{var}}` token — dot/word chars only (matches a capture var name). */ -const VAR_TOKEN = /\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g - -/** A string that is EXACTLY one `{{var}}` token, edge to edge (no surrounding text). */ -const WHOLE_VALUE_TOKEN = /^\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}$/ - -/** - * Interpolate every `{{var}}` in a string from the binding scope. A reference - * to an unbound var is an ERROR (fail-closed) — the literal token is never - * emitted onto the wire. Bound values render as their primitive text; a bound - * object/array renders as compact JSON. - * - * This is the STRING-coercing path, used unconditionally for the URL path and - * method (both are always strings on the wire) and for any partial/embedded - * token (a token surrounded by other text, e.g. `/things/{{id}}`). - */ -function interpolateString(s: string, bindings: Bindings): { value: string } | { error: string } { - let undef: string | undefined - const value = s.replace(VAR_TOKEN, (_m, name: string) => { - if (!Object.hasOwn(bindings, name)) { - undef ??= name - return '' - } - const v = bindings[name] - if (v === null || v === undefined) return '' - return typeof v === 'object' ? JSON.stringify(v) : String(v) - }) - if (undef !== undefined) return { error: `undefined capture var {{${undef}}}` } - return { value } -} - -/** - * Interpolate a string leaf in a TYPED context (a JSON value inside `body` or - * `expect` — e.g. `expect.paths[].equals`, an expected scalar, a body field). - * When the ENTIRE string is a single whole-value `{{var}}` token, the RAW bound - * value is substituted PRESERVING ITS TYPE (number / boolean / object / null), - * so a captured numeric/boolean id chained into a typed compare or a JSON body - * value is judged/serialized as the value it is — not falsely stringified to - * `"1"` where `judgeExpect` would then mismatch `1`. Any other string (a - * partial/embedded token, or plain text) falls through to string coercion. - * - * Surrounding whitespace is incidental ONLY for a NON-STRING binding: `'{{n}} '` - * or `' {{n}} '` bound to a number/boolean/object/null is still a lone - * whole-value token meant AS that value — whitespace cannot be part of the - * intended literal — so it is TRIMMED before classification and the RAW typed - * value is substituted (otherwise the trailing space would push it onto the - * string-coercing path and silently false-FAIL a compliant numeric target, - * `"1 "` vs `1`). But for a STRING binding the surrounding whitespace MAY be an - * intended literal (`' {{tid}} '` with tid = 'hello' meaning the literal - * ' hello '), so a string value keeps the string-coercing in-place path, which - * substitutes the token where it sits and PRESERVES the surrounding whitespace. - * (An edge-to-edge string token `'{{tid}}'` coerces to the identical raw string, - * so it is unaffected either way.) A token adjacent to NON-whitespace text - * (`'v{{n}}'`, `'{{a}}{{b}}'`) is genuine embedded interpolation and coerces. - */ -function interpolateTypedString(s: string, bindings: Bindings): { value: unknown } | { error: string } { - const whole = WHOLE_VALUE_TOKEN.exec(s.trim()) - if (whole) { - const name = whole[1]! - if (!Object.hasOwn(bindings, name)) return { error: `undefined capture var {{${name}}}` } - const v = bindings[name] - // Preserve TYPE (trimming incidental whitespace) only when whitespace cannot - // be part of an intended literal — i.e. the bound value is NON-STRING. A - // STRING binding falls through to the string-coercing path below, which - // preserves any surrounding whitespace in `s`. - if (typeof v !== 'string') return { value: v } - } - return interpolateString(s, bindings) -} - -/** Deep-interpolate strings inside an arbitrary JSON value (body / expect). */ -function interpolateDeep(value: unknown, bindings: Bindings): { value: unknown } | { error: string } { - if (typeof value === 'string') return interpolateTypedString(value, bindings) - if (Array.isArray(value)) { - const out: unknown[] = [] - for (const item of value) { - const r = interpolateDeep(item, bindings) - if ('error' in r) return r - out.push(r.value) - } - return { value: out } - } - if (value !== null && typeof value === 'object') { - const out: Record = {} - for (const [k, v] of Object.entries(value)) { - const r = interpolateDeep(v, bindings) - if ('error' in r) return r - out[k] = r.value - } - return { value: out } - } - return { value } -} - -type ResolvedEndpoint = - | { ok: true; method: string; url: string; body: unknown; expect: EndpointExpect } - | { ok: false; detail: string } - -/** - * Resolve an endpoint requirement against the current binding scope: interpolate - * method/path/body/expect, build the concrete URL, and RE-GATE it through the - * SAME same-origin + publicly-routable + non-private check every pinned fetch - * uses. Because a captured value is target-controlled, this gate is what stops a - * malicious target from steering an interpolated path off-origin or at a - * private/metadata address — such a resolution fails closed and is never - * fetched. Deterministic in `bindings`, so observe and judge agree. - */ -function resolveEndpoint(req: EndpointReq, origin: string, bindings: Bindings): ResolvedEndpoint { - const under = (detail: string): ResolvedEndpoint => ({ - ok: false, - detail: `requirement ${req.id} references ${detail}`, - }) - const m = interpolateString(req.method, bindings) - if ('error' in m) return under(m.error) - const p = interpolateString(req.path, bindings) - if ('error' in p) return under(p.error) - const b = interpolateDeep(req.body, bindings) - if ('error' in b) return under(b.error) - const e = interpolateDeep(req.expect, bindings) - if ('error' in e) return under(e.error) - - let url: URL - try { - url = new URL(p.value, `${origin}/`) - } catch { - return { ok: false, detail: `requirement ${req.id} resolved to an unparseable url from path "${p.value}"` } - } - const resolvedUrl = url.toString() - if (!isPubliclyRoutableSameOrigin(resolvedUrl, origin)) { - return { - ok: false, - detail: - `requirement ${req.id} resolved to off-origin/private url ${resolvedUrl} ` + - '(a captured value must not steer the request off-origin) — refused, fail closed', - } - } - return { ok: true, method: m.value.toUpperCase(), url: resolvedUrl, body: b.value, expect: e.value as EndpointExpect } -} - -/** - * Extract each `capture` dot-path from an observed response body and bind it. - * A path that does not resolve (or a non-JSON body) leaves the var UNBOUND, so a - * downstream `{{var}}` reference fails closed rather than silently skipping. - */ -function captureInto(bindings: Bindings, capture: Record, ev: Evidence | undefined): void { - let body: unknown - try { - body = JSON.parse(ev?.body ?? '') - } catch { - return - } - for (const [varName, path] of Object.entries(capture)) { - const r = readPath(body, path) - if (r.found) bindings[varName] = r.value - } -} - -/** - * Judge one observed exchange against an expectation block. Pure; returns the - * list of problems (empty = conforms). Shared by `endpoint` and `probe` - * requirement kinds. - */ -function judgeExpect(ev: Evidence | undefined, expect: EndpointExpect): string[] { - const problems: string[] = [] - if (!ev || ev.status === null) { - problems.push(`fetch failed (${ev?.error ?? 'not observed'})`) - return problems - } - const wanted = expect.status === undefined ? [200] : Array.isArray(expect.status) ? expect.status : [expect.status] - if (!wanted.includes(ev.status)) problems.push(`status ${ev.status}, wanted ${wanted.join('|')}`) - if (expect.contentTypeIncludes && !(ev.contentType ?? '').includes(expect.contentTypeIncludes)) { - problems.push(`content-type ${ev.contentType}, wanted *${expect.contentTypeIncludes}*`) - } - if (expect.schema || expect.paths) { - let body: unknown - try { body = JSON.parse(ev.body ?? '') } catch { problems.push('body is not JSON') } - if (body !== undefined) { - if (expect.schema) { - for (const v of validateSchema(body, expect.schema)) problems.push(`${v.path} ${v.message}`) - } - for (const p of expect.paths ?? []) { - const r = readPath(body, p.path) - if (p.exists !== undefined && r.found !== p.exists) problems.push(`path ${p.path} ${p.exists ? 'missing' : 'unexpectedly present'}`) - if (p.equals !== undefined && (!r.found || JSON.stringify(r.value) !== JSON.stringify(p.equals))) { - problems.push(`path ${p.path} = ${JSON.stringify(r.found ? r.value : undefined)}, wanted ${JSON.stringify(p.equals)}`) - } - // Closed-vocabulary membership (e.g. AXP pricing model ∈ [free, metered]). - if (p.oneOf !== undefined && - (!r.found || !p.oneOf.some((v) => JSON.stringify(v) === JSON.stringify(r.value)))) { - problems.push(`path ${p.path} = ${JSON.stringify(r.found ? r.value : undefined)}, wanted one of ${JSON.stringify(p.oneOf)}`) - } - // Numeric comparators — the pinned floor/ceiling. A comparator on a - // path that is absent or non-numeric is itself a failure (the target - // did not report the number the contract measures). - const comparators: Array<[keyof typeof p, string, (a: number, b: number) => boolean]> = [ - ['gte', '>=', (a, b) => a >= b], - ['lte', '<=', (a, b) => a <= b], - ['gt', '>', (a, b) => a > b], - ['lt', '<', (a, b) => a < b], - ] - for (const [key, sym, cmp] of comparators) { - const bound = p[key] as number | undefined - if (bound === undefined) continue - if (!r.found || typeof r.value !== 'number') { - problems.push(`path ${p.path} = ${JSON.stringify(r.found ? r.value : undefined)}, wanted a number ${sym} ${bound}`) - } else if (!cmp(r.value, bound)) { - problems.push(`path ${p.path} = ${r.value}, wanted ${sym} ${bound}`) - } - } - } - } - } - return problems -} diff --git a/src/sha256-sync.ts b/src/sha256-sync.ts new file mode 100644 index 0000000..939042c --- /dev/null +++ b/src/sha256-sync.ts @@ -0,0 +1,103 @@ +/** + * A SYNCHRONOUS SHA-256. + * + * WHY THIS EXISTS, since a second hash implementation is otherwise a smell. + * `digest.ts`'s `sha256Hex` is the estate's hash, and it stays the hash — but it + * is `async` because WebCrypto's `crypto.subtle.digest` is. `runChecks(bundle)` + * is SYNCHRONOUS and PURE over the evidence bundle, and that purity is the + * determinism contract the whole verifier rests on: the same bundle must yield + * the same verdicts, on a live run and on a replay of a stored bundle, with no + * I/O in between. Making `runChecks` async to await a digest would push `await` + * through every judge in the file and into every caller. + * + * The `published-test-suite` check has to verify that the suite document the + * target served hashes to the digest the target's own card pinned. The + * alternative — compute the digest during the OBSERVE phase and record the + * scalar for the judge to trust — is worse in the way that matters: on replay + * the judge would re-read a recorded verdict about the bytes instead of + * re-deriving it FROM the bytes. With a sync hash the check re-computes the + * digest from the suite text stored in the bundle every time it judges, so a + * tampered bundle fails the digest gate exactly as a tampered live response + * does, and the anti-Goodhart pin survives serialization. + * + * Correctness is not asserted, it is TESTED: `test/sha256-sync.test.ts` checks + * this implementation against the published FIPS-180-4 vectors AND against + * `digest.ts`'s WebCrypto `sha256Hex` over empty, ASCII, multi-byte UTF-8, + * block-boundary (55/56/63/64/119/120 byte) and multi-kilobyte inputs. If the + * two ever disagree, that test fails and this file is wrong. + */ + +const K = new Uint32Array([ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]) + +const rotr = (x: number, n: number): number => (x >>> n) | (x << (32 - n)) + +/** sha256 of a UTF-8 string, lowercase hex. Synchronous by design (see above). */ +export function sha256HexSync(text: string): string { + const msg = new TextEncoder().encode(text) + const bitLen = msg.length * 8 + + // Pad: 0x80, then zeros, then the 64-bit big-endian bit length. + const withLen = msg.length + 9 + const blocks = Math.ceil(withLen / 64) + const buf = new Uint8Array(blocks * 64) + buf.set(msg) + buf[msg.length] = 0x80 + // Bit length as a 64-bit big-endian value. Lengths beyond 2^53 bits are not + // representable in a JS number and cannot occur here (inputs are capped by + // the observer's byte cap), so the high word is written from a float-safe + // division rather than a BigInt. + const view = new DataView(buf.buffer) + view.setUint32(buf.length - 8, Math.floor(bitLen / 0x100000000), false) + view.setUint32(buf.length - 4, bitLen >>> 0, false) + + const h = new Uint32Array([ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, + ]) + const w = new Uint32Array(64) + + for (let b = 0; b < blocks; b++) { + const off = b * 64 + for (let i = 0; i < 16; i++) w[i] = view.getUint32(off + i * 4, false) + for (let i = 16; i < 64; i++) { + const x = w[i - 15]! + const y = w[i - 2]! + const s0 = rotr(x, 7) ^ rotr(x, 18) ^ (x >>> 3) + const s1 = rotr(y, 17) ^ rotr(y, 19) ^ (y >>> 10) + w[i] = (w[i - 16]! + s0 + w[i - 7]! + s1) >>> 0 + } + let [a, bb, c, d, e, f, g, hh] = [h[0]!, h[1]!, h[2]!, h[3]!, h[4]!, h[5]!, h[6]!, h[7]!] + for (let i = 0; i < 64; i++) { + const S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25) + const ch = (e & f) ^ (~e & g) + const t1 = (hh + S1 + ch + K[i]! + w[i]!) >>> 0 + const S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22) + const maj = (a & bb) ^ (a & c) ^ (bb & c) + const t2 = (S0 + maj) >>> 0 + hh = g; g = f; f = e + e = (d + t1) >>> 0 + d = c; c = bb; bb = a + a = (t1 + t2) >>> 0 + } + h[0] = (h[0]! + a) >>> 0 + h[1] = (h[1]! + bb) >>> 0 + h[2] = (h[2]! + c) >>> 0 + h[3] = (h[3]! + d) >>> 0 + h[4] = (h[4]! + e) >>> 0 + h[5] = (h[5]! + f) >>> 0 + h[6] = (h[6]! + g) >>> 0 + h[7] = (h[7]! + hh) >>> 0 + } + + let out = '' + for (let i = 0; i < 8; i++) out += h[i]!.toString(16).padStart(8, '0') + return out +} diff --git a/src/suite-doc.ts b/src/suite-doc.ts new file mode 100644 index 0000000..732f51b --- /dev/null +++ b/src/suite-doc.ts @@ -0,0 +1,365 @@ +/** + * The SUITE / PINNED-SPEC DOCUMENT LAYER — pure parsing and validation of a + * requirement list, with no dependency on the Observer, the network, or the + * judge. + * + * WHY IT IS SEPARATE. `parseSuite` and `validateRequirements` began in + * `pinned.ts`, whose only consumers were `verifyPinnedSpec` / `verifySuite`. + * They now have two more: the OBSERVE side (`discovery.ts`, which runs a + * card-declared suite) and the JUDGE side (`checks.ts`, which re-derives that + * run from the bundle). Both of those modules are imported BY `pinned.ts`, so + * importing it back would close a cycle. Extracting the pure document layer + * breaks the cycle without forking a single validation rule — a card-declared + * suite is held to the IDENTICAL id-uniqueness, derived-role-collision, + * colon-in-id and non-vacuity guards a ratified PinnedSpec is. + * + * `pinned.ts` re-exports both functions, so its public API is unchanged. + */ + +import type { PinnedRequirement, Suite } from './types.js' +import { + OPTIONAL_DECLARED_INTERFACES, + OPTIONAL_INTERFACE_PATH_RE, + eligibleOptionalChecks, +} from './optional-interfaces.js' + + +/** + * Validate an ordered requirement list — the id-uniqueness, derived-role-key + * collision-freeness, and colon-in-id guards. Extracted so BOTH a PinnedSpec + * and a reusable Suite (which is a PinnedSpec parameterized by an environment) + * run the SAME checks over the SAME requirement shape — the suite format does + * not fork the requirement contract, it reuses it. + */ +export function validateRequirements(requirements: PinnedRequirement[]): void { + const doc = { requirements } + // VACUOUS-PASS GUARD. `passed: results.every(r => r.verdict === 'pass')` is + // `true` for an EMPTY array — an all() over nothing is vacuously true. A + // PinnedSpec (or Suite) with zero requirements would therefore ALWAYS report + // `passed: true` regardless of what the target does, including a totally + // broken worker: `expect(anyWorker).toConform({spec: emptySpec})` would pass + // every time. That is the exact class of silent-faked-success this verifier + // exists to catch, so refuse it categorically, LOUDLY, at parse — before any + // probe fires — rather than let an empty spec verify nothing while looking + // like a green report. + if (doc.requirements.length === 0) { + throw new Error( + 'a PinnedSpec with no requirements verifies nothing; refusing to vacuously pass. ' + + 'Add at least one requirement (or delete this spec/suite rather than pin an empty one).', + ) + } + // Every requirement id MUST be a UNIQUE, NON-EMPTY STRING. The role key + // (`pinned:`) is what observe records evidence under and what the judge + // looks up by `find(role === 'pinned:')` (FIRST match). A PinnedSpec is + // EXTERNAL JSON parsed at runtime, so the `id: string` TS type is a + // compile-time fiction: a runtime id can be a number, boolean, null, missing, + // or the empty string. Any of those, or a duplicate, would let two + // requirements share one role — observe records under it by loop POSITION, + // the judge resolves BOTH to the first match — a self-contradictory report + // that re-opens the observe/judge divergence. Two numeric `1`s collapse to + // `pinned:1`; two missing ids to `pinned:undefined`. So reject any id that is + // not a unique non-empty string LOUDLY at parse, naming the offender — never + // `continue`-skip it. (Numeric `1` and string `"1"` both become the same + // role, so rejecting every non-string id also stops that cross-type + // collision.) + const seen = new Set() + for (const req of doc.requirements) { + const id = (req as { id?: unknown }).id + if (typeof id !== 'string' || id.length === 0) { + throw new Error( + `invalid requirement id ${JSON.stringify(id)} in PinnedSpec — every requirement id must be ` + + 'a unique NON-EMPTY STRING. This spec is external JSON: a numeric/boolean/null/missing/empty ' + + 'id collapses to a shared role (pinned:), making observe and the judge resolve different ' + + 'requirements — refusing to verify something incoherent', + ) + } + if (seen.has(id)) { + throw new Error( + `duplicate requirement id "${id}" in PinnedSpec — requirement ids must be unique ` + + '(observe indexes evidence by position, the judge by id; a repeat makes them disagree)', + ) + } + seen.add(id) + } + + // DERIVED-ROLE COLLISION GUARD. Raw-id uniqueness (above) is NOT enough: the + // role key a requirement records/is-judged under is DERIVED, not the raw id, + // and it is NON-INJECTIVE ACROSS KINDS: + // endpoint id X → the single role key `pinned:X` (observe/judge + // both use `pinned:${id}`) + // probe id Y → the role-key NAMESPACE `pinned:Y:` (one per manifest + // entry i), modeled here as the PREFIX `pinned:Y:` + // surface / ax-floor / check → record NO `pinned:` role at all, so they can + // never collide on a derived role key. + // So endpoint "x:0" derives `pinned:x:0`, which is ALSO probe "x"'s entry-0 + // role: both raw ids are distinct strings, the dup guard accepts the spec, + // then the judge's find(role === 'pinned:x:0') resolves BOTH requirements to + // the FIRST-recorded item — a probe judged against an endpoint's body (a + // false-FAIL, or a vacuous false-PASS: a conformance requirement that never + // judges the thing it names). Reject at parse if any two requirements' derived + // role keys can collide — an endpoint's point key falling inside a probe's + // namespace, or one probe namespace nested inside another. + const reservations: RoleReservation[] = [] + for (const req of doc.requirements) { + const kind = (req as { kind?: unknown }).kind + const id = (req as { id: string }).id + if (kind === 'endpoint') reservations.push({ id, kind: 'endpoint', point: `pinned:${id}` }) + else if (kind === 'probe') reservations.push({ id, kind: 'probe', prefix: `pinned:${id}:` }) + } + for (let i = 0; i < reservations.length; i++) { + for (let j = i + 1; j < reservations.length; j++) { + const a = reservations[i]! + const b = reservations[j]! + const shared = roleKeysCollide(a, b) + if (shared !== undefined) { + throw new Error( + `derived role-key collision in PinnedSpec: requirement "${a.id}" (${a.kind}) and ` + + `requirement "${b.id}" (${b.kind}) both derive role key(s) under "${shared}". The role ` + + 'key is DERIVED (endpoint → pinned:, probe → pinned::), not the raw id, so ' + + 'two distinct raw ids can still share a role and make observe and the judge resolve ' + + 'different requirements — refusing to verify something incoherent', + ) + } + } + } + + // Belt-and-suspenders: ':' is the role-key separator (`pinned:[:]`), so + // a colon INSIDE a raw id is the only way a derived role key can ever be + // ambiguous. The collision guard above already rejects the concrete colliding + // cases; this closes the whole class categorically — including ids like "a:b" + // that happen to collide with nothing yet still muddy role parsing. + for (const req of doc.requirements) { + const id = (req as { id: string }).id + if (id.includes(':')) { + throw new Error( + `requirement id "${id}" in PinnedSpec contains the ':' role-key separator — a requirement ` + + 'id must not contain ":" (the derived role key is pinned:[:]; a colon in the raw ' + + 'id makes that key ambiguous). Rename the requirement.', + ) + } + } + + // THE EVASION GUARD. See optional-interfaces.ts for why it exists. + for (const req of doc.requirements) validateAppliesWhen(req) +} + +/** + * THE EVASION GUARD — five rules, all THROWN at parse. + * + * `appliesWhen` is the one place in a PinnedSpec where a requirement can decide + * NOT to judge the target. The `cardDeclares` arm makes that decision from a + * key the TARGET writes. So it is only safe if the set of requirements that can + * reach it is fixed by the VERIFIER, not by the spec — otherwise any MUST + * clause becomes optional by omission and the standard quietly stops being one. + * + * This runs inside `validateRequirements`, which runs inside BOTH + * `parsePinnedSpec` and `parseSuite` — i.e. before `verifyPinnedSpec` fires a + * single probe. A spec that tries to gate an always-required check does not get + * a lenient verdict; it gets NO verdict, loudly, with the offending requirement + * named. There is no reviewer in the loop, which is what makes this ENFORCED + * rather than documented. + * + * The rules also give FORWARD protection the pre-union verifier could not have: + * an `appliesWhen` in a shape this verifier does not understand throws instead + * of silently degrading into "unobservable → applies → armed check skips → + * requirement fails", which would fail every conforming target for the wrong + * reason. + */ +function validateAppliesWhen(req: PinnedRequirement): void { + const raw = (req as { appliesWhen?: unknown }).appliesWhen + if (raw === undefined) return + const id = (req as { id: string }).id + const kind = (req as { kind?: unknown }).kind + const where = `requirement "${id}"` + + // Rule 2a: only `probe` and `check` requirements have ever consulted + // `appliesWhen`. On `surface` / `ax-floor` / `endpoint` it was silently + // ignored — a conditional-looking clause that conditions nothing is a + // false statement in a contract document. Make it explicit and throw. + if (kind !== 'probe' && kind !== 'check') { + throw new Error( + `${where} (kind:'${String(kind)}') carries an \`appliesWhen\`, which only kind:'probe' and ` + + "kind:'check' requirements evaluate. On this kind it would be silently ignored — a " + + 'conditional-looking clause that conditions nothing. Remove it, or change the kind.', + ) + } + + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error( + `${where} carries an \`appliesWhen\` that is not a JSON object (got ` + + `${raw === null ? 'null' : Array.isArray(raw) ? 'array' : typeof raw}). It must be exactly ` + + 'one of { fromProbe, path, equals } or { cardDeclares }.', + ) + } + const aw = raw as Record + const has = (k: string) => Object.prototype.hasOwnProperty.call(aw, k) + + // Rule 1: SHAPE TOTALITY. Exactly one arm. Both, neither, or a mixed shape is + // a spec this verifier cannot evaluate — and the union discriminates on key + // PRESENCE (no `source:` tag, so the standard's verbatim probe block stays + // byte-identical), which is only total because this rule rejects everything + // else at parse. + const fromProbeArm = has('fromProbe') + const cardArm = has('cardDeclares') + if (fromProbeArm === cardArm) { + throw new Error( + `${where} carries an \`appliesWhen\` with ${fromProbeArm ? 'BOTH' : 'NEITHER'} \`fromProbe\` ` + + 'and `cardDeclares`. Exactly one arm is legal: { fromProbe, path, equals } judges an ' + + 'OBSERVED VALUE and fails closed when it cannot be observed; { cardDeclares } judges a ' + + 'CARD DECLARATION and is not applicable when the key is absent. They are different in kind, ' + + 'so a requirement must say which one it means.', + ) + } + + if (cardArm) { + // Rule 2b: KIND RESTRICTION. `cardDeclares` is legal only on kind:'check'. + // This is what makes behavioural probe requirements — the ones that pin + // wire behaviour for always-required clauses — categorically un-gatable by + // any card key, with no registry lookup involved at all. + if (kind !== 'check') { + throw new Error( + `${where} is a kind:'probe' requirement carrying \`appliesWhen.cardDeclares\`. The ` + + "card-declaration arm is legal ONLY on kind:'check'. A behavioural probe requirement " + + 'pins what the wire must do for an always-required clause; letting a card key switch one ' + + 'off would let a target opt out of that clause by omission. An OPTIONAL capability that ' + + 'needs behavioural probing gets a CHECK that does the probing.', + ) + } + if (has('path') || has('equals')) { + throw new Error( + `${where} mixes \`cardDeclares\` with \`${has('path') ? 'path' : 'equals'}\`. The ` + + 'card-declaration arm tests PRESENCE only — there is deliberately no value test, because ' + + 'a present-but-unexpected value would have to mean either "not applicable" or "malformed" ' + + 'and two independent implementations would resolve that differently.', + ) + } + const cardDeclares = aw.cardDeclares + // Rule 5: PATH GRAMMAR. Deliberately redundant with rule 4 — it holds even + // if the registry is later mis-edited, and it forbids `cardDeclares: + // 'probes'`, which would gate an optional check on the AXP opt-in signal + // itself rather than on its own interface key. + if (typeof cardDeclares !== 'string' || !OPTIONAL_INTERFACE_PATH_RE.test(cardDeclares)) { + throw new Error( + `${where} carries \`appliesWhen.cardDeclares\` = ${JSON.stringify(cardDeclares)}, which is ` + + `not a legal optional-interface card path. It must match ${String(OPTIONAL_INTERFACE_PATH_RE)} ` + + '— exactly two segments, the first literally "interfaces", e.g. "interfaces.digitalLink". ' + + 'An optional interface is declared as a member of `interfaces`, nowhere else.', + ) + } + const check = (req as { check?: unknown }).check + // Rule 3: ALLOWLIST MEMBERSHIP. The registry is keyed by CHECK id — a + // string api.qa owns and a spec author cannot mint — not by requirement id, + // which the author chooses freely. + if (typeof check !== 'string' || !Object.prototype.hasOwnProperty.call(OPTIONAL_DECLARED_INTERFACES, check)) { + throw new Error( + `${where} tries to make check ${JSON.stringify(check)} conditional on the card declaration ` + + `${JSON.stringify(cardDeclares)}, but that check is NOT an api.qa optional-declared ` + + 'interface. A requirement can only be skipped by omission when the capability it verifies ' + + 'is ADDITIVE — otherwise the clause it binds stops being a MUST the moment a target leaves ' + + `a key out. Eligible checks: ${eligibleOptionalChecks().map((c) => `"${c}"`).join(', ')}. ` + + `Pin ${JSON.stringify(check)} WITHOUT \`appliesWhen\` if you mean to demand it of everyone.`, + ) + } + // Rule 4: PATH BINDING. Blocks cross-wiring — arming one optional check + // with a DIFFERENT optional interface's key, which would let a card skip a + // check by declaring something unrelated. + const bound = OPTIONAL_DECLARED_INTERFACES[check]! + if (cardDeclares !== bound) { + throw new Error( + `${where} arms check "${check}" with \`cardDeclares\` = ${JSON.stringify(cardDeclares)}, but ` + + `api.qa binds that check to ${JSON.stringify(bound)}. A check is armed by ITS OWN ` + + 'interface declaration; cross-wiring would let a card skip one capability by declaring ' + + 'another.', + ) + } + return + } + + // The OBSERVED-VALUE arm. Behaviour is unchanged; this only rejects shapes + // the evaluator could not have judged coherently anyway (a non-string source + // or path silently resolves to "unobservable → applies", which reads as a + // target failure when it is really a spec defect). + if (typeof aw.fromProbe !== 'string' || aw.fromProbe.length === 0) { + throw new Error( + `${where} carries \`appliesWhen.fromProbe\` = ${JSON.stringify(aw.fromProbe)} — it must be a ` + + 'non-empty string naming a probe channel this spec also declares a requirement for.', + ) + } + if (typeof aw.path !== 'string' || aw.path.length === 0) { + throw new Error( + `${where} carries \`appliesWhen.path\` = ${JSON.stringify(aw.path)} — the observed-value arm ` + + 'needs a non-empty dot-path into the source probe body.', + ) + } + if (!has('equals')) { + throw new Error( + `${where} carries \`appliesWhen.fromProbe\`/\`path\` with no \`equals\`. The observed-value arm ` + + 'applies the requirement only when the observed value deep-equals a PINNED value; without ' + + 'one there is nothing to compare against.', + ) + } +} + +/** + * One requirement's reservation in the DERIVED role-key space. An `endpoint` + * reserves a single POINT (`pinned:`); a `probe` reserves a whole NAMESPACE + * (`pinned::` for every manifest entry i), modeled as the PREFIX + * `pinned::`. + */ +interface RoleReservation { + id: string + kind: 'endpoint' | 'probe' + point?: string + prefix?: string +} + +/** + * Return the shared role key (a descriptive string) if two reservations' derived + * role-key spaces intersect, else undefined. A point falls inside a namespace + * when it starts with the namespace prefix; two namespaces collide when one + * prefix is a prefix of the other (nested). Two points can only match on an + * identical raw id, which the dup guard already rejects. + */ +function roleKeysCollide(a: RoleReservation, b: RoleReservation): string | undefined { + if (a.point !== undefined && b.point !== undefined) { + return a.point === b.point ? a.point : undefined + } + if (a.point !== undefined && b.prefix !== undefined) { + return a.point.startsWith(b.prefix) ? a.point : undefined + } + if (b.point !== undefined && a.prefix !== undefined) { + return b.point.startsWith(a.prefix) ? b.point : undefined + } + if (a.prefix !== undefined && b.prefix !== undefined) { + if (a.prefix.startsWith(b.prefix)) return `${a.prefix}` + if (b.prefix.startsWith(a.prefix)) return `${b.prefix}` + } + return undefined +} + + +/** + * Parse + validate a reusable Suite. Reuses `validateRequirements` (the SAME + * id-uniqueness / derived-role-collision / colon guards a PinnedSpec runs) so + * the suite format does not fork the requirement contract. Additionally checks + * the `environments` map shape: each entry must be `{ vars: { ... } }`. + */ +export function parseSuite(text: string): Suite { + const doc = JSON.parse(text) as Suite + if (doc.$type !== 'Suite' || !Array.isArray(doc.requirements)) { + throw new Error('not a Suite: expected {"$type":"Suite","environments":{...},"requirements":[...]}') + } + const envs = doc.environments as unknown + if (envs === null || typeof envs !== 'object' || Array.isArray(envs)) { + throw new Error('Suite.environments must be an object mapping env name -> { vars: { : } }') + } + for (const [name, env] of Object.entries(envs as Record)) { + const vars = (env as { vars?: unknown } | null)?.vars + if (env === null || typeof env !== 'object' || Array.isArray(env) || + vars === null || typeof vars !== 'object' || Array.isArray(vars)) { + throw new Error(`Suite environment "${name}" must be an object of the form { "vars": { : } }`) + } + } + validateRequirements(doc.requirements) + return doc +} diff --git a/src/test-suite.ts b/src/test-suite.ts new file mode 100644 index 0000000..ba274e1 --- /dev/null +++ b/src/test-suite.ts @@ -0,0 +1,322 @@ +/** + * The card-declared PUBLISHED TEST SUITE — the gates, shared by the observe + * side and the judge side. + * + * `interfaces.testSuite` lets a surface publish the workflows it holds itself + * to. The owner's case for it: *"the service COULD publish tests for more + * complex workflows/functionality, but it isn't required because simple crud + * apis and/or lookups don't need that much complexity."* So it is OPTIONAL, and + * a CRUD API that omits the key stays fully conformant — omission is + * conformance, and nothing here can make a surface that passes today fail. + * + * ─── WHY THE GATES LIVE HERE AND NOT IN THE TWO CALLERS ──────────────────── + * The observe side (`discovery.ts`) decides what to FETCH; the judge side + * (`checks.ts`) decides what the fetches MEAN. If those two derived the + * eligibility rules separately they could disagree, and a disagreement between + * observe and judge is the exact incoherence `validateRequirements` throws to + * prevent. So every rule is decided ONCE, here, purely, and both sides call it. + * + * ─── ⚠ THE EXECUTION BOUNDARY, STATED PLAINLY ────────────────────────────── + * A card-declared suite is A STRANGER'S DOCUMENT. It is not the api.qa + * operator's pinned spec, and it carries none of the consent that one does. + * There is NO third-party CODE EXECUTION: `verifySuite` interprets JSON — it + * does not `eval`, `import`, or spawn — so the sandbox question has a concrete + * answer, which is that the existing gates ARE the sandbox. But three of those + * gates are calibrated for consented self-verification and are NOT sufficient + * for a stranger's document, so this module tightens them rather than reusing + * `verifySuite` as-is: + * + * 1. WRITES. `verifyPinnedSpec` sets `allowWrites: true` — "pinned mode is + * consent mode: the target is yours". THAT CONSENT DOES NOT EXIST HERE. + * The sub-run gets its own Observer with `allowWrites: false`, AND any + * requirement whose method is not GET/HEAD fails the check outright. Two + * independent layers, because reusing `verifySuite` naively re-opens this. + * 2. TARGET STEERING. `verifySuite` takes `env.vars.baseUrl` as the target. + * A suite must never redirect the run: the target is pinned to the CARD'S + * OWN ORIGIN and `resolveEndpoint` re-gates every resolved URL same-origin, + * so a `baseUrl` naming another host fails closed instead of being fetched. + * 3. BUDGET. A dedicated cap (`MAX_SUITE_REQUIREMENTS`) and a wall-clock + * deadline, on a private budget so a long suite cannot starve the parent + * run's fixed high-value probes. An over-long suite FAILS — it is NEVER + * truncated, because truncating would let a target hide a failing + * requirement past the cutoff. + * + * And two structural refusals: + * 4. NO SELF-GRADING / NO RECURSION. `kind:'check'` is refused (a target must + * not make api.qa's own verdicts part of its self-test, and + * `published-test-suite` would recurse into itself); `kind:'ax-floor'` is + * refused (a target grading itself); `kind:'surface'` is refused + * (redundant with admission, which already judges those). + * 5. SSRF is unchanged and sufficient: `isPubliclyRoutableSameOrigin` re-gates + * every resolved URL including post-interpolation, so `capture` chaining + * and `{{var}}` interpolation stay enabled — the workflow mechanism is the + * whole point — without opening a new fetch surface. + * + * ─── ⚠ DEVIATION FROM THE DESIGN, DELIBERATE AND REPORTED ────────────────── + * The design permits `kind:'endpoint'` AND `kind:'probe'` in a card-declared + * suite. This implementation permits `endpoint` ONLY and refuses `probe` with + * its own named reason. A `probe` requirement resolves against the target's own + * AXP probe MANIFEST and drags in two-phase ordering, `paramValue.fromProbe` + * derivation, seeded over-ceiling randomization, `minDeclared` and the + * anti-decoy query-flip — machinery that exists to interrogate AXP Clause 4/5 + * ceilings, and that has no meaning in a suite a target writes about its own + * workflows. Refusing MORE than the design refuses cannot create an evasion: + * the failure direction is a target being told its suite is unsupported, never + * a requirement silently skipped. Widening to `probe` is a later, additive + * change and does not alter any verdict reached today. + */ + +import { isPubliclyRoutableSameOrigin } from './http.js' +import { sha256HexSync } from './sha256-sync.js' +import { parseSuite } from './suite-doc.js' +import type { EndpointReq } from './expect.js' +import type { Suite } from './types.js' +import type { TestSuiteClaim } from './discovery.js' + +/** The only suite dialect this verifier implements. */ +export const SUITE_RUNNER = 'api.qa/suite@1' + +/** + * Hard cap on requirements in a CARD-DECLARED suite. Deliberately far below the + * parent run's budget: a stranger's document gets a small, fixed allowance. + * Exceeding it FAILS the check — never truncates (see the header). + */ +export const MAX_SUITE_REQUIREMENTS = 25 + +/** Wall-clock deadline for the whole sub-run, ms. A breach FAILS, never passes what it got. */ +export const SUITE_DEADLINE_MS = 20_000 + +/** `sha256:` + exactly 64 lowercase hex. */ +const DIGEST_FORMAT = /^sha256:[0-9a-f]{64}$/ + +/** The methods a stranger's document may cause api.qa to issue. */ +const SAFE_METHODS = new Set(['GET', 'HEAD']) + +export type CardGate = + | { ok: true; url: string; digest: string } + | { ok: false; problem: string } + +export type DocumentGate = + | { ok: true; suite: Suite; requirements: EndpointReq[]; vars: Record; digest: string } + | { ok: false; problems: string[] } + +/** + * GATE 1 — decided from the CARD ALONE, before anything is fetched. + * + * Everything decidable without the document is decided here so a defective + * declaration costs no request: an off-origin url, a missing or malformed + * digest, and an unknown runner are all refused WITHOUT fetching. + */ +export function gateTestSuiteCard(claim: TestSuiteClaim, origin: string): CardGate { + if (claim.malformed) { + return { + ok: false, + problem: + `interfaces.testSuite is present but is not a JSON object (got ${claim.malformedAs}) — the card claims a published test suite ` + + 'in a shape no verifier can check. Declare an object (`{ "url": …, "digest": "sha256:…" }`), or OMIT the key entirely to ' + + 'declare no test suite — omitting it is fully conforming.', + } + } + if (claim.urlRaw === undefined || claim.url === '') { + return { + ok: false, + problem: + 'interfaces.testSuite declares no `url` — a published suite must say where it is published. ' + + 'Omit the whole key to declare no test suite.', + } + } + // The runner is a DIALECT, not a hint. An unknown one FAILS rather than + // skipping: a card that claims a suite api.qa cannot interpret has made a + // claim that cannot be verified, which is a defective claim, not an absence. + if (claim.runner !== SUITE_RUNNER) { + return { + ok: false, + problem: + `interfaces.testSuite declares runner ${JSON.stringify(claim.runner)}, which api.qa does not implement — the only defined ` + + `suite dialect is ${JSON.stringify(SUITE_RUNNER)} (a digest-pinned api.qa Suite document). This is a FAILURE, not a skip: ` + + 'the card claims a suite this verifier cannot interpret. Omit the key to declare no test suite.', + } + } + // The pin is REQUIRED — see judgeTestSuiteDocument for why a suite needs one + // where a Digital Link description file does not. + if (claim.digest === undefined) { + return { + ok: false, + problem: + 'interfaces.testSuite declares no `digest` — a published suite MUST be digest-pinned ("sha256:<64 hex>" over the document\'s ' + + 'exact bytes). Unpinned, the target can rewrite its assertions between advertising them and being held to them, so the ' + + 'artifact a report cites would not be the artifact that was judged. Refused without fetching.', + } + } + if (!DIGEST_FORMAT.test(claim.digest)) { + return { + ok: false, + problem: + `interfaces.testSuite declares digest ${JSON.stringify(claim.digest)}, which is not of the form "sha256:<64 lowercase hex>" — ` + + 'refused without fetching.', + } + } + const declaredOrigin = urlOriginOrUndefined(claim.url) + if (declaredOrigin === undefined) { + return { + ok: false, + problem: + `interfaces.testSuite declares url ${JSON.stringify(claim.urlRaw ?? claim.url)}, which does not resolve to a URL — ` + + 'refused without fetching.', + } + } + if (declaredOrigin !== origin) { + return { + ok: false, + problem: + `interfaces.testSuite declares its suite at ${claim.url} (origin ${declaredOrigin}), which is NOT the target origin ${origin} — ` + + "a card must not publish another origin's suite; api.qa refused to fetch it", + } + } + if (!isPubliclyRoutableSameOrigin(claim.url, origin)) { + return { + ok: false, + problem: + `interfaces.testSuite declares its suite at ${claim.url}, which is not a publicly-routable target for ${origin} — ` + + 'refused without fetching (SSRF guard)', + } + } + return { ok: true, url: claim.url, digest: claim.digest } +} + +/** + * GATE 2 — decided from the fetched DOCUMENT plus the card. + * + * PURE and synchronous, over the exact bytes the target served. The digest is + * RE-COMPUTED here (never trusted from an observe-phase scalar) so a replayed + * bundle re-derives the pin match from the stored text — which is what keeps + * the anti-Goodhart pin meaningful after serialization. + */ +export function gateTestSuiteDocument( + claim: TestSuiteClaim, + suiteText: string, + cardDigest: string, +): DocumentGate { + const problems: string[] = [] + + // ── The pin, before anything in the document is believed ──────────────── + // WHY A SUITE MUST BE PINNED although a Digital Link description file need + // not be: a description file is a DESCRIPTION, and a verdict citing the + // observed bytes is complete. A test suite is THE TARGET'S OWN ASSERTIONS + // ABOUT ITSELF. Unpinned, the target can rewrite it between advertising and + // running, so the artifact a report cites is not the artifact that was + // judged. The pin is also what makes the claim durable and citable — + // "passes suite sha256:1f0c…" survives; "passes its current suite" does not. + // Same anti-Goodhart argument `verifySuite`'s own `expectedDigest` makes. + const actual = sha256HexSync(suiteText) + const expected = cardDigest.slice('sha256:'.length) + if (actual !== expected) { + return { + ok: false, + problems: [ + `suite digest mismatch: the card pins sha256:${expected} but the document served at ${claim.url} hashes to sha256:${actual} — ` + + 'the published suite is not the one the card claims. Refusing to run it.', + ], + } + } + + let suite: Suite + try { + suite = parseSuite(suiteText) + } catch (e) { + return { + ok: false, + problems: [`suite document did not parse as an api.qa Suite: ${(e as Error).message}`], + } + } + + // ── Environment selection ─────────────────────────────────────────────── + if (!Object.hasOwn(suite.environments, claim.environment)) { + const defined = Object.keys(suite.environments) + problems.push( + `the card selects environment ${JSON.stringify(claim.environment)}, which suite "${suite.name}" does not define ` + + `(it defines ${defined.length ? defined.map((n) => JSON.stringify(n)).join(', ') : 'none'})`, + ) + } + + // ── The cap. FAIL, never truncate ─────────────────────────────────────── + if (suite.requirements.length > MAX_SUITE_REQUIREMENTS) { + problems.push( + `suite declares ${suite.requirements.length} requirements, over the ${MAX_SUITE_REQUIREMENTS} api.qa runs for a ` + + 'card-declared suite. This FAILS rather than running the first ' + + `${MAX_SUITE_REQUIREMENTS}: truncating would let a target hide a failing requirement past the cutoff.`, + ) + } + + // ── Kind eligibility: no self-grading, no recursion ────────────────────── + const requirements: EndpointReq[] = [] + for (const req of suite.requirements) { + if (req.kind === 'endpoint') { + requirements.push(req) + continue + } + if (req.kind === 'check') { + problems.push( + `requirement "${req.id}" is kind:'check' — a card-declared suite must not make api.qa's own verdicts part of a target's ` + + 'self-test (and published-test-suite would recurse into itself). Refused.', + ) + } else if (req.kind === 'ax-floor') { + problems.push( + `requirement "${req.id}" is kind:'ax-floor' — a card-declared suite must not assert the target's own AX grade. Refused.`, + ) + } else if (req.kind === 'surface') { + problems.push( + `requirement "${req.id}" is kind:'surface' — the admission spec already judges the surfaces; a self-published suite ` + + 'restating them verifies nothing. Refused.', + ) + } else if (req.kind === 'probe') { + problems.push( + `requirement "${req.id}" is kind:'probe' — api.qa does not run probe-manifest requirements from a card-declared suite in ` + + `${SUITE_RUNNER}. A probe resolves against the target's own AXP probe manifest and carries the Clause 4/5 ceiling ` + + 'machinery (paramValue derivation, seeded over-ceiling amounts, the anti-decoy query flip), which has no meaning in a ' + + "suite a target writes about its own workflows. Express the workflow as kind:'endpoint' requirements.", + ) + } + } + + // ── Writes. A stranger's document gets GET/HEAD only ──────────────────── + // Checked on the DECLARED method (pre-interpolation) as well as at the fetch + // site: a `{{var}}` in `method` cannot be resolved without running, so a + // method that is not a literal safe verb is refused up front rather than + // being allowed to resolve into one. + for (const req of requirements) { + const method = req.method.toUpperCase() + if (!SAFE_METHODS.has(method)) { + problems.push( + `requirement "${req.id}" declares method ${JSON.stringify(req.method)} — a card-declared suite runs GET/HEAD ONLY. ` + + 'Pinned-spec mode allows writes because the target is the operator\'s own; a suite named by a stranger\'s card carries ' + + 'no such consent, so api.qa will not be directed to issue a write against it.', + ) + } + } + + if (problems.length > 0) return { ok: false, problems } + if (requirements.length === 0) { + // Unreachable via parseSuite (its non-vacuity guard rejects an empty list), + // but a suite of entirely-refused kinds would have been caught above; keep + // the guard so a future kind widening cannot introduce a vacuous pass. + return { ok: false, problems: ['suite declares no runnable requirements — refusing to vacuously pass'] } + } + + return { + ok: true, + suite, + requirements, + vars: { ...(suite.environments[claim.environment]?.vars ?? {}) }, + digest: cardDigest, + } +} + +/** The origin of a URL, or undefined if it does not parse. */ +function urlOriginOrUndefined(url: string): string | undefined { + try { + return new URL(url).origin + } catch { + return undefined + } +} diff --git a/test/optional-interfaces.test.ts b/test/optional-interfaces.test.ts index c894bd4..b216d99 100644 --- a/test/optional-interfaces.test.ts +++ b/test/optional-interfaces.test.ts @@ -652,10 +652,24 @@ describe('a not-applicable requirement is distinguishable from a pass and from a expect([violated.verdict, violated.notApplicable !== undefined]).toEqual(['fail', false]) }) - it('a NEVER-PRODUCED check is still a hard fail, never a not-applicable', async () => { - // `published-test-suite` is REGISTERED as eligible but api.qa does not - // produce it yet. Against a DECLARING card the requirement must fail - // loudly — a verifier too old for the spec it was handed says so. + /* ⚠ REWRITTEN AT THE `published-test-suite` MERGE, deliberately, and the + reason is recorded rather than the assertion just relaxed. + + As written on afk/apiqa-optional-declared this case asserted + `detail` matched /unknown check/: `published-test-suite` was REGISTERED as + eligible but `runChecks` did not produce it, and the test documented that + hole — a spec pinning an unproducible check against a DECLARING card fails + loudly instead of skipping. afk/apiqa-testsuite-interface CLOSES the hole: + the check now exists, so the declaring card gets a real judgement and the + detail names a real defect (an unpinned suite) instead of a missing check. + + The INVARIANT under test is unchanged and is what both halves below assert: + against a card that DECLARED the interface, a requirement api.qa cannot + pass is a HARD FAIL and never a not-applicable. Only the two routes to it + are now distinct, so both are exercised. */ + it('a DECLARED-but-defective interface is a hard fail, never a not-applicable', async () => { + // The card declares `interfaces.testSuite` and the declaration is broken + // (no `digest`). A claim was made, so it is JUDGED — never excused. const spec = specText([ { id: 'suite-req', kind: 'check', check: 'published-test-suite', must: 'pass', appliesWhen: { cardDeclares: 'interfaces.testSuite' } }, @@ -670,6 +684,23 @@ describe('a not-applicable requirement is distinguishable from a pass and from a const r = report.requirements.find((x) => x.id === 'suite-req')! expect(r.verdict).toBe('fail') expect(r.notApplicable).toBeUndefined() + expect(r.detail).toMatch(/digest/) + }) + + it('a NEVER-PRODUCED check is still a hard fail, never a not-applicable', async () => { + // A check id `runChecks` does not produce. It cannot be declaration-armed — + // the evasion guard refuses an unregistered check — so the only way to pin + // it is bare, and a bare pin over a check that never appears must fail + // loudly. That is what a verifier too old for the spec it was handed does. + const spec = specText([ + { id: 'future-req', kind: 'check', check: 'a-check-from-the-future', must: 'pass' }, + ]) + const report = await verifyPinnedSpec(GOOD, spec, { + fetcher: makeFetcher(goodTargetRoutes()), delayMs: 0, seed: 7, mode: 'local', + }) + const r = report.requirements.find((x) => x.id === 'future-req')! + expect(r.verdict).toBe('fail') + expect(r.notApplicable).toBeUndefined() expect(r.detail).toMatch(/unknown check/) }) diff --git a/test/sha256-sync.test.ts b/test/sha256-sync.test.ts new file mode 100644 index 0000000..c3e7767 --- /dev/null +++ b/test/sha256-sync.test.ts @@ -0,0 +1,55 @@ +import { describe, it, expect } from 'vitest' +import { sha256HexSync } from '../src/sha256-sync.js' +import { sha256Hex } from '../src/digest.js' + +/** + * `sha256-sync.ts` exists only because `runChecks` is synchronous and WebCrypto + * is not (see that file's header). A second hash implementation is only + * acceptable if it is PROVED equal to the first, so this suite is the proof: + * published FIPS-180-4 vectors, then agreement with the estate's WebCrypto + * `sha256Hex` across every shape that has ever broken a hand-written SHA-256 — + * the padding block boundaries above all. + */ +describe('sha256HexSync', () => { + it('matches the published FIPS-180-4 vectors', () => { + expect(sha256HexSync('')).toBe('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') + expect(sha256HexSync('abc')).toBe('ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad') + expect(sha256HexSync('abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq')).toBe( + '248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1', + ) + expect(sha256HexSync('abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu')).toBe( + 'cf5b16a778af8380036ce59e7b0492370b249b11e8f07a51afac45037afee9d1', + ) + }) + + it('agrees with WebCrypto sha256Hex on the padding block boundaries', async () => { + // 55/56 straddle the "length fits in this block" boundary; 63/64 the block + // size itself; 119/120 the two-block equivalent. These are where a + // hand-written padding implementation goes wrong and nowhere else. + for (const n of [0, 1, 54, 55, 56, 57, 63, 64, 65, 118, 119, 120, 121, 127, 128, 129]) { + const s = 'a'.repeat(n) + expect(sha256HexSync(s), `length ${n}`).toBe(await sha256Hex(s)) + } + }) + + it('agrees with WebCrypto sha256Hex on multi-byte UTF-8 and large inputs', async () => { + const cases = [ + 'héllo wörld', // 2-byte sequences + '日本語テキスト', // 3-byte sequences + '🧪🔬 emoji surrogate pairs', // 4-byte sequences + JSON.stringify({ $type: 'Suite', name: 'x', requirements: [{ id: 'a' }] }), + 'x'.repeat(10_000), + Array.from({ length: 2000 }, (_, i) => `line ${i}\n`).join(''), + ] + for (const s of cases) { + expect(sha256HexSync(s), JSON.stringify(s.slice(0, 24))).toBe(await sha256Hex(s)) + } + }) + + it('is sensitive to a single-byte change (the whole point of the pin)', () => { + const a = sha256HexSync('{"$type":"Suite","name":"a"}') + const b = sha256HexSync('{"$type":"Suite","name":"b"}') + expect(a).not.toBe(b) + expect(a).toMatch(/^[0-9a-f]{64}$/) + }) +}) diff --git a/test/test-suite-interface.test.ts b/test/test-suite-interface.test.ts new file mode 100644 index 0000000..515856f --- /dev/null +++ b/test/test-suite-interface.test.ts @@ -0,0 +1,575 @@ +/** + * The OPTIONAL, card-DECLARED published test suite (`interfaces.testSuite`) + * and its check, `published-test-suite`. + * + * The owner's case for the interface: *"the service COULD publish tests for + * more complex workflows/functionality, but it isn't required because simple + * crud apis and/or lookups don't need that much complexity."* So the first + * property below is the one that matters most, and everything else exists to + * make sure buying it did not cost anything. + * + * 1. **Undeclared is not a failure.** A card that omits the key SKIPs, is + * never fetched for, spends no budget, and its grade is untouched. A CRUD + * API that publishes no suite stays fully conformant. + * 2. **Declared is judged strictly**, because a machine-readable claim is one + * a verifier will believe — and because an optionality mechanism is an + * EVASION mechanism if a defective declaration can buy a skip. Every + * failing case is here: digest mismatch, off-origin url, missing/malformed + * pin, unknown runner, absent environment, refused kinds, the requirement + * cap, and a target that violates its own published assertions. + * 3. **The execution boundary is real and tested**, not asserted in a + * comment: writes refused, target pinned to the card origin against a + * hostile `baseUrl`, sub-run evidence namespaced so it cannot shadow the + * parent's. + */ + +import { describe, it, expect } from 'vitest' +import { Observer } from '../src/http.js' +import { observeTarget, ROLE, SUITE_ROLE_PREFIX, parseAgentsJson } from '../src/discovery.js' +import { runChecks } from '../src/checks.js' +import { axScoreOf, gradeOf } from '../src/grade.js' +import { sha256HexSync } from '../src/sha256-sync.js' +import { MAX_SUITE_REQUIREMENTS, SUITE_RUNNER } from '../src/test-suite.js' +import { OPTIONAL_DECLARED_INTERFACES, OPTIONAL_INTERFACE_PATH_RE } from '../src/optional-interfaces.js' +import type { CheckResult } from '../src/types.js' +import { GOOD, goodTargetRoutes, makeFetcher, withOverrides, type Routes } from './helpers.js' + +const SUITE_PATH = '/.well-known/axp/suite.json' +const OMIT = Symbol('omit') + +const json = (value: unknown, contentType = 'application/json') => () => ({ + status: 200, + contentType, + body: JSON.stringify(value), +}) + +/** A suite the reference target genuinely passes: a two-step read workflow. */ +function validSuite(extra: Record = {}) { + return { + $type: 'Suite', + name: 'good.example public contracts', + version: '1.0.0', + environments: { public: { vars: {} } }, + requirements: [ + { + id: 'status-ok', + kind: 'endpoint', + method: 'GET', + path: '/api/status', + expect: { status: 200, paths: [{ path: 'ok', equals: true }] }, + }, + { + id: 'widgets-list', + kind: 'endpoint', + method: 'GET', + path: '/api/widgets', + expect: { status: 200, paths: [{ path: '0.id', exists: true }] }, + capture: { first: '0.id' }, + }, + ], + ...extra, + } +} + +/** + * Build routes whose card declares `interfaces.testSuite`, with the digest + * computed over the EXACT bytes served — so a test that wants a mismatch has to + * ask for one explicitly rather than getting one by accident. + */ +function routesFor( + opts: { + suite?: unknown + declaration?: unknown | typeof OMIT + /** Override the card's digest (e.g. to force a mismatch). */ + digest?: string + /** Omit the suite route entirely (404). */ + serveSuite?: boolean + /** Serve something other than the JSON suite text. */ + suiteBody?: { status: number; contentType: string; body: string } + extraRoutes?: Routes + } = {}, +): Routes { + const base = goodTargetRoutes() + const suite = opts.suite ?? validSuite() + const suiteText = typeof suite === 'string' ? suite : JSON.stringify(suite) + const card = JSON.parse( + base['GET /.well-known/agents.json']!({ method: 'GET', accept: 'application/json' }).body!, + ) as Record + + const declaration = + opts.declaration !== undefined + ? opts.declaration + : { url: SUITE_PATH, digest: opts.digest ?? `sha256:${sha256HexSync(suiteText)}` } + if (declaration !== OMIT) card.interfaces.testSuite = declaration + + const suiteRoute: Routes = {} + if (opts.serveSuite !== false) { + suiteRoute[`GET ${SUITE_PATH}`] = opts.suiteBody + ? () => opts.suiteBody! + : () => ({ status: 200, contentType: 'application/json', body: suiteText }) + } + + return withOverrides(base, { + 'GET /.well-known/agents.json': json(card), + ...suiteRoute, + ...(opts.extraRoutes ?? {}), + }) +} + +async function judge(routes: Routes, origin = GOOD) { + const calls: string[] = [] + const inner = makeFetcher(routes, origin) + const observer = new Observer({ + fetcher: async (url, init) => { + calls.push(url) + return inner(url, init) + }, + delayMs: 0, + }) + const bundle = await observeTarget(origin, observer, 7) + const checks = runChecks(bundle) + const { grade } = gradeOf(axScoreOf(checks), checks) + return { bundle, checks, grade, calls } +} + +const ts = (checks: CheckResult[]) => checks.find((c) => c.id === 'published-test-suite')! + +// --------------------------------------------------------------------------- +// 1. Undeclared is fully conforming — the property the interface exists for +// --------------------------------------------------------------------------- + +describe('a card that declares no test suite is fully conforming', () => { + it('SKIPs, with the wording that says a skip is not a free pass (§8.24)', async () => { + const { checks } = await judge(routesFor({ declaration: OMIT })) + const c = ts(checks) + expect(c.verdict).toBe('skip') + expect(c.detail).toBe( + 'no published test suite interface declared (agents.json `interfaces.testSuite` absent) — the interface is OPTIONAL and this card does not claim it, so nothing was fetched and nothing is judged; under a pinned must:pass this fails closed', + ) + }) + + it('costs no fetch and no budget — nothing under the suite path is ever requested', async () => { + const { calls } = await judge(routesFor({ declaration: OMIT })) + expect(calls.some((u) => u.includes('suite'))).toBe(false) + }) + + it('leaves the grade and every other check identical to a card with no such key', async () => { + const withKey = await judge(routesFor({ declaration: OMIT })) + const plain = await judge(goodTargetRoutes()) + expect(withKey.grade).toBe(plain.grade) + // Every check except the new one behaves identically. + const strip = (cs: CheckResult[]) => + cs.filter((c) => c.id !== 'published-test-suite').map((c) => `${c.id}:${c.verdict}`) + expect(strip(withKey.checks)).toEqual(strip(plain.checks)) + }) + + it('the check moves no AX point — it is an additive readiness dimension', async () => { + const declared = await judge(routesFor()) + const undeclared = await judge(routesFor({ declaration: OMIT })) + expect(ts(declared.checks).axItem).toBeUndefined() + expect(axScoreOf(declared.checks).points).toBe(axScoreOf(undeclared.checks).points) + }) +}) + +// --------------------------------------------------------------------------- +// 2. The happy path — declared, pinned, and actually passed +// --------------------------------------------------------------------------- + +describe('a declared suite the surface genuinely passes', () => { + it('PASSes, and the detail is legible about what was and was not judged', async () => { + const { checks } = await judge(routesFor()) + const c = ts(checks) + expect(c.verdict).toBe('pass') + expect(c.detail).toContain('interfaces.testSuite declared') + expect(c.detail).toContain('matches the card pin') + expect(c.detail).toContain('2 requirement(s) over 2 distinct pathname(s), all passed') + expect(c.detail).toContain('GET/HEAD-only with writes disabled') + // §6.5 — triviality is made LEGIBLE, never silently judged. + expect(c.detail).toContain('NOT judged: whether the suite is ambitious') + }) + + it('records each exchange under a `suite:`-namespaced role and cites them as evidence', async () => { + const { bundle, checks } = await judge(routesFor()) + const roles = bundle.items.map((e) => e.role) + expect(roles).toContain(ROLE.testSuite) + expect(roles).toContain(`${SUITE_ROLE_PREFIX}pinned:status-ok`) + expect(roles).toContain(`${SUITE_ROLE_PREFIX}pinned:widgets-list`) + expect(ts(checks).evidence).toContain(`${SUITE_ROLE_PREFIX}pinned:status-ok`) + }) + + it('re-judges identically from the stored bundle — replay needs no refetch', async () => { + const { bundle, checks } = await judge(routesFor()) + // runChecks is pure over the bundle; the digest is RE-COMPUTED from the + // stored suite text, so the pin still binds after serialization. + const replayed = runChecks(JSON.parse(JSON.stringify(bundle))) + expect(ts(replayed).verdict).toBe('pass') + expect(ts(replayed).detail).toBe(ts(checks).detail) + }) + + it('runs a CAPTURE-CHAINED workflow — the multi-step case the interface exists for', async () => { + const suite = { + $type: 'Suite', + name: 'chained', version: '1.0.0', + environments: { public: { vars: {} } }, + requirements: [ + { + id: 'list', kind: 'endpoint', method: 'GET', path: '/api/widgets', + expect: { status: 200 }, capture: { first: '0.id' }, + }, + { + id: 'fetch-one', kind: 'endpoint', method: 'GET', path: '/api/widgets/{{first}}', + expect: { status: 200, paths: [{ path: 'id', equals: 'w1' }] }, + }, + ], + } + const { bundle, checks } = await judge( + routesFor({ suite, extraRoutes: { 'GET /api/widgets/w1': json({ id: 'w1', name: 'widget one' }) } }), + ) + expect(ts(checks).verdict).toBe('pass') + // The chained URL really was resolved from the captured value. + const ev = bundle.items.find((e) => e.role === `${SUITE_ROLE_PREFIX}pinned:fetch-one`) + expect(ev?.url).toBe(`${GOOD}/api/widgets/w1`) + }) +}) + +// --------------------------------------------------------------------------- +// 3. THE FAILING CASES (§8.15-§8.25) +// --------------------------------------------------------------------------- + +describe('a declared suite is judged strictly — the failing cases', () => { + it('§8.15 digest MISMATCH fails, and the detail names both digests', async () => { + const bogus = `sha256:${'0'.repeat(64)}` + const { checks } = await judge(routesFor({ digest: bogus })) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('suite digest mismatch') + expect(c.detail).toContain(bogus.slice('sha256:'.length)) + expect(c.detail).toContain(sha256HexSync(JSON.stringify(validSuite()))) + }) + + it('§8.16 an OFF-ORIGIN url fails WITHOUT fetching it', async () => { + const { checks, calls } = await judge( + routesFor({ declaration: { url: 'https://evil.example/suite.json', digest: `sha256:${'a'.repeat(64)}` } }), + ) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('NOT the target origin') + expect(calls.some((u) => u.includes('evil.example'))).toBe(false) + }) + + it('§8.17 a POST requirement fails — writes are refused, never issued', async () => { + const suite = validSuite() + ;(suite.requirements as any[]).push({ + id: 'create', kind: 'endpoint', method: 'POST', path: '/api/widgets', + body: { name: 'x' }, expect: { status: 201 }, + }) + const { checks, bundle } = await judge(routesFor({ suite })) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('GET/HEAD ONLY') + expect(c.detail).toContain('"create"') + // And nothing ran at all — an ineligible suite is refused whole, not partly + // run up to the offending requirement. + expect(bundle.items.some((e) => e.role.startsWith(SUITE_ROLE_PREFIX))).toBe(false) + // No POST was ever issued against the target. + expect(bundle.items.some((e) => e.method === 'POST')).toBe(false) + }) + + it("§8.18 a kind:'check' requirement fails — no self-grading, no recursion", async () => { + const suite = validSuite() + ;(suite.requirements as any[]).push({ + id: 'self', kind: 'check', check: 'published-test-suite', must: 'pass', + }) + const { checks } = await judge(routesFor({ suite })) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain("kind:'check'") + expect(c.detail).toContain('recurse') + }) + + it("§8.18b kind:'ax-floor' and kind:'surface' are refused too", async () => { + for (const [req, needle] of [ + [{ id: 'floor', kind: 'ax-floor', minScore: 1 }, "kind:'ax-floor'"], + [{ id: 'surf', kind: 'surface', surface: 'llms.txt', must: 'present' }, "kind:'surface'"], + ] as const) { + const suite = validSuite() + ;(suite.requirements as any[]).push(req) + const c = ts((await judge(routesFor({ suite }))).checks) + expect(c.verdict, needle).toBe('fail') + expect(c.detail).toContain(needle) + } + }) + + it('§8.19 a suite over the requirement cap FAILS — it is never truncated', async () => { + const suite = validSuite() + suite.requirements = Array.from({ length: MAX_SUITE_REQUIREMENTS + 1 }, (_, i) => ({ + id: `r${i}`, kind: 'endpoint', method: 'GET', path: '/api/status', + expect: { status: 200 }, + })) as any + const { checks, bundle } = await judge(routesFor({ suite })) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain(`${MAX_SUITE_REQUIREMENTS + 1} requirements`) + expect(c.detail).toContain('hide a failing requirement past the cutoff') + // Proof it truncated nothing: no suite requirement ran at all. + expect(bundle.items.some((e) => e.role.startsWith(SUITE_ROLE_PREFIX))).toBe(false) + }) + + it('§8.20 an environment the suite does not define fails', async () => { + const { checks } = await judge( + routesFor({ + declaration: { + url: SUITE_PATH, + digest: `sha256:${sha256HexSync(JSON.stringify(validSuite()))}`, + environment: 'staging', + }, + }), + ) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('"staging"') + expect(c.detail).toContain('does not define') + }) + + it('§8.21 an unknown runner FAILS — it does not skip', async () => { + const { checks, calls } = await judge( + routesFor({ + declaration: { + url: SUITE_PATH, + digest: `sha256:${sha256HexSync(JSON.stringify(validSuite()))}`, + runner: 'vitest@3', + }, + }), + ) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.verdict).not.toBe('skip') + expect(c.detail).toContain('vitest@3') + expect(c.detail).toContain('This is a FAILURE, not a skip') + expect(calls.some((u) => u.includes('suite.json'))).toBe(false) + }) + + it('§8.22 a target that VIOLATES its own published suite fails, naming the requirement', async () => { + // The suite asserts /api/status returns ok:true; the target says ok:false. + const { checks } = await judge( + routesFor({ extraRoutes: { 'GET /api/status': json({ ok: false, widgets: 3 }) } }), + ) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('violated its OWN published suite') + expect(c.detail).toContain('"status-ok"') + expect(c.detail).toContain('path ok = false, wanted true') + }) + + it('§8.23 a hostile `baseUrl` env var cannot steer the run off the card origin', async () => { + const suite = { + $type: 'Suite', name: 'steered', version: '1.0.0', + environments: { public: { vars: { baseUrl: 'https://evil.example' } } }, + requirements: [ + // An ABSOLUTE off-origin interpolation: resolveEndpoint must refuse it. + { id: 'steer', kind: 'endpoint', method: 'GET', path: '{{baseUrl}}/api/status', expect: { status: 200 } }, + ], + } + const { checks, calls } = await judge(routesFor({ suite })) + expect(calls.some((u) => u.includes('evil.example'))).toBe(false) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('off-origin/private url') + }) + + it('MISSING suite evidence fails closed — the property the wall-clock deadline relies on', async () => { + // The sub-run stops at SUITE_DEADLINE_MS, leaving later requirements + // unobserved. Waiting 20s in CI would be absurd, so the load-bearing half + // is tested directly: an unobserved requirement must FAIL, never pass. That + // is what makes a deadline breach fail-closed rather than "pass what we + // got" — the same reason an over-long suite is refused instead of truncated. + const { bundle } = await judge(routesFor()) + const starved = { + ...bundle, + items: bundle.items.filter((e) => e.role !== `${SUITE_ROLE_PREFIX}pinned:widgets-list`), + } + const c = ts(runChecks(starved)) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('"widgets-list"') + expect(c.detail).toContain('not observed') + }) + + it('§8.25 suite evidence cannot shadow the parent run — roles are namespaced', async () => { + // A suite requirement id chosen to collide with a plausible admission-spec + // requirement id. Unprefixed, `find(role === 'pinned:status-ok')` would + // resolve suite evidence for an admission requirement (or vice versa). + const suite = { + $type: 'Suite', name: 'collide', version: '1.0.0', + environments: { public: { vars: {} } }, + requirements: [ + { id: 'status-ok', kind: 'endpoint', method: 'GET', path: '/api/status', expect: { status: 200 } }, + ], + } + const { bundle, checks } = await judge(routesFor({ suite })) + expect(ts(checks).verdict).toBe('pass') + const roles = bundle.items.map((e) => e.role) + // The suite's evidence exists ONLY under the namespace… + expect(roles).toContain(`${SUITE_ROLE_PREFIX}pinned:status-ok`) + // …and never leaks a bare `pinned:` role into the parent bundle. + expect(roles.filter((r) => r.startsWith('pinned:'))).toEqual([]) + expect(roles.filter((r) => r === 'pinned:status-ok')).toEqual([]) + }) +}) + +// --------------------------------------------------------------------------- +// 4. Defective declarations FAIL — a bad claim never buys a skip +// --------------------------------------------------------------------------- + +describe('a defective declaration fails rather than skipping (the evasion guard)', () => { + it('a present-but-non-object value is MALFORMED, never treated as absent', async () => { + for (const [value, name] of [ + [null, 'null'], [false, 'boolean'], [0, 'number'], ['', 'string'], + ['yes', 'string'], [[], 'array'], + ] as const) { + const c = ts((await judge(routesFor({ declaration: value }))).checks) + expect(c.verdict, JSON.stringify(value)).toBe('fail') + expect(c.verdict).not.toBe('skip') + expect(c.detail).toContain(`got ${name}`) + expect(c.detail).toContain('omitting it is fully conforming') + } + }) + + it('an EMPTY object declaration fails for having no url — presence is a claim', async () => { + const c = ts((await judge(routesFor({ declaration: {} }))).checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('declares no `url`') + }) + + it('a MISSING digest fails without fetching — the pin is required', async () => { + const { checks, calls } = await judge(routesFor({ declaration: { url: SUITE_PATH } })) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('MUST be digest-pinned') + expect(c.detail).toContain('rewrite its assertions') + expect(calls.some((u) => u.includes('suite.json'))).toBe(false) + }) + + it('a malformed digest STRING fails without fetching', async () => { + for (const digest of ['deadbeef', 'sha256:xyz', `sha256:${'A'.repeat(64)}`, `md5:${'a'.repeat(32)}`]) { + const { checks, calls } = await judge(routesFor({ declaration: { url: SUITE_PATH, digest } })) + expect(ts(checks).verdict, digest).toBe('fail') + expect(ts(checks).detail).toContain('sha256:<64 lowercase hex>') + expect(calls.some((u) => u.includes('suite.json'))).toBe(false) + } + }) + + it('a declared suite that 404s fails', async () => { + const { checks } = await judge(routesFor({ serveSuite: false })) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('did not answer 2xx') + }) + + it('a declared suite whose body is not JSON fails on the digest or the parse, never passes', async () => { + const body = 'not json at all' + const { checks } = await judge( + routesFor({ + digest: `sha256:${sha256HexSync(body)}`, + suiteBody: { status: 200, contentType: 'application/json', body }, + }), + ) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('did not parse as an api.qa Suite') + }) + + it('a JSON document that is not a Suite fails', async () => { + const body = JSON.stringify({ $type: 'PinnedSpec', requirements: [] }) + const { checks } = await judge( + routesFor({ digest: `sha256:${sha256HexSync(body)}`, suiteBody: { status: 200, contentType: 'application/json', body } }), + ) + expect(ts(checks).verdict).toBe('fail') + expect(ts(checks).detail).toContain('not a Suite') + }) + + it('an EMPTY requirement list is refused — a vacuous suite must not pass', async () => { + const suite = { ...validSuite(), requirements: [] } + const { checks } = await judge(routesFor({ suite })) + expect(ts(checks).verdict).toBe('fail') + expect(ts(checks).detail).toContain('verifies nothing') + }) + + it("§8.24' a suite whose requirement is kind:'probe' is refused with its own named reason", async () => { + const suite = validSuite() + ;(suite.requirements as any[]).push({ + id: 'probe-it', kind: 'probe', probe: 'pricing', expect: { status: 200 }, + }) + const c = ts((await judge(routesFor({ suite }))).checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain("kind:'probe'") + expect(c.detail).toContain(SUITE_RUNNER) + }) +}) + +// --------------------------------------------------------------------------- +// 5. The card parse — presence, not truthiness +// --------------------------------------------------------------------------- + +describe('parseAgentsJson reads the declaration by PRESENCE', () => { + it('absent key ⇒ undefined; present key ⇒ declared, whatever the value', () => { + expect(parseAgentsJson({ interfaces: {} }, GOOD).testSuite).toBeUndefined() + expect(parseAgentsJson({ interfaces: { testSuite: {} } }, GOOD).testSuite?.declared).toBe(true) + expect(parseAgentsJson({ interfaces: { testSuite: null } }, GOOD).testSuite?.malformed).toBe(true) + }) + + it('defaults environment and runner, and absolutizes a relative url same-origin', () => { + const c = parseAgentsJson({ interfaces: { testSuite: { url: '/s.json' } } }, GOOD).testSuite! + expect(c.environment).toBe('public') + expect(c.runner).toBe(SUITE_RUNNER) + expect(c.url).toBe(`${GOOD}/s.json`) + }) + + it('PRESERVES an absolute off-origin url verbatim so the gate can drop it', () => { + const c = parseAgentsJson( + { interfaces: { testSuite: { url: 'https://evil.example/s.json' } } }, GOOD, + ).testSuite! + expect(c.url).toBe('https://evil.example/s.json') + }) + + it('does NOT satisfy the non-empty interfaces obligation — a suite is not a way to call the API', () => { + // Clause 6's `check-card-interfaces-linked` reads endpoints/mcp only. A + // sibling key must be structurally incapable of satisfying it. + const c = parseAgentsJson({ interfaces: { testSuite: { url: '/s.json' } } }, GOOD) + expect(c.endpoints).toEqual([]) + expect(c.mcp).toBeUndefined() + }) +}) + +// --------------------------------------------------------------------------- +// 6. The registry — eligibility, and its grammar +// --------------------------------------------------------------------------- + +describe('the optional-interface registry', () => { + it('binds published-test-suite to interfaces.testSuite, and only that', () => { + expect(OPTIONAL_DECLARED_INTERFACES['published-test-suite']).toBe('interfaces.testSuite') + }) + + it('every registered path matches the two-segment grammar', () => { + for (const [check, path] of Object.entries(OPTIONAL_DECLARED_INTERFACES)) { + expect(OPTIONAL_INTERFACE_PATH_RE.test(path), `${check} -> ${path}`).toBe(true) + } + }) + + it('is frozen — eligibility is not editable at runtime', () => { + expect(Object.isFrozen(OPTIONAL_DECLARED_INTERFACES)).toBe(true) + }) + + it('registers only ADDITIVE capabilities — no check any AXP clause binds', () => { + // The evasion guard, from api.qa's side: these are the checks the ratified + // AXP spec pins as unconditional MUSTs. None may ever become + // declaration-armed, or the MUST stops being a MUST. + const axpMustChecks = [ + 'agents-json', 'machine-legible-home', 'conneg-accept', 'conneg-client-class', + 'conneg-alternates', 'conneg-forced-face', 'card-interfaces-linked', + 'probe-manifest', 'keyless-flow', 'offers-402', + ] + for (const id of axpMustChecks) { + expect(OPTIONAL_DECLARED_INTERFACES, id).not.toHaveProperty(id) + } + }) +})