diff --git a/package.json b/package.json index 75e36e5..1af6243 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "autonomous-qa", - "version": "0.2.0", + "version": "0.3.0", "description": "The external third-party verifier for agent-first APIs. Discovery from published machine surfaces, contract-derived deterministic checks, attested public grade reports. Reference client for the hosted service at https://api.qa.", "license": "MIT", "type": "module", diff --git a/src/index.ts b/src/index.ts index 675f7ba..89e10f3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,11 +17,22 @@ export { validateRequirements, verifySuite, parseSuite, + readCardDeclaration, type PinnedReport, type VerifyPinnedOpts, type SuiteReport, type VerifySuiteOpts, + type CardDeclarationState, } from './pinned.js' +// The optional-declared-interface registry is PUBLIC on purpose: a spec author +// needs to know which checks may be declaration-armed BEFORE writing a spec +// that throws, and an auditor needs to be able to read the closed list without +// reading the source. Exporting it does not widen it — it is frozen. +export { + OPTIONAL_DECLARED_INTERFACES, + OPTIONAL_INTERFACE_PATH_RE, + eligibleOptionalChecks, +} from './optional-interfaces.js' export { observeTarget, deriveDiscovery, diff --git a/src/optional-interfaces.ts b/src/optional-interfaces.ts new file mode 100644 index 0000000..e24cd0d --- /dev/null +++ b/src/optional-interfaces.ts @@ -0,0 +1,111 @@ +/** + * The OPTIONAL-DECLARED-INTERFACE registry — api.qa's own evasion guard. + * + * ── Why this file exists ──────────────────────────────────────────────────── + * + * `appliesWhen` has two arms (see types.ts). The `cardDeclares` arm lets a + * pinned requirement be SKIPPED when the target's capability card does not + * declare a named optional interface. That is exactly the shape of an EVASION + * MECHANISM: if any requirement could be gated on a card key, then every MUST + * clause reachable that way stops being a MUST — a target opts out of it by + * simply not writing the key. + * + * So the arm is not general. It is legal only for checks that appear in the + * frozen map below, and only when bound to the exact card path this map pairs + * with the check. Everything else THROWS at spec-parse time, before a single + * probe fires (see `validateRequirements` in pinned.ts). + * + * ── Why the map is keyed by CHECK ID, and why it lives in api.qa ──────────── + * + * A requirement `id` is a string the SPEC AUTHOR chooses; a `check` id is a + * string API.QA owns and an author cannot mint. The document under attack is + * the spec, so eligibility must be anchored in something the spec cannot + * declare into existence. A flag on the requirement (`optional: true`) would be + * a guard whose value is set by the party being guarded — no guard at all. + * + * And it lives HERE, in the verifier, never imported from the standard. api.qa + * must be able to fail the standard's own authors; a registry generated from + * `PROTOCOL.md` would make the verifier a mirror of the document it audits. The + * standard carries its own structural mirror of this list in its own repo, and + * the two agree by REVIEW and by two independent tests — never by a shared + * module. If you are about to generate one from the other, stop. + * + * ── What belongs in this map ──────────────────────────────────────────────── + * + * ONLY ADDITIVE CAPABILITIES: a check that verifies something a surface MAY + * offer, whose absence is fully conforming. Never a check any always-required + * clause binds. Adding an entry is a deliberate source change in this file, it + * gets a code review, and — if the check is one a MUST clause binds — it turns + * the standard's independent disjointness test red the moment the two lists are + * compared. Two tripwires, no shared import. + * + * ── The asymmetry that makes the skip legitimate ──────────────────────────── + * + * The `fromProbe` arm fails CLOSED on a missing source because a missing probe + * response is an OBSERVATION FAILURE: the verifier asked and did not get an + * answer, so it cannot tell "does not apply to me" from "I am broken" or "I am + * evading". Absence of evidence is not evidence. + * + * The `cardDeclares` arm skips on absence because there THE CARD IS THE ANSWER. + * A card that was fetched, parsed, and found well-formed, and that omits the + * key, has affirmatively said "I do not offer this". Absence IN a retrieved + * document is a datum; absence OF the document is not — which is why every case + * where the card itself could not be read collapses back to the fail-closed + * posture (see `readCardDeclaration` in pinned.ts). + */ + +/** + * The CLOSED registry of ADDITIVE, DECLARATION-ARMED capabilities. + * + * `checkId -> the ONE \`interfaces.\` card path that arms it.` + * + * A check in this map MAY be pinned with `appliesWhen: { cardDeclares: … }`. + * A check NOT in this map can NEVER be made conditional on a card key — which + * is what makes it structurally impossible to opt out of an always-required + * clause by omission. + * + * Note the direction of the guarantee: this map restricts what may be SKIPPED, + * never what may be REQUIRED. An allowlisted check pinned WITHOUT `appliesWhen` + * is legal and means "I demand this of everyone"; its `skip` then fails closed + * under `must: 'pass'` exactly as any other skip does. That is the escape hatch, + * and it needs no new machinery. + */ +export const OPTIONAL_DECLARED_INTERFACES: Readonly> = Object.freeze({ + /** + * GS1 Digital Link resolver discovery indicator. Implemented (checks.ts). + * The card key is the whole arming signal — NOT `axpClaimed`: an optional + * interface is armed by its own declaration, not by the AXP opt-in. + */ + 'digital-link-resolver': 'interfaces.digitalLink', + /** + * A card-published, digest-pinned api.qa Suite the target asserts about + * itself. + * + * ⚠ ELIGIBLE BUT NOT YET IMPLEMENTED. `runChecks` does not produce a + * `published-test-suite` check today — it lands in a later, separate change. + * The row is here because eligibility and admission are different things and + * this is the standing demonstration of that: the registry says what MAY be + * declaration-armed, the ratified spec says what IS pinned, and nothing pins + * this. A spec that pins it anyway does not get a lenient verdict: against a + * DECLARING card the requirement fails loudly with `unknown check + * "published-test-suite"`, which is the correct direction of failure for a + * verifier that is too old for the spec it was handed. + */ + 'published-test-suite': 'interfaces.testSuite', +}) + +/** + * The card-path grammar: EXACTLY two dot-separated segments, the first + * literally `interfaces`, the second a lowerCamelCase-shaped member name. No + * array indices, no deeper nesting, no other container. + * + * Deliberately redundant with the registry binding rule — it holds even if the + * registry above is later mis-edited, and it is the rule the standard states in + * prose, so an implementer reading only this file gets the same answer. + */ +export const OPTIONAL_INTERFACE_PATH_RE = /^interfaces\.[A-Za-z][A-Za-z0-9]*$/ + +/** Human-readable list of every eligible check id, for a thrown message. */ +export function eligibleOptionalChecks(): string[] { + return Object.keys(OPTIONAL_DECLARED_INTERFACES).sort() +} diff --git a/src/pinned.ts b/src/pinned.ts index 38dc612..1f65550 100644 --- a/src/pinned.ts +++ b/src/pinned.ts @@ -31,6 +31,11 @@ import { axScoreOf } from './grade.js' import { sha256Hex } from './digest.js' import { validateSchema, readPath } from './schema.js' import { VERIFIER_VERSION } from './verify.js' +import { + OPTIONAL_DECLARED_INTERFACES, + OPTIONAL_INTERFACE_PATH_RE, + eligibleOptionalChecks, +} from './optional-interfaces.js' import type { AppliesWhen, CheckResult, @@ -213,6 +218,166 @@ export function validateRequirements(requirements: PinnedRequirement[]): void { ) } } + + // THE EVASION GUARD. See optional-interfaces.ts for why it exists. + for (const req of doc.requirements) validateAppliesWhen(req) +} + +/** + * THE EVASION GUARD — five rules, all THROWN at parse. + * + * `appliesWhen` is the one place in a PinnedSpec where a requirement can decide + * NOT to judge the target. The `cardDeclares` arm makes that decision from a + * key the TARGET writes. So it is only safe if the set of requirements that can + * reach it is fixed by the VERIFIER, not by the spec — otherwise any MUST + * clause becomes optional by omission and the standard quietly stops being one. + * + * This runs inside `validateRequirements`, which runs inside BOTH + * `parsePinnedSpec` and `parseSuite` — i.e. before `verifyPinnedSpec` fires a + * single probe. A spec that tries to gate an always-required check does not get + * a lenient verdict; it gets NO verdict, loudly, with the offending requirement + * named. There is no reviewer in the loop, which is what makes this ENFORCED + * rather than documented. + * + * The rules also give FORWARD protection the pre-union verifier could not have: + * an `appliesWhen` in a shape this verifier does not understand throws instead + * of silently degrading into "unobservable → applies → armed check skips → + * requirement fails", which would fail every conforming target for the wrong + * reason. + */ +function validateAppliesWhen(req: PinnedRequirement): void { + const raw = (req as { appliesWhen?: unknown }).appliesWhen + if (raw === undefined) return + const id = (req as { id: string }).id + const kind = (req as { kind?: unknown }).kind + const where = `requirement "${id}"` + + // Rule 2a: only `probe` and `check` requirements have ever consulted + // `appliesWhen`. On `surface` / `ax-floor` / `endpoint` it was silently + // ignored — a conditional-looking clause that conditions nothing is a + // false statement in a contract document. Make it explicit and throw. + if (kind !== 'probe' && kind !== 'check') { + throw new Error( + `${where} (kind:'${String(kind)}') carries an \`appliesWhen\`, which only kind:'probe' and ` + + "kind:'check' requirements evaluate. On this kind it would be silently ignored — a " + + 'conditional-looking clause that conditions nothing. Remove it, or change the kind.', + ) + } + + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error( + `${where} carries an \`appliesWhen\` that is not a JSON object (got ` + + `${raw === null ? 'null' : Array.isArray(raw) ? 'array' : typeof raw}). It must be exactly ` + + 'one of { fromProbe, path, equals } or { cardDeclares }.', + ) + } + const aw = raw as Record + const has = (k: string) => Object.prototype.hasOwnProperty.call(aw, k) + + // Rule 1: SHAPE TOTALITY. Exactly one arm. Both, neither, or a mixed shape is + // a spec this verifier cannot evaluate — and the union discriminates on key + // PRESENCE (no `source:` tag, so the standard's verbatim probe block stays + // byte-identical), which is only total because this rule rejects everything + // else at parse. + const fromProbeArm = has('fromProbe') + const cardArm = has('cardDeclares') + if (fromProbeArm === cardArm) { + throw new Error( + `${where} carries an \`appliesWhen\` with ${fromProbeArm ? 'BOTH' : 'NEITHER'} \`fromProbe\` ` + + 'and `cardDeclares`. Exactly one arm is legal: { fromProbe, path, equals } judges an ' + + 'OBSERVED VALUE and fails closed when it cannot be observed; { cardDeclares } judges a ' + + 'CARD DECLARATION and is not applicable when the key is absent. They are different in kind, ' + + 'so a requirement must say which one it means.', + ) + } + + if (cardArm) { + // Rule 2b: KIND RESTRICTION. `cardDeclares` is legal only on kind:'check'. + // This is what makes behavioural probe requirements — the ones that pin + // wire behaviour for always-required clauses — categorically un-gatable by + // any card key, with no registry lookup involved at all. + if (kind !== 'check') { + throw new Error( + `${where} is a kind:'probe' requirement carrying \`appliesWhen.cardDeclares\`. The ` + + "card-declaration arm is legal ONLY on kind:'check'. A behavioural probe requirement " + + 'pins what the wire must do for an always-required clause; letting a card key switch one ' + + 'off would let a target opt out of that clause by omission. An OPTIONAL capability that ' + + 'needs behavioural probing gets a CHECK that does the probing.', + ) + } + if (has('path') || has('equals')) { + throw new Error( + `${where} mixes \`cardDeclares\` with \`${has('path') ? 'path' : 'equals'}\`. The ` + + 'card-declaration arm tests PRESENCE only — there is deliberately no value test, because ' + + 'a present-but-unexpected value would have to mean either "not applicable" or "malformed" ' + + 'and two independent implementations would resolve that differently.', + ) + } + const cardDeclares = aw.cardDeclares + // Rule 5: PATH GRAMMAR. Deliberately redundant with rule 4 — it holds even + // if the registry is later mis-edited, and it forbids `cardDeclares: + // 'probes'`, which would gate an optional check on the AXP opt-in signal + // itself rather than on its own interface key. + if (typeof cardDeclares !== 'string' || !OPTIONAL_INTERFACE_PATH_RE.test(cardDeclares)) { + throw new Error( + `${where} carries \`appliesWhen.cardDeclares\` = ${JSON.stringify(cardDeclares)}, which is ` + + `not a legal optional-interface card path. It must match ${String(OPTIONAL_INTERFACE_PATH_RE)} ` + + '— exactly two segments, the first literally "interfaces", e.g. "interfaces.digitalLink". ' + + 'An optional interface is declared as a member of `interfaces`, nowhere else.', + ) + } + const check = (req as { check?: unknown }).check + // Rule 3: ALLOWLIST MEMBERSHIP. The registry is keyed by CHECK id — a + // string api.qa owns and a spec author cannot mint — not by requirement id, + // which the author chooses freely. + if (typeof check !== 'string' || !Object.prototype.hasOwnProperty.call(OPTIONAL_DECLARED_INTERFACES, check)) { + throw new Error( + `${where} tries to make check ${JSON.stringify(check)} conditional on the card declaration ` + + `${JSON.stringify(cardDeclares)}, but that check is NOT an api.qa optional-declared ` + + 'interface. A requirement can only be skipped by omission when the capability it verifies ' + + 'is ADDITIVE — otherwise the clause it binds stops being a MUST the moment a target leaves ' + + `a key out. Eligible checks: ${eligibleOptionalChecks().map((c) => `"${c}"`).join(', ')}. ` + + `Pin ${JSON.stringify(check)} WITHOUT \`appliesWhen\` if you mean to demand it of everyone.`, + ) + } + // Rule 4: PATH BINDING. Blocks cross-wiring — arming one optional check + // with a DIFFERENT optional interface's key, which would let a card skip a + // check by declaring something unrelated. + const bound = OPTIONAL_DECLARED_INTERFACES[check]! + if (cardDeclares !== bound) { + throw new Error( + `${where} arms check "${check}" with \`cardDeclares\` = ${JSON.stringify(cardDeclares)}, but ` + + `api.qa binds that check to ${JSON.stringify(bound)}. A check is armed by ITS OWN ` + + 'interface declaration; cross-wiring would let a card skip one capability by declaring ' + + 'another.', + ) + } + return + } + + // The OBSERVED-VALUE arm. Behaviour is unchanged; this only rejects shapes + // the evaluator could not have judged coherently anyway (a non-string source + // or path silently resolves to "unobservable → applies", which reads as a + // target failure when it is really a spec defect). + if (typeof aw.fromProbe !== 'string' || aw.fromProbe.length === 0) { + throw new Error( + `${where} carries \`appliesWhen.fromProbe\` = ${JSON.stringify(aw.fromProbe)} — it must be a ` + + 'non-empty string naming a probe channel this spec also declares a requirement for.', + ) + } + if (typeof aw.path !== 'string' || aw.path.length === 0) { + throw new Error( + `${where} carries \`appliesWhen.path\` = ${JSON.stringify(aw.path)} — the observed-value arm ` + + 'needs a non-empty dot-path into the source probe body.', + ) + } + if (!has('equals')) { + throw new Error( + `${where} carries \`appliesWhen.fromProbe\`/\`path\` with no \`equals\`. The observed-value arm ` + + 'applies the requirement only when the observed value deep-equals a PINNED value; without ' + + 'one there is nothing to compare against.', + ) + } } /** @@ -372,6 +537,7 @@ export async function verifyPinnedSpec( probePlans.set(req.id, { declared: [], entryProblems: new Map(), finalUrls: new Map(), notApplicable: applicability.reason, + notApplicableMark: applicability.notApplicable, }) continue } @@ -526,25 +692,36 @@ export async function verifyPinnedSpec( // check-offers-402 instead of being wrongly failed on it. const applicability = evaluateAppliesWhen(req.appliesWhen, probeReqs, fullBundle.items) if (!applicability.applies) { + // `verdict` STAYS 'pass' — `passed` is every(v === 'pass'), and a fourth + // Verdict member would flip every free-model target passing today. The + // STRUCTURED `notApplicable` marker is how an agent tells "passed + // because verified" from "passed because never applicable" without + // string-matching prose. The CI reporters map it to a JUnit . results.push({ id: req.id, title: `check ${req.check} must ${req.must}`, verdict: 'pass', - detail: `${applicability.reason} — passes as not applicable`, - evidence: [], + detail: applicability.detail ?? `${applicability.reason} — passes as not applicable`, + evidence: applicability.notApplicable?.reason === 'not-declared' ? [ROLE.agentsJson] : [], + ...(applicability.notApplicable && { notApplicable: applicability.notApplicable }), }) continue } // Bind a MUST clause to a SPECIFIC api.qa check, not the coarse floor. const c = surfaceChecks.find((sc) => sc.id === req.check) const verdict: Verdict = c?.verdict === 'pass' ? 'pass' : 'fail' + // A card-declaration gate that could not READ the card applied this + // requirement by fail-closed rule, not because the interface was declared. + // Say so: the armed check's own skip line would otherwise report the key + // as "absent" when the truth is the card was unreadable. + const failClosedNote = applicability.failClosed ? ` — NOTE: ${applicability.reason}` : '' results.push({ id: req.id, title: `check ${req.check} must ${req.must}`, verdict, detail: - c === undefined + (c === undefined ? `unknown check "${req.check}" — not produced by api.qa runChecks; cannot pass` : c.verdict === 'pass' ? `check ${req.check} passed: ${c.detail}` - : `check ${req.check} verdict '${c.verdict}' (must be 'pass'): ${c.detail}`, + : `check ${req.check} verdict '${c.verdict}' (must be 'pass'): ${c.detail}`) + failClosedNote, evidence: c?.evidence ?? [], }) } else if (req.kind === 'endpoint') { @@ -579,6 +756,7 @@ export async function verifyPinnedSpec( id: req.id, title: `probe ${req.probe}`, verdict: 'pass', detail: `${plan.notApplicable} — passes as not applicable`, evidence: [], + ...(plan.notApplicableMark && { notApplicable: plan.notApplicableMark }), }) continue } @@ -890,23 +1068,183 @@ interface ProbePlan { * the judge reports a PASS with this reason. */ notApplicable?: string + /** + * The STRUCTURED form of the same fact, carried into the requirement result + * so an agent does not have to string-match `notApplicable` prose. Always the + * `observed-value` arm here: `cardDeclares` is refused on kind:'probe' at + * parse (see validateAppliesWhen). + */ + notApplicableMark?: CheckResult['notApplicable'] +} + +/** + * Three-way result of reading an optional-interface declaration off the card. + * + * THREE, not two, and that is the point: `readPath` (schema.ts) collapses "the + * key is absent" and "an intermediate is not an object" into the same + * `{ found: false }`, and those two states have OPPOSITE verdicts here. A + * well-formed card that omits the key means NOT APPLICABLE; a card whose + * `interfaces` member is the string "none" means the card is malformed and the + * requirement APPLIES. Reusing `readPath` would silently hand an evasion the + * same verdict as a conformance. + */ +export type CardDeclarationState = + | { state: 'declared'; value: unknown } + /** The card was read and well-formed; the final key is not present. */ + | { state: 'absent' } + /** Card missing / non-2xx / non-JSON / not an object / bad intermediate. */ + | { state: 'unreadable'; why: string } + +/** + * Read an optional-interface declaration (`interfaces.`) out of the + * recorded capability card. + * + * PURE over the recorded evidence. It reads `ROLE.agentsJson` from the bundle — + * the same evidence item every other card-reading check judges from — so the + * observe phase and the judge phase agree by construction, and a replay of a + * stored bundle re-judges identically without re-fetching. + * + * The verdict table this implements, and the argument for it: + * + * card not fetched / non-2xx / network error / body not JSON → unreadable + * body parses but is not a plain JSON object → unreadable + * an INTERMEDIATE segment exists but is not a plain object → unreadable + * card well-formed, final key ABSENT → absent + * final key PRESENT with ANY value ({}, null, false, 0, "", []) → declared + * + * Every `unreadable` row collapses back to the fail-closed posture at the call + * site, because in those rows THERE IS NO STATEMENT TO READ: absence IN a + * retrieved document is a datum, absence OF the document is not. Only the + * `absent` row is a deliberate statement of "I do not offer this", and only it + * earns a skip. + * + * Presence uses `hasOwnProperty`, not truthiness — matching how the card parser + * already arms `interfaces.digitalLink`. In JSON there is no `undefined`, so a + * `null` value is DECLARED (and a defective declaration, which the armed check + * then fails). That is what stops "declare it as false and get a free skip". + * + * An ABSENT intermediate (a card with no `interfaces` member at all) is treated + * as `absent`, not `unreadable`: the final key is not present in a document that + * WAS read, which is the same statement as omitting the key. A card with no + * `interfaces` at all already fails the always-required card checks on its own. + */ +export function readCardDeclaration(items: Evidence[], path: string): CardDeclarationState { + const ev = items.find((e) => e.role === ROLE.agentsJson) + if (!ev) return { state: 'unreadable', why: 'the capability card was never fetched in this run' } + if (ev.status === null) { + return { state: 'unreadable', why: `fetching the capability card failed (${ev.error ?? 'unknown error'})` } + } + if (ev.status < 200 || ev.status >= 300) { + return { state: 'unreadable', why: `GET ${ev.url} answered ${ev.status}` } + } + const doc = parseJsonBody(ev) + if (doc === undefined) { + return { state: 'unreadable', why: `${ev.url} answered ${ev.status} but its body did not parse as JSON` } + } + if (doc === null || typeof doc !== 'object' || Array.isArray(doc)) { + return { + state: 'unreadable', + why: `${ev.url} parsed as ${doc === null ? 'null' : Array.isArray(doc) ? 'a JSON array' : `a JSON ${typeof doc}`}, not a JSON object`, + } + } + const segments = path.split('.') + let cursor: Record = doc as Record + for (let i = 0; i < segments.length - 1; i++) { + const seg = segments[i]! + if (!Object.prototype.hasOwnProperty.call(cursor, seg)) return { state: 'absent' } + const next = cursor[seg] + if (next === null || typeof next !== 'object' || Array.isArray(next)) { + return { + state: 'unreadable', + why: `the card's \`${segments.slice(0, i + 1).join('.')}\` is ${next === null ? 'null' : Array.isArray(next) ? 'an array' : `a ${typeof next}`}, not an object — the declaration cannot be read from a malformed card`, + } + } + cursor = next as Record + } + const last = segments[segments.length - 1]! + if (!Object.prototype.hasOwnProperty.call(cursor, last)) return { state: 'absent' } + return { state: 'declared', value: cursor[last] } } /** - * Evaluate an `appliesWhen` condition against the recorded evidence: the value - * at `path` inside the FIRST spec requirement probing channel `fromProbe` - * (entry-0 evidence) must deep-equal `equals` for the requirement to apply. - * FAIL-CLOSED: an unobserved source, a non-JSON body, or an unresolvable path - * means the requirement APPLIES — not-applicable must be PROVEN by the - * observed value. Pure over (requirements, items): the observe phase and the - * judge run the identical derivation and agree by construction. + * Evaluate an `appliesWhen` condition against the recorded evidence. Two arms, + * and they are asymmetric ON PURPOSE — see the block comment on AppliesWhen and + * optional-interfaces.ts. + * + * fromProbe — the value at `path` inside the FIRST spec requirement + * probing channel `fromProbe` (entry-0 evidence) must + * deep-equal `equals` for the requirement to apply. + * FAIL-CLOSED: an unobserved source, a non-JSON body, or an + * unresolvable path means the requirement APPLIES — + * not-applicable must be PROVEN by the observed value. A + * missing probe response is an OBSERVATION FAILURE: the + * verifier asked and got no answer, so it cannot distinguish + * "does not apply to me" from "I am broken" or "I am evading". + * Absence of evidence is not evidence. + * + * cardDeclares — the requirement applies iff the capability card DECLARES the + * named optional interface. Here THE CARD IS THE ANSWER: a + * card that was fetched, parsed and found well-formed and that + * omits the key has affirmatively said "I do not offer this", + * and that statement is a datum the verifier is entitled to + * believe. Every case where the card itself could not be read + * collapses back into the fromProbe posture — applies, fail + * closed — because in those cases there is no statement to + * read. A present-but-empty value is a CLAIM, not an absence: + * it ARMS the requirement, and the armed check judges (and + * fails) the defective declaration. + * + * Pure over (requirements, items): the observe phase and the judge run the + * identical derivation and agree by construction. */ function evaluateAppliesWhen( aw: AppliesWhen | undefined, probeReqs: Array>, items: Evidence[], -): { applies: boolean; reason: string } { +): { + applies: boolean + reason: string + detail?: string + notApplicable?: CheckResult['notApplicable'] + /** The requirement applies only because its source could not be read. */ + failClosed?: boolean +} { if (aw === undefined) return { applies: true, reason: '' } + + if (aw.cardDeclares !== undefined) { + const cardPath = aw.cardDeclares + const st = readCardDeclaration(items, cardPath) + if (st.state === 'unreadable') { + return { + applies: true, + // `failClosed` is surfaced in the requirement detail. Without it the + // reader sees only the armed check's own skip line — "interfaces. + // digitalLink absent" — which is FALSE and misleading when the truth is + // that the card could not be read at all. The verdict is the same + // either way; the diagnosis is not. + failClosed: true, + reason: + `appliesWhen source \`${cardPath}\` could not be read from the capability card ` + + `(${st.why}) — requirement APPLIES (fail closed). A card that cannot be read has made no ` + + 'statement about what it offers; only an omission INSIDE a readable card is one.', + } + } + if (st.state === 'absent') { + return { + applies: false, + reason: `not applicable: the capability card declares no \`${cardPath}\``, + detail: + `not applicable: the capability card declares no \`${cardPath}\` — the optional interface ` + + 'is not claimed, so this requirement is not judged (omission is conformance)', + notApplicable: { reason: 'not-declared', source: cardPath }, + } + } + return { + applies: true, + reason: `the capability card declares \`${cardPath}\` (${JSON.stringify(st.value) ?? 'undefined'})`, + } + } + const srcReq = probeReqs.find((r) => r.probe === aw.fromProbe) const ev = srcReq ? items.find((e) => e.role === `pinned:${srcReq.id}:0`) : undefined let body: unknown @@ -924,6 +1262,7 @@ function evaluateAppliesWhen( return { applies: false, reason: `not applicable: probes.${aw.fromProbe} ${aw.path} = ${JSON.stringify(r.value)} (requirement applies only when it equals ${JSON.stringify(aw.equals)})`, + notApplicable: { reason: 'observed-value', source: `probes.${aw.fromProbe} ${aw.path}` }, } } diff --git a/src/render.ts b/src/render.ts index ee626f8..9dd8bf7 100644 --- a/src/render.ts +++ b/src/render.ts @@ -10,6 +10,18 @@ import type { DataDrivenReport } from './dataset.js' const MARK: Record = { pass: 'PASS', fail: 'FAIL', skip: 'skip' } +/** + * A pinned REQUIREMENT has three readable outcomes, not two: judged-and-passed, + * judged-and-failed, and NEVER JUDGED because `appliesWhen` said the + * requirement does not apply to this target. The verdict of the third is 'pass' + * (see CheckResult.notApplicable for why that must not change), so a renderer + * that keys only on `verdict` shows an unverified requirement as a verified + * one. Key on the structured marker instead. + */ +function requirementMark(c: { verdict: string; notApplicable?: unknown }): string { + return c.notApplicable !== undefined ? 'n/a' : MARK[c.verdict]! +} + export function reportMarkdown(r: VerificationReport): string { const host = r.target.replace(/^https?:\/\//, '') const lines: string[] = [ @@ -54,15 +66,21 @@ export function reportMarkdown(r: VerificationReport): string { } export function pinnedMarkdown(r: PinnedReport): string { + const notApplicable = r.requirements.filter((c) => c.notApplicable !== undefined).length + const failed = r.requirements.filter((c) => c.verdict !== 'pass').length + const passed = r.requirements.length - notApplicable - failed const lines = [ `# api.qa pinned-spec report — ${r.target.replace(/^https?:\/\//, '')}`, '', `> **${r.passed ? 'PASSED' : 'FAILED'}** against \`${r.spec.name}@${r.spec.version}\``, `> spec digest \`${r.spec.digest}\` · ${r.mode} mode · ${r.attested ? 'attested' : 'NOT attested (advisory)'}`, + // Three counts, never two: a not-applicable requirement was NOT verified, + // and folding it into "passed" would overstate what this run established. + `> ${passed} passed · ${notApplicable} not applicable · ${failed} failed`, '', '| requirement | verdict | detail |', '| --- | --- | --- |', - ...r.requirements.map((c) => `| ${c.title} (\`${c.id}\`) | ${MARK[c.verdict]} | ${c.detail.replace(/\|/g, '\\|')} |`), + ...r.requirements.map((c) => `| ${c.title} (\`${c.id}\`) | ${requirementMark(c)} | ${c.detail.replace(/\|/g, '\\|')} |`), '', ] return lines.join('\n') @@ -78,7 +96,7 @@ export function suiteMarkdown(r: SuiteReport): string { '| # | probe | verdict | detail |', '| --- | --- | --- | --- |', ...r.requirements.map( - (c, i) => `| ${i + 1} | ${c.title} (\`${c.id}\`) | ${MARK[c.verdict]} | ${c.detail.replace(/\|/g, '\\|')} |`, + (c, i) => `| ${i + 1} | ${c.title} (\`${c.id}\`) | ${requirementMark(c)} | ${c.detail.replace(/\|/g, '\\|')} |`, ), '', ] diff --git a/src/reporters.ts b/src/reporters.ts index 89b8e88..da67c29 100644 --- a/src/reporters.ts +++ b/src/reporters.ts @@ -137,12 +137,25 @@ function checkTime(check: CheckResult, times: Map): number { } function caseFromCheck(c: CheckResult, classname: string, times: Map): ReporterTestCase { + // NOT-APPLICABLE → JUnit `skip`, not `pass`. + // + // A requirement whose `appliesWhen` said it does not apply to this target was + // NEVER RUN. `` is the correct JUnit semantic for that, and it is + // what a CI reader needs: counting it as a pass tells a human that something + // was verified when nothing was. + // + // ⚠ INTENDED COUNT MOVE: this shifts such requirements out of the JUnit/summary + // `passed` total and into `skipped`. It is a REPORTING change only — + // `PinnedReport.passed` is untouched and the requirement's own `verdict` stays + // 'pass', so no target's admission changes. If you are bisecting a "passed + // count dropped" report, this is why, and it is deliberate. + const notApplicable = c.notApplicable !== undefined return { id: c.id, name: `${c.title} [${c.id}]`, classname, - status: c.verdict as CaseStatus, - detail: c.verdict === 'pass' ? '' : c.detail, + status: notApplicable ? 'skip' : (c.verdict as CaseStatus), + detail: notApplicable || c.verdict !== 'pass' ? c.detail : '', timeMs: checkTime(c, times), } } diff --git a/src/types.ts b/src/types.ts index 719bd09..084b8b0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -122,6 +122,26 @@ export interface CheckResult { * VerificationReport so the diff is a monitorable signal, not just a verdict. */ contractDiff?: ContractDiffReport + /** + * STRUCTURED not-applicable marker, set on a pinned REQUIREMENT result whose + * `appliesWhen` decided the requirement does not apply to this target. + * + * `verdict` deliberately STAYS `'pass'`. A PinnedSpec passes iff every + * requirement's verdict is `'pass'`; introducing a fourth `Verdict` member + * would flip every free-model target passing today to `passed: false`. This + * field is instead how an agent tells "passed because it was VERIFIED" from + * "passed because it was never APPLICABLE" — without parsing prose, which is + * not a contract. Absent on every ordinary verdict. + * + * The CI reporters map a marked requirement to a JUnit ``, which is + * the correct JUnit semantic for a test that did not run. + */ + notApplicable?: { + /** Which `appliesWhen` arm decided it. */ + reason: 'not-declared' | 'observed-value' + /** What was read: 'interfaces.digitalLink' | 'probes.pricing model'. */ + source: string + } } // --------------------------------------------------------------------------- @@ -333,19 +353,82 @@ export interface EndpointExpect { } /** - * Conditional applicability for `probe` / `check` requirements: the requirement - * applies only when the value at `path` inside the FIRST declared probe of - * channel `fromProbe` (as OBSERVED in this run — entry index 0) deep-equals - * `equals`. A non-applicable requirement passes as "not applicable" (this is - * how a free-model API passes AXP's metering requirements). FAIL-CLOSED rule: - * when the source probe was not observed, is not JSON, or the path does not - * resolve, the requirement APPLIES — applicability can only be proven by the - * observed value, never by its absence. + * Conditional applicability for `probe` / `check` requirements. EXACTLY ONE + * arm. The two arms differ in KIND, not degree, and the difference is the whole + * design: + * + * `fromProbe` — the requirement applies only when an OBSERVED VALUE says + * so. An unobservable source APPLIES the requirement + * (fail closed), because a missing probe response is an + * OBSERVATION FAILURE: the verifier asked and got no + * answer, so it cannot tell "does not apply to me" from + * "I am broken" or "I am evading". + * + * `cardDeclares` — the requirement applies only when the capability card + * DECLARES a named optional interface. An absent key means + * NOT APPLICABLE, because there the card IS the answer: a + * card that was fetched, parsed and found well-formed and + * that omits the key has affirmatively said "I do not offer + * this". Absence IN a retrieved document is a datum; + * absence OF the document is not — so every case where the + * card could not be read collapses back to fail-closed. + * + * A non-applicable requirement passes as "not applicable" and carries a + * STRUCTURED `CheckResult.notApplicable` marker, so an agent never has to + * string-match prose to tell a verified pass from an unjudged one. + */ +export type AppliesWhen = AppliesWhenFromProbe | AppliesWhenCardDeclares + +/** + * OBSERVED-VALUE arm. The requirement applies only when the value at `path` + * inside the FIRST declared probe of channel `fromProbe` (as OBSERVED in this + * run — entry index 0) deep-equals `equals`. This is how a free-model API + * passes AXP's metering requirements. + * + * FAIL-CLOSED: when the source probe was not observed, is not JSON, or the path + * does not resolve, the requirement APPLIES — applicability can only be PROVEN + * by the observed value, never by its absence. + * + * Legal on `kind: 'probe'` AND `kind: 'check'`. Semantics are byte-identical to + * the pre-union behaviour; nothing here changed. */ -export interface AppliesWhen { +export interface AppliesWhenFromProbe { fromProbe: string path: string equals: unknown + cardDeclares?: never +} + +/** + * CARD-DECLARATION arm. The requirement applies only when the capability card + * DECLARES the named optional interface — i.e. the dot-path resolves to a + * PRESENT member of a reachable, well-formed card. + * + * PRESENCE is the whole test. There is deliberately no `equals`: a + * present-but-unexpected value would have to mean either "not applicable" or + * "malformed", and two independent implementers would resolve that differently. + * A present-but-empty value (`{}`, `null`, `false`, `""`, `[]`) is a CLAIM, not + * an absence — it ARMS the requirement, and the armed check judges it (and + * fails it, if the declaration is defective). A card meaning "no" OMITS the key. + * + * There is also deliberately no `declared: false`. A negated form is an evasion + * primitive by construction — a MUST that switches off when you ADD a card key. + * Making it unrepresentable in the type is cheaper than forbidding it in prose. + * + * CONSTRAINTS, all enforced by THROWING in `validateRequirements` before any + * probe fires: + * - legal ONLY on `kind: 'check'` (never on `kind: 'probe'`, which is what + * makes behavioural probe requirements categorically un-gatable); + * - `check` MUST be a key of the verifier's own + * `OPTIONAL_DECLARED_INTERFACES` registry; + * - `cardDeclares` MUST equal the exact path that registry binds to `check`; + * - `cardDeclares` MUST match `/^interfaces\.[A-Za-z][A-Za-z0-9]*$/`. + */ +export interface AppliesWhenCardDeclares { + cardDeclares: string + fromProbe?: never + path?: never + equals?: never } export type PinnedRequirement = @@ -415,8 +498,18 @@ export type PinnedRequirement = * stricter). */ paramValue?: number | { fromProbe: string; path: string; multiply?: number; multiplyRange?: [number, number] } - /** Conditional applicability (see AppliesWhen). Absent = always applies. */ - appliesWhen?: AppliesWhen + /** + * Conditional applicability, OBSERVED-VALUE arm ONLY (see AppliesWhen). + * Absent = always applies. + * + * A `probe` requirement may NEVER carry the `cardDeclares` arm — it is + * excluded here at compile time and thrown on at parse time. A behavioural + * probe requirement is how an always-required clause is verified against + * the wire; letting a card key switch one off would let a target opt out + * of that clause by omission. An optional capability that needs + * behavioural probing gets a CHECK that does the probing instead. + */ + appliesWhen?: AppliesWhenFromProbe /** * When true, every declared entry's pathname must ALSO be observed * answering `200` with a top-level `type: "OK"` JSON envelope somewhere diff --git a/src/verify.ts b/src/verify.ts index 2833097..9975571 100644 --- a/src/verify.ts +++ b/src/verify.ts @@ -17,7 +17,25 @@ import type { VerificationReport } from './types.js' // conneg-client-class / conneg-alternates / conneg-forced-face), appliesWhen // gating on kind:check AND kind:probe, paramValue.multiplyRange, // expect.paths[].oneOf, metered-gated probe-manifest demands. -export const VERIFIER_VERSION = '0.2.0' +// +// 0.3.0: THE OPTIONAL-DECLARED-INTERFACE MECHANISM. `appliesWhen` becomes a +// two-arm union — the existing observed-value arm plus `cardDeclares`, which +// arms a requirement on the PRESENCE of a named `interfaces.` member of +// the capability card. Skipping is restricted to a frozen, verifier-owned +// registry of ADDITIVE capabilities (optional-interfaces.ts), enforced by +// throwing in validateRequirements before any probe fires, so no +// always-required clause can be opted out of by omission. Not-applicable +// requirement results carry a structured `notApplicable` marker and report as a +// JUnit ``. `digital-link-resolver` is registered, which is what makes +// it pinnable. +// +// ⚠ VERSION ORDERING: a spec carrying `appliesWhen.cardDeclares` reaching a +// verifier OLDER than 0.3.0 sees `aw.fromProbe === undefined`, finds no source +// probe, applies the requirement fail-closed, and therefore fails EVERY +// non-declaring target. That is the correct direction of failure — loud, never +// a silent pass — but it is an outage, and it is why a spec must not pin a +// declaration-armed requirement until 0.3.0 is deployed. +export const VERIFIER_VERSION = '0.3.0' export interface VerifyTargetOpts extends ObserverOpts { /** diff --git a/test/axp-fixture.ts b/test/axp-fixture.ts new file mode 100644 index 0000000..55e67ba --- /dev/null +++ b/test/axp-fixture.ts @@ -0,0 +1,118 @@ +/** + * The reference AXP target, shared. + * + * Extracted verbatim from conneg.test.ts (which still imports it) so a second + * suite can verify against the SAME fixture instead of forking it. A forked + * conformance fixture is how two suites end up disagreeing about what + * "conformant" means. + * + * good.example upgraded to the full AXP bar: the conneg law (three faces, + * deterministic selection, Link alternates), typed envelopes on one branching + * collection, a free Pricing Document, and the probe manifest. + */ + +import { GOOD, goodTargetRoutes, makeFetcher, withOverrides, type Routes } from './helpers.js' + +export const LINKS = + '; rel="alternate"; type="text/html", ' + + '; rel="alternate"; type="application/ld+json", ' + + '; rel="alternate"; type="text/markdown"' + +export const FACE_HTML = () => ({ + status: 200, contentType: 'text/html', headers: { link: LINKS }, + body: '

good.example

the page face

', +}) +export const FACE_JSON = () => ({ + status: 200, contentType: 'application/ld+json', headers: { link: LINKS }, + body: JSON.stringify({ $context: 'https://schema.org.ai', $type: 'Service', name: 'good.example' }), +}) +export const FACE_MD = () => ({ + status: 200, contentType: 'text/markdown', headers: { link: LINKS }, + body: '# good.example\n\n> the token-cheap agent face, substantive enough to be real content.', +}) + +/** The AXP A.7.3 selection algorithm, as a route handler for `GET /`. */ +export function connegRoot(req: { accept: string; headers?: Record }) { + const h = req.headers ?? {} + if (req.accept.includes('text/html')) return FACE_HTML() + if (req.accept.includes('application/json') || req.accept.includes('application/ld+json')) return FACE_JSON() + if (req.accept.includes('text/markdown')) return FACE_MD() + if (h['sec-fetch-mode'] === 'navigate' || h['sec-fetch-dest'] === 'document') return FACE_HTML() + if (/claude-user|gptbot|claudebot|agent/i.test(h['user-agent'] ?? '')) return FACE_MD() + return FACE_JSON() +} + +/** One branching collection (the api.lawyer /matters pattern): OK / EMPTY / BLOCKED. */ +export function recordsRoute(url: string) { + const u = new URL(url, GOOD) + const scope = u.searchParams.get('scope') + if (scope === 'admin' || scope === 'internal') { + return { status: 403, contentType: 'application/json', body: JSON.stringify({ type: 'BLOCKED', reason: 'not permitted for your agent class' }) } + } + if (u.searchParams.get('filter') === 'none' || u.searchParams.get('tag') === 'none') { + return { status: 200, contentType: 'application/json', body: JSON.stringify({ type: 'EMPTY', results: [], message: 'no records match' }) } + } + return { status: 200, contentType: 'application/json', body: JSON.stringify({ type: 'OK', results: [{ id: 'r1' }] }) } +} + +/** + * The reference AXP target: goodTargetRoutes + the conneg law + the probe + * manifest + the branching collection + a free Pricing Document. + * + * `mutateCard` is an OPTIONAL hook for a suite that needs to add a card member + * (e.g. an optional-interface declaration) without forking the fixture. It runs + * after every other card edit and before the card is serialized; existing + * callers pass nothing and get byte-identical routes. + */ +export function axpReferenceRoutes( + pricing: Record = { model: 'free' }, + mutateCard?: (card: Record) => void, +): Routes { + const base = goodTargetRoutes() + const card = JSON.parse( + base['GET /.well-known/agents.json']!({ method: 'GET', accept: 'application/json' }).body!, + ) as Record + card.llms = `${GOOD}/llms.txt` + card.interfaces.http.records = { method: 'GET', url: `${GOOD}/api/records`, auth: 'none' } + card.probes = { + keyless: { url: '/api/records' }, + pricing: { url: '/pricing' }, + knownEmpty: [{ url: '/api/records?filter=none' }, { url: '/api/records?tag=none' }], + knownForbidden: [{ url: '/api/records?scope=admin' }, { url: '/api/records?scope=internal' }], + ...(pricing.model === 'metered' ? { overCeiling: { url: '/api/records', param: 'spend' } } : {}), + } + mutateCard?.(card) + const openapi = JSON.parse(base['GET /openapi.json']!({ method: 'GET', accept: 'application/json' }).body!) as Record + openapi.paths['/api/records'] = { get: { responses: { '200': { description: 'records' } } } } + openapi.paths['/pricing'] = { get: { responses: { '200': { description: 'pricing document' } } } } + + return withOverrides(base, { + 'GET /': connegRoot, + 'GET /index.html': () => FACE_HTML(), + 'GET /index.json': () => FACE_JSON(), + 'GET /index.md': () => FACE_MD(), + 'GET /.well-known/agents.json': () => ({ status: 200, contentType: 'application/json', body: JSON.stringify(card) }), + 'GET /openapi.json': () => ({ status: 200, contentType: 'application/json', body: JSON.stringify(openapi) }), + 'GET /pricing': () => ({ status: 200, contentType: 'application/json', body: JSON.stringify(pricing) }), + // Plain-table route (no URL threading): the query-branching variants are + // exercised through urlAwareFetcher below. + 'GET /api/records': () => recordsRoute('/api/records'), + }) +} + +/** + * Route-handler plumbing: recordsRoute needs the request URL. makeFetcher hands + * handlers only method/accept/headers, so thread the url through this tiny + * wrapper instead of widening the shared helper for one test. + */ +export function urlAwareFetcher(routes: Routes) { + const inner = makeFetcher(routes) + return async (url: string, init?: RequestInit) => { + const u = new URL(url) + if (u.pathname === '/api/records') { + const out = recordsRoute(url) + return new Response(out.body ?? '', { status: out.status, headers: { 'content-type': out.contentType ?? 'text/plain' } }) + } + return inner(url, init) + } +} diff --git a/test/conneg.test.ts b/test/conneg.test.ts index c439ef5..15eeb49 100644 --- a/test/conneg.test.ts +++ b/test/conneg.test.ts @@ -24,93 +24,21 @@ import { axScoreOf, gradeOf } from '../src/grade.js' import { verifyPinnedSpec } from '../src/pinned.js' import { sha256Hex } from '../src/digest.js' import { GOOD, goodTargetRoutes, makeFetcher, withOverrides, type Routes } from './helpers.js' +import { + LINKS, FACE_HTML, FACE_JSON, FACE_MD, connegRoot, axpReferenceRoutes, urlAwareFetcher, +} from './axp-fixture.js' const AXP_SPEC_PATH = new URL('../examples/ax/apis-ax-standard.spec.json', import.meta.url) // --------------------------------------------------------------------------- -// Fixture: good.example upgraded to the full AXP 0.4.0 bar — conneg law -// (three faces, deterministic selection, Link alternates), typed envelopes on -// one branching collection, a free Pricing Document, and the probe manifest. +// Fixture: good.example upgraded to the full AXP bar — conneg law (three +// faces, deterministic selection, Link alternates), typed envelopes on one +// branching collection, a free Pricing Document, and the probe manifest. +// +// Moved to ./axp-fixture.ts so the optional-declared-interface suite verifies +// against the SAME reference target instead of forking it. Unchanged here. // --------------------------------------------------------------------------- -const LINKS = - '; rel="alternate"; type="text/html", ' + - '; rel="alternate"; type="application/ld+json", ' + - '; rel="alternate"; type="text/markdown"' - -const FACE_HTML = () => ({ - status: 200, contentType: 'text/html', headers: { link: LINKS }, - body: '

good.example

the page face

', -}) -const FACE_JSON = () => ({ - status: 200, contentType: 'application/ld+json', headers: { link: LINKS }, - body: JSON.stringify({ $context: 'https://schema.org.ai', $type: 'Service', name: 'good.example' }), -}) -const FACE_MD = () => ({ - status: 200, contentType: 'text/markdown', headers: { link: LINKS }, - body: '# good.example\n\n> the token-cheap agent face, substantive enough to be real content.', -}) - -/** The AXP A.7.3 selection algorithm, as a route handler for `GET /`. */ -function connegRoot(req: { accept: string; headers?: Record }) { - const h = req.headers ?? {} - if (req.accept.includes('text/html')) return FACE_HTML() - if (req.accept.includes('application/json') || req.accept.includes('application/ld+json')) return FACE_JSON() - if (req.accept.includes('text/markdown')) return FACE_MD() - if (h['sec-fetch-mode'] === 'navigate' || h['sec-fetch-dest'] === 'document') return FACE_HTML() - if (/claude-user|gptbot|claudebot|agent/i.test(h['user-agent'] ?? '')) return FACE_MD() - return FACE_JSON() -} - -/** One branching collection (the api.lawyer /matters pattern): OK / EMPTY / BLOCKED. */ -function recordsRoute(url: string) { - const u = new URL(url, GOOD) - const scope = u.searchParams.get('scope') - if (scope === 'admin' || scope === 'internal') { - return { status: 403, contentType: 'application/json', body: JSON.stringify({ type: 'BLOCKED', reason: 'not permitted for your agent class' }) } - } - if (u.searchParams.get('filter') === 'none' || u.searchParams.get('tag') === 'none') { - return { status: 200, contentType: 'application/json', body: JSON.stringify({ type: 'EMPTY', results: [], message: 'no records match' }) } - } - return { status: 200, contentType: 'application/json', body: JSON.stringify({ type: 'OK', results: [{ id: 'r1' }] }) } -} - -/** - * The reference AXP target: goodTargetRoutes + the conneg law + the probe - * manifest + the branching collection + a free Pricing Document. - */ -function axpReferenceRoutes(pricing: Record = { model: 'free' }): Routes { - const base = goodTargetRoutes() - const card = JSON.parse( - base['GET /.well-known/agents.json']!({ method: 'GET', accept: 'application/json' }).body!, - ) as Record - card.llms = `${GOOD}/llms.txt` - card.interfaces.http.records = { method: 'GET', url: `${GOOD}/api/records`, auth: 'none' } - card.probes = { - keyless: { url: '/api/records' }, - pricing: { url: '/pricing' }, - knownEmpty: [{ url: '/api/records?filter=none' }, { url: '/api/records?tag=none' }], - knownForbidden: [{ url: '/api/records?scope=admin' }, { url: '/api/records?scope=internal' }], - ...(pricing.model === 'metered' ? { overCeiling: { url: '/api/records', param: 'spend' } } : {}), - } - const openapi = JSON.parse(base['GET /openapi.json']!({ method: 'GET', accept: 'application/json' }).body!) as Record - openapi.paths['/api/records'] = { get: { responses: { '200': { description: 'records' } } } } - openapi.paths['/pricing'] = { get: { responses: { '200': { description: 'pricing document' } } } } - - return withOverrides(base, { - 'GET /': connegRoot, - 'GET /index.html': () => FACE_HTML(), - 'GET /index.json': () => FACE_JSON(), - 'GET /index.md': () => FACE_MD(), - 'GET /.well-known/agents.json': () => ({ status: 200, contentType: 'application/json', body: JSON.stringify(card) }), - 'GET /openapi.json': () => ({ status: 200, contentType: 'application/json', body: JSON.stringify(openapi) }), - 'GET /pricing': () => ({ status: 200, contentType: 'application/json', body: JSON.stringify(pricing) }), - // Plain-table route (no URL threading): the query-branching variants are - // exercised through urlAwareFetcher below. - 'GET /api/records': () => recordsRoute('/api/records'), - }) -} - async function judge(routes: Routes, seed = 7) { const observer = new Observer({ fetcher: makeFetcher(routes), delayMs: 0 }) const bundle = await observeTarget(GOOD, observer, seed) @@ -134,27 +62,9 @@ const AXP_CHECK_IDS = [ 'card-interfaces-linked', ] as const -// --------------------------------------------------------------------------- -// Route-handler plumbing: recordsRoute needs the request URL. makeFetcher -// hands handlers only method/accept/headers, so thread the url through a -// tiny wrapper fetcher instead of widening the shared helper for one test. -// --------------------------------------------------------------------------- - // (recordsRoute defaults to the plain-OK branch when no url is threaded; the // query-branching is exercised through the pinned probes below, which fetch -// concrete query URLs — see urlAwareFetcher.) - -function urlAwareFetcher(routes: Routes) { - const inner = makeFetcher(routes) - return async (url: string, init?: RequestInit) => { - const u = new URL(url) - if (u.pathname === '/api/records') { - const out = recordsRoute(url) - return new Response(out.body ?? '', { status: out.status, headers: { 'content-type': out.contentType ?? 'text/plain' } }) - } - return inner(url, init) - } -} +// concrete query URLs — see urlAwareFetcher in ./axp-fixture.ts.) // =========================================================================== // The six AXP structural checks diff --git a/test/optional-interfaces.test.ts b/test/optional-interfaces.test.ts new file mode 100644 index 0000000..c894bd4 --- /dev/null +++ b/test/optional-interfaces.test.ts @@ -0,0 +1,749 @@ +/** + * THE OPTIONAL-DECLARED-INTERFACE MECHANISM. + * + * `appliesWhen: { cardDeclares: "interfaces." }` — a pinned requirement + * that is verified only when the target's capability card DECLARES the optional + * interface it verifies. + * + * ── Why almost every test in this file is a FAILING case ──────────────────── + * + * An optionality mechanism is an EVASION mechanism if misapplied. If a + * requirement can be skipped by not declaring something, then every clause + * reachable that way stops being a MUST — a target opts out by omission and the + * report still says PASSED. So the properties worth holding are almost all + * negative: + * + * 1. **The guard refuses.** A spec that tries to gate a check api.qa has not + * registered as an ADDITIVE capability throws at PARSE, before a probe + * fires. Not a warning, not a lenient verdict — no verdict at all. + * 2. **Only an omission INSIDE a readable card skips.** An unreachable card, + * a non-JSON card, a card that is not an object, a card whose `interfaces` + * is a string — every one of those APPLIES the requirement and fails + * closed. Absence of the document is not a statement; only absence in it is. + * 3. **A present-but-empty declaration is a CLAIM, not an absence.** `null`, + * `false`, `0`, `""`, `[]`, `"yes"` all ARM the check and FAIL it. A card + * meaning "no" omits the key. This is what stops "declare it false, get a + * free skip". + * 4. **A skip is legible as a skip.** A not-applicable requirement is + * distinguishable from a verified pass and from a never-produced check + * WITHOUT string-matching prose, and it renders as a JUnit ``. + * + * ── Independence ──────────────────────────────────────────────────────────── + * + * Nothing here imports anything from the standard's repo. The registry under + * test is api.qa's own; a spec is external JSON parsed at runtime and is + * treated throughout as adversarial input written by a stranger. + */ + +import { describe, it, expect } from 'vitest' +import { readFileSync } from 'node:fs' +import { + verifyPinnedSpec, + parsePinnedSpec, + parseSuite, + readCardDeclaration, +} from '../src/pinned.js' +import { + OPTIONAL_DECLARED_INTERFACES, + OPTIONAL_INTERFACE_PATH_RE, +} from '../src/optional-interfaces.js' +import { GS1_RESOLVER_WELL_KNOWN_PATH } from '../src/gs1-resolver.js' +import { ROLE } from '../src/discovery.js' +import { junitXml, jsonReport } from '../src/reporters.js' +import { pinnedMarkdown } from '../src/render.js' +import type { Evidence } from '../src/types.js' +import type { Fetcher } from '../src/http.js' +import { GOOD, goodTargetRoutes, makeFetcher, withOverrides, type Routes } from './helpers.js' +import { axpReferenceRoutes, urlAwareFetcher } from './axp-fixture.js' +import { assertWellFormedXml } from './helpers.js' + +const AXP_SPEC_PATH = new URL('../examples/ax/apis-ax-standard.spec.json', import.meta.url) + +// --------------------------------------------------------------------------- +// Plumbing +// --------------------------------------------------------------------------- + +const WK_ROUTE = `GET ${GS1_RESOLVER_WELL_KNOWN_PATH}` + +/** A GS1 resolver description file that satisfies the published schema. */ +const VALID_DESCRIPTION_FILE = { + resolverRoot: GOOD, + supportedPrimaryKeys: ['01'], +} + +const json = (value: unknown, contentType = 'application/json') => () => ({ + status: 200, contentType, body: JSON.stringify(value), +}) + +const OMIT = Symbol('omit') +/** Serve NO well-known at all (the route 404s). Not `undefined`, which a JS + * default parameter would silently turn back into the valid document. */ +const NO_WELL_KNOWN = Symbol('no-well-known') + +/** + * The good target with `interfaces.digitalLink` set to `declaration` (or the + * key left off entirely for OMIT), and a well-known route unless suppressed. + */ +function cardRoutes( + declaration: unknown | typeof OMIT, + wellKnown: Routes[string] | typeof NO_WELL_KNOWN = json(VALID_DESCRIPTION_FILE), +): Routes { + const base = goodTargetRoutes() + const card = JSON.parse( + base['GET /.well-known/agents.json']!({ method: 'GET', accept: 'application/json' }).body!, + ) as Record + if (declaration !== OMIT) card.interfaces.digitalLink = declaration + return withOverrides(base, { + 'GET /.well-known/agents.json': json(card), + ...(wellKnown === NO_WELL_KNOWN ? {} : { [WK_ROUTE]: wellKnown }), + }) +} + +/** Serve a raw (possibly non-JSON, possibly non-object) body as the card. */ +function rawCardRoutes(status: number, contentType: string, body: string): Routes { + return withOverrides(goodTargetRoutes(), { + 'GET /.well-known/agents.json': () => ({ status, contentType, body }), + }) +} + +/** The minimum spec that exercises the mechanism and nothing else. */ +const DL_REQUIREMENT = { + id: 'check-digital-link-resolver', kind: 'check', check: 'digital-link-resolver', must: 'pass', + appliesWhen: { cardDeclares: 'interfaces.digitalLink' }, +} + +function specText(requirements: unknown[], name = 'optional-mechanism'): string { + return JSON.stringify({ $type: 'PinnedSpec', name, version: '1', requirements }) +} + +const DL_SPEC = specText([DL_REQUIREMENT]) + +/** Run a spec, recording every URL the verifier actually fetched. */ +async function run(spec: string, routes: Routes, fetcher?: Fetcher) { + const calls: string[] = [] + const inner = fetcher ?? makeFetcher(routes) + const report = await verifyPinnedSpec(GOOD, spec, { + fetcher: async (url, init) => { calls.push(url); return inner(url, init) }, + delayMs: 0, seed: 7, mode: 'local', + }) + const dl = report.requirements.find((r) => r.id === 'check-digital-link-resolver')! + return { report, dl, calls } +} + +/** A minimal Evidence item standing in for the observed capability card. */ +function cardEvidence(over: Partial = {}): Evidence[] { + return [{ + role: ROLE.agentsJson, + url: `${GOOD}/.well-known/agents.json`, + method: 'GET', + status: 200, + contentType: 'application/json', + headers: {}, + body: '{}', + elapsedMs: 1, + ...over, + }] +} + +// =========================================================================== +// 1. THE REGISTRY — the thing that decides what may be skipped at all +// =========================================================================== + +describe('the optional-declared-interface registry', () => { + it('is frozen: a runtime edit cannot widen what may be skipped', () => { + expect(Object.isFrozen(OPTIONAL_DECLARED_INTERFACES)).toBe(true) + expect(() => { + // @ts-expect-error — deliberately attacking the guard at runtime. + OPTIONAL_DECLARED_INTERFACES['keyless-flow'] = 'interfaces.anything' + }).toThrow() + expect(OPTIONAL_DECLARED_INTERFACES['keyless-flow']).toBeUndefined() + }) + + it('every registered card path obeys the two-segment `interfaces.` grammar', () => { + for (const [check, path] of Object.entries(OPTIONAL_DECLARED_INTERFACES)) { + expect(path, check).toMatch(OPTIONAL_INTERFACE_PATH_RE) + } + }) + + it('binds each check to exactly ONE card path, and no two checks share a path', () => { + const paths = Object.values(OPTIONAL_DECLARED_INTERFACES) + expect(new Set(paths).size, paths.join(', ')).toBe(paths.length) + }) + + it('registers digital-link-resolver against interfaces.digitalLink — the retrofit that makes it pinnable', () => { + expect(OPTIONAL_DECLARED_INTERFACES['digital-link-resolver']).toBe('interfaces.digitalLink') + }) + + /** + * The whole point of the registry, asserted as a property rather than a + * comment: none of the checks that always-required clauses bind may ever be + * made conditional on a card key. If someone adds one of these to the + * registry, this goes red before any spec is written. + */ + it('registers NONE of the always-required structural checks', () => { + const alwaysRequired = [ + 'agents-json', 'llms-txt', 'openapi', 'icp-json', + 'machine-legible-home', 'card-interfaces-linked', 'probe-manifest', + 'conneg-accept', 'conneg-client-class', 'conneg-alternates', 'conneg-forced-face', + 'keyless-flow', 'offers-402', 'content-negotiation', + ] + for (const id of alwaysRequired) { + expect(OPTIONAL_DECLARED_INTERFACES[id], `${id} must never be declaration-armed`).toBeUndefined() + } + }) +}) + +// =========================================================================== +// 2. THE EVASION GUARD — every one of these must THROW at parse +// =========================================================================== + +describe('the evasion guard refuses at PARSE, before any probe fires', () => { + const throwsNaming = (requirements: unknown[], id: string, match: RegExp) => { + expect(() => parsePinnedSpec(specText(requirements))).toThrow(match) + // The message must NAME the offending requirement — a guard that says + // "something is wrong" costs a spec author an afternoon. + expect(() => parsePinnedSpec(specText(requirements))).toThrow(new RegExp(id)) + } + + it('REFUSES the card-declaration arm on kind:probe — behavioural probes are categorically un-gatable', () => { + throwsNaming( + [{ id: 'gated-probe', kind: 'probe', probe: 'keyless', + appliesWhen: { cardDeclares: 'interfaces.digitalLink' }, + expect: { status: 200 } }], + 'gated-probe', + /legal ONLY on kind:'check'/, + ) + }) + + /** + * THE EVASION TEST. Each of these is an attempt to turn an always-required + * clause into an opt-out by pointing it at a card key. All must throw. + */ + it.each([ + 'keyless-flow', + 'machine-legible-home', + 'probe-manifest', + 'card-interfaces-linked', + 'agents-json', + 'offers-402', + 'conneg-accept', + ])('REFUSES to make the always-required check %s conditional on a card key', (check) => { + throwsNaming( + [{ id: `gate-${check}`, kind: 'check', check, must: 'pass', + appliesWhen: { cardDeclares: 'interfaces.digitalLink' } }], + `gate-${check}`, + /NOT an api\.qa optional-declared interface/, + ) + }) + + it('the refusal prints the eligible set, so the author is not left guessing', () => { + expect(() => parsePinnedSpec(specText([ + { id: 'x', kind: 'check', check: 'keyless-flow', must: 'pass', + appliesWhen: { cardDeclares: 'interfaces.digitalLink' } }, + ]))).toThrow(/"digital-link-resolver"/) + }) + + it('REFUSES gating an eligible check on `probes` — the AXP opt-in signal is not an interface key', () => { + throwsNaming( + [{ id: 'cross-probes', kind: 'check', check: 'digital-link-resolver', must: 'pass', + appliesWhen: { cardDeclares: 'probes' } }], + 'cross-probes', + /not a legal optional-interface card path/, + ) + }) + + it('REFUSES cross-wiring: arming one optional check with ANOTHER interface key', () => { + throwsNaming( + [{ id: 'crosswired', kind: 'check', check: 'digital-link-resolver', must: 'pass', + appliesWhen: { cardDeclares: 'interfaces.testSuite' } }], + 'crosswired', + /api\.qa binds that check to "interfaces\.digitalLink"/, + ) + }) + + it.each([ + ['neither arm', {}], + ['both arms', { fromProbe: 'pricing', path: 'model', equals: 'metered', cardDeclares: 'interfaces.digitalLink' }], + ])('REFUSES an appliesWhen with %s (shape totality)', (_label, appliesWhen) => { + throwsNaming( + [{ id: 'shapeless', kind: 'check', check: 'digital-link-resolver', must: 'pass', appliesWhen }], + 'shapeless', + /Exactly one arm is legal/, + ) + }) + + it.each([ + ['equals', { cardDeclares: 'interfaces.digitalLink', equals: true }], + ['path', { cardDeclares: 'interfaces.digitalLink', path: 'x' }], + ])('REFUSES cardDeclares mixed with `%s` — presence is the whole test', (_label, appliesWhen) => { + throwsNaming( + [{ id: 'mixed', kind: 'check', check: 'digital-link-resolver', must: 'pass', appliesWhen }], + 'mixed', + /tests PRESENCE only/, + ) + }) + + it.each([ + 'interfaces.a.b', + 'interfaces', + 'interfaces.0', + 'interfaces.', + '.digitalLink', + 'Interfaces.digitalLink', + 'interfaces.digitalLink.wellKnown', + 'interfaces[0]', + ])('REFUSES the malformed card path %s', (cardDeclares) => { + throwsNaming( + [{ id: 'badpath', kind: 'check', check: 'digital-link-resolver', must: 'pass', + appliesWhen: { cardDeclares } }], + 'badpath', + /not a legal optional-interface card path/, + ) + }) + + it.each([ + ['surface', { id: 'sfc', kind: 'surface', surface: 'agents.json', must: 'valid' }], + ['ax-floor', { id: 'floor', kind: 'ax-floor', minScore: 8 }], + ['endpoint', { id: 'ep', kind: 'endpoint', method: 'GET', path: '/api/status', expect: { status: 200 } }], + ])('REFUSES an appliesWhen on kind:%s, which would silently condition nothing', (_kind, base) => { + expect(() => parsePinnedSpec(specText([ + { ...base, appliesWhen: { fromProbe: 'pricing', path: 'model', equals: 'metered' } }, + ]))).toThrow(/only kind:'probe' and kind:'check'/) + }) + + it.each([ + ['a non-object', 'nope'], + ['null', null], + ['an array', []], + ])('REFUSES an appliesWhen that is %s', (_label, appliesWhen) => { + expect(() => parsePinnedSpec(specText([ + { id: 'notobj', kind: 'check', check: 'digital-link-resolver', must: 'pass', appliesWhen }, + ]))).toThrow(/not a JSON object|Exactly one arm/) + }) + + it.each([ + ['fromProbe is not a string', { fromProbe: 7, path: 'model', equals: 'metered' }], + ['path is missing', { fromProbe: 'pricing', equals: 'metered' }], + ['equals is missing', { fromProbe: 'pricing', path: 'model' }], + ])('REFUSES a malformed observed-value arm (%s) instead of degrading it to "unobservable"', (_label, appliesWhen) => { + expect(() => parsePinnedSpec(specText([ + { id: 'badprobearm', kind: 'check', check: 'offers-402', must: 'pass', appliesWhen }, + ]))).toThrow(/badprobearm/) + }) + + it('the guard runs for a SUITE too — parseSuite is not a side door around it', () => { + const suite = JSON.stringify({ + $type: 'Suite', name: 's', version: '1', + environments: { public: { vars: {} } }, + requirements: [ + { id: 'sneaky', kind: 'check', check: 'keyless-flow', must: 'pass', + appliesWhen: { cardDeclares: 'interfaces.digitalLink' } }, + ], + }) + expect(() => parseSuite(suite)).toThrow(/NOT an api\.qa optional-declared interface/) + }) + + it('ACCEPTS the one legal shape (the guard is a gate, not a wall)', () => { + const spec = parsePinnedSpec(DL_SPEC) + expect(spec.requirements).toHaveLength(1) + }) + + it('ACCEPTS an eligible check pinned WITHOUT appliesWhen — "I demand this of everyone" stays expressible', () => { + expect(() => parsePinnedSpec(specText([ + { id: 'demanded', kind: 'check', check: 'digital-link-resolver', must: 'pass' }, + ]))).not.toThrow() + }) +}) + +// =========================================================================== +// 3. readCardDeclaration — the three-way, unit level +// =========================================================================== + +describe('readCardDeclaration: absent and unreadable are DIFFERENT, and readPath cannot tell them apart', () => { + const read = (body: string | null, over: Partial = {}) => + readCardDeclaration(cardEvidence({ body, ...over }), 'interfaces.digitalLink') + + it('DECLARED when the key is present with any value at all', () => { + for (const v of ['{}', 'null', 'false', '0', '""', '[]', '"yes"', '{"wellKnown":"/x"}']) { + const st = read(`{"interfaces":{"digitalLink":${v}}}`) + expect(st.state, v).toBe('declared') + } + }) + + it('ABSENT when a well-formed card omits the key — the only row that earns a skip', () => { + expect(read('{"interfaces":{"http":{}}}').state).toBe('absent') + }) + + it('ABSENT when a well-formed card has no `interfaces` member at all', () => { + expect(read('{"name":"x"}').state).toBe('absent') + }) + + it.each([ + ['the card was never fetched', () => readCardDeclaration([], 'interfaces.digitalLink')], + ['the fetch failed', () => read(null, { status: null, error: 'ECONNREFUSED' })], + ['the card answered 404', () => read('{}', { status: 404 })], + ['the body is not JSON', () => read('nope')], + ['the body is a JSON array', () => read('[]')], + ['the body is a JSON scalar', () => read('"a card"')], + ['the body is JSON null', () => read('null')], + ['an intermediate segment is a string', () => read('{"interfaces":"none"}')], + ['an intermediate segment is an array', () => read('{"interfaces":[]}')], + ['an intermediate segment is null', () => read('{"interfaces":null}')], + ])('UNREADABLE when %s — there is no statement to read', (_label, f) => { + expect(f().state).toBe('unreadable') + }) + + it('an unreadable state carries a WHY, so the fail-closed line is diagnosable', () => { + const st = read('{"interfaces":"none"}') + expect(st.state).toBe('unreadable') + expect(st.state === 'unreadable' && st.why).toMatch(/not an object/) + }) +}) + +// =========================================================================== +// 4. THE UNDECLARED-BEHAVIOUR TABLE, end to end through a pinned spec +// =========================================================================== + +describe('an UNREADABLE card never buys a free skip (it is an observation failure)', () => { + it.each([ + ['the card 404s', () => withOverrides(goodTargetRoutes(), { + 'GET /.well-known/agents.json': () => ({ status: 404, contentType: 'application/json', body: '{}' }), + })], + ['the card body is not JSON', () => rawCardRoutes(200, 'application/json', 'not a card')], + ['the card body is a JSON array', () => rawCardRoutes(200, 'application/json', '[]')], + ['the card body is a JSON scalar', () => rawCardRoutes(200, 'application/json', '"a card"')], + ['`interfaces` is the string "none"', () => rawCardRoutes(200, 'application/json', + JSON.stringify({ name: 'x', interfaces: 'none' }))], + ])('APPLIES the requirement and FAILS when %s', async (_label, mk) => { + const { report, dl } = await run(DL_SPEC, mk()) + expect(dl.verdict).toBe('fail') + expect(dl.notApplicable).toBeUndefined() + expect(dl.detail).toMatch(/fail closed/) + expect(report.passed).toBe(false) + }) + + it('APPLIES and FAILS when the card fetch itself errors', async () => { + const routes = goodTargetRoutes() + const inner = makeFetcher(routes) + const fetcher: Fetcher = async (url, init) => { + if (new URL(url).pathname === '/.well-known/agents.json') throw new TypeError('fetch failed: network down') + return inner(url, init) + } + const { dl } = await run(DL_SPEC, routes, fetcher) + expect(dl.verdict).toBe('fail') + expect(dl.detail).toMatch(/fail closed/) + }) + + it('the fail-closed detail says WHY the card was unreadable, not just that it was', async () => { + const { dl } = await run(DL_SPEC, rawCardRoutes(200, 'application/json', '[]')) + expect(dl.detail).toMatch(/JSON array/) + }) +}) + +describe('an OMITTED key on a readable card is a deliberate statement, and it skips', () => { + it('passes as NOT APPLICABLE, with a structured marker (no prose-matching required)', async () => { + const { report, dl, calls } = await run(DL_SPEC, cardRoutes(OMIT)) + expect(dl.verdict).toBe('pass') + expect(dl.notApplicable).toEqual({ reason: 'not-declared', source: 'interfaces.digitalLink' }) + expect(dl.detail).toMatch(/omission is conformance/) + expect(report.passed).toBe(true) + // Nothing was fetched for the undeclared interface — no budget, no probe. + expect(calls.some((u) => u.includes(GS1_RESOLVER_WELL_KNOWN_PATH))).toBe(false) + }) + + it('an EMPTY `interfaces: {}` is still an omission of THIS key — not applicable', async () => { + const routes = rawCardRoutes(200, 'application/json', JSON.stringify({ name: 'x', interfaces: {} })) + const { dl } = await run(DL_SPEC, routes) + expect(dl.verdict).toBe('pass') + expect(dl.notApplicable?.reason).toBe('not-declared') + }) + + it('cites the card as its evidence — a not-applicable line must say what it read', async () => { + const { dl } = await run(DL_SPEC, cardRoutes(OMIT)) + expect(dl.evidence).toContain(ROLE.agentsJson) + }) +}) + +describe('a PRESENT-BUT-EMPTY declaration is a CLAIM, and the armed check fails it', () => { + it.each([ + ['null', null], + ['false', false], + ['0', 0], + ['the empty string', ''], + ['an array', []], + ['a string', 'yes'], + ['true', true], + ])('declaring `interfaces.digitalLink` as %s ARMS the requirement and FAILS it', async (_label, declaration) => { + const { report, dl } = await run(DL_SPEC, cardRoutes(declaration)) + expect(dl.verdict).toBe('fail') + expect(dl.notApplicable).toBeUndefined() + expect(dl.detail).toMatch(/not a JSON object/) + expect(report.passed).toBe(false) + }) + + it('declaring an EMPTY OBJECT arms it and fetches the RFC 8615 default well-known', async () => { + const { report, dl, calls } = await run(DL_SPEC, cardRoutes({})) + expect(calls.some((u) => u.endsWith(GS1_RESOLVER_WELL_KNOWN_PATH))).toBe(true) + expect(dl.notApplicable).toBeUndefined() + expect(dl.verdict).toBe('pass') + expect(report.passed).toBe(true) + }) + + it('declaring an empty object with NO well-known served fails — the claim is checked, not believed', async () => { + const { dl } = await run(DL_SPEC, cardRoutes({}, NO_WELL_KNOWN)) + expect(dl.verdict).toBe('fail') + expect(dl.detail).toMatch(/did not answer 2xx/) + }) +}) + +describe('the escape hatch: an eligible check pinned WITHOUT appliesWhen still demands it of everyone', () => { + it('a non-declaring card FAILS an ungated digital-link-resolver requirement', async () => { + const spec = specText([ + { id: 'check-digital-link-resolver', kind: 'check', check: 'digital-link-resolver', must: 'pass' }, + ]) + const { report, dl } = await run(spec, cardRoutes(OMIT)) + expect(dl.verdict).toBe('fail') + expect(dl.notApplicable).toBeUndefined() + expect(report.passed).toBe(false) + }) +}) + +// =========================================================================== +// 5. THE RETROFIT — digital-link-resolver is now PINNABLE +// =========================================================================== + +describe('the retrofit: digital-link-resolver as a pinned, declaration-armed requirement', () => { + it('NON-DECLARING target → passes as not applicable, and nothing about it was verified', async () => { + const { report, dl } = await run(DL_SPEC, cardRoutes(OMIT)) + expect(report.passed).toBe(true) + expect(dl.verdict).toBe('pass') + expect(dl.notApplicable?.reason).toBe('not-declared') + }) + + it('DECLARING target + truthful well-known → a REAL pass, not a laundered skip', async () => { + const { report, dl } = await run(DL_SPEC, cardRoutes({ wellKnown: GS1_RESOLVER_WELL_KNOWN_PATH })) + expect(report.passed).toBe(true) + expect(dl.verdict).toBe('pass') + // The distinction that makes the mechanism honest: this pass carries NO + // not-applicable marker, so an agent can tell it apart from the row above. + expect(dl.notApplicable).toBeUndefined() + expect(dl.detail).toMatch(/passed/) + }) + + it.each<[string, Routes[string] | typeof NO_WELL_KNOWN]>([ + ['the well-known 404s', NO_WELL_KNOWN], + ['the well-known is not JSON', () => ({ status: 200, contentType: 'text/html', body: '' })], + ['the well-known body does not parse', () => ({ status: 200, contentType: 'application/json', body: '{oops' })], + ['the description file violates the GS1 schema', json({ resolverRoot: GOOD })], + ['the description file names another origin', json({ resolverRoot: 'https://elsewhere.example', supportedPrimaryKeys: ['01'] })], + ])('DECLARING target + %s → FAILS the pinned requirement', async (_label, wellKnown) => { + const { report, dl } = await run(DL_SPEC, cardRoutes({}, wellKnown)) + expect(dl.verdict).toBe('fail') + expect(dl.notApplicable).toBeUndefined() + expect(report.passed).toBe(false) + }) + + it('a card claiming ANOTHER origin\'s resolver fails without the verifier fetching it', async () => { + const { dl, calls } = await run(DL_SPEC, cardRoutes({ wellKnown: 'https://elsewhere.example/.well-known/gs1resolver' })) + expect(dl.verdict).toBe('fail') + expect(dl.detail).toMatch(/NOT the target origin|refused/) + expect(calls.some((u) => u.startsWith('https://elsewhere.example'))).toBe(false) + }) +}) + +// =========================================================================== +// 6. THE AXP-SHAPED SPEC, end to end: 21 → 22 requirements +// =========================================================================== + +/** + * The vendored apis-ax-axp@2.2.0 requirement list with the declaration-armed + * requirement appended — the shape the standard's own ratification will take. + * + * This is api.qa proving the mechanism carries a REAL conformance spec, not a + * toy. It deliberately does NOT edit the vendored spec file: the canonical + * bytes (and therefore the digest) are the standard's to ratify, and api.qa + * front-running that would put two different documents in circulation under one + * version number. api.qa's vendored copy re-syncs after the standard ratifies. + */ +function axpSpecPlusDigitalLink(): string { + const doc = JSON.parse(readFileSync(AXP_SPEC_PATH, 'utf8')) as { requirements: unknown[] } + const at = doc.requirements.findIndex((r) => (r as { id: string }).id === 'probe-manifest-valid') + doc.requirements.splice(at + 1, 0, DL_REQUIREMENT) + return JSON.stringify(doc) +} + +describe('the AXP admission spec with the declaration-armed requirement appended', () => { + const spec = axpSpecPlusDigitalLink() + + it('is 22 requirements, and parses — the guard does not reject the real spec', () => { + const parsed = parsePinnedSpec(spec) + expect(parsed.requirements).toHaveLength(22) + expect(parsed.requirements.filter((r) => 'appliesWhen' in r && (r as any).appliesWhen?.cardDeclares)) + .toHaveLength(1) + }) + + it('a conformant NON-DECLARING target passes 22/22, with the new requirement not applicable', async () => { + const report = await verifyPinnedSpec(GOOD, spec, { + fetcher: urlAwareFetcher(axpReferenceRoutes()), delayMs: 0, seed: 11, mode: 'local', + }) + const failed = report.requirements.filter((r) => r.verdict !== 'pass') + expect(failed.map((r) => `${r.id}: ${r.detail}`)).toEqual([]) + expect(report.requirements).toHaveLength(22) + const dl = report.requirements.find((r) => r.id === 'check-digital-link-resolver')! + expect(dl.notApplicable).toEqual({ reason: 'not-declared', source: 'interfaces.digitalLink' }) + // NOTHING that was conforming yesterday started failing. + expect(report.passed).toBe(true) + }) + + it('a conformant DECLARING target that honours the claim passes 22/22, the new one for REAL', async () => { + const routes = withOverrides( + axpReferenceRoutes({ model: 'free' }, (card) => { card.interfaces.digitalLink = {} }), + { [WK_ROUTE]: json(VALID_DESCRIPTION_FILE) }, + ) + const report = await verifyPinnedSpec(GOOD, spec, { + fetcher: urlAwareFetcher(routes), delayMs: 0, seed: 11, mode: 'local', + }) + const failed = report.requirements.filter((r) => r.verdict !== 'pass') + expect(failed.map((r) => `${r.id}: ${r.detail}`)).toEqual([]) + const dl = report.requirements.find((r) => r.id === 'check-digital-link-resolver')! + expect(dl.notApplicable).toBeUndefined() + }) + + /** + * SHIP THE FAILING CASE. This is the population whose verdict the retrofit + * actually changes: a target that DECLARES the optional interface and does + * not honour it loses admission. If this test is absent, the retrofit is + * tested only where it cannot fail. + */ + it('a conformant DECLARING target with a BROKEN well-known FAILS 21/22 — and only on that requirement', async () => { + const routes = axpReferenceRoutes({ model: 'free' }, (card) => { card.interfaces.digitalLink = {} }) + const report = await verifyPinnedSpec(GOOD, spec, { + fetcher: urlAwareFetcher(routes), delayMs: 0, seed: 11, mode: 'local', + }) + const failed = report.requirements.filter((r) => r.verdict !== 'pass') + expect(failed.map((r) => r.id)).toEqual(['check-digital-link-resolver']) + expect(report.requirements.filter((r) => r.verdict === 'pass')).toHaveLength(21) + expect(report.passed).toBe(false) + }) + + it('the untouched 21 behave identically with and without the new requirement (no collateral)', async () => { + const base = readFileSync(AXP_SPEC_PATH, 'utf8') + const fetcher = urlAwareFetcher(axpReferenceRoutes()) + const before = await verifyPinnedSpec(GOOD, base, { fetcher, delayMs: 0, seed: 11, mode: 'local' }) + const after = await verifyPinnedSpec(GOOD, spec, { fetcher, delayMs: 0, seed: 11, mode: 'local' }) + const strip = (rs: typeof before.requirements) => + rs.filter((r) => r.id !== 'check-digital-link-resolver').map((r) => `${r.id}=${r.verdict}`) + expect(strip(after.requirements)).toEqual(strip(before.requirements)) + }) +}) + +// =========================================================================== +// 7. REPORTING — a skip must be legible AS a skip +// =========================================================================== + +describe('a not-applicable requirement is distinguishable from a pass and from a not-run', () => { + it('the THREE-WAY holds at the requirement level', async () => { + const notApplicable = (await run(DL_SPEC, cardRoutes(OMIT))).dl + const verified = (await run(DL_SPEC, cardRoutes({}))).dl + const violated = (await run(DL_SPEC, cardRoutes({}, NO_WELL_KNOWN))).dl + + expect([notApplicable.verdict, notApplicable.notApplicable !== undefined]).toEqual(['pass', true]) + expect([verified.verdict, verified.notApplicable !== undefined]).toEqual(['pass', false]) + expect([violated.verdict, violated.notApplicable !== undefined]).toEqual(['fail', false]) + }) + + it('a NEVER-PRODUCED check is still a hard fail, never a not-applicable', async () => { + // `published-test-suite` is REGISTERED as eligible but api.qa does not + // produce it yet. Against a DECLARING card the requirement must fail + // loudly — a verifier too old for the spec it was handed says so. + const spec = specText([ + { id: 'suite-req', kind: 'check', check: 'published-test-suite', must: 'pass', + appliesWhen: { cardDeclares: 'interfaces.testSuite' } }, + ]) + const base = goodTargetRoutes() + const card = JSON.parse(base['GET /.well-known/agents.json']!({ method: 'GET', accept: 'application/json' }).body!) as Record + card.interfaces.testSuite = { url: '/suite.json' } + const report = await verifyPinnedSpec(GOOD, spec, { + fetcher: makeFetcher(withOverrides(base, { 'GET /.well-known/agents.json': json(card) })), + delayMs: 0, seed: 7, mode: 'local', + }) + const r = report.requirements.find((x) => x.id === 'suite-req')! + expect(r.verdict).toBe('fail') + expect(r.notApplicable).toBeUndefined() + expect(r.detail).toMatch(/unknown check/) + }) + + it('the OBSERVED-VALUE arm gets a marker too, and keeps its wording byte-for-byte', async () => { + const spec = specText([ + { id: 'pricing-declared', kind: 'probe', probe: 'pricing', + expect: { status: 200, paths: [{ path: 'model', oneOf: ['free', 'metered'] }] } }, + { id: 'gated-offers', kind: 'check', check: 'offers-402', must: 'pass', + appliesWhen: { fromProbe: 'pricing', path: 'model', equals: 'metered' } }, + ]) + const report = await verifyPinnedSpec(GOOD, spec, { + fetcher: urlAwareFetcher(axpReferenceRoutes()), delayMs: 0, seed: 3, mode: 'local', + }) + const gated = report.requirements.find((r) => r.id === 'gated-offers')! + expect(gated.verdict).toBe('pass') + expect(gated.notApplicable).toEqual({ reason: 'observed-value', source: 'probes.pricing model' }) + expect(gated.detail).toBe( + 'not applicable: probes.pricing model = "free" (requirement applies only when it equals "metered") — passes as not applicable', + ) + }) + + it('a gated kind:probe carries the observed-value marker as well', async () => { + const spec = specText([ + { id: 'pricing-declared', kind: 'probe', probe: 'pricing', + expect: { status: 200, paths: [{ path: 'model', oneOf: ['free', 'metered'] }] } }, + { id: 'ceiling', kind: 'probe', probe: 'overCeiling', + appliesWhen: { fromProbe: 'pricing', path: 'model', equals: 'metered' }, + expect: { status: [402], paths: [{ path: 'type', equals: 'OFFER' }] } }, + ]) + const report = await verifyPinnedSpec(GOOD, spec, { + fetcher: urlAwareFetcher(axpReferenceRoutes()), delayMs: 0, seed: 5, mode: 'local', + }) + expect(report.requirements.find((r) => r.id === 'ceiling')!.notApplicable?.reason).toBe('observed-value') + }) +}) + +describe('the CI reporters render the third state', () => { + it('JUnit emits for a not-applicable requirement, and counts it as skipped not passed', async () => { + const { report } = await run(DL_SPEC, cardRoutes(OMIT)) + const xml = junitXml(report) + assertWellFormedXml(xml) + expect(xml).toMatch(/]*check-digital-link-resolver[^>]*><\/testcase>/) + expect(xml).toMatch(/skipped="1"/) + + const jr = jsonReport(report) + expect(jr.totals.skipped).toBe(1) + expect(jr.totals.passed).toBe(jr.totals.tests - 1) + expect(jr.suites[0]!.cases.find((c) => c.id === 'check-digital-link-resolver')!.status).toBe('skip') + }) + + it('a REAL pass is still a JUnit pass — the skip mapping keys on the marker, not the check id', async () => { + const { report } = await run(DL_SPEC, cardRoutes({})) + const xml = junitXml(report) + expect(xml).toMatch(/skipped="0"/) + expect(xml).not.toMatch(//) + }) + + it('the JSON report keeps the not-applicable DETAIL, so a skip is not an unexplained blank', async () => { + const { report } = await run(DL_SPEC, cardRoutes(OMIT)) + const c = jsonReport(report).suites[0]!.cases.find((x) => x.id === 'check-digital-link-resolver')! + expect(c.detail).toMatch(/omission is conformance/) + }) + + it('the markdown report counts three states and marks the row n/a, not PASS', async () => { + const { report } = await run(DL_SPEC, cardRoutes(OMIT)) + const md = pinnedMarkdown(report) + expect(md).toMatch(/0 passed · 1 not applicable · 0 failed/) + expect(md).toMatch(/\| n\/a \|/) + expect(md).not.toMatch(/\| PASS \|/) + }) + + it('`PinnedReport.passed` is UNCHANGED by the marker — no free-model target flips to failed', async () => { + const { report } = await run(DL_SPEC, cardRoutes(OMIT)) + expect(report.passed).toBe(true) + expect(report.requirements.every((r) => r.verdict === 'pass')).toBe(true) + }) +})