From 9e97e2f012f550d1d34887b3fb935f0dbbf404f5 Mon Sep 17 00:00:00 2001 From: Nathan Clevenger <4130910+nathanclevenger@users.noreply.github.com> Date: Sat, 8 Aug 2026 10:24:02 -0500 Subject: [PATCH 1/3] =?UTF-8?q?feat(exec):=20api.qa/vitest@1=20executable?= =?UTF-8?q?=20suites=20=E2=80=94=20shared=20harness,=20isolate=20runner,?= =?UTF-8?q?=20grading=20wire-in?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the runner phase of AXP 0.7.0 / apis-ax-axp@2.4.0 (spec digest dd3e5941…): the executable published-test-suite dialect the corrected spec ratified beside the declarative api.qa/suite@1 path. ONE SHARED HARNESS (A.8.6.2, normative parity). src/exec/vitest-subset.mjs is the single implementation of the guaranteed vitest subset (describe/it/expect + async, rejects/resolves, .not; snapshots/vi/unknown matchers fail BY NAME). scripts/gen-vitest-subset.mjs carries its exact bytes into a string constant so the hosted isolate's module map and the local runner instantiate byte-identical code; a pinning test fails on drift. Local==hosted by construction. CODE FIELDS, ONE ARTIFACT, ONE DIGEST (A.8.6.1). suite-doc.ts gains parseExecSuiteDocument: the additive suite@1 extension carrying `tests` (required) + `module` (optional) as string members; types.ts carries them plus the environment `sandbox` consent flag. One fetch, one buffer, hash-then-instantiate — digest fail-closed before anything runs. THE THREE COLLAPSED CHANNELS (A.8.6.6). test-suite.ts gateVitestSuiteCard / gateVitestSuiteDocument / gateVitestModuleArtifact: inline document strings, {url,digest} natively-served ESM (off-origin module-CDN allowed — the digest, never the host, is the authority), and {package,version,digest} as an IDENTITY ASSERTION recorded but never adjudicated (the registry is never contacted). The widened seam { url?, package?, version?, export?, digest, environment?, runner? } is read in discovery.ts; artifact kind is decided by the card, never by sniffing. ISOLATE POSTURE (A.8.6.3). src/exec/runner.ts (Worker Loader) + dialect.ts: zero ambient authority (env is always {}), a network floor that bars metadata/link-local/loopback/RFC1918/CGNAT/ULA/estate-internal ONLY and permits all other external egress (cross-estate composition is a feature; graded-origin scoping rejected), a metered circuit-breaker (300s wall / 60s CPU, billed, account-raisable), seeded Math.random, mutating verbs only where the environment declares sandbox:true, and fail-closed totality (a caught floor refusal still fails the run). worker_loaders is feature-detected: absent binding (or absent egress gateway) yields a typed runner-unavailable outcome, never a crash or a silent pass; wrangler.jsonc documents the flag-held binding. GRADING + ATTESTATION (A.8.5.2 / A.8.6.5). checks.ts arms `published-test-suite` on the vitest@1 runner and judges purely from the recorded typed run outcome (declared => run => must pass; undeclared => skip). The verdict carries executed digest, discriminated kind, environment + sandbox, seed, applied breaker limits + elapsed, folded rows+tests, and the npm coordinate where asserted. src/pinned.ts pins AXP_PINNED_SPEC to the final 2.4.0 digest dd3e5941…. LOCAL PARITY. cli/index.ts gains `vitest --target …` running the dialect through the same shared harness; exits non-zero on any failing test/row and fail-closed on a --expect-digest mismatch. Tests: harness subset behaviour, digest fail-closed, network floor (metadata/RFC1918 blocked, external allowed), the metered breaker, declared/undeclared arming, runner-unavailable, an end-to-end fixture, and the CLI verb. Full suite 1226 passing; typecheck + build green. Co-Authored-By: Claude Fable 5 --- cli/index.ts | 123 +++++- scripts/gen-vitest-subset.mjs | 44 ++ skill/SKILL.md | 7 +- src/checks.ts | 161 +++++++ src/discovery.ts | 239 ++++++++++- src/exec/dialect.ts | 622 +++++++++++++++++++++++++++ src/exec/runner.ts | 344 +++++++++++++++ src/exec/vitest-subset-source.ts | 11 + src/exec/vitest-subset.mjs | 395 +++++++++++++++++ src/http.ts | 18 +- src/index.ts | 53 +++ src/pinned.ts | 27 +- src/suite-doc.ts | 71 ++- src/test-suite.ts | 379 +++++++++++++++- src/types.ts | 22 + src/verify.ts | 11 +- src/worker.ts | 39 ++ test/cli-vitest.test.ts | 61 +++ test/fixtures/vitest-suite.json | 8 + test/vitest-subset.test.ts | 642 ++++++++++++++++++++++++++++ test/vitest-suite-interface.test.ts | 361 ++++++++++++++++ wrangler.jsonc | 22 + 22 files changed, 3636 insertions(+), 24 deletions(-) create mode 100644 scripts/gen-vitest-subset.mjs create mode 100644 src/exec/dialect.ts create mode 100644 src/exec/runner.ts create mode 100644 src/exec/vitest-subset-source.ts create mode 100644 src/exec/vitest-subset.mjs create mode 100644 test/cli-vitest.test.ts create mode 100644 test/fixtures/vitest-suite.json create mode 100644 test/vitest-subset.test.ts create mode 100644 test/vitest-suite-interface.test.ts diff --git a/cli/index.ts b/cli/index.ts index 969c30a..494a3cd 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -57,7 +57,9 @@ import { import { verifyAttestation } from '../src/attest.js' import { sha256Hex } from '../src/digest.js' import { runMcpServer } from '../src/mcp.js' -import type { VerificationReport } from '../src/types.js' +import { localExecRunner, type ExecRunOutcome } from '../src/exec/dialect.js' +import { parseExecSuiteDocument } from '../src/suite-doc.js' +import type { Suite, VerificationReport } from '../src/types.js' interface Flags { /** Last value wins (single-valued reads). */ @@ -246,6 +248,121 @@ async function main(): Promise { return emit(report, suiteMarkdown(report), flags) } + if (cmd === 'vitest') { + // vitest — run an `api.qa/vitest@1` artifact + // LOCALLY through the SAME shared subset harness the hosted verifier + // executes (AXP A.8.6.2 parity is by construction: one module, byte- + // identical, never a reimplementation). Local runs are ADVISORY (never + // attested); the definition of done is the SAME digest passing hosted. + // + // npx autonomous-qa vitest suite.json --target https://api.example + // [--env ] [--expect-digest sha256:<64hex>] [--seed ] [--json] + // npx autonomous-qa vitest dist/index.mjs --target https://api.example + // [--export suite] [--expect-digest sha256:<64hex>] + // + // Artifact KIND follows the card rule (A.8.5): a path ending `.mjs` (or + // `--module`) is a module artifact; anything else is a suite document. + // The digest gate is FAIL-CLOSED: with --expect-digest, bytes that do not + // hash to the pin never instantiate. + const file = rest[0] + if (!file) return die('vitest needs an artifact: a suite document (.json) or a module (.mjs)') + const text = readFileSync(file, 'utf8') + const digest = `sha256:${await sha256Hex(text)}` + const expect = flags.get('expect-digest') + if (expect !== undefined && expect !== digest) { + console.error( + `autonomous-qa: vitest artifact digest mismatch: expected ${expect}, ${file} hashes to ${digest}. ` + + 'NOTHING was instantiated (digest fail-closed, A.8.6.3).', + ) + return 1 + } + const isModule = flags.has('module') || /\.mjs$/i.test(file) + const envName = flags.get('env') ?? 'public' + + let doc: Suite | undefined + if (!isModule) { + try { + doc = parseExecSuiteDocument(text) + } catch (err) { + console.error(`autonomous-qa: ${err instanceof Error ? err.message : String(err)}`) + return 1 + } + if (!Object.hasOwn(doc.environments, envName)) { + return die( + `vitest: environment "${envName}" is not defined by the suite (it defines ${Object.keys(doc.environments).join(', ') || 'none'})`, + ) + } + } else if (flags.has('env') && envName !== 'public') { + return die('vitest: a module artifact defines no environments — only the implicit "public" exists (A.8.6.4)') + } + const env = doc?.environments[envName] + const vars = { ...(env?.vars ?? {}) } + const target = flags.get('target') ?? (typeof vars.baseUrl === 'string' ? (vars.baseUrl as string) : undefined) + if (!target) return die('vitest needs a target: --target (or a string `baseUrl` var in the selected environment)') + const runSeed = seed ?? (Math.floor(Math.random() * 0xffffffff) >>> 0) + + const outcome: ExecRunOutcome = await localExecRunner().run({ + artifactKind: isModule ? 'module' : 'document', + testsSource: isModule ? text : doc!.tests!, + ...(doc?.module !== undefined && { moduleSource: doc.module }), + ...(flags.has('export') && { exportName: flags.get('export') }), + origin: new URL(/^https?:\/\//.test(target) ? target : `https://${target}`).origin, + vars, + environment: envName, + sandbox: env?.sandbox === true, + seed: runSeed, + declarativeRows: doc?.requirements.length ?? 0, + digest, + }) + + // Declarative rows (document form) keep unchanged suite@1 engine + // semantics — run them through the SAME verifySuite door the `suite` verb + // uses, folded into one exit code (A.8.6.5's one result set). + let rowsPassed = true + let rowLines: string[] = [] + if (doc !== undefined && doc.requirements.length > 0) { + const rowSuite = JSON.stringify({ + $type: 'Suite', + name: doc.name, + version: doc.version, + environments: doc.environments, + requirements: doc.requirements, + }) + const rowReport = await verifySuite(rowSuite, envName, { + mode: 'local', + seed: runSeed, + target, + delayMs: isLocalTarget(target) ? 0 : 150, + }) + rowsPassed = rowReport.passed + rowLines = rowReport.requirements.map( + (r) => ` ${r.verdict === 'pass' ? 'PASS' : 'FAIL'} row ${r.id}${r.verdict === 'pass' ? '' : ` — ${r.detail}`}`, + ) + } + + if (flags.get('json') === 'true') { + console.log(JSON.stringify({ runner: 'api.qa/vitest@1', digest, seed: runSeed, environment: envName, outcome, rowsPassed }, null, 2)) + } else { + console.log(`# api.qa/vitest@1 — local run (advisory, never attested)`) + console.log(`artifact ${file} (${isModule ? 'module' : 'document'}), digest ${digest}`) + console.log(`target ${target}, environment "${envName}"${env?.sandbox === true ? ' (sandbox)' : ''}, seed ${runSeed}`) + for (const line of rowLines) console.log(line) + if (outcome.status === 'ran') { + for (const r of outcome.results) { + console.log(` ${r.status === 'pass' ? 'PASS' : 'FAIL'} ${r.name} (${r.durationMs} ms)${r.reason ? ` — ${r.reason}` : ''}`) + } + console.log( + `${outcome.results.filter((r) => r.status === 'pass').length}/${outcome.registered} tests passed ` + + `(${outcome.elapsedWallMs} ms wall, limits ${outcome.appliedLimits.wallMs} ms wall / ${outcome.appliedLimits.cpuMs} ms CPU)`, + ) + } else { + console.log(`RUN ${outcome.status.toUpperCase()}: ${outcome.reason}`) + } + } + const testsPassed = outcome.status === 'ran' && outcome.results.every((r) => r.status === 'pass') + return testsPassed && rowsPassed ? 0 : 1 + } + if (cmd === 'contract-diff') { // contract-diff (ax-gyh): the highest-value CI gate // — "did this deploy break the published contract?". The spec is LOCAL (a @@ -464,6 +581,10 @@ function usage(): string { [--iteration-data ] run once per dataset row (data-driven) [--target ] [--expect-digest ] [--seed ] (target defaults to the selected environment's baseUrl var) + npx autonomous-qa vitest run an api.qa/vitest@1 executable suite LOCALLY + --target [--env ] [--export ] through the SAME shared subset harness the hosted + [--expect-digest ] [--seed ] verifier executes (local==hosted by construction); + [--module] [--json] EXITS NON-ZERO iff any test or row fails npx autonomous-qa contract-diff did this deploy break the published contract? [--json] [--reporter ...] [--seed ] EXITS NON-ZERO on any breaking operation diff (spec is the published OpenAPI file; target is fetched live, SSRF-gated) diff --git a/scripts/gen-vitest-subset.mjs b/scripts/gen-vitest-subset.mjs new file mode 100644 index 0000000..aec10f5 --- /dev/null +++ b/scripts/gen-vitest-subset.mjs @@ -0,0 +1,44 @@ +#!/usr/bin/env node +/** + * Regenerate `src/exec/vitest-subset-source.ts` from the canonical harness + * `src/exec/vitest-subset.mjs`. + * + * node scripts/gen-vitest-subset.mjs + * + * WHY A GENERATED STRING CONSTANT EXISTS AT ALL. AXP A.8.6.2 requires the + * hosted verifier and the local CLI to share ONE implementation of the + * `api.qa/vitest@1` subset. The hosted runner must inject the harness into a + * Worker Loader isolate's module map as SOURCE TEXT (an isolate takes strings, + * not in-memory objects), and a Worker has no filesystem to read the .mjs from + * — so the exact bytes are carried as a TypeScript string constant, exactly + * like `scripts/gen-assets.mjs` inlines the raster assets. The local runner + * instantiates the SAME constant via a `data:text/javascript` import, so both + * hosts execute byte-identical harness code by construction. + * + * The constant is committed; a normal build does NOT run this. A pinning test + * (test/vitest-subset.test.ts) fails when the generated bytes drift from the + * canonical file, naming this script as the fix. + */ + +import { readFileSync, writeFileSync } from 'node:fs' + +const canonicalUrl = new URL('../src/exec/vitest-subset.mjs', import.meta.url) +const outUrl = new URL('../src/exec/vitest-subset-source.ts', import.meta.url) + +const source = readFileSync(canonicalUrl, 'utf8') + +const generated = `/** + * GENERATED — do not edit. Regenerate with: node scripts/gen-vitest-subset.mjs + * + * The exact bytes of the canonical \`api.qa/vitest@1\` subset harness + * (src/exec/vitest-subset.mjs), carried as a string so the hosted Worker + * Loader runner can inject them into an isolate's module map and the local + * runner can instantiate them via a data: import — ONE implementation for + * both hosts (AXP A.8.6.2), pinned byte-identical by test/vitest-subset.test.ts. + */ + +export const VITEST_SUBSET_SOURCE: string = ${JSON.stringify(source)} +` + +writeFileSync(outUrl, generated) +console.log(`gen-vitest-subset: wrote ${outUrl.pathname} (${source.length} bytes of harness source)`) diff --git a/skill/SKILL.md b/skill/SKILL.md index aebde92..ace3ca8 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -5,7 +5,7 @@ description: Build and verify AXP-conformant API surfaces (the agent-experience # AXP — the agent-experience standard -Every API property ships a machine face that agents can discover, price, call, and **verify without asking anyone**. The standard is AXP (https://axp.org.ai, pinned spec `apis-ax-axp@2.2.0`); the verifier is api.qa (`autonomous-qa` on npm). This skill encodes how to build to it. +Every API property ships a machine face that agents can discover, price, call, and **verify without asking anyone**. The standard is AXP (https://axp.org.ai, pinned spec `apis-ax-axp@2.4.0`); the verifier is api.qa (`autonomous-qa` on npm). This skill encodes how to build to it. ## The doctrine (why, in one paragraph) @@ -18,6 +18,11 @@ A published test suite converts **"trust us" into "run this."** It is the execut 3. **`/pricing`** — the machine-readable pricing document. `{"model":"free"}` where true — an agent must never have to ask a human what something costs; metered surfaces declare hard ceilings, offers, and the 402 boundary. 4. **`llms.txt`** — cross-linking the rest of your API family. +**Optional declared interfaces — additive only, and only where already true (AXP 0.6.0, Appendix A.8):** beside `http` and `mcp`, a card MAY declare further `interfaces.` members. **Presence is the declaration**, and the rule is two-sided: omitting the key is *fully conformant* and the armed check `skip`s, while declaring it is judged **strictly** — a defective declaration **fails**, it does not skip. There is no value meaning "no"; a card that means no omits the key. Optional interfaces are **additive capabilities only**: none of them can ever relieve you of Clauses 1–7. + +- `interfaces.digitalLink` — this origin's GS1 **Resolver Description File** at `/.well-known/gs1resolver`, so an agent which has never heard of GS1 learns from the card it already reads that this origin resolves GS1 keys. Declared, that file MUST answer 200, MUST validate against GS1's published description-file schema, and its `resolverRoot` MUST be this origin. **Since `apis-ax-axp@2.3.0` this is admission-pinned in declaration-armed form** (`check-digital-link-resolver`, `appliesWhen: { cardDeclares: "interfaces.digitalLink" }`) — so declare it only where the well-known already answers; a card that omits it passes as *not applicable*. AXP restates none of GS1's vocabulary and verifies none of GS1's resolution behaviour (linkType, RFC 9264 linksets, redirects) — that is GS1's standard and GS1's test suite. +- `interfaces.testSuite` — this origin's own digest-pinned conformance suite. Card seam `{ url?, package?, version?, export?, digest, environment?, runner? }`: at least one address, one `sha256:` digest as the **sole byte authority**. Two ratified dialects (AXP 0.7.0): `api.qa/suite@1` — declarative rows the verifier *interprets*, GET/HEAD-only with writes disabled, same-origin — and `api.qa/vitest@1` (Appendix A.8.6) — executable tests as a digest-pinned module, addressed by any of the three collapsed channels (A.8.6.6): string members inside the pinned suite document (`tests` + optional `module`), a natively served ES module at a versioned URL (e.g. `https://pkg.do/apis.vin@1.2.0/index.mjs` — the SDK case; an AXP package property MUST serve `.mjs` + `.d.ts` natively at immutable versioned URLs with `{package, version, digest}` provenance), or an npm `package@version` **identity assertion** over the served bytes (npm a verifiable mirror, never in the loop). A guaranteed vitest subset (`describe`/`it`/`expect` + async; imports closed to `vitest` / `suite:env` / `suite:module`; no node built-ins, snapshots, or mocking) the verifier *executes* in a fresh zero-authority isolate above a network floor (no metadata/link-local/private/verifier-internal destinations; **full external egress otherwise**), under a metered circuit-breaker deadline (default 300s wall / 60s CPU, billed; suite@1 keeps its fixed 20s), with seeded randomness and mutating verbs only against an environment the suite declares `sandbox: true`. The runner is a paid-tier capability — the paid tier, never scarcity, is its gate — so the remaining caps are abuse circuit-breakers, not rations: 1000 rows+tests combined, 1 MiB document, 4 MiB module artifact, 4 MiB output. Same file runs under local vitest and hosted api.qa — one shared harness, parity by construction. **Since `apis-ax-axp@2.4.0` this is admission-pinned in declaration-armed form** (`check-published-test-suite`) — declare it only where the artifact already answers at the declared pin; a card that omits it passes as *not applicable*. + Plus: **typed envelopes** (`OK / EMPTY / BLOCKED / OFFER` — three emptinesses never blend) and **the conneg law** on every dereferenceable address: extension forces (`.html/.json/.md`) → Accept infers (q-values, header-order ties) → client-class defaults on `*/*` (browser via Sec-Fetch-*, never UA sniffing → HTML; known agent UAs → markdown; everything else incl. bare curl → JSON). JSON faces are JSON-LD with resolvable `$context`. `Link rel="alternate"` advertises siblings. Never 406. HEAD mirrors GET. ## How to build (never hand-roll) diff --git a/src/checks.ts b/src/checks.ts index de77a6d..0e7be08 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -33,6 +33,7 @@ import { isMcpUiMime, SUITE_ROLE_PREFIX, type ServerJsonClaims, + type TestSuiteClaim, } from './discovery.js' // The card-declared test-suite gates and the shared expectation engine — the // SAME code the observe side ran, so what was fetched and what it means cannot @@ -40,9 +41,14 @@ import { // second expectation judge would make conformance depend on which door asked. import { MAX_SUITE_REQUIREMENTS, + VITEST_RUNNER, gateTestSuiteCard, gateTestSuiteDocument, + gateVitestModuleArtifact, + gateVitestSuiteCard, + gateVitestSuiteDocument, } from './test-suite.js' +import type { ExecRunOutcome, ExecTestResult } from './exec/dialect.js' import { captureInto, judgeExpect, resolveEndpoint, type Bindings } from './expect.js' import { isPubliclyRoutableSameOrigin, isPublicHttpsOffOriginAllowed } from './http.js' import { validateSchema } from './schema.js' @@ -917,6 +923,11 @@ export function runChecks(bundle: EvidenceBundle): CheckResult[] { detail: 'no published test suite interface declared (agents.json `interfaces.testSuite` absent) — the interface is OPTIONAL and this card does not claim it, so nothing was fetched and nothing is judged; under a pinned must:pass this fails closed', } + } else if (claim.runner === VITEST_RUNNER) { + // The EXECUTABLE dialect (A.8.6): judged purely from the recorded + // artifact + the recorded typed run outcome — the judge never + // re-executes (observe/judge split, same as every other check). + result = judgeVitestPublishedSuite(bundle, claim, evidence) } else { const cardGate = gateTestSuiteCard(claim, bundle.target) if (!cardGate.ok) { @@ -3347,6 +3358,156 @@ function trimTrailingSlash(url: string): string { return url.endsWith('/') ? url.slice(0, -1) : url } +/** + * Judge a card-declared `api.qa/vitest@1` suite (A.8.5.2 / A.8.6.5), PURELY + * from the bundle: + * + * - the card gate re-derives (defective declaration ⇒ fail, never skip); + * - the artifact evidence must be a 200 with a body that HASHES TO THE PIN + * (digest fail-closed — a mismatch means nothing was ever instantiated, + * and this judge proves it from the stored bytes); + * - declarative rows are re-judged with the IDENTICAL suite@1 engine the + * declarative branch uses (`resolveEndpoint`/`judgeExpect`, same binding + * scope, capture-on-pass); + * - the EXECUTED tests are judged from the recorded typed run outcome + * (ROLE.vitestRun): `runner-unavailable` and `failed` are named failures; + * `ran` folds per-test results with the rows into ONE result set. The + * judge never re-executes — replay of a stored bundle re-judges + * identically with no isolate and no fetch. + * + * The passing detail carries the A.8.6.5 attestation surface: executed + * digest, discriminated kind + address, environment + sandbox, seed, applied + * breaker limits + elapsed, counts (rows, tests, distinct pathnames), and + * the npm coordinate where asserted — RECORDED, never adjudicated. + */ +function judgeVitestPublishedSuite( + bundle: EvidenceBundle, + claim: TestSuiteClaim, + evidence: string[], +): { verdict: Verdict; detail: string } { + const cardGate = gateVitestSuiteCard(claim, bundle.target) + if (!cardGate.ok) return { verdict: 'fail', detail: cardGate.problem } + + const docEv = findEvidence(bundle, ROLE.testSuite) + if (!ok(docEv) || docEv?.body == null) { + return { + verdict: 'fail', + detail: + `GET ${cardGate.url} did not answer 2xx with a body — ${!docEv ? 'not fetched' : docEv.status === null ? `fetch failed (${docEv.error ?? 'unknown'})` : `status ${docEv.status}`}. ` + + 'The card DECLARES a published executable suite, so the pinned artifact must be served.', + } + } + + // Row judging state (document addressing only). + const rowProblems: string[] = [] + const pathnames = new Set() + let rowCount = 0 + let sandbox = false + + if (cardGate.kind === 'document') { + const plan = gateVitestSuiteDocument(claim, docEv.body, cardGate.digest) + if (!plan.ok) return { verdict: 'fail', detail: plan.problems.slice(0, 6).join('; ') } + sandbox = plan.sandbox + rowCount = plan.rows.length + // The IDENTICAL row engine the declarative branch judges with. + const bindings: Bindings = { ...plan.vars } + for (const req of plan.rows) { + const resolved = resolveEndpoint(req, bundle.target, bindings) + if (!resolved.ok) { + rowProblems.push(`"${req.id}": ${resolved.detail}`) + continue + } + const role = `${SUITE_ROLE_PREFIX}pinned:${req.id}` + evidence.push(role) + const ev = bundle.items.find((e) => e.role === role) + const ps = judgeExpect(ev, resolved.expect) + if (ps.length === 0) { + try { pathnames.add(new URL(resolved.url).pathname) } catch { /* resolved urls parse */ } + if (req.capture) captureInto(bindings, req.capture, ev) + } else { + rowProblems.push(`"${req.id}" ${resolved.method} ${resolved.url}: ${ps.join('; ')}`) + } + } + } else { + const mg = gateVitestModuleArtifact(claim, docEv.body, cardGate.digest) + if (!mg.ok) return { verdict: 'fail', detail: mg.problems.slice(0, 6).join('; ') } + } + + // The recorded typed run outcome — REQUIRED. Its absence after a valid, + // digest-matched artifact means the run never happened: fail closed. + const runEv = findEvidence(bundle, ROLE.vitestRun) + evidence.push(ROLE.vitestRun) + if (runEv?.body == null) { + return { + verdict: 'fail', + detail: + 'no executable run outcome was recorded for the declared api.qa/vitest@1 suite — the artifact gates passed but ' + + 'the run is missing from the bundle, so nothing proves the tests ran. Fail closed, never a silent pass.', + } + } + let record: { + executedDigest?: string + seed?: number + npm?: { package: string; version: string } + outcome?: ExecRunOutcome + } + try { + record = JSON.parse(runEv.body) as typeof record + } catch { + return { verdict: 'fail', detail: 'the recorded executable run outcome is not parseable JSON — fail closed' } + } + if (record.executedDigest !== cardGate.digest) { + return { + verdict: 'fail', + detail: + `the recorded run executed digest ${JSON.stringify(record.executedDigest)} but the card pins ${cardGate.digest} — ` + + 'the run is not a run of the pinned bytes (digest fail-closed, A.8.6.3)', + } + } + const outcome = record.outcome + if (outcome === undefined) { + return { verdict: 'fail', detail: 'the recorded executable run carries no outcome — fail closed' } + } + if (outcome.status === 'runner-unavailable') { + return { verdict: 'fail', detail: outcome.reason } + } + if (outcome.status === 'failed') { + return { + verdict: 'fail', + detail: `the api.qa/vitest@1 run failed: ${outcome.reason}`, + } + } + + // ONE folded result set (A.8.6.5): rows in document order, tests in + // registration order. + const failedTests = outcome.results.filter((r: ExecTestResult) => r.status === 'fail') + const problems = [ + ...rowProblems, + ...failedTests.map((r: ExecTestResult) => `test "${r.name}": ${r.reason ?? 'failed'}`), + ] + if (problems.length > 0) { + return { + verdict: 'fail', + detail: + `the surface violated its OWN published executable suite (${cardGate.digest.slice(0, 19)}…, environment ` + + `"${cardGate.environment}", seed ${record.seed ?? bundle.seed}): ` + problems.slice(0, 6).join('; '), + } + } + return pass( + `interfaces.testSuite declared (runner ${VITEST_RUNNER}); ${cardGate.kind} artifact at ${cardGate.url} → ${docEv!.status}, ` + + `${cardGate.digest.slice(0, 19)}… matches the card pin (hash-then-instantiate); environment "${cardGate.environment}"` + + `${sandbox ? ' (sandbox: true — mutating verbs consented)' : ''}, seed ${record.seed ?? bundle.seed}; ` + + `${rowCount} declarative row(s) over ${pathnames.size} distinct pathname(s) + ${outcome.registered} registered test(s), all passed ` + + `(${outcome.elapsedWallMs} ms wall elapsed under applied limits ${outcome.appliedLimits.wallMs} ms wall / ${outcome.appliedLimits.cpuMs} ms CPU` + + `${outcome.consumedCpuMs !== null ? `, ${outcome.consumedCpuMs} ms CPU consumed` : ''}). ` + + (record.npm !== undefined + ? `npm identity assertion recorded: ${record.npm.package}@${record.npm.version} (provenance claim — the registry was never contacted and did not influence this verdict). ` + : '') + + 'Isolated run: zero ambient authority, network floor only (full external egress above it), digest fail-closed. ' + + 'NOT judged: whether the suite is ambitious — api.qa verifies the surface keeps its OWN published promise, not that the promise is demanding.', + ) +} + function pass(detail: string): { verdict: Verdict; detail: string } { return { verdict: 'pass', detail } } diff --git a/src/discovery.ts b/src/discovery.ts index e3e3a31..507322a 100644 --- a/src/discovery.ts +++ b/src/discovery.ts @@ -17,10 +17,23 @@ import { canonicalJson, sha256Hex, sampleSeeded } from './digest.js' import { MAX_SUITE_REQUIREMENTS, SUITE_DEADLINE_MS, + VITEST_RUNNER, gateTestSuiteCard, gateTestSuiteDocument, + gateVitestModuleArtifact, + gateVitestSuiteCard, + gateVitestSuiteDocument, } from './test-suite.js' -import { captureInto, judgeExpect, resolveEndpoint, type Bindings } from './expect.js' +import { + EXEC_MAX_COMBINED, + EXEC_MAX_DOC_BYTES, + EXEC_MAX_MODULE_BYTES, + EXEC_WALL_MS, + type ExecRunOutcome, + type ExecSuiteRunner, +} from './exec/dialect.js' +import { unavailableExecRunner } from './exec/runner.js' +import { captureInto, judgeExpect, resolveEndpoint, type Bindings, type EndpointReq } from './expect.js' import type { ClaimedEndpoint, DiscoveryReport, @@ -175,6 +188,16 @@ export const ROLE = { * and the published-test-suite check SKIPs. */ testSuite: 'surface:testsuite', + /** + * The `api.qa/vitest@1` EXECUTABLE RUN OUTCOME (A.8.6.5) — a SYNTHETIC + * evidence item, not an HTTP exchange: its body is the typed + * `ExecRunOutcome` JSON (executed digest, environment + sandbox, seed, + * applied breaker limits, elapsed wall/CPU, per-test results in + * registration order, npm coordinate where asserted). Recorded at observe + * time — execution is I/O, judging is pure — so the judge and any replay + * read the SAME recorded run, and the whole thing rides the signed bundle. + */ + vitestRun: 'suite:vitest-run', } as const /** @@ -425,8 +448,25 @@ export interface TestSuiteClaim { digest?: string /** Environment name to select. Defaults to `"public"`. */ environment: string - /** Suite dialect. Only `"api.qa/suite@1"` is defined. */ + /** Suite dialect: `"api.qa/suite@1"` (declarative) or `"api.qa/vitest@1"` (executable, A.8.6). */ runner: string + /** + * npm IDENTITY ASSERTION (A.8.6.6, `api.qa/vitest@1` only): the pinned + * artifact's bytes are asserted byte-identical to `package@version`'s entry + * module as published to npm. PROVENANCE, never a delivery channel — the + * registry is NEVER contacted, and registry state MUST NOT influence a + * verdict; the verdict records the assertion, the digest decides what runs. + * `package` without an exact `version` is a defective declaration. + */ + packageName?: string + /** Exact published semver of `packageName` — never a range. */ + version?: string + /** + * Named nullary (possibly async) module export the harness calls to + * register tests (module artifacts only); absent, tests register at module + * instantiation. + */ + exportName?: string } /** The only suite dialect this verifier implements. */ @@ -538,6 +578,15 @@ export function parseAgentsJson(doc: unknown, origin: string): AgentsClaims { const digest = str(t.digest) const environment = str(t.environment) const runner = str(t.runner) + // The A.8.6.6 widened seam: { url?, package?, version?, export?, digest, + // environment?, runner? }. `package`/`version`/`export` are read here as + // RAW strings; every rule about them (vitest@1-only, exact semver, at + // least one address, module-kind-only export) is judged by the gates in + // test-suite.ts, so a defective declaration FAILS rather than being + // silently dropped at parse. + const packageName = str(t.package) + const version = str(t.version) + const exportName = str(t.export) out.testSuite = { declared: true, ...(urlRaw !== undefined && { urlRaw }), @@ -548,6 +597,9 @@ export function parseAgentsJson(doc: unknown, origin: string): AgentsClaims { ...(digest !== undefined && { digest }), environment: environment ?? TEST_SUITE_DEFAULT_ENVIRONMENT, runner: runner ?? TEST_SUITE_RUNNER, + ...(packageName !== undefined && { packageName }), + ...(version !== undefined && { version }), + ...(exportName !== undefined && { exportName }), } } else { out.testSuite = { @@ -1445,13 +1497,27 @@ async function observeDigitalLink(origin: string, observer: Observer): Promise { +async function observeTestSuite( + origin: string, + observer: Observer, + seed: number, + execRunner: ExecSuiteRunner, +): Promise { const items = observer.items const bundleView: EvidenceBundle = { target: origin, fetchedAt: '', seed: 0, items } const agents = parseAgentsJson(parseJsonBody(findEvidence(bundleView, ROLE.agentsJson)), origin) const claim = agents.testSuite if (!claim) return // interface not declared — optional, nothing to verify + // Dialect dispatch — on the CARD's `runner`, exactly as A.8.5 discriminates. + // The executable dialect has its own observe path (different caps, the + // network-floor artifact gate instead of same-origin, and the isolate run); + // suite@1 below is UNCHANGED. + if (claim.runner === VITEST_RUNNER) { + await observeVitestSuite(origin, observer, claim, seed, execRunner) + return + } + // Card-only gate. A malformed declaration, a missing/bad digest, an unknown // runner or an off-origin url is judged from the card alone and never // fetched; the check reports it from the same shared gate. @@ -1464,20 +1530,39 @@ async function observeTestSuite(origin: string, observer: Observer): Promise, + deadlineMs: number, + budget: number, +): Promise { + const items = observer.items // A PRIVATE observer for the sub-run. Three tightenings over the parent: - // read-only (a stranger's document may not direct a write), its own small - // budget (a long suite cannot drain the parent's politeness budget out from + // read-only (a stranger's document may not direct a write), its own budget + // (a long suite cannot drain the parent's politeness budget out from // under the fixed probes), and `allowPrivate` inherited so a consented local // target still works in dev without ever loosening the deployed posture. - const sub = observer.child({ allowWrites: false, budget: MAX_SUITE_REQUIREMENTS }) + const sub = observer.child({ allowWrites: false, budget }) // The binding scope starts from the SELECTED ENVIRONMENT's vars, exactly as a // `verifySuite` run does. A `baseUrl` among them cannot steer the run: the // target is `origin`, and `resolveEndpoint` re-gates every resolved URL // same-origin, so an off-origin interpolation fails closed WITHOUT a fetch. - const bindings: Bindings = { ...plan.vars } - const deadline = Date.now() + SUITE_DEADLINE_MS - for (const req of plan.requirements) { + const bindings: Bindings = { ...vars } + const deadline = Date.now() + deadlineMs + for (const req of requirements) { if (Date.now() > deadline) break // the judge fails the run for the missing evidence const resolved = resolveEndpoint(req, origin, bindings) if (!resolved.ok) continue // judge re-derives the identical resolution failure @@ -1511,7 +1596,139 @@ async function observeTestSuite(origin: string, observer: Observer): Promise { +/** + * Observe the `api.qa/vitest@1` EXECUTABLE suite (A.8.6): fetch the pinned + * artifact ONCE into one buffer, hash-then-instantiate (digest fail-closed — + * nothing executes unless the bytes match the card pin), run the declarative + * rows under unchanged suite@1 engine semantics, hand the code to the + * `ExecSuiteRunner` seam (Worker Loader isolate hosted; the shared-harness + * local runner in the CLI; a typed `runner-unavailable` outcome where the + * capability is not provisioned), and record the WHOLE typed outcome as + * synthetic evidence so the pure judge — and any replay — reads the same run. + */ +async function observeVitestSuite( + origin: string, + observer: Observer, + claim: TestSuiteClaim, + seed: number, + execRunner: ExecSuiteRunner, +): Promise { + const items = observer.items + const gate = gateVitestSuiteCard(claim, origin) + if (!gate.ok) return // defective declaration — judged from the card alone, nothing fetched + + // ONE fetch, ONE buffer (A.8.6.3). A dedicated child raises the body cap to + // the artifact's own limit + 1 byte — an over-cap artifact is visibly over + // (the gate fails it by size), never silently truncated into a digest + // mismatch. Budget 1: this path costs exactly one request. + const cap = (gate.kind === 'module' ? EXEC_MAX_MODULE_BYTES : EXEC_MAX_DOC_BYTES) + 1 + const artifactObserver = observer.child({ budget: 1, maxBodyBytes: cap }) + const docEv = await artifactObserver.observe(ROLE.testSuite, gate.url, { + accept: gate.kind === 'module' ? 'text/javascript, application/javascript, */*' : 'application/json', + }) + if (items.some((e) => e.role === ROLE.testSuite)) { + throw new Error(`evidence role "${ROLE.testSuite}" already exists in the parent bundle — refusing to shadow it`) + } + items.push(docEv) + if (docEv.status === null || docEv.status < 200 || docEv.status >= 300 || docEv.body == null) return + + let outcome: ExecRunOutcome + let sandbox = false + let declarativeRows = 0 + + if (gate.kind === 'document') { + const plan = gateVitestSuiteDocument(claim, docEv.body, gate.digest) + if (!plan.ok) return // digest mismatch / defective document — judged purely, NOTHING instantiated + sandbox = plan.sandbox + declarativeRows = plan.rows.length + // Declarative rows: the identical suite@1 loop, under the dialect's own + // combined bound (the judge fails the count breach; the loop is bounded + // by the metered wall clock, not the 20 s scarcity deadline). + await runSuiteRows(origin, observer, plan.rows, plan.vars, EXEC_WALL_MS, Math.min(plan.rows.length || 1, EXEC_MAX_COMBINED)) + outcome = await execRunner.run( + { + artifactKind: 'document', + testsSource: plan.testsSource, + ...(plan.moduleSource !== undefined && { moduleSource: plan.moduleSource }), + origin, + vars: plan.vars, + environment: gate.environment, + sandbox: plan.sandbox, + seed, + declarativeRows, + digest: gate.digest, + }, + { fetch: observer.transportFetcher }, + ) + } else { + const mg = gateVitestModuleArtifact(claim, docEv.body, gate.digest) + if (!mg.ok) return // digest mismatch / over-cap — judged purely, NOTHING instantiated + outcome = await execRunner.run( + { + artifactKind: 'module', + testsSource: docEv.body, + ...(gate.exportName !== undefined && { exportName: gate.exportName }), + origin, + // A module artifact defines no environments: implicit "public", + // sandbox false — a mutating verb always fails there (A.8.6.4). + vars: {}, + environment: 'public', + sandbox: false, + seed, + declarativeRows: 0, + digest: gate.digest, + }, + { fetch: observer.transportFetcher }, + ) + } + + // The A.8.6.5 attestation record: the typed outcome plus everything a + // signed verdict must carry — recorded as SYNTHETIC evidence (execution is + // I/O; judging is pure over the bundle; the signed report carries the run). + const record = { + $type: 'VitestRunOutcome', + runner: VITEST_RUNNER, + executedDigest: gate.digest, + address: gate.url, + artifactKind: gate.kind, + environment: gate.environment, + sandbox, + seed, + declarativeRows, + ...(gate.npm !== undefined && { npm: gate.npm }), + outcome, + } + items.push({ + role: ROLE.vitestRun, + url: gate.url, + method: 'EXEC', + status: 200, + contentType: 'application/json', + headers: {}, + body: JSON.stringify(record), + elapsedMs: outcome.status === 'ran' ? outcome.elapsedWallMs : 0, + }) +} + +export interface ObserveTargetOpts { + /** + * The `api.qa/vitest@1` execution seam. The deployed Worker wires the + * Worker Loader runner when the binding is provisioned; the CLI wires the + * shared-harness local runner; DEFAULT is the typed `runner-unavailable` + * runner — a card that declares the executable dialect against a verifier + * without the capability FAILS with the reason named (the same direction + * the ratified unknown-runner rule gives an older verifier), never a crash + * and never a silent pass. + */ + execRunner?: ExecSuiteRunner +} + +export async function observeTarget( + origin: string, + observer: Observer, + seed: number, + opts: ObserveTargetOpts = {}, +): Promise { // 1. The fixed surface plan — identical for every target (no fingerprint). const rootAgentEv = await observer.observe(ROLE.rootAgent, `${origin}/`, { accept: '*/*' }) const rootBrowserEv = await observer.observe(ROLE.rootBrowser, `${origin}/`, { @@ -1804,7 +2021,7 @@ export async function observeTarget(origin: string, observer: Observer, seed: nu // fully conforming. Ordered with the other fixed high-value probes, // BEFORE the unbounded contract-diff enumeration, and on its OWN observer // so a long suite can never starve them. - await observeTestSuite(origin, observer) + await observeTestSuite(origin, observer, seed, opts.execRunner ?? unavailableExecRunner()) // 5. Contract-diff probing (ax-e6b.28.4): for a FULL OpenAPI<->live diff, // fetch EVERY GET-safe candidate path once — not just the seeded keyless diff --git a/src/exec/dialect.ts b/src/exec/dialect.ts new file mode 100644 index 0000000..d6361c5 --- /dev/null +++ b/src/exec/dialect.ts @@ -0,0 +1,622 @@ +/** + * The `api.qa/vitest@1` DIALECT LAYER — everything that surrounds the shared + * subset harness (src/exec/vitest-subset.mjs) when a pinned suite's code + * actually runs: source validation, the network floor, the gated fetch, and + * the LOCAL instantiation path. + * + * AXP A.8.6 (apis-ax-axp@2.4.0, spec digest dd3e5941…) is the contract: + * + * - IMPORTS ARE CLOSED (A.8.6.2) to exactly `"vitest"` (the shared subset), + * `"suite:env"` ({origin, vars, seed, sandbox}), and `"suite:module"` + * (document form with a `module` member only). Everything else — `node:` + * built-ins, bare package specifiers, dynamic `import()`, `eval`, + * `new Function` — MUST fail by a NAMED reason, identically under the + * hosted runner and the local CLI. `validateDialectSource` is that shared + * refusal: one function, both hosts, so the two cannot diverge. + * + * - THE NETWORK FLOOR (A.8.6.3) refuses cloud-metadata, link-local, + * loopback, RFC 1918, CGNAT, ULA, and estate-internal destinations — + * re-checked per redirect hop — and permits EVERY other publicly-routable + * destination. Full external egress is a FEATURE of the dialect + * (cross-estate composition); the rejected graded-origin-only draft is + * exactly what this module must not reintroduce. + * + * - MUTATING VERBS are environment-gated (A.8.6.4): permitted only when the + * selected environment declares `"sandbox": true` in the pinned document; + * a module artifact carries only the implicit non-sandbox `"public"`. + * + * - SEEDED DETERMINISM (A.8.6.4): `Math.random` is replaced by a generator + * derived from the run seed, and the same seed is exported by + * `"suite:env"`. + * + * - FAIL-CLOSED TOTALITY (A.8.6.3): a floor refusal, a verb refusal, a + * breaker trip, or a cap breach fails the RUN by a named reason — even if + * the suite caught the thrown refusal — never a pass over the remainder. + * + * The local instantiation executes THE SAME BYTES the hosted isolate would: + * the harness from `VITEST_SUBSET_SOURCE`, the suite from the digest-matched + * buffer, both loaded as real ES modules via `data:` imports with the closed + * specifiers rewritten to generated shim modules. There is no transpile and + * no reimplementation — local==hosted by construction. + */ + +import { isPrivateHost } from '../http.js' +import { VITEST_SUBSET_SOURCE } from './vitest-subset-source.js' + +/** The executable suite dialect (A.8.6). */ +export const VITEST_RUNNER = 'api.qa/vitest@1' + +// --------------------------------------------------------------------------- +// The abuse circuit-breakers (A.8.6.1 / A.8.6.3) — NEVER rations. The paid +// tier is the gate on the capability; each number is sized so no legitimate +// suite meets it, and what bounds a legitimate run is the billed breaker. +// --------------------------------------------------------------------------- + +/** Metered circuit-breaker defaults: 5 minutes wall, 1 minute CPU (account-raisable). */ +export const EXEC_WALL_MS = 300_000 +export const EXEC_CPU_MS = 60_000 +/** Declarative rows + registered tests COMBINED (A.8.6.1). */ +export const EXEC_MAX_COMBINED = 1000 +/** The served suite DOCUMENT (A.8.6.1). */ +export const EXEC_MAX_DOC_BYTES = 1_048_576 +/** A pinned MODULE artifact (an SDK entry is a bundle) (A.8.6.3). */ +export const EXEC_MAX_MODULE_BYTES = 4_194_304 +/** Captured output — results, messages, logs (A.8.6.3). */ +export const EXEC_MAX_OUTPUT_BYTES = 4_194_304 +/** Redirect hops the gated fetch will follow, each re-floored. */ +export const EXEC_MAX_REDIRECTS = 5 + +// --------------------------------------------------------------------------- +// Source validation — the closed import surface, shared by both hosts +// --------------------------------------------------------------------------- + +/** Static import/re-export specifier scan: `from ''` and `import ''`. */ +const SPECIFIER_RE = /(\bfrom\s*|\bimport\s*)(["'])([^"']*)\2/g + +const FORBIDDEN_CONSTRUCTS: Array<[RegExp, string]> = [ + // Dynamic import() resolves at run time, outside the pinned module map. + [/\bimport\s*\(/, 'dynamic import()'], + // Runtime code generation: the only code that runs is code the digest covers. + [/\beval\s*\(/, 'eval'], + [/\bnew\s+Function\b/, 'new Function'], +] + +export type SourceValidation = { ok: true; specifiers: string[] } | { ok: false; problem: string } + +/** + * Validate a dialect module source against the closed surface of A.8.6.2. + * PURELY textual and deliberately conservative: a forbidden token inside a + * string literal is refused too — the refusal is identical under both hosts + * (this one function), and the fail direction is closed, never open. + * + * `allowSuiteModule` — `"suite:module"` exists only in the document form when + * the document carries a `module` member; anywhere else the specifier is + * refused by name. + */ +export function validateDialectSource( + source: string, + opts: { allowSuiteModule: boolean; what: string }, +): SourceValidation { + for (const [re, name] of FORBIDDEN_CONSTRUCTS) { + if (re.test(source)) { + return { + ok: false, + problem: + `${opts.what} uses ${name}, which is outside the api.qa/vitest@1 subset — the isolate disables ` + + 'runtime code generation; the only code that runs is code the digest covers (A.8.6.2)', + } + } + } + const allowed = new Set(['vitest', 'suite:env', ...(opts.allowSuiteModule ? ['suite:module'] : [])]) + const specifiers: string[] = [] + for (const m of source.matchAll(SPECIFIER_RE)) { + const spec = m[3]! + specifiers.push(spec) + if (!allowed.has(spec)) { + const why = spec.startsWith('node:') + ? 'node: built-ins do not exist in the isolate' + : spec === 'suite:module' + ? 'the "suite:module" specifier exists only in a document that carries a `module` member' + : /^[./]/.test(spec) + ? 'relative/path imports cannot resolve inside a single pinned artifact' + : 'bare package specifiers cannot resolve inside the isolate — bundle your dependencies into the pinned artifact' + return { + ok: false, + problem: + `${opts.what} imports ${JSON.stringify(spec)}, which is outside the api.qa/vitest@1 subset ` + + `(${why}). Imports are closed to exactly "vitest", "suite:env"${opts.allowSuiteModule ? ', "suite:module"' : ''} (A.8.6.2).`, + } + } + } + return { ok: true, specifiers } +} + +// --------------------------------------------------------------------------- +// The network floor (A.8.6.3) — and NOTHING above it +// --------------------------------------------------------------------------- + +/** + * True when the floor bars this hostname. Reuses `isPrivateHost` — the same + * refusal set the whole verifier stands on (metadata 169.254.169.254 incl. + * decimal/hex/octal encodings, link-local v4/v6, loopback, RFC 1918, ULA, + * `*.internal` / `*.local` / `localhost`, and EVERY raw IP-literal host, + * v4 or bracketed v6 — a public service is reached by NAME, and a literal is + * exactly the shape every encoding bypass arrives in; over-broad only in the + * closed direction) — plus + * the floor's own additions: CGNAT 100.64.0.0/10 and every single-label + * hostname (estate-internal service names are single-label; no public DNS + * name is). + * + * Everything else is PERMITTED: the floor is the ONLY network restriction on + * executable tests. No same-origin scoping — that drafted posture is recorded + * in-spec as REJECTED. + */ +export function isFloorBlockedHost(hostname: string): boolean { + const h = hostname.toLowerCase().replace(/\.$/, '') + if (h.length === 0) return true + if (isPrivateHost(h)) return true + // CGNAT 100.64.0.0/10 (RFC 6598) — not in the verifier's private set, barred + // by the floor explicitly. + const cgnat = /^100\.(\d{1,3})\./.exec(h) + if (cgnat !== null) { + const second = Number(cgnat[1]) + if (second >= 64 && second <= 127) return true + } + // A single-label hostname is never a public DNS name; it is exactly the + // shape of an estate-internal service hostname. + if (!h.includes('.')) return true + return false +} + +/** A floor/verb refusal recorded at the fetch boundary. Fails the RUN. */ +export interface GateViolation { + url: string + reason: string +} + +export type GatedFetch = (input: string | URL | Request, init?: RequestInit) => Promise + +/** + * Wrap a real fetch in the A.8.6.3 floor + the A.8.6.4 verb gate. + * + * Every request — and every redirect hop, followed MANUALLY so a hop can + * never slip under the floor — is checked against `isFloorBlockedHost`. + * A refusal is recorded in `violations` AND thrown: the throw fails the + * calling test immediately, and the recording fails the RUN even if the suite + * caught the throw (fail-closed totality — a suite cannot swallow a floor + * refusal into a pass). + * + * Mutating verbs are permitted only when `sandbox` is true — the suite's own + * pinned consent (A.8.6.4). A mutating request is never redirect-followed. + */ +export function createGatedFetch(opts: { + realFetch: (url: string, init?: RequestInit) => Promise + sandbox: boolean + violations: GateViolation[] +}): GatedFetch { + return async (input, init) => { + const url = + typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url + const method = ( + init?.method ?? (typeof input === 'object' && 'method' in input ? input.method : 'GET') + ).toUpperCase() + + const refuse = (reason: string): never => { + opts.violations.push({ url, reason }) + throw new Error(reason) + } + + if (method !== 'GET' && method !== 'HEAD' && !opts.sandbox) { + refuse( + `${method} ${url} refused: mutating verbs are permitted only against an environment the pinned ` + + 'suite itself declares "sandbox": true — this run\'s environment does not (A.8.6.4)', + ) + } + + let current = url + for (let hop = 0; ; hop++) { + let host: string + try { + host = new URL(current).hostname + } catch { + refuse(`fetch refused: ${JSON.stringify(current)} is not a parseable absolute URL`) + throw new Error('unreachable') + } + if (isFloorBlockedHost(host)) { + refuse( + `fetch toward ${current} refused by the network floor: metadata, link-local, loopback, ` + + 'private-range, CGNAT, ULA and estate-internal destinations are barred (A.8.6.3). ' + + 'Every other publicly-routable destination is permitted.', + ) + } + const res = await opts.realFetch(current, { + method, + headers: init?.headers as Record | undefined, + body: hop === 0 ? (init?.body as BodyInit | undefined) : undefined, + redirect: 'manual', + }) + const location = res.status >= 300 && res.status < 400 ? res.headers.get('location') : null + if (location === null) return res + // A mutating redirect is returned verbatim: following it would re-issue + // the write somewhere the suite never named. + if (method !== 'GET' && method !== 'HEAD') return res + if (hop >= EXEC_MAX_REDIRECTS) { + throw new Error(`too many redirects (> ${EXEC_MAX_REDIRECTS}) from ${url}`) + } + try { + current = new URL(location, current).toString() + } catch { + refuse(`fetch refused: unparseable redirect Location from ${current}`) + } + // Loop: the next hop's host is re-floored before any byte is fetched. + } + } +} + +// --------------------------------------------------------------------------- +// Local instantiation — the same bytes the hosted isolate executes +// --------------------------------------------------------------------------- + +/** One executed test's outcome, in registration order (A.8.6.5). */ +export interface ExecTestResult { + name: string + status: 'pass' | 'fail' + durationMs: number + reason?: string +} + +/** What the runner is asked to execute — already digest-matched by the caller. */ +export interface ExecRunRequest { + /** Discriminated by the CARD (A.8.5), never by sniffing. */ + artifactKind: 'document' | 'module' + /** The tests source: the document's `tests` member, or the module bytes. */ + testsSource: string + /** The document's optional `module` member (importable as "suite:module"). */ + moduleSource?: string + /** Module artifact: named nullary export the harness calls to register. */ + exportName?: string + /** The graded origin, exported by "suite:env". */ + origin: string + /** Selected environment's vars (module artifact: {}). */ + vars: Record + environment: string + sandbox: boolean + seed: number + /** Declarative row count, for the COMBINED cap (A.8.6.1). */ + declarativeRows: number + /** + * The card pin the executed bytes matched (`sha256:<64 hex>`). The caller + * verified it BEFORE building this request (hash-then-instantiate — there + * is no execute-then-check ordering); the hosted runner folds it into the + * content-hash isolate id. + */ + digest?: string + limits?: { wallMs?: number; cpuMs?: number } +} + +/** The typed outcome the judge reads. NEVER a thrown crash, NEVER a silent pass. */ +export type ExecRunOutcome = + | { + status: 'ran' + registered: number + results: ExecTestResult[] + appliedLimits: { wallMs: number; cpuMs: number } + elapsedWallMs: number + /** Milliseconds of CPU actually consumed; null where the host cannot meter it. */ + consumedCpuMs: number | null + } + | { + /** The execution capability is not provisioned on this deployment. */ + status: 'runner-unavailable' + reason: string + } + | { + /** Refused before instantiation (subset violation, cap breach) or failed + * as a whole run (floor/verb refusal, breaker trip, uncaught error, + * vacuity, combined cap). Always a NAMED reason. */ + status: 'failed' + reason: string + } + +/** Transport handed to a runner by its caller (the observe side). */ +export interface ExecRunIo { + /** + * The fetch the run's egress rides — the observe side passes the SAME + * transport the evidence bundle was recorded over, so an in-memory fixture + * target and the deployed Worker's real fetch apply to the suite run + * identically. The HOSTED runner ignores it (the isolate's egress is the + * parent-owned gateway); the LOCAL runner wraps it in the gated fetch. + */ + fetch?: (url: string, init?: RequestInit) => Promise +} + +/** The runner seam: hosted (Worker Loader) and local share this interface. */ +export interface ExecSuiteRunner { + run(req: ExecRunRequest, io?: ExecRunIo): Promise +} + +interface RunRegistryEntry { + api: Record + moduleNs?: Record +} + +/** Cross-module handoff for the generated `data:` shims (local path only). */ +const RUN_REGISTRY_KEY = '__APIQA_VITEST_RUNS__' + +function registry(): Record { + const g = globalThis as Record + if (g[RUN_REGISTRY_KEY] === undefined) g[RUN_REGISTRY_KEY] = {} + return g[RUN_REGISTRY_KEY] as Record +} + +/** `data:` module URL for a source (utf-8, no base64 — unicode-safe). */ +function dataModuleUrl(source: string): string { + return `data:text/javascript;charset=utf-8,${encodeURIComponent(source)}` +} + +/** Rewrite the closed specifiers to concrete module URLs (local path). */ +function rewriteSpecifiers(source: string, map: Record): string { + return source.replace(SPECIFIER_RE, (whole, lead: string, quote: string, spec: string) => { + const target = map[spec] + return target === undefined ? whole : `${lead}${quote}${target}${quote}` + }) +} + +const IDENTIFIER_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/ + +/** + * The generated "vitest" shim — re-exports the ONE harness instance out of the + * run registry. Orchestration glue, not a reimplementation: every exported + * name is the shared harness's own member. Used verbatim by BOTH the local + * `data:` module graph and the hosted isolate's module map. + */ +export function vitestShimSource(runId: string): string { + return ( + `const h = globalThis[${JSON.stringify(RUN_REGISTRY_KEY)}][${JSON.stringify(runId)}].api\n` + + `export const describe = h.describe\nexport const it = h.it\nexport const test = h.test\n` + + `export const expect = h.expect\nexport const vi = h.vi\nexport default h\n` + ) +} + +/** The generated "suite:env" module — pure data (A.8.6.2). Shared by both hosts. */ +export function suiteEnvSource(req: ExecRunRequest): string { + return ( + `export const origin = ${JSON.stringify(req.origin)}\n` + + `export const vars = ${JSON.stringify(req.vars)}\n` + + `export const seed = ${JSON.stringify(req.seed)}\n` + + `export const sandbox = ${JSON.stringify(req.sandbox)}\n` + ) +} + +/** The generated "suite:module" module — re-exports the instantiated `module` member's namespace. */ +function suiteModuleShimSource(runId: string, exportNames: string[]): string { + const head = `const ns = globalThis[${JSON.stringify(RUN_REGISTRY_KEY)}][${JSON.stringify(runId)}].moduleNs\n` + const lines = exportNames + .filter((n) => n !== 'default' && IDENTIFIER_RE.test(n)) + .map((n) => `export const ${n} = ns[${JSON.stringify(n)}]\n`) + const dflt = exportNames.includes('default') ? 'export default ns.default\n' : '' + return head + lines.join('') + dflt +} + +interface HarnessModule { + createHarness: () => HarnessInstance + seededRandom: (seed: number) => () => number +} + +let harnessModulePromise: Promise | undefined + +interface HarnessInstance { + api: Record & { describe: unknown; it: unknown; test: unknown; expect: unknown; vi: unknown } + run: () => Promise<{ registered: number; results: ExecTestResult[] }> + readonly registered: number +} + +/** + * Import the shared harness FROM ITS CANONICAL BYTES (`VITEST_SUBSET_SOURCE`) + * — not from a locally compiled sibling — so the local runner executes the + * byte-identical module the hosted isolate's module map carries. Memoized: + * `createHarness` is per-run state, the module itself is pure. + */ +export function loadHarnessModule(): Promise { + harnessModulePromise ??= import( + /* @vite-ignore */ dataModuleUrl(VITEST_SUBSET_SOURCE) + ) as Promise + return harnessModulePromise +} + +/** + * The FAIL-CLOSED TOTALITY fold (A.8.6.3), shared by the local runner and the + * hosted Worker Loader runner so the two judge a raw run identically: a floor + * or verb refusal fails the run even when the suite caught the throw; the + * combined count cap and the output cap fail, never truncate; an all-of- + * nothing is refused (non-vacuity over the union, A.8.6.1). + */ +export function foldRunOutcome( + raw: { registered: number; results: ExecTestResult[]; violations: GateViolation[] }, + req: ExecRunRequest, + appliedLimits: { wallMs: number; cpuMs: number }, + elapsedWallMs: number, + consumedCpuMs: number | null, +): ExecRunOutcome { + if (raw.violations.length > 0) { + return { status: 'failed', reason: raw.violations[0]!.reason } + } + const combined = raw.registered + req.declarativeRows + if (combined > EXEC_MAX_COMBINED) { + return { + status: 'failed', + reason: + `${req.declarativeRows} declarative row(s) + ${raw.registered} registered test(s) = ${combined}, over the ` + + `${EXEC_MAX_COMBINED} combined abuse circuit-breaker (A.8.6.1) — failed, never truncated`, + } + } + if (combined === 0) { + return { + status: 'failed', + reason: + 'the suite registered no tests and declares no declarative rows — an all-of-nothing is refused ' + + '(non-vacuity over the union, A.8.6.1)', + } + } + const outputBytes = new TextEncoder().encode(JSON.stringify(raw.results)).byteLength + if (outputBytes > EXEC_MAX_OUTPUT_BYTES) { + return { + status: 'failed', + reason: `captured output is ${outputBytes} bytes, over the ${EXEC_MAX_OUTPUT_BYTES}-byte cap (A.8.6.3) — failed, never truncated`, + } + } + return { + status: 'ran', + registered: raw.registered, + results: raw.results, + appliedLimits, + elapsedWallMs, + consumedCpuMs, + } +} + +let localRunCounter = 0 + +/** + * The LOCAL `api.qa/vitest@1` runner — the CLI verb and the tests run through + * this; it is also the parity reference for the hosted isolate. Same shared + * harness bytes, same `validateDialectSource`, same `createGatedFetch`, same + * seeded `Math.random`, same sequential run — "green locally, red hosted" is + * expressible only as a difference in the TARGET's behavior (A.8.6.2). + * + * `realFetch` is injectable so an in-memory fixture target works; it defaults + * to global fetch. During instantiation AND the run, `globalThis.fetch` and + * `Math.random` are swapped (gated fetch; seeded PRNG) and — in the document + * form — the subset names are installed as globals (A.8.6.2); everything is + * restored in a finally. + */ +export function localExecRunner(opts: { fetch?: (url: string, init?: RequestInit) => Promise } = {}): ExecSuiteRunner { + return { + async run(req: ExecRunRequest, io: ExecRunIo = {}): Promise { + const wallMs = req.limits?.wallMs ?? EXEC_WALL_MS + const cpuMs = req.limits?.cpuMs ?? EXEC_CPU_MS + const appliedLimits = { wallMs, cpuMs } + + // ── Subset validation, before a single expression evaluates ────────── + const hasModule = req.artifactKind === 'document' && typeof req.moduleSource === 'string' + if (req.artifactKind === 'document' && req.moduleSource !== undefined) { + const v = validateDialectSource(req.moduleSource, { + allowSuiteModule: false, + what: 'the suite document `module` member', + }) + if (!v.ok) return { status: 'failed', reason: v.problem } + } + const testsValidation = validateDialectSource(req.testsSource, { + allowSuiteModule: hasModule, + what: req.artifactKind === 'document' ? 'the suite document `tests` member' : 'the pinned module artifact', + }) + if (!testsValidation.ok) return { status: 'failed', reason: testsValidation.problem } + + const runId = `local:${++localRunCounter}:${req.seed}` + const violations: GateViolation[] = [] + const gatedFetch = createGatedFetch({ + realFetch: io.fetch ?? opts.fetch ?? ((url, init) => fetch(url, init)), + sandbox: req.sandbox, + violations, + }) + + const g = globalThis as Record + const saved: Record = {} + const installedGlobals: string[] = [] + const savedFetch = g.fetch + const savedRandom = Math.random + + const started = Date.now() + try { + const { createHarness, seededRandom } = await loadHarnessModule() + const harness = createHarness() + registry()[runId] = { api: harness.api } + + // The run's ambient surface: gated fetch, seeded randomness (from the + // SHARED harness bytes — the same generator the hosted isolate seeds), + // and (in the document form) the subset globals. + g.fetch = gatedFetch + Math.random = seededRandom(req.seed) + if (req.artifactKind === 'document') { + for (const name of ['describe', 'it', 'test', 'expect', 'vi']) { + saved[name] = g[name] + g[name] = harness.api[name] + installedGlobals.push(name) + } + } + + const specMap: Record = { + vitest: dataModuleUrl(vitestShimSource(runId)), + 'suite:env': dataModuleUrl(suiteEnvSource(req)), + } + + const importFresh = (source: string, map: Record, tag: string) => + // The trailing comment makes every run's URL unique: ES module + // instances are cached BY URL, and a re-run of the same bytes must + // re-register its tests, not reuse a spent registration. + import(/* @vite-ignore */ dataModuleUrl(`${rewriteSpecifiers(source, map)}\n//# ${runId}:${tag}`)) + + const execute = async (): Promise<{ registered: number; results: ExecTestResult[] }> => { + if (hasModule) { + const moduleNs = (await importFresh(req.moduleSource!, specMap, 'module')) as Record + registry()[runId]!.moduleNs = moduleNs + specMap['suite:module'] = dataModuleUrl(suiteModuleShimSource(runId, Object.keys(moduleNs))) + } + const testsNs = (await importFresh(req.testsSource, specMap, 'tests')) as Record + if (req.exportName !== undefined) { + const fn = testsNs[req.exportName] + if (typeof fn !== 'function') { + throw new Error( + `the card names export ${JSON.stringify(req.exportName)}, but the pinned module has no such function export`, + ) + } + await (fn as () => unknown | Promise)() + } + return harness.run() + } + + // ── The metered circuit-breaker (wall clock; local CPU is unmetered). + // A trip fails the WHOLE run — never a partial verdict (A.8.6.3). + let timer: ReturnType | undefined + const breaker = new Promise<'breaker'>((resolve) => { + timer = setTimeout(() => resolve('breaker'), wallMs) + }) + const raced = await Promise.race([execute(), breaker]).finally(() => clearTimeout(timer)) + if (raced === 'breaker') { + return { + status: 'failed', + reason: + `circuit breaker tripped: the run exceeded ${wallMs} ms wall-clock (applied limits ` + + `${wallMs} ms wall / ${cpuMs} ms CPU) — a tripped breaker fails the run, never a partial verdict (A.8.6.3)`, + } + } + const { registered, results } = raced + + // Fail-closed totality — the SAME fold the hosted runner applies. + return foldRunOutcome( + { registered, results, violations }, + req, + appliedLimits, + Date.now() - started, + null, // local CPU is unmetered; the hosted runner records consumed CPU + ) + } catch (err) { + // Instantiation-time throw (syntax error, top-level throw, export + // registration throw, a floor refusal at module top level). + const floored = violations[0] + return { + status: 'failed', + reason: floored !== undefined ? floored.reason : `the suite failed to instantiate or register: ${err instanceof Error ? err.message : String(err)}`, + } + } finally { + g.fetch = savedFetch + Math.random = savedRandom + for (const name of installedGlobals) { + if (saved[name] === undefined) delete g[name] + else g[name] = saved[name] + } + delete registry()[runId] + } + }, + } +} diff --git a/src/exec/runner.ts b/src/exec/runner.ts new file mode 100644 index 0000000..1dead9f --- /dev/null +++ b/src/exec/runner.ts @@ -0,0 +1,344 @@ +/** + * The HOSTED `api.qa/vitest@1` runner — a Cloudflare Worker Loader isolate + * per run — plus the typed "the capability is not provisioned here" runner. + * + * A.8.6.3's execution invariants, mapped onto the Worker Loader primitive: + * + * FRESH, SINGLE-USE ISOLATE — the loader id is derived from (artifact + * digest, harness version): same bytes MAY warm-reuse a compiled isolate as + * a pure optimization (Cloudflare's own recommendation), different bytes + * get a new isolate by construction, and NO run state survives — the run's + * entire result crosses the boundary once, as the entry's Response body. + * + * ZERO AMBIENT AUTHORITY — `env` is the EMPTY OBJECT, always. No + * SIGNING_KEY, no KV, no DO namespaces, no loader binding (no recursion), + * no vars. The run's inputs (origin, vars, seed, sandbox) travel as the + * generated `"suite:env"` MODULE — data compiled into the module map, not a + * binding. `buildWorkerCode` is exported so a unit test pins `env: {}` + * forever. + * + * THE NETWORK FLOOR — `globalOutbound` is REQUIRED: every fetch the isolate + * makes is delivered to a parent-owned gateway that applies + * `isFloorBlockedHost` per request and per redirect hop (`gatewayFetch`, + * the same floor the local runner's gated fetch applies). Leaving + * `globalOutbound` unspecified would inherit the verifier's OWN network + * access — the one catastrophic misconfiguration — so a runner constructed + * without an outbound gateway REFUSES to execute (typed + * `runner-unavailable`), it never runs open. + * + * METERED BREAKER — `limits.cpuMs` in the isolate (throws on breach), the + * wall-clock race in the parent; both recorded in the outcome. Trip ⇒ the + * whole run fails, never a partial verdict. + * + * FEATURE DETECTION. The Worker Loader binding is an open-beta, paid-plan + * capability. `wrangler.jsonc` documents (but does not enable) the binding so + * every account keeps valid deploys; `worker.ts` wires this runner ONLY when + * `env.SUITE_LOADER` exists. Anywhere the binding — or the outbound gateway — + * is absent, the runner is `unavailableExecRunner(...)`: a card that declares + * `runner: "api.qa/vitest@1"` then FAILS with the reason named (the same + * direction the ratified unknown-runner rule already gives an older + * verifier), never a crash, never a silent pass. + */ + +import { + EXEC_CPU_MS, + EXEC_WALL_MS, + createGatedFetch, + foldRunOutcome, + suiteEnvSource, + vitestShimSource, + type ExecRunOutcome, + type ExecRunRequest, + type ExecSuiteRunner, + type ExecTestResult, + type GateViolation, + validateDialectSource, +} from './dialect.js' +import { VITEST_SUBSET_SOURCE } from './vitest-subset-source.js' + +/** + * The harness revision folded into the isolate id: bump when the subset + * harness bytes change semantics, so a warm isolate can never run a suite + * against a stale harness. + */ +export const HARNESS_VERSION = 'vitest-subset@1' + +/** The pinned runtime the isolate runs under — api.qa's constant, never the card's. */ +export const EXEC_COMPATIBILITY_DATE = '2026-07-01' + +/** The registry key the hosted module map uses (one run per isolate invocation). */ +const HOSTED_RUN_ID = 'hosted' + +// --------------------------------------------------------------------------- +// Structural types for the (beta) Worker Loader binding — kept local so the +// repo compiles without @cloudflare/workers-types for a beta surface. +// --------------------------------------------------------------------------- + +export interface WorkerCodeLike { + compatibilityDate: string + mainModule: string + modules: Record + env: Record + globalOutbound: unknown + limits: { cpuMs: number } +} + +export interface WorkerStubLike { + getEntrypoint(): { fetch(url: string, init?: RequestInit): Promise } +} + +export interface WorkerLoaderLike { + get(id: string, getCode: () => Promise | WorkerCodeLike): WorkerStubLike +} + +// --------------------------------------------------------------------------- +// The typed absent-capability runner +// --------------------------------------------------------------------------- + +export const RUNNER_UNAVAILABLE_NO_BINDING = + 'runner-unavailable: the api.qa/vitest@1 execution capability is not provisioned on this deployment ' + + '(no Worker Loader binding). The card declares an executable suite this verifier cannot run, which ' + + 'fails by the same rule an unknown runner does — never a crash, never a silent pass.' + +export const RUNNER_UNAVAILABLE_NO_OUTBOUND = + 'runner-unavailable: the Worker Loader binding is present but no outbound egress gateway is bound — ' + + 'running would inherit the verifier\'s own network access, which the A.8.6.3 floor forbids. Refusing to run open.' + +/** A runner that always reports the capability absent, by a fixed named reason. */ +export function unavailableExecRunner(reason: string = RUNNER_UNAVAILABLE_NO_BINDING): ExecSuiteRunner { + return { + run(): Promise { + return Promise.resolve({ status: 'runner-unavailable', reason }) + }, + } +} + +// --------------------------------------------------------------------------- +// The parent-side egress gateway — the floor, applied where the parent owns it +// --------------------------------------------------------------------------- + +/** + * Handle ONE outbound request delivered by the isolate's `globalOutbound`. + * Reuses the SAME gated fetch the local runner uses (`createGatedFetch`), so + * the floor and the manual per-hop redirect re-check cannot drift between + * hosts. The eventual gateway entrypoint (deploy-time, beta account) is a + * thin wrapper over this function; the floor logic lives HERE, unit-tested. + */ +export async function gatewayFetch( + request: Request, + realFetch: (url: string, init?: RequestInit) => Promise = (url, init) => fetch(url, init), + opts: { sandbox?: boolean } = {}, +): Promise { + const violations: GateViolation[] = [] + const gated = createGatedFetch({ realFetch, sandbox: opts.sandbox ?? true, violations }) + try { + const headers: Record = {} + request.headers.forEach((v, k) => { + headers[k] = v + }) + return await gated(request.url, { + method: request.method, + headers, + body: request.method === 'GET' || request.method === 'HEAD' ? undefined : await request.text(), + }) + } catch (err) { + // The isolate sees an ordinary failed fetch with the refusal named; the + // verb/floor policy ALSO fails the run via the in-isolate wrapper's + // violation record, so a caught throw cannot become a pass. + return new Response( + JSON.stringify({ type: 'BLOCKED', reason: err instanceof Error ? err.message : String(err) }), + { status: 403, headers: { 'content-type': 'application/json' } }, + ) + } +} + +// --------------------------------------------------------------------------- +// WorkerCode assembly +// --------------------------------------------------------------------------- + +/** + * The generated isolate ENTRY — orchestration only: create the one shared + * harness, seed `Math.random` from the harness's own generator, wrap the + * (already gateway-brokered) global fetch with the A.8.6.4 verb gate, + * install the document-form globals, instantiate the pinned module(s), run, + * and return the raw results as the Response body. The verdict is computed by + * the PARENT (`foldRunOutcome`) — the isolate returns events, never "passed". + */ +export function entrySource(req: ExecRunRequest): string { + const hasModule = req.artifactKind === 'document' && typeof req.moduleSource === 'string' + return `import { createHarness, seededRandom, SUBSET_GLOBALS } from './harness.mjs' + +const SEED = ${JSON.stringify(req.seed)} +const SANDBOX = ${JSON.stringify(req.sandbox)} +const DOCUMENT = ${JSON.stringify(req.artifactKind === 'document')} +const HAS_MODULE = ${JSON.stringify(hasModule)} +const EXPORT_NAME = ${JSON.stringify(req.exportName ?? null)} + +export default { + async fetch() { + const violations = [] + const harness = createHarness() + globalThis[${JSON.stringify('__APIQA_VITEST_RUNS__')}] = { [${JSON.stringify(HOSTED_RUN_ID)}]: { api: harness.api } } + Math.random = seededRandom(SEED) + const realFetch = globalThis.fetch.bind(globalThis) + globalThis.fetch = async (input, init) => { + const url = typeof input === 'string' ? input : String(input && input.url ? input.url : input) + const method = ((init && init.method) || (input && input.method) || 'GET').toUpperCase() + if (method !== 'GET' && method !== 'HEAD' && !SANDBOX) { + const reason = method + ' ' + url + ' refused: mutating verbs are permitted only against an environment the pinned suite itself declares "sandbox": true (A.8.6.4)' + violations.push({ url, reason }) + throw new Error(reason) + } + return realFetch(input, init) + } + try { + if (DOCUMENT) for (const n of SUBSET_GLOBALS) globalThis[n] = harness.api[n] + if (HAS_MODULE) await import('./suite-module-impl.mjs') + const ns = await import('./suite-tests.mjs') + if (EXPORT_NAME !== null) { + const fn = ns[EXPORT_NAME] + if (typeof fn !== 'function') throw new Error('the card names export "' + EXPORT_NAME + '", but the pinned module has no such function export') + await fn() + } + const { registered, results } = await harness.run() + return Response.json({ registered, results, violations }) + } catch (err) { + return Response.json({ error: err instanceof Error ? err.message : String(err), violations }) + } + } +} +` +} + +/** + * Assemble the isolate's module map + posture. Exported so tests pin the + * invariants without a loader: `env` is EMPTY, `globalOutbound` is set, the + * harness bytes are `VITEST_SUBSET_SOURCE` verbatim, the compat date is the + * verifier's pinned constant. + */ +export function buildWorkerCode(req: ExecRunRequest, outbound: unknown): WorkerCodeLike { + const hasModule = req.artifactKind === 'document' && typeof req.moduleSource === 'string' + const modules: Record = { + 'entry.mjs': { js: entrySource(req) }, + './harness.mjs': { js: VITEST_SUBSET_SOURCE }, + // The closed specifiers resolve NATIVELY in the isolate's module map — no + // rewriting of the pinned bytes, ever. + vitest: { js: vitestShimSource(HOSTED_RUN_ID) }, + 'suite:env': { js: suiteEnvSource(req) }, + './suite-tests.mjs': { js: req.testsSource }, + } + if (hasModule) { + modules['./suite-module-impl.mjs'] = { js: req.moduleSource! } + modules['suite:module'] = { js: `export * from './suite-module-impl.mjs'\n` } + } + return { + compatibilityDate: EXEC_COMPATIBILITY_DATE, + mainModule: 'entry.mjs', + modules, + env: {}, + globalOutbound: outbound, + limits: { cpuMs: req.limits?.cpuMs ?? EXEC_CPU_MS }, + } +} + +// --------------------------------------------------------------------------- +// The hosted runner +// --------------------------------------------------------------------------- + +export interface WorkerLoaderRunnerOpts { + /** + * The service stub / entrypoint every isolate fetch is delivered to + * (`globalOutbound`). REQUIRED to execute: absent ⇒ typed + * `runner-unavailable`, because an unspecified outbound would inherit the + * verifier's own network access. + */ + outbound?: unknown +} + +export function workerLoaderExecRunner( + loader: WorkerLoaderLike, + opts: WorkerLoaderRunnerOpts = {}, +): ExecSuiteRunner { + return { + async run(req: ExecRunRequest): Promise { + if (opts.outbound === undefined || opts.outbound === null) { + return { status: 'runner-unavailable', reason: RUNNER_UNAVAILABLE_NO_OUTBOUND } + } + + // The SAME shared validation the local runner runs — a subset violation + // is refused before an isolate exists (and identically in both hosts). + const hasModule = req.artifactKind === 'document' && typeof req.moduleSource === 'string' + if (hasModule) { + const v = validateDialectSource(req.moduleSource!, { + allowSuiteModule: false, + what: 'the suite document `module` member', + }) + if (!v.ok) return { status: 'failed', reason: v.problem } + } + const tv = validateDialectSource(req.testsSource, { + allowSuiteModule: hasModule, + what: req.artifactKind === 'document' ? 'the suite document `tests` member' : 'the pinned module artifact', + }) + if (!tv.ok) return { status: 'failed', reason: tv.problem } + + const wallMs = req.limits?.wallMs ?? EXEC_WALL_MS + const cpuMs = req.limits?.cpuMs ?? EXEC_CPU_MS + const appliedLimits = { wallMs, cpuMs } + const started = Date.now() + + try { + // Content-hash id (Cloudflare's own recommendation): same digest + + // harness ⇒ same id (warm reuse is a pure optimization), different + // bytes ⇒ a different isolate by construction. + const id = `vitest1:${req.digest ?? 'inline'}:${HARNESS_VERSION}` + const stub = loader.get(id, () => buildWorkerCode(req, opts.outbound)) + + let timer: ReturnType | undefined + const breaker = new Promise<'breaker'>((resolve) => { + timer = setTimeout(() => resolve('breaker'), wallMs) + }) + const raced = await Promise.race([ + stub.getEntrypoint().fetch('https://run/'), + breaker, + ]).finally(() => clearTimeout(timer)) + if (raced === 'breaker') { + return { + status: 'failed', + reason: + `circuit breaker tripped: the run exceeded ${wallMs} ms wall-clock (applied limits ` + + `${wallMs} ms wall / ${cpuMs} ms CPU) — a tripped breaker fails the run, never a partial verdict (A.8.6.3)`, + } + } + const body = (await raced.json()) as { + registered?: number + results?: ExecTestResult[] + violations?: GateViolation[] + error?: string + } + if (typeof body.error === 'string') { + const floored = body.violations?.[0] + return { + status: 'failed', + reason: floored !== undefined ? floored.reason : `the suite failed to instantiate or register: ${body.error}`, + } + } + return foldRunOutcome( + { + registered: body.registered ?? 0, + results: body.results ?? [], + violations: body.violations ?? [], + }, + req, + appliedLimits, + Date.now() - started, + null, // consumed CPU: surfaced by the platform's limits API when enrolled; recorded null until then + ) + } catch (err) { + return { + status: 'failed', + reason: `the Worker Loader run failed: ${err instanceof Error ? err.message : String(err)}`, + } + } + }, + } +} diff --git a/src/exec/vitest-subset-source.ts b/src/exec/vitest-subset-source.ts new file mode 100644 index 0000000..2a0185f --- /dev/null +++ b/src/exec/vitest-subset-source.ts @@ -0,0 +1,11 @@ +/** + * GENERATED — do not edit. Regenerate with: node scripts/gen-vitest-subset.mjs + * + * The exact bytes of the canonical `api.qa/vitest@1` subset harness + * (src/exec/vitest-subset.mjs), carried as a string so the hosted Worker + * Loader runner can inject them into an isolate's module map and the local + * runner can instantiate them via a data: import — ONE implementation for + * both hosts (AXP A.8.6.2), pinned byte-identical by test/vitest-subset.test.ts. + */ + +export const VITEST_SUBSET_SOURCE: string = "/**\n * THE `api.qa/vitest@1` SUBSET HARNESS — the one shared implementation.\n *\n * AXP A.8.6.2 (apis-ax-axp@2.4.0, spec digest dd3e5941…) is normative here:\n * \"the hosted verifier's harness and the local CLI runner MUST share ONE\n * implementation of the subset — one module, never a reimplementation.\" This\n * file IS that module. It is deliberately:\n *\n * - SELF-CONTAINED — zero imports, so its exact bytes can be injected\n * verbatim into a Cloudflare Worker Loader isolate's module map (the\n * hosted runner) AND instantiated from the identical bytes via a\n * `data:text/javascript` import (the local CLI runner and the tests).\n * `src/exec/vitest-subset-source.ts` carries these bytes as a generated\n * string constant (node scripts/gen-vitest-subset.mjs); a pinning test\n * fails the build if the two ever drift by a byte.\n * - PLAIN .mjs, not TypeScript — the artifact both hosts execute is the\n * artifact in the repo; there is no compile step between what is reviewed\n * and what runs inside the isolate.\n *\n * WHAT IT IMPLEMENTS — the guaranteed subset, and only that:\n * describe / it (async-aware, ES2022) / expect with the core matchers\n * (equality, comparison, truthiness, containment, shape, rejects/resolves,\n * and `.not` over all of them). `test` is an alias of `it`, as in vitest.\n *\n * WHAT FAILS BY NAME — the constructs A.8.6.2 places outside the subset must\n * fail \"by a named reason, never by silently diverging\":\n * - snapshot matchers (`toMatchSnapshot`, `toMatchInlineSnapshot`) throw\n * naming the matcher — a snapshot is state outside the pinned document;\n * - the whole `vi` surface (mocking, fake timers) is a poisoned proxy: ANY\n * property access throws naming `vi.`;\n * - an unknown matcher is an ordinary TypeError naming the property, which\n * is the same named failure an unmodified local vitest produces for a\n * typo'd matcher.\n * Import-specifier closure, `eval`/`new Function`/dynamic-`import()` refusal,\n * the network floor, and seeded `Math.random` are the INSTANTIATOR's duties\n * (src/exec/dialect.ts locally; the isolate entry + gateway hosted) — they\n * gate what surrounds the module, not what the subset API itself does.\n *\n * The verdict is never computed here: `run()` returns raw per-test results\n * (name, pass/fail, duration, named reason, in REGISTRATION order — A.8.6.4\n * sequential politeness) and the caller judges. The suite cannot vote on its\n * own outcome through any export of this module.\n */\n\n/** The dialect this harness implements. */\nexport const SUBSET_RUNNER = 'api.qa/vitest@1'\n\n/** Names installed as globals in the document form (A.8.6.2). */\nexport const SUBSET_GLOBALS = ['describe', 'it', 'test', 'expect', 'vi']\n\n/**\n * Deterministic PRNG (mulberry32) — the A.8.6.4 seeded `Math.random`\n * replacement. It lives IN the shared harness module so the hosted isolate\n * entry and the local runner derive randomness from the same bytes: a replay\n * under the recorded seed and the recorded digest is the same run, in either\n * host. (Same algorithm the verifier's own probe sampling uses.)\n */\nexport function seededRandom(seed) {\n let a = seed >>> 0\n return () => {\n a |= 0\n a = (a + 0x6d2b79f5) | 0\n let t = Math.imul(a ^ (a >>> 15), 1 | a)\n t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t\n return ((t ^ (t >>> 14)) >>> 0) / 4294967296\n }\n}\n\n// --------------------------------------------------------------------------\n// Deep equality (toEqual / toStrictEqual / toContainEqual / toMatchObject)\n// --------------------------------------------------------------------------\n\nfunction isPlainObjectLike(v) {\n return v !== null && typeof v === 'object'\n}\n\nfunction deepEqual(a, b) {\n if (Object.is(a, b)) return true\n if (a instanceof Date && b instanceof Date) return a.getTime() === b.getTime()\n if (Array.isArray(a) || Array.isArray(b)) {\n if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false\n for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false\n return true\n }\n if (!isPlainObjectLike(a) || !isPlainObjectLike(b)) return false\n const ka = Object.keys(a).filter((k) => a[k] !== undefined)\n const kb = Object.keys(b).filter((k) => b[k] !== undefined)\n if (ka.length !== kb.length) return false\n for (const k of ka) {\n if (!Object.prototype.hasOwnProperty.call(b, k)) return false\n if (!deepEqual(a[k], b[k])) return false\n }\n return true\n}\n\n/** Subset-shape match: every key in `shape` must deep-match in `actual`. */\nfunction matchesShape(actual, shape) {\n if (!isPlainObjectLike(actual) || !isPlainObjectLike(shape)) return deepEqual(actual, shape)\n if (Array.isArray(shape)) {\n if (!Array.isArray(actual) || actual.length !== shape.length) return false\n for (let i = 0; i < shape.length; i++) if (!matchesShape(actual[i], shape[i])) return false\n return true\n }\n for (const k of Object.keys(shape)) {\n if (!(k in actual)) return false\n if (!matchesShape(actual[k], shape[k])) return false\n }\n return true\n}\n\nfunction show(v) {\n if (typeof v === 'string') return JSON.stringify(v)\n if (typeof v === 'function') return '[function]'\n if (typeof v === 'bigint') return `${v}n`\n try {\n const s = JSON.stringify(v)\n return s === undefined ? String(v) : s.length > 200 ? s.slice(0, 200) + '…' : s\n } catch {\n return String(v)\n }\n}\n\n// --------------------------------------------------------------------------\n// expect\n// --------------------------------------------------------------------------\n\nfunction readPath(obj, path) {\n const segs = Array.isArray(path) ? path : String(path).split('.')\n let cur = obj\n for (const s of segs) {\n if (cur === null || cur === undefined) return { found: false }\n if (!(typeof cur === 'object' && (s in cur))) return { found: false }\n cur = cur[s]\n }\n return { found: true, value: cur }\n}\n\nfunction makeExpect() {\n /** name → (actual, args) → { pass, message } */\n const matchers = {\n toBe: (a, [e]) => ({\n pass: Object.is(a, e),\n message: `expected ${show(a)} to be ${show(e)} (Object.is)`,\n }),\n toEqual: (a, [e]) => ({\n pass: deepEqual(a, e),\n message: `expected ${show(a)} to deeply equal ${show(e)}`,\n }),\n toStrictEqual: (a, [e]) => ({\n pass: deepEqual(a, e),\n message: `expected ${show(a)} to strictly equal ${show(e)}`,\n }),\n toMatchObject: (a, [shape]) => ({\n pass: matchesShape(a, shape),\n message: `expected ${show(a)} to match object ${show(shape)}`,\n }),\n toBeTruthy: (a) => ({ pass: !!a, message: `expected ${show(a)} to be truthy` }),\n toBeFalsy: (a) => ({ pass: !a, message: `expected ${show(a)} to be falsy` }),\n toBeNull: (a) => ({ pass: a === null, message: `expected ${show(a)} to be null` }),\n toBeUndefined: (a) => ({ pass: a === undefined, message: `expected ${show(a)} to be undefined` }),\n toBeDefined: (a) => ({ pass: a !== undefined, message: `expected value to be defined` }),\n toBeNaN: (a) => ({ pass: Number.isNaN(a), message: `expected ${show(a)} to be NaN` }),\n toBeInstanceOf: (a, [c]) => ({\n pass: a instanceof c,\n message: `expected value to be an instance of ${c && c.name ? c.name : show(c)}`,\n }),\n toContain: (a, [e]) => ({\n pass:\n typeof a === 'string'\n ? a.includes(e)\n : Array.isArray(a)\n ? a.some((x) => Object.is(x, e))\n : false,\n message: `expected ${show(a)} to contain ${show(e)}`,\n }),\n toContainEqual: (a, [e]) => ({\n pass: Array.isArray(a) && a.some((x) => deepEqual(x, e)),\n message: `expected ${show(a)} to contain an item deeply equal to ${show(e)}`,\n }),\n toHaveLength: (a, [n]) => ({\n pass: a != null && a.length === n,\n message: `expected length ${a == null ? '(none)' : show(a.length)} to be ${show(n)}`,\n }),\n toHaveProperty: (a, [path, ...rest]) => {\n const r = readPath(a, path)\n const pass = r.found && (rest.length === 0 || deepEqual(r.value, rest[0]))\n return {\n pass,\n message:\n rest.length === 0\n ? `expected ${show(a)} to have property ${show(path)}`\n : `expected property ${show(path)} (${r.found ? show(r.value) : 'absent'}) to equal ${show(rest[0])}`,\n }\n },\n toMatch: (a, [e]) => ({\n pass: typeof a === 'string' && (e instanceof RegExp ? e.test(a) : a.includes(String(e))),\n message: `expected ${show(a)} to match ${e instanceof RegExp ? String(e) : show(e)}`,\n }),\n toBeGreaterThan: (a, [e]) => ({ pass: a > e, message: `expected ${show(a)} > ${show(e)}` }),\n toBeGreaterThanOrEqual: (a, [e]) => ({ pass: a >= e, message: `expected ${show(a)} >= ${show(e)}` }),\n toBeLessThan: (a, [e]) => ({ pass: a < e, message: `expected ${show(a)} < ${show(e)}` }),\n toBeLessThanOrEqual: (a, [e]) => ({ pass: a <= e, message: `expected ${show(a)} <= ${show(e)}` }),\n toBeCloseTo: (a, [e, digits]) => {\n const d = digits === undefined ? 2 : digits\n return {\n pass: Math.abs(a - e) < Math.pow(10, -d) / 2,\n message: `expected ${show(a)} to be close to ${show(e)} (${d} digits)`,\n }\n },\n toThrow: (a, [e]) => {\n // `a` is the thrown error when reached via .rejects, else a function.\n let threw = false\n let err\n if (typeof a === 'function') {\n try {\n a()\n } catch (thrown) {\n threw = true\n err = thrown\n }\n } else {\n threw = true\n err = a\n }\n if (!threw) return { pass: false, message: 'expected function to throw, but it did not' }\n const msg = err instanceof Error ? err.message : String(err)\n const pass =\n e === undefined\n ? true\n : e instanceof RegExp\n ? e.test(msg)\n : typeof e === 'string'\n ? msg.includes(e)\n : typeof e === 'function'\n ? err instanceof e\n : false\n return { pass, message: `expected thrown ${show(msg)} to match ${e instanceof RegExp ? String(e) : show(e)}` }\n },\n }\n\n // Snapshot matchers are OUTSIDE the subset (A.8.6.2): a snapshot is state\n // outside the pinned document. They fail with the matcher NAMED — in the\n // hosted isolate and under the local runner alike.\n for (const name of ['toMatchSnapshot', 'toMatchInlineSnapshot', 'toMatchFileSnapshot']) {\n matchers[name] = () => {\n throw new Error(\n `${name} is outside the api.qa/vitest@1 subset — snapshot state cannot live inside a digest-pinned suite`,\n )\n }\n }\n\n function applyMatcher(name, actual, args, negated) {\n const m = matchers[name]\n const { pass, message } = m(actual, args)\n if (negated ? pass : !pass) {\n throw new Error(negated ? `${name}: NOT expected — ${message}` : `${name}: ${message}`)\n }\n }\n\n function chain(actual, negated) {\n const target = {}\n for (const name of Object.keys(matchers)) {\n target[name] = (...args) => applyMatcher(name, actual, args, negated)\n }\n Object.defineProperty(target, 'not', { get: () => chain(actual, !negated) })\n // Async assertion chains (A.8.6.2 rejects/resolves): every matcher becomes\n // a Promise-returning form the test MUST await.\n Object.defineProperty(target, 'resolves', {\n get: () => asyncChain(Promise.resolve(actual), false, negated),\n })\n Object.defineProperty(target, 'rejects', {\n get: () => asyncChain(Promise.resolve(actual), true, negated),\n })\n return target\n }\n\n function asyncChain(promise, expectRejection, negated) {\n const target = {}\n for (const name of Object.keys(matchers)) {\n target[name] = async (...args) => {\n let settled\n let rejected = false\n try {\n settled = await promise\n } catch (err) {\n rejected = true\n settled = err\n }\n if (expectRejection && !rejected) {\n throw new Error(`rejects.${name}: expected promise to reject, but it resolved with ${show(settled)}`)\n }\n if (!expectRejection && rejected) {\n const msg = settled instanceof Error ? settled.message : String(settled)\n throw new Error(`resolves.${name}: expected promise to resolve, but it rejected with ${show(msg)}`)\n }\n applyMatcher(name, settled, args, negated)\n }\n }\n Object.defineProperty(target, 'not', { get: () => asyncChain(promise, expectRejection, !negated) })\n return target\n }\n\n return (actual) => chain(actual, false)\n}\n\n// --------------------------------------------------------------------------\n// The vi poison — the whole mocking/timers surface fails by name (A.8.6.2)\n// --------------------------------------------------------------------------\n\nfunction makeViPoison() {\n return new Proxy(\n {},\n {\n get(_t, prop) {\n throw new Error(\n `vi.${String(prop)} is outside the api.qa/vitest@1 subset — module mocking and fake timers ` +\n 'have no meaning against a digest-pinned suite run over the network',\n )\n },\n },\n )\n}\n\n// --------------------------------------------------------------------------\n// The harness — registration + sequential run\n// --------------------------------------------------------------------------\n\n/**\n * Create one harness instance for one run. `api` carries the subset surface\n * (`describe`, `it`, `test`, `expect`, plus the poisoned `vi`); `run()`\n * executes every registered test SEQUENTIALLY in registration order (A.8.6.4)\n * and resolves to raw results the CALLER judges.\n */\nexport function createHarness() {\n const tests = [] // { name, fn } in registration order\n const path = [] // current describe nesting\n let running = false\n\n function describe(name, fn) {\n if (running) throw new Error('describe() called while the run is already executing')\n path.push(String(name))\n try {\n const r = fn()\n if (r && typeof r.then === 'function') {\n throw new Error('an async describe body is outside the api.qa/vitest@1 subset — register tests synchronously')\n }\n } finally {\n path.pop()\n }\n }\n\n function it(name, fn) {\n if (running) throw new Error('it() called while the run is already executing')\n if (typeof fn !== 'function') {\n throw new Error(`it(${show(name)}) registered without a test function`)\n }\n tests.push({ name: [...path, String(name)].join(' > '), fn })\n }\n\n const api = {\n describe,\n it,\n test: it,\n expect: makeExpect(),\n vi: makeViPoison(),\n }\n\n async function run() {\n running = true\n const results = []\n for (const t of tests) {\n const started = Date.now()\n try {\n await t.fn()\n results.push({ name: t.name, status: 'pass', durationMs: Date.now() - started })\n } catch (err) {\n results.push({\n name: t.name,\n status: 'fail',\n durationMs: Date.now() - started,\n reason: err instanceof Error ? err.message : String(err),\n })\n }\n }\n return { registered: tests.length, results }\n }\n\n return {\n api,\n run,\n get registered() {\n return tests.length\n },\n }\n}\n" diff --git a/src/exec/vitest-subset.mjs b/src/exec/vitest-subset.mjs new file mode 100644 index 0000000..ef2b049 --- /dev/null +++ b/src/exec/vitest-subset.mjs @@ -0,0 +1,395 @@ +/** + * THE `api.qa/vitest@1` SUBSET HARNESS — the one shared implementation. + * + * AXP A.8.6.2 (apis-ax-axp@2.4.0, spec digest dd3e5941…) is normative here: + * "the hosted verifier's harness and the local CLI runner MUST share ONE + * implementation of the subset — one module, never a reimplementation." This + * file IS that module. It is deliberately: + * + * - SELF-CONTAINED — zero imports, so its exact bytes can be injected + * verbatim into a Cloudflare Worker Loader isolate's module map (the + * hosted runner) AND instantiated from the identical bytes via a + * `data:text/javascript` import (the local CLI runner and the tests). + * `src/exec/vitest-subset-source.ts` carries these bytes as a generated + * string constant (node scripts/gen-vitest-subset.mjs); a pinning test + * fails the build if the two ever drift by a byte. + * - PLAIN .mjs, not TypeScript — the artifact both hosts execute is the + * artifact in the repo; there is no compile step between what is reviewed + * and what runs inside the isolate. + * + * WHAT IT IMPLEMENTS — the guaranteed subset, and only that: + * describe / it (async-aware, ES2022) / expect with the core matchers + * (equality, comparison, truthiness, containment, shape, rejects/resolves, + * and `.not` over all of them). `test` is an alias of `it`, as in vitest. + * + * WHAT FAILS BY NAME — the constructs A.8.6.2 places outside the subset must + * fail "by a named reason, never by silently diverging": + * - snapshot matchers (`toMatchSnapshot`, `toMatchInlineSnapshot`) throw + * naming the matcher — a snapshot is state outside the pinned document; + * - the whole `vi` surface (mocking, fake timers) is a poisoned proxy: ANY + * property access throws naming `vi.`; + * - an unknown matcher is an ordinary TypeError naming the property, which + * is the same named failure an unmodified local vitest produces for a + * typo'd matcher. + * Import-specifier closure, `eval`/`new Function`/dynamic-`import()` refusal, + * the network floor, and seeded `Math.random` are the INSTANTIATOR's duties + * (src/exec/dialect.ts locally; the isolate entry + gateway hosted) — they + * gate what surrounds the module, not what the subset API itself does. + * + * The verdict is never computed here: `run()` returns raw per-test results + * (name, pass/fail, duration, named reason, in REGISTRATION order — A.8.6.4 + * sequential politeness) and the caller judges. The suite cannot vote on its + * own outcome through any export of this module. + */ + +/** The dialect this harness implements. */ +export const SUBSET_RUNNER = 'api.qa/vitest@1' + +/** Names installed as globals in the document form (A.8.6.2). */ +export const SUBSET_GLOBALS = ['describe', 'it', 'test', 'expect', 'vi'] + +/** + * Deterministic PRNG (mulberry32) — the A.8.6.4 seeded `Math.random` + * replacement. It lives IN the shared harness module so the hosted isolate + * entry and the local runner derive randomness from the same bytes: a replay + * under the recorded seed and the recorded digest is the same run, in either + * host. (Same algorithm the verifier's own probe sampling uses.) + */ +export function seededRandom(seed) { + let a = seed >>> 0 + return () => { + a |= 0 + a = (a + 0x6d2b79f5) | 0 + let t = Math.imul(a ^ (a >>> 15), 1 | a) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +// -------------------------------------------------------------------------- +// Deep equality (toEqual / toStrictEqual / toContainEqual / toMatchObject) +// -------------------------------------------------------------------------- + +function isPlainObjectLike(v) { + return v !== null && typeof v === 'object' +} + +function deepEqual(a, b) { + if (Object.is(a, b)) return true + if (a instanceof Date && b instanceof Date) return a.getTime() === b.getTime() + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false + for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false + return true + } + if (!isPlainObjectLike(a) || !isPlainObjectLike(b)) return false + const ka = Object.keys(a).filter((k) => a[k] !== undefined) + const kb = Object.keys(b).filter((k) => b[k] !== undefined) + if (ka.length !== kb.length) return false + for (const k of ka) { + if (!Object.prototype.hasOwnProperty.call(b, k)) return false + if (!deepEqual(a[k], b[k])) return false + } + return true +} + +/** Subset-shape match: every key in `shape` must deep-match in `actual`. */ +function matchesShape(actual, shape) { + if (!isPlainObjectLike(actual) || !isPlainObjectLike(shape)) return deepEqual(actual, shape) + if (Array.isArray(shape)) { + if (!Array.isArray(actual) || actual.length !== shape.length) return false + for (let i = 0; i < shape.length; i++) if (!matchesShape(actual[i], shape[i])) return false + return true + } + for (const k of Object.keys(shape)) { + if (!(k in actual)) return false + if (!matchesShape(actual[k], shape[k])) return false + } + return true +} + +function show(v) { + if (typeof v === 'string') return JSON.stringify(v) + if (typeof v === 'function') return '[function]' + if (typeof v === 'bigint') return `${v}n` + try { + const s = JSON.stringify(v) + return s === undefined ? String(v) : s.length > 200 ? s.slice(0, 200) + '…' : s + } catch { + return String(v) + } +} + +// -------------------------------------------------------------------------- +// expect +// -------------------------------------------------------------------------- + +function readPath(obj, path) { + const segs = Array.isArray(path) ? path : String(path).split('.') + let cur = obj + for (const s of segs) { + if (cur === null || cur === undefined) return { found: false } + if (!(typeof cur === 'object' && (s in cur))) return { found: false } + cur = cur[s] + } + return { found: true, value: cur } +} + +function makeExpect() { + /** name → (actual, args) → { pass, message } */ + const matchers = { + toBe: (a, [e]) => ({ + pass: Object.is(a, e), + message: `expected ${show(a)} to be ${show(e)} (Object.is)`, + }), + toEqual: (a, [e]) => ({ + pass: deepEqual(a, e), + message: `expected ${show(a)} to deeply equal ${show(e)}`, + }), + toStrictEqual: (a, [e]) => ({ + pass: deepEqual(a, e), + message: `expected ${show(a)} to strictly equal ${show(e)}`, + }), + toMatchObject: (a, [shape]) => ({ + pass: matchesShape(a, shape), + message: `expected ${show(a)} to match object ${show(shape)}`, + }), + toBeTruthy: (a) => ({ pass: !!a, message: `expected ${show(a)} to be truthy` }), + toBeFalsy: (a) => ({ pass: !a, message: `expected ${show(a)} to be falsy` }), + toBeNull: (a) => ({ pass: a === null, message: `expected ${show(a)} to be null` }), + toBeUndefined: (a) => ({ pass: a === undefined, message: `expected ${show(a)} to be undefined` }), + toBeDefined: (a) => ({ pass: a !== undefined, message: `expected value to be defined` }), + toBeNaN: (a) => ({ pass: Number.isNaN(a), message: `expected ${show(a)} to be NaN` }), + toBeInstanceOf: (a, [c]) => ({ + pass: a instanceof c, + message: `expected value to be an instance of ${c && c.name ? c.name : show(c)}`, + }), + toContain: (a, [e]) => ({ + pass: + typeof a === 'string' + ? a.includes(e) + : Array.isArray(a) + ? a.some((x) => Object.is(x, e)) + : false, + message: `expected ${show(a)} to contain ${show(e)}`, + }), + toContainEqual: (a, [e]) => ({ + pass: Array.isArray(a) && a.some((x) => deepEqual(x, e)), + message: `expected ${show(a)} to contain an item deeply equal to ${show(e)}`, + }), + toHaveLength: (a, [n]) => ({ + pass: a != null && a.length === n, + message: `expected length ${a == null ? '(none)' : show(a.length)} to be ${show(n)}`, + }), + toHaveProperty: (a, [path, ...rest]) => { + const r = readPath(a, path) + const pass = r.found && (rest.length === 0 || deepEqual(r.value, rest[0])) + return { + pass, + message: + rest.length === 0 + ? `expected ${show(a)} to have property ${show(path)}` + : `expected property ${show(path)} (${r.found ? show(r.value) : 'absent'}) to equal ${show(rest[0])}`, + } + }, + toMatch: (a, [e]) => ({ + pass: typeof a === 'string' && (e instanceof RegExp ? e.test(a) : a.includes(String(e))), + message: `expected ${show(a)} to match ${e instanceof RegExp ? String(e) : show(e)}`, + }), + toBeGreaterThan: (a, [e]) => ({ pass: a > e, message: `expected ${show(a)} > ${show(e)}` }), + toBeGreaterThanOrEqual: (a, [e]) => ({ pass: a >= e, message: `expected ${show(a)} >= ${show(e)}` }), + toBeLessThan: (a, [e]) => ({ pass: a < e, message: `expected ${show(a)} < ${show(e)}` }), + toBeLessThanOrEqual: (a, [e]) => ({ pass: a <= e, message: `expected ${show(a)} <= ${show(e)}` }), + toBeCloseTo: (a, [e, digits]) => { + const d = digits === undefined ? 2 : digits + return { + pass: Math.abs(a - e) < Math.pow(10, -d) / 2, + message: `expected ${show(a)} to be close to ${show(e)} (${d} digits)`, + } + }, + toThrow: (a, [e]) => { + // `a` is the thrown error when reached via .rejects, else a function. + let threw = false + let err + if (typeof a === 'function') { + try { + a() + } catch (thrown) { + threw = true + err = thrown + } + } else { + threw = true + err = a + } + if (!threw) return { pass: false, message: 'expected function to throw, but it did not' } + const msg = err instanceof Error ? err.message : String(err) + const pass = + e === undefined + ? true + : e instanceof RegExp + ? e.test(msg) + : typeof e === 'string' + ? msg.includes(e) + : typeof e === 'function' + ? err instanceof e + : false + return { pass, message: `expected thrown ${show(msg)} to match ${e instanceof RegExp ? String(e) : show(e)}` } + }, + } + + // Snapshot matchers are OUTSIDE the subset (A.8.6.2): a snapshot is state + // outside the pinned document. They fail with the matcher NAMED — in the + // hosted isolate and under the local runner alike. + for (const name of ['toMatchSnapshot', 'toMatchInlineSnapshot', 'toMatchFileSnapshot']) { + matchers[name] = () => { + throw new Error( + `${name} is outside the api.qa/vitest@1 subset — snapshot state cannot live inside a digest-pinned suite`, + ) + } + } + + function applyMatcher(name, actual, args, negated) { + const m = matchers[name] + const { pass, message } = m(actual, args) + if (negated ? pass : !pass) { + throw new Error(negated ? `${name}: NOT expected — ${message}` : `${name}: ${message}`) + } + } + + function chain(actual, negated) { + const target = {} + for (const name of Object.keys(matchers)) { + target[name] = (...args) => applyMatcher(name, actual, args, negated) + } + Object.defineProperty(target, 'not', { get: () => chain(actual, !negated) }) + // Async assertion chains (A.8.6.2 rejects/resolves): every matcher becomes + // a Promise-returning form the test MUST await. + Object.defineProperty(target, 'resolves', { + get: () => asyncChain(Promise.resolve(actual), false, negated), + }) + Object.defineProperty(target, 'rejects', { + get: () => asyncChain(Promise.resolve(actual), true, negated), + }) + return target + } + + function asyncChain(promise, expectRejection, negated) { + const target = {} + for (const name of Object.keys(matchers)) { + target[name] = async (...args) => { + let settled + let rejected = false + try { + settled = await promise + } catch (err) { + rejected = true + settled = err + } + if (expectRejection && !rejected) { + throw new Error(`rejects.${name}: expected promise to reject, but it resolved with ${show(settled)}`) + } + if (!expectRejection && rejected) { + const msg = settled instanceof Error ? settled.message : String(settled) + throw new Error(`resolves.${name}: expected promise to resolve, but it rejected with ${show(msg)}`) + } + applyMatcher(name, settled, args, negated) + } + } + Object.defineProperty(target, 'not', { get: () => asyncChain(promise, expectRejection, !negated) }) + return target + } + + return (actual) => chain(actual, false) +} + +// -------------------------------------------------------------------------- +// The vi poison — the whole mocking/timers surface fails by name (A.8.6.2) +// -------------------------------------------------------------------------- + +function makeViPoison() { + return new Proxy( + {}, + { + get(_t, prop) { + throw new Error( + `vi.${String(prop)} is outside the api.qa/vitest@1 subset — module mocking and fake timers ` + + 'have no meaning against a digest-pinned suite run over the network', + ) + }, + }, + ) +} + +// -------------------------------------------------------------------------- +// The harness — registration + sequential run +// -------------------------------------------------------------------------- + +/** + * Create one harness instance for one run. `api` carries the subset surface + * (`describe`, `it`, `test`, `expect`, plus the poisoned `vi`); `run()` + * executes every registered test SEQUENTIALLY in registration order (A.8.6.4) + * and resolves to raw results the CALLER judges. + */ +export function createHarness() { + const tests = [] // { name, fn } in registration order + const path = [] // current describe nesting + let running = false + + function describe(name, fn) { + if (running) throw new Error('describe() called while the run is already executing') + path.push(String(name)) + try { + const r = fn() + if (r && typeof r.then === 'function') { + throw new Error('an async describe body is outside the api.qa/vitest@1 subset — register tests synchronously') + } + } finally { + path.pop() + } + } + + function it(name, fn) { + if (running) throw new Error('it() called while the run is already executing') + if (typeof fn !== 'function') { + throw new Error(`it(${show(name)}) registered without a test function`) + } + tests.push({ name: [...path, String(name)].join(' > '), fn }) + } + + const api = { + describe, + it, + test: it, + expect: makeExpect(), + vi: makeViPoison(), + } + + async function run() { + running = true + const results = [] + for (const t of tests) { + const started = Date.now() + try { + await t.fn() + results.push({ name: t.name, status: 'pass', durationMs: Date.now() - started }) + } catch (err) { + results.push({ + name: t.name, + status: 'fail', + durationMs: Date.now() - started, + reason: err instanceof Error ? err.message : String(err), + }) + } + } + return { registered: tests.length, results } + } + + return { + api, + run, + get registered() { + return tests.length + }, + } +} diff --git a/src/http.ts b/src/http.ts index bb83e2a..f2d6d12 100644 --- a/src/http.ts +++ b/src/http.ts @@ -96,18 +96,32 @@ export class Observer { * point — a mock/injected transport, and the deployed Worker's, must apply to * the sub-run identically — while the budget and write posture are isolated. */ - child(overrides: Pick): Observer { + child(overrides: Pick): Observer { return new Observer({ fetcher: this.opts.fetcher, delayMs: this.opts.delayMs, timeoutMs: this.opts.timeoutMs, - maxBodyBytes: this.opts.maxBodyBytes, + // A child may RAISE the body cap for one declared artifact fetch (the + // executable-suite document/module, whose A.8.6.1/A.8.6.3 caps exceed + // the probe default — a truncated body could never digest-match, which + // would turn every large-but-legal artifact into a false mismatch). + maxBodyBytes: overrides.maxBodyBytes ?? this.opts.maxBodyBytes, allowPrivate: this.opts.allowPrivate, budget: overrides.budget ?? this.opts.budget, allowWrites: overrides.allowWrites ?? false, }) } + /** + * The transport this observer fetches through — exposed so the executable- + * suite runner's gated fetch wraps the SAME channel the evidence was + * recorded over (an in-memory fixture target, and the deployed Worker's real + * fetch, apply to the suite run identically). + */ + get transportFetcher(): Fetcher { + return this.opts.fetcher + } + /** * Fetch once, record Evidence, return it. Never throws. * diff --git a/src/index.ts b/src/index.ts index 89e10f3..243392e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,7 +40,60 @@ export { parseAgentsJson, parseOpenapi, ROLE, + type ObserveTargetOpts, + type TestSuiteClaim, } from './discovery.js' +// ── The `api.qa/vitest@1` executable dialect (AXP A.8.6) ──────────────────── +// The SHARED subset harness + the runner seam are public: the CLI's local +// verb, a property's own CI, and the hosted verifier all run the ONE +// implementation (A.8.6.2's normative parity), and adopters need the gates +// and caps to publish artifacts that will verify. +export { + VITEST_RUNNER, + EXEC_WALL_MS, + EXEC_CPU_MS, + EXEC_MAX_COMBINED, + EXEC_MAX_DOC_BYTES, + EXEC_MAX_MODULE_BYTES, + EXEC_MAX_OUTPUT_BYTES, + isFloorBlockedHost, + createGatedFetch, + validateDialectSource, + localExecRunner, + loadHarnessModule, + foldRunOutcome, + type ExecRunRequest, + type ExecRunOutcome, + type ExecRunIo, + type ExecSuiteRunner, + type ExecTestResult, + type GateViolation, +} from './exec/dialect.js' +export { + workerLoaderExecRunner, + unavailableExecRunner, + buildWorkerCode, + entrySource, + gatewayFetch, + HARNESS_VERSION, + EXEC_COMPATIBILITY_DATE, + RUNNER_UNAVAILABLE_NO_BINDING, + RUNNER_UNAVAILABLE_NO_OUTBOUND, + type WorkerLoaderLike, + type WorkerCodeLike, +} from './exec/runner.js' +export { VITEST_SUBSET_SOURCE } from './exec/vitest-subset-source.js' +export { + gateVitestSuiteCard, + gateVitestSuiteDocument, + gateVitestModuleArtifact, + NATIVE_SERVING_BASE, + type VitestCardGate, + type VitestDocumentGate, + type VitestModuleGate, +} from './test-suite.js' +export { parseExecSuiteDocument } from './suite-doc.js' +export { AXP_PINNED_SPEC } from './pinned.js' export { runChecks } from './checks.js' export { contractDiff, enumerateOperations } from './contract.js' export { axScoreOf, gradeOf } from './grade.js' diff --git a/src/pinned.ts b/src/pinned.ts index 2988b2e..66f7a54 100644 --- a/src/pinned.ts +++ b/src/pinned.ts @@ -111,8 +111,33 @@ export interface VerifyPinnedOpts extends ObserverOpts { * into BOTH the observe scope and the judge scope so the two agree. */ initialBindings?: Record + /** The `api.qa/vitest@1` execution seam — see VerifyTargetOpts.execRunner. */ + execRunner?: import('./exec/dialect.js').ExecSuiteRunner } +/** + * THE RATIFIED ADMISSION CONTRACT — `apis-ax-axp@2.4.0`. + * + * The coordinated 2.3.0 → 2.4.0 bump (AXP 0.7.0, 2026-08-08): 22 → 23 + * requirements, retiring digest 9063cb3e… . The ONE added row is + * `check-published-test-suite` (kind: check, must: pass, + * appliesWhen: { cardDeclares: "interfaces.testSuite" }) — declaration-armed, + * so a card that omits the interface gains one not-applicable result and no + * new way to fail, and a card that declares a suite and does not keep it + * loses admission. The same release ratifies the executable dialect + * `api.qa/vitest@1` (Appendix A.8.6) this verifier implements in src/exec/. + * + * The digest is sha256 over the exact bytes of the spec's + * `apis-ax-standard.spec.json` (axp.org.ai `spec/conformance/`), and it is + * the ONLY authority: a run pinned to this contract supplies the spec TEXT + * plus this digest, and a text that does not hash to it never runs. + */ +export const AXP_PINNED_SPEC = { + name: 'apis-ax-axp', + version: '2.4.0', + digest: 'dd3e59417e2acacd0946e14c845c2e156a437ef55724ff06a52c053885e321bf', +} as const + export function parsePinnedSpec(text: string): PinnedSpec { const doc = JSON.parse(text) as PinnedSpec if (doc.$type !== 'PinnedSpec' || !Array.isArray(doc.requirements)) { @@ -168,7 +193,7 @@ export async function verifyPinnedSpec( // compliant target. budget: opts.budget ?? 64, }) - const bundle = await observeTarget(origin, observer, seed) + const bundle = await observeTarget(origin, observer, seed, { execRunner: opts.execRunner }) // Extra observations demanded by the spec's endpoint requirements. // diff --git a/src/suite-doc.ts b/src/suite-doc.ts index 732f51b..59868bd 100644 --- a/src/suite-doc.ts +++ b/src/suite-doc.ts @@ -349,7 +349,13 @@ export function parseSuite(text: string): Suite { if (doc.$type !== 'Suite' || !Array.isArray(doc.requirements)) { throw new Error('not a Suite: expected {"$type":"Suite","environments":{...},"requirements":[...]}') } - const envs = doc.environments as unknown + validateEnvironments(doc.environments as unknown) + validateRequirements(doc.requirements) + return doc +} + +/** The shared environments-map shape check — `parseSuite` and the vitest@1 parse agree by construction. */ +function validateEnvironments(envs: unknown): void { if (envs === null || typeof envs !== 'object' || Array.isArray(envs)) { throw new Error('Suite.environments must be an object mapping env name -> { vars: { : } }') } @@ -359,7 +365,66 @@ export function parseSuite(text: string): Suite { vars === null || typeof vars !== 'object' || Array.isArray(vars)) { throw new Error(`Suite environment "${name}" must be an object of the form { "vars": { : } }`) } + const sandbox = (env as { sandbox?: unknown }).sandbox + if (sandbox !== undefined && typeof sandbox !== 'boolean') { + throw new Error( + `Suite environment "${name}" declares "sandbox": ${JSON.stringify(sandbox)} — the A.8.6.4 write-consent ` + + 'flag must be a boolean. Consent is a pinned statement, not a truthy value.', + ) + } } - validateRequirements(doc.requirements) - return doc +} + +/** + * Parse + validate an `api.qa/vitest@1` SUITE DOCUMENT — the A.8.6.1 + * ADDITIVE extension of the suite@1 grammar (the mdxld house pattern: code as + * string members of the same JSON-serializable, digest-pinned document). + * + * Same `$type: "Suite"` envelope, same `environments` map (plus the boolean + * `sandbox` consent flag), same `requirements` grammar where rows are present + * — extended by two root members: + * + * `tests` (REQUIRED) — ES module source in the A.8.6.2 subset. Absent, + * empty, or not a string throws. + * `module` (optional) — ES module source instantiated first; its exports + * are importable by `tests` as `"suite:module"`. + * + * `requirements` MAY be absent or empty here (unlike suite@1): non-vacuity is + * judged over the UNION — at least one declarative row or one REGISTERED test + * — and registration counts exist only after the run, so the runner enforces + * it (foldRunOutcome), not this parse. Rows that ARE present get the full + * suite@1 guard set (`validateRequirements`), unchanged — the extension forks + * nothing. + */ +export function parseExecSuiteDocument(text: string): Suite { + const doc = JSON.parse(text) as Suite + if (doc.$type !== 'Suite') { + throw new Error( + 'not a Suite: an api.qa/vitest@1 document is an ADDITIVE extension of the suite@1 grammar — expected ' + + '{"$type":"Suite","environments":{...},"tests":"", ...}', + ) + } + validateEnvironments(doc.environments as unknown) + const tests = (doc as { tests?: unknown }).tests + if (typeof tests !== 'string' || tests.length === 0) { + throw new Error( + 'an api.qa/vitest@1 suite document MUST carry a non-empty string root member `tests` (ES module source ' + + 'in the A.8.6.2 subset) — absent, empty, or non-string fails (A.8.6.1)', + ) + } + const module = (doc as { module?: unknown }).module + if (module !== undefined && (typeof module !== 'string' || module.length === 0)) { + throw new Error( + 'the api.qa/vitest@1 `module` root member, when present, must be a non-empty string of ES module source (A.8.6.1)', + ) + } + const requirements = (doc as { requirements?: unknown }).requirements + if (requirements !== undefined && !Array.isArray(requirements)) { + throw new Error('Suite.requirements, when present, must be an array of requirement rows') + } + const rows = (requirements ?? []) as Suite['requirements'] + // Rows keep the UNCHANGED suite@1 guard set; an empty list is legal here + // because the tests member can carry the whole non-vacuity burden. + if (rows.length > 0) validateRequirements(rows) + return { ...doc, requirements: rows } } diff --git a/src/test-suite.ts b/src/test-suite.ts index ba274e1..f3389e5 100644 --- a/src/test-suite.ts +++ b/src/test-suite.ts @@ -68,14 +68,23 @@ import { isPubliclyRoutableSameOrigin } from './http.js' import { sha256HexSync } from './sha256-sync.js' -import { parseSuite } from './suite-doc.js' +import { parseSuite, parseExecSuiteDocument } from './suite-doc.js' +import { + EXEC_MAX_DOC_BYTES, + EXEC_MAX_MODULE_BYTES, + VITEST_RUNNER, + isFloorBlockedHost, +} from './exec/dialect.js' import type { EndpointReq } from './expect.js' import type { Suite } from './types.js' import type { TestSuiteClaim } from './discovery.js' -/** The only suite dialect this verifier implements. */ +/** The declarative suite dialect. */ export const SUITE_RUNNER = 'api.qa/suite@1' +/** The executable dialect (A.8.6) — re-exported so callers dispatch on one constant. */ +export { VITEST_RUNNER } + /** * Hard cap on requirements in a CARD-DECLARED suite. Deliberately far below the * parent run's budget: a stranger's document gets a small, fixed allowance. @@ -128,15 +137,32 @@ export function gateTestSuiteCard(claim: TestSuiteClaim, origin: string): CardGa // The runner is a DIALECT, not a hint. An unknown one FAILS rather than // skipping: a card that claims a suite api.qa cannot interpret has made a // claim that cannot be verified, which is a defective claim, not an absence. + // (The executable dialect never reaches this gate — callers dispatch + // `runner: "api.qa/vitest@1"` to `gateVitestSuiteCard` first.) if (claim.runner !== SUITE_RUNNER) { return { ok: false, problem: - `interfaces.testSuite declares runner ${JSON.stringify(claim.runner)}, which api.qa does not implement — the only defined ` + - `suite dialect is ${JSON.stringify(SUITE_RUNNER)} (a digest-pinned api.qa Suite document). This is a FAILURE, not a skip: ` + + `interfaces.testSuite declares runner ${JSON.stringify(claim.runner)}, which api.qa does not implement — the defined ` + + `suite dialects are ${JSON.stringify(SUITE_RUNNER)} (declarative, interpreted) and ${JSON.stringify(VITEST_RUNNER)} ` + + '(executable, A.8.6). This is a FAILURE, not a skip: ' + 'the card claims a suite this verifier cannot interpret. Omit the key to declare no test suite.', } } + // The npm identity assertion belongs to the executable dialect (A.8.6.6): + // under a declarative runner it asserts provenance about bytes that are + // never instantiated as a module, which is a claim in a shape this dialect + // cannot carry — refused, per A.8.5.2 ("a `package` under a declarative + // runner MUST fail"). + if (claim.packageName !== undefined) { + return { + ok: false, + problem: + `interfaces.testSuite declares package ${JSON.stringify(claim.packageName)} under the declarative runner ` + + `${JSON.stringify(SUITE_RUNNER)} — the npm identity assertion is meaningful only under ${JSON.stringify(VITEST_RUNNER)}. ` + + 'A defective declaration fails; omit the key to declare no test suite.', + } + } // The pin is REQUIRED — see judgeTestSuiteDocument for why a suite needs one // where a Digital Link description file does not. if (claim.digest === undefined) { @@ -320,3 +346,348 @@ function urlOriginOrUndefined(url: string): string | undefined { return undefined } } + +// --------------------------------------------------------------------------- +// The EXECUTABLE dialect gates — `api.qa/vitest@1` (A.8.5 seam + A.8.6) +// --------------------------------------------------------------------------- + +/** Exact published semver — never a range (A.8.5: `version` conditional rule). */ +const EXACT_SEMVER = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/ + +/** An npm package name (scoped or unscoped). */ +const NPM_NAME = /^(?:@[a-z0-9~-][a-z0-9._~-]*\/)?[a-z0-9~-][a-z0-9._~-]*$/ + +/** A JS identifier — the only legal `export` member shape. */ +const EXPORT_IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*$/ + +/** The reference module CDN the native-serving obligation (A.8.6.6) derives url-less coordinates from. */ +export const NATIVE_SERVING_BASE = 'https://pkg.do' + +export type VitestCardGate = + | { + ok: true + /** The resolved artifact address (declared url, or derived from the coordinate). */ + url: string + digest: string + /** Discriminated by the CARD: `package` present or pathname ends `.mjs` ⇒ module. */ + kind: 'document' | 'module' + environment: string + exportName?: string + /** The npm identity assertion — RECORDED in the verdict, never adjudicated. */ + npm?: { package: string; version: string } + } + | { ok: false; problem: string } + +/** + * GATE 1 for the executable dialect — decided from the CARD ALONE, before + * anything is fetched. The A.8.5 seam: + * + * { url?, package?, version?, export?, digest, environment?, runner } + * + * - at least one of `url` / `package`; `package` requires an EXACT `version`; + * - `digest` required, `"sha256:" + 64 lowhex`, the SOLE byte authority; + * - the address MAY be off-origin (a versioned module-CDN URL is the SDK + * norm — the digest, never the host, is the provenance); a non-routable or + * floor-blocked address is refused WITHOUT being fetched; + * - url-less coordinates resolve via the A.8.6.6 native-serving scheme + * (`https://pkg.do/@/index.mjs`); + * - artifact KIND is decided BY THE CARD (package present or pathname ends + * `.mjs` ⇒ module), never by sniffing bytes; + * - `export` is an identifier and module-kind-only; a module artifact defines + * no environments, so only the implicit `"public"` selects there. + */ +export function gateVitestSuiteCard(claim: TestSuiteClaim, _origin: string): VitestCardGate { + if (claim.malformed) { + return { + ok: false, + problem: + `interfaces.testSuite is present but is not a JSON object (got ${claim.malformedAs}) — the card claims a published test suite ` + + 'in a shape no verifier can check. Declare an object, or OMIT the key entirely — omitting it is fully conforming.', + } + } + // The pin, before anything else: the digest is the SOLE byte authority in + // every addressing (A.8.6.6) — absent or malformed refuses unfetched. + if (claim.digest === undefined) { + return { + ok: false, + problem: + 'interfaces.testSuite declares no `digest` — a published suite MUST be digest-pinned ("sha256:<64 hex>" over the artifact\'s ' + + 'exact bytes). The digest is the sole byte authority in every addressing (A.8.6.6). Refused without fetching.', + } + } + if (!/^sha256:[0-9a-f]{64}$/.test(claim.digest)) { + return { + ok: false, + problem: + `interfaces.testSuite declares digest ${JSON.stringify(claim.digest)}, which is not of the form "sha256:<64 lowercase hex>" — ` + + 'refused without fetching.', + } + } + + // The npm coordinate — an IDENTITY ASSERTION, never a delivery channel. + let npm: { package: string; version: string } | undefined + if (claim.packageName !== undefined) { + if (!NPM_NAME.test(claim.packageName)) { + return { + ok: false, + problem: `interfaces.testSuite declares package ${JSON.stringify(claim.packageName)}, which is not an npm package name — refused.`, + } + } + if (claim.version === undefined) { + return { + ok: false, + problem: + `interfaces.testSuite declares package ${JSON.stringify(claim.packageName)} with no \`version\` — the identity assertion ` + + 'requires the EXACT published version (A.8.5). Refused.', + } + } + if (!EXACT_SEMVER.test(claim.version)) { + return { + ok: false, + problem: + `interfaces.testSuite declares version ${JSON.stringify(claim.version)} — the identity assertion requires an EXACT ` + + 'published semver, never a range (A.8.5). Refused.', + } + } + npm = { package: claim.packageName, version: claim.version } + } + + // The address: declared url, or derived from the coordinate under the + // native-serving obligation. Neither ⇒ nothing is published. + let url = claim.urlRaw !== undefined && claim.url !== '' ? claim.url : undefined + if (url === undefined) { + if (npm === undefined) { + return { + ok: false, + problem: + 'interfaces.testSuite declares neither `url` nor `package` — a suite that names no location is not a published suite ' + + '(at least one address is required, A.8.5). Omit the whole key to declare no test suite.', + } + } + url = `${NATIVE_SERVING_BASE}/${npm.package}@${npm.version}/index.mjs` + } + let parsed: URL + try { + parsed = new URL(url) + } catch { + return { + ok: false, + problem: + `interfaces.testSuite declares url ${JSON.stringify(claim.urlRaw ?? url)}, which does not resolve to a URL — refused without fetching.`, + } + } + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { + return { + ok: false, + problem: `interfaces.testSuite declares url ${url}, which is not an http(s) address — refused without fetching.`, + } + } + // The network floor applies to the VERIFIER's artifact fetch exactly as it + // applies to the isolate's egress: a metadata/private/internal address is + // refused unfetched. An OFF-ORIGIN public address is fine — the A.8.5 + // same-origin restriction is RETIRED for this dialect (revision note 0.7.0); + // the digest, never the host, is the authority over the bytes. + if (isFloorBlockedHost(parsed.hostname)) { + return { + ok: false, + problem: + `interfaces.testSuite declares its artifact at ${url}, which the network floor bars (metadata, link-local, loopback, ` + + 'private-range, CGNAT, ULA, or estate-internal) — refused without fetching (A.8.6.3).', + } + } + + // Artifact kind — decided by the CARD, never by sniffing (A.8.5). + const kind: 'document' | 'module' = npm !== undefined || parsed.pathname.endsWith('.mjs') ? 'module' : 'document' + + if (claim.exportName !== undefined) { + if (kind !== 'module') { + return { + ok: false, + problem: + `interfaces.testSuite declares export ${JSON.stringify(claim.exportName)} on a suite DOCUMENT — \`export\` names a ` + + 'module export and is meaningful only for a module artifact (A.8.5). A defective declaration fails.', + } + } + if (!EXPORT_IDENTIFIER.test(claim.exportName)) { + return { + ok: false, + problem: `interfaces.testSuite declares export ${JSON.stringify(claim.exportName)}, which is not a legal identifier — refused.`, + } + } + } + + // A module artifact defines no environments: exactly the implicit + // non-sandbox "public" exists there (A.8.6.4). Selecting any other name is + // selecting an environment that does not exist ⇒ fail, at the card. + if (kind === 'module' && claim.environment !== 'public') { + return { + ok: false, + problem: + `interfaces.testSuite selects environment ${JSON.stringify(claim.environment)} on a MODULE artifact — a pinned module ` + + 'defines no environments and carries exactly the implicit "public" (sandbox: false, A.8.6.4). Refused.', + } + } + + return { + ok: true, + url, + digest: claim.digest, + kind, + environment: claim.environment, + ...(claim.exportName !== undefined && { exportName: claim.exportName }), + ...(npm !== undefined && { npm }), + } +} + +export type VitestDocumentGate = + | { + ok: true + suite: Suite + /** Declarative rows, eligible under UNCHANGED suite@1 engine semantics. */ + rows: EndpointReq[] + testsSource: string + moduleSource?: string + vars: Record + sandbox: boolean + digest: string + } + | { ok: false; problems: string[] } + +/** + * GATE 2 for the executable DOCUMENT addressing — pure over the exact bytes + * served. HASH-THEN-INSTANTIATE (A.8.6.3): the digest is re-computed over the + * one fetched buffer BEFORE anything in it is believed, the code strings are + * extracted from that same buffer, and a mismatch means nothing is ever + * instantiated — there is no execute-then-check ordering under any + * circumstance. + * + * Declarative rows keep the UNCHANGED suite@1 rules (endpoint-only, GET/HEAD + * only, same-origin re-gated at resolve time by the engine) — A.8.5.2: + * "declarative rows stay home; executable tests roam above the floor." + */ +export function gateVitestSuiteDocument( + claim: TestSuiteClaim, + suiteText: string, + cardDigest: string, +): VitestDocumentGate { + const problems: string[] = [] + + const docBytes = new TextEncoder().encode(suiteText).byteLength + if (docBytes > EXEC_MAX_DOC_BYTES) { + return { + ok: false, + problems: [ + `the served suite document is ${docBytes} bytes, over the ${EXEC_MAX_DOC_BYTES}-byte (1 MiB) cap of A.8.6.1 — ` + + 'an abuse circuit-breaker, not a ration; failed, never truncated.', + ], + } + } + + const actual = sha256HexSync(suiteText) + const expected = cardDigest.slice('sha256:'.length) + if (actual !== expected) { + return { + ok: false, + problems: [ + `suite digest mismatch: the card pins sha256:${expected} but the document served at ${claim.url} hashes to sha256:${actual} — ` + + 'the published suite is not the one the card claims. NOTHING was instantiated (digest fail-closed, A.8.6.3).', + ], + } + } + + let suite: Suite + try { + suite = parseExecSuiteDocument(suiteText) + } catch (e) { + return { + ok: false, + problems: [`suite document did not parse as an api.qa/vitest@1 Suite document: ${(e as Error).message}`], + } + } + + if (!Object.hasOwn(suite.environments, claim.environment)) { + const defined = Object.keys(suite.environments) + problems.push( + `the card selects environment ${JSON.stringify(claim.environment)}, which suite "${suite.name}" does not define ` + + `(it defines ${defined.length ? defined.map((n) => JSON.stringify(n)).join(', ') : 'none'})`, + ) + } + + // Declarative rows: the same kind-eligibility and write refusals the + // suite@1 gate applies — this dialect widens NOTHING about rows. + const rows: EndpointReq[] = [] + for (const req of suite.requirements) { + if (req.kind === 'endpoint') { + rows.push(req) + continue + } + problems.push( + `requirement "${(req as { id?: string }).id ?? '?'}" is kind:'${(req as { kind?: string }).kind}' — the declarative rows of an ` + + `api.qa/vitest@1 document keep unchanged suite@1 semantics: kind:'endpoint' only. Refused.`, + ) + } + for (const req of rows) { + const method = req.method.toUpperCase() + if (method !== 'GET' && method !== 'HEAD') { + problems.push( + `requirement "${req.id}" declares method ${JSON.stringify(req.method)} — declarative rows run GET/HEAD ONLY in both dialects ` + + '(A.8.5.2). Mutating flows belong in the executable tests, against an environment the suite declares "sandbox": true.', + ) + } + } + + if (problems.length > 0) return { ok: false, problems } + + const env = suite.environments[claim.environment]! + return { + ok: true, + suite, + rows, + testsSource: suite.tests!, + ...(suite.module !== undefined && { moduleSource: suite.module }), + vars: { ...env.vars }, + sandbox: env.sandbox === true, + digest: cardDigest, + } +} + +export type VitestModuleGate = + | { ok: true; digest: string } + | { ok: false; problems: string[] } + +/** + * GATE 2 for the MODULE addressing (A.8.6.6 channels 2 and 3): the pinned, + * natively served ES module. Digest fail-closed over the one fetched buffer; + * the 4 MiB module cap (an SDK entry is a bundle). The npm coordinate, where + * asserted, was already validated at the card gate and is RECORDED — never + * adjudicated: no registry is contacted, and registry state cannot influence + * the verdict. + */ +export function gateVitestModuleArtifact( + claim: TestSuiteClaim, + moduleText: string, + cardDigest: string, +): VitestModuleGate { + const bytes = new TextEncoder().encode(moduleText).byteLength + if (bytes > EXEC_MAX_MODULE_BYTES) { + return { + ok: false, + problems: [ + `the served module artifact is ${bytes} bytes, over the ${EXEC_MAX_MODULE_BYTES}-byte (4 MiB) cap of A.8.6.3 — ` + + 'failed, never truncated.', + ], + } + } + const actual = sha256HexSync(moduleText) + const expected = cardDigest.slice('sha256:'.length) + if (actual !== expected) { + return { + ok: false, + problems: [ + `module digest mismatch: the card pins sha256:${expected} but the module served at ${claim.url} hashes to sha256:${actual} — ` + + 'the published module is not the one the card claims. NOTHING was instantiated (digest fail-closed, A.8.6.3).', + ], + } + } + return { ok: true, digest: cardDigest } +} diff --git a/src/types.ts b/src/types.ts index 084b8b0..cfb9178 100644 --- a/src/types.ts +++ b/src/types.ts @@ -567,6 +567,14 @@ export interface SuiteEnvironment { /** `varName -> value`. A value keeps its JSON type: a number seeds a number * (typed whole-value interpolation preserves it), a string seeds a string. */ vars: Record + /** + * A.8.6.4 (`api.qa/vitest@1` documents only): the suite's OWN pinned + * consent to be written to in this environment. Executable tests may issue + * mutating verbs ONLY where this is `true`; the declarative dialect ignores + * it (suite@1 rows are GET/HEAD everywhere). Never inferred — consent lives + * in the pinned document or it does not exist. + */ + sandbox?: boolean } /** @@ -589,4 +597,18 @@ export interface Suite { /** Named environments; run selects one by name. */ environments: Record requirements: PinnedRequirement[] + /** + * A.8.6.1 — the `api.qa/vitest@1` ADDITIVE extension of this same document + * grammar (the mdxld house pattern: code as a Code-valued string member of a + * JSON-serializable document). ONE served artifact, ONE digest covering both + * natures: the declarative rows above and these code strings. The `suite@1` + * parser ignores them (unknown root members), and the dialect is + * discriminated by the CARD's `runner`, never by sniffing the document. + * + * `tests` — ES module source in the A.8.6.2 subset, registering the + * executable tests. REQUIRED in a vitest@1 document. + */ + tests?: string + /** Optional ES module source, instantiated first; importable as `"suite:module"`. */ + module?: string } diff --git a/src/verify.ts b/src/verify.ts index 9975571..efb52ef 100644 --- a/src/verify.ts +++ b/src/verify.ts @@ -7,6 +7,7 @@ import { Observer, normalizeTarget, type ObserverOpts } from './http.js' import { observeTarget, deriveDiscovery } from './discovery.js' +import type { ExecSuiteRunner } from './exec/dialect.js' import { runChecks } from './checks.js' import { axScoreOf, gradeOf } from './grade.js' import { attestReport } from './attest.js' @@ -48,6 +49,14 @@ export interface VerifyTargetOpts extends ObserverOpts { /** Held-out signing key. Only honored in remote mode. */ signingKeys?: CryptoKeyPair allowPrivateTargets?: boolean + /** + * The `api.qa/vitest@1` execution seam (A.8.6). Deployed Worker: the Worker + * Loader runner, when provisioned. CLI: the shared-harness local runner. + * Absent: the typed `runner-unavailable` runner — a card declaring the + * executable dialect fails with the reason named, never a crash or a + * silent pass. + */ + execRunner?: ExecSuiteRunner } export async function verifyTarget(target: string, opts: VerifyTargetOpts = {}): Promise { @@ -61,7 +70,7 @@ export async function verifyTarget(target: string, opts: VerifyTargetOpts = {}): // fetched at a private address (the structural SSRF backstop). Same signal // normalizeTarget used above — the deployed Worker leaves it false. const observer = new Observer({ ...opts, allowPrivate }) - const bundle = await observeTarget(normalized.origin, observer, seed) + const bundle = await observeTarget(normalized.origin, observer, seed, { execRunner: opts.execRunner }) const discovery = await deriveDiscovery(bundle) const checks = runChecks(bundle) const axScore = axScoreOf(checks) diff --git a/src/worker.ts b/src/worker.ts index 24a3421..7a770f5 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -21,6 +21,8 @@ */ import { verifyTarget, rejudge } from './verify.js' +import type { ExecSuiteRunner } from './exec/dialect.js' +import { unavailableExecRunner, workerLoaderExecRunner, type WorkerLoaderLike } from './exec/runner.js' import { verifyPinnedSpec, verifySuite, parseSuite, type PinnedReport, type SuiteReport } from './pinned.js' import { reportMarkdown, pinnedMarkdown, suiteMarkdown } from './render.js' import { landingHtml, reportPageHtml } from './views.js' @@ -146,6 +148,23 @@ export interface Env { TS_RAW_CAP?: string /** Time-series hourly rollup-bucket cap before oldest are evicted. */ TS_ROLLUP_CAP?: string + /** + * Dynamic Worker Loader binding (`worker_loaders` in wrangler.jsonc) — the + * `api.qa/vitest@1` isolate runner (A.8.6.3). OPEN-BETA, PAID-PLAN: the + * binding is documented but NOT enabled in the shipped config, so every + * account keeps valid deploys; absent, a card declaring the executable + * dialect fails with a typed `runner-unavailable` reason. + */ + SUITE_LOADER?: WorkerLoaderLike + /** + * The egress gateway service binding every isolate fetch is delivered to + * (`globalOutbound`). REQUIRED alongside SUITE_LOADER: without it the + * runner refuses to run rather than inherit this worker's own network + * access (the A.8.6.3 floor). The gateway's fetch handler is + * `gatewayFetch` (src/exec/runner.ts) — a thin service wrapper at deploy + * time; the floor logic itself is in-repo and unit-tested. + */ + SUITE_OUTBOUND?: unknown } /** Summary of one scheduled tick — returned for tests/observability. */ @@ -309,6 +328,23 @@ export function createApp( const routed: Fetcher = (u, init) => u.startsWith(SELF_ORIGIN) ? loopback(u, init) : (opts.externalFetcher ?? fetch)(u, init) + // The `api.qa/vitest@1` execution seam (A.8.6) — FEATURE-DETECTED. The + // Worker Loader binding is an open-beta, paid-plan capability, so the + // deployment stays valid without it (wrangler.jsonc documents, but does not + // enable, the binding). Three states, all typed and none a crash: + // binding + outbound gateway present → the isolate runner; + // binding present, outbound absent → runner-unavailable (running + // without a gateway would inherit THIS worker's network, which the + // A.8.6.3 floor forbids — refuse to run open); + // binding absent → runner-unavailable. + // A card declaring `runner: "api.qa/vitest@1"` on an unprovisioned + // deployment therefore FAILS the check with the reason named — the same + // direction the ratified unknown-runner rule gives an older verifier — + // never a silent pass. + const execRunner: ExecSuiteRunner = env.SUITE_LOADER + ? workerLoaderExecRunner(env.SUITE_LOADER, { outbound: env.SUITE_OUTBOUND }) + : unavailableExecRunner() + /** * The actual tick body: claim + re-verify every DUE monitor through the * SAME attested verifyTarget/verifySuite/cooldown/SSRF path a fetch run @@ -389,6 +425,7 @@ export function createApp( delayMs: isSelf ? 0 : externalDelayMs, signingKeys: await keys(), allowPrivateTargets: env.ALLOW_PRIVATE_TARGETS === 'true', + execRunner, }) let suiteVerdict: boolean | undefined @@ -688,6 +725,7 @@ export function createApp( delayMs: isSelf ? 0 : externalDelayMs, signingKeys: await keys(), allowPrivateTargets: env.ALLOW_PRIVATE_TARGETS === 'true', + execRunner, }) if (!bypass && cache) await cache.putDomain(domain, report, now()) return respondReport(report, accept, { cache: bypass ? undefined : 'MISS' }) @@ -777,6 +815,7 @@ export function createApp( seed: body.seed, signingKeys: await keys(), allowPrivateTargets: env.ALLOW_PRIVATE_TARGETS === 'true', + execRunner, }) if (!bypass && cache) await cache.putDomain(body.target, report, now()) return respondReport(report, accept, { cache: bypass ? undefined : 'MISS' }) diff --git a/test/cli-vitest.test.ts b/test/cli-vitest.test.ts new file mode 100644 index 0000000..4492237 --- /dev/null +++ b/test/cli-vitest.test.ts @@ -0,0 +1,61 @@ +/** + * The `vitest` CLI verb — LOCAL parity for the executable dialect (A.8.6.2). + * + * Spawns the REAL compiled CLI against the self-contained fixture suite (no + * network — the fixture asserts over `suite:env` / `suite:module` only), and + * pins the load-bearing process behavior a pipeline gates on: exit 0 on all + * pass, exit 1 on a failing test, and exit 1 fail-closed on a digest + * mismatch. The run goes through the SAME shared harness the hosted verifier + * executes, so "green here" is "green hosted" by construction. + */ +import { describe, it, expect, beforeAll } from 'vitest' +import { spawnSync, execSync } from 'node:child_process' +import { existsSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { join } from 'node:path' +import { sha256HexSync } from '../src/sha256-sync.js' +import { readFileSync } from 'node:fs' + +const repoRoot = fileURLToPath(new URL('..', import.meta.url)) +const bin = join(repoRoot, 'dist', 'cli', 'index.js') +const fixture = join(repoRoot, 'test', 'fixtures', 'vitest-suite.json') + +function run(args: string[]): { status: number; stdout: string; stderr: string } { + const r = spawnSync(process.execPath, [bin, ...args], { cwd: repoRoot, encoding: 'utf8' }) + return { status: r.status ?? -1, stdout: r.stdout ?? '', stderr: r.stderr ?? '' } +} + +describe('autonomous-qa vitest — local executable-suite runner', () => { + beforeAll(() => { + execSync('npm run build', { cwd: repoRoot, stdio: 'ignore' }) + expect(existsSync(bin)).toBe(true) + }, 120_000) + + it('runs a self-contained suite to green, exit 0', () => { + const r = run(['vitest', fixture, '--target', 'https://example.com', '--seed', '1']) + expect(r.status).toBe(0) + expect(r.stdout).toContain('2/2 tests passed') + }) + + it('honors --expect-digest fail-closed: a wrong pin exits 1 and instantiates nothing', () => { + const r = run(['vitest', fixture, '--target', 'https://example.com', '--expect-digest', `sha256:${'0'.repeat(64)}`]) + expect(r.status).toBe(1) + expect(r.stderr).toContain('digest mismatch') + }) + + it('accepts the CORRECT digest pin and exits 0', () => { + const digest = `sha256:${sha256HexSync(readFileSync(fixture, 'utf8'))}` + const r = run(['vitest', fixture, '--target', 'https://example.com', '--expect-digest', digest]) + expect(r.status).toBe(0) + }) + + it('--json emits the typed outcome (runner, digest, seed, per-test results)', () => { + const r = run(['vitest', fixture, '--target', 'https://example.com', '--seed', '9', '--json']) + expect(r.status).toBe(0) + const parsed = JSON.parse(r.stdout) as { runner: string; seed: number; outcome: { status: string; results: unknown[] } } + expect(parsed.runner).toBe('api.qa/vitest@1') + expect(parsed.seed).toBe(9) + expect(parsed.outcome.status).toBe('ran') + expect(parsed.outcome.results).toHaveLength(2) + }) +}) diff --git a/test/fixtures/vitest-suite.json b/test/fixtures/vitest-suite.json new file mode 100644 index 0000000..bc3f469 --- /dev/null +++ b/test/fixtures/vitest-suite.json @@ -0,0 +1,8 @@ +{ + "$type": "Suite", + "name": "fixture executable suite", + "version": "1.0.0", + "environments": { "public": { "vars": { "expected": 42 } } }, + "module": "export const answer = () => 6 * 7\n", + "tests": "import { describe, it, expect } from 'vitest'\nimport { vars, seed } from 'suite:env'\nimport { answer } from 'suite:module'\n\ndescribe('a self-contained api.qa/vitest@1 fixture', () => {\n it('runs the subset with no network at all', () => {\n expect(answer()).toBe(vars.expected)\n expect(typeof seed).toBe('number')\n })\n it('exercises core matchers, .not, and async', async () => {\n expect([1, 2, 3]).toHaveLength(3)\n expect({ a: { b: 1 } }).toMatchObject({ a: { b: 1 } })\n expect(7).not.toBe(8)\n await expect(Promise.resolve('ok')).resolves.toBe('ok')\n })\n})\n" +} diff --git a/test/vitest-subset.test.ts b/test/vitest-subset.test.ts new file mode 100644 index 0000000..cc18aae --- /dev/null +++ b/test/vitest-subset.test.ts @@ -0,0 +1,642 @@ +/** + * The `api.qa/vitest@1` SUBSET HARNESS + DIALECT LAYER (AXP A.8.6.2/A.8.6.3). + * + * What is pinned here, in dependency order: + * + * 1. **One implementation, byte-pinned.** The generated string constant the + * hosted isolate's module map carries is byte-identical to the canonical + * `src/exec/vitest-subset.mjs` — the A.8.6.2 "one module, never a + * reimplementation" law, enforced as a drift test exactly like the + * shipped-skill copy. + * 2. **The guaranteed subset behaves** — describe/it/expect, async, + * rejects/resolves, `.not`, registration order — and everything OUTSIDE + * the subset fails BY NAME (snapshots, the `vi` surface, unknown + * matchers), never by silently diverging. + * 3. **The closed import surface** — `"vitest"`, `"suite:env"`, + * `"suite:module"` (document-with-module only); `node:` builtins, bare + * specifiers, dynamic `import()`, `eval`, `new Function` are refused + * with the construct named. + * 4. **The execution invariants of the LOCAL runner** (the parity reference + * for the hosted isolate): digest-independent here — the observe side + * gates digests — but the floor, the sandbox verb gate, the metered + * breaker, the combined/output caps, non-vacuity, and seeded + * determinism all fail closed with named reasons. + */ + +import { describe, it, expect } from 'vitest' +import { readFileSync } from 'node:fs' +import { VITEST_SUBSET_SOURCE } from '../src/exec/vitest-subset-source.js' +import { + EXEC_CPU_MS, + EXEC_WALL_MS, + createGatedFetch, + isFloorBlockedHost, + loadHarnessModule, + localExecRunner, + validateDialectSource, + type ExecRunRequest, + type GateViolation, +} from '../src/exec/dialect.js' +import { + RUNNER_UNAVAILABLE_NO_BINDING, + RUNNER_UNAVAILABLE_NO_OUTBOUND, + buildWorkerCode, + gatewayFetch, + unavailableExecRunner, + workerLoaderExecRunner, + type WorkerCodeLike, +} from '../src/exec/runner.js' + +const ORIGIN = 'https://target.example' + +/** A run request with sane defaults; override what the case needs. */ +function req(overrides: Partial & Pick): ExecRunRequest { + return { + artifactKind: 'document', + origin: ORIGIN, + vars: {}, + environment: 'public', + sandbox: false, + seed: 7, + declarativeRows: 0, + ...overrides, + } +} + +/** A mock external fetch: any publicly-routable host answers 200 JSON. */ +const okFetch = async (url: string): Promise => + new Response(JSON.stringify({ ok: true, url }), { status: 200, headers: { 'content-type': 'application/json' } }) + +// --------------------------------------------------------------------------- +// 1. One implementation, byte-pinned +// --------------------------------------------------------------------------- + +describe('the shared harness is ONE module (A.8.6.2, normative)', () => { + it('the generated source constant byte-matches the canonical vitest-subset.mjs', () => { + const canonical = readFileSync(new URL('../src/exec/vitest-subset.mjs', import.meta.url), 'utf8') + // Drifted? Regenerate: node scripts/gen-vitest-subset.mjs — the hosted + // isolate executes the CONSTANT, the repo reviews the FILE; they must be + // the same bytes or "one implementation" is a lie. + expect(VITEST_SUBSET_SOURCE).toBe(canonical) + }) + + it('the canonical harness is self-contained: zero import statements', () => { + // The bytes go verbatim into an isolate module map where nothing else + // resolves; a single import would break the hosted runner. + expect(/^\s*import\s/m.test(VITEST_SUBSET_SOURCE)).toBe(false) + }) +}) + +// --------------------------------------------------------------------------- +// 2. The guaranteed subset — and named failure outside it +// --------------------------------------------------------------------------- + +describe('the subset harness', () => { + it('runs registered tests SEQUENTIALLY in registration order with nested describe names', async () => { + const { createHarness } = await loadHarnessModule() + const h = createHarness() + const api = h.api as { + describe: (n: string, f: () => void) => void + it: (n: string, f: () => unknown) => void + expect: (v: unknown) => Record void> + } + const order: string[] = [] + api.describe('outer', () => { + api.it('first', async () => { + order.push('first') + }) + api.describe('inner', () => { + api.it('second', () => { + order.push('second') + }) + }) + }) + api.it('third', () => { + order.push('third') + }) + const { registered, results } = await h.run() + expect(registered).toBe(3) + expect(order).toEqual(['first', 'second', 'third']) + expect(results.map((r) => r.name)).toEqual(['outer > first', 'outer > inner > second', 'third']) + expect(results.every((r) => r.status === 'pass' && typeof r.durationMs === 'number')).toBe(true) + }) + + it('core matchers judge, .not negates, a failure carries the matcher and values', async () => { + const { createHarness } = await loadHarnessModule() + const h = createHarness() + const api = h.api as { it: (n: string, f: () => unknown) => void; expect: (v: unknown) => any } + api.it('passes', () => { + api.expect(2).toBe(2) + api.expect({ a: [1, { b: 2 }] }).toEqual({ a: [1, { b: 2 }] }) + api.expect([1, 2]).toContain(2) + api.expect({ a: 1, b: 2 }).toMatchObject({ a: 1 }) + api.expect(3).not.toBe(4) + api.expect('abc').toMatch(/b/) + api.expect(3.14159).toBeCloseTo(3.14, 2) + }) + api.it('fails', () => { + api.expect(2).toBe(3) + }) + const { results } = await h.run() + expect(results[0]!.status).toBe('pass') + expect(results[1]!.status).toBe('fail') + expect(results[1]!.reason).toContain('toBe') + expect(results[1]!.reason).toContain('2') + expect(results[1]!.reason).toContain('3') + }) + + it('rejects/resolves are awaited async assertion chains', async () => { + const { createHarness } = await loadHarnessModule() + const h = createHarness() + const api = h.api as { it: (n: string, f: () => unknown) => void; expect: (v: unknown) => any } + api.it('async assertions', async () => { + await api.expect(Promise.resolve(41 + 1)).resolves.toBe(42) + await api.expect(Promise.reject(new Error('boom goes the door'))).rejects.toThrow('boom') + }) + api.it('a resolved promise fails rejects', async () => { + await api.expect(Promise.resolve(1)).rejects.toThrow() + }) + const { results } = await h.run() + expect(results[0]!.status).toBe('pass') + expect(results[1]!.status).toBe('fail') + expect(results[1]!.reason).toContain('expected promise to reject') + }) + + it('snapshot matchers fail BY NAME — snapshot state cannot live in a pinned document', async () => { + const { createHarness } = await loadHarnessModule() + const h = createHarness() + const api = h.api as { it: (n: string, f: () => unknown) => void; expect: (v: unknown) => any } + api.it('snap', () => { + api.expect({ a: 1 }).toMatchSnapshot() + }) + const { results } = await h.run() + expect(results[0]!.status).toBe('fail') + expect(results[0]!.reason).toContain('toMatchSnapshot') + expect(results[0]!.reason).toContain('outside the api.qa/vitest@1 subset') + }) + + it('the whole vi surface is poisoned: any property access fails naming vi.', async () => { + const { createHarness } = await loadHarnessModule() + const h = createHarness() + const api = h.api as { it: (n: string, f: () => unknown) => void; vi: Record } + api.it('mocks', () => { + void (api.vi as { useFakeTimers: () => void }).useFakeTimers + }) + const { results } = await h.run() + expect(results[0]!.status).toBe('fail') + expect(results[0]!.reason).toContain('vi.useFakeTimers') + }) +}) + +// --------------------------------------------------------------------------- +// 3. The closed import surface +// --------------------------------------------------------------------------- + +describe('validateDialectSource — imports closed to exactly three specifiers', () => { + const what = { allowSuiteModule: false, what: 'the suite document `tests` member' } + + it('accepts "vitest" and "suite:env"', () => { + const v = validateDialectSource(`import { describe } from 'vitest'\nimport { origin } from 'suite:env'\n`, what) + expect(v.ok).toBe(true) + }) + + it.each([ + ["import fs from 'node:fs'", 'node:fs', 'node: built-ins'], + ["import axios from 'axios'", 'axios', 'bare package specifiers'], + ["import x from './sibling.js'", './sibling.js', 'relative/path imports'], + ])('refuses %s naming the specifier', (src, spec, why) => { + const v = validateDialectSource(src, what) + expect(v.ok).toBe(false) + if (!v.ok) { + expect(v.problem).toContain(spec) + expect(v.problem).toContain(why) + } + }) + + it('refuses "suite:module" unless the document carries a `module` member', () => { + const src = "import { helper } from 'suite:module'" + const closed = validateDialectSource(src, what) + expect(closed.ok).toBe(false) + const open = validateDialectSource(src, { allowSuiteModule: true, what: 'x' }) + expect(open.ok).toBe(true) + }) + + it.each([ + ['await import("vitest")', 'dynamic import()'], + ['eval("1+1")', 'eval'], + ['new Function("return 1")', 'new Function'], + ])('refuses runtime code paths: %s', (src, name) => { + const v = validateDialectSource(`it('x', () => { ${src} })`, what) + expect(v.ok).toBe(false) + if (!v.ok) expect(v.problem).toContain(name) + }) +}) + +// --------------------------------------------------------------------------- +// The network floor — and NOTHING above it +// --------------------------------------------------------------------------- + +describe('isFloorBlockedHost — the A.8.6.3 refusal set, and no more', () => { + it.each([ + '169.254.169.254', // cloud metadata + '2852039166', // the same address, decimal-encoded + '169.254.0.7', // link-local + '127.0.0.1', + 'localhost', + '10.1.2.3', + '172.16.9.9', + '192.168.1.1', // RFC 1918 + '100.64.0.1', + '100.127.255.254', // CGNAT 100.64/10 + 'metadata.google.internal', + 'cooldown.internal', // estate-internal hostnames + 'suite-gateway', // single-label service names + '[fe80::1]', + '[fd00::1]', // link-local / ULA v6 + // Raw IP-literal hosts are barred CATEGORICALLY (inherited from the + // verifier's own SSRF gate): a public service is reached by name, and a + // literal is exactly the shape every encoding bypass arrives in. This is + // over-broad ONLY in the closed direction — public DNS names are what the + // floor leaves open. + '100.1.2.3', + '8.8.8.8', + ])('bars %s', (host) => { + expect(isFloorBlockedHost(host)).toBe(true) + }) + + it.each([ + 'api.example', // ordinary public host + 'other-estate.example', // cross-estate composition is a FEATURE + 'pkg.do', // the module CDN + ])('permits %s — full external egress above the floor', (host) => { + expect(isFloorBlockedHost(host)).toBe(false) + }) +}) + +describe('createGatedFetch', () => { + it('re-floors every redirect hop — a public host cannot 302 the run into metadata', async () => { + const violations: GateViolation[] = [] + const gated = createGatedFetch({ + realFetch: async (url) => + url.startsWith('https://public.example') + ? new Response(null, { status: 302, headers: { location: 'http://169.254.169.254/latest/meta-data/' } }) + : okFetch(url), + sandbox: false, + violations, + }) + await expect(gated('https://public.example/hop')).rejects.toThrow(/network floor/) + expect(violations).toHaveLength(1) + expect(violations[0]!.reason).toContain('169.254.169.254') + }) + + it('follows a public→public redirect and returns the final response', async () => { + const violations: GateViolation[] = [] + const gated = createGatedFetch({ + realFetch: async (url) => + url === 'https://a.example/' + ? new Response(null, { status: 301, headers: { location: 'https://b.example/final' } }) + : okFetch(url), + sandbox: false, + violations, + }) + const res = await gated('https://a.example/') + expect(((await res.json()) as { url: string }).url).toBe('https://b.example/final') + expect(violations).toHaveLength(0) + }) +}) + +// --------------------------------------------------------------------------- +// 4. The local runner — the parity reference +// --------------------------------------------------------------------------- + +describe('localExecRunner — document form', () => { + it('runs an import-form suite end-to-end: suite:env + external egress + assertions', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ + testsSource: ` +import { describe, it, expect } from 'vitest' +import { origin, vars, seed, sandbox } from 'suite:env' + +describe('the wired world', () => { + it('sees the selected environment', () => { + expect(origin).toBe('${ORIGIN}') + expect(vars.token).toBe('t-123') + expect(seed).toBe(7) + expect(sandbox).toBe(false) + }) + it('calls its own origin', async () => { + const r = await fetch(origin + '/api/status') + expect(r.status).toBe(200) + }) + it('calls ANOTHER estate — cross-origin egress is a feature, not a leak', async () => { + const r = await fetch('https://other-estate.example/compose') + await expect(r.json()).resolves.toMatchObject({ ok: true }) + }) +}) +`, + vars: { token: 't-123' }, + }), + ) + expect(outcome.status).toBe('ran') + if (outcome.status === 'ran') { + expect(outcome.registered).toBe(3) + expect(outcome.results.every((r) => r.status === 'pass')).toBe(true) + expect(outcome.appliedLimits).toEqual({ wallMs: EXEC_WALL_MS, cpuMs: EXEC_CPU_MS }) + } + }) + + it('the GLOBALS form runs with no import line at all (document form, A.8.6.2)', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ + testsSource: ` +describe('globals', () => { + it('describe/it/expect are ambient', () => { + expect(1 + 1).toBe(2) + }) +}) +`, + }), + ) + expect(outcome.status).toBe('ran') + if (outcome.status === 'ran') expect(outcome.results[0]!.status).toBe('pass') + }) + + it('a `module` member instantiates first and is importable as "suite:module"', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ + moduleSource: `export const checkDigit = (n) => (n * 3) % 10\nexport const label = 'gs1'\n`, + testsSource: ` +import { it, expect } from 'vitest' +import { checkDigit, label } from 'suite:module' + +it('uses the suite module', () => { + expect(checkDigit(4)).toBe(2) + expect(label).toBe('gs1') +}) +`, + }), + ) + expect(outcome.status).toBe('ran') + if (outcome.status === 'ran') expect(outcome.results[0]!.status).toBe('pass') + }) + + it('a failing expectation yields status ran with the test failed and the reason named', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ testsSource: `it('wrong', () => { expect(2).toBe(3) })` }), + ) + expect(outcome.status).toBe('ran') + if (outcome.status === 'ran') { + expect(outcome.results[0]!.status).toBe('fail') + expect(outcome.results[0]!.reason).toContain('toBe') + } + }) + + it('Math.random is SEEDED: same seed same draws, different seed different draws (A.8.6.4)', async () => { + const src = `it('draw', () => { globalThis.__vitest1_draw = Math.random() })` + const g = globalThis as Record + await localExecRunner({ fetch: okFetch }).run(req({ testsSource: src, seed: 42 })) + const first = g.__vitest1_draw + await localExecRunner({ fetch: okFetch }).run(req({ testsSource: src, seed: 42 })) + const second = g.__vitest1_draw + await localExecRunner({ fetch: okFetch }).run(req({ testsSource: src, seed: 43 })) + const third = g.__vitest1_draw + delete g.__vitest1_draw + expect(typeof first).toBe('number') + expect(second).toBe(first) + expect(third).not.toBe(first) + }) + + it('re-running the SAME bytes re-registers (module caching cannot spend a suite)', async () => { + const src = `it('x', () => { expect(true).toBeTruthy() })` + const a = await localExecRunner({ fetch: okFetch }).run(req({ testsSource: src })) + const b = await localExecRunner({ fetch: okFetch }).run(req({ testsSource: src })) + expect(a.status).toBe('ran') + expect(b.status).toBe('ran') + if (b.status === 'ran') expect(b.registered).toBe(1) + }) +}) + +describe('localExecRunner — fail-closed totality (A.8.6.3)', () => { + it('a fetch toward cloud metadata FAILS THE RUN by a named reason', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ testsSource: `it('steal', async () => { await fetch('http://169.254.169.254/latest/meta-data/') })` }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') { + expect(outcome.reason).toContain('network floor') + expect(outcome.reason).toContain('169.254.169.254') + } + }) + + it('an RFC1918 fetch fails the run even when the suite CATCHES the throw — no swallowing a refusal into a pass', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ + testsSource: `it('swallow', async () => { try { await fetch('http://10.0.0.8/internal') } catch {} expect(1).toBe(1) })`, + }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') expect(outcome.reason).toContain('10.0.0.8') + }) + + it('a mutating verb outside a sandbox environment fails the run by a named reason (A.8.6.4)', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ testsSource: `it('write', async () => { await fetch('${ORIGIN}/things', { method: 'POST', body: '{}' }) })` }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') expect(outcome.reason).toContain('sandbox') + }) + + it('the SAME mutating verb is permitted when the environment declares sandbox: true', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ + sandbox: true, + testsSource: `it('write', async () => { const r = await fetch('${ORIGIN}/things', { method: 'POST', body: '{}' }); expect(r.status).toBe(200) })`, + }), + ) + expect(outcome.status).toBe('ran') + if (outcome.status === 'ran') expect(outcome.results[0]!.status).toBe('pass') + }) + + it('the metered circuit-breaker trips the WHOLE run — never a partial verdict', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ + limits: { wallMs: 50 }, + testsSource: ` +it('quick', () => { expect(1).toBe(1) }) +it('hangs', async () => { await new Promise((r) => setTimeout(r, 60_000)) }) +`, + }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') { + expect(outcome.reason).toContain('circuit breaker') + expect(outcome.reason).toContain('50 ms') + } + }, 10_000) + + it('an all-of-nothing is refused: zero tests + zero rows fails (non-vacuity over the union)', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ testsSource: `export const nothing = true\n` }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') expect(outcome.reason).toContain('non-vacuity') + }) + + it('a subset violation refuses BEFORE anything executes', async () => { + const g = globalThis as Record + delete g.__vitest1_ran + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ testsSource: `globalThis.__vitest1_ran = true\nimport fs from 'node:fs'\nit('x', () => {})` }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') expect(outcome.reason).toContain('node:fs') + expect(g.__vitest1_ran).toBeUndefined() + }) +}) + +describe('localExecRunner — module artifact form', () => { + it('a module artifact with a named `export` registers through that nullary export', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ + artifactKind: 'module', + exportName: 'suite', + testsSource: ` +import { it, expect } from 'vitest' +export function suite() { + it('registered via the export seam', () => { expect('sdk').toHaveLength(3) }) +} +export const unrelatedSdkSurface = 42 +`, + }), + ) + expect(outcome.status).toBe('ran') + if (outcome.status === 'ran') { + expect(outcome.registered).toBe(1) + expect(outcome.results[0]!.status).toBe('pass') + } + }) + + it('a missing named export fails by name', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ artifactKind: 'module', exportName: 'suite', testsSource: `export const notASuite = 1\n` }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') expect(outcome.reason).toContain('"suite"') + }) + + it('a module artifact is ALWAYS non-sandbox: a mutating verb fails there (implicit "public")', async () => { + const outcome = await localExecRunner({ fetch: okFetch }).run( + req({ + artifactKind: 'module', + testsSource: ` +import { it } from 'vitest' +it('write', async () => { await fetch('${ORIGIN}/x', { method: 'DELETE' }) }) +`, + }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') expect(outcome.reason).toContain('sandbox') + }) +}) + +// --------------------------------------------------------------------------- +// The hosted runner surface — WorkerCode assembly + feature detection +// --------------------------------------------------------------------------- + +describe('the hosted Worker Loader runner (assembly + posture; the platform binding is flag-held)', () => { + const request = req({ testsSource: `it('x', () => { expect(1).toBe(1) })`, digest: 'sha256:' + 'ab'.repeat(32) }) + + it('buildWorkerCode: ZERO ambient authority — env is the empty object, always', () => { + const code = buildWorkerCode(request, { stub: true }) + expect(code.env).toEqual({}) + }) + + it('buildWorkerCode: globalOutbound is SET (never inherit the verifier network) and limits carry the CPU breaker', () => { + const code = buildWorkerCode(request, { stub: true }) + expect(code.globalOutbound).toEqual({ stub: true }) + expect(code.limits).toEqual({ cpuMs: EXEC_CPU_MS }) + }) + + it('buildWorkerCode: the module map carries the SHARED harness bytes verbatim and the pinned suite bytes verbatim', () => { + const code = buildWorkerCode(request, { stub: true }) + expect(code.modules['./harness.mjs']!.js).toBe(VITEST_SUBSET_SOURCE) + expect(code.modules['./suite-tests.mjs']!.js).toBe(request.testsSource) + expect(Object.keys(code.modules)).toContain('vitest') + expect(Object.keys(code.modules)).toContain('suite:env') + }) + + it('a runner without a loader binding reports the TYPED runner-unavailable outcome — never a crash', async () => { + const outcome = await unavailableExecRunner().run(request) + expect(outcome).toEqual({ status: 'runner-unavailable', reason: RUNNER_UNAVAILABLE_NO_BINDING }) + }) + + it('a loader WITHOUT an outbound gateway refuses to run open (would inherit the verifier network)', async () => { + const loader = { + get: () => ({ getEntrypoint: () => ({ fetch: async () => new Response('{}') }) }), + } + const outcome = await workerLoaderExecRunner(loader, {}).run(request) + expect(outcome).toEqual({ status: 'runner-unavailable', reason: RUNNER_UNAVAILABLE_NO_OUTBOUND }) + }) + + it('with a (fake) loader + outbound, the parent folds the isolate response through the SAME totality fold', async () => { + let builtCode: WorkerCodeLike | undefined + const loader = { + get: (_id: string, getCode: () => WorkerCodeLike | Promise) => ({ + getEntrypoint: () => ({ + fetch: async () => { + builtCode = await getCode() + return new Response( + JSON.stringify({ + registered: 1, + results: [{ name: 'x', status: 'pass', durationMs: 1 }], + violations: [], + }), + { headers: { 'content-type': 'application/json' } }, + ) + }, + }), + }), + } + const outcome = await workerLoaderExecRunner(loader, { outbound: { stub: true } }).run(request) + expect(outcome.status).toBe('ran') + if (outcome.status === 'ran') expect(outcome.results[0]!.name).toBe('x') + expect(builtCode?.env).toEqual({}) + expect(builtCode?.globalOutbound).toEqual({ stub: true }) + }) + + it('an in-isolate violation reported by the entry FAILS the run with the reason named', async () => { + const loader = { + get: () => ({ + getEntrypoint: () => ({ + fetch: async () => + new Response( + JSON.stringify({ + registered: 1, + results: [{ name: 'x', status: 'pass', durationMs: 1 }], + violations: [{ url: 'http://10.0.0.8/', reason: 'floor: refused 10.0.0.8' }], + }), + { headers: { 'content-type': 'application/json' } }, + ), + }), + }), + } + const outcome = await workerLoaderExecRunner(loader, { outbound: {} }).run(request) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') expect(outcome.reason).toContain('10.0.0.8') + }) +}) + +describe('gatewayFetch — the parent-owned egress gateway', () => { + it('bars a metadata request with a BLOCKED response (the isolate sees an ordinary failed fetch)', async () => { + const res = await gatewayFetch(new Request('http://169.254.169.254/latest/meta-data/'), okFetch) + expect(res.status).toBe(403) + const body = (await res.json()) as { type: string; reason: string } + expect(body.type).toBe('BLOCKED') + expect(body.reason).toContain('network floor') + }) + + it('passes an external public destination through untouched', async () => { + const res = await gatewayFetch(new Request('https://other-estate.example/compose'), okFetch) + expect(res.status).toBe(200) + }) +}) diff --git a/test/vitest-suite-interface.test.ts b/test/vitest-suite-interface.test.ts new file mode 100644 index 0000000..d644849 --- /dev/null +++ b/test/vitest-suite-interface.test.ts @@ -0,0 +1,361 @@ +/** + * The card-declared `api.qa/vitest@1` EXECUTABLE suite, end-to-end through the + * grading path (observe → judge), and the widened `interfaces.testSuite` seam. + * + * The observe side runs the isolate seam (here the LOCAL shared-harness + * runner — local==hosted by construction, A.8.6.2) and records the typed + * outcome as synthetic evidence; the judge (`runChecks`, pure) folds the + * declarative rows and the executed tests into one verdict from the bundle + * alone. Every property the ruling set fixes is asserted from the OUTSIDE: + * + * - undeclared ⇒ skip (omission is conformance); + * - a passing executable suite ⇒ pass, with the A.8.6.5 attestation surface + * in the detail (digest, kind, environment, seed, breaker limits, counts); + * - digest fail-closed: a tampered byte ⇒ fail, NOTHING instantiated; + * - a failing registered test ⇒ fail naming the test; + * - the network floor blocks metadata/RFC1918 but permits external egress; + * - runner-unavailable ⇒ fail with the typed reason, never a crash/skip; + * - the seam: defective declarations fail (package w/o version, package + * under the declarative runner, export on a document, unknown runner). + */ + +import { describe, it, expect } from 'vitest' +import { Observer } from '../src/http.js' +import { observeTarget, ROLE, parseAgentsJson } from '../src/discovery.js' +import { runChecks } from '../src/checks.js' +import { sha256HexSync } from '../src/sha256-sync.js' +import { localExecRunner } from '../src/exec/dialect.js' +import { unavailableExecRunner } from '../src/exec/runner.js' +import { VITEST_RUNNER, gateVitestSuiteCard } from '../src/test-suite.js' +import type { CheckResult } from '../src/types.js' +import { GOOD, goodTargetRoutes, makeFetcher, withOverrides, type Routes } from './helpers.js' + +const SUITE_PATH = '/.well-known/axp/suite.mjs.json' // a suite DOCUMENT (not .mjs) + +const json = (value: unknown) => () => ({ status: 200, contentType: 'application/json', body: JSON.stringify(value) }) + +/** An executable suite document the reference target genuinely passes. */ +function execDoc(extra: Record = {}) { + return { + $type: 'Suite', + name: 'good.example workflows', + version: '1.0.0', + environments: { public: { vars: { token: 't-1' } } }, + tests: ` +import { describe, it, expect } from 'vitest' +import { origin, vars } from 'suite:env' + +describe('status → widgets workflow', () => { + it('status is ok', async () => { + const r = await fetch(origin + '/api/status') + expect(r.status).toBe(200) + const b = await r.json() + expect(b.ok).toBe(true) + expect(b.widgets).toBeGreaterThan(0) + }) + it('widgets lists, and the environment var is wired', async () => { + expect(vars.token).toBe('t-1') + const r = await fetch(origin + '/api/widgets') + const list = await r.json() + expect(list).toHaveLength(3) + expect(list[0].id).toBe('w1') + }) +}) +`, + ...extra, + } +} + +function routesFor(opts: { + suite?: unknown + declaration?: unknown + digest?: string + suitePath?: string + serveSuite?: boolean + suiteBody?: { status: number; contentType: string; body: string } +} = {}): { routes: Routes; suiteText: string } { + const base = goodTargetRoutes() + const path = opts.suitePath ?? SUITE_PATH + const suite = opts.suite ?? execDoc() + const suiteText = typeof suite === 'string' ? suite : JSON.stringify(suite) + const card = JSON.parse( + base['GET /.well-known/agents.json']!({ method: 'GET', accept: 'application/json' }).body!, + ) as Record + card.interfaces.testSuite = + opts.declaration ?? { + url: path, + digest: opts.digest ?? `sha256:${sha256HexSync(suiteText)}`, + runner: VITEST_RUNNER, + } + const suiteRoute: Routes = {} + if (opts.serveSuite !== false) { + suiteRoute[`GET ${path}`] = opts.suiteBody ? () => opts.suiteBody! : () => ({ status: 200, contentType: 'application/json', body: suiteText }) + } + return { + routes: withOverrides(base, { 'GET /.well-known/agents.json': json(card), ...suiteRoute }), + suiteText, + } +} + +async function judge(routes: Routes, execRunner = localExecRunner()) { + const calls: string[] = [] + const inner = makeFetcher(routes) + const observer = new Observer({ + fetcher: async (url, init) => { + calls.push(url) + return inner(url, init) + }, + delayMs: 0, + }) + const bundle = await observeTarget(GOOD, observer, 7, { execRunner }) + const checks = runChecks(bundle) + return { bundle, checks, calls } +} + +const ts = (checks: CheckResult[]) => checks.find((c) => c.id === 'published-test-suite')! + +// --------------------------------------------------------------------------- +// The passing path + the attestation surface +// --------------------------------------------------------------------------- + +describe('a declared api.qa/vitest@1 suite the surface passes', () => { + it('PASSes, and the detail carries the A.8.6.5 attestation surface', async () => { + const { routes } = routesFor() + const { checks } = await judge(routes) + const c = ts(checks) + expect(c.verdict).toBe('pass') + expect(c.detail).toContain(VITEST_RUNNER) + expect(c.detail).toContain('document artifact') + expect(c.detail).toContain('matches the card pin') + expect(c.detail).toContain('2 registered test(s), all passed') + expect(c.detail).toContain('seed 7') + expect(c.detail).toContain('zero ambient authority') + }) + + it('records the executable run as synthetic evidence the judge reads (replay needs no re-exec)', async () => { + const { routes } = routesFor() + const { bundle, checks } = await judge(routes) + const runEv = bundle.items.find((e) => e.role === ROLE.vitestRun) + expect(runEv).toBeDefined() + const record = JSON.parse(runEv!.body!) as { outcome: { status: string }; executedDigest: string } + expect(record.outcome.status).toBe('ran') + // Re-judging the STORED bundle (no observe, no isolate) reaches the same verdict. + const replay = runChecks(bundle) + expect(ts(replay).verdict).toBe(ts(checks).verdict) + }) + + it('runs declarative rows AND executable tests folded into one verdict', async () => { + const withRows = execDoc({ + requirements: [ + { id: 'status-row', kind: 'endpoint', method: 'GET', path: '/api/status', expect: { status: 200, paths: [{ path: 'ok', equals: true }] } }, + ], + }) + const { routes } = routesFor({ suite: withRows }) + const { checks } = await judge(routes) + const c = ts(checks) + expect(c.verdict).toBe('pass') + expect(c.detail).toContain('1 declarative row(s)') + }) +}) + +// --------------------------------------------------------------------------- +// Digest fail-closed +// --------------------------------------------------------------------------- + +describe('digest fail-closed (A.8.6.3)', () => { + it('a tampered byte fails and NOTHING is instantiated', async () => { + const suite = execDoc() + const suiteText = JSON.stringify(suite) + const pin = `sha256:${sha256HexSync(suiteText)}` + // Serve a document that does NOT hash to the pinned digest. + const tampered = suiteText.replace('status → widgets workflow', 'status → widgets workflow (edited)') + const { routes } = routesFor({ suite, digest: pin, suiteBody: { status: 200, contentType: 'application/json', body: tampered } }) + const { checks, bundle } = await judge(routes) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('digest mismatch') + // No run outcome was recorded — the gate refused before the runner. + expect(bundle.items.find((e) => e.role === ROLE.vitestRun)).toBeUndefined() + }) +}) + +// --------------------------------------------------------------------------- +// A failing registered test +// --------------------------------------------------------------------------- + +describe('a suite the surface VIOLATES', () => { + it('fails, naming the failing test', async () => { + const bad = execDoc({ + tests: ` +import { it, expect } from 'vitest' +import { origin } from 'suite:env' +it('expects the wrong widget count', async () => { + const r = await fetch(origin + '/api/widgets') + const list = await r.json() + expect(list).toHaveLength(99) +}) +`, + }) + const { routes } = routesFor({ suite: bad }) + const { checks } = await judge(routes) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('violated its OWN published executable suite') + expect(c.detail).toContain('expects the wrong widget count') + }) +}) + +// --------------------------------------------------------------------------- +// The network floor, end-to-end through the check +// --------------------------------------------------------------------------- + +describe('the network floor through the grading path (A.8.6.3)', () => { + it('a test that fetches cloud metadata FAILS the suite by a named reason', async () => { + const evil = execDoc({ + tests: ` +import { it } from 'vitest' +it('reaches for metadata', async () => { await fetch('http://169.254.169.254/latest/meta-data/') }) +`, + }) + const { routes } = routesFor({ suite: evil }) + const { checks } = await judge(routes) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.detail).toContain('network floor') + }) + + it('a test that composes with ANOTHER estate passes — external egress is a feature', async () => { + const composed = execDoc({ + tests: ` +import { it, expect } from 'vitest' +it('calls a sibling estate', async () => { + const r = await fetch('https://other-estate.example/compose') + expect(r.status).toBe(200) +}) +`, + }) + const { routes } = routesFor({ suite: composed }) + // The fixture fetcher only knows GOOD; wrap it so the cross-estate call + // resolves (proving the FLOOR permits it — the refusal would be the floor, + // not the fixture 404). + const inner = makeFetcher(routes) + const observer = new Observer({ + fetcher: async (url, init) => + url.startsWith('https://other-estate.example') + ? new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } }) + : inner(url, init), + delayMs: 0, + }) + const bundle = await observeTarget(GOOD, observer, 7, { execRunner: localExecRunner() }) + expect(ts(runChecks(bundle)).verdict).toBe('pass') + }) +}) + +// --------------------------------------------------------------------------- +// runner-unavailable — the flag-held capability +// --------------------------------------------------------------------------- + +describe('runner-unavailable (the Worker Loader binding not provisioned)', () => { + it('a declared vitest@1 suite FAILS with the typed reason — never a crash, never a skip', async () => { + const { routes } = routesFor() + const { checks } = await judge(routes, unavailableExecRunner()) + const c = ts(checks) + expect(c.verdict).toBe('fail') + expect(c.verdict).not.toBe('skip') + expect(c.detail).toContain('runner-unavailable') + }) + + it('the DEFAULT (no execRunner passed) is runner-unavailable — a deploy without the binding fails closed', async () => { + const { routes } = routesFor() + const inner = makeFetcher(routes) + const observer = new Observer({ fetcher: inner, delayMs: 0 }) + const bundle = await observeTarget(GOOD, observer, 7) // no opts + expect(ts(runChecks(bundle)).verdict).toBe('fail') + }) +}) + +// --------------------------------------------------------------------------- +// The widened seam — defective declarations fail (A.8.5.2) +// --------------------------------------------------------------------------- + +describe('the widened interfaces.testSuite seam', () => { + it('parseAgentsJson reads url/package/version/export/digest/environment/runner', () => { + const card = { + interfaces: { + testSuite: { + url: 'https://pkg.do/apis.vin@1.2.0/index.mjs', + package: 'apis.vin', + version: '1.2.0', + export: 'suite', + digest: `sha256:${'ab'.repeat(32)}`, + runner: VITEST_RUNNER, + }, + }, + } + const claims = parseAgentsJson(card, GOOD) + expect(claims.testSuite?.packageName).toBe('apis.vin') + expect(claims.testSuite?.version).toBe('1.2.0') + expect(claims.testSuite?.exportName).toBe('suite') + expect(claims.testSuite?.runner).toBe(VITEST_RUNNER) + }) + + it('a package WITHOUT a version fails at the card gate', () => { + const claim = parseAgentsJson( + { interfaces: { testSuite: { package: 'apis.vin', digest: `sha256:${'a'.repeat(64)}`, runner: VITEST_RUNNER } } }, + GOOD, + ).testSuite! + const g = gateVitestSuiteCard(claim, GOOD) + expect(g.ok).toBe(false) + if (!g.ok) expect(g.problem).toContain('no `version`') + }) + + it('an OFF-ORIGIN module CDN url is ACCEPTED — the digest, never the host, is the authority (A.8.6.6)', () => { + const claim = parseAgentsJson( + { + interfaces: { + testSuite: { url: 'https://pkg.do/apis.vin@1.2.0/index.mjs', digest: `sha256:${'a'.repeat(64)}`, runner: VITEST_RUNNER }, + }, + }, + GOOD, + ).testSuite! + const g = gateVitestSuiteCard(claim, GOOD) + expect(g.ok).toBe(true) + if (g.ok) { + expect(g.kind).toBe('module') // .mjs pathname ⇒ module, by the card + expect(g.url).toBe('https://pkg.do/apis.vin@1.2.0/index.mjs') + } + }) + + it('a url-less package@version DERIVES the native-serving address (A.8.6.6)', () => { + const claim = parseAgentsJson( + { interfaces: { testSuite: { package: 'apis.vin', version: '2.0.0', digest: `sha256:${'a'.repeat(64)}`, runner: VITEST_RUNNER } } }, + GOOD, + ).testSuite! + const g = gateVitestSuiteCard(claim, GOOD) + expect(g.ok).toBe(true) + if (g.ok) { + expect(g.url).toBe('https://pkg.do/apis.vin@2.0.0/index.mjs') + expect(g.npm).toEqual({ package: 'apis.vin', version: '2.0.0' }) + } + }) + + it('an export on a DOCUMENT artifact fails (export is module-kind-only)', () => { + const claim = parseAgentsJson( + { interfaces: { testSuite: { url: '/suite.json', export: 'suite', digest: `sha256:${'a'.repeat(64)}`, runner: VITEST_RUNNER } } }, + GOOD, + ).testSuite! + const g = gateVitestSuiteCard(claim, GOOD) + expect(g.ok).toBe(false) + if (!g.ok) expect(g.problem).toContain('module export') + }) + + it('a metadata artifact address is refused WITHOUT fetching', () => { + const claim = parseAgentsJson( + { interfaces: { testSuite: { url: 'http://169.254.169.254/suite.json', digest: `sha256:${'a'.repeat(64)}`, runner: VITEST_RUNNER } } }, + GOOD, + ).testSuite! + const g = gateVitestSuiteCard(claim, GOOD) + expect(g.ok).toBe(false) + if (!g.ok) expect(g.problem).toContain('network floor') + }) +}) diff --git a/wrangler.jsonc b/wrangler.jsonc index 03be86a..ddbb885 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -53,6 +53,28 @@ { "binding": "REPORTS", "id": "90eb42161ef045eb8b38b01cf87f985a" } ], + // ── api.qa/vitest@1 executable-suite runner (AXP A.8.6) — FLAG-HELD ────── + // The Dynamic Worker Loader binding is an OPEN-BETA, PAID-PLAN capability: + // enabling it on an unenrolled account breaks `wrangler deploy`, so the + // binding ships DOCUMENTED-BUT-DISABLED and the code feature-detects + // `env.SUITE_LOADER` (absent ⇒ a card declaring runner "api.qa/vitest@1" + // fails with a typed `runner-unavailable` reason — never a crash, never a + // silent pass; see src/exec/runner.ts). To provision, on an enrolled + // account, uncomment BOTH pieces: + // + // "worker_loaders": [ { "binding": "SUITE_LOADER" } ], + // // SUITE_OUTBOUND — a service binding to the egress-gateway worker + // // (its fetch handler wraps `gatewayFetch` from src/exec/runner.ts: + // // the A.8.6.3 network floor, re-checked per redirect hop). REQUIRED: + // // without it the runner REFUSES to execute rather than let an isolate + // // inherit this worker's own network access. + // "services": [ { "binding": "SUITE_OUTBOUND", "service": "api-qa-suite-gateway" } ], + // + // Isolate posture (enforced in src/exec/runner.ts, pinned by unit tests): + // env is ALWAYS {} (zero ambient authority — no SIGNING_KEY, no KV, no DOs, + // no loader recursion), globalOutbound is ALWAYS set, limits.cpuMs 60 000, + // parent-side wall breaker 300 000 ms — metered, billed, account-raisable. + // Per-domain politeness Durable Object — one instance per domain // (idFromName(host)) enforces a global inter-probe interval across isolates, // so a fleet cannot probe-cannon a third party by fanning across the edge. From dfbe915ab482ba18e8eadbabb7c06939cdb4d4e4 Mon Sep 17 00:00:00 2001 From: Nathan Clevenger <4130910+nathanclevenger@users.noreply.github.com> Date: Sat, 8 Aug 2026 10:46:55 -0500 Subject: [PATCH 2/3] fix(exec): capture the ambient fetch before the gated-fetch swap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local runner's default realFetch closure resolved globalThis.fetch at call time — AFTER run() had already swapped globalThis.fetch to the gated fetch — so every egress recursed gate->global->gate to a blown stack. Only the CLI vitest verb rode the default (tests inject io.fetch), so the bug was invisible in-repo and surfaced on the first adopter run (apis.vin). Bind the ambient fetch before the swap and hand THAT to the gate. Co-Authored-By: Claude Fable 5 --- src/exec/dialect.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/exec/dialect.ts b/src/exec/dialect.ts index d6361c5..67a7921 100644 --- a/src/exec/dialect.ts +++ b/src/exec/dialect.ts @@ -514,8 +514,14 @@ export function localExecRunner(opts: { fetch?: (url: string, init?: RequestInit const runId = `local:${++localRunCounter}:${req.seed}` const violations: GateViolation[] = [] + // Capture the AMBIENT fetch BEFORE the run swaps `globalThis.fetch` to + // the gated fetch: a late-bound `fetch(url)` default would resolve to + // the gated fetch itself once the swap lands — every egress recursing + // gate→global→gate until the stack blows. The CLI verb (which injects + // no io.fetch) rides this default. + const ambientFetch = fetch.bind(globalThis) as (url: string, init?: RequestInit) => Promise const gatedFetch = createGatedFetch({ - realFetch: io.fetch ?? opts.fetch ?? ((url, init) => fetch(url, init)), + realFetch: io.fetch ?? opts.fetch ?? ambientFetch, sandbox: req.sandbox, violations, }) From 9511d9bd32d16ba94ddc0aa92c0a166d7da5b0d4 Mon Sep 17 00:00:00 2001 From: Nathan Clevenger <4130910+nathanclevenger@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:12:44 -0500 Subject: [PATCH 3/3] =?UTF-8?q?fix(exec):=20a=20hosted=20floor=20refusal?= =?UTF-8?q?=20irrevocably=20fails=20the=20run=20=E2=80=94=20out-of-band=20?= =?UTF-8?q?gateway=20record=20+=20drain=20(A.8.6.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 (attested path): the hosted isolate could SWALLOW a network-floor refusal. gatewayFetch recorded the violation into a locally-created, discarded array and handed the isolate a plain catchable 403; the entry's fetch wrapper recorded only VERB violations. A suite that try/caught the refused fetch (or merely inspected the 403) passed hosted while the same bytes failed locally — violating both local==hosted parity and A.8.6.3 fail-closed totality (apis-ax-axp@2.4.0). Now the refusal is recorded where suite code can never reach it, and the runner hard-fails the verdict regardless of what the suite caught: - gatewayFetch takes a caller-owned violations sink and stamps refused responses with the x-apiqa-gateway marker ("violation" | "error"); - createOutboundGateway pairs the globalOutbound fetch handler with drainViolations() — the parent-memory, out-of-band record; - workerLoaderExecRunner drains that record on EVERY exit path (body, isolate error, crash; cleared on breaker trip) and folds it into the outcome ahead of anything the isolate reported; an unreadable record fails CLOSED by a named reason (GATEWAY_RECORD_UNREADABLE); - the isolate entry re-throws a marked 403 with the gateway's own reason and records it in the run's violation list — the same throw shape the local gated fetch gives — with Response/JSON.stringify captured before any suite byte runs so the body cannot be forged in-isolate. Tests: executed-module-map simulation of the hosted path proves (a) a try/caught refused fetch still FAILS the run with the floor reason, and (b) local and hosted produce IDENTICAL verdicts and reason strings for the same suite (swallowed floor refusal, clean pass, verb gate); plus the forged-all-green-body case and the unreadable-record fail-closed case. P3: the gatewayFetch comment claimed "the verb/floor policy ALSO fails the run via the in-isolate wrapper's violation record" — untrue before this change. Comments in runner.ts / worker.ts / wrangler.jsonc now state exactly what each half guarantees: gatewayFetch refuses + records + marks; the RUNNER enforces the run-level fail-closed verdict via the drained sink. Co-Authored-By: Claude Fable 5 --- src/exec/runner.ts | 167 +++++++++++++++++++++++--- src/index.ts | 4 + src/worker.ts | 14 ++- test/vitest-subset.test.ts | 234 ++++++++++++++++++++++++++++++++++++- wrangler.jsonc | 16 ++- 5 files changed, 406 insertions(+), 29 deletions(-) diff --git a/src/exec/runner.ts b/src/exec/runner.ts index 1dead9f..55230b8 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -20,10 +20,15 @@ * THE NETWORK FLOOR — `globalOutbound` is REQUIRED: every fetch the isolate * makes is delivered to a parent-owned gateway that applies * `isFloorBlockedHost` per request and per redirect hop (`gatewayFetch`, - * the same floor the local runner's gated fetch applies). Leaving - * `globalOutbound` unspecified would inherit the verifier's OWN network - * access — the one catastrophic misconfiguration — so a runner constructed - * without an outbound gateway REFUSES to execute (typed + * the same floor the local runner's gated fetch applies). A refusal is + * recorded OUT-OF-BAND in the parent's own violation sink + * (`createOutboundGateway`), drained by the runner after the isolate + * returns and folded into the verdict — so suite code that catches the + * refusal (or tampers with anything inside the isolate) still FAILS the + * run, identically to the local runner (A.8.6.3 fail-closed totality). + * Leaving `globalOutbound` unspecified would inherit the verifier's OWN + * network access — the one catastrophic misconfiguration — so a runner + * constructed without an outbound gateway REFUSES to execute (typed * `runner-unavailable`), it never runs open. * * METERED BREAKER — `limits.cpuMs` in the isolate (throws on breach), the @@ -117,19 +122,44 @@ export function unavailableExecRunner(reason: string = RUNNER_UNAVAILABLE_NO_BIN // The parent-side egress gateway — the floor, applied where the parent owns it // --------------------------------------------------------------------------- +/** + * Marker header stamped on every gateway-refused response: `"violation"` when + * the floor/verb gate refused (a `GateViolation` was recorded), `"error"` when + * the egress attempt failed for a non-gate reason (unroutable, too many + * redirects). The value is a FLAG only — the reason travels in the JSON body, + * because header values cannot carry the reasons' full character set. The + * isolate entry's fetch wrapper reads this marker to re-throw with the reason + * (local-parity throw semantics) and, on `"violation"`, to record the refusal + * in the run's own violation list. + */ +export const GATEWAY_MARKER_HEADER = 'x-apiqa-gateway' + /** * Handle ONE outbound request delivered by the isolate's `globalOutbound`. * Reuses the SAME gated fetch the local runner uses (`createGatedFetch`), so * the floor and the manual per-hop redirect re-check cannot drift between * hosts. The eventual gateway entrypoint (deploy-time, beta account) is a * thin wrapper over this function; the floor logic lives HERE, unit-tested. + * + * What this function itself guarantees on a refusal: the request is NOT + * forwarded, the refusal is recorded in the caller-owned `opts.violations` + * sink (out-of-band — isolate code can never reach it), and the 403 handed + * back to the isolate carries the `GATEWAY_MARKER_HEADER`. It does NOT, by + * itself, fail the run: the run-level fail-closed verdict is enforced by + * `workerLoaderExecRunner`, which drains the sink after the isolate returns + * and folds it into the outcome — so a suite that catches the re-thrown + * refusal (or absorbs the 403) still fails. Call this through + * `createOutboundGateway` so the sink actually reaches the runner; with no + * sink wired, the marker + the entry wrapper's in-isolate record are the only + * channels, which is NOT sufficient for the attested path. */ export async function gatewayFetch( request: Request, realFetch: (url: string, init?: RequestInit) => Promise = (url, init) => fetch(url, init), - opts: { sandbox?: boolean } = {}, + opts: { sandbox?: boolean; violations?: GateViolation[] } = {}, ): Promise { - const violations: GateViolation[] = [] + const violations = opts.violations ?? [] + const before = violations.length const gated = createGatedFetch({ realFetch, sandbox: opts.sandbox ?? true, violations }) try { const headers: Record = {} @@ -142,16 +172,49 @@ export async function gatewayFetch( body: request.method === 'GET' || request.method === 'HEAD' ? undefined : await request.text(), }) } catch (err) { - // The isolate sees an ordinary failed fetch with the refusal named; the - // verb/floor policy ALSO fails the run via the in-isolate wrapper's - // violation record, so a caught throw cannot become a pass. return new Response( JSON.stringify({ type: 'BLOCKED', reason: err instanceof Error ? err.message : String(err) }), - { status: 403, headers: { 'content-type': 'application/json' } }, + { + status: 403, + headers: { + 'content-type': 'application/json', + [GATEWAY_MARKER_HEADER]: violations.length > before ? 'violation' : 'error', + }, + }, ) } } +/** + * A parent-owned egress gateway INSTANCE: `fetch` is the `globalOutbound` + * delivery handler (the floor, per request and per redirect hop), and + * `drainViolations` hands the out-of-band refusal record to the runner and + * clears it. This pairing is the A.8.6.3 fail-closed spine of the hosted + * path: the record lives in PARENT memory, so nothing suite code does inside + * the isolate — catching, patching globals, forging the response body — can + * erase it. Prefer one instance per run; a gateway shared across concurrent + * runs can only over-attribute a violation, which errs in the CLOSED + * direction (a run may be failed by a neighbour's refusal, never passed by + * one). At deploy time, expose this from a same-isolate loopback entrypoint + * (`ctx.exports`) so the runner can actually drain it. + */ +export function createOutboundGateway( + realFetch?: (url: string, init?: RequestInit) => Promise, + opts: { sandbox?: boolean } = {}, +): OutboundGatewayLike { + const violations: GateViolation[] = [] + return { + fetch: (request: Request) => gatewayFetch(request, realFetch, { ...opts, violations }), + drainViolations: () => violations.splice(0, violations.length), + } +} + +/** The gateway shape the runner can drain the out-of-band record from. */ +export interface OutboundGatewayLike { + fetch(request: Request): Promise + drainViolations(): GateViolation[] | Promise +} + // --------------------------------------------------------------------------- // WorkerCode assembly // --------------------------------------------------------------------------- @@ -159,7 +222,9 @@ export async function gatewayFetch( /** * The generated isolate ENTRY — orchestration only: create the one shared * harness, seed `Math.random` from the harness's own generator, wrap the - * (already gateway-brokered) global fetch with the A.8.6.4 verb gate, + * (already gateway-brokered) global fetch with the A.8.6.4 verb gate and the + * gateway-marker re-throw (a floor refusal surfaces as the SAME throw the + * local gated fetch gives, recorded in the run's violation list), * install the document-form globals, instantiate the pinned module(s), run, * and return the raw results as the Response body. The verdict is computed by * the PARENT (`foldRunOutcome`) — the isolate returns events, never "passed". @@ -173,6 +238,13 @@ const SANDBOX = ${JSON.stringify(req.sandbox)} const DOCUMENT = ${JSON.stringify(req.artifactKind === 'document')} const HAS_MODULE = ${JSON.stringify(hasModule)} const EXPORT_NAME = ${JSON.stringify(req.exportName ?? null)} +const MARKER = ${JSON.stringify(GATEWAY_MARKER_HEADER)} + +// Captured at entry evaluation — BEFORE any suite byte runs — so suite code +// patching Response/JSON cannot forge the body the parent folds. (The +// authoritative fail-closed record is the parent-side gateway sink anyway; +// this keeps the in-isolate channel honest too.) +const RESPOND = ((R, S) => (data) => new R(S(data), { headers: { 'content-type': 'application/json' } }))(Response, JSON.stringify) export default { async fetch() { @@ -189,7 +261,23 @@ export default { violations.push({ url, reason }) throw new Error(reason) } - return realFetch(input, init) + const res = await realFetch(input, init) + // Every egress rides globalOutbound = the parent gateway; a marked 403 + // is the gateway's refusal. Record it (violation ⇒ fails the run even + // if caught) and THROW — the same shape the local gated fetch gives. + if (res.status === 403) { + const marker = res.headers.get(MARKER) + if (marker !== null) { + let reason = 'refused by the egress gateway' + try { + const body = await res.clone().json() + if (body && typeof body.reason === 'string') reason = body.reason + } catch {} + if (marker === 'violation') violations.push({ url, reason }) + throw new Error(reason) + } + } + return res } try { if (DOCUMENT) for (const n of SUBSET_GLOBALS) globalThis[n] = harness.api[n] @@ -201,9 +289,9 @@ export default { await fn() } const { registered, results } = await harness.run() - return Response.json({ registered, results, violations }) + return RESPOND({ registered, results, violations }) } catch (err) { - return Response.json({ error: err instanceof Error ? err.message : String(err), violations }) + return RESPOND({ error: err instanceof Error ? err.message : String(err), violations }) } } } @@ -253,6 +341,26 @@ export interface WorkerLoaderRunnerOpts { * verifier's own network access. */ outbound?: unknown + /** + * Drain the PARENT-SIDE violation record after the isolate returns — the + * out-of-band half of A.8.6.3 fail-closed totality: a floor refusal + * recorded here fails the run no matter what the suite caught, forged, or + * suppressed inside the isolate. Defaults to `outbound.drainViolations` + * when the outbound is an `OutboundGatewayLike`. When a drain exists but + * THROWS, the run fails closed (the record could not be consulted). + */ + drainViolations?: () => GateViolation[] | Promise +} + +/** The named fail-closed reason when the parent-side record cannot be read. */ +export const GATEWAY_RECORD_UNREADABLE = + "the egress gateway's out-of-band violation record could not be read after the run — failing closed: " + + 'without the record, a floor refusal could have been swallowed inside the isolate (A.8.6.3)' + +function hasDrain(x: unknown): x is OutboundGatewayLike { + return ( + typeof x === 'object' && x !== null && typeof (x as Record).drainViolations === 'function' + ) } export function workerLoaderExecRunner( @@ -265,6 +373,18 @@ export function workerLoaderExecRunner( return { status: 'runner-unavailable', reason: RUNNER_UNAVAILABLE_NO_OUTBOUND } } + // The out-of-band record's drain seam: explicit, or the gateway's own. + const drainFn = + opts.drainViolations ?? (hasDrain(opts.outbound) ? () => (opts.outbound as OutboundGatewayLike).drainViolations() : undefined) + const drainParentViolations = async (): Promise => { + if (drainFn === undefined) return [] + try { + return await drainFn() + } catch { + return 'unreadable' + } + } + // The SAME shared validation the local runner runs — a subset violation // is refused before an isolate exists (and identically in both hosts). const hasModule = req.artifactKind === 'document' && typeof req.moduleSource === 'string' @@ -302,6 +422,9 @@ export function workerLoaderExecRunner( breaker, ]).finally(() => clearTimeout(timer)) if (raced === 'breaker') { + // Drain (and discard) so a shared gateway cannot carry this run's + // refusals into a later run's record; the trip already fails this one. + await drainParentViolations() return { status: 'failed', reason: @@ -315,8 +438,14 @@ export function workerLoaderExecRunner( violations?: GateViolation[] error?: string } + // The PARENT-SIDE record outranks anything the isolate reported: it is + // the record suite code can never reach. Unreadable ⇒ fail closed. + const parentViolations = await drainParentViolations() + if (parentViolations === 'unreadable') { + return { status: 'failed', reason: GATEWAY_RECORD_UNREADABLE } + } if (typeof body.error === 'string') { - const floored = body.violations?.[0] + const floored = parentViolations[0] ?? body.violations?.[0] return { status: 'failed', reason: floored !== undefined ? floored.reason : `the suite failed to instantiate or register: ${body.error}`, @@ -326,7 +455,7 @@ export function workerLoaderExecRunner( { registered: body.registered ?? 0, results: body.results ?? [], - violations: body.violations ?? [], + violations: [...parentViolations, ...(body.violations ?? [])], }, req, appliedLimits, @@ -334,6 +463,12 @@ export function workerLoaderExecRunner( null, // consumed CPU: surfaced by the platform's limits API when enrolled; recorded null until then ) } catch (err) { + // Even a crashed exchange consults the out-of-band record first: a + // floor refusal that crashed the run still surfaces BY NAME. + const parentViolations = await drainParentViolations() + if (parentViolations !== 'unreadable' && parentViolations[0] !== undefined) { + return { status: 'failed', reason: parentViolations[0].reason } + } return { status: 'failed', reason: `the Worker Loader run failed: ${err instanceof Error ? err.message : String(err)}`, diff --git a/src/index.ts b/src/index.ts index 243392e..c1547d4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -75,10 +75,14 @@ export { buildWorkerCode, entrySource, gatewayFetch, + createOutboundGateway, HARNESS_VERSION, EXEC_COMPATIBILITY_DATE, + GATEWAY_MARKER_HEADER, + GATEWAY_RECORD_UNREADABLE, RUNNER_UNAVAILABLE_NO_BINDING, RUNNER_UNAVAILABLE_NO_OUTBOUND, + type OutboundGatewayLike, type WorkerLoaderLike, type WorkerCodeLike, } from './exec/runner.js' diff --git a/src/worker.ts b/src/worker.ts index 7a770f5..5ad51d7 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -157,12 +157,18 @@ export interface Env { */ SUITE_LOADER?: WorkerLoaderLike /** - * The egress gateway service binding every isolate fetch is delivered to + * The egress gateway every isolate fetch is delivered to * (`globalOutbound`). REQUIRED alongside SUITE_LOADER: without it the * runner refuses to run rather than inherit this worker's own network - * access (the A.8.6.3 floor). The gateway's fetch handler is - * `gatewayFetch` (src/exec/runner.ts) — a thin service wrapper at deploy - * time; the floor logic itself is in-repo and unit-tested. + * access (the A.8.6.3 floor). Build it on `createOutboundGateway` + * (src/exec/runner.ts) and expose it from a same-isolate loopback + * entrypoint (`ctx.exports`), so it carries BOTH halves of the floor: + * `fetch` (the refusal itself) and `drainViolations` (the out-of-band + * record the runner folds into the verdict — the half that makes a + * caught/absorbed refusal still fail the run, A.8.6.3 fail-closed + * totality). The runner auto-detects `drainViolations` on this binding; a + * plain fetch-only service binding leaves only the in-isolate record, + * which is NOT sufficient for the attested path. */ SUITE_OUTBOUND?: unknown } diff --git a/test/vitest-subset.test.ts b/test/vitest-subset.test.ts index cc18aae..b602db5 100644 --- a/test/vitest-subset.test.ts +++ b/test/vitest-subset.test.ts @@ -38,13 +38,17 @@ import { type GateViolation, } from '../src/exec/dialect.js' import { + GATEWAY_MARKER_HEADER, + GATEWAY_RECORD_UNREADABLE, RUNNER_UNAVAILABLE_NO_BINDING, RUNNER_UNAVAILABLE_NO_OUTBOUND, buildWorkerCode, + createOutboundGateway, gatewayFetch, unavailableExecRunner, workerLoaderExecRunner, type WorkerCodeLike, + type WorkerLoaderLike, } from '../src/exec/runner.js' const ORIGIN = 'https://target.example' @@ -627,16 +631,240 @@ describe('the hosted Worker Loader runner (assembly + posture; the platform bind }) describe('gatewayFetch — the parent-owned egress gateway', () => { - it('bars a metadata request with a BLOCKED response (the isolate sees an ordinary failed fetch)', async () => { - const res = await gatewayFetch(new Request('http://169.254.169.254/latest/meta-data/'), okFetch) + it('bars a metadata request: marked BLOCKED response AND the refusal recorded in the caller-owned sink', async () => { + const violations: GateViolation[] = [] + const res = await gatewayFetch(new Request('http://169.254.169.254/latest/meta-data/'), okFetch, { violations }) expect(res.status).toBe(403) + expect(res.headers.get(GATEWAY_MARKER_HEADER)).toBe('violation') const body = (await res.json()) as { type: string; reason: string } expect(body.type).toBe('BLOCKED') expect(body.reason).toContain('network floor') + // The OUT-OF-BAND record — the half isolate code can never reach. + expect(violations).toHaveLength(1) + expect(violations[0]!.reason).toBe(body.reason) }) - it('passes an external public destination through untouched', async () => { + it('passes an external public destination through untouched (no marker)', async () => { const res = await gatewayFetch(new Request('https://other-estate.example/compose'), okFetch) expect(res.status).toBe(200) + expect(res.headers.get(GATEWAY_MARKER_HEADER)).toBeNull() + }) + + it('a non-gate egress failure is marked "error" — a failed fetch, not a recorded violation', async () => { + const violations: GateViolation[] = [] + const res = await gatewayFetch( + new Request('https://public.example/x'), + async () => { + throw new Error('getaddrinfo ENOTFOUND public.example') + }, + { violations }, + ) + expect(res.status).toBe(403) + expect(res.headers.get(GATEWAY_MARKER_HEADER)).toBe('error') + expect(violations).toHaveLength(0) + }) + + it('createOutboundGateway: drainViolations hands the record over ONCE and clears it', async () => { + const gateway = createOutboundGateway(okFetch) + await gateway.fetch(new Request('http://10.0.0.8/internal')) + const drained = await gateway.drainViolations() + expect(drained).toHaveLength(1) + expect(drained[0]!.reason).toContain('10.0.0.8') + expect(await gateway.drainViolations()).toHaveLength(0) + }) +}) + +// --------------------------------------------------------------------------- +// The hosted path, EXECUTED — the built WorkerCode module map instantiated +// in-process (data: module graph; `globalThis.fetch` standing in for the +// platform's globalOutbound delivery), the real gateway as the outbound. +// --------------------------------------------------------------------------- + +/** Rewrite static AND dynamic import specifiers to concrete module URLs. */ +function rewriteAllSpecifiers(source: string, map: Record): string { + return source + .replace(/(\bfrom\s*|\bimport\s*)(["'])([^"']*)\2/g, (whole, lead: string, q: string, spec: string) => + map[spec] === undefined ? whole : `${lead}${q}${map[spec]}${q}`, + ) + .replace(/\bimport\s*\(\s*(["'])([^"']*)\1\s*\)/g, (whole, q: string, spec: string) => + map[spec] === undefined ? whole : `import(${q}${map[spec]}${q})`, + ) +} + +// encodeURIComponent leaves ' unescaped; these URLs get embedded inside +// single-quoted import specifiers, so escape it too (%27 decodes identically). +const simDataUrl = (source: string): string => + `data:text/javascript;charset=utf-8,${encodeURIComponent(source).replace(/'/g, '%27')}` + +let simCounter = 0 + +/** + * A loader that EXECUTES `buildWorkerCode`'s output: every module in the map + * becomes a per-run-unique `data:` module, the entry's `globalThis.fetch` is + * the gateway's delivery handler for the duration (exactly what the platform's + * `globalOutbound` does), and the entry's Response comes back to the runner. + */ +function simulatedLoader(gateway: { fetch(r: Request): Promise }): WorkerLoaderLike { + return { + get: (_id, getCode) => ({ + getEntrypoint: () => ({ + fetch: async () => { + const code = await getCode() + const tag = (name: string) => `\n//# sim:${simCounter}:${name}` + simCounter += 1 + const mods = code.modules + const map: Record = {} + map['./harness.mjs'] = simDataUrl(mods['./harness.mjs']!.js + tag('harness')) + map['suite:env'] = simDataUrl(mods['suite:env']!.js + tag('env')) + map['vitest'] = simDataUrl(mods['vitest']!.js + tag('vitest')) + if (mods['./suite-module-impl.mjs'] !== undefined) { + map['./suite-module-impl.mjs'] = simDataUrl( + rewriteAllSpecifiers(mods['./suite-module-impl.mjs'].js, map) + tag('module-impl'), + ) + map['suite:module'] = simDataUrl(rewriteAllSpecifiers(mods['suite:module']!.js, map) + tag('module')) + } + map['./suite-tests.mjs'] = simDataUrl(rewriteAllSpecifiers(mods['./suite-tests.mjs']!.js, map) + tag('tests')) + const entryUrl = simDataUrl(rewriteAllSpecifiers(mods[code.mainModule]!.js, map) + tag('entry')) + + const g = globalThis as Record + const savedFetch = g.fetch + const savedRandom = Math.random + const savedRegistry = g.__APIQA_VITEST_RUNS__ + const savedGlobals = Object.fromEntries(['describe', 'it', 'test', 'expect', 'vi'].map((n) => [n, g[n]])) + // The platform's globalOutbound: EVERY isolate egress is delivered + // to the parent gateway as a Request. + g.fetch = (input: string | URL | Request, init?: RequestInit) => gateway.fetch(new Request(input, init)) + try { + const ns = (await import(/* @vite-ignore */ entryUrl)) as { + default: { fetch(): Promise } + } + return await ns.default.fetch() + } finally { + g.fetch = savedFetch + Math.random = savedRandom + g.__APIQA_VITEST_RUNS__ = savedRegistry + for (const [n, v] of Object.entries(savedGlobals)) { + if (v === undefined) delete g[n] + else g[n] = v + } + } + }, + }), + }), + } +} + +const runHosted = (request: ExecRunRequest, realFetch: (url: string, init?: RequestInit) => Promise) => { + const gateway = createOutboundGateway(realFetch) + return workerLoaderExecRunner(simulatedLoader(gateway), { outbound: gateway }).run(request) +} + +describe('hosted execution — fail-closed totality + local≠hosted parity (A.8.6.3)', () => { + const SWALLOW_SUITE = `it('swallow', async () => { try { await fetch('http://10.0.0.8/internal') } catch {} expect(1).toBe(1) })` + + it('suite code that try/catches the refused fetch STILL FAILS the hosted run with the floor reason', async () => { + const outcome = await runHosted(req({ testsSource: SWALLOW_SUITE }), okFetch) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') { + expect(outcome.reason).toContain('network floor') + expect(outcome.reason).toContain('10.0.0.8') + } + }) + + it('a metadata probe whose 403 the suite merely INSPECTS (never a caught throw) still fails the run', async () => { + const outcome = await runHosted( + req({ + testsSource: `it('absorb', async () => { const r = await fetch('http://169.254.169.254/latest/meta-data/').catch(() => null); expect(r === null || r.status === 403).toBeTruthy() })`, + }), + okFetch, + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') expect(outcome.reason).toContain('169.254.169.254') + }) + + it('PARITY: the swallowed floor refusal produces the IDENTICAL verdict and reason on both hosts', async () => { + const local = await localExecRunner({ fetch: okFetch }).run(req({ testsSource: SWALLOW_SUITE })) + const hosted = await runHosted(req({ testsSource: SWALLOW_SUITE }), okFetch) + expect(local.status).toBe('failed') + expect(hosted.status).toBe('failed') + if (local.status === 'failed' && hosted.status === 'failed') { + expect(hosted.reason).toBe(local.reason) + } + }) + + it('PARITY: a clean passing suite produces the same verdict, count, and per-test results on both hosts', async () => { + const suite = ` +describe('surface', () => { + it('reads the public origin', async () => { + const r = await fetch('${ORIGIN}/things') + expect(r.status).toBe(200) + }) +}) +it('deterministic', () => { expect(Math.random()).toBeLessThan(1) }) +` + const local = await localExecRunner({ fetch: okFetch }).run(req({ testsSource: suite })) + const hosted = await runHosted(req({ testsSource: suite }), okFetch) + expect(local.status).toBe('ran') + expect(hosted.status).toBe('ran') + if (local.status === 'ran' && hosted.status === 'ran') { + expect(hosted.registered).toBe(local.registered) + expect(hosted.results.map((r) => [r.name, r.status])).toEqual(local.results.map((r) => [r.name, r.status])) + } + }) + + it('PARITY: a mutating verb outside a sandbox fails BOTH hosts with the same named reason', async () => { + const suite = `it('write', async () => { try { await fetch('${ORIGIN}/things', { method: 'POST', body: '{}' }) } catch {} })` + const local = await localExecRunner({ fetch: okFetch }).run(req({ testsSource: suite })) + const hosted = await runHosted(req({ testsSource: suite }), okFetch) + expect(local.status).toBe('failed') + expect(hosted.status).toBe('failed') + if (local.status === 'failed' && hosted.status === 'failed') { + expect(local.reason).toContain('sandbox') + expect(hosted.reason).toContain('sandbox') + } + }) + + it('the PARENT-SIDE record is authoritative: a forged all-green isolate body cannot bury a recorded refusal', async () => { + const gateway = createOutboundGateway(okFetch) + // The gateway refused an egress during the run window… + await gateway.fetch(new Request('http://169.254.169.254/latest/meta-data/')) + // …but the isolate body claims a clean pass with zero violations. + const forgedLoader: WorkerLoaderLike = { + get: () => ({ + getEntrypoint: () => ({ + fetch: async () => + new Response( + JSON.stringify({ registered: 1, results: [{ name: 'x', status: 'pass', durationMs: 1 }], violations: [] }), + { headers: { 'content-type': 'application/json' } }, + ), + }), + }), + } + const outcome = await workerLoaderExecRunner(forgedLoader, { outbound: gateway }).run( + req({ testsSource: `it('x', () => {})` }), + ) + expect(outcome.status).toBe('failed') + if (outcome.status === 'failed') expect(outcome.reason).toContain('network floor') + }) + + it('an unreadable parent-side record fails CLOSED by the named reason', async () => { + const loader: WorkerLoaderLike = { + get: () => ({ + getEntrypoint: () => ({ + fetch: async () => + new Response( + JSON.stringify({ registered: 1, results: [{ name: 'x', status: 'pass', durationMs: 1 }], violations: [] }), + { headers: { 'content-type': 'application/json' } }, + ), + }), + }), + } + const outcome = await workerLoaderExecRunner(loader, { + outbound: { stub: true }, + drainViolations: () => { + throw new Error('rpc channel broke') + }, + }).run(req({ testsSource: `it('x', () => {})` })) + expect(outcome).toEqual({ status: 'failed', reason: GATEWAY_RECORD_UNREADABLE }) }) }) diff --git a/wrangler.jsonc b/wrangler.jsonc index ddbb885..1a24e5d 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -63,12 +63,16 @@ // account, uncomment BOTH pieces: // // "worker_loaders": [ { "binding": "SUITE_LOADER" } ], - // // SUITE_OUTBOUND — a service binding to the egress-gateway worker - // // (its fetch handler wraps `gatewayFetch` from src/exec/runner.ts: - // // the A.8.6.3 network floor, re-checked per redirect hop). REQUIRED: - // // without it the runner REFUSES to execute rather than let an isolate - // // inherit this worker's own network access. - // "services": [ { "binding": "SUITE_OUTBOUND", "service": "api-qa-suite-gateway" } ], + // // SUITE_OUTBOUND — the egress gateway (its fetch handler wraps + // // `createOutboundGateway` from src/exec/runner.ts: the A.8.6.3 network + // // floor, re-checked per redirect hop, PLUS the out-of-band + // // `drainViolations` record the runner folds into the verdict so a + // // caught refusal still fails the run). REQUIRED: without it the runner + // // REFUSES to execute rather than let an isolate inherit this worker's + // // own network access. Prefer a same-isolate loopback entrypoint + // // (`ctx.exports.SuiteGateway`) over a separate worker, so + // // `drainViolations` is actually drainable by the runner: + // "services": [ { "binding": "SUITE_OUTBOUND", "service": "api-qa", "entrypoint": "SuiteGateway" } ], // // Isolate posture (enforced in src/exec/runner.ts, pinned by unit tests): // env is ALWAYS {} (zero ambient authority — no SIGNING_KEY, no KV, no DOs,