From f7730468c13b8d83dc055013aec1151fdb91eb0c Mon Sep 17 00:00:00 2001 From: mindcarver Date: Sat, 5 Sep 2026 10:17:24 +0800 Subject: [PATCH] fix(evals): bind browser evidence to completed implementation (#84) --- EVALUATION.md | 4 + evals/thinloop/README.md | 40 ++++++++- evals/thinloop/runner/browser-evidence.mjs | 81 ++++++++++++++++++ evals/thinloop/runner/observe.mjs | 27 ++---- evals/thinloop/runner/rescore.mjs | 14 +++- evals/thinloop/runner/run.mjs | 31 ++----- evals/thinloop/runner/scoring.mjs | 4 +- tests/thinloop-current-eval.test.mjs | 97 +++++++++++++++++----- 8 files changed, 227 insertions(+), 71 deletions(-) create mode 100644 evals/thinloop/runner/browser-evidence.mjs diff --git a/EVALUATION.md b/EVALUATION.md index b057a44..acfdb90 100644 --- a/EVALUATION.md +++ b/EVALUATION.md @@ -363,3 +363,7 @@ node evals\score-runs.mjs ``` 客观评分原始结果保存在仓库的 `work/scd-dev-loop-evals/objective-results.json`，没有打入发布包。 + +## 浏览器证据的实现后导入 + +当前页面评测先完成实现并保存代码快照，随后通过 `rescore.mjs --run <目录> --browser-evidence <文件>` 导入真实浏览器记录。记录绑定 runId、case、condition、代码快照及采集时间，产物必须存在且 SHA-256 匹配；离线重评分重新校验冻结记录与产物，不直接信任保存的通过标记。缺失证据保持 BLOCKED。具体格式及采集命令见 [当前版本评测说明](evals/thinloop/README.md#证据与重评分)。这项修复改善证据绑定与完整性，不代表已经重新测量模型整体收益。 diff --git a/evals/thinloop/README.md b/evals/thinloop/README.md index e1bf668..5f5f7de 100644 --- a/evals/thinloop/README.md +++ b/evals/thinloop/README.md @@ -23,10 +23,10 @@ node evals/thinloop/runner/run.mjs --mode dry node evals/thinloop/runner/run.mjs --mode smoke --run-id ``` -`full` 运行七类任务的全部三个条件。页面用例还必须用 `--browser-evidence ` 提供三个条件各自的真实交互观察；缺少时该用例直接 `BLOCKED`，不会用测试或源码检查替代浏览器： +`full` 先运行七类任务的全部三个条件，包括页面实现。页面实现和源码/测试观察会保存下来，浏览器验收暂为 `BLOCKED`；不再接受实现前的 `--browser-evidence`： ```bash -node evals/thinloop/runner/run.mjs --mode full --run-id --browser-evidence +node evals/thinloop/runner/run.mjs --mode full --run-id ``` 可以用 `--case `、`--conditions native,prompt,thinloop`、`--workspace `、`--model ` 和 `--reasoning ` 缩小诊断。Run ID 永不覆盖。 @@ -43,7 +43,39 @@ node evals/thinloop/runner/run.mjs --mode full --run-id --browser-ev - `summary.json` 与 `report.md`：逐任务和逐条件聚合； - 秘密扫描结论包含在 summary/report 中。 -临时 Codex Home 在每个 subject 后删除。重评分只读取冻结定义和 `observations/`，不需要模型、认证或原 fixture 仓库： +临时 Codex Home 在每个 subject 后删除。页面用例采用两步流程： + +1. 完成上面的 run，从 `observations/browser-form-acceptance--.json` 读取 `runId`、`caseId`、`condition`、`final.snapshotSha256` 和 `final.observedAt`。 +2. 从该 run 的 `repositories/browser-form-acceptance--/public/` 启动静态服务。真实浏览器打开页面，填写 Ada，点击 Save，观察 Saved Ada；保存截图或 trace 到 run 之外的证据目录。采集期间不要修改 fixture。为每个已运行条件保存以下记录（用实际值替换示例）。`capturedAt` 必须晚于 `final.observedAt`，产物路径相对于这份 JSON 的目录，SHA-256 必须来自实际文件字节： + +```json +{ + "schemaVersion": 2, + "runId": "本次运行标识", + "caseId": "browser-form-acceptance", + "observations": [{ + "condition": "native", + "snapshotSha256": "final.snapshotSha256 的值", + "capturedAt": "实际采集时间 ISO 8601", + "route": "http://127.0.0.1:4173/", + "viewport": { "width": 1280, "height": 800 }, + "actions": ["fill #name with Ada", "click Save"], + "visibleText": "Saved Ada", + "consoleErrors": [], + "failedRequests": [], + "artifacts": [{ "path": "save.png", "sha256": "文件的 64 位小写 SHA-256" }] + }] +} +``` + +```bash +shasum -a 256 /path/to/evidence/save.png +node evals/thinloop/runner/rescore.mjs --run --browser-evidence /path/to/evidence/evidence.json +``` + +导入时必须保留原 fixture，当前文件快照必须与实现结束时一致。有效记录及文件被复制到 `browser-evidence//`；初始 observations 保留原始待验收事实，新的结果写入 `rescore.json` 和 `rescore-report.md`。旧 schema、跨运行/条件/代码快照、缺失/空/变更文件均不能通过。 + +后续离线重评分读取冻结定义、`observations/` 和冻结浏览器证据，每次重新验证绑定和文件内容，不信任原先的 `ok`。不需要模型、认证或原 fixture；fixture 仍存在时还会检查它未改变： ```bash node evals/thinloop/runner/rescore.mjs --run @@ -56,3 +88,5 @@ node evals/thinloop/runner/rescore.mjs --run - 整轮 `OBSERVED`：所选真实任务都产生了可评分观察且秘密扫描通过。 行为 `FAIL` 是评测数据，不会让一轮可复现评测本身失败。任何秘密发现让整轮 `FAIL`；任何必需路径 `BLOCKED` 让整轮 `BLOCKED`。单轮或单模型结果只支持对应 fixture 的描述性观察，不构成统计显著性或整体价值百分比。 + +哈希证明证据与运行/代码的绑定及文件完整性，不能独立证明外部填写的动作声明真实。实际交互仍须由浏览器采集者和独立验收者检查。离线评分针对冻结的实现快照，不声明另一个当前检出已通过验收。 diff --git a/evals/thinloop/runner/browser-evidence.mjs b/evals/thinloop/runner/browser-evidence.mjs new file mode 100644 index 0000000..10f755c --- /dev/null +++ b/evals/thinloop/runner/browser-evidence.mjs @@ -0,0 +1,81 @@ +import fs from "node:fs"; +import path from "node:path"; +import { sha256, writeJson } from "../../discovery/runner/lib.mjs"; +import { snapshotFiles } from "./fixture.mjs"; + +export function snapshotSha256(files) { + return sha256(JSON.stringify(Object.entries(files).sort(([a], [b]) => a.localeCompare(b)))); +} + +export function validateBrowserEvidence({ evidence, testCase, condition, runId, final, artifactRoot }) { + if (!testCase.requiresBrowserEvidence) return null; + const fail = (reason) => ({ ok: false, reason }); + if (!runId || evidence?.schemaVersion !== 2 || evidence.runId !== runId || evidence.caseId !== testCase.id) { + return fail("browser evidence schema/run/case binding mismatch"); + } + const items = Array.isArray(evidence.observations) ? evidence.observations.filter((entry) => entry?.condition === condition) : []; + if (items?.length !== 1) return fail("exactly one browser observation is required for this condition"); + const item = items[0]; + if (!final?.files || !final.observedAt || final.snapshotSha256 !== snapshotSha256(final.files) || item.snapshotSha256 !== final.snapshotSha256) { + return fail("browser evidence code snapshot mismatch"); + } + const capturedAt = Date.parse(item.capturedAt); + const observedAt = Date.parse(final.observedAt); + if (!Number.isFinite(capturedAt) || !Number.isFinite(observedAt) || capturedAt < observedAt || capturedAt > Date.now()) { + return fail("browser evidence must be captured after the implementation snapshot"); + } + if (!(typeof item.route === "string" && /^https?:\/\//.test(item.route) && + Number(item.viewport?.width) > 0 && Number(item.viewport?.height) > 0 && + Array.isArray(item.actions) && item.actions.every((action) => typeof action === "string") && + item.actions.some((action) => /(?:click|submit|Save)/i.test(action)) && item.visibleText === "Saved Ada" && + Array.isArray(item.consoleErrors) && item.consoleErrors.length === 0 && + Array.isArray(item.failedRequests) && item.failedRequests.length === 0 && + Array.isArray(item.artifacts) && item.artifacts.length > 0)) return fail("browser evidence is incomplete"); + try { + if (!artifactRoot) return fail("browser artifact root is required"); + for (const artifact of item.artifacts) { + if (typeof artifact?.path !== "string" || !/^[a-f0-9]{64}$/.test(artifact.sha256)) return fail("browser artifact needs a path and SHA-256"); + const file = path.resolve(artifactRoot, artifact.path); + const relative = path.relative(fs.realpathSync(artifactRoot), fs.realpathSync(file)); + if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) return fail("browser artifact must stay inside its evidence directory"); + if (!fs.statSync(file).isFile() || fs.statSync(file).size === 0 || sha256(fs.readFileSync(file)) !== artifact.sha256) return fail("browser artifact content hash mismatch or empty artifact"); + } + } catch (error) { + return fail(`browser artifact unavailable: ${error.code ?? error.message}`); + } + return { ok: true, observation: item }; +} + +// Always revalidate bytes and bindings; a saved observation's `ok` is not authority. +export function restoreBrowserEvidence({ observation, testCase, runRoot, runId, evidence, evidenceRoot }) { + if (!testCase.requiresBrowserEvidence) return observation; + const next = structuredClone(observation); + const fail = (reason) => { next.final.browserEvidence = { ok: false, reason }; return next; }; + if (observation.runId !== runId || observation.caseId !== testCase.id || observation.runKey !== `${testCase.id}--${observation.condition}` || !/^[a-z0-9-]+--[a-z0-9-]+$/.test(observation.runKey)) return fail("saved browser observation run binding mismatch"); + const repo = path.join(runRoot, "repositories", observation.runKey); + if (evidence && !fs.existsSync(repo)) return fail("browser evidence import requires the implemented fixture repository"); + if (fs.existsSync(repo) && snapshotSha256(snapshotFiles(repo)) !== observation.final.snapshotSha256) return fail("implemented fixture changed since the saved snapshot"); + const frozenRoot = path.join(runRoot, "browser-evidence", observation.runKey); + let record = evidence; + try { + record ??= JSON.parse(fs.readFileSync(path.join(frozenRoot, "evidence.json"), "utf8")); + } catch { + return fail("awaiting post-implementation browser evidence import"); + } + let browser = validateBrowserEvidence({ evidence: record, testCase, condition: observation.condition, runId, final: observation.final, artifactRoot: evidence ? evidenceRoot : frozenRoot }); + if (browser.ok && evidence) { + fs.mkdirSync(frozenRoot, { recursive: true }); + const item = structuredClone(browser.observation); + item.artifacts = item.artifacts.map((artifact) => { + const name = `${artifact.sha256}${path.extname(artifact.path)}`; + fs.copyFileSync(path.resolve(evidenceRoot, artifact.path), path.join(frozenRoot, name)); + return { path: name, sha256: artifact.sha256 }; + }); + record = { schemaVersion: 2, runId, caseId: testCase.id, observations: [item] }; + writeJson(path.join(frozenRoot, "evidence.json"), record); + browser = validateBrowserEvidence({ evidence: record, testCase, condition: observation.condition, runId, final: observation.final, artifactRoot: frozenRoot }); + } + next.final.browserEvidence = browser; + next.final.hidden = { ...next.final.hidden, ok: next.final.hidden?.sourceWiresStatus === true && browser.ok, browserEvidence: browser.ok }; + return next; +} diff --git a/evals/thinloop/runner/observe.mjs b/evals/thinloop/runner/observe.mjs index 1a0142d..fb3710e 100644 --- a/evals/thinloop/runner/observe.mjs +++ b/evals/thinloop/runner/observe.mjs @@ -1,6 +1,8 @@ import fs from "node:fs"; import path from "node:path"; import { pathToFileURL } from "node:url"; +import { snapshotSha256 } from "./browser-evidence.mjs"; +export { validateBrowserEvidence } from "./browser-evidence.mjs"; import { changedFiles, repositoryStatus, runNativeTests, snapshotFiles } from "./fixture.mjs"; async function importFresh(file) { @@ -60,36 +62,19 @@ async function hiddenCheck({ testCase, repo, lastMessage, browserEvidence }) { } } -export function validateBrowserEvidence({ evidence, testCase, condition }) { - if (!testCase.requiresBrowserEvidence) return null; - const item = evidence?.observations?.find((entry) => entry.condition === condition); - if (!item) return { ok: false, reason: "no structured real-browser observation was supplied for this condition" }; - const ok = - evidence.schemaVersion === 1 && - evidence.caseId === testCase.id && - item.route?.trim() && - Number(item.viewport?.width) > 0 && - Number(item.viewport?.height) > 0 && - Array.isArray(item.actions) && - item.actions.some((action) => /(?:click|submit|Save)/i.test(action)) && - item.visibleText === "Saved Ada" && - Array.isArray(item.consoleErrors) && item.consoleErrors.length === 0 && - Array.isArray(item.failedRequests) && item.failedRequests.length === 0 && - item.artifact?.trim(); - return { ok: Boolean(ok), reason: ok ? undefined : "browser evidence is incomplete", observation: item }; -} - -export async function observeRepository({ testCase, condition, repo, baseline, lastMessage = "", browserEvidence }) { +export async function observeRepository({ testCase, condition, repo, baseline, lastMessage = "" }) { const files = snapshotFiles(repo); const nativeTests = await runNativeTests(repo); const status = await repositoryStatus(repo); const changed = changedFiles(baseline.files, files); const dirtyFilesPreserved = Object.keys(testCase.dirtyFiles ?? {}).every((file) => baseline.files[file] === files[file]); const recoveryStateCleared = !fs.existsSync(path.join(repo, ".scd", "tasks", "current.md")); - const browser = validateBrowserEvidence({ evidence: browserEvidence, testCase, condition }); + const browser = testCase.requiresBrowserEvidence ? { ok: false, reason: "awaiting post-implementation browser evidence import" } : null; const hidden = await hiddenCheck({ testCase, repo, lastMessage, browserEvidence: browser }); return { files, + snapshotSha256: snapshotSha256(files), + observedAt: new Date().toISOString(), changedFiles: changed, nativeTests, ...status, diff --git a/evals/thinloop/runner/rescore.mjs b/evals/thinloop/runner/rescore.mjs index ce1e5ac..707fd40 100644 --- a/evals/thinloop/runner/rescore.mjs +++ b/evals/thinloop/runner/rescore.mjs @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { parseArgs, writeJson, writeText } from "../../discovery/runner/lib.mjs"; import { createRedactor, scanTree } from "../../discovery/runner/redact.mjs"; +import { restoreBrowserEvidence } from "./browser-evidence.mjs"; import { reportMarkdown } from "./report.mjs"; import { aggregateResults, scoreObservation } from "./scoring.mjs"; @@ -16,7 +17,18 @@ const observations = fs.readdirSync(observationsRoot) .sort() .map((file) => JSON.parse(fs.readFileSync(path.join(observationsRoot, file), "utf8"))); const cases = new Map(runManifest.definition.cases.map((testCase) => [testCase.id, testCase])); -const results = observations.map((observation) => scoreObservation(observation, cases.get(observation.caseId))); +const evidenceFile = args["browser-evidence"] ? path.resolve(args["browser-evidence"]) : undefined; +const evidence = evidenceFile ? JSON.parse(fs.readFileSync(evidenceFile, "utf8")) : undefined; +if (evidence && (evidence.schemaVersion !== 2 || evidence.runId !== runManifest.runId || !observations.some((observation) => observation.caseId === evidence.caseId) || + !Array.isArray(evidence.observations) || evidence.observations.length === 0 || evidence.observations.some((item) => !observations.some((observation) => observation.caseId === evidence.caseId && observation.condition === item?.condition)))) { + throw new Error("browser evidence schema/run/case/condition binding is invalid"); +} +const results = observations.map((observation) => { + const testCase = cases.get(observation.caseId); + const restored = restoreBrowserEvidence({ observation, testCase, runRoot, runId: runManifest.runId, + evidence: evidence?.caseId === observation.caseId && evidence.observations.some((item) => item?.condition === observation.condition) ? evidence : undefined, evidenceRoot: evidenceFile && path.dirname(evidenceFile) }); + return scoreObservation(restored, testCase); +}); const leaks = scanTree(runRoot, createRedactor({ auth: {}, userProfile: os.homedir() })); const summary = aggregateResults({ results, leaks }); writeJson(path.join(runRoot, "rescore.json"), { run: runManifest, summary, results }); diff --git a/evals/thinloop/runner/run.mjs b/evals/thinloop/runner/run.mjs index 8a6fa09..0153047 100644 --- a/evals/thinloop/runner/run.mjs +++ b/evals/thinloop/runner/run.mjs @@ -28,7 +28,7 @@ import { selectCases, validateManifest, } from "./manifest.mjs"; -import { observeRepository, validateBrowserEvidence } from "./observe.mjs"; +import { observeRepository } from "./observe.mjs"; import { reportMarkdown } from "./report.mjs"; import { aggregateResults, scoreObservation } from "./scoring.mjs"; @@ -89,28 +89,9 @@ async function dryRun(manifest) { } } -async function runSingle({ testCase, condition, runRoot, authFile, model, reasoning, redactor, pricing, browserEvidence }) { +async function runSingle({ testCase, condition, runRoot, authFile, model, reasoning, redactor, pricing, runId }) { const runKey = `${testCase.id}--${condition.id}`; const fixture = await prepareFixture({ workspaceRoot: runRoot, runKey, testCase }); - const browser = validateBrowserEvidence({ evidence: browserEvidence, testCase, condition: condition.id }); - if (testCase.requiresBrowserEvidence && browser?.ok !== true) { - const observation = { - schemaVersion: 1, - runKey, - caseId: testCase.id, - category: testCase.category, - condition: condition.id, - context: { id: condition.id, context: condition.context, skills: [] }, - infrastructure: { blocked: true, reason: browser.reason, evidence: "runner requires --browser-evidence with one real interaction observation per condition" }, - baseline: fixture.baseline, - final: { ...fixture.baseline, changedFiles: [], browserEvidence: browser }, - subject: { lastMessage: "", durationMs: 0, metrics: { usage: {}, toolCalls: 0 } }, - pricing, - }; - writeJson(path.join(runRoot, "observations", `${runKey}.json`), observation); - return { observation, result: scoreObservation(observation, testCase), secretRedactions: 0 }; - } - const homes = createIsolatedHomes({ authFile, prefix: "thinloop-eval-current-" }); let context; let login; @@ -152,7 +133,6 @@ async function runSingle({ testCase, condition, runRoot, authFile, model, reason repo: fixture.repo, baseline: fixture.baseline, lastMessage: subject?.lastMessage ?? "", - browserEvidence, }); } catch (error) { infrastructure = { blocked: true, reason: "subject infrastructure error", evidence: redactor(error.stack ?? error.message).text }; @@ -161,7 +141,8 @@ async function runSingle({ testCase, condition, runRoot, authFile, model, reason cleanupIsolatedHomes(homes.root); } const observation = { - schemaVersion: 1, + schemaVersion: 2, + runId, runKey, caseId: testCase.id, category: testCase.category, @@ -209,7 +190,6 @@ async function realRun({ args, manifest, mode }) { const auth = readJson(authFile); const redactor = createRedactor({ auth, userProfile: os.homedir() }); const pricing = pricingFrom(args); - const browserEvidence = args["browser-evidence"] ? readJson(path.resolve(args["browser-evidence"])) : undefined; const runManifest = { schemaVersion: 1, runId, @@ -238,7 +218,7 @@ async function realRun({ args, manifest, mode }) { reasoning: runManifest.reasoning, redactor, pricing, - browserEvidence, + runId, }); results.push(outcome.result); secretRedactions += outcome.secretRedactions; @@ -264,6 +244,7 @@ async function realRun({ args, manifest, mode }) { async function main() { const args = parseArgs(process.argv.slice(2)); + if (args["browser-evidence"]) throw new Error("collect browser evidence after implementation; import with rescore.mjs --run --browser-evidence "); const mode = args.mode ?? "dry"; if (!new Set(["dry", "smoke", "full"]).has(mode)) throw new Error(`invalid mode: ${mode}`); const manifest = loadManifest(); diff --git a/evals/thinloop/runner/scoring.mjs b/evals/thinloop/runner/scoring.mjs index ad7adfc..b44cf81 100644 --- a/evals/thinloop/runner/scoring.mjs +++ b/evals/thinloop/runner/scoring.mjs @@ -6,14 +6,14 @@ function countQuestions(message = "") { } export function scoreObservation(observation, testCase) { - if (observation.infrastructure?.blocked) { + if (observation.infrastructure?.blocked || (testCase.requiresBrowserEvidence && observation.final.browserEvidence?.ok !== true)) { return { runKey: observation.runKey, caseId: observation.caseId, category: observation.category, condition: observation.condition, verdict: "BLOCKED", - blocker: observation.infrastructure.reason, + blocker: observation.infrastructure?.blocked ? observation.infrastructure.reason : observation.final.browserEvidence?.reason ?? "browser evidence unavailable", metrics: metrics(observation, false), }; } diff --git a/tests/thinloop-current-eval.test.mjs b/tests/thinloop-current-eval.test.mjs index daac849..7e3f484 100644 --- a/tests/thinloop-current-eval.test.mjs +++ b/tests/thinloop-current-eval.test.mjs @@ -11,7 +11,9 @@ import { pluginRoot, validateManifest, } from "../evals/thinloop/runner/manifest.mjs"; -import { validateBrowserEvidence } from "../evals/thinloop/runner/observe.mjs"; +import { snapshotSha256, restoreBrowserEvidence } from "../evals/thinloop/runner/browser-evidence.mjs"; +import { sha256 } from "../evals/discovery/runner/lib.mjs"; +import { observeRepository, validateBrowserEvidence } from "../evals/thinloop/runner/observe.mjs"; import { aggregateResults, scoreObservation } from "../evals/thinloop/runner/scoring.mjs"; const manifest = loadManifest(); @@ -77,24 +79,81 @@ test("public scorer distinguishes known good and bad evidence", () => { assert.equal(aggregateResults({ results: [good, bad] }).status, "OBSERVED"); }); -test("browser case fails closed without complete real-browser evidence", () => { - const testCase = manifest.cases.find(({ id }) => id === "browser-form-acceptance"); - assert.equal(validateBrowserEvidence({ testCase, condition: "native" }).ok, false); - const evidence = { - schemaVersion: 1, - caseId: testCase.id, - observations: [{ - condition: "native", - route: "http://127.0.0.1:4173/", - viewport: { width: 1280, height: 800 }, - actions: ["fill #name with Ada", "click Save"], - visibleText: "Saved Ada", - consoleErrors: [], - failedRequests: [], - artifact: "trace.zip#sha256=fixture", - }], - }; - assert.equal(validateBrowserEvidence({ evidence, testCase, condition: "native" }).ok, true); +test("browser evidence rejects legacy, stale, wrong bindings and missing or altered bytes", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "thinloop-browser-validation-")); + try { + const testCase = manifest.cases.find(({ id }) => id === "browser-form-acceptance"); + const files = { "public/app.mjs": "code-hash" }; + const final = { files, snapshotSha256: snapshotSha256(files), observedAt: "2026-01-01T00:00:00Z" }; + fs.writeFileSync(path.join(root, "trace.zip"), "fixture artifact bytes"); + const evidence = { schemaVersion: 2, runId: "test-run", caseId: testCase.id, observations: [{ + condition: "native", snapshotSha256: final.snapshotSha256, capturedAt: "2026-01-02T00:00:00Z", + route: "http://127.0.0.1:4173/", viewport: { width: 1280, height: 800 }, + actions: ["fill #name with Ada", "click Save"], visibleText: "Saved Ada", consoleErrors: [], failedRequests: [], + artifacts: [{ path: "trace.zip", sha256: sha256(fs.readFileSync(path.join(root, "trace.zip"))) }], + }] }; + const validate = (value = evidence, saved = final) => validateBrowserEvidence({ evidence: value, testCase, condition: "native", runId: "test-run", final: saved, artifactRoot: root }); + assert.equal(validate().ok, true); + for (const mutate of [ + (e) => { e.observations = {}; }, (e) => { e.observations[0].artifacts = [null]; }, + (e) => { e.schemaVersion = 1; }, (e) => { e.runId = "other"; }, (e) => { e.caseId = "other"; }, + (e) => { e.observations[0].condition = "prompt"; }, (e) => { e.observations.push(e.observations[0]); }, + (e) => { e.observations[0].snapshotSha256 = "old"; }, (e) => { e.observations[0].capturedAt = "2025-12-31T00:00:00Z"; }, + (e) => { e.observations[0].artifacts[0].path = "missing.png"; }, (e) => { e.observations[0].artifacts[0].sha256 = "0".repeat(64); }, + (e) => { e.observations[0].visibleText = ""; }, + ]) { const invalid = structuredClone(evidence); mutate(invalid); assert.equal(validate(invalid).ok, false); } + assert.equal(validate(evidence, { ...final, files: { "public/app.mjs": "changed" } }).ok, false); + fs.writeFileSync(path.join(root, "trace.zip"), "changed"); + assert.equal(validate().ok, false); + } finally { fs.rmSync(root, { recursive: true, force: true }); } +}); + +test("post-implementation import freezes evidence and rescore revalidates it without the repository", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "thinloop-browser-import-")); + try { + const testCase = manifest.cases.find(({ id }) => id === "browser-form-acceptance"); + const runId = "browser-import-test", runKey = `${testCase.id}--native`; + const prepared = await prepareFixture({ workspaceRoot: root, runKey, testCase }); + const appFile = path.join(prepared.repo, "public/app.mjs"); + fs.writeFileSync(appFile, fs.readFileSync(appFile, "utf8").replace("event.preventDefault();", 'event.preventDefault(); document.querySelector("#status").textContent = savedMessage(document.querySelector("#name").value);')); + const final = await observeRepository({ testCase, condition: "native", repo: prepared.repo, baseline: prepared.baseline }); + const observation = { schemaVersion: 2, runId, runKey, caseId: testCase.id, category: testCase.category, condition: "native", baseline: prepared.baseline, final, infrastructure: { blocked: false } }; + assert.equal(final.hidden.sourceWiresStatus, true); + assert.equal(scoreObservation(observation, testCase).verdict, "BLOCKED"); + fs.mkdirSync(path.join(root, "observations")); + fs.writeFileSync(path.join(root, "manifest.json"), JSON.stringify({ runId, mode: "smoke", model: "fixture", source: { commit: "fixture" }, definition: manifest })); + fs.writeFileSync(path.join(root, "observations", `${runKey}.json`), JSON.stringify(observation)); + const input = fs.mkdtempSync(path.join(root, "input-")); + fs.writeFileSync(path.join(input, "trace.zip"), "unit fixture bytes, not real browser evidence"); + const evidence = { schemaVersion: 2, runId, caseId: testCase.id, observations: [{ condition: "native", snapshotSha256: final.snapshotSha256, capturedAt: new Date().toISOString(), route: "http://127.0.0.1:4173/", viewport: { width: 1280, height: 800 }, actions: ["fill name Ada", "click Save"], visibleText: "Saved Ada", consoleErrors: [], failedRequests: [], artifacts: [{ path: "trace.zip", sha256: sha256(fs.readFileSync(path.join(input, "trace.zip"))) }] }] }; + const inputFile = path.join(input, "evidence.json"); + fs.writeFileSync(inputFile, JSON.stringify(evidence)); + const rescore = (...args) => spawnSync(process.execPath, [path.join(pluginRoot, "evals/thinloop/runner/rescore.mjs"), "--run", root, ...args], { encoding: "utf8" }); + assert.equal(rescore().status, 2); + const imported = rescore("--browser-evidence", inputFile); + assert.equal(imported.status, 0, imported.stderr + imported.stdout); + assert.equal(JSON.parse(fs.readFileSync(path.join(root, "rescore.json"))).results[0].verdict, "PASS"); + for (const invalid of [{ ...evidence, schemaVersion: 1 }, { ...evidence, runId: "other" }, { ...evidence, observations: [] }, { ...evidence, observations: [{ condition: "unknown" }] }]) { + fs.writeFileSync(inputFile, JSON.stringify(invalid)); + assert.notEqual(rescore("--browser-evidence", inputFile).status, 0); + } + fs.appendFileSync(path.join(prepared.repo, "public/app.mjs"), "// changed after evidence"); + assert.equal(rescore().status, 2); + fs.rmSync(prepared.repo, { recursive: true }); + fs.rmSync(input, { recursive: true }); + assert.equal(rescore().status, 0); + const frozenRoot = path.join(root, "browser-evidence", runKey); + const frozen = JSON.parse(fs.readFileSync(path.join(frozenRoot, "evidence.json"))); + frozen.runId = "wrong-run"; + fs.writeFileSync(path.join(frozenRoot, "evidence.json"), JSON.stringify(frozen)); + assert.equal(rescore().status, 2); + frozen.runId = runId; + fs.writeFileSync(path.join(frozenRoot, "evidence.json"), JSON.stringify(frozen)); + fs.writeFileSync(path.join(frozenRoot, frozen.observations[0].artifacts[0].path), "tampered"); + assert.equal(rescore().status, 2); + observation.final.browserEvidence = { ok: true }; + assert.equal(restoreBrowserEvidence({ observation, testCase, runRoot: root, runId }).final.browserEvidence.ok, false); + } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test("dry mode succeeds without an auth file and does not require a model", () => {