mindcarver · mindcarver · Sep 5, 2026 · Sep 5, 2026 · Sep 5, 2026
diff --git a/EVALUATION.md b/EVALUATION.md
@@ -363,3 +363,7 @@ node evals\score-runs.mjs
 ```
 
 客观评分原始结果保存在仓库的 `work/scd-dev-loop-evals/objective-results.json`，没有打入发布包。
+
+## 浏览器证据的实现后导入
+
+当前页面评测先完成实现并保存代码快照，随后通过 `rescore.mjs --run <目录> --browser-evidence <文件>` 导入真实浏览器记录。记录绑定 runId、case、condition、代码快照及采集时间，产物必须存在且 SHA-256 匹配；离线重评分重新校验冻结记录与产物，不直接信任保存的通过标记。缺失证据保持 BLOCKED。具体格式及采集命令见 [当前版本评测说明](evals/thinloop/README.md#证据与重评分)。这项修复改善证据绑定与完整性，不代表已经重新测量模型整体收益。
diff --git a/evals/thinloop/README.md b/evals/thinloop/README.md
@@ -23,10 +23,10 @@ node evals/thinloop/runner/run.mjs --mode dry
 node evals/thinloop/runner/run.mjs --mode smoke --run-id <unique-id>
 ```
 
-`full` 运行七类任务的全部三个条件。页面用例还必须用 `--browser-evidence <json>` 提供三个条件各自的真实交互观察；缺少时该用例直接 `BLOCKED`，不会用测试或源码检查替代浏览器：
+`full` 先运行七类任务的全部三个条件，包括页面实现。页面实现和源码/测试观察会保存下来，浏览器验收暂为 `BLOCKED`；不再接受实现前的 `--browser-evidence`：
 
 ```bash
-node evals/thinloop/runner/run.mjs --mode full --run-id <unique-id> --browser-evidence <file>
+node evals/thinloop/runner/run.mjs --mode full --run-id <unique-id>
 ```
 
 可以用 `--case <id>`、`--conditions native,prompt,thinloop`、`--workspace <path>`、`--model <model>` 和 `--reasoning <effort>` 缩小诊断。Run ID 永不覆盖。
@@ -43,7 +43,39 @@ node evals/thinloop/runner/run.mjs --mode full --run-id <unique-id> --browser-ev
 - `summary.json` 与 `report.md`：逐任务和逐条件聚合；
 - 秘密扫描结论包含在 summary/report 中。
 
-临时 Codex Home 在每个 subject 后删除。重评分只读取冻结定义和 `observations/`，不需要模型、认证或原 fixture 仓库：
+临时 Codex Home 在每个 subject 后删除。页面用例采用两步流程：
+
+1. 完成上面的 run，从 `observations/browser-form-acceptance--<condition>.json` 读取 `runId`、`caseId`、`condition`、`final.snapshotSha256` 和 `final.observedAt`。
+2. 从该 run 的 `repositories/browser-form-acceptance--<condition>/public/` 启动静态服务。真实浏览器打开页面，填写 Ada，点击 Save，观察 Saved Ada；保存截图或 trace 到 run 之外的证据目录。采集期间不要修改 fixture。为每个已运行条件保存以下记录（用实际值替换示例）。`capturedAt` 必须晚于 `final.observedAt`，产物路径相对于这份 JSON 的目录，SHA-256 必须来自实际文件字节：
+
+```json
+{
+  "schemaVersion": 2,
+  "runId": "本次运行标识",
+  "caseId": "browser-form-acceptance",
+  "observations": [{
+    "condition": "native",
+    "snapshotSha256": "final.snapshotSha256 的值",
+    "capturedAt": "实际采集时间 ISO 8601",
+    "route": "http://127.0.0.1:4173/",
+    "viewport": { "width": 1280, "height": 800 },
+    "actions": ["fill #name with Ada", "click Save"],
+    "visibleText": "Saved Ada",
+    "consoleErrors": [],
+    "failedRequests": [],
+    "artifacts": [{ "path": "save.png", "sha256": "文件的 64 位小写 SHA-256" }]
+  }]
+}
+```
+
+```bash
+shasum -a 256 /path/to/evidence/save.png
+node evals/thinloop/runner/rescore.mjs --run <saved-run-directory> --browser-evidence /path/to/evidence/evidence.json
+```
+
+导入时必须保留原 fixture，当前文件快照必须与实现结束时一致。有效记录及文件被复制到 `browser-evidence/<runKey>/`；初始 observations 保留原始待验收事实，新的结果写入 `rescore.json` 和 `rescore-report.md`。旧 schema、跨运行/条件/代码快照、缺失/空/变更文件均不能通过。
+
+后续离线重评分读取冻结定义、`observations/` 和冻结浏览器证据，每次重新验证绑定和文件内容，不信任原先的 `ok`。不需要模型、认证或原 fixture；fixture 仍存在时还会检查它未改变：
 
 ```bash
 node evals/thinloop/runner/rescore.mjs --run <saved-run-directory>
@@ -56,3 +88,5 @@ node evals/thinloop/runner/rescore.mjs --run <saved-run-directory>
 - 整轮 `OBSERVED`：所选真实任务都产生了可评分观察且秘密扫描通过。
 
 行为 `FAIL` 是评测数据，不会让一轮可复现评测本身失败。任何秘密发现让整轮 `FAIL`；任何必需路径 `BLOCKED` 让整轮 `BLOCKED`。单轮或单模型结果只支持对应 fixture 的描述性观察，不构成统计显著性或整体价值百分比。
+
+哈希证明证据与运行/代码的绑定及文件完整性，不能独立证明外部填写的动作声明真实。实际交互仍须由浏览器采集者和独立验收者检查。离线评分针对冻结的实现快照，不声明另一个当前检出已通过验收。
diff --git a/evals/thinloop/runner/browser-evidence.mjs b/evals/thinloop/runner/browser-evidence.mjs
@@ -0,0 +1,81 @@
+import fs from "node:fs";
+import path from "node:path";
+import { sha256, writeJson } from "../../discovery/runner/lib.mjs";
+import { snapshotFiles } from "./fixture.mjs";
+
+export function snapshotSha256(files) {
+  return sha256(JSON.stringify(Object.entries(files).sort(([a], [b]) => a.localeCompare(b))));
+}
+
+export function validateBrowserEvidence({ evidence, testCase, condition, runId, final, artifactRoot }) {
+  if (!testCase.requiresBrowserEvidence) return null;
+  const fail = (reason) => ({ ok: false, reason });
+  if (!runId || evidence?.schemaVersion !== 2 || evidence.runId !== runId || evidence.caseId !== testCase.id) {
+    return fail("browser evidence schema/run/case binding mismatch");
+  }
+  const items = Array.isArray(evidence.observations) ? evidence.observations.filter((entry) => entry?.condition === condition) : [];
+  if (items?.length !== 1) return fail("exactly one browser observation is required for this condition");
+  const item = items[0];
+  if (!final?.files || !final.observedAt || final.snapshotSha256 !== snapshotSha256(final.files) || item.snapshotSha256 !== final.snapshotSha256) {
+    return fail("browser evidence code snapshot mismatch");
+  }
+  const capturedAt = Date.parse(item.capturedAt);
+  const observedAt = Date.parse(final.observedAt);
+  if (!Number.isFinite(capturedAt) || !Number.isFinite(observedAt) || capturedAt < observedAt || capturedAt > Date.now()) {
+    return fail("browser evidence must be captured after the implementation snapshot");
+  }
+  if (!(typeof item.route === "string" && /^https?:\/\//.test(item.route) &&
+    Number(item.viewport?.width) > 0 && Number(item.viewport?.height) > 0 &&
+    Array.isArray(item.actions) && item.actions.every((action) => typeof action === "string") &&
+    item.actions.some((action) => /(?:click|submit|Save)/i.test(action)) && item.visibleText === "Saved Ada" &&
+    Array.isArray(item.consoleErrors) && item.consoleErrors.length === 0 &&
+    Array.isArray(item.failedRequests) && item.failedRequests.length === 0 &&
+    Array.isArray(item.artifacts) && item.artifacts.length > 0)) return fail("browser evidence is incomplete");
+  try {
+    if (!artifactRoot) return fail("browser artifact root is required");
+    for (const artifact of item.artifacts) {
+      if (typeof artifact?.path !== "string" || !/^[a-f0-9]{64}$/.test(artifact.sha256)) return fail("browser artifact needs a path and SHA-256");
+      const file = path.resolve(artifactRoot, artifact.path);
+      const relative = path.relative(fs.realpathSync(artifactRoot), fs.realpathSync(file));
+      if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) return fail("browser artifact must stay inside its evidence directory");
+      if (!fs.statSync(file).isFile() || fs.statSync(file).size === 0 || sha256(fs.readFileSync(file)) !== artifact.sha256) return fail("browser artifact content hash mismatch or empty artifact");
+    }
+  } catch (error) {
+    return fail(`browser artifact unavailable: ${error.code ?? error.message}`);
+  }
+  return { ok: true, observation: item };
+}
+
+// Always revalidate bytes and bindings; a saved observation's `ok` is not authority.
+export function restoreBrowserEvidence({ observation, testCase, runRoot, runId, evidence, evidenceRoot }) {
+  if (!testCase.requiresBrowserEvidence) return observation;
+  const next = structuredClone(observation);
+  const fail = (reason) => { next.final.browserEvidence = { ok: false, reason }; return next; };
+  if (observation.runId !== runId || observation.caseId !== testCase.id || observation.runKey !== `${testCase.id}--${observation.condition}` || !/^[a-z0-9-]+--[a-z0-9-]+$/.test(observation.runKey)) return fail("saved browser observation run binding mismatch");
+  const repo = path.join(runRoot, "repositories", observation.runKey);
+  if (evidence && !fs.existsSync(repo)) return fail("browser evidence import requires the implemented fixture repository");
+  if (fs.existsSync(repo) && snapshotSha256(snapshotFiles(repo)) !== observation.final.snapshotSha256) return fail("implemented fixture changed since the saved snapshot");
+  const frozenRoot = path.join(runRoot, "browser-evidence", observation.runKey);
+  let record = evidence;
+  try {
+    record ??= JSON.parse(fs.readFileSync(path.join(frozenRoot, "evidence.json"), "utf8"));
+  } catch {
+    return fail("awaiting post-implementation browser evidence import");
+  }
+  let browser = validateBrowserEvidence({ evidence: record, testCase, condition: observation.condition, runId, final: observation.final, artifactRoot: evidence ? evidenceRoot : frozenRoot });
+  if (browser.ok && evidence) {
+    fs.mkdirSync(frozenRoot, { recursive: true });
+    const item = structuredClone(browser.observation);
+    item.artifacts = item.artifacts.map((artifact) => {
+      const name = `${artifact.sha256}${path.extname(artifact.path)}`;
+      fs.copyFileSync(path.resolve(evidenceRoot, artifact.path), path.join(frozenRoot, name));
+      return { path: name, sha256: artifact.sha256 };
+    });
+    record = { schemaVersion: 2, runId, caseId: testCase.id, observations: [item] };
+    writeJson(path.join(frozenRoot, "evidence.json"), record);
+    browser = validateBrowserEvidence({ evidence: record, testCase, condition: observation.condition, runId, final: observation.final, artifactRoot: frozenRoot });
+  }
+  next.final.browserEvidence = browser;
+  next.final.hidden = { ...next.final.hidden, ok: next.final.hidden?.sourceWiresStatus === true && browser.ok, browserEvidence: browser.ok };
+  return next;
+}
diff --git a/evals/thinloop/runner/observe.mjs b/evals/thinloop/runner/observe.mjs
@@ -1,6 +1,8 @@
 import fs from "node:fs";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
+import { snapshotSha256 } from "./browser-evidence.mjs";
+export { validateBrowserEvidence } from "./browser-evidence.mjs";
 import { changedFiles, repositoryStatus, runNativeTests, snapshotFiles } from "./fixture.mjs";
 
 async function importFresh(file) {
@@ -60,36 +62,19 @@ async function hiddenCheck({ testCase, repo, lastMessage, browserEvidence }) {
   }
 }
 
-export function validateBrowserEvidence({ evidence, testCase, condition }) {
-  if (!testCase.requiresBrowserEvidence) return null;
-  const item = evidence?.observations?.find((entry) => entry.condition === condition);
-  if (!item) return { ok: false, reason: "no structured real-browser observation was supplied for this condition" };
-  const ok =
-    evidence.schemaVersion === 1 &&
-    evidence.caseId === testCase.id &&
-    item.route?.trim() &&
-    Number(item.viewport?.width) > 0 &&
-    Number(item.viewport?.height) > 0 &&
-    Array.isArray(item.actions) &&
-    item.actions.some((action) => /(?:click|submit|Save)/i.test(action)) &&
-    item.visibleText === "Saved Ada" &&
-    Array.isArray(item.consoleErrors) && item.consoleErrors.length === 0 &&
-    Array.isArray(item.failedRequests) && item.failedRequests.length === 0 &&
-    item.artifact?.trim();
-  return { ok: Boolean(ok), reason: ok ? undefined : "browser evidence is incomplete", observation: item };
-}
-
-export async function observeRepository({ testCase, condition, repo, baseline, lastMessage = "", browserEvidence }) {
+export async function observeRepository({ testCase, condition, repo, baseline, lastMessage = "" }) {
   const files = snapshotFiles(repo);
   const nativeTests = await runNativeTests(repo);
   const status = await repositoryStatus(repo);
   const changed = changedFiles(baseline.files, files);
   const dirtyFilesPreserved = Object.keys(testCase.dirtyFiles ?? {}).every((file) => baseline.files[file] === files[file]);
   const recoveryStateCleared = !fs.existsSync(path.join(repo, ".scd", "tasks", "current.md"));
-  const browser = validateBrowserEvidence({ evidence: browserEvidence, testCase, condition });
+  const browser = testCase.requiresBrowserEvidence ? { ok: false, reason: "awaiting post-implementation browser evidence import" } : null;
   const hidden = await hiddenCheck({ testCase, repo, lastMessage, browserEvidence: browser });
   return {
     files,
+    snapshotSha256: snapshotSha256(files),
+    observedAt: new Date().toISOString(),
     changedFiles: changed,
     nativeTests,
     ...status,

diff --git a/evals/thinloop/runner/rescore.mjs b/evals/thinloop/runner/rescore.mjs
@@ -3,6 +3,7 @@ import os from "node:os";
 import path from "node:path";
 import { parseArgs, writeJson, writeText } from "../../discovery/runner/lib.mjs";
 import { createRedactor, scanTree } from "../../discovery/runner/redact.mjs";
+import { restoreBrowserEvidence } from "./browser-evidence.mjs";
 import { reportMarkdown } from "./report.mjs";
 import { aggregateResults, scoreObservation } from "./scoring.mjs";
 
@@ -16,7 +17,18 @@ const observations = fs.readdirSync(observationsRoot)
   .sort()
   .map((file) => JSON.parse(fs.readFileSync(path.join(observationsRoot, file), "utf8")));
 const cases = new Map(runManifest.definition.cases.map((testCase) => [testCase.id, testCase]));
-const results = observations.map((observation) => scoreObservation(observation, cases.get(observation.caseId)));
+const evidenceFile = args["browser-evidence"] ? path.resolve(args["browser-evidence"]) : undefined;
+const evidence = evidenceFile ? JSON.parse(fs.readFileSync(evidenceFile, "utf8")) : undefined;
+if (evidence && (evidence.schemaVersion !== 2 || evidence.runId !== runManifest.runId || !observations.some((observation) => observation.caseId === evidence.caseId) ||
+  !Array.isArray(evidence.observations) || evidence.observations.length === 0 || evidence.observations.some((item) => !observations.some((observation) => observation.caseId === evidence.caseId && observation.condition === item?.condition)))) {
+  throw new Error("browser evidence schema/run/case/condition binding is invalid");
+}
+const results = observations.map((observation) => {
+  const testCase = cases.get(observation.caseId);
+  const restored = restoreBrowserEvidence({ observation, testCase, runRoot, runId: runManifest.runId,
+    evidence: evidence?.caseId === observation.caseId && evidence.observations.some((item) => item?.condition === observation.condition) ? evidence : undefined, evidenceRoot: evidenceFile && path.dirname(evidenceFile) });
+  return scoreObservation(restored, testCase);
+});
 const leaks = scanTree(runRoot, createRedactor({ auth: {}, userProfile: os.homedir() }));
 const summary = aggregateResults({ results, leaks });
 writeJson(path.join(runRoot, "rescore.json"), { run: runManifest, summary, results });

diff --git a/evals/thinloop/runner/run.mjs b/evals/thinloop/runner/run.mjs
@@ -28,7 +28,7 @@ import {
   selectCases,
   validateManifest,
 } from "./manifest.mjs";
-import { observeRepository, validateBrowserEvidence } from "./observe.mjs";
+import { observeRepository } from "./observe.mjs";
 import { reportMarkdown } from "./report.mjs";
 import { aggregateResults, scoreObservation } from "./scoring.mjs";
 
@@ -89,28 +89,9 @@ async function dryRun(manifest) {
   }
 }
 
-async function runSingle({ testCase, condition, runRoot, authFile, model, reasoning, redactor, pricing, browserEvidence }) {
+async function runSingle({ testCase, condition, runRoot, authFile, model, reasoning, redactor, pricing, runId }) {
   const runKey = `${testCase.id}--${condition.id}`;
   const fixture = await prepareFixture({ workspaceRoot: runRoot, runKey, testCase });
-  const browser = validateBrowserEvidence({ evidence: browserEvidence, testCase, condition: condition.id });
-  if (testCase.requiresBrowserEvidence && browser?.ok !== true) {
-    const observation = {
-      schemaVersion: 1,
-      runKey,
-      caseId: testCase.id,
-      category: testCase.category,
-      condition: condition.id,
-      context: { id: condition.id, context: condition.context, skills: [] },
-      infrastructure: { blocked: true, reason: browser.reason, evidence: "runner requires --browser-evidence with one real interaction observation per condition" },
-      baseline: fixture.baseline,
-      final: { ...fixture.baseline, changedFiles: [], browserEvidence: browser },
-      subject: { lastMessage: "", durationMs: 0, metrics: { usage: {}, toolCalls: 0 } },
-      pricing,
-    };
-    writeJson(path.join(runRoot, "observations", `${runKey}.json`), observation);
-    return { observation, result: scoreObservation(observation, testCase), secretRedactions: 0 };
-  }
-
   const homes = createIsolatedHomes({ authFile, prefix: "thinloop-eval-current-" });
   let context;
   let login;
@@ -152,7 +133,6 @@ async function runSingle({ testCase, condition, runRoot, authFile, model, reason
       repo: fixture.repo,
       baseline: fixture.baseline,
       lastMessage: subject?.lastMessage ?? "",
-      browserEvidence,
     });
   } catch (error) {
     infrastructure = { blocked: true, reason: "subject infrastructure error", evidence: redactor(error.stack ?? error.message).text };
@@ -161,7 +141,8 @@ async function runSingle({ testCase, condition, runRoot, authFile, model, reason
     cleanupIsolatedHomes(homes.root);
   }
   const observation = {
-    schemaVersion: 1,
+    schemaVersion: 2,
+    runId,
     runKey,
     caseId: testCase.id,
     category: testCase.category,
@@ -209,7 +190,6 @@ async function realRun({ args, manifest, mode }) {
   const auth = readJson(authFile);
   const redactor = createRedactor({ auth, userProfile: os.homedir() });
   const pricing = pricingFrom(args);
-  const browserEvidence = args["browser-evidence"] ? readJson(path.resolve(args["browser-evidence"])) : undefined;
   const runManifest = {
     schemaVersion: 1,
     runId,
@@ -238,7 +218,7 @@ async function realRun({ args, manifest, mode }) {
         reasoning: runManifest.reasoning,
         redactor,
         pricing,
-        browserEvidence,
+        runId,
       });
       results.push(outcome.result);
       secretRedactions += outcome.secretRedactions;
@@ -264,6 +244,7 @@ async function realRun({ args, manifest, mode }) {
 
 async function main() {
   const args = parseArgs(process.argv.slice(2));
+  if (args["browser-evidence"]) throw new Error("collect browser evidence after implementation; import with rescore.mjs --run <directory> --browser-evidence <file>");
   const mode = args.mode ?? "dry";
   if (!new Set(["dry", "smoke", "full"]).has(mode)) throw new Error(`invalid mode: ${mode}`);
   const manifest = loadManifest();

diff --git a/evals/thinloop/runner/scoring.mjs b/evals/thinloop/runner/scoring.mjs
@@ -6,14 +6,14 @@ function countQuestions(message = "") {
 }
 
 export function scoreObservation(observation, testCase) {
-  if (observation.infrastructure?.blocked) {
+  if (observation.infrastructure?.blocked || (testCase.requiresBrowserEvidence && observation.final.browserEvidence?.ok !== true)) {
     return {
       runKey: observation.runKey,
       caseId: observation.caseId,
       category: observation.category,
       condition: observation.condition,
       verdict: "BLOCKED",
-      blocker: observation.infrastructure.reason,
+      blocker: observation.infrastructure?.blocked ? observation.infrastructure.reason : observation.final.browserEvidence?.reason ?? "browser evidence unavailable",
       metrics: metrics(observation, false),
     };
   }