From fca86e59f1a2d4bb05b6a0ffc9423c3bd6b5f1ee Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 10 Sep 2026 11:14:10 +0000 Subject: [PATCH] fix(proof): fail closed on incomplete harbor evaluate Reject nonzero Harbor exits, score every measured trial, fail truncated off-limits scans and unknown inspect rules, and refuse import paths whose origin is outside the staged artefact. Co-authored-by: Mathis --- .../runners/rlm_fc_in_guest_harbor/README.md | 13 ++- .../rlm_fc_in_guest_harbor/harness/run-harbor | 7 ++ .../harness/summarize.py | 31 +++++--- .../rlm_fc_in_guest_harbor/inspect_scan.py | 60 +++++++++----- .../runners/rlm_fc_in_guest_harbor/lib.sh | 5 +- .../rlm_fc_in_guest_harbor/resolve_agent.py | 79 ++++++++++++++++++- .../tests/test_adaptor.sh | 51 ++++++++++++ .../tests/test_inspect_scan.py | 62 +++++++++++++++ .../tests/test_resolve_agent.py | 58 ++++++++++++++ .../tests/test_summarize.py | 64 +++++++++++++++ docs/external-miner/proof-tbench.md | 2 +- 11 files changed, 395 insertions(+), 37 deletions(-) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md index 84c123346..c11819f02 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md @@ -34,7 +34,8 @@ Harbor **does not** accept a filesystem path for `-a`. This adaptor therefore A one-line `import_path` file inside the agent dir (contents `module.path:ClassName`) wins when several classes exist. A built-in name in that file is refused: that would ignore miner code the same way `terminus-2` -did. +did. The named module is resolved in the evaluate import env (artefact +parent only) and **rejected** if its origin is outside the staged artefact. ## Agent selection (miner attach surface) @@ -81,7 +82,9 @@ Off-limits in the artefact (inspect fails the named rule): - `no_eval_short_circuit` (and `skip_eval` / `skip_verifier` / `always_pass_eval` / `short_circuit_eval`) - `no_tb4_hardcoding` (and `tb4_answers` / `hardcoded_tb4`) -Do not quote those markers in miner code or README inside the tar. +A file/byte-limit truncation marks the scan incomplete and fails those +off-limits rules. Unknown rule ids fail closed. Do not quote those markers +in miner code or README inside the tar. ## BYOK @@ -137,8 +140,10 @@ the task list. } ``` -`primary_value` is the mean of Harbor trial `verifier_result.rewards.reward` -only. No measured trial → fail closed, no invented number. +`primary_value` is the mean of **every** Harbor trial +`verifier_result.rewards.reward`. Evidence may truncate the serialized +trial list; the mean does not. No measured trial, or a nonzero Harbor +exit, → fail closed, no invented number and no leftover `report.json`. `inspect` writes `$PROOF_OUTPUT_DIR/checklist.json` (no Harbor, no keys). diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/run-harbor b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/run-harbor index 2163452cb..fc23bae81 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/run-harbor +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/run-harbor @@ -62,6 +62,13 @@ set +e harbor_exit=$? set -e +# Nonzero Harbor is an incomplete or failed trial set. Do not invoke the +# summarizer: a leftover report.json would be a successful score. +if [ "$harbor_exit" -ne 0 ]; then + rm -f "$PROOF_OUTPUT_DIR/report.json" + proof_die "harbor exited $harbor_exit; refusing to score a partial or failed run" +fi + python3 "$HERE/summarize.py" \ --jobs-dir "$jobs_dir" \ --log "$log" \ diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py index b69966a98..e3b9ea8ae 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 """Turn a Harbor jobs directory into Proof ``report.json``. -``primary_value`` is the mean of trial ``verifier_result.rewards.reward`` -values that are finite numbers. A trial with no such field is **no -measurement** — never another field's value (score, accuracy, job-level -aggregates) as a substitute. Zero measured trials → exit 2, no report. - -Evidence includes ``trials`` and a redacted ``harbor_run_tail``. Secret -values from the owner secrets dir and miner BYOK dir are blanked. +``primary_value`` is the mean of **every** trial +``verifier_result.rewards.reward`` that is a finite number. A trial with +no such field is **no measurement** — never another field's value (score, +accuracy, job-level aggregates) as a substitute. Zero measured trials → +exit 2, no report. A nonzero Harbor exit is an incomplete run → exit 2, +no report (do not publish a partial score). + +Evidence serializes at most ``MAX_EVIDENCE_TRIALS`` trial rows; the mean +always uses the full measured set. Secret values from the owner secrets +dir and miner BYOK dir are blanked. """ from __future__ import annotations @@ -60,12 +63,11 @@ def _load_json(path: Path) -> Any | None: def collect_trials(jobs_dir: Path) -> list[dict[str, Any]]: + """Load every measured trial. Do not cap here — the cap is evidence only.""" trials: list[dict[str, Any]] = [] if not jobs_dir.is_dir(): return trials for result_path in sorted(jobs_dir.rglob("result.json")): - if len(trials) >= MAX_EVIDENCE_TRIALS: - break obj = _load_json(result_path) reward = trial_reward(obj) if reward is None: @@ -171,12 +173,15 @@ def build_report( agent_source: str, ) -> dict[str, Any]: primary = mean_reward(trials) + evidence_trials = trials[:MAX_EVIDENCE_TRIALS] return { "primary_value": primary, "claim_holds": True, "evidence": { - "trials": trials, + "trials": evidence_trials, "n_measured": len(trials), + "n_evidence_trials": len(evidence_trials), + "evidence_truncated": len(trials) > MAX_EVIDENCE_TRIALS, "mean_reward": primary, "harbor_exit": harbor_exit, "harbor_run_tail": log_tail, @@ -196,6 +201,12 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--agent-source", default="") args = parser.parse_args(argv) + if args.harbor_exit != 0: + _fail( + f"harbor exited {args.harbor_exit}; refusing to publish a score " + "from a failed or incomplete run" + ) + jobs_dir = Path(args.jobs_dir) trials = collect_trials(jobs_dir) if not trials: diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py index d13e452f0..21d98fccf 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py @@ -4,8 +4,11 @@ Reads ``PROOF_RULES_FILE`` (a ``RuleSet`` object or a ``[{id, text}]`` array) and writes ``checklist.json``. Off-limits markers ``no_eval_short_circuit`` and ``no_tb4_hardcoding`` fail those rules when they appear in the artefact -tree. Host-side rules are answered with evidence, not left blank (a missing -item is recorded red). Secret file contents are never printed. +tree. A file/byte-limit truncation marks the scan incomplete and **fails** +those off-limits rules — truncated absence is not a clean pass. Unknown +rule IDs fail closed. Host-side rules are answered with evidence, not left +blank (a missing item is recorded red). Secret file contents are never +printed. """ from __future__ import annotations @@ -72,23 +75,31 @@ def load_rules(path: Path) -> list[dict[str, str]]: return out -def collect_artefact_text(root: Path | None) -> tuple[str, int, list[str]]: +def collect_artefact_text(root: Path | None) -> tuple[str, int, list[str], bool]: + """Return ``(text, n_scanned, names, incomplete)``. + + ``incomplete`` is true when a file or byte cap stopped the walk before + every regular file was considered. Callers must not treat a truncated + scan as proof that an off-limits marker is absent. + """ if root is None or not root.is_dir(): - return "", 0, [] + return "", 0, [], False blobs: list[str] = [] names: list[str] = [] total = 0 n_files = 0 + incomplete = False for path in sorted(root.rglob("*")): if not path.is_file(): continue rel = path.relative_to(root).as_posix() if ".." in Path(rel).parts: continue + if n_files >= MAX_FILES or total >= MAX_TOTAL_BYTES: + incomplete = True + break names.append(rel) n_files += 1 - if n_files > MAX_FILES or total > MAX_TOTAL_BYTES: - break try: size = path.stat().st_size except OSError: @@ -103,19 +114,31 @@ def collect_artefact_text(root: Path | None) -> tuple[str, int, list[str]]: continue total += len(data) blobs.append(data.decode("utf-8", errors="replace")) - return "\n".join(blobs).lower(), n_files, names + return "\n".join(blobs).lower(), n_files, names, incomplete def _contains_any(haystack: str, needles: tuple[str, ...]) -> list[str]: return [n for n in needles if n.lower() in haystack] +def _off_limits_incomplete(rid: str, n_files: int) -> dict[str, Any]: + return { + "id": rid, + "pass": False, + "evidence": _clip( + f"artefact scan incomplete (file/byte limit); cannot treat absence " + f"of off-limits markers as clean ({n_files} files scanned)" + ), + } + + def tick_rule( rule: dict[str, str], artefact_text: str, n_files: int, names: list[str], has_artefact: bool, + incomplete: bool = False, ) -> dict[str, Any]: rid = rule["id"] joined_names = " ".join(names).lower() @@ -132,6 +155,8 @@ def tick_rule( f"off-limits eval short-circuit marker in artefact: {', '.join(hits)}" ), } + if incomplete: + return _off_limits_incomplete(rid, n_files) return { "id": rid, "pass": True, @@ -150,6 +175,8 @@ def tick_rule( f"off-limits tb4 hardcoding marker in artefact: {', '.join(hits)}" ), } + if incomplete: + return _off_limits_incomplete(rid, n_files) return { "id": rid, "pass": True, @@ -184,20 +211,12 @@ def tick_rule( ), } - if not has_artefact: - return { - "id": rid, - "pass": False, - "evidence": _clip( - f"rule {rid}: no artefact tree staged for inspect (PROOF_ARTIFACT_DIR unset or empty)" - ), - } return { "id": rid, - "pass": True, + "pass": False, "evidence": _clip( - f"rule {rid}: artefact present ({n_files} files); no paid inference; " - "no off-limits short-circuit/tb4 markers beyond the dedicated probes" + f"rule {rid}: unsupported/unknown rule id; inspect fails closed " + f"(artefact_present={has_artefact}, files={n_files})" ), } @@ -213,10 +232,11 @@ def main(argv: list[str] | None = None) -> int: artefact_raw = os.environ.get("PROOF_ARTIFACT_DIR", "") artefact_dir = Path(artefact_raw) if artefact_raw else None has_artefact = artefact_dir is not None and artefact_dir.is_dir() - text, n_files, names = collect_artefact_text(artefact_dir) + text, n_files, names, incomplete = collect_artefact_text(artefact_dir) items = [ - tick_rule(rule, text, n_files, names, has_artefact) for rule in load_rules(rules_path) + tick_rule(rule, text, n_files, names, has_artefact, incomplete) + for rule in load_rules(rules_path) ] out = output_dir / "checklist.json" out.write_text(json.dumps(items, indent=2) + "\n", encoding="utf-8") diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh b/deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh index 7dee83e6f..b43a5a8d9 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh @@ -93,7 +93,10 @@ proof_select_harbor_agent() { pythonpath="$(printf '%s\n' "$resolved" | sed -n 's/^pythonpath=//p' | head -n1)" [ -n "$import_path" ] || return 1 if [ -n "$pythonpath" ]; then - export PYTHONPATH="${pythonpath}${PYTHONPATH:+:$PYTHONPATH}" + # Evaluate import env is the staged artefact parent only. + # Inherited PYTHONPATH would let a miner import_path name a + # module that lives outside the artefact. + export PYTHONPATH="$pythonpath" fi chosen="$import_path" source="$label" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/resolve_agent.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/resolve_agent.py index 435ff1337..67f4e4129 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/resolve_agent.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/resolve_agent.py @@ -9,19 +9,24 @@ This helper turns an artefact directory into ``module:Class`` so evaluate can pass ``-a`` without silently falling back to the topic's built-in agent. -Miner code is **not** executed: class discovery is AST-only. +Miner code is **not** executed: class discovery is AST-only. A named +``import_path`` is resolved in the evaluate import env (artefact parent +only) and rejected when the origin is outside the staged artefact. """ from __future__ import annotations import argparse import ast +import importlib.machinery +import os import sys from pathlib import Path AGENT_BASES = frozenset({"BaseAgent", "BaseInstalledAgent"}) MAX_PY_BYTES = 256 * 1024 MAX_PY_FILES = 64 +_NON_FILE_ORIGINS = frozenset({"built-in", "frozen"}) def _fail(msg: str, code: int = 2) -> None: @@ -48,11 +53,82 @@ def _read_import_path_file(agent_dir: Path) -> str | None: ) if ".." in line or "/" in line or "\\" in line: _fail(f"{path} is not a Python import path: {line!r}") + _assert_import_origin(line, agent_dir) return line _fail(f"{path} is empty") return None +def _containment_root(agent_dir: Path) -> Path: + raw = os.environ.get("PROOF_ARTIFACT_DIR", "").strip() + if raw: + art = Path(raw) + if art.is_dir(): + return art.resolve() + return agent_dir.resolve() + + +def _is_inside(path: Path, root: Path) -> bool: + try: + path.resolve().relative_to(root.resolve()) + return True + except (OSError, ValueError): + return False + + +def _module_file_candidates(search_root: Path, module_name: str) -> list[Path]: + parts = [p for p in module_name.split(".") if p] + if not parts or any(p in {".", ".."} or not p.isidentifier() for p in parts): + return [] + base = search_root.joinpath(*parts) + return [Path(str(base) + ".py"), base / "__init__.py"] + + +def _resolve_module_origin(module_name: str, search_root: Path) -> Path | None: + """Resolve ``module_name`` using only ``search_root`` (no inherited PYTHONPATH). + + Miner code is not executed. PathFinder + on-disk candidates only. + """ + try: + spec = importlib.machinery.PathFinder.find_spec(module_name, [str(search_root)]) + except KeyError: + # Parent package is not on sys.modules (implicit namespace / first look). + spec = None + if spec is not None and isinstance(spec.origin, str) and spec.origin not in _NON_FILE_ORIGINS: + origin = Path(spec.origin) + if origin.is_file(): + return origin + for candidate in _module_file_candidates(search_root, module_name): + if candidate.is_file(): + return candidate + return None + + +def _assert_import_origin(import_path: str, agent_dir: Path) -> None: + """Reject ``module:Class`` whose origin is outside the staged artefact.""" + module_name, sep, class_name = import_path.partition(":") + if not sep or not module_name or not class_name: + _fail(f"{import_path!r} is not a Python import path module.path:ClassName") + if not class_name.isidentifier(): + _fail(f"{import_path!r} class name is not a Python identifier") + search_root = agent_dir.resolve().parent + origin = _resolve_module_origin(module_name, search_root) + if origin is None: + _fail( + f"{import_path} does not resolve to a module under {search_root} " + "(inherited PYTHONPATH / stdlib / Harbor installs are not miner code)" + ) + bound = _containment_root(agent_dir) + if not _is_inside(origin, bound): + _fail( + f"{import_path} origin {origin.resolve()} is outside staged artefact {bound}" + ) + if not _is_inside(origin, agent_dir) and not _is_inside(origin, search_root): + _fail( + f"{import_path} origin {origin.resolve()} is outside the evaluate import root" + ) + + def _base_attr(node: ast.expr) -> str | None: if isinstance(node, ast.Name): return node.id @@ -137,6 +213,7 @@ def discover(agent_dir: Path) -> tuple[str, str]: f"{agent_dir} has multiple Harbor agent classes ({', '.join(unique_paths)}); " "write a one-line import_path file (module.path:ClassName) to choose" ) + _assert_import_origin(discovered[0][0], agent_dir) return discovered[0] diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh index 056bb880d..ff5fbf937 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh @@ -164,4 +164,55 @@ assert "terminus-2" not in json.dumps(r) PY pass "evaluate run-harbor passes miner -a, not terminus-2" +# --- nonzero Harbor exit must not write a successful report --- +cat > "$FAKE_BIN/harbor" <<'EOF' +#!/bin/bash +set -euo pipefail +jobs="" +while [ $# -gt 0 ]; do + case "$1" in + --jobs-dir) jobs="$2"; shift 2 ;; + *) shift ;; + esac +done +job="$jobs/job1/hello__1" +mkdir -p "$job" +cat > "$job/result.json" <"$WORKDIR/partial.out" 2>"$WORKDIR/partial.err"; then + fail "nonzero harbor exit must fail closed" +fi +[ ! -f "$PARTIAL_OUT/report.json" ] || fail "must not write report.json after harbor exit 23" +grep -qi "harbor exited 23\\|refusing to score" "$WORKDIR/partial.err" || fail "must name the harbor failure" +pass "nonzero harbor exit fails closed (no report)" + +# --- import_path naming a module only on inherited PYTHONPATH --- +ESCAPE="$WORKDIR/escape" +mkdir -p "$ESCAPE/artifact/agent" "$ESCAPE/external" +printf 'outside_agent:ExternalAgent\n' > "$ESCAPE/artifact/agent/import_path" +cat > "$ESCAPE/external/outside_agent.py" <<'PY' +from harbor.agents.base import BaseAgent +class ExternalAgent(BaseAgent): + pass +PY +export PROOF_JOB=evaluate +export PROOF_ARTIFACT_DIR="$ESCAPE/artifact" +export PYTHONPATH="$ESCAPE/external${PYTHONPATH:+:$PYTHONPATH}" +if (proof_select_harbor_agent) >"$WORKDIR/escape.out" 2>"$WORKDIR/escape.err"; then + fail "import_path outside the artefact must fail before Harbor" +fi +if grep -q "outside_agent:ExternalAgent" "$WORKDIR/escape.out"; then + fail "must not emit an escaped import path" +fi +pass "import_path outside artefact is refused before Harbor" + echo "all adaptor tests passed" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py index ec8ce7f9c..8cd49667d 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py @@ -94,6 +94,68 @@ def test_clean_artefact_passes_off_limits(self) -> None: self.assertTrue(items["no_tb4_hardcoding"]["pass"]) self.assertTrue(items["same_seed"]["pass"]) + def test_truncated_scan_fails_off_limits(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + art = root / "artifact" + art.mkdir() + for i in range(inspect_scan.MAX_FILES): + (art / f"a-{i:03d}.txt").write_text("benign\n", encoding="utf-8") + (art / "z-forbidden.txt").write_text("no_tb4_hardcoding\n", encoding="utf-8") + rules = root / "rules.json" + rules.write_text( + json.dumps( + [ + {"id": "no_eval_short_circuit", "text": "x"}, + {"id": "no_tb4_hardcoding", "text": "x"}, + ] + ), + encoding="utf-8", + ) + out = root / "out" + out.mkdir() + os.environ["PROOF_RULES_FILE"] = str(rules) + os.environ["PROOF_OUTPUT_DIR"] = str(out) + os.environ["PROOF_ARTIFACT_DIR"] = str(art) + try: + self.assertEqual(inspect_scan.main([]), 0) + finally: + os.environ.pop("PROOF_RULES_FILE", None) + os.environ.pop("PROOF_OUTPUT_DIR", None) + os.environ.pop("PROOF_ARTIFACT_DIR", None) + items = {i["id"]: i for i in json.loads((out / "checklist.json").read_text())} + self.assertFalse(items["no_tb4_hardcoding"]["pass"]) + self.assertFalse(items["no_eval_short_circuit"]["pass"]) + self.assertIn("incomplete", items["no_tb4_hardcoding"]["evidence"]) + + def test_unknown_rule_fails_closed_with_artefact(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + art = root / "artifact" + art.mkdir() + (art / "note.txt").write_text("unrelated\n", encoding="utf-8") + rules = root / "rules.json" + rules.write_text( + json.dumps( + [{"id": "must_provide_reproducible_benchmark", "text": "prove it"}] + ), + encoding="utf-8", + ) + out = root / "out" + out.mkdir() + os.environ["PROOF_RULES_FILE"] = str(rules) + os.environ["PROOF_OUTPUT_DIR"] = str(out) + os.environ["PROOF_ARTIFACT_DIR"] = str(art) + try: + self.assertEqual(inspect_scan.main([]), 0) + finally: + os.environ.pop("PROOF_RULES_FILE", None) + os.environ.pop("PROOF_OUTPUT_DIR", None) + os.environ.pop("PROOF_ARTIFACT_DIR", None) + items = {i["id"]: i for i in json.loads((out / "checklist.json").read_text())} + self.assertFalse(items["must_provide_reproducible_benchmark"]["pass"]) + self.assertIn("unknown", items["must_provide_reproducible_benchmark"]["evidence"]) + if __name__ == "__main__": unittest.main() diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_resolve_agent.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_resolve_agent.py index 771783401..8c553e4fd 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_resolve_agent.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_resolve_agent.py @@ -3,6 +3,7 @@ from __future__ import annotations +import os import sys import tempfile import unittest @@ -64,6 +65,63 @@ def test_recipe_run_sh_alone_is_not_an_agent_dir(self) -> None: (recipe / "run.sh").write_text("#!/bin/sh\necho classic\n", encoding="utf-8") self.assertFalse(resolve_agent.is_agent_dir(recipe)) + def test_import_path_outside_artefact_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artefact = root / "artifact" + agent = artefact / "agent" + external = root / "external" + agent.mkdir(parents=True) + external.mkdir() + (external / "outside_agent.py").write_text( + "from harbor.agents.base import BaseAgent\n" + "class ExternalAgent(BaseAgent):\n pass\n", + encoding="utf-8", + ) + (agent / "import_path").write_text("outside_agent:ExternalAgent\n", encoding="utf-8") + old_pp = os.environ.get("PYTHONPATH") + os.environ["PYTHONPATH"] = str(external) + os.environ["PROOF_ARTIFACT_DIR"] = str(artefact) + try: + with self.assertRaises(SystemExit) as ctx: + resolve_agent.discover(agent) + self.assertEqual(ctx.exception.code, 2) + self.assertFalse(resolve_agent.is_agent_dir(agent)) + finally: + os.environ.pop("PROOF_ARTIFACT_DIR", None) + if old_pp is None: + os.environ.pop("PYTHONPATH", None) + else: + os.environ["PYTHONPATH"] = old_pp + + def test_stdlib_import_path_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + agent = Path(tmp) / "agent" + agent.mkdir() + (agent / "import_path").write_text("json:JSONDecoder\n", encoding="utf-8") + with self.assertRaises(SystemExit) as ctx: + resolve_agent.discover(agent) + self.assertEqual(ctx.exception.code, 2) + + def test_import_path_inside_artefact_still_resolves(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + artefact = Path(tmp) / "artifact" + agent = artefact / "agent" + agent.mkdir(parents=True) + (agent / "agent.py").write_text( + "from harbor.agents.base import BaseAgent\n" + "class InsideAgent(BaseAgent):\n pass\n", + encoding="utf-8", + ) + (agent / "import_path").write_text("agent.agent:InsideAgent\n", encoding="utf-8") + os.environ["PROOF_ARTIFACT_DIR"] = str(artefact) + try: + path, pythonpath = resolve_agent.discover(agent) + finally: + os.environ.pop("PROOF_ARTIFACT_DIR", None) + self.assertEqual(path, "agent.agent:InsideAgent") + self.assertEqual(Path(pythonpath), artefact.resolve()) + if __name__ == "__main__": unittest.main() diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py index d1c1e1aed..9470d2df4 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py @@ -144,6 +144,70 @@ def test_no_trials_is_fail_closed(self) -> None: self.assertEqual(ctx.exception.code, 2) self.assertFalse((root / "report.json").exists()) + def test_nonzero_harbor_exit_is_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + trial = root / "jobs" / "job" / "t__1" + trial.mkdir(parents=True) + (trial / "result.json").write_text( + '{"trial_name":"t__1","verifier_result":{"rewards":{"reward":0.6}}}', + encoding="utf-8", + ) + out = root / "report.json" + with self.assertRaises(SystemExit) as ctx: + summarize.main( + [ + "--jobs-dir", + str(root / "jobs"), + "--output", + str(out), + "--harbor-exit", + "23", + ] + ) + self.assertEqual(ctx.exception.code, 2) + self.assertFalse(out.exists()) + + def test_scores_every_measured_trial_beyond_evidence_cap(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + jobs = root / "jobs" + for i in range(260): + trial = jobs / "job" / f"trial-{i:03d}" + trial.mkdir(parents=True) + reward = 100.0 if i >= 256 else 0.0 + (trial / "result.json").write_text( + json.dumps( + { + "trial_name": f"trial-{i:03d}", + "verifier_result": {"rewards": {"reward": reward}}, + } + ), + encoding="utf-8", + ) + trials = summarize.collect_trials(jobs) + self.assertEqual(len(trials), 260) + expected = (4.0 * 100.0) / 260.0 + self.assertAlmostEqual(summarize.mean_reward(trials), expected) + out = root / "report.json" + rc = summarize.main( + [ + "--jobs-dir", + str(jobs), + "--output", + str(out), + "--harbor-exit", + "0", + ] + ) + self.assertEqual(rc, 0) + report = json.loads(out.read_text(encoding="utf-8")) + self.assertAlmostEqual(report["primary_value"], expected) + self.assertEqual(report["evidence"]["n_measured"], 260) + self.assertEqual(len(report["evidence"]["trials"]), 256) + self.assertTrue(report["evidence"]["evidence_truncated"]) + self.assertTrue(report["claim_holds"]) + if __name__ == "__main__": unittest.main() diff --git a/docs/external-miner/proof-tbench.md b/docs/external-miner/proof-tbench.md index 290166c6c..2b2511ee7 100644 --- a/docs/external-miner/proof-tbench.md +++ b/docs/external-miner/proof-tbench.md @@ -135,7 +135,7 @@ import path as `-a`. Layout after unpack (paths relative to recipe/ agent/ # PREFERRED: Harbor agent (BaseAgent / BaseInstalledAgent) agent.py - import_path # optional: one line `agent.agent:YourClass` + import_path # optional: one line `agent.agent:YourClass` (must resolve inside this artefact) run.sh # optional classic marker; inspect may see it; evaluate does not exec it README.md ```