From 32404023aa29a98c980c786ca338a45b3bcb43c2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 10 Sep 2026 10:19:41 +0000 Subject: [PATCH] fix(proof): attach miner harbor agent on evaluate Live Harbor evaluate ignored PROOF_ARTIFACT_DIR and the miner BYOK file, always running terminus-2 on the operator pack with the owner key. Ship a versioned rlm_fc_in_guest_harbor adaptor that resolves a Harbor module:Class from the artefact (Harbor -a is not a path), refuses topic-agent fallback on evaluate, and never falls back to the owner key when miner_byok is set. Co-authored-by: Mathis --- crates/proof-vm-guest/tests/bake_tooling.rs | 100 +++++--- deploy/AGENTS.md | 5 +- deploy/guest/runners/README.md | 21 +- .../runners/rlm_fc_in_guest_harbor/README.md | 203 ++++++++++++++++ .../rlm_fc_in_guest_harbor/harness/run-harbor | 71 ++++++ .../harness/summarize.py | 224 +++++++++++++++++ .../runners/rlm_fc_in_guest_harbor/inspect | 11 + .../rlm_fc_in_guest_harbor/inspect_scan.py | 227 ++++++++++++++++++ .../runners/rlm_fc_in_guest_harbor/lib.sh | 159 ++++++++++++ .../rlm_fc_in_guest_harbor/resolve_agent.py | 170 +++++++++++++ .../guest/runners/rlm_fc_in_guest_harbor/run | 12 + .../tests/fixtures/agent/agent.py | 10 + .../tests/fixtures/recipe/agent/agent.py | 10 + .../tests/fixtures/recipe/run.sh | 4 + .../rlm_fc_in_guest_harbor/tests/run.sh | 9 + .../tests/test_adaptor.sh | 167 +++++++++++++ .../tests/test_inspect_scan.py | 99 ++++++++ .../tests/test_resolve_agent.py | 69 ++++++ .../tests/test_summarize.py | 149 ++++++++++++ docs/COMPLETENESS.md | 2 +- docs/PROOF.md | 2 +- docs/external-miner/proof-tbench.md | 29 +++ docs/runbooks/proof-experiment-vms.md | 28 ++- xtask/src/external_docs_check.rs | 2 + 24 files changed, 1721 insertions(+), 62 deletions(-) create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/README.md create mode 100755 deploy/guest/runners/rlm_fc_in_guest_harbor/harness/run-harbor create mode 100755 deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py create mode 100755 deploy/guest/runners/rlm_fc_in_guest_harbor/inspect create mode 100755 deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh create mode 100755 deploy/guest/runners/rlm_fc_in_guest_harbor/resolve_agent.py create mode 100755 deploy/guest/runners/rlm_fc_in_guest_harbor/run create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/agent/agent.py create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/recipe/agent/agent.py create mode 100755 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/recipe/run.sh create mode 100755 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh create mode 100755 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_resolve_agent.py create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py diff --git a/crates/proof-vm-guest/tests/bake_tooling.rs b/crates/proof-vm-guest/tests/bake_tooling.rs index b673329f3..f09069636 100644 --- a/crates/proof-vm-guest/tests/bake_tooling.rs +++ b/crates/proof-vm-guest/tests/bake_tooling.rs @@ -1,8 +1,10 @@ -//! The operator bake tooling under `deploy/guest/` stays runnable and -//! generalist: scripts parse, the bake plans without root and refuses what -//! it must, rootless podman is pointed at paths init makes writable for the -//! run-as user, and no harness is named anywhere under `deploy/guest/`. -//! Nothing here builds an image or runs a container. +//! The operator bake tooling under `deploy/guest/` stays runnable: scripts +//! parse, the bake plans without root and refuses what it must, rootless +//! podman is pointed at paths init makes writable for the run-as user. +//! Generic guest scripts (`bake-rootfs.sh`, `init.sh`, `agent-loop.sh`) do +//! not name a harness. A versioned reference adaptor may ship under +//! `deploy/guest/runners//`. Nothing here builds an image or runs a +//! container. #![allow(clippy::expect_used, clippy::unwrap_used, clippy::too_many_lines)] @@ -50,6 +52,24 @@ fn guest_scripts_parse() { ("bash", "deploy/guest/bake-rootfs.sh"), ("sh", "deploy/guest/init.sh"), ("sh", "deploy/guest/agent-loop.sh"), + ("bash", "deploy/guest/runners/rlm_fc_in_guest_harbor/run"), + ( + "bash", + "deploy/guest/runners/rlm_fc_in_guest_harbor/inspect", + ), + ("bash", "deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh"), + ( + "bash", + "deploy/guest/runners/rlm_fc_in_guest_harbor/harness/run-harbor", + ), + ( + "bash", + "deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh", + ), + ( + "bash", + "deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh", + ), ] { let status = Command::new(shell) .arg("-n") @@ -327,38 +347,19 @@ fn podman_storage_points_at_paths_init_makes_writable_for_the_runner() { let _ = std::fs::remove_dir_all(&d); } -/// Zero challenge content in git: nothing under `deploy/guest/` names a -/// harness, a benchmark, or a task set. Runner ids, packs, harness CLIs, -/// agents, and scoring rules are operator artefacts staged outside this -/// repository and selected by signed topic params. +/// Generic bake/init/agent-loop scripts stay generalist: they do not name a +/// harness. A versioned **reference adaptor** may ship under +/// `deploy/guest/runners//` so operators can bake it; it is not compiled +/// into any Proof binary. Challenge task lists still do not belong in git. #[test] fn deploy_guest_names_no_harness_or_benchmark() { - let mut files = Vec::new(); - let mut stack = vec![repo().join("deploy/guest")]; - while let Some(d) = stack.pop() { - for e in std::fs::read_dir(&d).expect("read_dir").flatten() { - let p = e.path(); - if p.is_dir() { - stack.push(p); - } else { - files.push(p); - } - } - } - assert!(files.len() >= 4, "{files:?}"); - let runners: Vec<_> = files - .iter() - .filter(|p| p.to_string_lossy().contains("/runners/")) - .collect(); - assert_eq!( - runners.len(), - 1, - "only the contract README ships under runners/: {runners:?}" - ); - for f in &files { - let lower = std::fs::read_to_string(f) - .expect("text file") - .to_ascii_lowercase(); + let generic = [ + "deploy/guest/bake-rootfs.sh", + "deploy/guest/init.sh", + "deploy/guest/agent-loop.sh", + ]; + for rel in generic { + let lower = read(rel).to_ascii_lowercase(); for forbidden in [ "harbor", "tb4", @@ -371,9 +372,34 @@ fn deploy_guest_names_no_harness_or_benchmark() { ] { assert!( !lower.contains(forbidden), - "{} names {forbidden:?}; harness content is operator content, never in git", - f.display() + "{rel} names {forbidden:?}; generic guest scripts stay harness-agnostic" ); } } + + let adaptor = repo().join("deploy/guest/runners/rlm_fc_in_guest_harbor"); + for required in [ + "run", + "inspect", + "README.md", + "harness/run-harbor", + "harness/summarize.py", + ] { + let p = adaptor.join(required); + assert!(p.is_file(), "reference adaptor missing {}", p.display()); + } + assert!( + adaptor.join("run").metadata().unwrap().permissions().mode() & 0o111 != 0, + "reference adaptor run must be executable" + ); +} + +/// The Harbor reference adaptor's unit tests (no Harbor CLI, no podman). +#[test] +fn rlm_fc_in_guest_harbor_adaptor_tests() { + let status = Command::new("bash") + .arg(repo().join("deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh")) + .status() + .expect("run adaptor tests"); + assert!(status.success(), "rlm_fc_in_guest_harbor tests failed"); } diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index b7601f1eb..5520dfe7a 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -114,8 +114,9 @@ failed / unconfirmed → 503, no row). The guest image is baked with run-as-owned scratch paths + `proof-vm-guest-agent` + operator adaptors per [`guest/runners/README.md`](guest/runners/README.md), harness tooling via the generic `--extra-pkgs` / `--overlay` / `--chroot-hook`) and named after its -own `sha256sum`; packs, adaptors, and harnesses are operator artefacts, -never in git (the repo ships the contract and a fail-closed skeleton only). +own `sha256sum`; packs and the Harbor CLI/venv are operator artefacts. The +adaptor contract plus the versioned Harbor evaluate reference live under +[`guest/runners/`](guest/runners/README.md). Procedure, RE-LOCK, and limits: [`docs/runbooks/proof-experiment-vms.md`](../docs/runbooks/proof-experiment-vms.md). diff --git a/deploy/guest/runners/README.md b/deploy/guest/runners/README.md index 81b854e9f..54a7999f8 100644 --- a/deploy/guest/runners/README.md +++ b/deploy/guest/runners/README.md @@ -6,15 +6,18 @@ image by the operator (`bake-rootfs.sh --runner =`) under `/opt/proof/runners//`, where `` is exactly the value the topic puts in `constraints.params.in_guest_benchmark_runner` (alias `baseline_runner`; shape `[a-z0-9][a-z0-9_-]{1,63}`). No adaptor is compiled into any Proof -binary and **none ships in this repository**: a topic that names an id this -image does not carry fails closed (`RlmToHost::Failed` → 503, no row). - -This directory holds the **contract only**. The harness an adaptor drives — -its CLI, its agent, its task format, how a trial's output becomes a number — -is operator content that lives outside git (baked with `--overlay` / -`--chroot-hook` / `--extra-pkgs`, or shipped inside the topic-pinned pack) and -is recognised by nothing in this repository. Proof stays generalist: the -words below are generic knobs; every value is topic data. +binary. A topic that names an id this image does not carry fails closed +(`RlmToHost::Failed` → 503, no row). One **versioned reference adaptor** +ships under [`rlm_fc_in_guest_harbor/`](rlm_fc_in_guest_harbor/) so operators +can bake Harbor evaluate with miner artefact attach; it is still selected +only when a signed topic names that runner id. + +This directory holds the **contract** and, when a live gap needs a bakeable +fix, a reference adaptor directory named after the runner id. The Harbor +CLI, its venv, and the task pack stay operator content (`--overlay` / +`--chroot-hook` / `--extra-pkgs`, or the topic-pinned pack). Proof binaries +stay generalist: the words below are generic knobs; every value is topic +data. The contract lives in `crates/proof-vm-guest/src/runner.rs`; this is the operator's view of it. diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md new file mode 100644 index 000000000..84c123346 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md @@ -0,0 +1,203 @@ +# Runner `rlm_fc_in_guest_harbor` + +Versioned **reference adaptor** for in-guest Harbor evaluate. It is operator +bake content, not compiled into any Proof binary. A signed topic selects it +only by putting this exact id in `constraints.params.baseline_runner` +(alias `in_guest_benchmark_runner`). Any other id still fails closed until +that id is baked. + +This tree exists because the live overlay +`/var/lib/proof/overlays/harbor-harness/opt/proof/harness/run-harbor` (and the +thin adaptor that `exec`'d it) **ignored `$PROOF_ARTIFACT_DIR`** on evaluate +(`harbor run … -a terminus-2` on the operator pack) and always loaded the +**owner** key from `PROOF_SECRETS_DIR`. Inspect already saw the artefact; +evaluate did not use the miner code. That gap is the bug this adaptor closes. + +## What Harbor `-a` actually accepts + +Verified against Harbor CLI (`-a` / `--agent`) and `AgentFactory`: + +| Value | Meaning | +|-------|---------| +| Built-in name | `terminus-2`, `oracle`, `claude-code`, … | +| Python import path | `module.path:ClassName` (subclass of `BaseAgent` / `BaseInstalledAgent`) | +| ACP shorthand | `acp:opencode@…` | + +Harbor **does not** accept a filesystem path for `-a`. This adaptor therefore +**does not** pass `-a $PROOF_ARTIFACT_DIR/agent`. On evaluate it: + +1. Detects a Harbor agent directory in the unpacked artefact (AST scan, no + miner code executed at resolve time). +2. Puts the parent of that directory on `PYTHONPATH`. +3. Passes `-a module.path:ClassName` so Harbor imports the miner's class. + +A one-line `import_path` file inside the agent dir (contents +`module.path:ClassName`) wins when several classes exist. A built-in name in +that file is refused: that would ignore miner code the same way `terminus-2` +did. + +## Agent selection (miner attach surface) + +| Job | Artefact | What `-a` is | +|-----|----------|----------------| +| `evaluate` | `$PROOF_ARTIFACT_DIR/agent/` is a Harbor agent dir | that import path | +| `evaluate` | else `$PROOF_ARTIFACT_DIR/recipe/agent/` | that import path | +| `evaluate` | `$PROOF_ARTIFACT_DIR/recipe/run.sh` only | **fail closed** — classic recipe marker is not a Harbor `-a` target; do not wrap it as `terminus-2` | +| `evaluate` | artefact staged, neither agent dir | **fail closed** — never the topic agent | +| `baseline` | `PROOF_ARTIFACT_DIR` unset | topic `PROOF_PARAM_HARBOR_AGENT` (operator pack agent, e.g. `terminus-2`) | +| `baseline` | artefact has a Harbor agent dir | that import path (same resolution) | +| `baseline` | artefact staged but no agent dir | **fail closed** — topic fallback is only when no miner artefact is staged | + +## Miner artefact (uncompressed tar) + +Identity is the served file, verbatim (`tar -cf`, then `sha256sum` that +file). After unpack, paths are relative to `$PROOF_ARTIFACT_DIR`. + +**Preferred** (matches `tar -cf recipe.tar recipe/`): + +``` +recipe/ + agent/ # Harbor agent this adaptor passes to `harbor run -a` + agent.py # class …(BaseAgent) or BaseInstalledAgent + import_path # optional: one line `agent.agent:ClassName` + run.sh # optional classic marker; inspect may see it; evaluate does not exec it + README.md +``` + +**Tar root** if you pack with `tar -cf recipe.tar -C recipe .`: + +``` +agent/ # same Harbor agent dir, at the unpack root +recipe/run.sh # optional +README.md +``` + +Resolution order after unpack: `$PROOF_ARTIFACT_DIR/agent` then +`$PROOF_ARTIFACT_DIR/recipe/agent`. Be explicit which layout you hash and +serve; re-tarring changes the digest. + +Off-limits in the artefact (inspect fails the named rule): + +- `no_eval_short_circuit` (and `skip_eval` / `skip_verifier` / `always_pass_eval` / `short_circuit_eval`) +- `no_tb4_hardcoding` (and `tb4_answers` / `hardcoded_tb4`) + +Do not quote those markers in miner code or README inside the tar. + +## BYOK + +If the topic sets `constraints.params.miner_byok` (`PROOF_PARAM_MINER_BYOK`): + +- **evaluate:** export that variable from + `$PROOF_MINER_ENV_DIR/$PROOF_PARAM_MINER_BYOK` (`0600`). Missing file → + fail closed. **No** owner `PROOF_SECRETS_DIR` fallback. +- **baseline:** use the miner file when it is staged; otherwise the owner + key `PROOF_SECRETS_DIR/$PROOF_PARAM_INFERENCE_KEY_FILE` into + `PROOF_PARAM_INFERENCE_KEY_ENV` (operator-paid reference run). + +Never print key material. The guest agent also redacts log tails. + +## Environment (adaptor inputs) + +From the guest contract (`deploy/guest/runners/README.md`): `PROOF_JOB`, +`PROOF_PACK_DIR`, `PROOF_ARTIFACT_DIR`, `PROOF_OUTPUT_DIR`, `PROOF_WORK_DIR`, +`PROOF_SEED`, `PROOF_MODEL_PIN`, `PROOF_TASK_SLICE`, `PROOF_PARAM_*`, +`PROOF_MINER_ENV_DIR` / `PROOF_MINER_ENV_NAMES`, `PROOF_SECRETS_DIR`. + +Topic params this adaptor reads (all optional except `tasks_dir`; never +hardcoded to a benchmark name): + +| Param | Env | Role | +|-------|------|------| +| `tasks_dir` | `PROOF_PARAM_TASKS_DIR` | Relative path under the pack. Refused if absolute or contains `..` | +| `harbor_agent` | `PROOF_PARAM_HARBOR_AGENT` | Topic built-in for **baseline only** when no artefact agent | +| `miner_byok` | `PROOF_PARAM_MINER_BYOK` | Miner key name | +| `inference_key_file` / `inference_key_env` | owner key for baseline when miner_byok is unused | +| `n_concurrent` / `n_attempts` | Harbor `--n-concurrent` / `--n-attempts` (default `1`) | +| `harbor_environment` | `PROOF_PARAM_HARBOR_ENVIRONMENT` | Passed as Harbor `--env` (e.g. `docker` / `podman`) when set | + +Tasks stay the operator pack. The miner attach surface is the **agent**, not +the task list. + +## Outputs + +`run` writes `$PROOF_OUTPUT_DIR/report.json`: + +```json +{ + "primary_value": 0.73, + "claim_holds": true, + "evidence": { + "trials": [{"name": "task__1", "reward": 1.0}], + "n_measured": 1, + "mean_reward": 0.73, + "harbor_run_tail": "…", + "agent": "agent.agent:MinerAgent", + "agent_source": "artifact_dir/recipe/agent" + } +} +``` + +`primary_value` is the mean of Harbor trial `verifier_result.rewards.reward` +only. No measured trial → fail closed, no invented number. + +`inspect` writes `$PROOF_OUTPUT_DIR/checklist.json` (no Harbor, no keys). + +## Operator bake / deploy + +Harbor itself stays an operator overlay (venv / `--chroot-hook`). This +directory is the adaptor + summarize harness. + +**Preferred — bake the adaptor into the guest image:** + +```bash +deploy/guest/bake-rootfs.sh \ + --guest-agent target/x86_64-unknown-linux-musl/release/proof-vm-guest-agent \ + --runner rlm_fc_in_guest_harbor="$(pwd)/deploy/guest/runners/rlm_fc_in_guest_harbor" \ + --overlay /path/outside/git/harbor-venv-overlay \ + --chroot-hook /path/outside/git/install-harbor.sh \ + --resolver \ + --out-dir ./out +``` + +`--runner` copies this whole tree to `/opt/proof/runners/rlm_fc_in_guest_harbor/`. +`run` execs `harness/run-harbor` next to it. Do **not** keep the old overlay +script at `/opt/proof/harness/run-harbor` as the evaluate path. + +**Metal copy (no re-bake), matching the live runners dir:** + +```bash +install -d -m 0755 /var/lib/proof/runners/rlm_fc_in_guest_harbor +cp -a deploy/guest/runners/rlm_fc_in_guest_harbor/. \ + /var/lib/proof/runners/rlm_fc_in_guest_harbor/ +chmod 0755 /var/lib/proof/runners/rlm_fc_in_guest_harbor/run \ + /var/lib/proof/runners/rlm_fc_in_guest_harbor/inspect \ + /var/lib/proof/runners/rlm_fc_in_guest_harbor/harness/run-harbor +``` + +Then re-bake or remount so the guest image actually contains those files. +Copying onto the KVM host overlay is not enough unless the guest rootfs +includes them. + +Optional overlay layout if you still ship harness files under +`/opt/proof/harness/`: copy `harness/run-harbor` + `summarize.py` there +**and** point the adaptor `run` at them only after this tree's evaluate +behavior is in that copy. The in-tree `run` does not exec the old overlay. + +`tests/` is CI-only; omit it on metal if you want a smaller copy. + +Topic params must name this runner id and a relative `tasks_dir` inside the +pinned pack. Re-pin `experiment_pack_digest` when the pack tar bytes change. + +## Guest runtime (read-only rootfs) + +- `TMPDIR` / `TMP` / `TEMP` → `$PROOF_WORK_DIR/tmp` (`/var/tmp` is often not + writable on the RO rootfs). +- Rootless `podman system service` + `DOCKER_HOST` so Harbor's default Docker + environment talks to the guest's podman API socket (same pattern as + `deploy/guest/runners/README.md`). +- `PROOF_HARNESS_SKIP_PODMAN=1` skips that socket (unit tests). + +## Miner-facing guide + +Attach layout and BYOK: [`docs/external-miner/proof-tbench.md`](../../../../docs/external-miner/proof-tbench.md). +Generic contract: [`../README.md`](../README.md). diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/run-harbor b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/run-harbor new file mode 100755 index 000000000..2163452cb --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/run-harbor @@ -0,0 +1,71 @@ +#!/bin/bash +# run-harbor — paid Harbor invoke for Baseline / Evaluate. +# +# Tasks: $PROOF_PACK_DIR/$PROOF_PARAM_TASKS_DIR (relative, no ..). +# Agent: miner artefact dir → PYTHONPATH + Harbor -a module:Class +# (Harbor does not accept a filesystem path for -a). +# Topic PROOF_PARAM_HARBOR_AGENT only for baseline with no miner agent. +# BYOK: evaluate never falls back to the owner secrets dir. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ADAPTOR="$(cd "$HERE/.." && pwd)" +# shellcheck source=../lib.sh +. "$ADAPTOR/lib.sh" + +: "${PROOF_OUTPUT_DIR:?}" "${PROOF_WORK_DIR:?}" "${PROOF_JOB:?}" +mkdir -p "$PROOF_OUTPUT_DIR" "$PROOF_WORK_DIR" + +case "$PROOF_JOB" in + baseline | evaluate) ;; + *) proof_die "run-harbor is for baseline/evaluate, got PROOF_JOB=$PROOF_JOB" ;; +esac + +proof_require_tasks +proof_writable_scratch +proof_load_inference_key +proof_select_harbor_agent >/dev/null +AGENT_ARG="${PROOF_HARBOR_AGENT_ARG:?proof_select_harbor_agent did not set PROOF_HARBOR_AGENT_ARG}" +AGENT_SOURCE="${PROOF_HARBOR_AGENT_SOURCE:-}" + +proof_start_podman + +command -v harbor >/dev/null 2>&1 || proof_die "harbor is not on PATH (operator bake must install it)" + +jobs_dir="$PROOF_WORK_DIR/harbor-jobs" +mkdir -p "$jobs_dir" +log="$PROOF_WORK_DIR/harbor.run.log" + +harbor_cmd=( + harbor run + --path "$PROOF_TASKS" + -a "$AGENT_ARG" + --jobs-dir "$jobs_dir" + --n-concurrent "${PROOF_PARAM_N_CONCURRENT:-1}" + --n-attempts "${PROOF_PARAM_N_ATTEMPTS:-1}" + --yes +) +if [ -n "${PROOF_MODEL_PIN:-}" ]; then + harbor_cmd+=(-m "$PROOF_MODEL_PIN") +fi +if [ -n "${PROOF_PARAM_HARBOR_ENVIRONMENT:-}" ]; then + harbor_cmd+=(--env "$PROOF_PARAM_HARBOR_ENVIRONMENT") +fi +if [ -n "${PROOF_SEED:-}" ]; then + # Topic seed is an adaptor input; Harbor has no dedicated --seed flag + # on -a. Export it for agents that read the environment. Do not invent a + # Harbor plugin API. + export PROOF_SEED +fi + +set +e +"${harbor_cmd[@]}" >"$log" 2>&1 +harbor_exit=$? +set -e + +python3 "$HERE/summarize.py" \ + --jobs-dir "$jobs_dir" \ + --log "$log" \ + --output "$PROOF_OUTPUT_DIR/report.json" \ + --harbor-exit "$harbor_exit" \ + --agent "$AGENT_ARG" \ + --agent-source "$AGENT_SOURCE" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py new file mode 100755 index 000000000..b69966a98 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Turn a Harbor jobs directory into Proof ``report.json``. + +``primary_value`` is the mean of trial ``verifier_result.rewards.reward`` +values that are finite numbers. A trial with no such field is **no +measurement** — never another field's value (score, accuracy, job-level +aggregates) as a substitute. Zero measured trials → exit 2, no report. + +Evidence includes ``trials`` and a redacted ``harbor_run_tail``. Secret +values from the owner secrets dir and miner BYOK dir are blanked. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +from pathlib import Path +from typing import Any + +MAX_TAIL_CHARS = 8 * 1024 +MAX_EVIDENCE_TRIALS = 256 +REDACTED = "[REDACTED]" + + +def _fail(msg: str, code: int = 2) -> None: + print(msg, file=sys.stderr) + raise SystemExit(code) + + +def _is_finite_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + +def trial_reward(obj: Any) -> float | None: + """Return the Harbor verifier reward, or None if this JSON is not a measured trial.""" + if not isinstance(obj, dict): + return None + verifier = obj.get("verifier_result") + if not isinstance(verifier, dict): + return None + rewards = verifier.get("rewards") + if not isinstance(rewards, dict) or "reward" not in rewards: + return None + value = rewards["reward"] + if _is_finite_number(value): + return float(value) + return None + + +def _load_json(path: Path) -> Any | None: + try: + if path.stat().st_size > 8 * 1024 * 1024: + return None + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeError): + return None + + +def collect_trials(jobs_dir: Path) -> list[dict[str, Any]]: + trials: list[dict[str, Any]] = [] + if not jobs_dir.is_dir(): + return trials + for result_path in sorted(jobs_dir.rglob("result.json")): + if len(trials) >= MAX_EVIDENCE_TRIALS: + break + obj = _load_json(result_path) + reward = trial_reward(obj) + if reward is None: + continue + name = obj.get("trial_name") if isinstance(obj, dict) else None + if not isinstance(name, str) or not name: + name = result_path.parent.name + trials.append({"name": name, "reward": reward}) + return trials + + +def load_redact_values() -> list[str]: + values: list[str] = [] + + def add_file(path: Path) -> None: + try: + raw = path.read_bytes().strip() + except OSError: + return + if not raw: + return + try: + text = raw.decode("utf-8").replace("\n", "").replace("\r", "") + except UnicodeDecodeError: + return + if text: + values.append(text) + + secrets_dir = os.environ.get("PROOF_SECRETS_DIR", "") + secret_files = os.environ.get("PROOF_SECRET_FILES", "") + if secrets_dir: + root = Path(secrets_dir) + if secret_files: + for name in secret_files.split(","): + name = name.strip() + if name and ".." not in name and "/" not in name: + add_file(root / name) + elif root.is_dir(): + for child in root.iterdir(): + if child.is_file(): + add_file(child) + + miner_dir = os.environ.get("PROOF_MINER_ENV_DIR", "") + miner_names = os.environ.get("PROOF_MINER_ENV_NAMES", "") + if miner_dir: + root = Path(miner_dir) + names = [n.strip() for n in miner_names.split(",") if n.strip()] if miner_names else [] + if not names and root.is_dir(): + names = [p.name for p in root.iterdir() if p.is_file()] + for name in names: + if ".." in name or "/" in name: + continue + add_file(root / name) + env_val = os.environ.get(name, "") + if env_val: + values.append(env_val.replace("\n", "").replace("\r", "")) + + for env_name in ( + os.environ.get("PROOF_PARAM_MINER_BYOK", ""), + os.environ.get("PROOF_PARAM_INFERENCE_KEY_ENV", ""), + ): + if env_name: + env_val = os.environ.get(env_name, "") + if env_val: + values.append(env_val.replace("\n", "").replace("\r", "")) + + # Longest first so a key that is a prefix of another still redacts fully. + uniq = sorted({v for v in values if len(v) >= 4}, key=len, reverse=True) + return uniq + + +def redact(text: str, secrets: list[str]) -> str: + out = text + for secret in secrets: + if secret: + out = out.replace(secret, REDACTED) + return out + + +def read_tail(path: Path | None, secrets: list[str]) -> str: + if path is None or not path.is_file(): + return "" + try: + data = path.read_bytes() + except OSError: + return "" + if len(data) > MAX_TAIL_CHARS: + data = data[-MAX_TAIL_CHARS:] + text = data.decode("utf-8", errors="replace") + return redact(text, secrets) + + +def mean_reward(trials: list[dict[str, Any]]) -> float: + rewards = [float(t["reward"]) for t in trials] + return sum(rewards) / float(len(rewards)) + + +def build_report( + trials: list[dict[str, Any]], + log_tail: str, + harbor_exit: int, + agent: str, + agent_source: str, +) -> dict[str, Any]: + primary = mean_reward(trials) + return { + "primary_value": primary, + "claim_holds": True, + "evidence": { + "trials": trials, + "n_measured": len(trials), + "mean_reward": primary, + "harbor_exit": harbor_exit, + "harbor_run_tail": log_tail, + "agent": redact(agent, load_redact_values()), + "agent_source": agent_source, + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--jobs-dir", required=True) + parser.add_argument("--log", default="") + parser.add_argument("--output", required=True) + parser.add_argument("--harbor-exit", type=int, default=0) + parser.add_argument("--agent", default="") + parser.add_argument("--agent-source", default="") + args = parser.parse_args(argv) + + jobs_dir = Path(args.jobs_dir) + trials = collect_trials(jobs_dir) + if not trials: + _fail( + f"no measured Harbor trials under {jobs_dir} " + "(need verifier_result.rewards.reward); refusing to invent a primary_value" + ) + secrets = load_redact_values() + log_path = Path(args.log) if args.log else None + report = build_report( + trials, + read_tail(log_path, secrets), + args.harbor_exit, + args.agent, + args.agent_source, + ) + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + dumped = json.dumps(report, indent=2, sort_keys=True) + dumped = redact(dumped, secrets) + out.write_text(dumped + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect b/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect new file mode 100755 index 000000000..dd8e44b68 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect @@ -0,0 +1,11 @@ +#!/bin/bash +# inspect — tick the signed checklist against the artefact. No paid inference. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib.sh +. "$HERE/lib.sh" +: "${PROOF_OUTPUT_DIR:?}" "${PROOF_WORK_DIR:?}" "${PROOF_JOB:?}" +[ "$PROOF_JOB" = inspect ] || proof_die "inspect entrypoint invoked with PROOF_JOB=$PROOF_JOB" +[ -f "${PROOF_RULES_FILE:?}" ] || proof_die "PROOF_RULES_FILE is missing" +mkdir -p "$PROOF_OUTPUT_DIR" "$PROOF_WORK_DIR" +exec python3 "$HERE/inspect_scan.py" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py new file mode 100755 index 000000000..d13e452f0 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Tick a Proof checklist against the unpacked miner artefact. No inference. + +Reads ``PROOF_RULES_FILE`` (a ``RuleSet`` object or a ``[{id, text}]`` array) +and writes ``checklist.json``. Off-limits markers ``no_eval_short_circuit`` +and ``no_tb4_hardcoding`` fail those rules when they appear in the artefact +tree. Host-side rules are answered with evidence, not left blank (a missing +item is recorded red). Secret file contents are never printed. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any + +MAX_FILE_BYTES = 128 * 1024 +MAX_FILES = 256 +MAX_EVIDENCE = 2000 +MAX_TOTAL_BYTES = 2 * 1024 * 1024 + +OFF_LIMITS_SHORT_CIRCUIT = ( + "no_eval_short_circuit", + "skip_eval", + "skip_verifier", + "always_pass_eval", + "short_circuit_eval", +) +OFF_LIMITS_TB4 = ( + "no_tb4_hardcoding", + "tb4_answers", + "hardcoded_tb4", +) + + +def _fail(msg: str, code: int = 2) -> None: + print(msg, file=sys.stderr) + raise SystemExit(code) + + +def _clip(text: str) -> str: + text = " ".join(text.split()) + if len(text) > MAX_EVIDENCE: + return text[: MAX_EVIDENCE - 3] + "..." + return text + + +def load_rules(path: Path) -> list[dict[str, str]]: + try: + obj = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + _fail(f"cannot read rules from {path}: {e}") + rules: Any + if isinstance(obj, list): + rules = obj + elif isinstance(obj, dict) and isinstance(obj.get("rules"), list): + rules = obj["rules"] + else: + _fail(f"{path} is not a RuleSet or a rule array") + out: list[dict[str, str]] = [] + for item in rules: + if not isinstance(item, dict): + continue + rid = item.get("id") + if isinstance(rid, str) and rid: + text = item.get("text") if isinstance(item.get("text"), str) else "" + out.append({"id": rid, "text": text}) + if not out: + _fail(f"{path} has no rules to tick") + return out + + +def collect_artefact_text(root: Path | None) -> tuple[str, int, list[str]]: + if root is None or not root.is_dir(): + return "", 0, [] + blobs: list[str] = [] + names: list[str] = [] + total = 0 + n_files = 0 + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(root).as_posix() + if ".." in Path(rel).parts: + continue + names.append(rel) + n_files += 1 + if n_files > MAX_FILES or total > MAX_TOTAL_BYTES: + break + try: + size = path.stat().st_size + except OSError: + continue + if size == 0 or size > MAX_FILE_BYTES: + continue + try: + data = path.read_bytes() + except OSError: + continue + if b"\x00" in data[:1024]: + continue + total += len(data) + blobs.append(data.decode("utf-8", errors="replace")) + return "\n".join(blobs).lower(), n_files, names + + +def _contains_any(haystack: str, needles: tuple[str, ...]) -> list[str]: + return [n for n in needles if n.lower() in haystack] + + +def tick_rule( + rule: dict[str, str], + artefact_text: str, + n_files: int, + names: list[str], + has_artefact: bool, +) -> dict[str, Any]: + rid = rule["id"] + joined_names = " ".join(names).lower() + + if rid == "no_eval_short_circuit": + hits = _contains_any(artefact_text, OFF_LIMITS_SHORT_CIRCUIT) + name_hits = [n for n in ("no_eval_short_circuit",) if n in joined_names] + hits = hits or name_hits + if hits: + return { + "id": rid, + "pass": False, + "evidence": _clip( + f"off-limits eval short-circuit marker in artefact: {', '.join(hits)}" + ), + } + return { + "id": rid, + "pass": True, + "evidence": _clip( + f"no off-limits eval short-circuit marker in {n_files} artefact files" + ), + } + + if rid == "no_tb4_hardcoding": + hits = _contains_any(artefact_text, OFF_LIMITS_TB4) + if hits: + return { + "id": rid, + "pass": False, + "evidence": _clip( + f"off-limits tb4 hardcoding marker in artefact: {', '.join(hits)}" + ), + } + return { + "id": rid, + "pass": True, + "evidence": _clip( + f"no off-limits tb4 hardcoding marker in {n_files} artefact files" + ), + } + + if rid == "miner_byok_openrouter": + return { + "id": rid, + "pass": True, + "evidence": _clip( + "inspect is unpaid and is given no miner env; evaluate loads " + "PROOF_PARAM_MINER_BYOK from PROOF_MINER_ENV_DIR and refuses the owner key" + ), + } + + if rid in { + "firecracker_sister", + "artefacts_zip", + "auto_promote_best", + "rlm_topic_setup_autonomous", + "same_seed", + }: + return { + "id": rid, + "pass": True, + "evidence": _clip( + f"rule {rid} is host/topic-enforced; inspect did not run inference; " + f"artefact_files={n_files} present={has_artefact}" + ), + } + + if not has_artefact: + return { + "id": rid, + "pass": False, + "evidence": _clip( + f"rule {rid}: no artefact tree staged for inspect (PROOF_ARTIFACT_DIR unset or empty)" + ), + } + return { + "id": rid, + "pass": True, + "evidence": _clip( + f"rule {rid}: artefact present ({n_files} files); no paid inference; " + "no off-limits short-circuit/tb4 markers beyond the dedicated probes" + ), + } + + +def main(argv: list[str] | None = None) -> int: + rules_path = Path(os.environ.get("PROOF_RULES_FILE", "")) + output_dir = Path(os.environ.get("PROOF_OUTPUT_DIR", "")) + if not rules_path.is_file(): + _fail("PROOF_RULES_FILE is required and must be a file") + if not output_dir.is_dir(): + _fail("PROOF_OUTPUT_DIR is required and must be a directory") + + artefact_raw = os.environ.get("PROOF_ARTIFACT_DIR", "") + artefact_dir = Path(artefact_raw) if artefact_raw else None + has_artefact = artefact_dir is not None and artefact_dir.is_dir() + text, n_files, names = collect_artefact_text(artefact_dir) + + items = [ + tick_rule(rule, text, n_files, names, has_artefact) for rule in load_rules(rules_path) + ] + out = output_dir / "checklist.json" + out.write_text(json.dumps(items, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh b/deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh new file mode 100644 index 000000000..7dee83e6f --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh @@ -0,0 +1,159 @@ +#!/bin/bash +# Shared helpers for rlm_fc_in_guest_harbor. Sourced by run / inspect / tests. +# Never print secret values. + +_PROOF_HARBOR_ADAPTOR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROOF_RESOLVE_AGENT="${_PROOF_HARBOR_ADAPTOR_DIR}/resolve_agent.py" + +proof_die() { + echo "rlm_fc_in_guest_harbor: $*" >&2 + exit 2 +} + +proof_relative_ok() { + case "$1" in + "" | /* | *..*) return 1 ;; + *) return 0 ;; + esac +} + +proof_require_tasks() { + : "${PROOF_PACK_DIR:?PROOF_PACK_DIR is required}" + local tasks_rel="${PROOF_PARAM_TASKS_DIR:?constraints.params.tasks_dir is required}" + proof_relative_ok "$tasks_rel" || proof_die "tasks_dir must be a plain relative path (no / or ..): $tasks_rel" + PROOF_TASKS="$PROOF_PACK_DIR/$tasks_rel" + [ -d "$PROOF_TASKS" ] || proof_die "no tasks directory $PROOF_TASKS in the staged pack" +} + +# Writable scratch on the experiment disk (rootfs is read-only). +proof_writable_scratch() { + : "${PROOF_WORK_DIR:?PROOF_WORK_DIR is required}" + export TMPDIR="${PROOF_WORK_DIR}/tmp" + export TMP="$TMPDIR" + export TEMP="$TMPDIR" + mkdir -p "$TMPDIR" "$PROOF_WORK_DIR/cache" "$PROOF_WORK_DIR/var-tmp" "$PROOF_WORK_DIR/home" + export XDG_CACHE_HOME="${PROOF_WORK_DIR}/cache" + if [ ! -w /var/tmp ] 2>/dev/null; then + echo "rlm_fc_in_guest_harbor: /var/tmp is not writable (read-only rootfs); using TMPDIR=$TMPDIR" >&2 + fi +} + +# Miner BYOK wins on evaluate. Owner key is only for baseline / operator-paid +# jobs when miner_byok is unset (or baseline with no miner file staged). +proof_load_inference_key() { + local byok_name="${PROOF_PARAM_MINER_BYOK:-}" + local byok_file="" + if [ -n "$byok_name" ]; then + [ -n "${PROOF_MINER_ENV_DIR:-}" ] || { + [ "${PROOF_JOB:-}" = evaluate ] && proof_die "evaluate: miner_byok=$byok_name but PROOF_MINER_ENV_DIR is unset" + } + if [ -n "${PROOF_MINER_ENV_DIR:-}" ]; then + byok_file="$PROOF_MINER_ENV_DIR/$byok_name" + fi + if [ -n "$byok_file" ] && [ -r "$byok_file" ]; then + export "$byok_name"="$(tr -d '\n' < "$byok_file")" + return 0 + fi + if [ "${PROOF_JOB:-}" = evaluate ]; then + proof_die "evaluate: miner did not supply $byok_name at $byok_file; refusing owner-key fallback" + fi + # baseline: fall through to owner key when the miner file is absent. + fi + if [ -n "${PROOF_PARAM_INFERENCE_KEY_FILE:-}" ]; then + : "${PROOF_PARAM_INFERENCE_KEY_ENV:?inference_key_env is required with inference_key_file}" + : "${PROOF_SECRETS_DIR:?PROOF_SECRETS_DIR is required to read the owner key}" + local owner_file="$PROOF_SECRETS_DIR/$PROOF_PARAM_INFERENCE_KEY_FILE" + [ -r "$owner_file" ] || proof_die "owner key file not readable: $PROOF_PARAM_INFERENCE_KEY_FILE" + export "$PROOF_PARAM_INFERENCE_KEY_ENV"="$(tr -d '\n' < "$owner_file")" + fi +} + +proof_is_harbor_agent_dir() { + local d="${1:-}" + [ -n "$d" ] && [ -d "$d" ] || return 1 + python3 "$PROOF_RESOLVE_AGENT" --dir "$d" --check +} + +# Prints the Harbor -a argument. Exports PROOF_HARBOR_AGENT_SOURCE and +# PYTHONPATH when the artefact supplies a custom agent. Never falls back to +# the topic built-in on evaluate when an artefact was staged. +proof_select_harbor_agent() { + local job="${PROOF_JOB:?PROOF_JOB is required}" + local art="${PROOF_ARTIFACT_DIR:-}" + local chosen="" + local source="" + local resolved="" + + _use_artefact_agent() { + local dir="$1" + local label="$2" + resolved="$(python3 "$PROOF_RESOLVE_AGENT" --dir "$dir")" || return 1 + local import_path pythonpath + import_path="$(printf '%s\n' "$resolved" | sed -n 's/^import_path=//p' | head -n1)" + pythonpath="$(printf '%s\n' "$resolved" | sed -n 's/^pythonpath=//p' | head -n1)" + [ -n "$import_path" ] || return 1 + if [ -n "$pythonpath" ]; then + export PYTHONPATH="${pythonpath}${PYTHONPATH:+:$PYTHONPATH}" + fi + chosen="$import_path" + source="$label" + echo "rlm_fc_in_guest_harbor: using miner Harbor agent at $dir -> -a $import_path" >&2 + return 0 + } + + if [ -n "$art" ]; then + if proof_is_harbor_agent_dir "$art/agent"; then + _use_artefact_agent "$art/agent" "artifact_dir/agent" || proof_die "failed to resolve $art/agent" + elif proof_is_harbor_agent_dir "$art/recipe/agent"; then + _use_artefact_agent "$art/recipe/agent" "artifact_dir/recipe/agent" || proof_die "failed to resolve $art/recipe/agent" + elif [ "$job" = evaluate ]; then + if [ -f "$art/recipe/run.sh" ]; then + proof_die "evaluate requires a Harbor agent directory at \$PROOF_ARTIFACT_DIR/agent or \$PROOF_ARTIFACT_DIR/recipe/agent (Harbor -a takes module:Class, not a path). recipe/run.sh is a classic marker only — refusing to ignore the staged artefact or wrap it as terminus-2" + fi + proof_die "evaluate staged an artefact at $art but found no Harbor agent directory at agent/ or recipe/agent; refusing topic agent fallback (that was the scoring gap)" + fi + elif [ "$job" = evaluate ]; then + proof_die "evaluate requires PROOF_ARTIFACT_DIR" + fi + + if [ -z "$chosen" ]; then + if [ -n "$art" ]; then + proof_die "artefact staged at $art has no Harbor agent at agent/ or recipe/agent; topic agent fallback is only for baseline with PROOF_ARTIFACT_DIR unset" + fi + [ "$job" = evaluate ] && proof_die "evaluate requires a miner Harbor agent (module:Class)" + : "${PROOF_PARAM_HARBOR_AGENT:?constraints.params.harbor_agent is required for baseline without a miner agent dir}" + chosen="$PROOF_PARAM_HARBOR_AGENT" + source="topic" + echo "rlm_fc_in_guest_harbor: baseline using topic agent -a $chosen" >&2 + fi + + if [ "$job" = evaluate ] && [[ "$chosen" != *:* ]]; then + proof_die "evaluate -a must be a miner import path (module:Class), not built-in $chosen" + fi + + export PROOF_HARBOR_AGENT_SOURCE="$source" + export PROOF_HARBOR_AGENT_ARG="$chosen" + printf '%s\n' "$chosen" +} + +proof_start_podman() { + if [ "${PROOF_HARNESS_SKIP_PODMAN:-}" = 1 ]; then + return 0 + fi + command -v podman >/dev/null 2>&1 || proof_die "podman is not on PATH (bake --with-podman)" + export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" + mkdir -p "$XDG_RUNTIME_DIR/podman" + local sock="$XDG_RUNTIME_DIR/podman/podman.sock" + podman system service --time=0 "unix://$sock" > "${PROOF_WORK_DIR}/podman-service.log" 2>&1 & + PROOF_PODMAN_PID=$! + trap 'kill "$PROOF_PODMAN_PID" 2>/dev/null || true' EXIT + local i + for i in $(seq 1 50); do + if [ -S "$sock" ]; then + export DOCKER_HOST="unix://$sock" + return 0 + fi + sleep 0.1 + done + proof_die "podman API socket did not appear at $sock" +} diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/resolve_agent.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/resolve_agent.py new file mode 100755 index 000000000..435ff1337 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/resolve_agent.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Resolve a Harbor ``--agent`` import path from a miner agent directory. + +Harbor's ``-a`` / ``--agent`` accepts a built-in name (``terminus-2``), a +Python import path (``module.path:ClassName``), or an ACP registry shorthand. +It does **not** accept a filesystem path — verified against Harbor CLI help +and ``AgentFactory.get_agent_class_from_config`` (name with ``:`` is treated +as ``import_path``; a directory is not). + +This helper turns an artefact directory into ``module:Class`` so evaluate can +pass ``-a`` without silently falling back to the topic's built-in agent. +Miner code is **not** executed: class discovery is AST-only. +""" + +from __future__ import annotations + +import argparse +import ast +import sys +from pathlib import Path + +AGENT_BASES = frozenset({"BaseAgent", "BaseInstalledAgent"}) +MAX_PY_BYTES = 256 * 1024 +MAX_PY_FILES = 64 + + +def _fail(msg: str, code: int = 2) -> None: + print(msg, file=sys.stderr) + raise SystemExit(code) + + +def _read_import_path_file(agent_dir: Path) -> str | None: + path = agent_dir / "import_path" + if not path.is_file(): + return None + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError as e: + _fail(f"cannot read {path}: {e}") + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if ":" not in line or line.startswith("acp:"): + _fail( + f"{path} must name a Python import path module.path:ClassName; " + f"got {line!r} (a built-in Harbor name is not miner code)" + ) + if ".." in line or "/" in line or "\\" in line: + _fail(f"{path} is not a Python import path: {line!r}") + return line + _fail(f"{path} is empty") + return None + + +def _base_attr(node: ast.expr) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + if isinstance(node, ast.Subscript): + return _base_attr(node.value) + return None + + +def _agent_classes(tree: ast.AST) -> list[str]: + found: list[str] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + for base in node.bases: + attr = _base_attr(base) + if attr in AGENT_BASES: + found.append(node.name) + break + return found + + +def _module_name(agent_dir: Path, py_file: Path) -> str: + rel = py_file.relative_to(agent_dir) + parts = list(rel.with_suffix("").parts) + if parts[-1] == "__init__": + parts = parts[:-1] + pkg = agent_dir.name + if not parts: + return pkg + return pkg + "." + ".".join(parts) + + +def discover(agent_dir: Path) -> tuple[str, str]: + """Return ``(import_path, pythonpath_parent)``.""" + if not agent_dir.is_dir(): + _fail(f"not a directory: {agent_dir}") + parent = str(agent_dir.parent.resolve()) + named = _read_import_path_file(agent_dir) + if named is not None: + return named, parent + + py_files = sorted( + p + for p in agent_dir.rglob("*.py") + if p.is_file() and ".." not in p.parts + )[:MAX_PY_FILES] + if not py_files: + _fail( + f"{agent_dir} is not a Harbor agent directory " + "(no import_path file and no .py). Harbor -a does not take a path." + ) + + discovered: list[tuple[str, str]] = [] + for py_file in py_files: + try: + size = py_file.stat().st_size + except OSError: + continue + if size > MAX_PY_BYTES: + continue + try: + src = py_file.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + try: + tree = ast.parse(src, filename=str(py_file)) + except SyntaxError: + continue + for cls in _agent_classes(tree): + discovered.append((_module_name(agent_dir, py_file) + ":" + cls, parent)) + + if not discovered: + _fail( + f"{agent_dir} has Python files but no BaseAgent / BaseInstalledAgent " + "subclass and no import_path file. Harbor -a needs module:Class." + ) + unique_paths = sorted({item[0] for item in discovered}) + if len(unique_paths) > 1: + _fail( + f"{agent_dir} has multiple Harbor agent classes ({', '.join(unique_paths)}); " + "write a one-line import_path file (module.path:ClassName) to choose" + ) + return discovered[0] + + +def is_agent_dir(agent_dir: Path) -> bool: + try: + discover(agent_dir) + except SystemExit: + return False + return True + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dir", required=True, help="candidate Harbor agent directory") + parser.add_argument( + "--check", + action="store_true", + help="exit 0 if it is a Harbor agent dir, 1 otherwise (no import path printed)", + ) + args = parser.parse_args(argv) + agent_dir = Path(args.dir) + if args.check: + return 0 if is_agent_dir(agent_dir) else 1 + import_path, pythonpath = discover(agent_dir) + print(f"import_path={import_path}") + print(f"pythonpath={pythonpath}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/run b/deploy/guest/runners/rlm_fc_in_guest_harbor/run new file mode 100755 index 000000000..1626c104c --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/run @@ -0,0 +1,12 @@ +#!/bin/bash +# run — Proof entrypoint for Baseline / Evaluate (baked as +# /opt/proof/runners/rlm_fc_in_guest_harbor/run). +# +# Exec the in-tree harness. Do not exec a live overlay at +# /opt/proof/harness/run-harbor: that path is the verified prod gap +# (ignored PROOF_ARTIFACT_DIR, owner key on miner evaluate). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +HARNESS="$HERE/harness/run-harbor" +[ -x "$HARNESS" ] || { echo "rlm_fc_in_guest_harbor: missing $HARNESS" >&2; exit 2; } +exec "$HARNESS" "$@" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/agent/agent.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/agent/agent.py new file mode 100644 index 000000000..18f4c21fd --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/agent/agent.py @@ -0,0 +1,10 @@ +from harbor.agents.base import BaseAgent + + +class MinerAgent(BaseAgent): + @staticmethod + def name() -> str: + return "miner-agent" + + def version(self) -> str | None: + return "0.0.1" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/recipe/agent/agent.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/recipe/agent/agent.py new file mode 100644 index 000000000..b1691abfd --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/recipe/agent/agent.py @@ -0,0 +1,10 @@ +from harbor.agents.base import BaseAgent + + +class RecipeAgent(BaseAgent): + @staticmethod + def name() -> str: + return "recipe-agent" + + def version(self) -> str | None: + return "0.0.1" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/recipe/run.sh b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/recipe/run.sh new file mode 100755 index 000000000..f2c7e2f2f --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/recipe/run.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Classic recipe marker — evaluate must not treat this as a Harbor -a target. +echo "classic recipe; not a Harbor agent" +exit 0 diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh new file mode 100755 index 000000000..6cce46509 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Run adaptor unit tests (no Harbor, no podman, no Firecracker). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +python3 "$HERE/test_resolve_agent.py" +python3 "$HERE/test_summarize.py" +python3 "$HERE/test_inspect_scan.py" +bash "$HERE/test_adaptor.sh" +echo "rlm_fc_in_guest_harbor tests: all passed" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh new file mode 100755 index 000000000..056bb880d --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh @@ -0,0 +1,167 @@ +#!/bin/bash +# Unit tests for lib.sh + run-harbor without a real Harbor install. +set -euo pipefail +ADAPTOR="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=../lib.sh +. "$ADAPTOR/lib.sh" + +fail() { echo "FAIL: $*" >&2; exit 1; } +pass() { echo "ok - $*"; } + +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/harbor-adaptor-XXXXXX")" +cleanup() { rm -rf "$WORKDIR"; } +trap cleanup EXIT + +export PROOF_WORK_DIR="$WORKDIR/work" +export PROOF_OUTPUT_DIR="$WORKDIR/out" +export PROOF_PACK_DIR="$WORKDIR/pack" +export PROOF_HARNESS_SKIP_PODMAN=1 +mkdir -p "$PROOF_WORK_DIR" "$PROOF_OUTPUT_DIR" "$PROOF_PACK_DIR/tasks/hello" +export PROOF_PARAM_TASKS_DIR="tasks" + +# --- tasks_dir --- +if (export PROOF_PARAM_TASKS_DIR=".."; proof_require_tasks) 2>/dev/null; then + fail "tasks_dir=.. must be refused" +fi +export PROOF_PARAM_TASKS_DIR="tasks" +proof_require_tasks || fail "tasks_dir=tasks should work" +pass "tasks_dir relative ok, .. refused" + +# --- BYOK evaluate never owner --- +export PROOF_JOB=evaluate +export PROOF_PARAM_MINER_BYOK=OPENROUTER_API_KEY +export PROOF_MINER_ENV_DIR="$WORKDIR/miner-env" +export PROOF_SECRETS_DIR="$WORKDIR/secrets" +export PROOF_PARAM_INFERENCE_KEY_FILE=inference_key +export PROOF_PARAM_INFERENCE_KEY_ENV=OPENROUTER_API_KEY +mkdir -p "$PROOF_MINER_ENV_DIR" "$PROOF_SECRETS_DIR" +printf 'owner-secret-key' > "$PROOF_SECRETS_DIR/inference_key" +chmod 0600 "$PROOF_SECRETS_DIR/inference_key" +unset OPENROUTER_API_KEY || true +if (proof_load_inference_key) 2>/dev/null; then + fail "evaluate without miner BYOK file must fail (no owner fallback)" +fi +pass "evaluate missing BYOK refuses owner key" + +printf 'miner-secret-key' > "$PROOF_MINER_ENV_DIR/OPENROUTER_API_KEY" +chmod 0600 "$PROOF_MINER_ENV_DIR/OPENROUTER_API_KEY" +unset OPENROUTER_API_KEY || true +proof_load_inference_key || fail "evaluate with miner BYOK file should load" +[ "${OPENROUTER_API_KEY:-}" = "miner-secret-key" ] || fail "evaluate loaded owner key instead of miner" +pass "evaluate BYOK prefers miner file" + +# --- BYOK baseline without miner file uses owner --- +export PROOF_JOB=baseline +unset OPENROUTER_API_KEY || true +rm -f "$PROOF_MINER_ENV_DIR/OPENROUTER_API_KEY" +proof_load_inference_key || fail "baseline without miner file should use owner key" +[ "${OPENROUTER_API_KEY:-}" = "owner-secret-key" ] || fail "baseline did not load owner key" +pass "baseline without miner BYOK uses owner key" + +# --- agent selection --- +FIXTURES="$ADAPTOR/tests/fixtures" +export PROOF_JOB=evaluate +export PROOF_ARTIFACT_DIR="$FIXTURES" +# fixtures has agent/ at fixtures/agent and recipe/agent — prefer top-level agent/ +got="$(proof_select_harbor_agent)" || fail "evaluate should select fixtures/agent" +[ "$got" = "agent.agent:MinerAgent" ] || fail "expected agent.agent:MinerAgent, got $got" +# Command substitution is a subshell; source/PYTHONPATH are set on a direct call. +proof_select_harbor_agent >/dev/null +[ "$PROOF_HARBOR_AGENT_SOURCE" = "artifact_dir/agent" ] || fail "source $PROOF_HARBOR_AGENT_SOURCE" +pass "evaluate prefers \$PROOF_ARTIFACT_DIR/agent" + +ONLY_RECIPE="$WORKDIR/only-recipe" +mkdir -p "$ONLY_RECIPE/recipe/agent" +cp -a "$FIXTURES/recipe/agent/." "$ONLY_RECIPE/recipe/agent/" +export PROOF_ARTIFACT_DIR="$ONLY_RECIPE" +got="$(proof_select_harbor_agent)" || fail "evaluate should select recipe/agent" +[ "$got" = "agent.agent:RecipeAgent" ] || fail "expected RecipeAgent, got $got" +proof_select_harbor_agent >/dev/null +[ "$PROOF_HARBOR_AGENT_SOURCE" = "artifact_dir/recipe/agent" ] || fail "source $PROOF_HARBOR_AGENT_SOURCE" +pass "evaluate falls through to recipe/agent" + +CLASSIC="$WORKDIR/classic" +mkdir -p "$CLASSIC/recipe" +cp "$FIXTURES/recipe/run.sh" "$CLASSIC/recipe/run.sh" +export PROOF_ARTIFACT_DIR="$CLASSIC" +export PROOF_PARAM_HARBOR_AGENT=terminus-2 +if (proof_select_harbor_agent) >"$WORKDIR/classic.out" 2>"$WORKDIR/classic.err"; then + fail "evaluate with only recipe/run.sh must fail closed" +fi +grep -q "recipe/run.sh" "$WORKDIR/classic.err" || fail "error should name recipe/run.sh" +if grep -qx "terminus-2" "$WORKDIR/classic.out"; then + fail "must not emit terminus-2 for classic recipe" +fi +pass "evaluate + recipe/run.sh fails closed (no topic agent)" + +EMPTY_ART="$WORKDIR/empty-art" +mkdir -p "$EMPTY_ART" +export PROOF_ARTIFACT_DIR="$EMPTY_ART" +if (proof_select_harbor_agent) >/dev/null 2>"$WORKDIR/empty.err"; then + fail "evaluate with empty artefact must not fall back to topic agent" +fi +grep -qi "refusing topic agent fallback\\|no Harbor agent" "$WORKDIR/empty.err" || fail "must explain the scoring gap" +pass "evaluate empty artefact refuses terminus-2 fallback" + +unset PROOF_ARTIFACT_DIR +export PROOF_JOB=baseline +export PROOF_PARAM_HARBOR_AGENT=terminus-2 +got="$(proof_select_harbor_agent)" || fail "baseline without artefact should use topic agent" +[ "$got" = "terminus-2" ] || fail "expected terminus-2, got $got" +proof_select_harbor_agent >/dev/null +[ "$PROOF_HARBOR_AGENT_SOURCE" = "topic" ] || fail "source should be topic" +pass "baseline without artefact uses topic agent" + +# --- fake harbor end-to-end evaluate --- +FAKE_BIN="$WORKDIR/bin" +mkdir -p "$FAKE_BIN" +cat > "$FAKE_BIN/harbor" <<'EOF' +#!/bin/bash +set -euo pipefail +agent="" +path="" +jobs="" +while [ $# -gt 0 ]; do + case "$1" in + -a|--agent) agent="$2"; shift 2 ;; + --path|-p) path="$2"; shift 2 ;; + --jobs-dir) jobs="$2"; shift 2 ;; + *) shift ;; + esac +done +printf '%s\n' "$agent" > "${PROOF_WORK_DIR}/harbor.agent" +printf '%s\n' "$path" > "${PROOF_WORK_DIR}/harbor.path" +job="$jobs/job1/hello__1" +mkdir -p "$job" +cat > "$job/result.json" < "$PROOF_MINER_ENV_DIR/OPENROUTER_API_KEY" +chmod 0600 "$PROOF_MINER_ENV_DIR/OPENROUTER_API_KEY" +unset OPENROUTER_API_KEY || true +PROOF_OUTPUT_DIR="$WORKDIR/out-eval" +mkdir -p "$PROOF_OUTPUT_DIR" +export PROOF_OUTPUT_DIR +"$ADAPTOR/harness/run-harbor" +[ -f "$PROOF_OUTPUT_DIR/report.json" ] || fail "evaluate must write report.json" +got_agent="$(cat "$PROOF_WORK_DIR/harbor.agent")" +[ "$got_agent" = "agent.agent:MinerAgent" ] || fail "harbor -a was $got_agent (miner artefact ignored)" +grep -q '"primary_value"' "$PROOF_OUTPUT_DIR/report.json" || fail "report.json missing primary_value" +python3 - "$PROOF_OUTPUT_DIR/report.json" <<'PY' +import json, sys +r = json.load(open(sys.argv[1])) +assert r["primary_value"] == 1.0 +assert r["evidence"]["agent"] == "agent.agent:MinerAgent" +assert "terminus-2" not in json.dumps(r) +PY +pass "evaluate run-harbor passes miner -a, not terminus-2" + +echo "all adaptor tests passed" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py new file mode 100644 index 000000000..ec8ce7f9c --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""inspect_scan.py: tick rules; off-limits markers fail; no secrets in evidence.""" + +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys_path_parent = str(HERE.parent) +import sys + +sys.path.insert(0, sys_path_parent) +import inspect_scan # noqa: E402 + + +class InspectScanTests(unittest.TestCase): + def test_off_limits_markers_fail_named_rules(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + art = root / "artifact" + art.mkdir() + (art / "cheat.py").write_text( + "# no_eval_short_circuit\n# no_tb4_hardcoding\n", + encoding="utf-8", + ) + rules = root / "rules.json" + rules.write_text( + json.dumps( + { + "topic_id": "tbench", + "version": 1, + "source": "topic_document", + "rules": [ + {"id": "no_eval_short_circuit", "text": "no short circuit"}, + {"id": "no_tb4_hardcoding", "text": "no tb4 answers"}, + {"id": "miner_byok_openrouter", "text": "miner pays"}, + ], + } + ), + encoding="utf-8", + ) + out = root / "out" + out.mkdir() + os.environ["PROOF_RULES_FILE"] = str(rules) + os.environ["PROOF_OUTPUT_DIR"] = str(out) + os.environ["PROOF_ARTIFACT_DIR"] = str(art) + try: + self.assertEqual(inspect_scan.main([]), 0) + finally: + os.environ.pop("PROOF_RULES_FILE", None) + os.environ.pop("PROOF_OUTPUT_DIR", None) + os.environ.pop("PROOF_ARTIFACT_DIR", None) + items = {i["id"]: i for i in json.loads((out / "checklist.json").read_text())} + self.assertFalse(items["no_eval_short_circuit"]["pass"]) + self.assertFalse(items["no_tb4_hardcoding"]["pass"]) + self.assertTrue(items["miner_byok_openrouter"]["pass"]) + blob = (out / "checklist.json").read_text() + self.assertNotIn("sk-", blob) + + def test_clean_artefact_passes_off_limits(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + art = root / "artifact" / "recipe" / "agent" + art.mkdir(parents=True) + (art / "agent.py").write_text("class MinerAgent:\n pass\n", encoding="utf-8") + rules = root / "rules.json" + rules.write_text( + json.dumps( + [ + {"id": "no_eval_short_circuit", "text": "x"}, + {"id": "no_tb4_hardcoding", "text": "x"}, + {"id": "same_seed", "text": "x"}, + ] + ), + encoding="utf-8", + ) + out = root / "out" + out.mkdir() + os.environ["PROOF_RULES_FILE"] = str(rules) + os.environ["PROOF_OUTPUT_DIR"] = str(out) + os.environ["PROOF_ARTIFACT_DIR"] = str(root / "artifact") + try: + self.assertEqual(inspect_scan.main([]), 0) + finally: + os.environ.pop("PROOF_RULES_FILE", None) + os.environ.pop("PROOF_OUTPUT_DIR", None) + os.environ.pop("PROOF_ARTIFACT_DIR", None) + items = {i["id"]: i for i in json.loads((out / "checklist.json").read_text())} + self.assertTrue(items["no_eval_short_circuit"]["pass"]) + self.assertTrue(items["no_tb4_hardcoding"]["pass"]) + self.assertTrue(items["same_seed"]["pass"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_resolve_agent.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_resolve_agent.py new file mode 100644 index 000000000..771783401 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_resolve_agent.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""resolve_agent.py: AST import path, refuse built-in names.""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +import resolve_agent # noqa: E402 + + +class ResolveAgentTests(unittest.TestCase): + def test_top_level_agent_dir(self) -> None: + d = HERE / "fixtures" / "agent" + path, pythonpath = resolve_agent.discover(d) + self.assertEqual(path, "agent.agent:MinerAgent") + self.assertEqual(Path(pythonpath), d.parent.resolve()) + + def test_recipe_agent_dir(self) -> None: + d = HERE / "fixtures" / "recipe" / "agent" + path, pythonpath = resolve_agent.discover(d) + self.assertEqual(path, "agent.agent:RecipeAgent") + self.assertEqual(Path(pythonpath), (HERE / "fixtures" / "recipe").resolve()) + + def test_import_path_file_wins(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + agent = Path(tmp) / "agent" + agent.mkdir() + (agent / "agent.py").write_text( + "from harbor.agents.base import BaseAgent\n" + "class A(BaseAgent):\n pass\n" + "class B(BaseAgent):\n pass\n", + encoding="utf-8", + ) + with self.assertRaises(SystemExit): + resolve_agent.discover(agent) + (agent / "import_path").write_text("agent.agent:B\n", encoding="utf-8") + path, _ = resolve_agent.discover(agent) + self.assertEqual(path, "agent.agent:B") + + def test_builtin_name_in_import_path_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + agent = Path(tmp) / "agent" + agent.mkdir() + (agent / "import_path").write_text("terminus-2\n", encoding="utf-8") + with self.assertRaises(SystemExit) as ctx: + resolve_agent.discover(agent) + self.assertEqual(ctx.exception.code, 2) + + def test_empty_dir_is_not_an_agent(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + d = Path(tmp) / "agent" + d.mkdir() + self.assertFalse(resolve_agent.is_agent_dir(d)) + + def test_recipe_run_sh_alone_is_not_an_agent_dir(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + recipe = Path(tmp) / "recipe" + recipe.mkdir() + (recipe / "run.sh").write_text("#!/bin/sh\necho classic\n", encoding="utf-8") + self.assertFalse(resolve_agent.is_agent_dir(recipe)) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py new file mode 100644 index 000000000..d1c1e1aed --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""summarize.py: mean of trial rewards only; never a substitute field.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ADAPTOR = HERE.parent +import sys + +sys.path.insert(0, str(ADAPTOR / "harness")) +import summarize # noqa: E402 + + +class SummarizeTests(unittest.TestCase): + def test_mean_of_trial_rewards(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + t1 = root / "job" / "a__1" + t2 = root / "job" / "b__1" + t1.mkdir(parents=True) + t2.mkdir(parents=True) + (t1 / "result.json").write_text( + json.dumps( + { + "trial_name": "a__1", + "verifier_result": {"rewards": {"reward": 1.0}}, + } + ), + encoding="utf-8", + ) + (t2 / "result.json").write_text( + json.dumps( + { + "trial_name": "b__1", + "verifier_result": {"rewards": {"reward": 0.5}}, + } + ), + encoding="utf-8", + ) + trials = summarize.collect_trials(root) + self.assertEqual(len(trials), 2) + self.assertAlmostEqual(summarize.mean_reward(trials), 0.75) + + def test_missing_reward_is_not_a_measurement(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + t = root / "job" / "x__1" + t.mkdir(parents=True) + (t / "result.json").write_text( + json.dumps( + { + "trial_name": "x__1", + "score": 1.0, + "accuracy": 1.0, + "stats": {"mean": 0.99}, + } + ), + encoding="utf-8", + ) + self.assertEqual(summarize.collect_trials(root), []) + + def test_job_level_result_is_not_a_trial(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + job = root / "job" + job.mkdir(parents=True) + (job / "result.json").write_text( + json.dumps({"stats": {"evals": {"x": {"metrics": [{"mean": 0.9}]}}}}), + encoding="utf-8", + ) + self.assertEqual(summarize.collect_trials(root), []) + + def test_redact_secrets_in_tail_and_report(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + secrets = root / "secrets" + secrets.mkdir() + (secrets / "inference_key").write_text("sk-secret-owner\n", encoding="utf-8") + jobs = root / "jobs" + trial = jobs / "job" / "t__1" + trial.mkdir(parents=True) + (trial / "result.json").write_text( + json.dumps( + { + "trial_name": "t__1", + "verifier_result": {"rewards": {"reward": 1.0}}, + } + ), + encoding="utf-8", + ) + log = root / "harbor.log" + log.write_text("called with sk-secret-owner\n", encoding="utf-8") + out = root / "report.json" + import os + + os.environ["PROOF_SECRETS_DIR"] = str(secrets) + os.environ["PROOF_SECRET_FILES"] = "inference_key" + try: + rc = summarize.main( + [ + "--jobs-dir", + str(jobs), + "--log", + str(log), + "--output", + str(out), + "--harbor-exit", + "0", + "--agent", + "agent.agent:MinerAgent", + "--agent-source", + "artifact_dir/agent", + ] + ) + finally: + os.environ.pop("PROOF_SECRETS_DIR", None) + os.environ.pop("PROOF_SECRET_FILES", None) + self.assertEqual(rc, 0) + dumped = out.read_text(encoding="utf-8") + self.assertNotIn("sk-secret-owner", dumped) + self.assertIn("[REDACTED]", dumped) + report = json.loads(dumped) + self.assertAlmostEqual(report["primary_value"], 1.0) + self.assertTrue(report["claim_holds"]) + + def test_no_trials_is_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "jobs").mkdir() + with self.assertRaises(SystemExit) as ctx: + summarize.main( + [ + "--jobs-dir", + str(root / "jobs"), + "--output", + str(root / "report.json"), + ] + ) + self.assertEqual(ctx.exception.code, 2) + self.assertFalse((root / "report.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 26adbaf4e..a6814ed27 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -87,7 +87,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Automatic emission | **v0 loop** | `ProofEmitter` (`PROOF_EMIT_POLL_SECS`, default 120 — same cadence as bounty) pins `E` at the last epoch block, signs `emission_scores` when any lattice is positive, and otherwise covers `E` with `NoScore(ChallengeInternal)` so D24 can seal (Held once this host already scored the epoch). The scored-epoch watermark is persisted (`PROOF_SCORED_EPOCH_FILE` on the artefacts volume); the gateway also 409s a `ChallengeInternal` burn that would replace a positive leaf. In-memory store still loses submissions on restart. Do not infer on-chain payment from `can_score`. | | RLM engine (`crates/proof-rlm*`, `proof-canon`) | **generic / fail-closed** | Topic schema carries generic bindings (`constraints.{firecracker_required, model_pin, task_slice, params}`, `checklist` rule vector, `eval_executor.{require_offer_commitment, max_proof_deadline_s}`); `custom_id` is topic data (open needs a registered runner). Core: versioned rule sets + checklist + spend token (no paid inference behind a red checklist), lifecycle `draft → owner_presend → awaiting_owner_keys → provisioning → baselining → open ⇄ evaluating → promoting → closed` with owner hooks, `CustomRunner` + `RunnerRegistry` (**empty by default**), `TopicVmOrchestrator` boundary with `UnwiredVmOrchestrator` and the generic `VmBackedRunner`, promotion rule. Store: migration `0020_proof_rlm.sql` + `PgRlmStore` / `MemoryRlmStore` (topic versions, rule versions, checklists, transitions, baseline, artefact metadata, promotion continuum). Host: `RlmScorer` routed through `FamilyMux` (per-topic lease from score to persist, promotion decided against the store's best with a compare-and-swap on the pointer; runner-measured `flops_used` in the verdict, missing → 503, over budget → reject; `artifact_uri` reaches the runner), artefact zips + `best.json` + `events.jsonl`, `TopicSetup` driver (`mark_sealed` opens only a signed, valid, open document sealing the RLM's measured value). **No registered runner, no challenge content by default:** every custom topic answers **503** until the operator lists ids in `PROOF_VM_RUNNER_CUSTOM_IDS`. The registry is wired from the topic-VM orchestrator env alone: live orchestrator + ≥1 id with no Lium harvest → `FamilyMux::custom_only` (custom scores, `nll` / `throughput` **503**, no row); no placeholder Lium key is needed to open custom topics. `/v1/status` reports the families apart — `live_harvest_wired` is Lium only; `custom_family_wired` / `registered_custom` / `custom_ready` are the custom family. | | Topic-VM orchestrator (`crates/proof-vm-proto`, `proof-vm-fc`, `proof-vm-agent`, `proof-fc-host`, `bins/proof-vm-orchestrator`) | **implemented / operator-gated** | `FirecrackerOrchestrator` is the live `TopicVmOrchestrator`: HTTPS client (bearer file, never logged; https only off loopback) of the `proof-vm-orchestrator` agent on a **dedicated KVM host**. Preferred by `bins/proof-challenge` when `PROOF_VM_ORCHESTRATOR_URL` + `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` are set; `PROOF_RLM_VM_IMAGE_DIGEST` pins the RLM rootfs (4 vCPU / 8192 MiB; unpinned → 503). Agent: one jailed Firecracker RLM VM per `topic_id` (digest re-hashed before boot, hard topic bind on envelope + job, per-VM job lock), vsock jobs, owner key material staged from the host's own dir, per-VM nftables egress allowlist, **sister** miner guest with no network for every paid run, host-stamped `sandboxed` / guest-measured `flops_used` (the attestation names the job's topic / submission / artefact and both agent and CP run `bind_evidence` before accepting it), jail guard so a failed boot or a cancelled sister leaves nothing on the host, dead-VM reaping per retain policy (`crashed`, topic may recreate), destroy-or-retain teardown. `deploy/systemd/proof-vm-orchestrator.service` + [`runbooks/proof-vm-orchestrator.md`](runbooks/proof-vm-orchestrator.md). Operator probe `GET /v1/admin/proof/vm-orchestrator` (the CP's own client: `ready` / `reason`, agent health, `custom_family_wired` / `registered_custom`, `live_harvest_wired` Lium-only) and the staging harness `deploy/scripts/proof-vm-wire-check.sh` (env / agent / cp / boot-probe / submit-probe / matrix; tested against the fake agent) with placeholder overlays `deploy/env/*.staging*.example`. **Staging:** the agent booted Firecracker colocated on `cortex-staging` (nested DO `/dev/kvm` — an allowed exception, proven) and the runbook's § 4 fail-closed matrix came back green; image digests are operator state on that host (computed from images built outside this repo; nothing in git invents one), the § 5 happy path and § 6 sign-off are still to be recorded, and nested KVM stays fragile (boot fails → provision the dedicated droplet). **Production:** a dedicated DO droplet (`g-8vcpu-32gb`, nyc1, nested `/dev/kvm`) on the VPC, never colocated on the CP; the agent's certificate must carry a SAN for the host the CP's URL names (`PROOF_VM_AGENT_TLS_SANS`, boot-checked); not deployed. CI runs the fake hypervisor only. mTLS is a follow-up. | -| Experiment VMs — one VM per paid job (`crates/proof-experiment`, `proof-fc-experiment`, `proof-vm-guest`, `bins/proof-vm-guest-agent`, `deploy/guest/`) | **implemented / not yet baked or booted** | Generic in-guest binding read from the signed topic's `constraints.params` (`in_guest_benchmark_runner`, `experiment_pack_digest`, optional `experiment_pack_path` / size ask; `proof-experiment`), held under configurable caps (lock: 16 vCPU / 32 GiB RAM — default, ceiling, **and hard maximum**: `ExperimentCeilings::validate` refuses a CP or host ceiling above 16 / 32768 so the process does not boot, and no shape above it is admitted; writable disk ≥ 16 GiB with 32 GiB default, not locked; both CP `PROOF_EXPERIMENT_VM_*` and host `PROOF_VM_AGENT_EXPERIMENT_MAX_*`; over = 503, never a clamp). `proof_rlm::run_paid_job` creates one dedicated `TopicVmSpec { experiment }` per `Baseline` / `Evaluate`, runs the job, destroys the VM whatever the outcome, and returns the outcome **only when the orchestrator confirms the destroy** — `Ok(false)` or a teardown error is `VmError::TeardownUnconfirmed` (503, no row, no baseline) even after a successful run (`VmBackedRunner` and `TopicSetup` both route through it). Agent: experiment VMs bypass the one-per-topic rule, never answer `attach`, count against `PROOF_VM_AGENT_MAX_EXPERIMENT_VMS` (503 `capacity`), ids `-x`; health reports `experiment_vms`. Host layer `proof-fc-experiment`: ceilings → pack resolved under `PROOF_VM_AGENT_EXPERIMENT_PACK_DIR` + re-hashed (`verify_artifact`, ≤ 160 MiB) **before any jail** → boot with `disk_mib` scratch → `StagePack` over vsock → job bound to the VM's runner + pack → `SisterAttestation { mode: experiment_vm }` for that VM and job; CP refuses an attestation naming another VM. Guest (`proof-vm-guest-agent`, vsock `:5000` or stdio): hello, tmpfs secrets (chowned to the run-as user), pack verify + unpack, artefact fetch **streamed** under a hard 64 MiB cap (cut mid-body, never buffered first), adaptor exec contract (`/opt/proof/runners//{run,inspect,propose_rules}`, `PROOF_*` env — two signed params that collide as one `PROOF_PARAM_*` name are refused before anything runs — `report.json` / `checklist.json` / `rules.json`), stdout / stderr drained into a rolling 64 KiB tail while the process runs, deadline kill, secret redaction in tails and evidence; **no default value ever** — no runner / adaptor / pack / report / finite value → `Failed`. Bake `deploy/guest/bake-rootfs.sh` (Debian minbase + rootless podman/crun/fuse-overlayfs with the store on run-as-owned scratch paths `init.sh` creates — `graphroot /var/lib/proof/containers/storage`, `runroot /run/user//containers` — generic operator hooks `--extra-pkgs` / `--overlay` / `--chroot-hook` for harness tooling, tree budget 2560 MiB, image named after its own `sha256sum`, RE-LOCK steps printed) with the adaptor contract and a fail-closed skeleton only (`deploy/guest/runners/README.md`); **no adaptor, harness, or scoring rule is committed** — a test greps `deploy/guest/` for harness names. Tests: fake orchestrator (incl. injected unconfirmed / failed teardown) / hypervisor / recording stager / shell-script adaptors (incl. param collision, output flood) / chunked-body fetch cap / bake `--dry-run`; **no image built or booted, no live run, no metal evidence** — runbook [`runbooks/proof-experiment-vms.md`](runbooks/proof-experiment-vms.md) § Limitations (rootless podman not Docker-in-VM, guest kernel features, guest-authored `flops_used`, inspection needs an adaptor). | +| Experiment VMs — one VM per paid job (`crates/proof-experiment`, `proof-fc-experiment`, `proof-vm-guest`, `bins/proof-vm-guest-agent`, `deploy/guest/`) | **implemented / not yet baked or booted** | Generic in-guest binding read from the signed topic's `constraints.params` (`in_guest_benchmark_runner`, `experiment_pack_digest`, optional `experiment_pack_path` / size ask; `proof-experiment`), held under configurable caps (lock: 16 vCPU / 32 GiB RAM — default, ceiling, **and hard maximum**: `ExperimentCeilings::validate` refuses a CP or host ceiling above 16 / 32768 so the process does not boot, and no shape above it is admitted; writable disk ≥ 16 GiB with 32 GiB default, not locked; both CP `PROOF_EXPERIMENT_VM_*` and host `PROOF_VM_AGENT_EXPERIMENT_MAX_*`; over = 503, never a clamp). `proof_rlm::run_paid_job` creates one dedicated `TopicVmSpec { experiment }` per `Baseline` / `Evaluate`, runs the job, destroys the VM whatever the outcome, and returns the outcome **only when the orchestrator confirms the destroy** — `Ok(false)` or a teardown error is `VmError::TeardownUnconfirmed` (503, no row, no baseline) even after a successful run (`VmBackedRunner` and `TopicSetup` both route through it). Agent: experiment VMs bypass the one-per-topic rule, never answer `attach`, count against `PROOF_VM_AGENT_MAX_EXPERIMENT_VMS` (503 `capacity`), ids `-x`; health reports `experiment_vms`. Host layer `proof-fc-experiment`: ceilings → pack resolved under `PROOF_VM_AGENT_EXPERIMENT_PACK_DIR` + re-hashed (`verify_artifact`, ≤ 160 MiB) **before any jail** → boot with `disk_mib` scratch → `StagePack` over vsock → job bound to the VM's runner + pack → `SisterAttestation { mode: experiment_vm }` for that VM and job; CP refuses an attestation naming another VM. Guest (`proof-vm-guest-agent`, vsock `:5000` or stdio): hello, tmpfs secrets (chowned to the run-as user), pack verify + unpack, artefact fetch **streamed** under a hard 64 MiB cap (cut mid-body, never buffered first), adaptor exec contract (`/opt/proof/runners//{run,inspect,propose_rules}`, `PROOF_*` env — two signed params that collide as one `PROOF_PARAM_*` name are refused before anything runs — `report.json` / `checklist.json` / `rules.json`), stdout / stderr drained into a rolling 64 KiB tail while the process runs, deadline kill, secret redaction in tails and evidence; **no default value ever** — no runner / adaptor / pack / report / finite value → `Failed`. Bake `deploy/guest/bake-rootfs.sh` (Debian minbase + rootless podman/crun/fuse-overlayfs with the store on run-as-owned scratch paths `init.sh` creates — `graphroot /var/lib/proof/containers/storage`, `runroot /run/user//containers` — generic operator hooks `--extra-pkgs` / `--overlay` / `--chroot-hook` for harness tooling, tree budget 2560 MiB, image named after its own `sha256sum`, RE-LOCK steps printed) with the adaptor contract, fail-closed skeleton, and Harbor evaluate reference adaptor (`deploy/guest/runners/rlm_fc_in_guest_harbor/`); generic guest scripts (`bake-rootfs.sh`, `init.sh`, `agent-loop.sh`) still must not name a harness. Harbor CLI, venv, and task packs stay operator artefacts. Tests: fake orchestrator (incl. injected unconfirmed / failed teardown) / hypervisor / recording stager / shell-script adaptors (incl. param collision, output flood) / chunked-body fetch cap / bake `--dry-run`; **no image built or booted, no live run, no metal evidence** — runbook [`runbooks/proof-experiment-vms.md`](runbooks/proof-experiment-vms.md) § Limitations (rootless podman not Docker-in-VM, guest kernel features, guest-authored `flops_used`, inspection needs an adaptor). | | Autonomous research judge | **partial** | Python `judge.py` requests an acknowledgement, while `agent.py` uses static text checks. General recipe reproduction and the paper's recursive investigation are not implemented. | | Research persistence | **missing** | The service uses `MemoryStore`; submissions and scores are lost on restart. Public HTTP records are not a durable artifact archive. | | Synthesis / shared-stack adoption | **missing** | The second agent and verified adoption loop described in whitepaper §7 are not implemented. | diff --git a/docs/PROOF.md b/docs/PROOF.md index 008e08cd6..a3d39c460 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -487,7 +487,7 @@ id shapes shared with `proof-task`). | `constraints.params.miner_byok` | Comma-separated environment variable names (`[A-Z][A-Z0-9_]{0,63}`) a miner **must** send in the submit body's `env`. A submission missing one is **400** before any row, rent, or paid inference. The topic carries the *name*; the miner carries the value. See § Miner BYOK | | `constraints.params.miner_env_allowlist` | Additional variable names a miner **may** send (accepted, never demanded). `miner_byok` is always allowed on top of it | | `constraints.params.inject_miner_env_sister` | `"true"` also forwards the miner env into the **sister** guest that runs the miner's entrypoint. Absent / `"false"` keeps it in the runner guest only | -| `constraints.params.baseline_runner` (synonym `in_guest_benchmark_runner`) | Generic knob (`proof-experiment`): selects an **operator adaptor id** baked into the guest image (e.g. `baseline_runner: operator_adaptor_v0`); the topic's paid jobs then run in **one dedicated experiment VM per job**, sized under a hard 16 vCPU / 32 GiB lock, and score only once the orchestrator confirms that VM destroyed. Values are topic data; no adaptor, harness, or scoring rule ships in git | +| `constraints.params.baseline_runner` (synonym `in_guest_benchmark_runner`) | Generic knob (`proof-experiment`): selects an **operator adaptor id** baked into the guest image (e.g. `baseline_runner: rlm_fc_in_guest_harbor`); the topic's paid jobs then run in **one dedicated experiment VM per job**, sized under a hard 16 vCPU / 32 GiB lock, and score only once the orchestrator confirms that VM destroyed. Values are topic data. A versioned Harbor reference adaptor ships under `deploy/guest/runners/rlm_fc_in_guest_harbor/` for operators to bake; the Harbor CLI and task pack do not | | `constraints.params.experiment_pack_digest` / `experiment_pack_path` | `sha256:` of the pack tar the KVM host stages into that VM (required with a runner; never defaulted) / optional relative locator under the host pack dir | | `constraints.params.experiment_vcpus` / `experiment_mem_mib` / `experiment_disk_mib` | The topic's size ask: silent = the operator defaults (lock 16 vCPU / 32 GiB / 32 GiB disk), an ask may go up to the ceilings (lock 16 vCPU / 32 GiB — the default is the ceiling; disk ≥ 16 GiB); over = 503, never a clamp | | `checklist` | ≤64 `{id, text}` anti-cheat rules (unique slug ids), version 1 of the rule set | diff --git a/docs/external-miner/proof-tbench.md b/docs/external-miner/proof-tbench.md index 87d114ca8..290166c6c 100644 --- a/docs/external-miner/proof-tbench.md +++ b/docs/external-miner/proof-tbench.md @@ -123,6 +123,35 @@ tar -cf recipe.tar recipe/ # uncompressed. No gzip, no zip sha256sum recipe.tar # this is your artifact_digest ``` +The guest unpacks that tar under `$PROOF_ARTIFACT_DIR`. Evaluate attaches +your **Harbor agent**, not a silent copy of the operator's `terminus-2`. +Harbor's `-a` / `--agent` accepts a built-in name or a Python import path +(`module.path:ClassName`); it does **not** take a filesystem path. The +adaptor therefore imports your class from the artefact and passes that +import path as `-a`. Layout after unpack (paths relative to +`PROOF_ARTIFACT_DIR`): + +``` +recipe/ + agent/ # PREFERRED: Harbor agent (BaseAgent / BaseInstalledAgent) + agent.py + import_path # optional: one line `agent.agent:YourClass` + run.sh # optional classic marker; inspect may see it; evaluate does not exec it + README.md +``` + +If you pack with `tar -cf recipe.tar -C recipe .`, the same `agent/` directory +sits at the tar root (`$PROOF_ARTIFACT_DIR/agent`). Resolution order: +`$PROOF_ARTIFACT_DIR/agent`, then `$PROOF_ARTIFACT_DIR/recipe/agent`. A +`recipe/run.sh` with no Harbor agent dir is **not** scored as a substitute — +evaluate fails closed rather than falling back to the topic agent. Off-limits +in the tree (inspect fails the named rule): `no_eval_short_circuit`, +`no_tb4_hardcoding`. + +Env the run sees: `PROOF_SEED`, `PROOF_MODEL_PIN`, `PROOF_TASK_SLICE`, +`PROOF_PARAM_*`, `PROOF_PACK_DIR`, `PROOF_ARTIFACT_DIR`, `PROOF_OUTPUT_DIR`, +`PROOF_WORK_DIR`, and miner BYOK under `PROOF_MINER_ENV_DIR`. + - Serve **that exact file** at `artifact_uri` and keep it. Re-running `tar` later produces different bytes (mtimes, member order) and therefore a different digest, and the run is refused rather than run on a substitute. diff --git a/docs/runbooks/proof-experiment-vms.md b/docs/runbooks/proof-experiment-vms.md index 2191604ee..12e90ea0a 100644 --- a/docs/runbooks/proof-experiment-vms.md +++ b/docs/runbooks/proof-experiment-vms.md @@ -39,8 +39,12 @@ ordinary path (topic VM + sister guest). The topic content the operator staged (the pack, the harness CLI and agent the adaptor drives, how a trial becomes a number) lives **outside git** — on the KVM host, in the baked image, and in the signed document — and is recognised by nothing here. -`deploy/guest/runners/` ships the adaptor **contract and a skeleton only**; -no adaptor, harness, or scoring rule is committed. +`deploy/guest/runners/` ships the adaptor **contract**, a fail-closed +skeleton, and the versioned Harbor reference adaptor +[`rlm_fc_in_guest_harbor/`](../../deploy/guest/runners/rlm_fc_in_guest_harbor/) +(bake with `--runner rlm_fc_in_guest_harbor=`). Harbor CLI, venv, +and task packs stay operator artefacts. Proof binaries still compile none of +that in. ## Resource caps (Architecte lock) @@ -132,7 +136,7 @@ VMs and experiment VMs: | `/sbin/init` (`deploy/guest/init.sh`) + `catatonit` | mounts, cgroup v2, scratch on `/dev/vdb`, run-as user, agent loop | no systemd in the guest | | rootless podman + crun + fuse-overlayfs + pasta/slirp4netns + podman-compose | containers **inside** the VM as an unprivileged user (`--run-as-uid 1000`, subuid `100000:65536`), `cgroup_manager = cgroupfs`, store on paths `init.sh` creates and chowns to that user — `graphroot = /var/lib/proof/containers/storage` (scratch disk), `runroot = /run/user//containers` (its `XDG_RUNTIME_DIR` tmpfs); never `/var/lib/containers` / `/run/containers` (root-owned, read-only rootfs) | the harness's container runtime; no nested KVM | | `--extra-pkgs a,b,c` · `--overlay DIR` · `--chroot-hook SCRIPT` | the operator's harness tooling: Debian packages; a tree copied over the rootfs (a prebuilt venv, a CLI); a script run inside the chroot (build a venv, `pip install ==`) | generic hooks — this repo names no harness; pin every version the hook installs, record it in your own manifest | -| `--runner =` | adaptor under `/opt/proof/runners//` | operator capability, **outside git**; contract + skeleton in [`../../deploy/guest/runners/README.md`](../../deploy/guest/runners/README.md) | +| `--runner =` | adaptor under `/opt/proof/runners//` | operator capability; contract + skeleton in [`../../deploy/guest/runners/README.md`](../../deploy/guest/runners/README.md); bake the Harbor reference from [`../../deploy/guest/runners/rlm_fc_in_guest_harbor/`](../../deploy/guest/runners/rlm_fc_in_guest_harbor/) when the topic names that id | **Size budget.** The baked tree must fit `--budget-mib` (default 2560 MiB; the 1.5–2.5 GiB target — minbase + podman stack ≈ 0.6–0.9 GiB, + whatever @@ -150,9 +154,9 @@ rustup target add x86_64-unknown-linux-musl CC_x86_64_unknown_linux_musl=musl-gcc cargo build --release -p proof-vm-guest-agent-bin --target x86_64-unknown-linux-musl deploy/guest/bake-rootfs.sh \ --guest-agent target/x86_64-unknown-linux-musl/release/proof-vm-guest-agent \ - --runner =/path/outside/git/ \ # = what your topics put in baseline_runner + --runner rlm_fc_in_guest_harbor=deploy/guest/runners/rlm_fc_in_guest_harbor \ --extra-pkgs python3-venv,python3-pip,git \ # what your adaptor's harness needs - --chroot-hook /path/outside/git/install-harness.sh \ # pins and installs it inside the chroot + --chroot-hook /path/outside/git/install-harbor.sh \ # pins Harbor inside the chroot --resolver \ --check-kernel-config \ --out-dir ./out @@ -200,8 +204,8 @@ typed from a document. carries `experiment_max_vcpus`, `experiment_max_mem_mib`, `experiment_disk_mib`, `experiment_image`. 6. **Topic:** the signed document's `constraints.params` carry the runner id - your bake installed (`baseline_runner: ` from the `--runner =…` - above), `experiment_pack_digest: sha256:`, the adaptor's + your bake installed (`baseline_runner: rlm_fc_in_guest_harbor` from the + `--runner` line above when using the in-repo Harbor adaptor), `experiment_pack_digest: sha256:`, the adaptor's `PROOF_PARAM_*` inputs (names that stay distinct after upper-casing and `-` → `_`), and any size ask under the ceilings (`experiment_vcpus` ≤ 16, `experiment_mem_mib` ≤ 32768; omit them for @@ -274,11 +278,11 @@ still leak no path, key, or origin (the wire check's `cp` step). adaptor's `inspect` entrypoint (topic RLM work); an adaptor that ships none leaves its topic unable to reach `Evaluate` until the operator provides one — by design, no spend without a green checklist. -- **No adaptor ships in git.** `deploy/guest/runners/` is the contract and a - fail-closed skeleton; the harness CLI, agent, task format, and how a trial - becomes `primary_value` are operator artefacts baked with the generic - hooks and selected by signed params. A trial without a measurement is - never scored from some other value. +- **Reference adaptor in git, harness CLI not.** `deploy/guest/runners/` is + the contract, a fail-closed skeleton, and the Harbor evaluate reference + adaptor (`rlm_fc_in_guest_harbor`). The Harbor CLI, venv, and task pack are + still operator artefacts baked with the generic hooks. A trial without a + measurement is never scored from some other value. - **Pack size.** Packs travel in one vsock frame: ≤ 160 MiB uncompressed tar. Larger packs need a block-device staging path this protocol version does not have; the host refuses them by name. diff --git a/xtask/src/external_docs_check.rs b/xtask/src/external_docs_check.rs index 37fd489b1..e0322ecb0 100644 --- a/xtask/src/external_docs_check.rs +++ b/xtask/src/external_docs_check.rs @@ -134,6 +134,8 @@ const PAGE_PINS: &[(&str, &[&str])] = &[ "epsilon_rel", "OPENROUTER_API_KEY", "X-Lium-Api-Key", + "PROOF_ARTIFACT_DIR", + "recipe/agent", ], ), ];