diff --git a/.github/workflows/evidence.yml b/.github/workflows/evidence.yml index 4003079..8258972 100644 --- a/.github/workflows/evidence.yml +++ b/.github/workflows/evidence.yml @@ -90,4 +90,4 @@ jobs: suffix="" [ "$RUNNER_OS" = "Windows" ] && suffix=".exe" exe="target/release/opencode-setup-system$suffix" - "$py" scripts/evidence.py --binary "$exe" --harness opencode --writes-absent "it reads configuration from the target and writes the global ~/.config/opencode/opencode.json regardless, so a write here would reach a real home rather than the target" --probe 'debug agent build' --probe-kind postures --probe-baseline '"permission": "bash"' --probe-full-auto '"pattern": "*.env" THEN "permission": "*"' + "$py" scripts/evidence.py --binary "$exe" --harness opencode --writes-absent "it reads configuration from the target and writes the global ~/.config/opencode/opencode.json regardless, so a write here would reach a real home rather than the target" --probe 'debug agent build' --probe-kind autonomy --probe-baseline '"pattern": "*.env" THEN "permission": "*" THEN "action": "allow" THEN "pattern": "*"' diff --git a/scripts/evidence.py b/scripts/evidence.py index 61e2208..faa1038 100644 --- a/scripts/evidence.py +++ b/scripts/evidence.py @@ -529,7 +529,11 @@ def the_product_reads_our_setup( directory above where the product looks. Each installed, verified and restored cleanly, and changed nothing about the product. - `postures` is the stronger form: the probe must say something *different* + `autonomy` checks every standard variant against the common autonomous + policy, then requires a clean target to produce a different answer. The + standard variants no longer differ in approval or sandbox policy. + + `postures` is the older form: the probe must say something *different* with `full-auto` installed than with `baseline`, which no constant can do. `reads` only asks the product to name our file at our target. @@ -552,19 +556,51 @@ def says(text: str, marker: str) -> bool: at = found + len(fragment) return True - def ask(setup: str) -> str: - run_text([binary, "select", setup, "--target", str(target)]) + def ask(setup: str | None, destination: Path = target) -> str: + if setup is not None: + run_text([binary, "select", setup, "--target", str(destination)]) started = subprocess.run( - [binary, "launch", "--target", str(target), "--prefix", str(prefix), + [binary, "launch", "--target", str(destination), "--prefix", str(prefix), "--json", "--", *probe], capture_output=True, text=True, timeout=180, - env=contained(target.parent), + env=contained(destination.parent), ) + if kind == "autonomy": + try: + report = json.loads(started.stdout) + except ValueError as error: + raise Failed("the autonomy probe did not return its machine report") from error + if not isinstance(report, dict) or not report or "error" in report: + raise Failed("the autonomy probe returned no usable configuration report") + # Doctor can exit 1 because this credential-free run has no login. + # Its configuration check must still have succeeded in both the + # installed case and the empty-target control. + if "checks" in report and ( + report["checks"].get("config.load", {}).get("status") != "ok" + ): + raise Failed("the autonomy probe could not load the product configuration") + return started.stdout return started.stdout + started.stderr print("reads ", end="", flush=True) + if kind == "autonomy": + for setup in ("minimal", "baseline", "full-auto", "nddev-builder"): + said = ask(setup) + if not says(said, baseline): + raise Failed( + f"with {setup!r} installed the product did not report the " + f"autonomous policy {baseline!r}; it said:\n{said[:600]}" + ) + with tempfile.TemporaryDirectory(prefix="empty-control-", dir=target.parent) as room: + empty = Path(room) / "target" + empty.mkdir() + control = ask(None, empty) + if says(control, baseline): + raise Failed("the empty-target control also reports our autonomous policy") + print("-> all four variants resolve the autonomous policy; the empty target does not") + return said = ask("baseline") if kind != "postures": # `reads` names a file, and a file name is not a string comparison on @@ -997,7 +1033,7 @@ def main() -> int: parser.add_argument("--binary", required=True) parser.add_argument("--harness", required=True) parser.add_argument("--probe", default="", help="argv that makes the product report its resolved configuration") - parser.add_argument("--probe-kind", default="", choices=["", "postures", "reads"]) + parser.add_argument("--probe-kind", default="", choices=["", "postures", "reads", "autonomy"]) parser.add_argument("--probe-baseline", default="", help="what the probe must say with `baseline` installed") parser.add_argument("--probe-full-auto", default="", help="what it must say with `full-auto` installed, when the kind is `postures`") parser.add_argument("--probe-absent", default="", help="the measured reason this product reports nothing without credentials")