Skip to content

fix(proof): fail closed when inspect skips artefact files #269

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
echobt wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(proof): fail closed when inspect skips artefact files #269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion deploy/guest/runners/rlm_fc_in_guest_harbor/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ Off-limits in the artefact (inspect fails the named rule):
- `no_eval_short_circuit` (and `skip_eval` / `skip_verifier` / `always_pass_eval` / `short_circuit_eval`)
- `no_tb4_hardcoding` (and `tb4_answers` / `hardcoded_tb4`)

A file/byte-limit truncation marks the scan incomplete and fails those
A file/byte-limit truncation, or a regular file that is oversized,
unreadable, or binary, marks the scan incomplete and fails those
off-limits rules. Unknown rule ids fail closed. Do not quote those markers
in miner code or README inside the tar.

Expand Down
15 changes: 11 additions & 4 deletions deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,10 @@ def load_rules(path: Path) -> list[dict[str, str]]:
def collect_artefact_text(root: Path | None) -> tuple[str, int, list[str], bool]:
"""Return ``(text, n_scanned, names, incomplete)``.

``incomplete`` is true when a file or byte cap stopped the walk before
every regular file was considered. Callers must not treat a truncated
scan as proof that an off-limits marker is absent.
``incomplete`` is true when a file or byte cap stopped the walk, or a
regular file was oversized / unreadable so its contents were not
inspected. Callers must not treat that absence as a clean off-limits
pass.
"""
if root is None or not root.is_dir():
return "", 0, [], False
Expand All @@ -103,14 +104,20 @@ def collect_artefact_text(root: Path | None) -> tuple[str, int, list[str], bool]
try:
size = path.stat().st_size
except OSError:
incomplete = True
continue
if size == 0:
continue
if size == 0 or size > MAX_FILE_BYTES:
if size > MAX_FILE_BYTES:
incomplete = True
continue
try:
data = path.read_bytes()
except OSError:
incomplete = True
continue
if b"\x00" in data[:1024]:
incomplete = True
Comment on lines 106 to +120

@greptile-apps greptile-apps Bot Sep 10, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Report actual scan failure

Stat failures, read failures, and NUL-classified binary files set the same incomplete flag as file or byte limits. The later checklist evidence consequently says file/byte limit even when no limit was reached. This is a non-blocking diagnostic concern: operators investigating a rejected artefact are directed to the wrong cause, increasing time to remediate access or content problems. Retain the reason for an incomplete scan and report it in the failed off-limits evidence.

Artifacts

Evidence from the check

  • The authored executable imports the repository scanner, runs a clean baseline and controlled failure branches, and asserts the resulting checklist evidence; it provides the reproducible test source.

Command output from the check

  • The baseline command scanned a harmless text file and passed both off-limits rules with clean-absence evidence, establishing the comparison behavior.

Command output from the check

  • The failure command exercised stat, read, and binary paths and showed every resulting failed rule claims a file/byte-limit condition; the evidence is misleading.

Command output from the check

  • The repository diff check completed cleanly and status showed only validation artifacts, confirming repository source was not modified.

Command output from the check

  • The Python compile command completed successfully for the inspected scanner module, confirming the executed target is syntactically valid.

View artifacts

T-Rex Ran code and verified through T-Rex

continue
total += len(data)
blobs.append(data.decode("utf-8", errors="replace"))
Expand Down
133 changes: 133 additions & 0 deletions deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

HERE = Path(__file__).resolve().parent
sys_path_parent = str(HERE.parent)
Expand Down Expand Up @@ -156,6 +157,138 @@ def test_unknown_rule_fails_closed_with_artefact(self) -> None:
self.assertFalse(items["must_provide_reproducible_benchmark"]["pass"])
self.assertIn("unknown", items["must_provide_reproducible_benchmark"]["evidence"])

def _run_inspect(self, art: Path, rules: Path, out: Path) -> dict[str, dict]:
os.environ["PROOF_RULES_FILE"] = str(rules)
os.environ["PROOF_OUTPUT_DIR"] = str(out)
os.environ["PROOF_ARTIFACT_DIR"] = str(art)
try:
self.assertEqual(inspect_scan.main([]), 0)
finally:
os.environ.pop("PROOF_RULES_FILE", None)
os.environ.pop("PROOF_OUTPUT_DIR", None)
os.environ.pop("PROOF_ARTIFACT_DIR", None)
return {i["id"]: i for i in json.loads((out / "checklist.json").read_text())}

def test_oversized_file_fails_off_limits(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
art = root / "artifact"
art.mkdir()
(art / "ok.py").write_text("class MinerAgent:\n pass\n", encoding="utf-8")
oversized = art / "oversized.txt"
payload = ("x" * inspect_scan.MAX_FILE_BYTES) + "\nno_tb4_hardcoding\n"
oversized.write_text(payload, encoding="utf-8")
rules = root / "rules.json"
rules.write_text(
json.dumps(
[
{"id": "no_eval_short_circuit", "text": "x"},
{"id": "no_tb4_hardcoding", "text": "x"},
]
),
encoding="utf-8",
)
out = root / "out"
out.mkdir()
items = self._run_inspect(art, rules, out)
self.assertFalse(items["no_tb4_hardcoding"]["pass"])
self.assertFalse(items["no_eval_short_circuit"]["pass"])
self.assertIn("incomplete", items["no_tb4_hardcoding"]["evidence"])

def test_unreadable_file_fails_off_limits(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
art = root / "artifact"
art.mkdir()
(art / "ok.py").write_text("class MinerAgent:\n pass\n", encoding="utf-8")
hidden = art / "stat-failed.txt"
hidden.write_text("skip_eval\n", encoding="utf-8")
rules = root / "rules.json"
rules.write_text(
json.dumps(
[
{"id": "no_eval_short_circuit", "text": "x"},
{"id": "no_tb4_hardcoding", "text": "x"},
]
),
encoding="utf-8",
)
out = root / "out"
out.mkdir()
real_stat = Path.stat
size_lookups = {"n": 0}

def _stat(self: Path, *args: object, **kwargs: object) -> os.stat_result:
result = real_stat(self, *args, **kwargs)
if self.name == "stat-failed.txt":
size_lookups["n"] += 1
# is_file() must succeed so the walker reaches the size lookup.
if size_lookups["n"] > 1:
raise OSError("simulated stat failure")
return result

with patch.object(Path, "stat", _stat):
items = self._run_inspect(art, rules, out)
self.assertFalse(items["no_eval_short_circuit"]["pass"])
self.assertFalse(items["no_tb4_hardcoding"]["pass"])
self.assertIn("incomplete", items["no_eval_short_circuit"]["evidence"])

def test_read_failed_file_fails_off_limits(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
art = root / "artifact"
art.mkdir()
(art / "ok.py").write_text("class MinerAgent:\n pass\n", encoding="utf-8")
hidden = art / "read-failed.txt"
hidden.write_text("tb4_answers\n", encoding="utf-8")
rules = root / "rules.json"
rules.write_text(
json.dumps(
[
{"id": "no_eval_short_circuit", "text": "x"},
{"id": "no_tb4_hardcoding", "text": "x"},
]
),
encoding="utf-8",
)
out = root / "out"
out.mkdir()
real_read = Path.read_bytes

def _read(self: Path) -> bytes:
if self.name == "read-failed.txt":
raise OSError("simulated read failure")
return real_read(self)

with patch.object(Path, "read_bytes", _read):
items = self._run_inspect(art, rules, out)
self.assertFalse(items["no_tb4_hardcoding"]["pass"])
self.assertFalse(items["no_eval_short_circuit"]["pass"])
self.assertIn("incomplete", items["no_tb4_hardcoding"]["evidence"])

def test_binary_file_fails_off_limits(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
art = root / "artifact"
art.mkdir()
(art / "ok.py").write_text("class MinerAgent:\n pass\n", encoding="utf-8")
(art / "blob.bin").write_bytes(b"\x00no_tb4_hardcoding\n")
rules = root / "rules.json"
rules.write_text(
json.dumps(
[
{"id": "no_eval_short_circuit", "text": "x"},
{"id": "no_tb4_hardcoding", "text": "x"},
]
),
encoding="utf-8",
)
out = root / "out"
out.mkdir()
items = self._run_inspect(art, rules, out)
self.assertFalse(items["no_tb4_hardcoding"]["pass"])
self.assertIn("incomplete", items["no_tb4_hardcoding"]["evidence"])


if __name__ == "__main__":
unittest.main()
Loading