diff --git a/crates/proof-rlm/src/vm.rs b/crates/proof-rlm/src/vm.rs index a9ebb0375..b9b577cb8 100644 --- a/crates/proof-rlm/src/vm.rs +++ b/crates/proof-rlm/src/vm.rs @@ -949,6 +949,14 @@ mod tests { "inspection runs no miner code" ); let outputs = [ + // The whole authored set, first: it is the variant a host baked + // before `945e143f` cannot decode, and the one whose payload is a + // whole document rather than a report. A wire round trip that + // skips it is exactly how a host↔guest skew ships. + VmJobOutput::Authored(Box::new(crate::fixtures::authored_set( + &crate::fixtures::topic(), + rules().rules, + ))), VmJobOutput::Rules(rules().rules), VmJobOutput::Baseline(crate::fixtures::report_for(&req, 0.5)), VmJobOutput::Inspected(crate::runner::InspectOutcome { @@ -966,7 +974,44 @@ mod tests { assert!(json.contains("\"output\""), "{json}"); let back: VmJobOutput = serde_json::from_str(&json).expect("round trip"); assert_eq!(back, out); + // The set travels with **every** part, not only the vector: a + // guest that emits `Authored` and a host that reads it must agree + // on the whole document, and a part that did not survive the wire + // would arrive as an incomplete set (or not at all). + if let VmJobOutput::Authored(set) = &back { + assert!( + set.is_complete(), + "the set lost parts on the wire: {:?}", + set.missing_parts() + ); + assert_eq!(set.missing_parts(), Vec::<&str>::new()); + } + } + } + + /// The wire tag is a **contract**: `Authored` travels as `"authored"`, and + /// a host that does not know that tag fails to decode the frame. + /// + /// This pins the tag rather than the Rust name, because the tag is what + /// two independently built binaries agree on — the guest emits it, the + /// control plane and the orchestrator decode it, and none of them share a + /// build. + #[test] + fn the_authored_tag_is_a_wire_contract() { + let set = crate::fixtures::authored_set(&crate::fixtures::topic(), rules().rules); + let json = serde_json::to_string(&VmJobOutput::Authored(Box::new(set))).expect("json"); + let value: serde_json::Value = serde_json::from_str(&json).expect("json"); + assert_eq!(value["output"], "authored", "{json}"); + // Adjacently tagged (`output` / `body`): the document is the body, and + // it is an object with the five parts rather than a bare vector. + let body = value["body"].as_object().expect("the set is the body"); + for part in proof_topic_authoring::PARTS { + assert!(body.contains_key(part), "{part} did not travel: {json}"); } + // And the old tag still means what it meant: a fragment. + let rules_json = serde_json::to_string(&VmJobOutput::Rules(rules().rules)).expect("json"); + let rules_value: serde_json::Value = serde_json::from_str(&rules_json).expect("json"); + assert_eq!(rules_value["output"], "rules", "{rules_json}"); } /// A topic whose params select an in-guest runner gets **one experiment diff --git a/crates/proof-vm-agent/src/fixtures_tests.rs b/crates/proof-vm-agent/src/fixtures_tests.rs index 1e4247b35..6980b77cc 100644 --- a/crates/proof-vm-agent/src/fixtures_tests.rs +++ b/crates/proof-vm-agent/src/fixtures_tests.rs @@ -19,7 +19,7 @@ use std::time::Duration; use async_trait::async_trait; use proof_canon::ChecklistRule; -use proof_rlm::fixtures::report_for; +use proof_rlm::fixtures::{authored_set, report_for}; use proof_rlm::{ ArtifactFile, Checklist, CustomRunRequest, InspectOutcome, LogFile, RetainPolicy, RunOutcome, TopicVmSpec, VmJob, VmJobOutput, @@ -62,6 +62,13 @@ pub struct FakeHypervisor { /// Whether an experiment VM attests the paid job it ran (the host's /// view). `false` models a host that lost track of what it booted. experiment_attests: AtomicBool, + /// Whether `ProposeRules` answers with the whole authored set + /// (`VmJobOutput::Authored`) instead of a bare rule vector + /// (`VmJobOutput::Rules`). Default **true**: the tip's guest emits the + /// set, so a fake that only ever answered `Rules` would let the whole + /// orchestrator hop be exercised without ever carrying the variant a + /// host baked before `945e143f` cannot decode. + authored_complete: AtomicBool, boots: Mutex>, /// Specs of every VM booted, by id (experiment VMs carry `experiment`). specs: Mutex>, @@ -96,6 +103,7 @@ impl FakeHypervisor { dead: Mutex::new(BTreeSet::new()), dies_under_job: AtomicBool::new(false), experiment_attests: AtomicBool::new(true), + authored_complete: AtomicBool::new(true), boots: Mutex::new(Vec::new()), specs: Mutex::new(Vec::new()), jobs: Mutex::new(Vec::new()), @@ -171,6 +179,12 @@ impl FakeHypervisor { *self.proposed.lock().unwrap() = rules; } + /// Answer `ProposeRules` with the whole authored set (default) or with a + /// bare rule vector (an adaptor/guest baked before the set existed). + pub fn set_authored_complete(&self, v: bool) { + self.authored_complete.store(v, Ordering::SeqCst); + } + /// Make every job take this long (to exercise `Busy`). pub fn set_job_delay(&self, d: Option) { *self.job_delay.lock().unwrap() = d; @@ -338,8 +352,19 @@ impl Hypervisor for FakeHypervisor { ))); } Ok(match job { - VmJob::ProposeRules { .. } => JobOutcome { - output: VmJobOutput::Rules(self.proposed.lock().unwrap().clone()), + VmJob::ProposeRules { topic, .. } => JobOutcome { + // The tip's guest answers `Authored` (the whole set) when its + // adaptor wrote `authoring.json`, and `Rules` (a fragment) + // when it only had `rules.json`. Both travel this hop, so both + // are reachable here. + output: if self.authored_complete.load(Ordering::SeqCst) { + VmJobOutput::Authored(Box::new(authored_set( + topic, + self.proposed.lock().unwrap().clone(), + ))) + } else { + VmJobOutput::Rules(self.proposed.lock().unwrap().clone()) + }, sister: None, }, VmJob::Baseline { request } => JobOutcome { diff --git a/crates/proof-vm-agent/src/lib.rs b/crates/proof-vm-agent/src/lib.rs index 9593d2bea..6f83aa7cc 100644 --- a/crates/proof-vm-agent/src/lib.rs +++ b/crates/proof-vm-agent/src/lib.rs @@ -360,6 +360,81 @@ mod tests { assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); } + /// The whole authored set crosses the orchestrator's **HTTP** hop, every + /// part, and comes back as the set. + /// + /// This is the hop the live FAIL travelled: the guest emits `Authored`, + /// the orchestrator stamps and re-encodes it, and the control plane reads + /// it. A host baked before `945e143f` has no `authored` variant and cannot + /// decode this response — so the shape is pinned here, at the layer that + /// does the re-encoding, rather than only inside the guest. + #[tokio::test] + async fn the_whole_authored_set_crosses_the_job_hop() { + let hv = FakeHypervisor::new(0.8); + let (app, _) = app(hv.clone(), "authored-hop"); + let rec = create(&app).await; + let topic = proof_rlm::fixtures::topic(); + let body = serde_json::to_vec(&RunJobRequest { + topic_id: rec.handle.topic_id.clone(), + job: VmJob::ProposeRules { + topic: Box::new(topic.clone()), + current_version: None, + current: None, + }, + }) + .expect("json"); + let (status, bytes) = post_bytes(&app, &paths::vm_jobs(&rec.handle.vm_id), body).await; + assert_eq!( + status, + StatusCode::OK, + "{}", + String::from_utf8_lossy(&bytes) + ); + // The wire the control plane reads: `output` / `body`, with the set as + // the body and the tag `authored`. + let value: serde_json::Value = serde_json::from_slice(&bytes).expect("json"); + assert_eq!( + value["output"]["output"], "authored", + "the hop must carry the set, not a fragment: {value}" + ); + for part in proof_rlm::AUTHORING_PARTS { + assert!( + value["output"]["body"].get(part).is_some(), + "{part} did not cross the hop: {value}" + ); + } + let out: RunJobResponse = serde_json::from_slice(&bytes).expect("response"); + let VmJobOutput::Authored(set) = out.output else { + panic!("expected the set, got a fragment"); + }; + assert!(set.is_complete(), "missing {:?}", set.missing_parts()); + assert_eq!(set.topic_id, topic.id); + assert!(!set.migrations.is_empty() && !set.apis.is_empty()); + // And a guest baked before the set existed still crosses as `rules`. + hv.set_authored_complete(false); + let body = serde_json::to_vec(&RunJobRequest { + topic_id: rec.handle.topic_id.clone(), + job: VmJob::ProposeRules { + topic: Box::new(topic), + current_version: None, + current: None, + }, + }) + .expect("json"); + let (status, bytes) = post_bytes(&app, &paths::vm_jobs(&rec.handle.vm_id), body).await; + assert_eq!( + status, + StatusCode::OK, + "{}", + String::from_utf8_lossy(&bytes) + ); + let value: serde_json::Value = serde_json::from_slice(&bytes).expect("json"); + assert_eq!( + value["output"]["output"], "rules", + "a fragment keeps its own tag: {value}" + ); + } + /// A job or teardown that names another topic than the VM's never reaches /// the hypervisor — whether the mismatch is in the envelope or the job. #[tokio::test] diff --git a/crates/proof-vm-fc/src/lib.rs b/crates/proof-vm-fc/src/lib.rs index d85ac8927..7146f6664 100644 --- a/crates/proof-vm-fc/src/lib.rs +++ b/crates/proof-vm-fc/src/lib.rs @@ -403,8 +403,15 @@ impl FirecrackerOrchestrator { ))); } serde_json::from_slice::(&bytes).map(Some).map_err(|e| { + // The control plane and the orchestrator are separately built + // binaries. An answer carrying a variant this build does not know + // is a **build skew**, not a malformed answer — the shape of the + // live FAIL, where the guest emitted the authored set and the + // reader could not decode it. `proof_vm_proto` owns the wording so + // both host decoders say the same thing. backend(format!( - "orchestrator answer ({method} {path}) did not parse: {e}" + "orchestrator answer ({method} {path}) did not parse: {}", + proof_vm_proto::guest::decode_skew(&bytes, &e) )) }) } diff --git a/crates/proof-vm-guest/src/agent_tests.rs b/crates/proof-vm-guest/src/agent_tests.rs index 03b653404..549b44c05 100644 --- a/crates/proof-vm-guest/src/agent_tests.rs +++ b/crates/proof-vm-guest/src/agent_tests.rs @@ -1294,10 +1294,127 @@ JSON }; assert_eq!(proposed[0].id, "rules_only_rule"); assert_eq!(crate::runner::AUTHORING_FILE, "authoring.json"); + assert_eq!(crate::runner::RULES_FILE, "rules.json"); assert!(crate::runner::RULES_ONLY_IS_NOT_AUTHORSHIP.contains("authoring.json")); let _ = std::fs::remove_dir_all(&r); } +/// The dual-emit pair is one answer: `authoring.json` is authorship and +/// `rules.json` is its compat copy, so a pair whose vectors disagree is +/// refused rather than resolved by preference. +/// +/// This is the pair half of the live FAIL: the set's own harvest (a current +/// guest) and the compat harvest (a guest baked before the set existed) must +/// not disagree about the topic's anti-cheat surface — which file a stale +/// guest read would otherwise decide. +#[tokio::test] +async fn a_dual_emit_pair_whose_vectors_disagree_is_refused() { + let r = root("dual-emit"); + let a = agent(&r); + hello(&a).await; + let mut selecting = topic(); + selecting + .constraints + .params + .insert(proof_experiment::PARAM_RUNNER.into(), RUNNER.into()); + selecting + .constraints + .params + .insert(proof_experiment::PARAM_PACK_DIGEST.into(), pack().1); + install(&r, "run", "true"); + + let set = |rules: &str| { + format!( + r#" +cat > "$PROOF_OUTPUT_DIR/authoring.json" <<'JSON' +{{"schema_version":1,"topic_id":"topic-a","rules":{rules},"migrations":[{{"name":"0001_scratch","sql":"CREATE TABLE topic_a_scratch (id TEXT)"}}],"apis":[{{"path":"status","method":"GET"}}],"submission_format":{{"kind":"tar"}},"pin_policy":{{}}}} +JSON +"# + ) + }; + + // A pair that agrees: the compat copy is the set's own vector, so the run + // is authorship. This is what the reference adaptor writes. + install( + &r, + "propose_rules", + &format!( + "{}{}", + set(r#"[{"id":"rlm_rule","text":"the rlm wrote this"}]"#), + r#"echo '[{"id": "rlm_rule", "text": "the rlm wrote this"}]' > "$PROOF_OUTPUT_DIR/rules.json""# + ), + ); + let out = a + .handle(HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(selecting.clone()), + current_version: None, + current: None, + }), + }) + .await; + let RlmToHost::Done { + output: VmJobOutput::Authored(agreed), + } = out + else { + panic!("an agreeing pair is the whole set, got {out:?}"); + }; + assert_eq!(agreed.rules[0].id, "rlm_rule"); + + // A pair that disagrees on the vector is refused: which guest harvested + // the run would otherwise decide what the topic's rules are. + install( + &r, + "propose_rules", + &format!( + "{}{}", + set(r#"[{"id":"rlm_rule","text":"the rlm wrote this"}]"#), + r#"echo '[{"id": "a_different_rule", "text": "a second answer"}]' > "$PROOF_OUTPUT_DIR/rules.json""# + ), + ); + let err = failed( + a.handle(HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(selecting.clone()), + current_version: None, + current: None, + }), + }) + .await, + ); + assert!( + err.contains("their rule vectors disagree"), + "the refusal names the disagreement: {err}" + ); + assert!( + err.contains("rules.json is the compat copy"), + "the refusal says what the copy is for: {err}" + ); + + // A run that writes **neither** file authored nothing: there is no answer + // to fall back on, and the signed checklist is the operator's vector. + install(&r, "propose_rules", "true"); + let err = failed( + a.handle(HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(selecting), + current_version: None, + current: None, + }), + }) + .await, + ); + assert!( + err.contains("wrote neither authoring.json"), + "a run that wrote nothing is named as authoring nothing: {err}" + ); + assert!( + err.contains("never a substitute for RLM authorship"), + "the refusal says why nothing cannot be widened: {err}" + ); + let _ = std::fs::remove_dir_all(&r); +} + /// A leftover unsyncable entry under the shared work root (earlier failed /// job on a reused topic VM) must not turn Archive — or any later success /// that does not own that path — into Failed. @@ -1346,6 +1463,92 @@ async fn archive_and_later_jobs_ignore_stale_unsyncable_siblings() { } /// Framing over a stream, as the host drives it: hello, staging, a job, EOF. +/// The real agent's authored set, **framed**: what the guest writes over the +/// wire is what the host decodes, with every part and the `authored` tag. +/// +/// The in-process tests above assert `VmJobOutput::Authored` on the Rust type; +/// this one closes the loop the live FAIL fell through — the guest encodes, +/// the frame carries it, and a reader with only the wire types gets the whole +/// set back. A variant the host cannot decode would die here, before a VM. +#[tokio::test] +async fn the_real_agent_frames_the_whole_set_the_host_decodes() { + let r = root("framed-set"); + let a = agent(&r); + hello(&a).await; + let t = topic(); + let mut selecting = t.clone(); + selecting + .constraints + .params + .insert(proof_experiment::PARAM_RUNNER.into(), RUNNER.into()); + selecting + .constraints + .params + .insert(proof_experiment::PARAM_PACK_DIGEST.into(), pack().1); + install(&r, "run", "true"); + install( + &r, + "propose_rules", + r#" +cat > "$PROOF_OUTPUT_DIR/authoring.json" <<'JSON' +{ + "schema_version": 1, + "topic_id": "topic-a", + "rules": [{"id": "framed_rule", "text": "the rlm wrote this"}], + "migrations": [{"name": "0001_scratch", "sql": "CREATE TABLE topic_a_scratch (id TEXT)"}], + "apis": [{"path": "status", "method": "GET"}], + "submission_format": {"kind": "tar", "max_bytes": 5242880}, + "pin_policy": {} +} +JSON +echo '[{"id": "framed_rule", "text": "the rlm wrote this"}]' > "$PROOF_OUTPUT_DIR/rules.json" +"#, + ); + + // Drive the real agent over a real frame channel, exactly as the host does. + let (mut host, guest) = tokio::io::duplex(1 << 20); + let server = { + let a = a.clone(); + tokio::spawn(async move { a.serve_connection(guest).await }) + }; + write_frame( + &mut host, + &HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(selecting), + current_version: None, + current: None, + }), + }, + ) + .await + .expect("job"); + let done: RlmToHost = read_frame(&mut host).await.expect("done"); + let RlmToHost::Done { + output: VmJobOutput::Authored(set), + } = done + else { + panic!("the frame did not carry the set: {done:?}"); + }; + assert!( + set.is_complete(), + "the framed set lost parts: {:?}", + set.missing_parts() + ); + assert_eq!(set.rules[0].id, "framed_rule"); + assert_eq!(set.migrations.len(), 1); + assert_eq!(set.apis[0].path, "status"); + // The raw frame carries the tag two separately built binaries agree on. + let raw = serde_json::to_string(&RlmToHost::Done { + output: VmJobOutput::Authored(set), + }) + .expect("json"); + assert!(raw.contains(r#""output":"authored""#), "{raw}"); + drop(host); + let _ = server.await; + let _ = std::fs::remove_dir_all(&r); +} + #[tokio::test] async fn serve_connection_speaks_frames_until_the_host_hangs_up() { let r = root("frames"); @@ -1715,7 +1918,127 @@ sleep 30 ); assert!( err.contains("sync"), - "timeout must persist work before Failed, got {err}" + "timeout must persist work before Fail, got {err}" + ); + let _ = std::fs::remove_dir_all(&r); +} + +/// The reference adaptor's own `propose_rules`, through the real guest agent: +/// discovery finds it under the runner id the topic names, and the whole set +/// it writes comes back as `Authored` — the answer the install applies. +/// +/// This is the discovery half of the authorship pin. `runner::propose_rules` +/// resolves `//propose_rules`; a tree without it is +/// `NO_RLM_RULES`, which is the state the live guest image was in (every +/// `--drive-rlm` refused, so no topic could open and no `authorship: rlm` row +/// could ever be written). The test runs the shipped adaptor rather than a +/// stand-in, so a rename or a lost execute bit fails here. +#[tokio::test] +async fn the_reference_adaptor_propose_rules_is_discovered_and_authors_the_whole_set() { + let Some(python3) = which("python3") else { + eprintln!("python3 not on PATH: skipping the reference-adaptor discovery test"); + return; + }; + let r = root("reference-authoring"); + let a = agent(&r); + hello(&a).await; + + // The shipped adaptor tree, copied to this guest's runners dir under the + // id the topic selects. `run` must exist for the adaptor to be installed + // at all (a `propose_rules`-only directory is not a runner). + let src = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../deploy/guest/runners/rlm_fc_in_guest_harbor"); + let dir = r.join("runners").join(RUNNER); + std::fs::create_dir_all(&dir).expect("adaptor dir"); + let status = std::process::Command::new("cp") + .args([ + "-a", + &format!("{}/.", src.display()), + &dir.display().to_string(), + ]) + .status() + .expect("cp adaptor"); + assert!(status.success(), "copy the reference adaptor"); + for entry in ["run", "inspect", "propose_rules"] { + let path = dir.join(entry); + assert!( + path.is_file(), + "the reference adaptor ships no {entry} ({})", + path.display() + ); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + } + + let mut t = topic(); + t.constraints + .params + .insert(proof_experiment::PARAM_RUNNER.into(), RUNNER.into()); + t.constraints.params.insert( + proof_experiment::PARAM_PACK_DIGEST.into(), + format!("sha256:{}", "ab".repeat(32)), + ); + // The signed inspect policy: what makes each declared rule tickable. The + // adaptor authors the vector its inspector ticks, so a rule the topic does + // not say how to tick is a refusal rather than an invented check. + t.constraints.params.insert( + "inspect_marker_rules".into(), + "rule_a:off-limits-marker-a;rule_b:off-limits-marker-b".into(), + ); + t.constraints + .params + .insert("inspect_attested_rules".into(), "rule_c".into()); + + // The guest execs the entrypoint directly with its own environment, so + // the interpreter must be on the path it sets. Point the entrypoint at + // the interpreter we resolved when it is not where the guest looks. + if python3 != Path::new("/usr/bin/python3") { + let entry = dir.join("propose_rules"); + let body = std::fs::read_to_string(&entry).expect("entrypoint"); + std::fs::write( + &entry, + body.replace("python3 ", &format!("{} ", python3.display())), + ) + .expect("rewrite interpreter"); + } + + let out = a + .handle(HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(t), + current_version: None, + current: None, + }), + }) + .await; + let RlmToHost::Done { + output: VmJobOutput::Authored(set), + } = out + else { + panic!("the reference adaptor must answer with the whole set, got {out:?}"); + }; + assert_eq!(set.topic_id, "topic-a"); + assert!(set.is_complete(), "missing {:?}", set.missing_parts()); + assert_eq!( + set.rules.iter().map(|r| r.id.as_str()).collect::>(), + vec!["rule_a", "rule_b", "rule_c"] + ); + assert!( + set.rules.iter().all(|r| r.text.contains("rlm:")), + "the vector is the RLM's framing, not the document's sentences: {:?}", + set.rules.iter().map(|r| &r.text).collect::>() ); let _ = std::fs::remove_dir_all(&r); } + +/// `command -v` without assuming where the interpreter lives. +fn which(bin: &str) -> Option { + let out = std::process::Command::new("sh") + .args(["-c", &format!("command -v {bin}")]) + .output() + .ok()?; + if !out.status.success() { + return None; + } + let path = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + (!path.is_empty()).then(|| PathBuf::from(path)) +} diff --git a/crates/proof-vm-guest/src/lib.rs b/crates/proof-vm-guest/src/lib.rs index babd907dd..2d3c54a40 100644 --- a/crates/proof-vm-guest/src/lib.rs +++ b/crates/proof-vm-guest/src/lib.rs @@ -19,14 +19,14 @@ //! resolves `//` and execs the entrypoint for the job //! kind — `run` (baseline / evaluate), `inspect`, `propose_rules` — with the //! environment contract in [`runner`]. The adaptor writes its answer under -//! `PROOF_OUTPUT_DIR` (`report.json`, `checklist.json`, `rules.json`); the -//! agent turns it into the protocol document with the identities copied -//! from the request. **Nothing is defaulted:** no runner selected, no adaptor -//! installed under that id, no staged pack matching the topic's digest, no -//! artefact that verifies, no report, a non-finite value, or a run cut at the -//! deadline is `RlmToHost::Failed` — the host answers 502, the control plane -//! 503, and no row is written. A placeholder `primary_value` never leaves -//! this process. +//! `PROOF_OUTPUT_DIR` (`report.json`, `checklist.json`, `authoring.json`, +//! `rules.json`); the agent turns it into the protocol document with the +//! identities copied from the request. **Nothing is defaulted:** no runner +//! selected, no adaptor installed under that id, no staged pack matching the +//! topic's digest, no artefact that verifies, no report, a non-finite value, +//! or a run cut at the deadline is `RlmToHost::Failed` — the host answers +//! 502, the control plane 503, and no row is written. A placeholder +//! `primary_value` never leaves this process. //! //! **No default exists for authorship.** `ProposeRules` is answered only by //! the adaptor's own `propose_rules` entrypoint: a runner that ships none is @@ -37,6 +37,15 @@ //! the parts it would fill in are the operator's. What a topic *is* (rules, //! migrations, APIs, submission format, pin policy) comes from its own RLM. //! +//! **One answer, two files.** `authoring.json` is the authorship; the +//! `rules.json` an adaptor writes beside it is the **compat copy** that a +//! guest baked before the set existed harvests the run out of, so a +//! `propose_rules` run that must answer both guests writes both. They are +//! held to being the same vector ([`runner::DUAL_EMIT_RULES_DISAGREE`]), and +//! a run that writes neither file is refused as authoring nothing +//! ([`runner::NO_AUTHORING_OR_RULES`]) rather than widened from the signed +//! document. +//! //! Secrets are files the adaptor reads (`PROOF_SECRETS_DIR`); their bytes //! are redacted from every log tail and evidence document the agent sends //! back, and never appear in an environment variable the agent sets. diff --git a/crates/proof-vm-guest/src/runner.rs b/crates/proof-vm-guest/src/runner.rs index da88aac76..bbcec44be 100644 --- a/crates/proof-vm-guest/src/runner.rs +++ b/crates/proof-vm-guest/src/runner.rs @@ -9,7 +9,7 @@ //! |------------|-----|---------------------------------| //! | `run` | `Baseline`, `Evaluate` | `report.json` — [`RunnerReport`]; **Evaluate** also `results.json` | //! | `inspect` | `Inspect` | `checklist.json` — `[{"id", "pass", "evidence"}]` | -//! | `propose_rules` | `ProposeRules` (optional) | `rules.json` — `[{"id", "text"}]` | +//! | `propose_rules` | `ProposeRules` (optional) | `authoring.json` — the whole set ([`AuthoredSet::Complete`]); `rules.json` — `[{"id", "text"}]`, the same vector, read by a guest baked before the set existed | //! //! Every entrypoint receives the same environment contract //! ([`env::*`](env)): the job kind, the identities (topic, custom id, @@ -117,12 +117,24 @@ pub mod env { pub const MAX_TAIL_BYTES: usize = 64 * 1024; /// Rolling tail kept per stream while draining (half of [`MAX_TAIL_BYTES`]). pub const STREAM_TAIL_BYTES: usize = MAX_TAIL_BYTES / 2; -/// Largest `report.json` / `checklist.json` / `rules.json` read back. +/// Largest `report.json` / `checklist.json` / `authoring.json` / `rules.json` +/// read back. pub const MAX_OUTPUT_DOC_BYTES: u64 = 8 * 1024 * 1024; /// The file the RLM writes its whole authored set to (`ProposeRules`). pub const AUTHORING_FILE: &str = "authoring.json"; +/// The file a `propose_rules` adaptor writes a bare rule vector to — the +/// **compat fragment**, also written beside `authoring.json` so a guest baked +/// before the set existed answers instead of failing the job +/// (`adaptor wrote no rules.json`). It is not authorship: the host records +/// what it carries with honest `rlm` provenance and refuses to open a topic +/// on it ([`RULES_ONLY_IS_NOT_AUTHORSHIP`]). +/// +/// Not to be confused with [`env::RULES_FILE`], the variable naming the +/// **input** vector an `Inspect` job ticks. +pub const RULES_FILE: &str = "rules.json"; + /// The file the guest writes the **previous** set to, for a re-authoring run. pub const CURRENT_AUTHORING_FILE: &str = "current-authoring.json"; /// Deadline for jobs that carry none (`ProposeRules`). @@ -999,6 +1011,23 @@ pub const NO_RLM_RULES: &str = "the topic's runner ships no propose_rules entryp /// operator-cloned document the authorship pin exists to refuse. pub const RULES_ONLY_IS_NOT_AUTHORSHIP: &str = "the runner wrote rules.json but no authoring.json: a topic's behavior is authored by its own RLM (rules, migrations, apis, submission_format, pin_policy), and a rules-only proposal is not that. Ship an adaptor whose propose_rules writes authoring.json; nothing is installed from the operator's bundle in its place"; +/// Why a `propose_rules` run that wrote neither file fails closed. +/// +/// The guest has no answer to fall back on: the signed `checklist` is the +/// operator's vector, and echoing it back would record the operator's own +/// rules as [`proof_rlm::RuleSource::Rlm`]. A run that wrote nothing authored +/// nothing. +pub const NO_AUTHORING_OR_RULES: &str = "the runner wrote neither authoring.json (the whole set: rules, migrations, apis, submission_format, pin_policy) nor rules.json (the compat fragment): a propose_rules run that writes nothing authors nothing, and the signed checklist is the operator's vector, never a substitute for RLM authorship"; + +/// Why a dual-written pair whose vectors disagree fails closed. +/// +/// `rules.json` exists so a guest baked before `authoring.json` existed still +/// answers; it is the **same** vector as the set's `rules`, or it is a second +/// answer. A pair that disagrees is refused rather than resolved by preferring +/// one: which file a stale guest read would then decide the topic's anti-cheat +/// surface, and the two halves of one authorship claim could disagree forever. +pub const DUAL_EMIT_RULES_DISAGREE: &str = "the runner wrote authoring.json and rules.json but their rule vectors disagree: rules.json is the compat copy of the set's own rules, not a second answer. A guest baked before the set existed reads rules.json, so a pair that disagrees would make the topic's anti-cheat surface depend on which guest harvested it"; + /// What the RLM authored, as the guest read it. #[derive(Debug, Clone, PartialEq)] pub enum AuthoredSet { @@ -1028,7 +1057,9 @@ pub enum AuthoredSet { /// Two output shapes are read, and the difference matters: /// /// - `authoring.json` — the whole set ([`AuthoredSet::Complete`]). This is -/// what a topic that must **open** needs. +/// what a topic that must **open** needs, and it is read whichever guest +/// harvests the run: a guest baked before the set existed reads the same +/// answer out of `rules.json` (below) rather than failing the job. /// - `rules.json` — a bare vector ([`AuthoredSet::RulesOnly`]), kept because /// an adaptor baked before the set existed still writes it. The host /// records the rules with honest `rlm` provenance and refuses to open the @@ -1036,6 +1067,11 @@ pub enum AuthoredSet { /// ([`RULES_ONLY_IS_NOT_AUTHORSHIP`]) — it does not widen a rules-only /// answer into a whole set, because the parts it would fill in would be the /// operator's. +/// +/// A dual-written pair is the **same** answer twice, so the compat copy is +/// held to the set's own rules: a pair whose vectors disagree is refused +/// ([`DUAL_EMIT_RULES_DISAGREE`]) rather than resolved by preference, because +/// which guest harvested the run would otherwise decide the topic's vector. pub async fn propose_rules( cfg: &GuestConfig, topic: &TopicDocument, @@ -1103,17 +1139,40 @@ pub async fn propose_rules( if exec.timed_out { return Err("propose_rules cut at its deadline".into()); } - // The whole set is what a topic's behavior is. Read it first: an adaptor - // that writes it is authoring, and one that writes only rules is - // proposing a fragment. + let ctx = describe(&exec, &secrets, DEFAULT_UNPAID_DEADLINE.as_secs()); + read_authored_set(topic, &output, &secrets, &ctx) +} + +/// Read what a `propose_rules` run wrote, or refuse. +/// +/// The two files are one answer, and the precedence between them is the whole +/// point: +/// +/// - `authoring.json` present → [`AuthoredSet::Complete`], held to the shape, +/// the deny-list, and the policy-vs-document checks the control plane runs. +/// A `rules.json` beside it must carry **the same** vector +/// ([`DUAL_EMIT_RULES_DISAGREE`]), because a guest baked before the set +/// existed harvests this run out of that file. +/// - only `rules.json` → [`AuthoredSet::RulesOnly`], the compat fragment an +/// older adaptor answers with. The host records it with honest `rlm` +/// provenance and refuses to open a topic on it. +/// - neither → [`NO_AUTHORING_OR_RULES`]. A run that wrote nothing authored +/// nothing, and the signed `checklist` is the operator's vector. +/// +/// `ctx` is the failure context of the run that produced the files (its exit +/// status and redacted tail), appended to a read failure so the operator sees +/// why the adaptor wrote nothing. +fn read_authored_set( + topic: &TopicDocument, + output: &Path, + secrets: &[Vec], + ctx: &str, +) -> Result { let authoring_path = output.join(AUTHORING_FILE); + let rules_path = output.join(RULES_FILE); if authoring_path.is_file() { - let body = read_output_text(&authoring_path, AUTHORING_FILE).map_err(|e| { - format!( - "{e} ({})", - describe(&exec, &secrets, DEFAULT_UNPAID_DEADLINE.as_secs()) - ) - })?; + let body = read_output_text(&authoring_path, AUTHORING_FILE) + .map_err(|e| format!("{e} ({ctx})"))?; let mut set = proof_rlm::authoring_from_json(&body).map_err(|e| format!("{AUTHORING_FILE}: {e}"))?; if set.topic_id.trim() != topic.id.trim() { @@ -1130,22 +1189,37 @@ pub async fn propose_rules( set.pin_policy .agrees_with_document(topic) .map_err(|e| format!("{AUTHORING_FILE}: {e}"))?; + if rules_path.is_file() { + let compat: Vec = + read_output_doc(&rules_path, RULES_FILE).map_err(|e| format!("{e} ({ctx})"))?; + // Compared before redaction: the two files carry the same text, so + // a secret in one is a secret in the other, and redacting first + // would compare two redactions rather than the RLM's answer. + if compat != set.rules { + return Err(format!( + "{DUAL_EMIT_RULES_DISAGREE} ({AUTHORING_FILE} carries {} rules, \ + {RULES_FILE} carries {})", + set.rules.len(), + compat.len() + )); + } + } for rule in &mut set.rules { - rule.text = redact(&rule.text, &secrets); + rule.text = redact(&rule.text, secrets); } return Ok(AuthoredSet::Complete(Box::new(set))); } - let mut rules: Vec = read_output_doc(&output.join("rules.json"), "rules.json") - .map_err(|e| { - format!( - "{e} ({})", - describe(&exec, &secrets, DEFAULT_UNPAID_DEADLINE.as_secs()) - ) - })?; + if !rules_path.is_file() { + // Neither file: the run authored nothing. The signed checklist is the + // operator's vector, so there is no answer to fall back on. + return Err(format!("{NO_AUTHORING_OR_RULES} ({ctx})")); + } + let mut rules: Vec = + read_output_doc(&rules_path, RULES_FILE).map_err(|e| format!("{e} ({ctx})"))?; for r in &mut rules { - r.text = redact(&r.text, &secrets); + r.text = redact(&r.text, secrets); } - validate_rules(&rules).map_err(|e| format!("rules.json {}: {}", e.field, e.why))?; + validate_rules(&rules).map_err(|e| format!("{RULES_FILE} {}: {}", e.field, e.why))?; Ok(AuthoredSet::RulesOnly(rules)) } @@ -1185,3 +1259,187 @@ mod sync_tests { let _ = std::fs::remove_dir_all(&d); } } + +/// The harvest reader itself, without a VM: which file decides, and what a +/// pair of files means. +#[cfg(test)] +mod authored_set_tests { + #![allow(clippy::expect_used, clippy::unwrap_used)] + + use super::{ + read_authored_set, AuthoredSet, AUTHORING_FILE, DUAL_EMIT_RULES_DISAGREE, + NO_AUTHORING_OR_RULES, RULES_FILE, + }; + use proof_task::TopicDocument; + use std::path::Path; + + fn topic() -> TopicDocument { + let mut doc = TopicDocument { + id: "topic-a".into(), + ..TopicDocument::default() + }; + doc.metric.family = proof_task::MetricFamily::Custom; + doc.metric.custom_id = "custom_a".into(); + doc + } + + fn dir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!( + "proof-vm-guest-authored-{}-{tag}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).expect("dir"); + d + } + + fn set_body(rules: &str) -> String { + format!( + r#"{{"schema_version":1,"topic_id":"topic-a","rules":{rules},"migrations":[{{"name":"0001_scratch","sql":"CREATE TABLE topic_a_scratch (id TEXT)"}}],"apis":[{{"path":"status","method":"GET"}}],"submission_format":{{"kind":"tar"}},"pin_policy":{{}}}}"# + ) + } + + /// `authoring.json` alone is the whole set, and it is what the harvest + /// reads first — the live FAIL was a guest that read only `rules.json`. + #[test] + fn the_set_is_harvested_from_authoring_json() { + let d = dir("set-only"); + std::fs::write( + d.join(AUTHORING_FILE), + set_body(r#"[{"id":"rlm_rule","text":"t"}]"#), + ) + .expect("write set"); + let out = read_authored_set(&topic(), &d, &[], "ctx").expect("the set is read"); + let AuthoredSet::Complete(set) = out else { + panic!("a set is authorship, got {out:?}"); + }; + assert_eq!(set.rules[0].id, "rlm_rule"); + assert!(set.is_complete()); + let _ = std::fs::remove_dir_all(&d); + } + + /// A dual-written pair whose vectors agree is the same answer twice, so + /// the harvest answers the set (and the compat copy is never the answer). + #[test] + fn a_dual_emit_pair_that_agrees_harvests_the_set() { + let d = dir("pair-agrees"); + let rules = r#"[{"id":"rlm_rule","text":"t"}]"#; + std::fs::write(d.join(AUTHORING_FILE), set_body(rules)).expect("write set"); + std::fs::write(d.join(RULES_FILE), rules).expect("write fragment"); + let out = read_authored_set(&topic(), &d, &[], "ctx").expect("the pair is read"); + assert!( + matches!(out, AuthoredSet::Complete(_)), + "an agreeing pair is the whole set, got {out:?}" + ); + let _ = std::fs::remove_dir_all(&d); + } + + /// A pair that disagrees is refused by name: which guest harvested the run + /// must not decide the topic's vector. + #[test] + fn a_dual_emit_pair_that_disagrees_is_refused() { + let d = dir("pair-disagrees"); + std::fs::write( + d.join(AUTHORING_FILE), + set_body(r#"[{"id":"rlm_rule","text":"t"}]"#), + ) + .expect("write set"); + std::fs::write( + d.join(RULES_FILE), + r#"[{"id":"a_different_rule","text":"u"}]"#, + ) + .expect("write fragment"); + let err = + read_authored_set(&topic(), &d, &[], "ctx").expect_err("a disagreement is refused"); + assert!(err.contains(DUAL_EMIT_RULES_DISAGREE), "{err}"); + assert!(err.contains("carries 1 rules"), "{err}"); + let _ = std::fs::remove_dir_all(&d); + } + + /// Neither file: nothing was authored, and there is no fallback. + #[test] + fn a_run_that_wrote_neither_file_is_refused() { + let d = dir("neither"); + let err = read_authored_set(&topic(), &d, &[], "the run exited 0").expect_err("nothing"); + assert!(err.contains(NO_AUTHORING_OR_RULES), "{err}"); + assert!( + err.contains("the run exited 0"), + "the refusal carries the run's own context: {err}" + ); + let _ = std::fs::remove_dir_all(&d); + } + + /// `rules.json` alone stays a fragment: the guest never widens it. + #[test] + fn a_rules_only_run_stays_a_fragment() { + let d = dir("rules-only"); + std::fs::write(d.join(RULES_FILE), r#"[{"id":"compat_rule","text":"t"}]"#).expect("write"); + let out = read_authored_set(&topic(), &d, &[], "ctx").expect("a fragment is read"); + let AuthoredSet::RulesOnly(rules) = out else { + panic!("a fragment stays a fragment, got {out:?}"); + }; + assert_eq!(rules[0].id, "compat_rule"); + let _ = std::fs::remove_dir_all(&d); + } + + /// A set for another topic is refused however it is paired. + #[test] + fn a_set_for_another_topic_is_refused() { + let d = dir("wrong-topic"); + std::fs::write( + d.join(AUTHORING_FILE), + set_body(r#"[{"id":"rlm_rule","text":"t"}]"#).replace("topic-a", "topic-b"), + ) + .expect("write set"); + let err = read_authored_set(&topic(), &d, &[], "ctx").expect_err("wrong topic"); + assert!(err.contains("is for topic"), "{err}"); + let _ = std::fs::remove_dir_all(&d); + } + + /// The compat file's own shape is still checked when it is the answer: a + /// malformed fragment is refused with the file named, not silently empty. + #[test] + fn a_malformed_fragment_is_refused_by_name() { + let d = dir("bad-fragment"); + std::fs::write(d.join(RULES_FILE), r#"[{"id":"Bad Id","text":"t"}]"#).expect("write"); + let err = read_authored_set(&topic(), &d, &[], "ctx").expect_err("bad id"); + assert!(err.contains(RULES_FILE), "{err}"); + assert!(err.contains("Bad Id"), "{err}"); + let _ = std::fs::remove_dir_all(&d); + } + + /// A redacted secret never reaches the set's rules. + #[test] + fn a_staged_secret_is_redacted_out_of_the_harvested_rules() { + let d = dir("redacted"); + std::fs::write( + d.join(AUTHORING_FILE), + set_body(r#"[{"id":"rlm_rule","text":"uses s3cret-value here"}]"#), + ) + .expect("write set"); + let out = + read_authored_set(&topic(), &d, &[b"s3cret-value".to_vec()], "ctx").expect("the set"); + let AuthoredSet::Complete(set) = out else { + panic!("a set, got {out:?}"); + }; + assert!( + !set.rules[0].text.contains("s3cret-value"), + "a staged secret travels in a rule text: {}", + set.rules[0].text + ); + let _ = std::fs::remove_dir_all(&d); + } + + /// The reader takes a directory, and an absent one is "nothing written". + #[test] + fn a_missing_output_directory_reads_as_nothing_written() { + let err = read_authored_set( + &topic(), + Path::new("/no/such-proof-vm-guest-output-dir"), + &[], + "ctx", + ) + .expect_err("nothing written"); + assert!(err.contains(NO_AUTHORING_OR_RULES), "{err}"); + } +} diff --git a/crates/proof-vm-guest/tests/bake_tooling.rs b/crates/proof-vm-guest/tests/bake_tooling.rs index 39817473a..db7633531 100644 --- a/crates/proof-vm-guest/tests/bake_tooling.rs +++ b/crates/proof-vm-guest/tests/bake_tooling.rs @@ -57,6 +57,10 @@ fn guest_scripts_parse() { "bash", "deploy/guest/runners/rlm_fc_in_guest_harbor/inspect", ), + ( + "bash", + "deploy/guest/runners/rlm_fc_in_guest_harbor/propose_rules", + ), ("bash", "deploy/guest/runners/rlm_fc_in_guest_harbor/lib.sh"), ( "bash", @@ -389,8 +393,14 @@ fn deploy_guest_names_no_harness_or_benchmark() { for required in [ "run", "inspect", + // Authorship: without this entrypoint the guest refuses every + // `ProposeRules` job (NO_RLM_RULES), so no topic on this runner can + // ever open — the RLM authored nothing and the bundle is not a + // substitute for its answer. + "propose_rules", "README.md", "harness/run-harbor", + "harness/authoring_set.py", "harness/summarize.py", "harness/filter_tasks.py", "harness/resolve_model.py", @@ -404,10 +414,12 @@ fn deploy_guest_names_no_harness_or_benchmark() { let p = adaptor.join(required); assert!(p.is_file(), "reference adaptor missing {}", p.display()); } - assert!( - adaptor.join("run").metadata().unwrap().permissions().mode() & 0o111 != 0, - "reference adaptor run must be executable" - ); + for entry in ["run", "inspect", "propose_rules"] { + assert!( + adaptor.join(entry).metadata().unwrap().permissions().mode() & 0o111 != 0, + "reference adaptor {entry} must be executable (the guest execs it directly)" + ); + } // The adaptor's selection / inspection code carries no task list, slice // name, filter mode, or rule id of any topic: those are topic data // (signed params + the pinned pack). Test fixtures are exempt. @@ -419,6 +431,8 @@ fn deploy_guest_names_no_harness_or_benchmark() { "harness/filter_tasks.py", "harness/summarize.py", "harness/run-harbor", + "harness/authoring_set.py", + "propose_rules", "lib.sh", "inspect_scan.py", ] { diff --git a/crates/proof-vm-guest/tests/reference_adaptor_authoring.rs b/crates/proof-vm-guest/tests/reference_adaptor_authoring.rs new file mode 100644 index 000000000..dcd488b7c --- /dev/null +++ b/crates/proof-vm-guest/tests/reference_adaptor_authoring.rs @@ -0,0 +1,436 @@ +//! The authored set this repository's reference adaptor writes must pass the +//! gates the guest and the install hold it to — in Rust, not in Python. +//! +//! The adaptor's `propose_rules` is operator content and its own unit tests +//! are Python; the **authoritative** checks are `proof-topic-authoring` (which +//! the guest links) and `proof-topic-sql-guard` (which the install runs). A +//! Python test can only prove the module agrees with itself. This one runs the +//! real thing: +//! +//! 1. write a fixture topic the way a signed document carries one, +//! 2. run the adaptor's entrypoint exactly as the guest does (same env +//! contract, `PROOF_JOB=propose_rules`), +//! 3. parse the `authoring.json` it wrote with `authoring_from_json`, +//! 4. hold it to `TopicAuthoring::validate` (shape, completeness, the +//! migration deny-list) and `PinPolicy::agrees_with_document`, and +//! 5. hold it to `validate_against_pin` with the real pin file. +//! +//! A change to the adaptor that would produce a set the guest refuses fails +//! here, and so does a change to the gates that would start refusing the set +//! the adaptor ships. +//! +//! `python3` is required (the adaptor is a Python entrypoint); the test skips +//! with a message rather than passing vacuously if it is absent. + +#![allow(clippy::expect_used, clippy::unwrap_used)] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use proof_rlm::{authoring_from_json, TopicAuthoring}; +use proof_task::{ChecklistRule, ProofPin, TopicDocument}; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn adaptor() -> PathBuf { + repo().join("deploy/guest/runners/rlm_fc_in_guest_harbor") +} + +fn tmp(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("proof-rlm-authoring-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).expect("dir"); + d +} + +fn python3() -> Option { + let out = Command::new("sh") + .args(["-c", "command -v python3"]) + .output() + .ok()?; + if !out.status.success() { + return None; + } + let path = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + (!path.is_empty()).then(|| PathBuf::from(path)) +} + +/// The topic the adaptor authors against: a custom-family document with a +/// checklist and the signed inspect policy each rule needs. +fn topic() -> TopicDocument { + let mut doc = TopicDocument { + id: "fixture-topic-v0".into(), + statement: "score the pinned pack with the pinned runner".into(), + epsilon_nll: 0.02, + epsilon_topic_max_regress: 0.05, + holdout_size: 120, + ..TopicDocument::default() + }; + doc.metric.family = proof_task::MetricFamily::Custom; + doc.metric.custom_id = "fixture_metric".into(); + doc.metric.primary = "success_rate".into(); + doc.metric.direction = proof_task::MetricDirection::Max; + doc.metric.epsilon_rel = 0.05; + doc.eval_executor.max_proof_deadline_s = Some(3600); + doc.checklist = vec![ + ChecklistRule { + id: "no_short_circuit".into(), + text: "the evaluator and the metric path are untouched".into(), + }, + ChecklistRule { + id: "miner_pays_provider".into(), + text: "the miner pays for its own provider calls".into(), + }, + ]; + doc.constraints + .params + .insert("baseline_runner".into(), "rlm_fc_in_guest_harbor".into()); + doc.constraints.params.insert( + "experiment_pack_digest".into(), + format!("sha256:{}", "ab".repeat(32)), + ); + doc.constraints + .params + .insert("tasks_dir".into(), "tasks".into()); + doc +} + +/// Run the adaptor's `propose_rules` exactly as the guest does. +fn author(doc: &TopicDocument, root: &Path, current: Option<&str>) -> Result { + let topic_file = root.join("topic.json"); + std::fs::write( + &topic_file, + serde_json::to_vec_pretty(doc).expect("topic json"), + ) + .expect("write topic"); + let output = root.join("output"); + let work = root.join("work"); + std::fs::create_dir_all(&output).expect("output dir"); + std::fs::create_dir_all(&work).expect("work dir"); + let current_file = match current { + Some(body) => { + let path = work.join("current-authoring.json"); + std::fs::write(&path, body).expect("write current set"); + path.display().to_string() + } + None => String::new(), + }; + let out = Command::new(adaptor().join("propose_rules")) + .current_dir(root) + .env_clear() + .env("PATH", "/usr/local/bin:/usr/bin:/bin") + .env("LANG", "C.UTF-8") + .env("PROOF_JOB", "propose_rules") + .env("PROOF_TOPIC_ID", &doc.id) + .env("PROOF_CUSTOM_ID", &doc.metric.custom_id) + .env("PROOF_TOPIC_FILE", &topic_file) + .env("PROOF_OUTPUT_DIR", &output) + .env("PROOF_WORK_DIR", &work) + .env("PROOF_CURRENT_AUTHORING_FILE", current_file) + .env( + "PROOF_PARAM_INSPECT_MARKER_RULES", + "no_short_circuit:skip_eval|skip_verifier", + ) + .env("PROOF_PARAM_INSPECT_ATTESTED_RULES", "miner_pays_provider") + .output() + .map_err(|e| format!("spawn propose_rules: {e}"))?; + if !out.status.success() { + return Err(format!( + "propose_rules exited {:?}: {}{}", + out.status.code(), + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + )); + } + let path = output.join("authoring.json"); + if path.is_file() { + // The compat copy travels with the set, and it is the **same** vector: + // a guest baked before the set existed reads `rules.json`, and one + // that reads the set refuses a pair that disagrees. Either way the two + // files are one answer, so the gate compares them here too. + let compat = output.join("rules.json"); + if compat.is_file() { + let set: TopicAuthoring = + authoring_from_json(&std::fs::read_to_string(&path).expect("set")).expect("parses"); + let fragment: Vec = + serde_json::from_str(&std::fs::read_to_string(&compat).expect("fragment")) + .expect("the compat fragment is a rule vector"); + assert_eq!( + fragment, set.rules, + "rules.json is not the set's own vector: a guest baked before the set existed \ + would harvest a different anti-cheat surface than the set's" + ); + } else { + panic!( + "the adaptor wrote no rules.json: a guest baked before the set existed fails \ + this run (`adaptor wrote no rules.json`), which is the live FAIL this pairs with" + ); + } + } + std::fs::read_to_string(&path).map_err(|e| format!("read authoring.json: {e}")) +} + +/// The set the reference adaptor writes is the set the guest accepts. +#[test] +fn the_reference_adaptor_authors_a_set_the_guest_and_the_install_accept() { + if python3().is_none() { + eprintln!("python3 not on PATH: skipping the adaptor authoring gate"); + return; + } + let doc = topic(); + let root = tmp("complete"); + let body = author(&doc, &root, None).expect("the adaptor authors a set"); + + // 1. It parses as the shape the guest reads (`deny_unknown_fields`). + let set: TopicAuthoring = authoring_from_json(&body).expect("parses as a TopicAuthoring"); + + // 2. Every part the install applies is present. + assert!( + set.is_complete(), + "the set is missing {:?}", + set.missing_parts() + ); + assert_eq!(set.topic_id, doc.id); + assert_eq!(set.schema_version, proof_rlm::AUTHORING_SCHEMA); + assert_eq!( + set.rules.iter().map(|r| r.id.as_str()).collect::>(), + vec!["no_short_circuit", "miner_pays_provider"], + "the vector is the declared rules, in declaration order" + ); + assert!( + !set.migrations.is_empty() && !set.apis.is_empty(), + "a complete set carries a migration and a route" + ); + + // 3. The guest's own gate: shape, completeness, and the migration + // deny-list, run before the answer becomes a job output. + set.validate(&doc.id).expect("the guest accepts the set"); + + // 4. The policy restates the signed document — scoring reads the document, + // so a divergence either way is a threshold nobody is judged by. + set.pin_policy + .agrees_with_document(&doc) + .expect("the policy restates the document"); + + // 5. And the control plane's gate: the policy against the **pin**. + let pin = ProofPin::from_toml( + &std::fs::read_to_string(repo().join("config/proof-pin.toml")).expect("pin file"), + ) + .expect("pin parses"); + pin.validate().expect("the shipped pin validates"); + set.validate_against_pin(&doc.id, &pin) + .expect("the control plane accepts the set against the pin"); + let _ = std::fs::remove_dir_all(&root); +} + +/// The authored set is the RLM's answer, not the operator's bundle. +#[test] +fn the_authored_set_is_not_a_copy_of_the_signed_document() { + if python3().is_none() { + eprintln!("python3 not on PATH: skipping the adaptor authoring gate"); + return; + } + let doc = topic(); + let root = tmp("not-a-copy"); + let body = author(&doc, &root, None).expect("the adaptor authors a set"); + let set: TopicAuthoring = authoring_from_json(&body).expect("parses"); + + // The rule text is the RLM's framing, not the operator's sentence alone. + for rule in &set.rules { + let declared = doc + .checklist + .iter() + .find(|d| d.id == rule.id) + .expect("the rule is one the topic declares"); + assert_ne!( + rule.text.trim(), + declared.text.trim(), + "rule {} is the operator's sentence verbatim: that is a restatement, and the \ + provenance it would carry is the operator-cloned document the gate refuses", + rule.id + ); + assert!( + rule.text.contains("rlm:"), + "rule {} does not say how this RLM enforces it: {}", + rule.id, + rule.text + ); + } + + // The migration is inside the topic's own namespace, and touches nothing + // the repository owns. + let prefix = doc.id.replace('-', "_"); + for migration in &set.migrations { + assert!( + migration.sql.contains(&prefix), + "migration {} is not inside the topic's namespace: {}", + migration.name, + migration.sql + ); + assert!( + !migration.sql.to_lowercase().contains("proof_"), + "migration {} names an object the repository owns", + migration.name + ); + } + + // The submission format describes this runtime's intake, not a bundle + // section: the staged cap and the submit domain are facts about the host. + let fmt = set + .submission_format + .as_object() + .expect("format is an object"); + assert!( + fmt.contains_key("signature_domain"), + "the format does not name the signature domain the intake checks" + ); + assert_eq!( + set.pin_policy.eval_image_digest, None, + "the policy invented an eval image digest: the VM does not hold the pin" + ); + assert_eq!( + set.pin_policy.gpu_class, None, + "the policy invented a gpu class: the VM does not hold the pin" + ); + let _ = std::fs::remove_dir_all(&root); +} + +/// Re-authoring retains what it is not changing, and the retained set is +/// still held to the same gates (retention is not a bypass). +#[test] +fn a_re_authoring_run_retains_the_prior_set_and_is_still_validated() { + if python3().is_none() { + eprintln!("python3 not on PATH: skipping the adaptor authoring gate"); + return; + } + let doc = topic(); + let root = tmp("re-author"); + let first = author(&doc, &root, None).expect("first authoring run"); + let first_set: TopicAuthoring = authoring_from_json(&first).expect("parses"); + let second = author(&doc, &root, Some(&first)).expect("second authoring run"); + let second_set: TopicAuthoring = authoring_from_json(&second).expect("parses"); + + // The first run's parts survive: nothing the RLM still needs vanishes. + for migration in &first_set.migrations { + assert!( + second_set + .migrations + .iter() + .any(|m| m.name == migration.name && m.sql == migration.sql), + "migration {} was dropped by the re-authoring run", + migration.name + ); + } + for api in &first_set.apis { + assert!( + second_set + .apis + .iter() + .any(|a| a.path == api.path && a.method == api.method), + "route {} /{} was dropped by the re-authoring run", + api.method, + api.path + ); + } + // And the retained set still passes the gates. + second_set + .validate(&doc.id) + .expect("the guest accepts the re-authored set"); + second_set + .pin_policy + .agrees_with_document(&doc) + .expect("the re-authored policy restates the document"); + let _ = std::fs::remove_dir_all(&root); +} + +/// A prior set whose migrations include a topic-scoped `DELETE` is retained and +/// still accepted: the part the topic still needs must survive a re-authoring +/// run, and the RLM's own scope check must read `DELETE FROM _table` as +/// an in-namespace touch rather than a touch on `FROM`. +/// +/// Greptile P1: the scan skipped modifiers after a table keyword but not +/// `FROM`, so a retained pruning migration was refused and the topic could not +/// re-author. +#[test] +fn a_retained_topic_scoped_delete_survives_re_authoring() { + if python3().is_none() { + eprintln!("python3 not on PATH: skipping the adaptor authoring gate"); + return; + } + let doc = topic(); + let root = tmp("retained-delete"); + let first = author(&doc, &root, None).expect("first authoring run"); + let prefix = doc.id.replace('-', "_"); + let mut prior: TopicAuthoring = authoring_from_json(&first).expect("parses"); + prior.migrations.push(proof_rlm::AuthoredMigration { + name: "0002_prune".into(), + sql: format!("DELETE FROM {prefix}_rlm_state WHERE key = 'stale'"), + }); + let prior_body = serde_json::to_string(&prior).expect("encode prior"); + let second = author(&doc, &root, Some(&prior_body)).expect("second authoring run"); + let second_set: TopicAuthoring = authoring_from_json(&second).expect("parses"); + let pruned = second_set + .migrations + .iter() + .find(|m| m.name == "0002_prune") + .expect("the retained pruning migration was dropped"); + assert_eq!( + pruned.sql, + format!("DELETE FROM {prefix}_rlm_state WHERE key = 'stale'") + ); + // And the set is still what the guest and the install accept. + second_set + .validate(&doc.id) + .expect("the guest accepts a set carrying a topic-scoped DELETE"); + let _ = std::fs::remove_dir_all(&root); +} + +/// A topic that says nothing about how a rule is ticked is a refusal, not a +/// silent pass: the adaptor never invents a check and never drops a rule. +#[test] +fn a_topic_with_no_rule_policy_authors_nothing() { + if python3().is_none() { + eprintln!("python3 not on PATH: skipping the adaptor authoring gate"); + return; + } + let doc = topic(); + let root = tmp("no-policy"); + let topic_file = root.join("topic.json"); + std::fs::write( + &topic_file, + serde_json::to_vec_pretty(&doc).expect("topic json"), + ) + .expect("write topic"); + let output = root.join("output"); + let work = root.join("work"); + std::fs::create_dir_all(&output).expect("output dir"); + std::fs::create_dir_all(&work).expect("work dir"); + let out = Command::new(adaptor().join("propose_rules")) + .current_dir(&root) + .env_clear() + .env("PATH", "/usr/local/bin:/usr/bin:/bin") + .env("PROOF_JOB", "propose_rules") + .env("PROOF_TOPIC_ID", &doc.id) + .env("PROOF_TOPIC_FILE", &topic_file) + .env("PROOF_OUTPUT_DIR", &output) + .env("PROOF_WORK_DIR", &work) + .env("PROOF_CURRENT_AUTHORING_FILE", "") + .output() + .expect("spawn propose_rules"); + assert!( + !out.status.success(), + "a topic with no inspect policy must fail closed, not author a set" + ); + assert!( + !output.join("authoring.json").exists(), + "a refused run writes no set" + ); + let stderr = String::from_utf8_lossy(&out.stderr).to_lowercase(); + assert!( + stderr.contains("inspect_marker_rules") || stderr.contains("inspect_attested_rules"), + "the refusal must name the missing policy: {stderr}" + ); + let _ = std::fs::remove_dir_all(&root); +} diff --git a/crates/proof-vm-proto/src/guest.rs b/crates/proof-vm-proto/src/guest.rs index 43e11caf6..7a0eab9d6 100644 --- a/crates/proof-vm-proto/src/guest.rs +++ b/crates/proof-vm-proto/src/guest.rs @@ -421,7 +421,61 @@ pub async fn read_frame Deserialize<'de>>( r.read_exact(&mut body) .await .map_err(|e| ProtoError::Io(e.to_string()))?; - serde_json::from_slice(&body).map_err(|e| ProtoError::Decode(e.to_string())) + serde_json::from_slice(&body).map_err(|e| decode_error(&body, &e)) +} + +/// A decode failure, named for the skew it usually is. +/// +/// The guest and the host are **separately built** binaries pinned by digest: +/// the guest image and the orchestrator on the KVM host are promoted on their +/// own schedules, so a frame can carry a variant the reading side does not +/// know. `serde_json` reports `unknown variant` and stops there, which reads +/// like a corrupt frame rather than a build mismatch — the shape of the live +/// FAIL this helper exists to name. An unknown variant is called out as a +/// version skew with both remedies (rebake the image, or rebuild the reader); +/// anything else stays a decode error. +/// +/// Public because **both** host decoders face the same skew: the orchestrator +/// reads the guest's frame ([`read_frame`]) and the control plane reads the +/// orchestrator's HTTP answer, which is the same JSON one hop later. +#[must_use] +pub fn decode_skew(body: &[u8], e: &serde_json::Error) -> String { + let msg = e.to_string(); + if !msg.contains("unknown variant") { + return msg; + } + // The frame is well-formed JSON the reader does not understand: the writer + // is newer than the reader, and only one of them can be promoted to fix + // it. `serde_json` writes the offending name between **backticks** + // (`unknown variant \`authored\`, expected one of …`), so the split is on + // a backtick; a double quote never matches, and the refusal would fall + // back to naming no variant at all. + let variant = msg + .split('`') + .nth(1) + .unwrap_or("a variant this build does not know"); + format!( + "peer sent variant `{variant}`, which this build (api_version {API_VERSION}) does not \ + decode: the two sides are built separately (the guest image and the host binaries are \ + pinned by digest), so this is a build skew, not a corrupt frame. Rebake the guest image \ + from the tip, or rebuild this reader from it, so both sides speak the same wire ({})", + preview(body) + ) +} + +fn decode_error(body: &[u8], e: &serde_json::Error) -> ProtoError { + ProtoError::Decode(decode_skew(body, e)) +} + +/// A short, bounded view of a frame body for an error message. +fn preview(body: &[u8]) -> String { + const MAX: usize = 160; + let text = String::from_utf8_lossy(&body[..body.len().min(MAX)]); + if body.len() > MAX { + format!("{text}…") + } else { + text.into_owned() + } } #[cfg(test)] @@ -462,6 +516,102 @@ mod tests { assert_eq!(check_version(2), Err(ProtoError::WrongVersion { got: 2 })); } + /// The guest and the host are **separately built** binaries. A frame + /// carrying a variant the reader does not know is a build skew, and it + /// must say so: the raw `unknown variant` reads like a corrupt frame, and + /// that is what made the live FAIL opaque. + #[tokio::test] + async fn an_unknown_variant_names_the_build_skew_not_a_corrupt_frame() { + // A frame from a newer guest: a `Done` whose output tag this build + // has no variant for. Hand-written rather than encoded, because the + // whole point is a body *this* build cannot produce. `RlmToHost` is + // internally tagged (`"type"`) and `VmJobOutput` is adjacently tagged + // inside `Done`'s `output` field, so the skew lands on the inner tag. + let body = + br#"{"type":"done","output":{"output":"a_variant_from_a_newer_guest","body":{}}}"#; + let mut frame = u32::try_from(body.len()) + .expect("fits") + .to_be_bytes() + .to_vec(); + frame.extend_from_slice(body); + let err = read_frame::<_, RlmToHost>(&mut std::io::Cursor::new(frame)) + .await + .expect_err("unknown variant"); + let text = err.to_string(); + assert!(text.contains("build skew"), "{text}"); + // The **extracted** name, not merely a substring of the preview: the + // preview also carries the body, so asserting on the whole message + // would pass even if the variant were never parsed out of serde's + // message. This is the assertion that would have caught that. + assert!( + text.contains("peer sent variant `a_variant_from_a_newer_guest`"), + "the refusal must name the variant it extracted: {text}" + ); + assert!( + text.contains("Rebake the guest image"), + "the refusal names both remedies: {text}" + ); + assert!(text.contains("api_version 1"), "{text}"); + // The outer tag can skew too — a whole message this build does not + // know — and it is named the same way. + let body = br#"{"type":"a_message_from_a_newer_guest","x":1}"#; + let mut frame = u32::try_from(body.len()) + .expect("fits") + .to_be_bytes() + .to_vec(); + frame.extend_from_slice(body); + let err = read_frame::<_, RlmToHost>(&mut std::io::Cursor::new(frame)) + .await + .expect_err("unknown message"); + let text = err.to_string(); + assert!(text.contains("build skew"), "{text}"); + assert!( + text.contains("peer sent variant `a_message_from_a_newer_guest`"), + "{text}" + ); + // And a genuinely corrupt body stays a decode error, not a skew. + let body = b"{not json at all"; + let mut frame = u32::try_from(body.len()) + .expect("fits") + .to_be_bytes() + .to_vec(); + frame.extend_from_slice(body); + let err = read_frame::<_, RlmToHost>(&mut std::io::Cursor::new(frame)) + .await + .expect_err("corrupt"); + assert!(!err.to_string().contains("build skew"), "{err}"); + } + + /// The full authored set survives the **frame** codec, every part: the + /// guest emits it, the orchestrator relays it, the control plane reads it, + /// and each hop re-encodes the same document. + #[tokio::test] + async fn the_whole_authored_set_survives_a_frame() { + let set = proof_rlm::fixtures::authored_set( + &proof_rlm::fixtures::topic(), + proof_rlm::fixtures::rules().rules, + ); + let msg = RlmToHost::Done { + output: VmJobOutput::Authored(Box::new(set.clone())), + }; + let (mut a, mut b) = tokio::io::duplex(1 << 20); + write_frame(&mut a, &msg).await.expect("write"); + let back: RlmToHost = read_frame(&mut b).await.expect("read"); + let RlmToHost::Done { + output: VmJobOutput::Authored(got), + } = back + else { + panic!("the set did not survive the frame: {back:?}"); + }; + assert_eq!(*got, set); + assert!(got.is_complete(), "missing {:?}", got.missing_parts()); + // The relay hop re-encodes the same document: what the orchestrator + // sends the control plane is byte-identical to what the guest sent. + let relayed = serde_json::to_vec(&VmJobOutput::Authored(got)).expect("relay"); + let direct = serde_json::to_vec(&VmJobOutput::Authored(Box::new(set))).expect("direct"); + assert_eq!(relayed, direct, "the relay changed the document"); + } + #[test] fn staged_files_round_trip_and_sister_documents_are_public() { let f = StagedFile::new("artifact.tar", b"\x00\x01binary"); diff --git a/deploy/guest/runners/README.md b/deploy/guest/runners/README.md index 34c632292..076e6d9d4 100644 --- a/deploy/guest/runners/README.md +++ b/deploy/guest/runners/README.md @@ -12,6 +12,26 @@ ships under [`rlm_fc_in_guest_harbor/`](rlm_fc_in_guest_harbor/) so operators can bake Harbor evaluate with miner artefact attach; it is still selected only when a signed topic names that runner id. +**A change here needs a guest rebake to have any effect.** The adaptor is the +baked image, not the control-plane tip: tipping `proof-challenge` or the +gateway leaves `/opt/proof/runners` on the old pin, and a missing or older +entrypoint keeps failing closed (`NO_RLM_RULES` for `propose_rules`, +`adaptor wrote no results.json` for a stale `run`). Rebake, stage the new +rootfs on the KVM host, and re-pin `PROOF_RLM_VM_IMAGE_DIGEST` to that +image's own `sha256sum` — never invent a digest. Runbook: +[`docs/runbooks/proof-experiment-vms.md`](../../../docs/runbooks/proof-experiment-vms.md) +§ Guest rebake after runner changes. + +**The guest agent and the adaptor are two halves of one pin.** A run is +harvested by whichever guest agent the image carries, and an agent baked +before `authoring.json` existed reads only `rules.json`: a `propose_rules` +run that writes the set alone answers that agent with `adaptor wrote no +rules.json` and the job fails (the live FAIL). Write **both** files — the set +as the authorship, the fragment as its compat copy — so one run answers +either agent, and keep them one answer, because the pair is compared and a +disagreement is refused. The rebake is what removes the need; the dual write +is what makes the run correct on both sides of it. + This directory holds the **contract** and, when a live gap needs a bakeable fix, a reference adaptor directory named after the runner id. The Harbor CLI, its venv, and the task pack stay operator content (`--overlay` / @@ -28,7 +48,7 @@ operator's view of it. |------|-----|--------------------------------------| | `run` (required) | `Baseline`, `Evaluate` | `report.json` — `{"primary_value": , "claim_holds": bool, "flops_used": , "evidence": {...}}`. **Evaluate** also writes the topic-defined complete results JSON (default `results.json`; pin `results_path` / `results_contract` in `constraints.params`). Missing or non-conforming on evaluate is fail-closed (no Done) | | `inspect` | `Inspect` (anti-cheat rules, **before any paid inference**) | `checklist.json` — `[{"id": "", "pass": bool, "evidence": "..."}]`; a rule left out is recorded **red** | -| `propose_rules` | `ProposeRules` (RLM authorship) | `rules.json` — `[{"id": "", "text": "..."}]`. **A runner whose topic must open needs this entrypoint**: without it the guest refuses the job (`Failed` → 503, no row, nothing scored), because there is **no** fallback that echoes the signed `checklist` back. Echoing it would let the control plane record the operator's own vector as `source = rlm`, which is an operator-cloned document masquerading as RLM authorship. The signed `checklist` stays the topic's version 1 with honest `topic_document` provenance, and only a run of this entrypoint advances the store to `rlm` — which the publish gate requires before a topic may be `open`. | +| `propose_rules` | `ProposeRules` (RLM authorship) | **`authoring.json`** — `{schema_version: 1, topic_id, rules, migrations, apis, submission_format, pin_policy}`; **every part is required** (an absent `pin_policy` key does not parse, an empty `{}` is a legitimate answer). **Write `rules.json` beside it too** — `[{"id": "", "text": "..."}]`, the set's own `rules` vector verbatim: a guest **baked before the set existed** harvests this run out of `rules.json` and fails the job without it (`adaptor wrote no rules.json`, the live FAIL), so writing both is what makes one run answer both guests. The pair is **one answer**, never two: a guest that reads the set refuses a pair whose vectors disagree (`DUAL_EMIT_RULES_DISAGREE`) rather than letting which guest harvested decide the topic's vector, and a run that writes neither file is refused as authoring nothing. `rules.json` alone is read as a **fragment** and is not authorship. **A runner whose topic must open needs this entrypoint**: without it the guest refuses the job (`Failed` → 503, no row, nothing scored), because there is **no** fallback that echoes the signed `checklist` back. Echoing it would let the control plane record the operator's own vector as `source = rlm`, which is an operator-cloned document masquerading as RLM authorship. The signed `checklist` stays the topic's version 1 with honest `topic_document` provenance, and only a run of this entrypoint advances the store to `rlm` — which the publish gate requires before a topic may be `open`. A rules-only answer is recorded with honest `rlm` provenance and refused by name (`IncompleteAuthoring`, naming the parts that have no author) rather than widened from the operator's bundle. The reference adaptor's own entrypoint is [`rlm_fc_in_guest_harbor/propose_rules`](rlm_fc_in_guest_harbor/propose_rules) (+ [`harness/authoring_set.py`](rlm_fc_in_guest_harbor/harness/authoring_set.py)); it reads `$PROOF_CURRENT_AUTHORING_FILE` to **retain** the parts a re-authoring run is not changing, and writes the fragment **before** the set so a run cut between the two writes leaves a fragment (which opens nothing) rather than a set whose compat copy a stale guest cannot find | A non-zero exit with no document, a missing document, a non-finite `primary_value`, a missing or non-conforming Evaluate `results.json`, or a @@ -52,10 +72,10 @@ happened to be lying around. | `PROOF_SUBMISSION_DIGEST`, `PROOF_ARTIFACT_DIGEST` | the run's identities (echoed into the report by the agent) | | `PROOF_ARTIFACT_DIR` | the miner's artefact, fetched **by the agent** (streamed under a 64 MiB cap), verified against `PROOF_ARTIFACT_DIGEST`, unpacked (set only when the request carries a locator; always set for `evaluate`). `$PROOF_WORK_DIR/artifact.tar` holds the verbatim bytes | | `PROOF_PACK_DIR`, `PROOF_PACK_DIGEST` | the topic-pinned experiment pack, staged by the host at boot and verified by the agent (paid jobs) | -| `PROOF_MODEL_PIN`, `PROOF_TASK_SLICE` | `constraints.model_pin` / `constraints.task_slice` when the topic carries them | +| `PROOF_MODEL_PIN`, `PROOF_TASK_SLICE` | `constraints.model_pin` / `constraints.task_slice` when the topic carries them. **`task_slice` is an assertion, not a hint**: a label the pinned pack does not define is a refusal (the LIVE Gate 1 fail-open scored a different, larger set). Name the set explicitly with `PROOF_PARAM_TASKS` (+ `n_tasks`) instead — the guest reads the tasks **instead of** the label, so a stale label beside them is never resolved | | `PROOF_SEED`, `PROOF_DEADLINE_S`, `PROOF_DECLARED_FLOPS`, `PROOF_FLOPS_BUDGET` | run parameters from the signed topic and the submission | | `PROOF_CLAIM_FILE` | the miner's claim text (`run`) | -| `PROOF_RULES_FILE` | the rule set to tick (`inspect`); `PROOF_TOPIC_FILE` the signed topic (`propose_rules`) | +| `PROOF_RULES_FILE` | the rule set to tick (`inspect`); `PROOF_TOPIC_FILE` the signed topic (`propose_rules`); `PROOF_CURRENT_AUTHORING_FILE` where the set this RLM authored **last time** was written (`propose_rules`; always set, empty when there is none — read it to retain the parts a re-authoring run is not changing); `PROOF_CURRENT_RULES_VERSION` the version it supersedes (empty = none) | | `PROOF_OUTPUT_DIR`, `PROOF_WORK_DIR` | where to write the answer; scratch on the writable disk | | `PROOF_SECRETS_DIR`, `PROOF_SECRET_FILES` | owner key material staged by the KVM host (`PROOF_VM_AGENT_OWNER_KEY_DIR`), by file name. **Read them; never print them** — the agent redacts their values from every log tail and evidence string it sends back, but not from anything you write elsewhere | | `PROOF_PARAM_` | one per `constraints.params` entry (key upper-cased, `-` → `_`). This is how a topic tells its adaptor which tasks, agent, concurrency, key file, … to use — **the adaptor never hardcodes them**. Two signed names that collide after that mapping (`foo-bar` / `foo_bar`) are refused before anything runs. The well-known **run-policy** keys below are shape-checked by the guest before the adaptor runs | diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md index 71196d609..e66709ed9 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md @@ -444,6 +444,69 @@ pass. Until it carries a task selection, the whole `tasks_dir` is scored. - `PROOF_HARNESS_SKIP_PODMAN=1` (or `PROOF_HARNESS_SKIP_RUNTIME=1`) skips the socket (unit tests). +## Authorship (`propose_rules`) + +The entrypoint that makes a topic installable at all: without it the guest +refuses every `ProposeRules` job (`NO_RLM_RULES` → 503, no row), so the topic +has no RLM-authored behavior and the publish gate will not open it. + +It writes **`$PROOF_OUTPUT_DIR/authoring.json`** — `schema_version` 1 plus the +five parts an install applies: `rules`, `migrations`, `apis`, +`submission_format`, `pin_policy`. + +It also writes **`$PROOF_OUTPUT_DIR/rules.json`** — the set's own `rules` +vector, verbatim — as a **compat copy**. A guest agent baked before the set +existed harvests a `propose_rules` run out of that file and fails the job +without it (`adaptor wrote no rules.json`), so writing both is what makes one +run answer both guests. The pair is **one answer**, never two: a guest that +reads the set compares the two vectors and refuses a disagreement, because +which guest harvested the run would otherwise decide the topic's anti-cheat +surface. The fragment is written **first**, so a run cut between the two writes +leaves a fragment — recorded with honest `rlm` provenance and refused +downstream by name (`IncompleteAuthoring` / `RULES_ONLY_IS_NOT_AUTHORSHIP`) — +rather than a set whose compat copy a stale guest cannot find. + +| Part | Authored from | What makes it the RLM's | +|------|---------------|-------------------------| +| `rules` | the signed `checklist` (ids and order) + the signed inspect policy | the **text** is the RLM's own statement of what it will prove and how, with the topic's sentence quoted as the declaration it enforces. The vector is exactly the declared rules: none invented, none dropped | +| `migrations` | the topic's own namespace (`{id}` with `-` → `_`) | the RLM's own state table, inside its namespace. Prior entries the derivation does not name are **retained** | +| `apis` | the topic's own prefix | the route row the install registers; prior routes retained the same way | +| `submission_format` | this runtime's real intake | the staged cap, the `base-proof-submit-v1` domain, the single-use nonce — facts about the host, not a bundle section | +| `pin_policy` | the signed document's own knobs | a **restatement**: scoring reads the document, so a policy may restate a knob and never diverge. `eval_image_digest` / `gpu_class` are equalities against the **pin**, which the VM does not hold, so they stay absent rather than invented | + +**Re-authoring retains what it is not changing.** The guest writes the set this +RLM authored last time to `$PROOF_WORK_DIR/current-authoring.json` and exports +`PROOF_CURRENT_AUTHORING_FILE` (always set; empty on a first run). `migrations` +and `apis` are merged — prior order preserved, the current answer winning per +name — while `rules` and `pin_policy` are re-derived, because a retained rule +could be one the re-signed document dropped and a retained policy could diverge +from it. A prior set for **another topic** is refused. + +**Two refusals, both deliberate:** + +- a declared rule the topic names in **neither** `inspect_marker_rules` nor + `inspect_attested_rules` — this RLM does not invent a check for a rule it was + handed, and does not drop one either (dropping it would narrow the topic's + anti-cheat surface without saying so; leaving it in would record it red + forever, so the topic could never open). The topic must say how each rule is + ticked; +- a **marker** rule the checklist does not declare — a signed marker check for + a rule the topic does not carry is a typo, and ignoring it would discard a + check the operator asked for. + +The gates that matter are **not** in this directory: the guest runs +`proof-topic-authoring`'s shape and completeness checks plus the migration +deny-list before the answer becomes a job output, and the control plane runs +the same shape checks plus the policy against the **global pin** +(`set.validate_against_pin`). `harness/authoring_set.py` holds the set to the +same shapes before writing it, so a malformed answer fails inside the VM with +the part named. + +> **A guest rebake is required for this to reach a live topic.** The entrypoint +> is an operator artefact copied into the image by `bake-rootfs.sh --runner`; +> tipping `proof-challenge` alone leaves `/opt/proof/runners` on the old pin and +> `--drive-rlm` keeps failing closed. See § Operator bake / deploy. + ## Miner-facing guide Attach layout and BYOK: [`docs/external-miner/proof-tbench.md`](../../../../docs/external-miner/proof-tbench.md). diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/authoring_set.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/authoring_set.py new file mode 100755 index 000000000..a81f7d094 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/authoring_set.py @@ -0,0 +1,867 @@ +#!/usr/bin/env python3 +"""Author the topic's whole behavior set — the RLM's answer to `ProposeRules`. + +Writes ``$PROOF_OUTPUT_DIR/authoring.json``: ``schema_version`` 1 plus the five +parts an install applies — and, beside it, ``rules.json`` as a **compat copy** +of the set's own ``rules`` vector. The set is authorship; the fragment is not, +and nothing may open a topic on it (``SetupError::IncompleteAuthoring`` / +``RULES_ONLY_IS_NOT_AUTHORSHIP``). The copy exists because a guest **baked +before the set existed** harvests a ``propose_rules`` run out of ``rules.json`` +and fails the job when it is absent (``adaptor wrote no rules.json``): one run +therefore answers both guests, and a guest that reads the set refuses a pair +whose vectors disagree rather than letting which guest harvested decide the +topic's vector. The two files are always written together, fragment first, so a +run cut between the writes leaves a fragment (which opens nothing) rather than +a set whose compat copy a stale guest cannot find. + +What each part is authored *from*, and why it is the RLM's answer rather than +a copy of the operator's bundle: + +=============================== ========================================= +part source of the answer +=============================== ========================================= +``rules`` the signed ``checklist`` **partitioned by + what this RLM can actually tick**: only a + rule the topic names in + ``inspect_marker_rules`` / + ``inspect_attested_rules`` is enforceable, + and the vector is the RLM's own framing of + it. A declared rule with no policy is a + refusal, never a silent drop — dropping it + would narrow the topic's anti-cheat + surface behind the operator's back. +``migrations`` the topic's own namespace (its id, ``-`` → + ``_``), because a complete set needs at + least one and the RLM's minimum schema is + its own state table. Prior entries the + derivation does not name are **retained**. +``apis`` the topic's own prefix; the mux serves the + row, so the RLM authors the route it + exposes and no handler it cannot have. + Prior routes are retained the same way. +``submission_format`` what this RLM's runtime accepts: the host's + real intake shape (uncompressed tar under + the staged cap, sr25519 hotkey signature + over the submit domain, single-use nonce), + never a bundle section. +``pin_policy`` a **restatement** of the signed document's + own knobs. Scoring reads the document, so a + policy may restate what the document + declares — proving the RLM considered the + knob — and may not diverge from it in + either direction. Knobs the RLM cannot + truthfully name (``eval_image_digest``, + ``gpu_class``: equalities against the + **pin**, which the VM does not hold) are + left absent rather than invented. +=============================== ========================================= + +**Re-authoring retains what it is not changing.** ``PROOF_CURRENT_AUTHORING_FILE`` +(e.g. ``current-authoring.json``; empty on a first run) carries the set this +RLM authored last time. ``migrations`` and ``apis`` are merged: the prior order +is preserved, the RLM's current answer replaces the entries it names, and +entries it does not name are kept — without that a re-authoring run is a +rewrite from nothing, and a migration the topic still needs would silently +vanish. + +``rules``, ``submission_format`` and ``pin_policy`` are **always re-derived**, +because each is a fact about *now* rather than a decision to keep: + +* a retained rule could be one the re-signed document dropped (the vector is + the topic's anti-cheat surface, and it must match the declaration); +* a retained ``submission_format`` would publish a **previous host's** intake + contract — the staged cap, the submit domain, the nonce are properties of + the runtime this run is executing on, so they are re-read every run; +* a retained policy could diverge from the document that scoring actually + reads. + +**What is checked here, and what is not.** The authoritative gates are the +guest's (``crates/proof-vm-guest``, the same ``proof-topic-authoring`` the +install links) and the install's SQL deny-list. This module holds the set to +the same shape before writing it, so a malformed answer fails inside the VM +with the part named instead of becoming a job output. The migration check here +is deliberately conservative rather than complete: it refuses an unscoped or +``proof_*`` name and leaves the full deny-list to the guard that owns it. + +Secrets: ``propose_rules`` is unpaid and reads none. Nothing here is logged. +""" + +from __future__ import annotations + +import json +import math +import os +import re +import sys +import tempfile +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +# `inspect_scan` is the adaptor's own inspector. Reusing its parser is what +# makes "the RLM authors the vector its inspector ticks" true by construction: +# a rule this file authors is tickable by the code that will tick it, and the +# two cannot drift. +sys.path.insert(0, str(HERE.parent)) +import inspect_scan # noqa: E402 + +AUTHORING_SCHEMA = 1 +AUTHORING_FILE = "authoring.json" +RULES_FILE = "rules.json" +CURRENT_AUTHORING_FILE = "current-authoring.json" + +MAX_RULES = 64 +MAX_RULE_TEXT_CHARS = 2048 +MAX_MIGRATIONS = 64 +MAX_MIGRATION_SQL_BYTES = 256 * 1024 +MAX_APIS = 64 +MAX_API_SUMMARY_CHARS = 256 +MAX_PIN_STRING_CHARS = 256 + +# `proof_vm_proto::guest::MAX_STAGED_ARTIFACT_TAR_BYTES`: the host refuses a +# larger staged tar, so a format claiming more would be a promise it does not +# keep. +MAX_ARTIFACT_BYTES = 5 * 1024 * 1024 +# `base-proof-submit-v1`: the domain the miner's hotkey signature is over. +SUBMIT_DOMAIN = "base-proof-submit-v1" + +RULE_ID = re.compile(r"^[a-z0-9][a-z0-9_-]{1,63}$") +API_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE", "*") +# `proof_topic_authoring::RESERVED_API_PREFIXES`: the challenge's own operator +# surface, which is not a topic's to claim. +RESERVED_API_PREFIXES = ("v1/admin",) +# `proof_topic_sql_guard`: the prefix every object this repository owns +# carries. A topic migration may not name one, whatever the verb. +OWNED_TABLE_PREFIX = "proof_" +TABLE_KEYWORDS = ("FROM", "JOIN", "INTO", "UPDATE", "TABLE", "INDEX", "TRUNCATE", "DELETE") +# `proof_topic_sql_guard::is_sql_keyword`: tokens that are never a table name in +# the position the scan reads. `FROM` is here for the same reason it is in the +# guard — in `DELETE FROM x` the token after `DELETE` is `FROM`, and the table +# is the token after *that* (which the scan reaches because `FROM` is itself a +# table keyword). Without this a retained `DELETE FROM ` +# is refused for "touching FROM". +SQL_KEYWORDS = ( + "select", + "from", + "where", + "values", + "set", + "and", + "or", + "not", + "null", + "default", + "lateral", + "unnest", + "true", + "false", +) +# Modifiers skipped between a table keyword and the name it introduces. +TABLE_MODIFIERS = ( + "IF", + "NOT", + "EXISTS", + "OR", + "REPLACE", + "ONLY", + "INTO", + "UNIQUE", + "CONCURRENTLY", +) +DENIED_OBJECTS = ( + "_sqlx_migrations", + "base_app", + "pg_roles", + "pg_authid", + "information_schema", + "pg_catalog", + "pg_proc", + "pg_shadow", +) + +PARAM_MARKER_RULES = inspect_scan.PARAM_MARKER_RULES +PARAM_ATTESTED_RULES = inspect_scan.PARAM_ATTESTED_RULES + + +def _fail(msg: str, code: int = 2) -> None: + print(f"authoring_set: {msg}", file=sys.stderr) + raise SystemExit(code) + + +def _load_json(path: Path, what: str) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + _fail(f"cannot read {what} from {path}: {e}") + + +def _as_object(value: Any, what: str) -> dict[str, Any]: + if not isinstance(value, dict): + _fail(f"{what} is not a JSON object") + return value + + +def _clip(text: str, limit: int) -> str: + if len(text) <= limit: + return text + return text[: limit - 3] + "..." + + +def _is_int(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _is_fraction(value: Any) -> bool: + """A finite knob in `(0, 1]` — the shape a pin-policy floor may carry.""" + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and 0.0 < value <= 1.0 + ) + + +def topic_document() -> dict[str, Any]: + path = os.environ.get("PROOF_TOPIC_FILE", "").strip() + if not path: + _fail("PROOF_TOPIC_FILE is required") + doc = _as_object(_load_json(Path(path), "the signed topic"), "the signed topic") + topic_id = doc.get("id") + if not isinstance(topic_id, str) or not topic_id.strip(): + _fail("the signed topic carries no id; the set cannot be bound to a topic") + return doc + + +def current_set() -> dict[str, Any] | None: + """The set this RLM authored last time, or ``None`` on a first run. + + The guest always sets ``PROOF_CURRENT_AUTHORING_FILE`` (empty when there is + none) so this branches on one variable rather than on its presence. A file + that exists but cannot be read is a refusal: silently rewriting from + nothing is the lossy re-authoring the retention exists to prevent. + """ + path = os.environ.get("PROOF_CURRENT_AUTHORING_FILE", "").strip() + if not path: + return None + p = Path(path) + if not p.is_file(): + _fail(f"PROOF_CURRENT_AUTHORING_FILE={path} is not a file") + return _as_object(_load_json(p, "the previous authored set"), "the previous authored set") + + +def sql_prefix(topic_id: str) -> str: + """`proof_topic_sql_guard::topic_sql_prefix`: the id's identifier-safe form.""" + return topic_id.strip().lower().replace("-", "_") + + +def is_topic_scoped(name: str, topic_id: str) -> bool: + """`proof_topic_sql_guard::is_topic_scoped`, in Python. + + Two spellings are the topic's and only two: a `{id}`-qualified name + (`.scores`) or a bare `{id}_`-prefixed one (`_scores`), with the + hyphen id's identifier-safe form accepted in both positions. + """ + n = name.strip().strip('"').lower() + if not n: + return False + topic = topic_id.strip().lower() + mapped = sql_prefix(topic) + schema, _, bare = n.partition(".") + if not bare: + schema, bare = "", schema + if schema in (topic, mapped): + return True + return bare.startswith(f"{topic}_") or bare.startswith(f"{mapped}_") + + +# --------------------------------------------------------------------------- +# rules +# --------------------------------------------------------------------------- + + +def derive_rules(doc: dict[str, Any]) -> list[dict[str, str]]: + """The vector this RLM ticks, framed by the RLM. + + The topic's signed policy is what makes a rule *tickable*: + ``inspect_marker_rules`` names a rule this RLM proves by scanning the + artefact for off-limits markers, ``inspect_attested_rules`` one the host or + topic enforces outside the scan. The signed ``checklist`` is the + operator's declaration of intent; it seeds the ids and the order. + + The authored **text** is the RLM's own statement of what it will show and + how it will show it, with the topic's sentence quoted as the declaration it + enforces. That is the part the RLM owns: the operator said what the rule + means, the RLM says what it will prove. + + The vector is exactly the declared rules, in declaration order — no rule is + added and none is dropped, because the vector in force is the inspection + surface miners verified against. Two refusals follow from that: + + * a declared rule with **no** policy — this RLM will not invent a check for + a rule it was handed, and it will not drop one either: dropping it would + narrow the topic's anti-cheat surface without saying so. (Left in, the + inspector would record it red forever and the topic could never open.) + * a **marker** rule the checklist does not declare — a signed marker policy + for a rule the topic does not carry is a check that would never run, and + silently ignoring it would discard something the operator asked for. An + *attested* id outside the checklist is not a refusal: the inspector never + ticks a rule that is not in the vector, so a host fact the topic declares + and does not carry as a rule is simply not one of this RLM's rules. + """ + declared: list[dict[str, str]] = [] + for item in doc.get("checklist") or []: + if not isinstance(item, dict): + _fail("checklist carries an entry that is not an object") + rid = item.get("id") + if not isinstance(rid, str) or not rid.strip(): + _fail("checklist carries a rule with no id") + text = item.get("text") + declared.append( + {"id": rid.strip(), "text": text.strip() if isinstance(text, str) else ""} + ) + declared_ids = {r["id"] for r in declared} + + # The adaptor's own parser: a policy it accepts here is one its inspector + # accepts at tick time. Both fail closed on a malformed entry. + marker_rules = inspect_scan.parse_marker_rules(os.environ.get(PARAM_MARKER_RULES)) + attested = inspect_scan.parse_attested_rules(os.environ.get(PARAM_ATTESTED_RULES)) + both = sorted(set(marker_rules) & attested) + if both: + _fail( + f"{PARAM_MARKER_RULES} and {PARAM_ATTESTED_RULES} both name {', '.join(both)}; " + "a rule is ticked one way" + ) + if not marker_rules and not attested: + _fail( + f"the signed topic names no rule policy: set {PARAM_MARKER_RULES} (rules this RLM " + f"proves by scanning the artefact) and/or {PARAM_ATTESTED_RULES} (rules the host or " + "topic enforces outside the scan) and re-run. Without a policy every rule the " + "inspector ticks is red, so the topic could never open — this entrypoint does not " + "invent a check for a rule the topic never said how to tick." + ) + if not declared: + _fail( + "the signed topic declares no checklist rules, so this RLM has no rule vector to " + "author: the vector is the topic's anti-cheat surface, and inventing one would " + "score miners against rules they never verified. Sign the topic's checklist (and " + "the inspect policy for each rule) and re-run." + ) + + unpolicy = [r["id"] for r in declared if r["id"] not in marker_rules and r["id"] not in attested] + if unpolicy: + _fail( + f"the signed topic declares {', '.join(unpolicy)} but names them in neither " + f"{PARAM_MARKER_RULES} nor {PARAM_ATTESTED_RULES}; this RLM ticks what the topic " + "says it is ticked by. Re-sign the topic with a policy for every rule it declares " + "(or drop the rule from the checklist)" + ) + orphan_markers = sorted(rid for rid in marker_rules if rid not in declared_ids) + if orphan_markers: + _fail( + f"{PARAM_MARKER_RULES} names {', '.join(orphan_markers)}, which the signed checklist " + "does not declare; a marker check for a rule the topic does not carry is a typo" + ) + + rules: list[dict[str, str]] = [] + for item in declared: + rid, quoted = item["id"], item["text"] + if rid in marker_rules: + show = ( + f"none of the {len(marker_rules[rid])} off-limits markers the signed topic names " + "for this rule appear in the miner's artefact, in its text or its file names (a " + "truncated scan is not a pass)" + ) + how = "rlm: artefact scan, no inference" + else: + show = ( + "the host or topic enforces this outside the artefact scan, and this RLM records " + "the signed attestation instead of scanning" + ) + how = "rlm: host/topic attestation, no inference" + text = f"{show} [{how}]" + if quoted: + text += f" — the topic's declaration: {quoted}" + rules.append({"id": rid, "text": _clip(text, MAX_RULE_TEXT_CHARS)}) + if len(rules) > MAX_RULES: + _fail(f"the vector would carry {len(rules)} rules; at most {MAX_RULES} are applied") + return rules + + +# --------------------------------------------------------------------------- +# migrations +# --------------------------------------------------------------------------- + + +def derive_migrations(doc: dict[str, Any]) -> list[dict[str, str]]: + """The RLM's minimum schema, inside the topic's own namespace. + + One table, because a complete set carries at least one migration and this + is the schema the RLM's runtime keeps for itself: the topic's own state, + under the topic's prefix. It touches nothing else — the shared database's + objects are the repository's, and the deny-list refuses them. + """ + prefix = sql_prefix(doc["id"].strip()) + sql = ( + f"CREATE TABLE {prefix}_rlm_state (\n" + " key TEXT PRIMARY KEY,\n" + " value TEXT NOT NULL,\n" + " updated_at TIMESTAMPTZ NOT NULL DEFAULT now()\n" + ")" + ) + return [{"name": "0001_rlm_state", "sql": sql}] + + +def merge_migrations( + derived: list[dict[str, str]], retained: Any +) -> list[dict[str, str]]: + """Prior order preserved; the RLM's answer wins per name; the rest kept.""" + prior: list[dict[str, str]] = [] + if retained is not None: + if not isinstance(retained, list): + _fail("the previous set's migrations are not a list") + for item in retained: + if not isinstance(item, dict): + _fail("the previous set carries a migration that is not an object") + prior.append(item) + current = {m["name"]: m for m in derived} + merged: list[dict[str, str]] = [] + seen: set[str] = set() + for item in prior: + name = item.get("name") + if not isinstance(name, str) or not name.strip(): + _fail("the previous set carries a migration with no name") + if name in seen: + _fail(f"the previous set names the migration {name!r} twice") + seen.add(name) + merged.append(current.get(name, item)) + for name, item in current.items(): + if name not in seen: + merged.append(item) + if len(merged) > MAX_MIGRATIONS: + _fail(f"the set would carry {len(merged)} migrations; at most {MAX_MIGRATIONS} are applied") + return merged + + +IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*(?:\.[A-Za-z_][A-Za-z0-9_$]*)*") +QUOTED_IDENTIFIER = re.compile(r'"([^"]*)"') + + +def _identifiers(sql: str) -> list[str]: + """Identifiers in source order, double-quoted runs kept whole. + + `proof_topic_sql_guard::tokens` does the same, and for the same reason: a + hyphenated topic's mapped name is one identifier, and the **literal** id + (`"fixture-topic-v0_scratch"`) is a single legal quoted identifier that + splitting at the hyphen would turn into three unscoped ones. Order matters + — the namespace check reads the name *after* a table keyword — so a quoted + run is emitted where it appears, not collected separately. + """ + out: list[str] = [] + cursor = 0 + for match in QUOTED_IDENTIFIER.finditer(sql): + out.extend(IDENTIFIER.findall(sql[cursor : match.start()])) + out.append(match.group(1)) + cursor = match.end() + out.extend(IDENTIFIER.findall(sql[cursor:])) + return out + + +def check_migration_scope(sql: str, topic_id: str) -> None: + """Refuse a name that is not inside the topic's namespace. + + Conservative on purpose — the authoritative deny-list is + `proof_topic_sql_guard`, which runs in the guest and again at install. + This is the pre-flight that keeps an obviously unscoped migration out of a + job output, and it checks the same object positions that guard checks: + every identifier for the owned/denied names, then the name after each + table keyword for the namespace. + """ + for token in _identifiers(sql): + base = token.rsplit(".", 1)[-1].lower() + if base.startswith(OWNED_TABLE_PREFIX): + _fail( + f"migration names {token!r}: every proof_* object belongs to the scoring path " + "and is not a topic's to touch" + ) + if base in DENIED_OBJECTS or token.lower() in DENIED_OBJECTS: + _fail(f"migration names {token!r}: the shared database's own objects are not a topic's") + words = _identifiers(sql) + for index, word in enumerate(words): + if word.upper() not in TABLE_KEYWORDS: + continue + cursor = index + 1 + while cursor < len(words) and words[cursor].upper() in TABLE_MODIFIERS: + cursor += 1 + if cursor >= len(words): + continue + name = words[cursor] + # A token that is itself a SQL keyword is never a table name in this + # position — `DELETE FROM x` reads `FROM` here, and the outer loop + # reaches `x` because `FROM` is a table keyword too. Refusing it would + # reject every topic-scoped `DELETE FROM _table`. + if name.lower() in SQL_KEYWORDS: + continue + if name.lower().startswith(OWNED_TABLE_PREFIX): + continue # already refused above, by identifier + if not is_topic_scoped(name, topic_id): + _fail( + f"migration touches {name!r}, which is not inside the topic's namespace " + f"({sql_prefix(topic_id)}_* / {sql_prefix(topic_id)}.*); an unscoped name would " + "collide with — or read — another topic's install" + ) + + +def check_migration(item: Any) -> dict[str, str]: + if not isinstance(item, dict): + _fail("a migration is not an object") + name = item.get("name") + sql = item.get("sql") + if not isinstance(name, str) or not RULE_ID.match(name.strip()): + _fail(f"migration name {name!r} must match [a-z0-9][a-z0-9_-]{{1,63}}") + if not isinstance(sql, str) or not sql.strip(): + _fail(f"migration {name!r} carries no SQL; remove it instead") + if len(sql.encode("utf-8")) > MAX_MIGRATION_SQL_BYTES: + _fail(f"migration {name!r} is larger than {MAX_MIGRATION_SQL_BYTES} bytes") + return {"name": name.strip(), "sql": sql} + + +# --------------------------------------------------------------------------- +# apis +# --------------------------------------------------------------------------- + + +def derive_apis(doc: dict[str, Any]) -> list[dict[str, str]]: + """The route this topic exposes for itself, under its own prefix. + + The mux serves the **row** the install wrote, so the RLM authors a route + whose answer is its own record and never a handler it cannot have. + """ + del doc + return [ + { + "path": "status", + "method": "GET", + "summary": ( + "the topic's own record: the route row this install registered under the topic's " + "prefix, served by the challenge's registry" + ), + } + ] + + +def merge_apis(derived: list[dict[str, str]], retained: Any) -> list[dict[str, str]]: + """Same rule as migrations, keyed by `(path, method)`.""" + prior: list[dict[str, str]] = [] + if retained is not None: + if not isinstance(retained, list): + _fail("the previous set's apis are not a list") + for item in retained: + if not isinstance(item, dict): + _fail("the previous set carries a route that is not an object") + prior.append(item) + current = {(a["path"], a["method"]): a for a in derived} + merged: list[dict[str, str]] = [] + seen: set[tuple[str, str]] = set() + for item in prior: + key = (str(item.get("path", "")), str(item.get("method", ""))) + if key in seen: + _fail(f"the previous set names the route {key[1]} /{key[0]} twice") + seen.add(key) + merged.append(current.get(key, item)) + for key, item in current.items(): + if key not in seen: + merged.append(item) + if len(merged) > MAX_APIS: + _fail(f"the set would carry {len(merged)} routes; at most {MAX_APIS} are applied") + return merged + + +def check_api(item: Any) -> dict[str, str]: + if not isinstance(item, dict): + _fail("a route is not an object") + path = item.get("path") + method = item.get("method") + summary = item.get("summary", "") + if not isinstance(path, str) or not _is_relative_api_path(path): + _fail( + f"route path {path!r} must be a relative path of plain segments (no leading '/', no " + "'..', no empty segment): a topic's routes live under its own prefix" + ) + if any( + path == prefix or path.startswith(f"{prefix}/") + for prefix in RESERVED_API_PREFIXES + ): + _fail( + f"route path {path!r} is inside the challenge's admin namespace " + f"({', '.join(RESERVED_API_PREFIXES)}), which is not a topic's to claim" + ) + if not isinstance(method, str) or method.strip().upper() not in API_METHODS: + _fail(f"route method {method!r} must be one of {', '.join(API_METHODS)}") + if not isinstance(summary, str): + _fail(f"route {path!r} carries a summary that is not a string") + if len(summary) > MAX_API_SUMMARY_CHARS: + _fail(f"route {path!r} carries a summary longer than {MAX_API_SUMMARY_CHARS} chars") + return {"path": path, "method": method.strip().upper(), "summary": summary} + + +def _is_control(c: str) -> bool: + return ord(c) < 0x20 or ord(c) == 0x7F + + +def _is_relative_api_path(path: str) -> bool: + """`proof_topic_authoring::is_relative_api_path`, in Python.""" + p = path.strip() + if not p or len(p) > 512 or p.startswith("/") or p.endswith("/"): + return False + if any(_is_control(c) or c in "\\?#" for c in p): + return False + segments = p.split("/") + if any(seg in ("", ".", "..") for seg in segments): + return False + allowed = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._~-") + return all(set(seg) <= allowed for seg in segments) + + +# --------------------------------------------------------------------------- +# submission_format / pin_policy +# --------------------------------------------------------------------------- + + +def derive_submission_format() -> dict[str, Any]: + """What this RLM's runtime accepts, as the host's intake actually is.""" + return { + "kind": "tar", + "compression": "none", + "max_bytes": MAX_ARTIFACT_BYTES, + "artifact_digest": "sha256 of the exact served bytes; the guest verifies what it fetched", + "identity": "sr25519 hotkey_signature over the canonical submit payload", + "signature_domain": SUBMIT_DOMAIN, + "replay": "single-use 64-hex submit_nonce; a replay is refused before any row", + "artifact": "multipart part `artifact` (bytes win) or artifact_uri (the served file, verbatim)", + "miner_supplies": "claim + code + artifact; the judge and executor offers are not the miner's to bind", + } + + +def derive_pin_policy(doc: dict[str, Any]) -> dict[str, Any]: + """Restate the signed document's own knobs, and nothing it does not declare. + + Every knob here is an **equality** with the document (`PinPolicy::agrees_with_document`): + scoring reads the document, so a policy that named a different number would + be a threshold no challenger is judged by. The knobs that are equalities + against the **pin** (`eval_image_digest`, `gpu_class`) are left absent — + this VM does not hold the pin, and a value it cannot read is not a value it + may invent. + """ + policy: dict[str, Any] = {} + for key, source in ( + ("epsilon_nll_min", doc.get("epsilon_nll")), + ("epsilon_topic_max_regress_min", doc.get("epsilon_topic_max_regress")), + ("epsilon_throughput_rel_min", (doc.get("metric") or {}).get("epsilon_rel")), + ): + if _is_fraction(source): + policy[key] = source + deadline = (doc.get("eval_executor") or {}).get("max_proof_deadline_s") + if _is_int(deadline) and deadline >= 1: + policy["max_proof_deadline_s"] = deadline + budget = doc.get("flops_budget") + if _is_int(budget) and budget >= 1: + policy["flops_budget_max"] = budget + holdout = doc.get("holdout_size") + if _is_int(holdout) and holdout >= 1: + policy["holdout_size"] = holdout + return policy + + +def check_pin_policy(item: Any) -> dict[str, Any]: + if not isinstance(item, dict): + _fail("pin_policy is not an object") + allowed = { + "epsilon_nll_min", + "epsilon_throughput_rel_min", + "epsilon_topic_max_regress_min", + "max_proof_deadline_s", + "flops_budget_max", + "holdout_size", + "eval_image_digest", + "gpu_class", + } + unknown = sorted(set(item) - allowed) + if unknown: + _fail(f"pin_policy carries {', '.join(unknown)}, which this build does not read") + for key in ( + "epsilon_nll_min", + "epsilon_throughput_rel_min", + "epsilon_topic_max_regress_min", + ): + if key in item and not _is_fraction(item[key]): + _fail(f"pin_policy.{key} must be a finite fraction in (0, 1]") + for key in ("max_proof_deadline_s", "flops_budget_max", "holdout_size"): + if key in item and (not _is_int(item[key]) or item[key] < 1): + _fail(f"pin_policy.{key} must be an integer >= 1") + for key in ("eval_image_digest", "gpu_class"): + if key in item: + value = item[key] + if not isinstance(value, str) or not value.strip(): + _fail(f"pin_policy.{key} must be a non-empty string") + if len(value) > MAX_PIN_STRING_CHARS: + _fail(f"pin_policy.{key} is longer than {MAX_PIN_STRING_CHARS} chars") + return item + + +# --------------------------------------------------------------------------- +# the set +# --------------------------------------------------------------------------- + + +def build_set(doc: dict[str, Any], previous: dict[str, Any] | None) -> dict[str, Any]: + """The whole set: the RLM's answer, with the prior set's parts retained.""" + topic_id = doc["id"].strip() + prior = previous or {} + if prior and isinstance(prior.get("topic_id"), str) and prior["topic_id"].strip() != topic_id: + _fail( + f"the previous set is for topic {prior['topic_id']!r}, this VM is bound to " + f"{topic_id!r}; it is not this RLM's to retain" + ) + migrations = merge_migrations(derive_migrations(doc), prior.get("migrations")) + for item in migrations: + check_migration(item) + check_migration_scope(item["sql"], topic_id) + apis = merge_apis(derive_apis(doc), prior.get("apis")) + apis = [check_api(item) for item in apis] + return { + "schema_version": AUTHORING_SCHEMA, + "topic_id": topic_id, + "rules": derive_rules(doc), + "migrations": migrations, + "apis": apis, + # **Always derived.** `submission_format` states the intake contract of + # the host this run is executing on — the staged cap, the submit + # domain, the nonce — and a retained copy would be a previous host's + # contract published as the current one. Unlike a migration (which the + # topic still needs and the RLM therefore keeps), this part is a fact + # about the runtime, so it is re-read every run and never inherited. + "submission_format": derive_submission_format(), + "pin_policy": check_pin_policy(derive_pin_policy(doc)), + } + + +def check_set(set_: dict[str, Any], doc: dict[str, Any]) -> None: + """The guest's own gates, before the answer becomes a job output.""" + topic_id = doc["id"].strip() + if set_.get("schema_version") != AUTHORING_SCHEMA: + _fail( + f"authored set schema_version {set_.get('schema_version')!r}, this build writes " + f"{AUTHORING_SCHEMA}" + ) + if str(set_.get("topic_id", "")).strip() != topic_id: + _fail( + f"the set is for topic {set_.get('topic_id')!r}, this VM is bound to {topic_id!r}" + ) + missing = [ + part + for part in ("rules", "migrations", "apis", "submission_format") + if not set_.get(part) + ] + if missing: + _fail( + f"the RLM authored no {', '.join(missing)}: the set is incomplete, so nothing is " + "installed — a topic's behavior is authored by its own RLM (rules, migrations, apis, " + "submission_format, pin_policy)" + ) + if "pin_policy" not in set_: + _fail("the set carries no pin_policy key; an empty policy is an answer, an absent one is not") + rules = set_["rules"] + if len(rules) > MAX_RULES: + _fail(f"the vector carries {len(rules)} rules; at most {MAX_RULES} are applied") + seen: set[str] = set() + for rule in rules: + rid = rule.get("id") if isinstance(rule, dict) else None + text = rule.get("text") if isinstance(rule, dict) else None + if not isinstance(rid, str) or not RULE_ID.match(rid): + _fail(f"rule id {rid!r} must match [a-z0-9][a-z0-9_-]{{1,63}}") + if rid in seen: + _fail(f"rule id {rid!r} appears twice in the vector") + seen.add(rid) + if not isinstance(text, str) or not text.strip(): + _fail(f"rule {rid!r} carries no text") + if len(text) > MAX_RULE_TEXT_CHARS: + _fail(f"rule {rid!r} carries more than {MAX_RULE_TEXT_CHARS} chars") + + +def write_rules_fragment(set_: dict[str, Any], output_dir: Path) -> Path: + """Write the compat `rules.json`: the set's own vector, and nothing else. + + A guest baked before `authoring.json` existed harvests a `propose_rules` + run out of this file and fails the job when it is absent + (``adaptor wrote no rules.json``). Writing the set's rules here as well is + what makes one run answer both guests. + + It is a **copy**, never a second answer: the guest that reads the set + compares the two vectors and refuses a pair that disagrees, because + otherwise which guest harvested the run would decide the topic's + anti-cheat surface. Written first, so a run cut between the two writes + leaves a fragment (which nothing may open a topic on) rather than a set + whose compat copy a stale guest cannot find. + """ + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / RULES_FILE + body = json.dumps(set_["rules"], indent=2, sort_keys=False) + "\n" + _write_atomically(path, body) + return path + + +def _write_atomically(path: Path, body: str) -> None: + """Replace `path` with `body` in one step: a partial file never lands.""" + handle, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}-", suffix=".json") + try: + with os.fdopen(handle, "w", encoding="utf-8") as fh: + fh.write(body) + fh.flush() + os.fsync(fh.fileno()) + os.chmod(tmp, 0o644) + os.replace(tmp, path) + except OSError as e: + try: + os.unlink(tmp) + except OSError: + pass + _fail(f"cannot write {path}: {e}") + + +def write_set(set_: dict[str, Any], output_dir: Path) -> Path: + """Write `authoring.json` in one step: a partial set never lands.""" + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / AUTHORING_FILE + body = json.dumps(set_, indent=2, sort_keys=False) + "\n" + _write_atomically(path, body) + return path + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if argv: + _fail(f"propose_rules takes no arguments (got {' '.join(argv)})") + output_dir = os.environ.get("PROOF_OUTPUT_DIR", "").strip() + if not output_dir: + _fail("PROOF_OUTPUT_DIR is required") + job = os.environ.get("PROOF_JOB", "").strip() + if job != "propose_rules": + _fail(f"PROOF_JOB is {job!r}, not propose_rules") + doc = topic_document() + set_ = build_set(doc, current_set()) + check_set(set_, doc) + # The fragment first, the set second: the pair is one answer, and the + # ordering is what a run cut between the two writes leaves behind. + fragment = write_rules_fragment(set_, Path(output_dir)) + path = write_set(set_, Path(output_dir)) + print( + f"authoring_set: authored {len(set_['rules'])} rules, " + f"{len(set_['migrations'])} migrations, {len(set_['apis'])} routes for " + f"topic {set_['topic_id']} -> {path} (+ compat {fragment.name})", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/propose_rules b/deploy/guest/runners/rlm_fc_in_guest_harbor/propose_rules new file mode 100755 index 000000000..36f7ca9a1 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/propose_rules @@ -0,0 +1,46 @@ +#!/bin/bash +# propose_rules — the topic's RLM authors its **whole** behavior set. +# +# Writes `$PROOF_OUTPUT_DIR/authoring.json`: schema_version 1 plus every part +# the install applies — rules, migrations, apis, submission_format, +# pin_policy. A rules-only answer is a fragment, and the host refuses to open +# a topic on one (`IncompleteAuthoring` / `RULES_ONLY_IS_NOT_AUTHORSHIP`), so +# `authoring.json` is the answer. +# +# It also writes `rules.json` — the set's own `rules` vector, verbatim — as a +# **compat copy**. A guest baked before the set existed harvests a +# `propose_rules` run out of that file and fails the job when it is absent +# (`adaptor wrote no rules.json`), so writing both is what makes one run +# answer both guests. The pair is one answer: a guest that reads the set +# compares the two vectors and refuses a disagreement, so this file must never +# carry a vector the set does not. Fragment first, set second, so a run cut +# between the writes leaves a fragment (which opens nothing). +# +# Where every part comes from is **topic data**, never a list compiled here: +# +# * `$PROOF_TOPIC_FILE` — the signed document the VM is bound to. Its +# `checklist` seeds the rule vector (the RLM may rename, drop, split or +# add rules, and must say so), its `metric` / `eval_executor` / +# `flops_budget` / `holdout_size` are what a `pin_policy` may **restate** +# (never diverge from — scoring reads the document), and its id is the +# namespace every migration must sit inside. +# * `$PROOF_CURRENT_AUTHORING_FILE` — the set this RLM authored last time, +# empty on a first run. Every part not named by a seed is **retained** +# from it, so a re-authoring run is not a rewrite from nothing. +# * `PROOF_PARAM_*` — signed `constraints.params` seeds, one part each +# (see `harness/authoring_set.py` for the key list). A param the topic +# did not sign is absent, and an absent param authors nothing. +# +# An RLM with a model wired can replace `harness/authoring_set.py`'s answer +# for any part (the seeds are inputs to the prompt, not a ceiling on it); the +# parts it does not touch stay the ones above. What this entrypoint must +# never do is echo the operator's `checklist` back as the whole answer. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib.sh +. "$HERE/lib.sh" +: "${PROOF_OUTPUT_DIR:?}" "${PROOF_WORK_DIR:?}" "${PROOF_JOB:?}" +[ "$PROOF_JOB" = propose_rules ] || proof_die "propose_rules entrypoint invoked with PROOF_JOB=$PROOF_JOB" +[ -f "${PROOF_TOPIC_FILE:?}" ] || proof_die "PROOF_TOPIC_FILE is missing" +mkdir -p "$PROOF_OUTPUT_DIR" "$PROOF_WORK_DIR" +exec python3 "$HERE/harness/authoring_set.py" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh index 35f109046..c2be0876c 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/run.sh @@ -6,6 +6,7 @@ python3 "$HERE/test_resolve_agent.py" python3 "$HERE/test_resolve_harness.py" python3 "$HERE/test_summarize.py" python3 "$HERE/test_inspect_scan.py" +python3 "$HERE/test_authoring_set.py" python3 "$HERE/test_filter_tasks.py" python3 "$HERE/test_resolve_model.py" python3 "$HERE/test_rewrite_network.py" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh index 439ff3bac..23c570cae 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh @@ -411,6 +411,71 @@ assert res["n_scored"] == 1 PY pass "evaluate run-harbor passes miner -a and full OpenRouter -m" +# --- the refusal is EARLY: it lands before Harbor is invoked --------------- +# "Fail-closed" is only half the property. The Gate 1 cost was a provision + +# boot + a Harbor run that overran its wall clock, so the refusal has to happen +# in the selection, before any Harbor process exists — not after it. This runs +# the real entrypoint with a fake `harbor` that records that it ran, and +# requires: no record, no jobs dir, and no report. +early_bin="$WORKDIR/early-bin" +mkdir -p "$early_bin" +cat > "$early_bin/harbor" <<'EOF' +#!/bin/bash +set -euo pipefail +printf 'invoked\n' >> "${PROOF_WORK_DIR}/early.harbor-invoked" +jobs="" +while [ $# -gt 0 ]; do + case "$1" in + --jobs-dir) jobs="$2"; shift 2 ;; + *) shift ;; + esac +done +# One complete trial, so the control case (the escape) can summarize. The +# refusal case must never get here at all. +job="$jobs/job1/alpha__1" +mkdir -p "$job/verifier" +printf '%s\n' '{"trial_name": "alpha__1", "verifier_result": {"rewards": {"reward": 1.0}}}' > "$job/result.json" +printf '1.0\n' > "$job/verifier/reward.txt" +EOF +chmod 0755 "$early_bin/harbor" +early_out="$WORKDIR/out-early" +early_work="$WORKDIR/work-early" +rm -rf "$early_out" "$early_work" +mkdir -p "$early_out" "$early_work" +if (export PATH="$early_bin:$PATH" PROOF_JOB=evaluate PROOF_PACK_DIR="$no_slices" \ + PROOF_PARAM_TASKS_DIR=tasks PROOF_TASK_SLICE=tb4-first-5 PROOF_PARAM_TASKS= \ + PROOF_OUTPUT_DIR="$early_out" PROOF_WORK_DIR="$early_work" \ + PROOF_HARNESS_SKIP_PODMAN=1 + "$ADAPTOR/harness/run-harbor") 2>"$early_work/early.err"; then + fail "an unresolved slice must refuse the whole run, not just the filter" +fi +[ ! -f "$early_work/early.harbor-invoked" ] \ + || fail "Harbor was invoked on a refused selection — the refusal is not early" +[ ! -d "$early_work/harbor-jobs" ] \ + || fail "a refused selection must not create a Harbor jobs dir" +[ ! -f "$early_out/report.json" ] || fail "a refused selection must write no report" +grep -q "defines no slices" "$early_work/early.err" \ + || fail "the early refusal must name why: $(cat "$early_work/early.err")" +pass "an unresolved slice refuses before Harbor exists (no run, no jobs dir, no report)" + +# --- the escape runs through the same entrypoint, and DOES reach Harbor ---- +# The negative control: with the set named explicitly the same entrypoint gets +# as far as Harbor. Without this, "Harbor was never invoked" could pass because +# the whole entrypoint was broken. +rm -rf "$early_out" "$early_work" "$PROOF_WORK_DIR/early.harbor-invoked" +mkdir -p "$early_out" "$early_work" +if ! (export PATH="$early_bin:$PATH" PROOF_JOB=evaluate PROOF_PACK_DIR="$no_slices" \ + PROOF_PARAM_TASKS_DIR=tasks PROOF_TASK_SLICE=tb4-first-5 PROOF_PARAM_TASKS=alpha \ + PROOF_ARTIFACT_DIR="$FIXTURES" PROOF_MODEL_PIN="vendor/model" \ + PROOF_OUTPUT_DIR="$early_out" PROOF_WORK_DIR="$early_work" \ + PROOF_HARNESS_SKIP_PODMAN=1 + "$ADAPTOR/harness/run-harbor") 2>"$early_work/escape.err"; then + fail "a named set beside a stale label must run: $(cat "$early_work/escape.err")" +fi +[ -f "$early_work/early.harbor-invoked" ] \ + || fail "the escape must reach Harbor (the early test's control)" +pass "the explicit-set escape reaches Harbor with the stale label still set" + # --- n_concurrent is topic data, passed to Harbor verbatim ------------------- # The signed value is honored as-is (no clamp, no ceiling): a topic that asks # for 5 gets 5, and a topic that asks for nothing gets Harbor's own default of diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_authoring_set.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_authoring_set.py new file mode 100644 index 000000000..a12a8b98b --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_authoring_set.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python3 +"""authoring_set.py: the RLM authors its whole set, or it refuses. + +What is pinned here, in the order the failure modes matter: + +* a complete set — every one of the five parts present and shaped the way the + guest and the install hold them (the same `proof-topic-authoring` checks); +* **the compat fragment**: `authoring.json` is the authorship and `rules.json` + is the set's own vector written beside it, so a guest baked before the set + existed harvests the run instead of failing it — and a missing part is still + a refusal naming the part; +* the operator's bundle is not the source of truth: the vector is framed by + the RLM, the migration sits in the topic's namespace, the pin policy + **restates** the document, and a policy that would diverge is not written; +* re-authoring retains what it is not changing, and a previous set for another + topic is refused; +* a declared rule with no signed policy, and a marker policy with no declared + rule, are both refusals — never a silent drop or an invented check. + +No guest agent, no VM, no Harbor: `main()` is driven directly with the +environment contract the guest exports for `ProposeRules`. +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent / "harness")) +import authoring_set # noqa: E402 + +TOPIC_ID = "fixture-topic-v0" +MARKERS = "no_short_circuit:skip_eval|skip_verifier;no_answer_table:answer_key" +ATTESTED = "miner_pays_provider,same_seed" + + +def topic_document(**overrides: object) -> dict: + doc = { + "schema_version": 1, + "id": TOPIC_ID, + "statement": "score the pinned pack with the pinned runner", + "status": "draft", + "constraints": { + "firecracker_required": True, + "model_pin": "vendor/model", + "params": { + "baseline_runner": "rlm_fc_in_guest_harbor", + "experiment_pack_digest": "sha256:" + "ab" * 32, + "tasks_dir": "tasks", + }, + }, + "metric": { + "family": "custom", + "primary": "success_rate", + "direction": "max", + "epsilon_rel": 0.05, + "custom_id": "fixture_metric", + }, + "checklist": [ + {"id": "no_short_circuit", "text": "the evaluator and the metric path are untouched"}, + {"id": "no_answer_table", "text": "no hardcoded answers for the scored set"}, + {"id": "miner_pays_provider", "text": "the miner pays for its own provider calls"}, + ], + "epsilon_nll": 0.02, + "epsilon_topic_max_regress": 0.05, + "flops_budget": 2_000_000_000_000_000_000, + "holdout_size": 120, + "eval_executor": {"max_proof_deadline_s": 3600}, + } + doc.update(overrides) + return doc + + +class Harness: + """One `main()` run: a temp root, the guest's env, the parsed answer.""" + + def __init__(self, doc: dict, previous: dict | None = None, markers=MARKERS, attested=ATTESTED): + self.root = Path(tempfile.mkdtemp(prefix="authoring-set-")) + self.topic = self.root / "topic.json" + self.topic.write_text(json.dumps(doc), encoding="utf-8") + self.output = self.root / "output" + self.work = self.root / "work" + self.output.mkdir() + self.work.mkdir() + env = { + "PROOF_JOB": "propose_rules", + "PROOF_TOPIC_ID": doc.get("id", ""), + "PROOF_CUSTOM_ID": (doc.get("metric") or {}).get("custom_id", ""), + "PROOF_TOPIC_FILE": str(self.topic), + "PROOF_OUTPUT_DIR": str(self.output), + "PROOF_WORK_DIR": str(self.work), + "PROOF_CURRENT_RULES_VERSION": "", + } + if previous is not None: + path = self.work / authoring_set.CURRENT_AUTHORING_FILE + path.write_text(json.dumps(previous), encoding="utf-8") + env["PROOF_CURRENT_AUTHORING_FILE"] = str(path) + else: + env["PROOF_CURRENT_AUTHORING_FILE"] = "" + if markers is not None: + env[authoring_set.PARAM_MARKER_RULES] = markers + if attested is not None: + env[authoring_set.PARAM_ATTESTED_RULES] = attested + self.env = env + + def run(self) -> int: + saved = {k: os.environ.get(k) for k in self.env} + os.environ.update(self.env) + try: + return authoring_set.main([]) + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + def set_path(self) -> Path: + return self.output / authoring_set.AUTHORING_FILE + + def authored(self) -> dict: + return json.loads(self.set_path().read_text(encoding="utf-8")) + + def cleanup(self) -> None: + import shutil + + shutil.rmtree(self.root, ignore_errors=True) + + +class CompleteSet(unittest.TestCase): + def test_a_complete_set_carries_every_part_the_install_applies(self): + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + set_ = h.authored() + self.assertEqual(set_["schema_version"], 1) + self.assertEqual(set_["topic_id"], TOPIC_ID) + # The five parts, each present and non-empty: what the install reads. + self.assertEqual( + [r["id"] for r in set_["rules"]], + ["no_short_circuit", "no_answer_table", "miner_pays_provider"], + ) + self.assertTrue(set_["migrations"], "a complete set carries a migration") + self.assertTrue(set_["apis"], "a complete set carries a route") + self.assertTrue(set_["submission_format"]) + self.assertIn("pin_policy", set_) + # `deny_unknown_fields`: the parts this build reads are exactly these. + self.assertEqual( + sorted(set_), + [ + "apis", + "migrations", + "pin_policy", + "rules", + "schema_version", + "submission_format", + "topic_id", + ], + ) + + def test_the_entrypoint_writes_the_set_and_the_compat_fragment(self): + """The set is authorship; `rules.json` is its compat copy, not a second answer.""" + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + self.assertTrue(h.set_path().is_file()) + fragment = h.output / "rules.json" + self.assertTrue( + fragment.is_file(), + "a guest baked before the set existed harvests this run out of rules.json and " + "fails the job when it is absent (`adaptor wrote no rules.json`)", + ) + # The fragment is the set's own vector, verbatim: one answer, two files. + self.assertEqual( + json.loads(fragment.read_text(encoding="utf-8")), + h.authored()["rules"], + "the compat copy must be the set's rules, or a stale guest would score a " + "different anti-cheat surface than the set's own", + ) + + def test_the_fragment_is_written_before_the_set(self): + """A run cut between the writes leaves a fragment, not a set alone. + + The ordering is the fail-closed half of the dual emit: a fragment is + recorded honestly and refused downstream (`IncompleteAuthoring`), + while a set with no compat copy is the live FAIL (a guest baked before + the set existed fails the job with `adaptor wrote no rules.json`). + """ + order: list[str] = [] + real_fragment = authoring_set.write_rules_fragment + real_set = authoring_set.write_set + + def fragment(set_, output_dir): + order.append(authoring_set.RULES_FILE) + return real_fragment(set_, output_dir) + + def set_(set_dict, output_dir): + order.append(authoring_set.AUTHORING_FILE) + return real_set(set_dict, output_dir) + + authoring_set.write_rules_fragment = fragment + authoring_set.write_set = set_ + try: + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + finally: + authoring_set.write_rules_fragment = real_fragment + authoring_set.write_set = real_set + self.assertEqual( + order, + [authoring_set.RULES_FILE, authoring_set.AUTHORING_FILE], + "the fragment must be written before the set, so a run cut between the two " + "writes leaves a fragment (which opens nothing) rather than a set whose compat " + "copy a stale guest cannot find", + ) + # And both files are on disk with the set's own vector in the copy. + self.assertTrue((h.output / "rules.json").is_file()) + self.assertEqual( + json.loads((h.output / "rules.json").read_text(encoding="utf-8")), + h.authored()["rules"], + ) + + def test_the_rules_are_the_rlms_framing_with_the_declaration_quoted(self): + """The operator's sentence is the declaration, not the whole answer.""" + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + rules = {r["id"]: r["text"] for r in h.authored()["rules"]} + marker_text = rules["no_short_circuit"] + self.assertIn("2 off-limits markers", marker_text) + self.assertIn("rlm: artefact scan", marker_text) + self.assertIn("the topic's declaration:", marker_text) + attested_text = rules["miner_pays_provider"] + self.assertIn("rlm: host/topic attestation", attested_text) + self.assertIn("the topic's declaration:", attested_text) + # Not a verbatim echo: the RLM says what it will prove, and how. + for rid, text in rules.items(): + declared = next(r for r in topic_document()["checklist"] if r["id"] == rid) + self.assertNotEqual( + text.strip(), + declared["text"].strip(), + f"{rid} is the operator's sentence alone, which is a restatement not authorship", + ) + + def test_the_migration_is_inside_the_topics_own_namespace(self): + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + migrations = h.authored()["migrations"] + self.assertEqual(len(migrations), 1) + self.assertTrue(migrations[0]["name"], "a migration carries a name") + prefix = TOPIC_ID.replace("-", "_") + self.assertIn(f"{prefix}_", migrations[0]["sql"]) + for denied in ("proof_", "pg_catalog", "_sqlx_migrations"): + self.assertNotIn(denied, migrations[0]["sql"].lower()) + + def test_the_route_is_relative_and_outside_the_admin_namespace(self): + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + for route in h.authored()["apis"]: + self.assertFalse(route["path"].startswith("/"), "a topic route is relative") + self.assertNotEqual(route["path"].split("/")[0], "v1") + self.assertIn(route["method"], ("GET", "POST", "PUT", "PATCH", "DELETE", "*")) + + def test_the_pin_policy_restates_the_document_and_invents_nothing(self): + """Scoring reads the document, so the policy restates it exactly.""" + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + policy = h.authored()["pin_policy"] + doc = topic_document() + self.assertEqual(policy["epsilon_nll_min"], doc["epsilon_nll"]) + self.assertEqual( + policy["epsilon_topic_max_regress_min"], doc["epsilon_topic_max_regress"] + ) + self.assertEqual(policy["epsilon_throughput_rel_min"], doc["metric"]["epsilon_rel"]) + self.assertEqual(policy["max_proof_deadline_s"], 3600) + self.assertEqual(policy["flops_budget_max"], doc["flops_budget"]) + self.assertEqual(policy["holdout_size"], doc["holdout_size"]) + # The two pin equalities the VM cannot read are absent, never invented. + self.assertNotIn("eval_image_digest", policy) + self.assertNotIn("gpu_class", policy) + + def test_the_submission_format_is_the_hosts_real_intake_shape(self): + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + fmt = h.authored()["submission_format"] + self.assertEqual(fmt["max_bytes"], authoring_set.MAX_ARTIFACT_BYTES) + self.assertEqual(fmt["signature_domain"], authoring_set.SUBMIT_DOMAIN) + self.assertIn("submit_nonce", json.dumps(fmt)) + + def test_a_knob_the_document_does_not_declare_is_not_authored(self): + """A policy restates; an absent knob is absent, never guessed.""" + doc = topic_document() + doc.pop("epsilon_topic_max_regress") + doc["eval_executor"] = {} + h = Harness(doc) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + policy = h.authored()["pin_policy"] + self.assertNotIn("epsilon_topic_max_regress_min", policy) + self.assertNotIn("max_proof_deadline_s", policy) + self.assertIn("epsilon_nll_min", policy) + + +class Refusals(unittest.TestCase): + def test_a_declared_rule_with_no_signed_policy_is_refused_by_name(self): + """Neither an invented check nor a silent drop: the topic must say.""" + h = Harness(topic_document(), markers="no_short_circuit:skip_eval", attested="") + self.addCleanup(h.cleanup) + with self.assertRaises(SystemExit) as ctx: + h.run() + self.assertEqual(ctx.exception.code, 2) + self.assertFalse(h.set_path().exists(), "a refused run writes no set") + self.assertFalse( + (h.output / "rules.json").exists(), + "a refused run writes no fragment either: the refusal is decided before either " + "file is written, so a partial answer never lands", + ) + + def test_no_rule_policy_at_all_is_refused(self): + h = Harness(topic_document(), markers=None, attested=None) + self.addCleanup(h.cleanup) + with self.assertRaises(SystemExit): + h.run() + self.assertFalse(h.set_path().exists()) + + def test_a_marker_policy_for_an_undeclared_rule_is_refused(self): + h = Harness(topic_document(), markers=MARKERS + ";never_declared:oops", attested=ATTESTED) + self.addCleanup(h.cleanup) + with self.assertRaises(SystemExit): + h.run() + self.assertFalse(h.set_path().exists()) + + def test_a_rule_named_as_both_marker_and_attested_is_refused(self): + h = Harness(topic_document(), markers=MARKERS, attested=ATTESTED + ",no_short_circuit") + self.addCleanup(h.cleanup) + with self.assertRaises(SystemExit): + h.run() + + def test_an_empty_checklist_is_refused(self): + h = Harness(topic_document(checklist=[])) + self.addCleanup(h.cleanup) + with self.assertRaises(SystemExit): + h.run() + self.assertFalse(h.set_path().exists()) + + def test_the_wrong_job_is_refused(self): + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + h.env["PROOF_JOB"] = "evaluate" + with self.assertRaises(SystemExit): + h.run() + + def test_a_missing_topic_file_is_refused(self): + h = Harness(topic_document()) + self.addCleanup(h.cleanup) + h.env["PROOF_TOPIC_FILE"] = str(h.root / "nope.json") + with self.assertRaises(SystemExit): + h.run() + + def test_a_previous_set_for_another_topic_is_refused(self): + prior = {"schema_version": 1, "topic_id": "someone-elses-topic"} + h = Harness(topic_document(), previous=prior) + self.addCleanup(h.cleanup) + with self.assertRaises(SystemExit): + h.run() + self.assertFalse(h.set_path().exists()) + + def test_a_migration_outside_the_namespace_is_refused(self): + """The scope check refuses what the install's deny-list would.""" + with self.assertRaises(SystemExit): + authoring_set.check_migration_scope("DROP TABLE proof_rule_version", TOPIC_ID) + with self.assertRaises(SystemExit): + authoring_set.check_migration_scope("SELECT * FROM other_topic_rows", TOPIC_ID) + with self.assertRaises(SystemExit): + authoring_set.check_migration_scope("SELECT * FROM pg_catalog.pg_class", TOPIC_ID) + # The topic's own objects pass, in both accepted spellings. An index + # name is an object too: the guard scopes it the same way. + prefix = TOPIC_ID.replace("-", "_") + authoring_set.check_migration_scope( + f"CREATE TABLE {prefix}_scratch (id TEXT)", TOPIC_ID + ) + authoring_set.check_migration_scope( + f"CREATE INDEX {prefix}_scratch_idx ON {prefix}_scratch (id)", TOPIC_ID + ) + # The literal hyphen id is one legal **quoted** identifier, not three + # unscoped words (the guard's own `tokens` keeps a quoted run whole). + authoring_set.check_migration_scope( + f'CREATE TABLE "{TOPIC_ID}_scratch" (id TEXT)', TOPIC_ID + ) + with self.assertRaises(SystemExit): + authoring_set.check_migration_scope( + f"CREATE INDEX unscoped_idx ON {prefix}_scratch (id)", TOPIC_ID + ) + + def test_a_malformed_route_is_refused(self): + with self.assertRaises(SystemExit): + authoring_set.check_api({"path": "/absolute", "method": "GET"}) + with self.assertRaises(SystemExit): + authoring_set.check_api({"path": "v1/admin/exec", "method": "GET"}) + with self.assertRaises(SystemExit): + authoring_set.check_api({"path": "status", "method": "TRACE"}) + with self.assertRaises(SystemExit): + authoring_set.check_api({"path": "a/../b", "method": "GET"}) + # A method is normalised the way the install reads it. + self.assertEqual( + authoring_set.check_api({"path": "status", "method": "get"})["method"], "GET" + ) + + def test_a_malformed_pin_policy_is_refused(self): + with self.assertRaises(SystemExit): + authoring_set.check_pin_policy({"epsilon_nll_min": 0}) + with self.assertRaises(SystemExit): + authoring_set.check_pin_policy({"epsilon_nll_min": 1.5}) + with self.assertRaises(SystemExit): + authoring_set.check_pin_policy({"max_proof_deadline_s": 0}) + with self.assertRaises(SystemExit): + authoring_set.check_pin_policy({"not_a_knob": 1}) + # `{}` is an answer: the RLM saying this topic tightens nothing. + self.assertEqual(authoring_set.check_pin_policy({}), {}) + + +class ReAuthoring(unittest.TestCase): + def previous_set(self) -> dict: + return { + "schema_version": 1, + "topic_id": TOPIC_ID, + "rules": [{"id": "stale_rule", "text": "a rule the re-signed topic dropped"}], + "migrations": [ + { + "name": "0001_scratch", + "sql": f"CREATE TABLE {TOPIC_ID.replace('-', '_')}_scratch (id TEXT)", + }, + {"name": "0002_kept", "sql": f"CREATE TABLE {TOPIC_ID.replace('-', '_')}_kept (id TEXT)"}, + ], + "apis": [ + {"path": "status", "method": "GET", "summary": "the old summary"}, + {"path": "kept", "method": "GET", "summary": "a route still needed"}, + ], + "submission_format": {"kind": "tar", "max_bytes": 1}, + "pin_policy": {"epsilon_nll_min": 0.02}, + } + + def test_a_re_authoring_run_retains_the_parts_it_is_not_changing(self): + h = Harness(topic_document(), previous=self.previous_set()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + set_ = h.authored() + names = [m["name"] for m in set_["migrations"]] + self.assertIn("0001_scratch", names, "a migration the topic still needs must survive") + self.assertIn("0002_kept", names, "a retained migration is kept") + self.assertIn("0001_rlm_state", names, "the RLM's current answer is added") + self.assertEqual( + names[: len(self.previous_set()["migrations"])], + ["0001_scratch", "0002_kept"], + "prior order is preserved", + ) + routes = [(a["path"], a["method"]) for a in set_["apis"]] + self.assertIn(("kept", "GET"), routes) + self.assertIn(("status", "GET"), routes) + # The current answer wins for the route it names. + status = next(a for a in set_["apis"] if a["path"] == "status") + self.assertNotEqual(status["summary"], "the old summary") + + def test_a_retained_migration_is_still_scope_checked(self): + """Retention is not a bypass: a prior set is not a trusted input.""" + prior = self.previous_set() + prior["migrations"].append( + {"name": "0003_evil", "sql": "DROP TABLE proof_rule_version"} + ) + h = Harness(topic_document(), previous=prior) + self.addCleanup(h.cleanup) + with self.assertRaises(SystemExit): + h.run() + self.assertFalse(h.set_path().exists()) + + def test_the_submission_format_is_re_derived_not_retained(self): + """A retained format would publish a previous host's intake contract. + + Greptile P1: a populated prior `submission_format` was copied into the + new set, so a re-authoring run published the old contract (here a + 1-byte cap) as the current one. + """ + prior = self.previous_set() + prior["submission_format"] = { + "kind": "tar", + "max_bytes": 1, + "stale_marker": "prior-host-contract", + } + h = Harness(topic_document(), previous=prior) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + fmt = h.authored()["submission_format"] + self.assertNotEqual(fmt, prior["submission_format"], "the stale contract was retained") + self.assertNotIn("stale_marker", fmt) + self.assertEqual(fmt["max_bytes"], authoring_set.MAX_ARTIFACT_BYTES) + self.assertEqual(fmt["signature_domain"], authoring_set.SUBMIT_DOMAIN) + self.assertEqual(fmt, authoring_set.derive_submission_format()) + + def test_a_retained_topic_scoped_delete_is_not_refused(self): + """`DELETE FROM _table` is in the namespace, not a touch on FROM. + + Greptile P1: the scan skipped modifiers after a table keyword but not + `FROM`, so a retained `DELETE FROM fixture_topic_v0_kept …` was refused + with "migration touches 'FROM'". + """ + prior = self.previous_set() + prefix = TOPIC_ID.replace("-", "_") + prior["migrations"].append( + { + "name": "0003_prune", + "sql": f"DELETE FROM {prefix}_kept WHERE id = 'retained-row'", + } + ) + h = Harness(topic_document(), previous=prior) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + names = [m["name"] for m in h.authored()["migrations"]] + self.assertIn("0003_prune", names, "the retained DELETE migration must survive") + # And the scope check itself, directly, for both spellings. + authoring_set.check_migration_scope( + f"DELETE FROM {prefix}_kept WHERE id = 'x'", TOPIC_ID + ) + authoring_set.check_migration_scope( + f'DELETE FROM "{TOPIC_ID}_kept" WHERE id = \'x\'', TOPIC_ID + ) + authoring_set.check_migration_scope( + f"DELETE FROM {prefix}_kept USING {prefix}_other WHERE 1 = 1", TOPIC_ID + ) + authoring_set.check_migration_scope( + f"UPDATE {prefix}_kept SET value = 'x' WHERE id = 'y'", TOPIC_ID + ) + authoring_set.check_migration_scope( + f"TRUNCATE {prefix}_kept", TOPIC_ID + ) + # A DELETE that reaches a sibling is still refused. + with self.assertRaises(SystemExit): + authoring_set.check_migration_scope("DELETE FROM other_topic_rows", TOPIC_ID) + with self.assertRaises(SystemExit): + authoring_set.check_migration_scope( + "DELETE FROM proof_rule_version WHERE id = 'x'", TOPIC_ID + ) + + def test_the_rules_and_the_policy_are_re_derived_not_retained(self): + """A retained rule or policy could contradict the re-signed document.""" + h = Harness(topic_document(), previous=self.previous_set()) + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + set_ = h.authored() + self.assertNotIn( + "stale_rule", + [r["id"] for r in set_["rules"]], + "a rule the re-signed topic dropped must not be retained", + ) + self.assertNotEqual(set_["pin_policy"], {"epsilon_nll_min": 0.02}) + self.assertEqual(set_["pin_policy"]["epsilon_nll_min"], 0.02) + self.assertIn("max_proof_deadline_s", set_["pin_policy"]) + + +class Bounds(unittest.TestCase): + def test_the_rule_text_stays_within_the_signed_cap(self): + long_text = "x" * 4000 + doc = topic_document( + checklist=[{"id": "no_short_circuit", "text": long_text}] + ) + h = Harness(doc, markers="no_short_circuit:skip_eval", attested="") + self.addCleanup(h.cleanup) + self.assertEqual(h.run(), 0) + text = h.authored()["rules"][0]["text"] + self.assertLessEqual(len(text), authoring_set.MAX_RULE_TEXT_CHARS) + + def test_too_many_rules_is_refused(self): + ids = [f"rule_{i:02d}" for i in range(authoring_set.MAX_RULES + 1)] + doc = topic_document(checklist=[{"id": rid, "text": "a rule"} for rid in ids]) + markers = ";".join(f"{rid}:marker" for rid in ids) + h = Harness(doc, markers=markers, attested="") + self.addCleanup(h.cleanup) + with self.assertRaises(SystemExit): + h.run() + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/deploy/scripts/proof-slice-preflight.sh b/deploy/scripts/proof-slice-preflight.sh index 7e25cd948..52e185206 100755 --- a/deploy/scripts/proof-slice-preflight.sh +++ b/deploy/scripts/proof-slice-preflight.sh @@ -176,13 +176,38 @@ note "kept: $names" echo # A slice that was set but did not resolve is the LIVE Gate 1 failure. The guest -# refuses it now, so reaching here with a label means it resolved — assert it. -if [ -n "$task_slice" ]; then - case "$resolved" in - true) pass "task_slice '$task_slice' resolved through the pack (resolved=true)" ;; - *) die "task_slice '$task_slice' did not resolve (resolved=$resolved) — the guest would refuse this run" ;; - esac -fi +# refuses it now — **unless** the topic also named the set explicitly, in which +# case `params.tasks` is the documented escape and the label is not read at all +# (`filter_tasks.select_base` returns before resolving it). Refusing here would +# block the very fix this preflight is meant to prove, so the assertion follows +# the guest: a label that did not resolve is a refusal only when it was the +# selector. +# +# "Was it the selector" is read from the **summary**, never from these shell +# variables: the guest normalizes a whitespace-only value as absent +# (`present()` trims, then rejects empty), so `--tasks " "` beside a slice +# selects the slice. Testing `-n "$tasks"` here would claim `params.tasks` +# named the set — the wrong explanation, on a run that took a different branch. +# `source` is what the guest actually used; that is what the operator is told. +case "$source" in + "params.tasks") + # The tasks decided it; the label (resolved or not) was not read. + [ -z "$task_slice" ] \ + || pass "task_slice '$task_slice' is not read: params.tasks named the set (source=$source)" + ;; + "pack slice "*) + # The label was the selector and it resolved. + pass "task_slice '$task_slice' resolved through the pack (resolved=$resolved)" + ;; + *) + # The label was the selector and it did not resolve: the LIVE refusal. + # The guest would already have refused above, so this is unreachable — + # asserted rather than assumed, because reaching it means the guest and + # this preflight disagree about what the topic asked for. + [ -z "$task_slice" ] \ + || die "task_slice '$task_slice' did not resolve (resolved=$resolved, source=$source) — the guest would refuse this run" + ;; +esac if [ -n "$expect" ]; then [ "$kept" = "$expect" ] \ diff --git a/deploy/scripts/test_proof_slice_preflight.sh b/deploy/scripts/test_proof_slice_preflight.sh index 09ed7ea99..ce8ff3342 100755 --- a/deploy/scripts/test_proof_slice_preflight.sh +++ b/deploy/scripts/test_proof_slice_preflight.sh @@ -70,6 +70,42 @@ out="$("$PREFLIGHT" --pack-dir "$no_slices" --tasks t-one,t-two --n-tasks 2 --ex grep -q "t-one,t-two" <<<"$out" || fail "the explicit set must be what is kept: $out" pass "an explicit params.tasks set resolves on a slice-less pack" +# --- the escape WITH a stale label still set: the label is not read ---------- +# The Owner's migration shape: a topic carries an old `task_slice` it cannot +# resolve *and* names the set explicitly. The guest reads `params.tasks` and +# never resolves the label, so the run is correct — but this preflight refused +# it, which blocks the very fix it exists to prove. Assert the guest's own +# answer, both directions, so the two cannot drift apart again. +out="$("$PREFLIGHT" --pack-dir "$no_slices" --task-slice five --tasks t-one,t-two --expect 2 2>&1)" \ + || fail "a stale label beside an explicit set must not be refused: $out" +grep -q "is not read: params.tasks named the set" <<<"$out" \ + || fail "the preflight must say the label is not read: $out" +pass "a stale label beside params.tasks passes (the guest reads the tasks)" +# …and the label alone is still the LIVE refusal: the escape is the tasks, not +# the label. +if out="$("$PREFLIGHT" --pack-dir "$no_slices" --task-slice five 2>&1)"; then + fail "the label alone must still be refused, got: $out" +fi +pass "the stale label alone is still refused (the escape is the explicit set)" + +# --- a whitespace-only selector is ABSENT, not a selection ------------------- +# The guest normalizes a whitespace-only value as absent (`present()` trims, +# then rejects empty), so `--tasks " "` beside a slice selects the **slice**. +# Reading the raw shell variables instead of the summary made the preflight +# claim "params.tasks named the set" on a run that took the slice branch — +# the wrong explanation for the operator, on the exact shape a re-sign +# produces. Both directions, against the guest's own answer. +out="$("$PREFLIGHT" --pack-dir "$with_slices" --tasks " " --task-slice five --expect 5 2>&1)" \ + || fail "a whitespace-only --tasks must fall through to the slice: $out" +grep -q "task_slice 'five' resolved through the pack" <<<"$out" \ + || fail "the slice must be reported as the selector, not params.tasks: $out" +pass "a whitespace-only --tasks beside a slice selects the slice (guest semantics)" +out="$("$PREFLIGHT" --pack-dir "$no_slices" --task-slice " " --tasks t-one,t-two --expect 2 2>&1)" \ + || fail "a whitespace-only --task-slice must not be read as a label: $out" +grep -q "is not read" <<<"$out" \ + || fail "with the tasks naming the set, no label should be reported: $out" +pass "a whitespace-only --task-slice beside params.tasks is not read" + # --- a count that does not match is caught, not silently accepted ------------ if out="$("$PREFLIGHT" --pack-dir "$with_slices" --task-slice five --expect 3 2>&1)"; then fail "--expect must fail when the selection is a different size, got: $out" diff --git a/docs/PROOF.md b/docs/PROOF.md index b19bc3748..b6016ab83 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -881,7 +881,7 @@ test instead of quietly outdating this paragraph. | `constraints.firecracker_required` | Miner code runs only inside a Firecracker guest under the topic VM | | `constraints.model_pin` | `vendor/model[:tag]` every paid call must name (shape-checked only; `proof-canon` rejects `a/b/c`) | | `constraints.params.model` | Harbor / LiteLLM id (`openrouter/vendor/model`). The guest injects it as `PROOF_PARAM_MODEL`. Canon `model_pin` stays two-segment; do not put the LiteLLM id in `model_pin` | -| `constraints.task_slice` | Opaque label the runner interprets; the control plane does not. The Harbor reference adaptor resolves it **through the pinned pack** (`slices/