From 87e320249d9a92d0eb6f5b2ef08e6d3e42ee3f55 Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:08:37 +0000 Subject: [PATCH 1/8] feat(proof): publish through the gateway without a rewrite proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `proof-admin topic install` publishes to `POST /challenge/proof/v1/admin/proof/topics`, and both hops refused it: the gateway answered 403 for every `/v1/admin/*` path, and the challenge's own topic mux swallowed `/challenge/proof/...` as a topic id it holds no routes for (404 `topic_route_not_registered`). The operator worked around it with an ephemeral rewrite proxy in front of the gateway. Three changes, each narrow: - The gateway forwards exactly one admin route: `POST` on the publish path, for the Proof challenge id, with a bearer present (presence only — the gateway never holds the operator token; the challenge still compares the hash). Every other admin route, the topic-id form, and a `GET` of the publish route keep their 403. - The challenge serves its own routes under its own prefix (`/challenge/proof/…`), so the same publish URL works against the gateway and against the challenge service directly. A topic id is unaffected: `proof` resolves to the static prefix, not to the mux. - A topic may not claim a path inside the challenge's admin namespace (`v1/admin…`): refused at install time, and never resolved by the mux, so a row written before this rule cannot answer like an operator route either. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- crates/gateway/src/proxy.rs | 173 +++++++++++++++++- crates/gateway/tests/proxy_rr.rs | 89 +++++++++ crates/proof-challenge/src/topic_routes.rs | 96 +++++++++- crates/proof-topic-install/src/routes.rs | 10 +- crates/proof-topic-install/src/section.rs | 69 +++++++ .../proof-topic-install/tests/topic_routes.rs | 43 +++++ 6 files changed, 471 insertions(+), 9 deletions(-) diff --git a/crates/gateway/src/proxy.rs b/crates/gateway/src/proxy.rs index e475fcefd..3c85a28ea 100644 --- a/crates/gateway/src/proxy.rs +++ b/crates/gateway/src/proxy.rs @@ -71,12 +71,9 @@ async fn proxy_inner( req: Request, ) -> Response { let method = req.method().clone(); - if is_admin_path(&rest) { - return ( - StatusCode::FORBIDDEN, - "admin API is not exposed via gateway; use master-local challenge port", - ) - .into_response(); + let headers = req.headers().clone(); + if let Some(refusal) = admin_gate(&method, &challenge_id, &rest, &headers) { + return refusal; } if is_blocked_report_read(&method, &rest) { return ( @@ -85,7 +82,6 @@ async fn proxy_inner( ) .into_response(); } - let headers = req.headers().clone(); let body = match axum::body::to_bytes(req.into_body(), PROXY_MAX_BODY_BYTES).await { Ok(b) => b, Err(e) => { @@ -184,6 +180,43 @@ async fn proxy_inner( } } +/// The admin gate: `None` when the request may proceed, a refusal otherwise. +/// +/// Every `/v1/admin/*` path is master-local except the one operator route +/// [`is_forwardable_admin_route`] names (the publish the install CLI calls), +/// which additionally needs a bearer **at the gateway**: the challenge +/// compares the token hash, but an anonymous `POST` should not even cost a +/// hop. A topic id never reaches the admin surface either way. +fn admin_gate( + method: &Method, + challenge_id: &str, + rest: &str, + headers: &HeaderMap, +) -> Option { + if !is_admin_path(rest) { + return None; + } + if !is_forwardable_admin_route(method, challenge_id, rest) { + return Some( + ( + StatusCode::FORBIDDEN, + "admin API is not exposed via gateway; use master-local challenge port", + ) + .into_response(), + ); + } + if !has_operator_bearer(headers) { + return Some( + ( + StatusCode::UNAUTHORIZED, + "the operator publish route needs an `authorization: Bearer ` header", + ) + .into_response(), + ); + } + None +} + /// Join base URL, remaining path, and optional query. #[must_use] pub fn upstream_url(base: &str, rest: &str, query: Option<&str>) -> String { @@ -278,6 +311,51 @@ pub fn is_admin_path(rest: &str) -> bool { n.starts_with("v1/admin/") || n == "v1/admin" } +/// The **one** admin route the gateway forwards: the operator publish. +/// +/// `proof-admin topic install` publishes a signed document with +/// `POST /challenge/proof/v1/admin/proof/topics` (`proof_topic_bundle:: +/// PUBLISH_PATH`). The route is operator-authenticated at the challenge +/// (`admin_hashes`, the same bearer the master-local routes use), so the +/// gateway forwards it instead of refusing it — otherwise an operator on +/// staging needs a rewrite proxy in front of the gateway, which is exactly +/// the hop this replaces. Two gates still hold here: +/// +/// - the challenge id must be the **Proof challenge**, not a topic id, so +/// `/challenge/{topic_id}/v1/admin/…` keeps its 403 (a topic's own routes +/// are never a way to reach the admin surface); and +/// - the path must be the publish route **exactly**, after the same +/// normalization [`is_admin_path`] uses, so `v1/admin/proof/queue/drain` +/// and every future admin route stay master-local until one is named here. +/// +/// The method is pinned to `POST`: a read of the publish route has no +/// meaning (the document is in `proof_topic_version`), and a `GET` stays a +/// 403 like the rest of the admin surface. +#[must_use] +pub fn is_forwardable_admin_route(method: &Method, challenge_id: &str, rest: &str) -> bool { + *method == Method::POST + && challenge_id == gateway_core::topic_routes::PROOF_CHALLENGE_ID + && normalize_proxy_path(rest) == PUBLISH_ADMIN_PATH +} + +/// The publish route, relative to the challenge (no leading slash): the path +/// `proof_topic_bundle::PUBLISH_PATH` carries after the challenge prefix. +pub const PUBLISH_ADMIN_PATH: &str = "v1/admin/proof/topics"; + +/// Whether the request carries an operator bearer at all. +/// +/// Presence only: the gateway does not hold the operator token and must not +/// learn it. The challenge compares the hash; this is the cheap floor that +/// keeps an anonymous `POST` from reaching the admin route. +#[must_use] +pub fn has_operator_bearer(headers: &HeaderMap) -> bool { + headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|raw| raw.strip_prefix("Bearer ").or(Some(raw))) + .is_some_and(|token| !token.trim().is_empty()) +} + /// Report bodies are operator-local. POST submit stays on the miner path. /// /// HEAD is a read: Axum would otherwise map it onto the GET handler and leak @@ -399,6 +477,87 @@ mod tests { assert!(!is_admin_path("v1/not-admin/rounds/1/winners")); } + /// Exactly one admin route is forwarded, and only for the Proof + /// challenge: the operator publish the install CLI calls. + #[test] + fn only_the_operator_publish_route_is_forwarded() { + // The CLI's path, as the gateway sees it (the challenge id is + // stripped, so this is the challenge-relative form). + assert!(is_forwardable_admin_route( + &Method::POST, + "proof", + "v1/admin/proof/topics" + )); + // Normalized the same way the 403 gate is, so a client that collapses + // `.` cannot slip a different route past the allowlist. + assert!(is_forwardable_admin_route( + &Method::POST, + "proof", + "v1/./admin/proof/topics" + )); + assert!(is_forwardable_admin_route( + &Method::POST, + "proof", + "v1/admin/../admin/proof/topics" + )); + + // A topic id is never a way to reach the admin surface. + assert!(!is_forwardable_admin_route( + &Method::POST, + "tb4", + "v1/admin/proof/topics" + )); + // Another challenge's admin surface is not this route. + assert!(!is_forwardable_admin_route( + &Method::POST, + "bounty", + "v1/admin/proof/topics" + )); + // Every other admin route stays master-local. + for rest in [ + "v1/admin", + "v1/admin/proof/executor", + "v1/admin/proof/queue/drain", + "v1/admin/proof/vm-orchestrator", + "v1/admin/proof/submissions/pf/score", + "v1/admin/proof/topics/extra", + ] { + assert!( + !is_forwardable_admin_route(&Method::POST, "proof", rest), + "{rest:?} must stay master-local" + ); + } + // The publish route is a POST; a read of it is not forwarded either. + assert!(!is_forwardable_admin_route( + &Method::GET, + "proof", + "v1/admin/proof/topics" + )); + } + + /// The forwarded route still needs a bearer: the gateway never holds the + /// operator token, it only refuses an anonymous call before the hop. + #[test] + fn the_operator_bearer_floor_is_presence_only() { + let mut headers = HeaderMap::new(); + assert!(!has_operator_bearer(&headers)); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("")); + assert!(!has_operator_bearer(&headers)); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer ")); + assert!(!has_operator_bearer(&headers)); + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static("Bearer operator-token"), + ); + assert!(has_operator_bearer(&headers)); + // A raw token (no scheme) is what `admin_ok` accepts too. + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static("operator-token"), + ); + assert!(has_operator_bearer(&headers)); + } + #[test] fn view_paths_detected() { assert!(is_view_path("v1/view/abc/index.html")); diff --git a/crates/gateway/tests/proxy_rr.rs b/crates/gateway/tests/proxy_rr.rs index bfc75b9d3..ab0c778a0 100644 --- a/crates/gateway/tests/proxy_rr.rs +++ b/crates/gateway/tests/proxy_rr.rs @@ -730,3 +730,92 @@ async fn a_topic_route_reaches_proof_with_its_topic_id() { assert!(body.contains("no healthy backends"), "{body}"); let _ = shutdown.send(()); } + +/// The operator publish is the one admin route the gateway forwards, so +/// `proof-admin topic install` works on staging against the gateway itself +/// (no rewrite proxy). It is forwarded with the challenge id stripped, it +/// needs a bearer at the gateway, and every other admin route — plus the +/// topic-id form — keeps its 403. +#[tokio::test] +async fn the_operator_publish_route_is_forwarded_with_a_bearer() { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/admin/proof/topics")) + .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({ + "id": "tb4", + "status": "draft", + }))) + .mount(&upstream) + .await; + // Any other admin path reaching the upstream would be a leak; wiremock + // answers 404 for an unmounted path, so an assertion on the gateway's own + // 403 below is what proves it never dialed. + + let reg = fast_registry(); + reg.create(&CreateBackend { + challenge_id: "proof".into(), + base_url: upstream.uri(), + weight: 1, + }) + .unwrap(); + + let (addr, shutdown) = spawn_gateway(reg).await; + let client = reqwest::Client::new(); + + // The CLI's publish path, with the operator bearer: forwarded to the + // challenge with the challenge id stripped. + let resp = client + .post(format!( + "http://{addr}/challenge/proof/v1/admin/proof/topics" + )) + .header("authorization", "Bearer operator-token") + .json(&serde_json::json!({"id": "tb4", "status": "draft"})) + .send() + .await + .expect("proxy"); + assert_eq!( + resp.status().as_u16(), + 201, + "the publish must reach the challenge" + ); + let body = resp.text().await.unwrap(); + assert!(body.contains("\"id\":\"tb4\""), "{body}"); + + // No bearer: refused at the gateway, never forwarded. + let resp = client + .post(format!( + "http://{addr}/challenge/proof/v1/admin/proof/topics" + )) + .json(&serde_json::json!({"id": "tb4", "status": "draft"})) + .send() + .await + .expect("proxy"); + assert_eq!(resp.status().as_u16(), 401); + + // Every other admin route stays master-local, bearer or not. + for rest in [ + "v1/admin/proof/executor", + "v1/admin/proof/queue/drain", + "v1/admin/proof/vm-orchestrator", + ] { + let resp = client + .post(format!("http://{addr}/challenge/proof/{rest}")) + .header("authorization", "Bearer operator-token") + .json(&serde_json::json!({})) + .send() + .await + .expect("proxy"); + assert_eq!(resp.status().as_u16(), 403, "{rest} must stay master-local"); + } + + // A topic id is not a way to reach the admin surface. + let resp = client + .post(format!("http://{addr}/challenge/tb4/v1/admin/proof/topics")) + .header("authorization", "Bearer operator-token") + .json(&serde_json::json!({"id": "tb4"})) + .send() + .await + .expect("proxy"); + assert_eq!(resp.status().as_u16(), 403); + let _ = shutdown.send(()); +} diff --git a/crates/proof-challenge/src/topic_routes.rs b/crates/proof-challenge/src/topic_routes.rs index d1acd0e94..a3c126f59 100644 --- a/crates/proof-challenge/src/topic_routes.rs +++ b/crates/proof-challenge/src/topic_routes.rs @@ -60,13 +60,34 @@ pub fn topic_route_router(mux: Arc) -> Router { /// Proof routes alone, and a topic route is a 404 from the base router rather /// than an answer from a table that was never read. pub fn challenge_router(state: AppState, topic_routes: Option>) -> Router { - let app = proof_router(state); + // The challenge's own routes answer under their own prefix too + // (`/challenge/proof/…` → the Proof routes). That is the path the + // operator CLI publishes to (`proof_topic_bundle::PUBLISH_PATH`), and it + // is the *same* path the gateway forwards — so one publish URL works + // against the gateway and against the challenge service directly, with no + // rewrite proxy in between. Without this mount the prefix would fall + // through to the topic mux below, which holds no routes for the + // challenge's own id and would answer `404 topic_route_not_registered`. + let app = proof_router(state.clone()).merge(prefixed_proof_router(state)); match topic_routes { Some(mux) => app.merge(topic_route_router(mux)), None => app, } } +/// The Proof routes under the challenge's own prefix: `/challenge/proof/…`. +/// +/// The prefix is the challenge id (`proof`), never a topic id: a topic's +/// routes live under its own slug, and [`topic_route_router`] is what serves +/// those. A request to `/challenge/{topic_id}/…` for any other id is +/// unaffected. +pub fn prefixed_proof_router(state: AppState) -> Router { + Router::new().nest(PROOF_PREFIX, proof_router(state)) +} + +/// The challenge's own prefix: `/challenge/proof`. +pub const PROOF_PREFIX: &str = "/challenge/proof"; + /// The install journal, read through `proof_topic_install`. /// /// This is what the **publish gate** consults: an `open` document is refused @@ -256,6 +277,28 @@ mod tests { (status, body) } + /// The same, with a JSON body — the admin routes take a `Json` extractor, + /// so a POST without one is a 415 before any gate runs. + async fn ask_json(app: Router, method: &str, uri: &str) -> (StatusCode, serde_json::Value) { + let response = app + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .unwrap(); + let body = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, body) + } + /// A registered route is served; a path the topic did not register is not /// invented, and a method it did not claim is a 405. #[tokio::test] @@ -352,4 +395,55 @@ mod tests { let (status, _) = ask(app, "GET", "/challenge/tb4/status").await; assert_eq!(status, StatusCode::NOT_FOUND); } + + /// The challenge's own routes answer under its own prefix too, so one + /// publish URL works against the gateway and against the challenge + /// service directly (no rewrite proxy): `/challenge/proof/v1/admin/proof/ + /// topics` reaches the Proof publish route, while a *topic* id keeps + /// resolving through the mux. + #[tokio::test] + async fn the_challenge_prefix_reaches_the_proof_routes_and_not_the_mux() { + let fake = Fake::new(vec![("tb4", "status", "GET")]); + let state = AppState { + store: MemoryStore::new(), + pin: ProofPin::default(), + backend: EvalBackend::Sim, + live_scorer: None, + offer: None, + executor: executor_slot(None), + judge_api_key: None, + admin_hashes: Arc::new(Vec::new()), + vm_probe: None, + install_journal: None, + epoch: 0, + }; + let app = challenge_router(state, Some(mux(fake))); + + // The Proof routes are served under the prefix, with their own + // answers: `auth_unconfigured` (503) is the publish route's, not the + // mux's 404. + let (status, body) = ask(app.clone(), "GET", "/challenge/proof/health").await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["challenge_id"], CHALLENGE_ID); + let (status, body) = ask_json( + app.clone(), + "POST", + "/challenge/proof/v1/admin/proof/topics", + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert_eq!(body["error"], "auth_unconfigured", "{body}"); + + // The topic mux still owns every other id under `/challenge/…`. + let (status, _) = ask(app.clone(), "GET", "/challenge/tb4/status").await; + assert_eq!(status, StatusCode::OK); + let (status, body) = ask(app.clone(), "GET", "/challenge/proof/status").await; + assert_eq!(status, StatusCode::NOT_FOUND, "{body}"); + assert_eq!(body["error"], "topic_route_not_registered", "{body}"); + + // A topic id is never a way to reach the admin surface. + let (status, body) = ask(app, "POST", "/challenge/tb4/v1/admin/proof/topics").await; + assert_eq!(status, StatusCode::NOT_FOUND, "{body}"); + assert_eq!(body["error"], "topic_route_not_registered", "{body}"); + } } diff --git a/crates/proof-topic-install/src/routes.rs b/crates/proof-topic-install/src/routes.rs index 1149f7b26..f2e02aa3b 100644 --- a/crates/proof-topic-install/src/routes.rs +++ b/crates/proof-topic-install/src/routes.rs @@ -168,6 +168,11 @@ impl TopicRouteMux { /// id the CHECK cannot hold is refused rather than trimmed into one that /// resolves. /// + /// A path inside the challenge's admin namespace + /// ([`crate::section::is_reserved_api_path`]) is never served, whatever + /// the table holds: the install refuses to record one, and this is the + /// read-side half for a row that predates that rule. + /// /// # Errors /// /// [`InstallError::Db`] when the registry cannot be read. The caller @@ -181,9 +186,12 @@ impl TopicRouteMux { if !is_topic_id(topic_id) { return Ok(Resolved::NotRegistered); } + let path = path.trim().trim_matches('/'); + if crate::section::is_reserved_api_path(path) { + return Ok(Resolved::NotRegistered); + } let routes = self.routes(topic_id).await?; let method = method.trim().to_ascii_uppercase(); - let path = path.trim().trim_matches('/'); match routes.iter().find(|r| r.path == path) { None => Ok(Resolved::NotRegistered), Some(route) if route.method == "*" || route.method == method => { diff --git a/crates/proof-topic-install/src/section.rs b/crates/proof-topic-install/src/section.rs index 3a7eb1ae6..fcd95bd75 100644 --- a/crates/proof-topic-install/src/section.rs +++ b/crates/proof-topic-install/src/section.rs @@ -270,6 +270,17 @@ pub fn read_apis(value: &Value) -> Result, InstallError> { ), )); } + if is_reserved_api_path(&path) { + return Err(bad( + &part, + format!( + "path {path:?} is inside the challenge's admin namespace ({}), which is not a \ + topic's to claim: a topic route that reads like an operator route is a route \ + a reader cannot tell apart from the real one. Register a different path.", + RESERVED_API_PREFIXES.join(", ") + ), + )); + } let method = string_field(obj, "method", &part)? .trim() .to_ascii_uppercase(); @@ -334,6 +345,31 @@ pub fn is_api_method(m: &str) -> bool { matches!(m, "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "*") } +/// Path prefixes inside a topic's own namespace that are **not a topic's to +/// claim**: the challenge's operator surface. +/// +/// A topic route is served under the topic's prefix +/// (`/challenge/{topic_id}/{path}`), so a stored `v1/admin/…` would answer at +/// `/challenge/{topic_id}/v1/admin/…` — a path a reader cannot tell apart +/// from the challenge's own admin surface, which is master-local. The install +/// refuses to record one, and the mux refuses to resolve one that is already +/// in the table (a row written before this rule existed). +pub const RESERVED_API_PREFIXES: [&str; 1] = ["v1/admin"]; + +/// Whether `p` is inside a [`RESERVED_API_PREFIXES`] namespace. +/// +/// Segment-aware: `v1/admin` and `v1/admin/…` are reserved, `v1/administrator` +/// is not. +#[must_use] +pub fn is_reserved_api_path(p: &str) -> bool { + let p = p.trim(); + RESERVED_API_PREFIXES.iter().any(|prefix| { + p == *prefix + || p.strip_prefix(prefix) + .is_some_and(|rest| rest.starts_with('/')) + }) +} + /// Read a section's rule vector. /// /// # Errors @@ -555,6 +591,39 @@ mod tests { } } + /// The challenge's admin namespace is not a topic's to claim: a topic + /// route that reads like an operator route is refused at install time, + /// and the same predicate is what the mux checks on read. + #[test] + fn a_route_inside_the_admin_namespace_is_refused() { + for bad in [ + "v1/admin", + "v1/admin/proof/topics", + "v1/admin/proof/queue/drain", + ] { + let err = read_section(&format!( + r#"{{"apis": [{{"path": "{bad}", "method": "POST"}}]}}"# + )) + .expect_err(bad); + let InstallError::Section { why, .. } = err else { + panic!("{bad:?}: expected Section"); + }; + assert!(why.contains("admin namespace"), "{bad:?}: {why}"); + assert!(is_reserved_api_path(bad), "{bad:?}"); + } + // Segment-aware: a path that merely starts with the same characters + // is not the reserved namespace. + for good in ["v1/administrator", "v1/adminx", "admin", "v1/admins"] { + read_section(&format!( + r#"{{"apis": [{{"path": "{good}", "method": "GET"}}]}}"# + )) + .unwrap_or_else(|e| panic!("{good:?} is not reserved: {e}")); + assert!(!is_reserved_api_path(good), "{good:?}"); + } + // The predicate is the one the mux runs, on a trimmed path. + assert!(is_reserved_api_path(" v1/admin/x ")); + } + #[test] fn a_rule_vector_the_scoring_path_would_refuse_is_refused_here() { let err = diff --git a/crates/proof-topic-install/tests/topic_routes.rs b/crates/proof-topic-install/tests/topic_routes.rs index 9253c59f6..980679465 100644 --- a/crates/proof-topic-install/tests/topic_routes.rs +++ b/crates/proof-topic-install/tests/topic_routes.rs @@ -131,6 +131,49 @@ async fn only_a_registered_route_resolves() { } } +/// A route inside the challenge's admin namespace is never served, even when +/// the table holds a row for it: the install refuses to record one +/// (`section::is_reserved_api_path`), and this is the read-side half for a row +/// written before that rule existed. The answer is a 404 from the challenge, +/// never a 200 that reads like an operator route. +#[tokio::test] +async fn a_route_inside_the_admin_namespace_never_resolves() { + let registry = FakeRegistry::new(); + registry.install( + "tb4", + vec![("v1/admin/proof/topics", "POST"), ("v1/admin", "*")], + ); + let mux = TopicRouteMux::new(registry.clone()); + + for path in [ + "v1/admin/proof/topics", + "v1/admin", + "/v1/admin", + "v1/admin/x", + ] { + assert_eq!( + mux.resolve("tb4", "POST", path).await.expect("resolve"), + Resolved::NotRegistered, + "{path:?}" + ); + } + // The reserved check runs before the table read, so a path in the admin + // namespace does not even cost a query. + assert_eq!( + registry.route_reads.load(Ordering::SeqCst), + 0, + "the reservation is decided without reading the registry" + ); + // A path that merely starts with the same characters is not reserved. + registry.install("tb4", vec![("v1/administrator", "GET")]); + assert!(matches!( + mux.resolve("tb4", "GET", "v1/administrator") + .await + .expect("resolve"), + Resolved::Route(_) + )); +} + /// **The regression this cache exists for:** an install in another process /// writes the table, and the next request sees it — with no signal beyond the /// generation probe and no restart. From b6d84340c419c589d57819b824a131da05520cb4 Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:14:32 +0000 Subject: [PATCH 2/8] feat(proof): topic disable is an operator switch the challenge obeys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `proof-admin topic disable` / `topic enable` were stubs that exited 3, so the only way to stop a topic taking submissions was to re-sign its document — a ceremony that does not wait for an incident. The gate is now real, and fail-closed end to end: - `proof_topic_gate` (migration 0026) is an append-only journal: the newest row per topic is the state, the rows before it are who turned it off, when, and why. `enabled` clears a disable; nothing is deleted. - The CLI appends the row (`topic disable --reason …`, `topic enable`), and refuses an id that is not published (or an alias of one) so a typo cannot silently "disable" nothing. `topic show` prints the gate state. - The challenge reads it on `POST /v1/submissions` **before the nonce is spent**: a disabled topic is a 403 naming the operator's reason, an unreadable gate is a 503 (never an admission), and a re-submit after `topic enable` lands with the same signature and the same submit_nonce. - `GET /v1/proof/topics` and `/v1/proof/topics/{id}` carry `disabled` (+ `disabled_reason`), so a miner does not read a stopped topic as open work. The gate changes no document, cancels no in-flight evaluation, and leaves emission alone, so a topic disabled mid-epoch cannot silently break the leaf a seal depends on. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- bins/proof-admin/src/main.rs | 227 ++++++++++- bins/proof-admin/tests/cli.rs | 60 ++- .../db/migrations/0026_proof_topic_gate.sql | 58 +++ crates/proof-challenge/src/topic_routes.rs | 22 + crates/proof-http/src/lib.rs | 376 +++++++++++++++++- crates/proof-topic-install/src/gate.rs | 264 ++++++++++++ crates/proof-topic-install/src/lib.rs | 11 +- .../tests/install_engine.rs | 90 +++++ crates/proof-topic-sql-guard/src/lib.rs | 3 +- 9 files changed, 1067 insertions(+), 44 deletions(-) create mode 100644 crates/db/migrations/0026_proof_topic_gate.sql create mode 100644 crates/proof-topic-install/src/gate.rs diff --git a/bins/proof-admin/src/main.rs b/bins/proof-admin/src/main.rs index 6060b41fd..cf4348eb2 100644 --- a/bins/proof-admin/src/main.rs +++ b/bins/proof-admin/src/main.rs @@ -74,7 +74,8 @@ List the installed topics (a read-only view of proof_topic_version): Nothing here writes a topic, opens a route, or changes how a score is computed. `install` prints the publish call and the host env for an operator -to run; `topic enable` / `disable` / `seal` exit 3 as not-implemented." +to run; `disable` / `enable` throw the operator gate the challenge reads on +the submit path; `topic seal` exits 3 as not-implemented." )] struct Cli { /// Postgres URL for the topic registry view. Falls back to `BASE_DATABASE_URL`. @@ -191,15 +192,38 @@ enum TopicCmd { #[command(subcommand)] cmd: AliasCmd, }, - /// Not implemented in this slice. - Enable { - /// Topic slug. + /// Stop a topic taking submissions, now. + /// + /// Appends a `disabled` row to `proof_topic_gate`; the challenge reads it + /// on the next `POST /v1/submissions` and refuses with your reason. No + /// re-sign, no restart, no redeploy — the document keeps its own `status`, + /// in-flight evaluations finish, and rows already scored keep their + /// verdicts. Use it when something is wrong with the topic, not to retire + /// one: retiring is a signed `closed` document. + Disable { + /// Topic slug, or an alias of one. topic_id: String, + /// Why, in your words. Shown to a miner in the 403, so no secrets. + #[arg(long, value_name = "TEXT")] + reason: Option, + /// Who is throwing the switch, for the audit trail. Never a token. + #[arg(long, env = "PROOF_GATE_ACTOR", value_name = "LABEL")] + actor: Option, }, - /// Not implemented in this slice. - Disable { - /// Topic slug. + /// Let a disabled topic take submissions again. + /// + /// Appends an `enabled` row — the only way back, so the history of who + /// turned it off (and who turned it on) stays readable. The topic's own + /// document is untouched. + Enable { + /// Topic slug, or an alias of one. topic_id: String, + /// Why it is being re-enabled, for the audit trail. + #[arg(long, value_name = "TEXT")] + reason: Option, + /// Who is clearing the switch. Never a token. + #[arg(long, env = "PROOF_GATE_ACTOR", value_name = "LABEL")] + actor: Option, }, /// Not implemented in this slice. Seal { @@ -332,8 +356,34 @@ async fn run_topic(opts: &Options, cmd: &TopicCmd) -> Result<(), Failure> { TopicCmd::InstallLog { topic } => cmd_install_log(opts, topic).await, TopicCmd::Show { topic_id } => cmd_show(opts, topic_id).await, TopicCmd::Alias { cmd } => run_alias(opts, cmd).await, - TopicCmd::Enable { topic_id } => Err(not_implemented("topic enable", topic_id)), - TopicCmd::Disable { topic_id } => Err(not_implemented("topic disable", topic_id)), + TopicCmd::Disable { + topic_id, + reason, + actor, + } => { + cmd_gate( + opts, + topic_id, + proof_topic_install::GateState::Disabled, + reason.as_deref(), + actor.as_deref(), + ) + .await + } + TopicCmd::Enable { + topic_id, + reason, + actor, + } => { + cmd_gate( + opts, + topic_id, + proof_topic_install::GateState::Enabled, + reason.as_deref(), + actor.as_deref(), + ) + .await + } TopicCmd::Seal { topic_id, value } => Err(not_implemented( &format!("topic seal (value {value})"), topic_id, @@ -664,7 +714,8 @@ async fn cmd_list(opts: &Options) -> Result<(), Failure> { } async fn cmd_show(opts: &Options, topic_id: &str) -> Result<(), Failure> { - let store = open_store(opts).await?; + let pool = open_pool(opts).await?; + let store = PgRlmStore::new(pool.clone()); // An alias resolves to its canonical slug first, so `show tbench` finds // `tb4`. Resolution is fail-closed in the store: an alias whose topic has // no published version resolves to nothing rather than to an empty row. @@ -691,6 +742,12 @@ async fn cmd_show(opts: &Options, topic_id: &str) -> Result<(), Failure> { version, document, }; + // The operator gate, read from the same table the challenge reads: `show` + // must not report a topic as open for work when the submit path refuses + // it. An unreadable gate is reported rather than assumed enabled. + let gate = proof_topic_install::gate(&pool, canonical) + .await + .map_err(|e| Failure::Error(format!("{canonical} gate: {e}")))?; if let Some(canonical) = resolved.as_deref() { if !opts.json { println!("{topic_id} is an alias of {canonical}"); @@ -698,18 +755,76 @@ async fn cmd_show(opts: &Options, topic_id: &str) -> Result<(), Failure> { } } if opts.json { - print_json(&topic_json(&row))?; + let mut body = topic_json(&row); + if let Some(obj) = body.as_object_mut() { + obj.insert( + "disabled".to_owned(), + serde_json::Value::Bool( + gate.as_ref() + .is_some_and(proof_topic_install::Gate::is_disabled), + ), + ); + if let Some(gate) = gate.as_ref().filter(|g| g.is_disabled()) { + obj.insert( + "disabled_reason".to_owned(), + serde_json::Value::String(gate.reason.clone()), + ); + } + } + print_json(&body)?; return Ok(()); } print_row(&row); + print_gate(gate.as_ref(), canonical); Ok(()) } +/// The operator gate line(s) for `topic show`. +fn print_gate(gate: Option<&proof_topic_install::Gate>, topic_id: &str) { + println!(); + match gate { + None => println!("Operator gate: enabled (no `proof_topic_gate` row)."), + Some(gate) if !gate.is_disabled() => { + println!( + "Operator gate: enabled (gate row {}; the newest row is an enable).", + gate.id + ); + } + Some(gate) => { + println!("Operator gate: DISABLED (gate row {}).", gate.id); + if gate.reason.is_empty() { + println!(" reason (none given)"); + } else { + println!(" reason {}", gate.reason); + } + if !gate.actor.is_empty() { + println!(" actor {}", gate.actor); + } + println!(); + println!("Submissions to this topic are refused. Re-enable with:"); + println!(" proof-admin topic enable {topic_id}"); + } + } +} + /// The topic registry: the existing `proof_topic_version` rows. /// /// A configured but unreachable database is fatal: falling back to an empty /// in-memory view would report "nothing installed" for a host that has topics. async fn open_store(opts: &Options) -> Result, Failure> { + let pool = open_pool(opts).await?; + // `PgRlmStore` is the production registry; the memory store exists for + // CI/local and is never selected here, so a real host never reads an + // empty view by accident. + let _ = MemoryRlmStore::new; + Ok(Box::new(PgRlmStore::new(pool))) +} + +/// A connection pool over the topic database. +/// +/// The gate commands write (`proof_topic_gate`) as well as read, so they need +/// the pool itself and not only the registry trait object. +async fn open_pool(opts: &Options) -> Result { let Some(url) = database_url(opts)? else { return Err(Failure::Usage( "this command reads the topic registry and needs a database: set \ @@ -718,14 +833,90 @@ async fn open_store(opts: &Options) -> Result, Failure> { .into(), )); }; - let pool = db::connect(&url) + db::connect(&url) .await - .map_err(|e| Failure::Error(format!("connect: {e}")))?; - // `PgRlmStore` is the production registry; the memory store exists for - // CI/local and is never selected here, so a real host never reads an - // empty view by accident. - let _ = MemoryRlmStore::new; - Ok(Box::new(PgRlmStore::new(pool))) + .map_err(|e| Failure::Error(format!("connect: {e}"))) +} + +/// `topic disable` / `topic enable`: throw the operator gate. +/// +/// The topic must be published (or be an alias of one): a typo must not +/// silently disable nothing, because the operator would then believe a topic +/// is stopped while it is still taking submissions. The write is append-only +/// — the newest row is the state, the rows before it are the history — and it +/// is visible to the challenge on the next request, which is the point of the +/// switch. +async fn cmd_gate( + opts: &Options, + topic_id: &str, + state: proof_topic_install::GateState, + reason: Option<&str>, + actor: Option<&str>, +) -> Result<(), Failure> { + let pool = open_pool(opts).await?; + let store = PgRlmStore::new(pool.clone()); + let resolved = store + .resolve_alias(topic_id) + .await + .map_err(|e| Failure::Error(format!("resolve {topic_id}: {e}")))?; + let canonical = resolved.as_deref().unwrap_or(topic_id); + let row = store + .latest_topic(canonical) + .await + .map_err(|e| Failure::Error(format!("{canonical}: {e}")))?; + if row.is_none() { + return Err(Failure::Error(format!( + "no installed topic {topic_id:?}{}. Nothing was changed — check the id with \ + `proof-admin topic list`.", + resolved + .as_deref() + .map(|c| format!(" (alias of {c:?})")) + .unwrap_or_default() + ))); + } + let reason = reason.unwrap_or_default(); + let actor = actor.unwrap_or_default(); + let gate = proof_topic_install::set(&pool, canonical, state, reason, actor) + .await + .map_err(|e| Failure::Error(format!("{canonical}: {e}")))?; + let disabled = gate.is_disabled(); + if opts.json { + print_json(&serde_json::json!({ + "ok": true, + "topic_id": canonical, + "state": gate.state.as_str(), + "disabled": disabled, + "reason": gate.reason, + "actor": gate.actor, + "gate_row": gate.id, + }))?; + return Ok(()); + } + if disabled { + println!("topic {canonical} is disabled (gate row {}).", gate.id); + if gate.reason.is_empty() { + println!(" reason (none given)"); + } else { + println!(" reason {}", gate.reason); + } + println!(); + println!( + "Submissions are refused from the next request on, with this reason. The document \ + keeps its own status, in-flight evaluations finish, and rows already scored keep \ + their verdicts. Nothing was re-signed and nothing was restarted." + ); + println!(); + println!("To let it take submissions again:"); + println!(" proof-admin topic enable {canonical}"); + } else { + println!("topic {canonical} is enabled again (gate row {}).", gate.id); + println!(); + println!( + "Submissions are admitted from the next request on, under the topic's own document \ + (`status`), which was never changed. The disable rows stay in the history." + ); + } + Ok(()) } /// `BASE_DATABASE_URL` value, or the contents of `BASE_DATABASE_URL_FILE`. diff --git a/bins/proof-admin/tests/cli.rs b/bins/proof-admin/tests/cli.rs index 00433c681..26a25f897 100644 --- a/bins/proof-admin/tests/cli.rs +++ b/bins/proof-admin/tests/cli.rs @@ -972,31 +972,53 @@ fn database_url_and_file_are_mutually_exclusive() { } #[test] -fn enable_disable_and_seal_fail_closed_with_exit_3() { +fn seal_still_fails_closed_with_exit_3() { + let args = vec!["topic", "seal", "tb4", "--value", "0.42"]; + let out = run(&args); + assert_eq!( + code(&out), + EXIT_NOT_IMPLEMENTED, + "{args:?}: {}", + stderr(&out) + ); + let err = stderr(&out); + assert!( + err.contains("not implemented in this slice"), + "{args:?}: {err}" + ); + assert!( + err.contains("Nothing was changed"), + "a stub must say it changed nothing: {args:?}: {err}" + ); + assert!( + stdout(&out).is_empty(), + "a stub prints nothing to stdout: {args:?}" + ); +} + +/// `topic disable` / `topic enable` are implemented now, and they are +/// **fail-closed without a database**: the gate is the table the challenge +/// reads, so a CLI that could not write it must refuse rather than report a +/// topic as stopped. Exit 2 (usage), nothing on stdout, and the message names +/// the variable to set. +#[test] +fn disable_and_enable_need_the_gate_database() { for args in [ + vec!["topic", "disable", "tb4", "--reason", "incident 42"], vec!["topic", "enable", "tb4"], - vec!["topic", "disable", "tb4"], - vec!["topic", "seal", "tb4", "--value", "0.42"], ] { - let out = run(&args); - assert_eq!( - code(&out), - EXIT_NOT_IMPLEMENTED, - "{args:?}: {}", - stderr(&out) - ); + let out = Command::new(env!("CARGO_BIN_EXE_proof-admin")) + .args(&args) + .env_remove("BASE_DATABASE_URL") + .env_remove("BASE_DATABASE_URL_FILE") + .output() + .expect("run"); + assert_eq!(code(&out), EXIT_USAGE, "{args:?}: {}", stderr(&out)); let err = stderr(&out); - assert!( - err.contains("not implemented in this slice"), - "{args:?}: {err}" - ); - assert!( - err.contains("Nothing was changed"), - "a stub must say it changed nothing: {args:?}: {err}" - ); + assert!(err.contains("BASE_DATABASE_URL"), "{args:?}: {err}"); assert!( stdout(&out).is_empty(), - "a stub prints nothing to stdout: {args:?}" + "nothing is reported as changed: {args:?}" ); } } diff --git a/crates/db/migrations/0026_proof_topic_gate.sql b/crates/db/migrations/0026_proof_topic_gate.sql new file mode 100644 index 000000000..0ee407220 --- /dev/null +++ b/crates/db/migrations/0026_proof_topic_gate.sql @@ -0,0 +1,58 @@ +-- Proof topic gate: the operator switch that stops a topic taking submissions. +-- +-- A topic's lifecycle lives in its **signed document** (`proof_topic_version`, +-- migration 0020): `draft` / `open` / `closed`. Moving that status is a signing +-- ceremony — the operator re-signs with the `proof` key — and an incident does +-- not wait for one. This table is the other switch: an operator at the CLI +-- records "this topic is disabled", and the challenge refuses every submission +-- to it on the next request, with no re-sign, no restart, and no redeploy. +-- +-- It is a **journal**, not a state column, and for the same reason +-- `proof_topic_install` is one: the newest row for a topic is its current +-- gate state, and the rows before it are the history of who turned it off, +-- when, and why. A `disable` after an `enable` is a new row; nothing is +-- rewritten. An `enabled` row is what clears a disable — there is no DELETE, +-- because "who turned it back on" is exactly what an incident review asks. +-- +-- What a disabled topic does *not* do: it does not un-sign, un-publish, or +-- re-score anything. The document keeps its status, in-flight evaluations +-- finish, and rows already scored keep their verdicts. The gate is a +-- **submission** gate: `POST /v1/submissions` answers 403 with the reason +-- before the nonce is spent, and the topic is not advertised as open for work. +-- Emission is untouched, so a topic disabled mid-epoch cannot silently break +-- the leaf a seal depends on. +-- +-- Fail-closed at the reader: the challenge reads this table on the submit +-- path, and a read that fails is a **503**, never an admission. A host with no +-- database has no gate (and no published topics either). +-- +-- `topic_id` is the shared challenge DB's discriminant, exactly like every +-- other `proof_*` table; there is no per-topic schema. The id shape is the +-- same CHECK `proof_topic_version` enforces. + +CREATE TABLE proof_topic_gate ( + id BIGSERIAL PRIMARY KEY, + topic_id TEXT NOT NULL, + -- `disabled` stops submissions; `enabled` clears a disable and is the + -- only way back, so the history is readable in one direction. + state TEXT NOT NULL, + -- Why, in the operator's words. Shown to a miner in the 403, so it must + -- never carry a secret; it is operator-authored text, not a key. + reason TEXT NOT NULL DEFAULT '', + -- Who, for the incident review: an operator label, never a token. + actor TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT proof_topic_gate_topic_check CHECK (topic_id ~ '^[a-z0-9][a-z0-9-]{1,62}$'), + CONSTRAINT proof_topic_gate_state_check CHECK (state IN ('disabled', 'enabled')), + CONSTRAINT proof_topic_gate_reason_check CHECK (length(reason) <= 512), + CONSTRAINT proof_topic_gate_actor_check CHECK (length(actor) <= 128) +); + +-- The read on the submit path is "the newest gate row for this topic"; the +-- operator listing is "newest first per topic". +CREATE INDEX ix_proof_topic_gate_topic ON proof_topic_gate (topic_id, id DESC); + +-- Append-only for the application role: a gate that could be edited in place +-- would not be a history, and the runtime read is the newest row. +GRANT SELECT, INSERT ON TABLE proof_topic_gate TO base_app; +GRANT USAGE, SELECT ON SEQUENCE proof_topic_gate_id_seq TO base_app; diff --git a/crates/proof-challenge/src/topic_routes.rs b/crates/proof-challenge/src/topic_routes.rs index a3c126f59..22b012c7d 100644 --- a/crates/proof-challenge/src/topic_routes.rs +++ b/crates/proof-challenge/src/topic_routes.rs @@ -94,6 +94,12 @@ pub const PROOF_PREFIX: &str = "/challenge/proof"; /// until the topic's newest install row is `applied`. The read is the same /// one `proof-admin topic install-log` shows, so the operator and the route /// cannot disagree about whether a topic is installed. +/// +/// It is also the **operator gate** the submit path reads: a topic an operator +/// disabled (`proof-admin topic disable`) is refused with the operator's +/// reason, and an unreadable gate is a 503 rather than an admission. Both +/// reads are over the same database, so a host that can prove an install can +/// also answer whether the topic is switched off. pub struct PgInstallJournal { /// Pool over the shared challenge database. pub pool: sqlx::PgPool, @@ -114,6 +120,22 @@ impl proof_http::InstallJournal for PgInstallJournal { .await .map_err(|e| e.to_string()) } + + async fn disabled(&self, topic_id: &str) -> Result, String> { + proof_topic_install::gate(&self.pool, topic_id) + .await + .map(|gate| { + gate.filter(proof_topic_install::Gate::is_disabled) + .map(|g| g.reason) + }) + .map_err(|e| e.to_string()) + } + + async fn disabled_topics(&self) -> Result, String> { + proof_topic_install::disabled_topics(&self.pool) + .await + .map_err(|e| e.to_string()) + } } /// One topic route. diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 4a003758a..00f1149cc 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -45,6 +45,7 @@ clippy::too_many_arguments )] +use std::collections::BTreeMap; use std::future::Future; use std::sync::{Arc, PoisonError, RwLock}; @@ -192,6 +193,32 @@ pub trait InstallJournal: Send + Sync { /// The reason the journal could not be read. The caller refuses the /// publish: an unreadable journal is not an installed topic. async fn applied(&self, topic_id: &str) -> Result; + + /// `Ok(Some(reason))` when an operator **disabled** `topic_id`, `Ok(None)` + /// when the topic was never disabled (or was enabled again). + /// + /// The submit path reads this and refuses a disabled topic with a 403 + /// carrying the reason — before the single-use nonce is spent, so a miner + /// loses nothing to an operator's switch. It is the operator's incident + /// switch (`proof-admin topic disable`), and it needs no re-sign, no + /// restart, and no redeploy. + /// + /// # Errors + /// + /// The reason the gate could not be read. The caller answers **503**: an + /// unreadable gate is not an enabled topic. + async fn disabled(&self, topic_id: &str) -> Result, String>; + + /// Every topic currently disabled, with the operator's reason. + /// + /// One read for the public listing, which annotates each topic with the + /// flag rather than paying a query per topic. The submit path uses + /// [`Self::disabled`] for the single topic it is admitting. + /// + /// # Errors + /// + /// The reason the gate could not be read. The caller answers **503**. + async fn disabled_topics(&self) -> Result, String>; } /// The journal read behind the publish gate, or `None` on a host that @@ -420,9 +447,26 @@ async fn status(State(st): State) -> impl IntoResponse { })) } -async fn list_topics(State(st): State) -> impl IntoResponse { - let items = st.store.topics().unwrap_or_default(); - Json(serde_json::json!({ "items": items })) +/// `GET /v1/proof/topics` — every published topic, annotated with the +/// operator gate. +/// +/// The annotation is the point: a topic an operator disabled still has its +/// document (and its `status`), so a miner reading the list would otherwise +/// see work that the submit path refuses. The flag is read from the same gate +/// the submit path consults, and an unreadable gate is a **503** rather than +/// a list that reads as "nothing is disabled". +async fn list_topics( + State(st): State, +) -> Result)> { + let disabled = disabled_topics(&st).await?; + let items: Vec = st + .store + .topics() + .unwrap_or_default() + .iter() + .map(|t| annotate_gate(t, disabled.get(&t.id).map(String::as_str))) + .collect(); + Ok(Json(serde_json::json!({ "items": items }))) } async fn get_topic( @@ -433,7 +477,51 @@ async fn get_topic( .store .topic(&id) .map_err(|_| err(StatusCode::NOT_FOUND, "unknown topic"))?; - Ok(Json(doc)) + let reason = disabled_topics(&st).await?.get(&id).cloned(); + Ok(Json(annotate_gate(&doc, reason.as_deref()))) +} + +/// The public view of a topic: its signed document, plus the operator gate. +/// +/// `disabled` is always present (`false` on a host that never threw the +/// switch, and on one with no gate at all); `disabled_reason` appears only +/// when an operator gave one. The document is never rewritten: the gate is +/// operator state *about* the topic, not part of what was signed. +fn annotate_gate(doc: &TopicDocument, disabled_reason: Option<&str>) -> serde_json::Value { + let mut value = serde_json::to_value(doc).unwrap_or(serde_json::Value::Null); + if let Some(obj) = value.as_object_mut() { + obj.insert( + "disabled".to_owned(), + serde_json::Value::Bool(disabled_reason.is_some()), + ); + if let Some(reason) = disabled_reason.filter(|r| !r.trim().is_empty()) { + obj.insert( + "disabled_reason".to_owned(), + serde_json::Value::String(reason.trim().to_owned()), + ); + } + } + value +} + +/// The disabled set the listing annotates from, or a 503. +async fn disabled_topics( + st: &AppState, +) -> Result, (StatusCode, Json)> { + let Some(journal) = st.install_journal.as_deref() else { + // No gate on this host: no database, so no operator switch was ever + // thrown (and no topic was published either). + return Ok(BTreeMap::new()); + }; + journal.disabled_topics().await.map_err(|e| { + err( + StatusCode::SERVICE_UNAVAILABLE, + &format!( + "the topic gate could not be read: {e}. The listing is refused rather than \ + served without the operator's switch; fix the database and re-read." + ), + ) + }) } /// Public executor contract: the live offer (every field is public), whether @@ -641,6 +729,10 @@ async fn submit( if !topic.is_open_at(st.epoch) { return Err(err(StatusCode::BAD_REQUEST, "topic is not open")); } + // The operator gate, before anything is spent: a disabled topic refuses + // the submission here, so the single-use nonce the miner signed is still + // unspent and a re-submit after the operator enables the topic works. + disabled_gate(&st, &topic_id).await?; // Miner BYOK, held to what the signed topic declares. Checked before the // signature so a body with the wrong variable names is a plain 400 the // miner can fix and re-post: the `submit_nonce` they signed is still @@ -1696,6 +1788,44 @@ async fn install_gate(st: &AppState, topic_id: &str) -> Result<(), String> { } } +/// The operator gate on the **submit** path: is this topic disabled? +/// +/// `Ok(())` admits the submission. A disabled topic is a **403** naming the +/// operator's reason; an unreadable gate is a **503** — never an admission, +/// and never a 404 that would read as "no such topic". A host that resolved +/// no journal (no database) has no gate to read: it also has no published +/// topics, so the submission is refused by the topic lookup above it. +/// +/// This is deliberately **not cached**: a disable has to take effect on the +/// next request, which is what makes it usable during an incident. +async fn disabled_gate(st: &AppState, topic_id: &str) -> Result<(), ErrResp> { + let Some(journal) = st.install_journal.as_deref() else { + return Ok(()); + }; + match journal.disabled(topic_id).await { + Ok(None) => Ok(()), + Ok(Some(reason)) => Err(err( + StatusCode::FORBIDDEN, + &format!( + "topic {topic_id:?} is disabled by the operator{}", + if reason.trim().is_empty() { + String::new() + } else { + format!(": {}", reason.trim()) + } + ), + )), + Err(e) => Err(err( + StatusCode::SERVICE_UNAVAILABLE, + &format!( + "the topic gate could not be read for {topic_id:?}: {e}. The submission is \ + refused rather than admitted on an unread fact; fix the database and re-post \ + (the submit_nonce is unspent)." + ), + )), + } +} + fn store_err(e: &proof_store::StoreError) -> (StatusCode, Json) { err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()) } @@ -2700,6 +2830,148 @@ mod tests { assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); } + /// `proof-admin topic disable` is an operator switch the submit path + /// reads: a disabled topic is refused with the operator's reason, no row + /// is written, and the miner's single-use nonce is **not** spent — the + /// same body lands once the operator enables the topic again. + #[tokio::test] + async fn a_disabled_topic_refuses_submit_and_does_not_spend_the_nonce() { + let gate = SwitchableGate::new("dt-no-ib-v0", "incident 42: harness regression"); + let app = app_with_gate(Some(gate.clone())); + + // Enabled (never thrown): the submission lands. + let body = submit_body("first", &serde_json::json!({})); + let (st, created) = + json_req(app.clone(), "POST", "/v1/submissions", body.clone(), None).await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + + // Disabled: 403 naming the operator's reason, and no row. + gate.set(true); + let body = submit_body("second", &serde_json::json!({})); + let (st, refused) = + json_req(app.clone(), "POST", "/v1/submissions", body.clone(), None).await; + assert_eq!(st, StatusCode::FORBIDDEN, "{refused}"); + let error = refused["error"].as_str().unwrap_or_default(); + assert!(error.contains("disabled by the operator"), "{refused}"); + assert!(error.contains("incident 42"), "{refused}"); + let (_, list) = json_req( + app.clone(), + "GET", + "/v1/submissions", + serde_json::json!({}), + None, + ) + .await; + assert_eq!( + list["items"].as_array().map(Vec::len), + Some(1), + "the refusal writes no row: {list}" + ); + + // Enabled again: the *same* body — same signature, same submit_nonce + // — lands, so the refusal never spent it. + gate.set(false); + let (st, created) = json_req(app, "POST", "/v1/submissions", body, None).await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + } + + /// The public listing carries the gate, so a miner does not read a + /// disabled topic as open for work. + #[tokio::test] + async fn the_topic_listing_carries_the_operator_gate() { + let gate = SwitchableGate::new("dt-no-ib-v0", "incident 42"); + let app = app_with_gate(Some(gate.clone())); + + let (st, body) = json_req( + app.clone(), + "GET", + "/v1/proof/topics", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!(body["items"][0]["disabled"], false, "{body}"); + assert!( + body["items"][0].get("disabled_reason").is_none(), + "no reason when there is no disable: {body}" + ); + + gate.set(true); + let (st, body) = json_req( + app.clone(), + "GET", + "/v1/proof/topics", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!(body["items"][0]["disabled"], true, "{body}"); + assert_eq!(body["items"][0]["disabled_reason"], "incident 42", "{body}"); + + let (st, body) = json_req( + app, + "GET", + "/v1/proof/topics/dt-no-ib-v0", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!(body["disabled"], true, "{body}"); + assert_eq!(body["disabled_reason"], "incident 42", "{body}"); + assert_eq!( + body["status"], "open", + "the gate is operator state about the topic, not a rewrite of it: {body}" + ); + } + + /// An unreadable gate is a **503** on both reads — never an admission, + /// and never a listing that reads as "nothing is disabled". + #[tokio::test] + async fn an_unreadable_gate_refuses_submit_and_the_listing() { + let app = app_with_gate(Some(Arc::new(BrokenJournal))); + let body = submit_body("x", &serde_json::json!({})); + let (st, refused) = json_req(app.clone(), "POST", "/v1/submissions", body, None).await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{refused}"); + let error = refused["error"].as_str().unwrap_or_default(); + assert!(error.contains("gate could not be read"), "{refused}"); + assert!(error.contains("unspent"), "{refused}"); + + let (st, body) = json_req( + app.clone(), + "GET", + "/v1/proof/topics", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let (st, body) = json_req( + app.clone(), + "GET", + "/v1/proof/topics/dt-no-ib-v0", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + + // A host with no gate at all (no database) is not this case: it + // serves the listing with the flag false rather than refusing. + let (st, body) = json_req( + app_with_gate(None), + "GET", + "/v1/proof/topics", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!(body["items"][0]["disabled"], false, "{body}"); + } + #[tokio::test] async fn submit_requires_a_hotkey_signature() { let app = app("op"); @@ -3374,6 +3646,14 @@ mod tests { async fn applied(&self, _topic_id: &str) -> Result { Ok(true) } + + async fn disabled(&self, _topic_id: &str) -> Result, String> { + Ok(None) + } + + async fn disabled_topics(&self) -> Result, String> { + Ok(BTreeMap::new()) + } } /// An install journal that refuses every read, for the gate's @@ -3385,6 +3665,14 @@ mod tests { async fn applied(&self, _topic_id: &str) -> Result { Err("journal unavailable".into()) } + + async fn disabled(&self, _topic_id: &str) -> Result, String> { + Err("gate unavailable".into()) + } + + async fn disabled_topics(&self) -> Result, String> { + Err("gate unavailable".into()) + } } /// An install journal that has no row for any topic. @@ -3395,6 +3683,86 @@ mod tests { async fn applied(&self, _topic_id: &str) -> Result { Ok(false) } + + async fn disabled(&self, _topic_id: &str) -> Result, String> { + Ok(None) + } + + async fn disabled_topics(&self) -> Result, String> { + Ok(BTreeMap::new()) + } + } + + /// A gate an operator can throw while the host is running: the point of + /// the switch is that the **next** request sees it, with no restart. + struct SwitchableGate { + topic_id: String, + reason: String, + disabled: Arc, + } + + impl SwitchableGate { + fn new(topic_id: &str, reason: &str) -> Arc { + Arc::new(Self { + topic_id: topic_id.to_owned(), + reason: reason.to_owned(), + disabled: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }) + } + + fn set(&self, disabled: bool) { + self.disabled + .store(disabled, std::sync::atomic::Ordering::SeqCst); + } + } + + #[async_trait] + impl InstallJournal for SwitchableGate { + async fn applied(&self, _topic_id: &str) -> Result { + Ok(true) + } + + async fn disabled(&self, topic_id: &str) -> Result, String> { + let on = self.disabled.load(std::sync::atomic::Ordering::SeqCst); + Ok((on && topic_id == self.topic_id).then(|| self.reason.clone())) + } + + async fn disabled_topics(&self) -> Result, String> { + let on = self.disabled.load(std::sync::atomic::Ordering::SeqCst); + Ok(if on { + BTreeMap::from([(self.topic_id.clone(), self.reason.clone())]) + } else { + BTreeMap::new() + }) + } + } + + /// The `app("op")` host — one open topic, Sim, submitable — with `journal` + /// as the operator gate, so the submit path and the listing can be driven + /// against a gate the test controls. + fn app_with_gate(journal: InstallJournalSlot) -> Router { + let p = pin(""); + let store = MemoryStore::new(); + let recs = synthetic_holdout(STRATUM_SIZE, 1); + let (topic, meas) = seal_topic(&p, unsigned_topic(&recs)); + store.put_topic(topic.clone()).expect("topic"); + store.load_holdout(&topic.id, recs).expect("holdout"); + store + .set_baseline(&topic.id, meas.into_sealed()) + .expect("baseline"); + proof_router(AppState { + store, + pin: p, + backend: EvalBackend::Sim, + live_scorer: None, + offer: Some(offer()), + executor: executor_slot(None), + judge_api_key: None, + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + vm_probe: None, + install_journal: journal, + epoch: 0, + }) } /// A host whose install journal says every topic is installed, and whose diff --git a/crates/proof-topic-install/src/gate.rs b/crates/proof-topic-install/src/gate.rs new file mode 100644 index 000000000..adec98ac8 --- /dev/null +++ b/crates/proof-topic-install/src/gate.rs @@ -0,0 +1,264 @@ +//! The operator gate: whether a topic is **disabled**, and the write the CLI +//! uses to change that. +//! +//! A topic's lifecycle lives in its signed document (`draft` / `open` / +//! `closed`), and moving it is a signing ceremony. This is the other switch: +//! an operator records "this topic is disabled" and the challenge refuses +//! every submission to it on the next request — no re-sign, no restart, no +//! redeploy. The table is `proof_topic_gate` (migration `0026`), a journal +//! whose newest row per topic is the current state. +//! +//! | Question | Function | +//! |----------|----------| +//! | Is the topic disabled right now? | [`disabled`] | +//! | Turn it off, with a reason | [`disable`] | +//! | Turn it back on | [`enable`] | +//! | What is the state, for an operator listing? | [`gate`] | +//! +//! # Fail-closed, and where +//! +//! The runtime read is [`disabled`], and the caller +//! (`proof_http::InstallJournal::disabled`) answers **503** on an `Err`: an +//! unreadable gate is not an enabled topic. That is the same direction the +//! install journal takes on the publish path, and it is the reason this +//! module returns `Result` rather than a `bool` that a caller could +//! read as "not disabled" when the database is down. +//! +//! A topic with **no** row is enabled: the gate is a switch an operator +//! throws, not a registration every topic needs. Nothing here interprets a +//! document, a rule, or a route; the gate is one bit of operator state plus +//! its history. + +use std::collections::BTreeMap; + +use sqlx::PgPool; + +use crate::InstallError; + +/// The two states a gate row can hold. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GateState { + /// Submissions are refused. + Disabled, + /// Submissions are admitted (the state a topic starts in). + Enabled, +} + +impl GateState { + /// The word stored in the table. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Disabled => "disabled", + Self::Enabled => "enabled", + } + } + + /// Parse a stored word. + #[must_use] + pub fn parse(s: &str) -> Option { + match s.trim() { + "disabled" => Some(Self::Disabled), + "enabled" => Some(Self::Enabled), + _ => None, + } + } +} + +/// The newest gate row for a topic. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Gate { + /// Current state. + pub state: GateState, + /// Why the operator set it, in their words. Empty when they gave none. + pub reason: String, + /// Who set it (an operator label, never a token). Empty when unset. + pub actor: String, + /// Row id, for an audit trail. + pub id: i64, +} + +impl Gate { + /// Whether submissions are refused right now. + #[must_use] + pub const fn is_disabled(&self) -> bool { + matches!(self.state, GateState::Disabled) + } +} + +/// The gate for `topic_id`, or `None` when no operator ever set one. +/// +/// `Ok(None)` is an **enabled** topic: no row means the switch was never +/// thrown. `Err` is the database refusing, which the caller must not read as +/// "enabled" — see the module docs. +/// +/// # Errors +/// +/// [`InstallError::Db`]. +pub async fn gate(pool: &PgPool, topic_id: &str) -> Result, InstallError> { + let row: Option<(i64, String, String, String)> = sqlx::query_as( + "SELECT id, state, reason, actor FROM proof_topic_gate \ + WHERE topic_id = $1 ORDER BY id DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(pool) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + let Some((id, state, reason, actor)) = row else { + return Ok(None); + }; + // A stored word outside the CHECK cannot reach here; if one somehow did, + // the honest answer is the database refusing, not a silent "enabled". + let state = GateState::parse(&state).ok_or_else(|| { + InstallError::Db(format!( + "gate row {id} for topic {topic_id:?} holds an unknown state" + )) + })?; + Ok(Some(Gate { + state, + reason, + actor, + id, + })) +} + +/// Whether `topic_id` is disabled right now. +/// +/// The submit-path read: `Ok(false)` for a topic with no row, `Ok(true)` for +/// a disabled one, `Err` when the table cannot be read. +/// +/// # Errors +/// +/// [`InstallError::Db`]. +pub async fn disabled(pool: &PgPool, topic_id: &str) -> Result { + Ok(gate(pool, topic_id).await?.is_some_and(|g| g.is_disabled())) +} + +/// Every topic currently disabled, with the operator's reason, in one read. +/// +/// For a listing that annotates a set of topics: one query rather than one +/// per topic. A topic whose newest row is `enabled` is absent, which is the +/// same answer [`disabled`] gives for it. +/// +/// # Errors +/// +/// [`InstallError::Db`], including a stored state the CHECK cannot produce. +pub async fn disabled_topics(pool: &PgPool) -> Result, InstallError> { + let rows: Vec<(String, String, String)> = sqlx::query_as( + "SELECT DISTINCT ON (topic_id) topic_id, state, reason FROM proof_topic_gate \ + ORDER BY topic_id, id DESC", + ) + .fetch_all(pool) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + let mut out = BTreeMap::new(); + for (topic_id, state, reason) in rows { + let state = GateState::parse(&state).ok_or_else(|| { + InstallError::Db(format!( + "the gate row for topic {topic_id:?} holds an unknown state" + )) + })?; + if state == GateState::Disabled { + out.insert(topic_id, reason); + } + } + Ok(out) +} + +/// Append a gate row and return it. +/// +/// Append-only, like the install journal: `disable` after `enable` is a new +/// row, so "who turned it off, when, and why" survives the turn-on. The +/// `reason` is bounded (512 chars) and the `actor` (128); both are stored +/// verbatim, and neither is a secret — the reason is shown to a miner in the +/// 403. +/// +/// # Errors +/// +/// [`InstallError::Db`], including the CHECK refusing an over-long reason. +pub async fn set( + pool: &PgPool, + topic_id: &str, + state: GateState, + reason: &str, + actor: &str, +) -> Result { + let row: (i64,) = sqlx::query_as( + "INSERT INTO proof_topic_gate (topic_id, state, reason, actor) VALUES ($1, $2, $3, $4) \ + RETURNING id", + ) + .bind(topic_id) + .bind(state.as_str()) + .bind(reason.trim()) + .bind(actor.trim()) + .fetch_one(pool) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + Ok(Gate { + state, + reason: reason.trim().to_owned(), + actor: actor.trim().to_owned(), + id: row.0, + }) +} + +/// [`set`] with [`GateState::Disabled`]. +/// +/// # Errors +/// +/// [`InstallError::Db`]. +pub async fn disable( + pool: &PgPool, + topic_id: &str, + reason: &str, + actor: &str, +) -> Result { + set(pool, topic_id, GateState::Disabled, reason, actor).await +} + +/// [`set`] with [`GateState::Enabled`]: the only way to clear a disable. +/// +/// # Errors +/// +/// [`InstallError::Db`]. +pub async fn enable( + pool: &PgPool, + topic_id: &str, + reason: &str, + actor: &str, +) -> Result { + set(pool, topic_id, GateState::Enabled, reason, actor).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_stored_words_round_trip() { + for state in [GateState::Disabled, GateState::Enabled] { + assert_eq!(GateState::parse(state.as_str()), Some(state)); + } + assert_eq!(GateState::parse(" disabled "), Some(GateState::Disabled)); + assert_eq!(GateState::parse("DISABLED"), None); + assert_eq!(GateState::parse(""), None); + } + + #[test] + fn a_missing_row_is_not_a_disable() { + let gate = Gate { + state: GateState::Enabled, + reason: String::new(), + actor: String::new(), + id: 1, + }; + assert!(!gate.is_disabled()); + let gate = Gate { + state: GateState::Disabled, + reason: "incident 42".into(), + actor: "ops".into(), + id: 2, + }; + assert!(gate.is_disabled()); + } +} diff --git a/crates/proof-topic-install/src/lib.rs b/crates/proof-topic-install/src/lib.rs index e2e199c7c..413493497 100644 --- a/crates/proof-topic-install/src/lib.rs +++ b/crates/proof-topic-install/src/lib.rs @@ -26,6 +26,9 @@ //! - [`routes`] is the **read** side of the routes an install recorded: the //! dynamic mux the challenge answers `/challenge/{topic_id}/…` from, behind //! a cache an install invalidates. +//! - [`gate`] is the operator switch that stops a topic taking submissions +//! (`proof-admin topic disable`), read by the challenge on the submit path +//! and fail-closed there. //! - [`proof_topic_sql_guard`] is the migration deny-list (its own crate: it //! is pure text analysis, and keeping it separate means it can be reasoned //! about — and tested — without a database). @@ -48,11 +51,13 @@ clippy::doc_markdown )] +pub mod gate; pub mod handler; pub mod install; pub mod routes; pub mod section; +pub use gate::{disable, disabled, disabled_topics, enable, gate, set, Gate, GateState}; pub use handler::{bound_runner, check_handler, resolve_handler, Handler, HandlerError}; pub use install::{ applied_install, install_history, is_installed, latest_install, topic_routes, ExecutorBinding, @@ -66,8 +71,9 @@ pub use proof_topic_sql_guard::{ }; pub use routes::{is_topic_id, PgTopicRoutes, Resolved, TopicRouteMux, TopicRouteSource}; pub use section::{ - is_api_method, is_relative_api_path, read_section, ApiRoute, Migration, SectionPlan, MAX_APIS, - MAX_MIGRATIONS, MAX_MIGRATION_SQL_BYTES, READ_KEYS, + is_api_method, is_relative_api_path, is_reserved_api_path, read_section, ApiRoute, Migration, + SectionPlan, MAX_APIS, MAX_MIGRATIONS, MAX_MIGRATION_SQL_BYTES, READ_KEYS, + RESERVED_API_PREFIXES, }; /// Why an install refused or failed. @@ -162,6 +168,7 @@ mod tests { assert!(OWNED_TABLES.contains(&"proof_topic_version")); assert!(OWNED_TABLES.contains(&"proof_rule_version")); assert!(OWNED_TABLES.contains(&"proof_topic_install")); + assert!(OWNED_TABLES.contains(&"proof_topic_gate")); } #[test] diff --git a/crates/proof-topic-install/tests/install_engine.rs b/crates/proof-topic-install/tests/install_engine.rs index 93aada76f..8c8369339 100644 --- a/crates/proof-topic-install/tests/install_engine.rs +++ b/crates/proof-topic-install/tests/install_engine.rs @@ -809,3 +809,93 @@ async fn the_engine_drives_the_store_it_is_given() { assert_eq!(row.state, InstallState::Applied.as_str()); tp.drop_schema().await.expect("drop"); } + +/// The operator gate, end to end against Postgres: a topic with no row is +/// enabled, a `disable` row is what the submit path refuses on, and an +/// `enable` row clears it **without deleting the history** — the point of the +/// append-only shape is that an incident review can still see who turned it +/// off, when, and why. +#[tokio::test] +async fn the_operator_gate_is_a_journal_and_the_newest_row_wins() { + let Some((tp, pool)) = test_pool().await else { + return; + }; + // Never thrown: no row, not disabled. + assert!( + !proof_topic_install::disabled(&pool, "tb4") + .await + .expect("read"), + "a topic with no gate row is enabled" + ); + assert!(proof_topic_install::gate(&pool, "tb4") + .await + .expect("read") + .is_none()); + assert!(proof_topic_install::disabled_topics(&pool) + .await + .expect("list") + .is_empty()); + + // Disabled, with a reason: the read the submit path makes. + let disabled = proof_topic_install::disable(&pool, "tb4", "incident 42", "ops") + .await + .expect("disable"); + assert!(disabled.is_disabled()); + assert!(proof_topic_install::disabled(&pool, "tb4") + .await + .expect("read")); + assert_eq!( + proof_topic_install::disabled_topics(&pool) + .await + .expect("list") + .get("tb4") + .map(String::as_str), + Some("incident 42") + ); + + // Enabled again: the newest row wins, and the disable row is still there. + proof_topic_install::enable(&pool, "tb4", "fixed", "ops") + .await + .expect("enable"); + assert!(!proof_topic_install::disabled(&pool, "tb4") + .await + .expect("read")); + assert!(proof_topic_install::disabled_topics(&pool) + .await + .expect("list") + .is_empty()); + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT state, reason FROM proof_topic_gate WHERE topic_id = 'tb4' ORDER BY id", + ) + .fetch_all(&pool) + .await + .expect("history"); + assert_eq!( + rows, + vec![ + ("disabled".to_owned(), "incident 42".to_owned()), + ("enabled".to_owned(), "fixed".to_owned()), + ], + "the journal keeps both rows" + ); + + // The table is topic-scoped: another topic is unaffected by this one. + proof_topic_install::disable(&pool, "tb9", "other incident", "ops") + .await + .expect("disable"); + assert!(proof_topic_install::disabled(&pool, "tb4") + .await + .expect("read") + .eq(&false)); + assert_eq!( + proof_topic_install::disabled_topics(&pool) + .await + .expect("list") + .keys() + .cloned() + .collect::>(), + vec!["tb9".to_owned()] + ); + + tp.drop_schema().await.expect("drop"); +} diff --git a/crates/proof-topic-sql-guard/src/lib.rs b/crates/proof-topic-sql-guard/src/lib.rs index a265049d5..cfceabf0f 100644 --- a/crates/proof-topic-sql-guard/src/lib.rs +++ b/crates/proof-topic-sql-guard/src/lib.rs @@ -105,7 +105,7 @@ pub const OWNED_TABLE_PREFIX: &str = "proof_"; /// The prefix check above is the enforcement; this list is what makes the /// refusal *legible*, and a test asserts it covers every `proof_*` table the /// migrations create. -pub const OWNED_TABLES: [&str; 9] = [ +pub const OWNED_TABLES: [&str; 10] = [ "proof_topic_version", "proof_rule_version", "proof_checklist", @@ -115,6 +115,7 @@ pub const OWNED_TABLES: [&str; 9] = [ "proof_promotion_event", "proof_topic_alias", "proof_topic_install", + "proof_topic_gate", ]; /// Names a topic migration may never name, whatever the verb. From 8805b84995711a59e6e3bb0484cd90af67414190 Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:18:54 +0000 Subject: [PATCH 3/8] feat(proof): baseline -> scorable, sealed by the same call the runtime uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `topic install --drive-rlm` (without `--skip-baseline`) leaves the topic at `baselining` with a measurement in `proof_baseline_measurement`, and nothing in the CLI could turn that into a scorable topic: the operator had to hand-build a `BaselineMeasurement`, find the commitment, and call the runtime's `mark_sealed` from somewhere else. The gap was the wiring, not a missing check. Two commands close it: - `topic baseline ` reads the stored measurement and prints the primary, the rule version, and the **`metrics_commitment`** an `open` document must carry. It is computed from the stored run and the pin's eval image digest, so "the sealed value is the value the RLM measured" is a property of the command. - `topic seal --document [--publish]` builds the measurement from that stored run and hands it to `TopicSetup::mark_sealed` — the same call the runtime's own tests drive, so the CLI cannot seal something the scoring path would refuse. `--publish` then posts the sealed document through the admin route (the install gate still applies: an `open` document needs an `applied` install). The seal is deliberately unwired from any VM: `mark_sealed` provisions nothing, and the setup it is handed carries `UnwiredVmOrchestrator` plus a key probe that refuses, so a future change that made sealing reach for a host would stop instead of running on the control plane. `topic seal` replaces the exit-3 stub, so the CLI's not-implemented machinery is gone: every command exits 0/1/2. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- Cargo.lock | 1 + bins/proof-admin/Cargo.toml | 2 + bins/proof-admin/src/install.rs | 15 +- bins/proof-admin/src/main.rs | 102 ++++--- bins/proof-admin/src/seal.rs | 497 ++++++++++++++++++++++++++++++++ bins/proof-admin/tests/cli.rs | 53 ++-- 6 files changed, 598 insertions(+), 72 deletions(-) create mode 100644 bins/proof-admin/src/seal.rs diff --git a/Cargo.lock b/Cargo.lock index dd06a0448..97886e50f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3690,6 +3690,7 @@ dependencies = [ "crypto", "db", "hex", + "proof-eval", "proof-experiment", "proof-rlm", "proof-rlm-store", diff --git a/bins/proof-admin/Cargo.toml b/bins/proof-admin/Cargo.toml index 5d7ceaa80..33a31f9a5 100644 --- a/bins/proof-admin/Cargo.toml +++ b/bins/proof-admin/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] clap = { version = "4", features = ["derive", "env"] } db = { path = "../../crates/db" } +proof-eval = { path = "../../crates/proof-eval" } proof-rlm = { path = "../../crates/proof-rlm" } proof-rlm-store = { path = "../../crates/proof-rlm-store" } proof-task = { path = "../../crates/proof-task" } @@ -30,6 +31,7 @@ tokio = { version = "1", features = ["macros", "rt-multi-thread"] } [dev-dependencies] crypto = { path = "../../crates/crypto" } +proof-rlm = { path = "../../crates/proof-rlm", features = ["test-fixtures"] } db = { path = "../../crates/db", features = ["testing"] } hex = "0.4" proof-experiment = { path = "../../crates/proof-experiment" } diff --git a/bins/proof-admin/src/install.rs b/bins/proof-admin/src/install.rs index fafaf5f1e..1b82a16bd 100644 --- a/bins/proof-admin/src/install.rs +++ b/bins/proof-admin/src/install.rs @@ -172,7 +172,7 @@ async fn run_real( ) -> Result<(), Failure> { // The bearer and the URL are resolved before anything is written, so a // misconfiguration cannot leave a half-installed topic. - let admin = AdminTarget::resolve(args)?; + let admin = AdminTarget::resolve(args.admin_url, args.admin_token_file)?; let database_url = crate::database_url(opts)?.ok_or_else(|| { Failure::Usage( "a real install writes to the topic registry, so it needs a database: set \ @@ -472,15 +472,18 @@ fn next_steps(plan: &TopicInstallPlan, args: &InstallArgs<'_>) -> String { } /// Where the admin publish call goes, and the bearer it uses. -struct AdminTarget { +pub(crate) struct AdminTarget { base_url: String, token: String, } impl AdminTarget { /// Resolve the URL and bearer, refusing a half-configured pair. - fn resolve(args: &InstallArgs<'_>) -> Result { - let Some(base_url) = args.admin_url.map(str::trim).filter(|u| !u.is_empty()) else { + pub(crate) fn resolve( + admin_url: Option<&str>, + admin_token_file: Option<&Path>, + ) -> Result { + let Some(base_url) = admin_url.map(str::trim).filter(|u| !u.is_empty()) else { return Err(Failure::Usage( "a real install publishes through the admin route, so it needs the master's \ base URL: pass --admin-url (or set PROOF_ADMIN_URL), e.g. \ @@ -489,7 +492,7 @@ impl AdminTarget { .to_owned(), )); }; - let Some(path) = args.admin_token_file else { + let Some(path) = admin_token_file else { return Err(Failure::Usage( "a real install needs the operator bearer for /v1/admin/*: pass \ --admin-token-file (or set PROOF_ADMIN_TOKEN_FILE). The file is read and never \ @@ -524,7 +527,7 @@ impl AdminTarget { } /// Publish the document through the existing admin route. - async fn publish(&self, doc: &proof_task::TopicDocument) -> Result<(), String> { + pub(crate) async fn publish(&self, doc: &proof_task::TopicDocument) -> Result<(), String> { let url = format!("{}{}", self.base_url, proof_topic_bundle::PUBLISH_PATH); let client = reqwest::Client::builder() .timeout(std::time::Duration::from_mins(1)) diff --git a/bins/proof-admin/src/main.rs b/bins/proof-admin/src/main.rs index cf4348eb2..ffb86abec 100644 --- a/bins/proof-admin/src/main.rs +++ b/bins/proof-admin/src/main.rs @@ -27,8 +27,7 @@ //! - It touches no route, no allocator, and no scoring path. //! - It removes none of the compiled-in bindings the current live topic uses. //! -//! Exit codes: `0` ok, `1` error, `2` usage or configuration, `3` not -//! implemented in this slice. +//! Exit codes: `0` ok, `1` error, `2` usage or configuration. #![forbid(unsafe_code)] #![allow(clippy::print_stdout, clippy::print_stderr)] @@ -43,6 +42,7 @@ use proof_topic_bundle::{InstallEnvironment, TopicInstallBundle, TopicInstallPla mod drive; mod install; +mod seal; use install::InstallArgs; @@ -52,8 +52,6 @@ const EXIT_OK: u8 = 0; const EXIT_ERROR: u8 = 1; /// Bad usage or missing configuration. const EXIT_USAGE: u8 = 2; -/// The command exists but its behaviour belongs to a later slice. -const EXIT_NOT_IMPLEMENTED: u8 = 3; /// Proof operator CLI. #[derive(Debug, Parser)] @@ -72,10 +70,12 @@ Resolve the publish call and host env without touching anything: List the installed topics (a read-only view of proof_topic_version): proof-admin topic list -Nothing here writes a topic, opens a route, or changes how a score is -computed. `install` prints the publish call and the host env for an operator -to run; `disable` / `enable` throw the operator gate the challenge reads on -the submit path; `topic seal` exits 3 as not-implemented." +Every command is implemented. Nothing here writes a topic document, opens a +route, or changes how a score is computed: `install` publishes the document +the bundle carries and applies the bundle's RLM section, `disable` / `enable` +throw the operator gate the challenge reads on the submit path, and +`seal` records the operator's seal of the baseline the RLM measured and +publishes the open document." )] struct Cli { /// Postgres URL for the topic registry view. Falls back to `BASE_DATABASE_URL`. @@ -225,13 +225,44 @@ enum TopicCmd { #[arg(long, env = "PROOF_GATE_ACTOR", value_name = "LABEL")] actor: Option, }, - /// Not implemented in this slice. + /// Show what the RLM measured, and the commitment an open document must + /// seal. The read half of `topic seal`. + Baseline { + /// Topic slug, or an alias of one. + topic_id: String, + /// Pin the document is checked against. Defaults to `config/proof-pin.toml`. + #[arg(long, value_name = "PATH", default_value = "config/proof-pin.toml")] + pin: PathBuf, + }, + /// Seal the RLM's measured baseline and open the topic. + /// + /// Takes the signed `status: open` document whose `baseline` carries the + /// commitment `topic baseline` printed, checks it exactly the way the + /// runtime does (`TopicSetup::mark_sealed`: open, valid on this host, + /// operator-signed, and sealing the value the RLM measured), records the + /// move to `open`, and — with `--publish` — publishes it through the + /// admin route, which is what makes the topic reachable and scorable. Seal { - /// Topic slug. + /// Topic slug, or an alias of one. topic_id: String, - /// Measured baseline primary. - #[arg(long, value_name = "VALUE")] - value: f64, + /// The signed `status: open` document (JSON). + #[arg(long, value_name = "PATH")] + document: PathBuf, + /// Pin the document is checked against. Defaults to `config/proof-pin.toml`. + #[arg(long, value_name = "PATH", default_value = "config/proof-pin.toml")] + pin: PathBuf, + /// Publish the sealed document through the admin route. + #[arg(long)] + publish: bool, + /// Master base URL for the publish call, e.g. + /// `http://10.116.0.3:8080` (the gateway) or + /// `http://127.0.0.1:8100` (the challenge service directly). + #[arg(long, env = "PROOF_ADMIN_URL", value_name = "URL")] + admin_url: Option, + /// File holding the operator bearer for `/v1/admin/*`. Never logged, + /// never printed. Defaults to `PROOF_ADMIN_TOKENS_FILE`. + #[arg(long, env = "PROOF_ADMIN_TOKEN_FILE", value_name = "PATH")] + admin_token_file: Option, }, } @@ -284,10 +315,6 @@ fn main() -> ExitCode { eprintln!("proof-admin: {msg}"); ExitCode::from(EXIT_USAGE) } - Err(Failure::NotImplemented(msg)) => { - eprintln!("proof-admin: {msg}"); - ExitCode::from(EXIT_NOT_IMPLEMENTED) - } Err(Failure::Error(msg)) => { eprintln!("proof-admin: {msg}"); ExitCode::from(EXIT_ERROR) @@ -300,8 +327,6 @@ fn main() -> ExitCode { enum Failure { /// Bad usage or missing configuration. Usage(String), - /// A later slice owns this behaviour. - NotImplemented(String), /// Anything else (bad bundle, refused document, database error). Error(String), } @@ -384,10 +409,28 @@ async fn run_topic(opts: &Options, cmd: &TopicCmd) -> Result<(), Failure> { ) .await } - TopicCmd::Seal { topic_id, value } => Err(not_implemented( - &format!("topic seal (value {value})"), + TopicCmd::Seal { topic_id, - )), + document, + pin, + publish, + admin_url, + admin_token_file, + } => { + seal::seal( + opts, + &seal::SealArgs { + topic_id, + document, + pin, + publish: *publish, + admin_url: admin_url.as_deref(), + admin_token_file: admin_token_file.as_deref(), + }, + ) + .await + } + TopicCmd::Baseline { topic_id, pin } => seal::baseline(opts, topic_id, pin).await, } } @@ -451,15 +494,6 @@ async fn run_alias(opts: &Options, cmd: &AliasCmd) -> Result<(), Failure> { } } -/// A stub that names what is missing instead of guessing. -fn not_implemented(command: &str, topic_id: &str) -> Failure { - Failure::NotImplemented(format!( - "`{command}` for topic {topic_id:?} is not implemented in this slice (P0: bundle + \ - admin CLI skeleton). Nothing was changed. A topic's lifecycle is the signed document's \ - `status`; re-sign and re-publish through POST /v1/admin/proof/topics instead." - )) -} - /// Read a bundle file. pub(crate) fn load_bundle(path: &Path) -> Result { let body = std::fs::read_to_string(path) @@ -824,7 +858,7 @@ async fn open_store(opts: &Options) -> Result, Failure> { /// /// The gate commands write (`proof_topic_gate`) as well as read, so they need /// the pool itself and not only the registry trait object. -async fn open_pool(opts: &Options) -> Result { +pub(crate) async fn open_pool(opts: &Options) -> Result { let Some(url) = database_url(opts)? else { return Err(Failure::Usage( "this command reads the topic registry and needs a database: set \ @@ -988,7 +1022,7 @@ fn summarize(row: &TopicVersionRow) -> String { } /// Lifecycle word, matching the wire spelling the document uses. -fn status_word(status: proof_task::TopicStatus) -> &'static str { +pub(crate) fn status_word(status: proof_task::TopicStatus) -> &'static str { match status { proof_task::TopicStatus::Draft => "draft", proof_task::TopicStatus::Open => "open", @@ -1017,7 +1051,7 @@ pub(crate) fn print_json(value: &T) -> Result<(), Failure> Ok(()) } -fn dash_if_empty(s: &str) -> String { +pub(crate) fn dash_if_empty(s: &str) -> String { if s.trim().is_empty() { "-".to_owned() } else { diff --git a/bins/proof-admin/src/seal.rs b/bins/proof-admin/src/seal.rs new file mode 100644 index 000000000..e0e2dbb79 --- /dev/null +++ b/bins/proof-admin/src/seal.rs @@ -0,0 +1,497 @@ +//! `topic seal` and `topic baseline` — the step between "the RLM measured a +//! baseline" and "the topic is scorable". +//! +//! `proof-admin topic install --drive-rlm` (without `--skip-baseline`) leaves +//! the topic at **`baselining`** with a measured baseline in +//! `proof_baseline_measurement`. Nothing is scorable yet: the signed document +//! is still the bundle's `draft`, and a `draft` takes no submissions. The +//! remaining ceremony is the operator's, and it is two commands: +//! +//! ```text +//! # 1. What did the RLM measure, and what must the open document seal? +//! proof-admin topic baseline tb4 +//! +//! # 2. Sign the open document carrying that commitment, then: +//! proof-admin topic seal tb4 --document open.json --publish \ +//! --admin-url https:// --admin-token-file /run/base/proof/admin_tokens +//! ``` +//! +//! [`baseline`] is the read half: the measured primary, the rule version it +//! was measured under, and the `metrics_commitment` an `open` document must +//! carry — the value that goes into the draft before it is signed (the CLI +//! never signs: the `proof` key stays with the operator, and `xtask +//! proof-topic` is what signs a draft). +//! +//! [`seal`] is the write half, and it is deliberately thin: it hands the +//! signed document and the measurement to [`TopicSetup::mark_sealed`], the +//! **same** call the challenge's own tests drive, so the CLI cannot seal +//! something the runtime would refuse. `mark_sealed` is what checks, in +//! order: the document is `status: open`; it validates as an open topic on +//! this host (a registered custom id, a sealed baseline, tighten-only +//! floors); its signature verifies under the pin's topic key; the sealed +//! measurement binds to the document and its `custom_value` is the primary +//! the RLM actually measured in the topic VM; and the lifecycle is at +//! `baselining`. Only then does the topic move to `open`, and only then can +//! `--publish` reach the admin route — which itself refuses an `open` +//! document whose install is not `applied`. +//! +//! # Why the measurement is built here, and not handed in +//! +//! A `BaselineMeasurement` is the seal: it carries the eval image digest, the +//! topic's holdout commitment, the metric vector, and the primary. For a +//! custom-family topic the primary is `custom_value` (the NLL fields are not +//! this family's metric — they are zero, which is the shape the RLM's own +//! e2e drives). Building it from the **stored** measurement rather than from +//! an operator-supplied file is what makes "the sealed value is the value the +//! RLM measured" a property of the command instead of a promise: the number +//! comes from `proof_baseline_measurement`, and `mark_sealed` compares it +//! against the document. + +use std::path::Path; + +use proof_eval::BaselineMeasurement; +use proof_rlm_store::{BaselineRow, PgRlmStore, RlmStore}; +use proof_task::{HoldoutSplit, ProofPin, TopicDocument, TopicStatus}; +use proof_topic_setup::TopicSetup; + +use crate::install::AdminTarget; +use crate::{Failure, Options}; + +/// Everything `topic seal` was asked to do. +pub struct SealArgs<'a> { + /// Topic slug, or an alias of one. + pub topic_id: &'a str, + /// The signed `status: open` document. + pub document: &'a Path, + /// Pin the document is checked against. + pub pin: &'a Path, + /// Publish the sealed document through the admin route. + pub publish: bool, + /// Master base URL for the publish call (with `--publish`). + pub admin_url: Option<&'a str>, + /// File holding the operator bearer (with `--publish`). + pub admin_token_file: Option<&'a Path>, +} + +/// `topic baseline`: what the RLM measured, and what to seal. +/// +/// # Errors +/// +/// [`Failure::Usage`] without a database, [`Failure::Error`] when the topic +/// or its measurement cannot be read. +pub async fn baseline(opts: &Options, topic_id: &str, pin_path: &Path) -> Result<(), Failure> { + let pool = crate::open_pool(opts).await?; + let store = PgRlmStore::new(pool.clone()); + let pin = crate::load_pin(pin_path)?; + let (canonical, _, document) = resolve_topic(&store, topic_id).await?; + let Some(measured) = store + .baseline(&canonical) + .await + .map_err(|e| Failure::Error(format!("{canonical} baseline: {e}")))? + else { + return Err(Failure::Error(format!( + "no baseline measured for topic {canonical:?}. It is written by the RLM's baseline \ + job: run `proof-admin topic install --bundle --env --drive-rlm \ + --owner-approved` (without --skip-baseline) first. Nothing to seal yet." + ))); + }; + let commitment = seal_measurement(&pin, &document, &measured).commitment(); + if opts.json { + crate::print_json(&serde_json::json!({ + "topic_id": canonical, + "rules_version": measured.rules_version, + "primary_value": measured.primary_value, + "metric_primary": document.metric.primary, + "custom_id": document.metric.custom_id, + "holdout_commitment": document.holdout_commitment, + "metrics_commitment": commitment, + "document_status": document.status, + "document_version_signature": document.signature, + "next": next_seal_steps(&canonical, &commitment), + }))?; + return Ok(()); + } + println!("topic {canonical} — measured baseline"); + println!(" primary_value {}", measured.primary_value); + println!(" metric_primary {}", document.metric.primary); + println!( + " custom_id {}", + crate::dash_if_empty(&document.metric.custom_id) + ); + println!(" rules_version {}", measured.rules_version); + println!(" holdout {}", document.holdout_commitment); + println!( + " document_status {}", + crate::status_word(document.status) + ); + println!(); + println!("An `open` document must seal this measurement. Its baseline block needs:"); + println!(" metrics_commitment {commitment}"); + println!( + " script_sha256 {}", + crate::dash_if_empty(&document.baseline.script_sha256) + ); + println!(); + println!("{}", next_seal_steps(&canonical, &commitment)); + Ok(()) +} + +/// `topic seal`: record the operator's seal and open the topic. +/// +/// # Errors +/// +/// [`Failure::Usage`] without a database or without a publish target when +/// `--publish` was given, [`Failure::Error`] for a refused document, a +/// missing measurement, a lifecycle that is not at `baselining`, or a +/// refused publish. +pub async fn seal(opts: &Options, args: &SealArgs<'_>) -> Result<(), Failure> { + let pool = crate::open_pool(opts).await?; + let store = PgRlmStore::new(pool.clone()); + let (canonical, version, _) = resolve_topic(&store, args.topic_id).await?; + let pin = crate::load_pin(args.pin)?; + let body = std::fs::read_to_string(args.document) + .map_err(|e| Failure::Error(format!("read {}: {e}", args.document.display())))?; + let document: TopicDocument = serde_json::from_str(&body) + .map_err(|e| Failure::Error(format!("{}: {e}", args.document.display())))?; + if document.id != canonical { + return Err(Failure::Usage(format!( + "{} carries topic {:?}, but this command is sealing {canonical:?}{}. Nothing was \ + changed.", + args.document.display(), + document.id, + alias_note(args.topic_id, &canonical) + ))); + } + if document.status != TopicStatus::Open { + return Err(Failure::Usage(format!( + "{} is `{}`, not `open`. Sealing opens a topic, so the document has to be the open \ + one: set `status: open`, seal `baseline.metrics_commitment` from `proof-admin topic \ + baseline`, sign it, and re-run. Nothing was changed.", + args.document.display(), + crate::status_word(document.status) + ))); + } + let Some(measured) = store + .baseline(&canonical) + .await + .map_err(|e| Failure::Error(format!("{canonical} baseline: {e}")))? + else { + return Err(Failure::Error(format!( + "no baseline measured for topic {canonical:?}, so there is nothing to seal. Run the \ + install with --drive-rlm (without --skip-baseline) first. Nothing was changed." + ))); + }; + let sealed = seal_measurement(&pin, &document, &measured); + // The one call that decides: the same `mark_sealed` the runtime's own + // tests drive, so a document this command accepts is one the scoring path + // would accept. + let registered = crate::registered_custom_from_env(); + let registered: Vec<&str> = registered.iter().map(String::as_str).collect(); + let setup = seal_setup(store); + let state = setup + .mark_sealed(&document, &pin, ®istered, &sealed) + .await + .map_err(|e| Failure::Error(seal_failure(&e, &canonical)))?; + let commitment = sealed.commitment(); + + if args.publish { + let admin = AdminTarget::resolve(args.admin_url, args.admin_token_file)?; + admin + .publish(&document) + .await + .map_err(|e| Failure::Error(publish_failure(&e, &canonical)))?; + } + if opts.json { + crate::print_json(&serde_json::json!({ + "ok": true, + "topic_id": canonical, + "state": format!("{state:?}").to_lowercase(), + "document_version": version + 1, + "metrics_commitment": commitment, + "primary_value": measured.primary_value, + "published": args.publish, + }))?; + return Ok(()); + } + println!("topic {canonical} sealed and opened."); + println!(" state open"); + println!(" document_version {}", version + 1); + println!(" primary_value {}", measured.primary_value); + println!(" commitment {commitment}"); + if args.publish { + println!(" published yes (the topic's routes and document are live)"); + } else { + println!(" published no (--publish was not given)"); + } + println!(); + println!("{}", after_seal(&canonical, args.publish)); + Ok(()) +} + +/// The `TopicSetup` `mark_sealed` needs: the store, and a VM boundary that is +/// deliberately **unwired**. +/// +/// Sealing touches no VM — it validates a signed document against a stored +/// measurement — so handing it a stub that refuses every call is the honest +/// wiring: if a future change made sealing provision something, it would stop +/// here instead of quietly running on the control-plane host. +fn seal_setup(store: PgRlmStore) -> TopicSetup { + TopicSetup { + orchestrator: std::sync::Arc::new(proof_rlm::UnwiredVmOrchestrator), + store: std::sync::Arc::new(store) as std::sync::Arc, + template: proof_rlm::VmTemplate::from_env(), + experiments: proof_rlm::ExperimentPolicy::default(), + owner: std::sync::Arc::new(proof_rlm::StaticOwnerHook( + proof_rlm::OwnerDecision::Approve, + )), + keys: std::sync::Arc::new(SealKeys), + spend_cap_usd: None, + skip_baseline: false, + } +} + +/// A key probe for a path that never asks for a key: sealing is not a +/// provisioning step, so nothing here may reach the owner's key file. +struct SealKeys; + +impl proof_rlm::OwnerKeysProbe for SealKeys { + fn owner_keys_present(&self) -> Result<(), proof_rlm::HookError> { + Err(proof_rlm::HookError::Failed( + "`topic seal` provisions nothing and asks for no owner key".into(), + )) + } +} + +/// The measurement an `open` document must seal, built from the RLM's own +/// stored run. +/// +/// `custom_value` is the measured primary: for the custom family that *is* +/// the metric. The NLL fields are zero because this family does not measure +/// them, and [`BaselineMeasurement::verify`] checks the split **count** +/// against the topic's holdout shape, so the vector has the right shape +/// without inventing numbers that would then be signed. The eval image digest +/// is the pin's, so the seal binds to the image the run was made under. +fn seal_measurement( + pin: &ProofPin, + document: &TopicDocument, + measured: &BaselineRow, +) -> BaselineMeasurement { + BaselineMeasurement { + eval_image_digest: pin.eval_image_digest.clone(), + topic_id: document.id.clone(), + holdout_commitment: document.holdout_commitment.clone(), + holdout_nll: 0.0, + split_nll: HoldoutSplit::SCORED + .iter() + .map(|s| (s.as_str().to_owned(), 0.0)) + .collect(), + tokens_per_sec: None, + step_latency_ms: None, + custom_value: Some(measured.primary_value), + } +} + +/// Resolve an alias and read the topic's newest document, with its version. +async fn resolve_topic( + store: &PgRlmStore, + topic_id: &str, +) -> Result<(String, u32, TopicDocument), Failure> { + let resolved = store + .resolve_alias(topic_id) + .await + .map_err(|e| Failure::Error(format!("resolve {topic_id}: {e}")))?; + let canonical = resolved.as_deref().unwrap_or(topic_id); + let row = store + .latest_topic(canonical) + .await + .map_err(|e| Failure::Error(format!("{canonical}: {e}")))?; + let Some((version, document)) = row else { + return Err(Failure::Error(format!( + "no installed topic {topic_id:?}{}. Use `proof-admin topic list` to see the exact \ + ids.", + alias_note(topic_id, canonical) + ))); + }; + Ok((canonical.to_owned(), version, document)) +} + +fn alias_note(topic_id: &str, canonical: &str) -> String { + if topic_id == canonical { + String::new() + } else { + format!(" (alias of {canonical:?})") + } +} + +/// What the operator does with the commitment `topic baseline` printed. +fn next_seal_steps(topic_id: &str, commitment: &str) -> String { + format!( + "1. Put that commitment into the draft's `baseline.metrics_commitment`, set `status: \ + open`, and sign it (the `proof` key stays with you; `xtask proof-topic` signs a draft).\n\ + 2. Seal it and publish:\n proof-admin topic seal {topic_id} --document \ + --publish --admin-url --admin-token-file \n\ + 3. Confirm the host is scorable: `ctx proof status` reports `can_score`, or read \ + `GET /v1/status` (commitment {commitment})." + ) +} + +/// What to do once the topic is open. +fn after_seal(topic_id: &str, published: bool) -> String { + if published { + format!( + "The topic is open and published, so miners can submit to it. Confirm the host \ + reports it scorable:\n GET /v1/status → `can_score`, `open_topics` contains \ + {topic_id:?}\n ctx proof status (or ctx proof topics) from a miner host" + ) + } else { + format!( + "The topic is open in the registry, but the published document is still the old one \ + — re-run with --publish (the seal is already recorded; the same document publishes \ + as-is):\n proof-admin topic seal {topic_id} --document --publish \ + --admin-url --admin-token-file " + ) + } +} + +/// Turn a `mark_sealed` refusal into an operator instruction. +fn seal_failure(err: &proof_topic_setup::SetupError, topic_id: &str) -> String { + use proof_topic_setup::SetupError; + let guidance = match err { + SetupError::NotOpen(_) => { + "The document is not `status: open`. Sealing opens a topic; a draft is not one." + } + SetupError::Topic(e) => { + return format!( + "the document was refused: {e}\n Nothing moved: the topic is still at its \ + previous state and the open version was not stored.\n What the open document \ + needs: a registered custom id for this host (PROOF_VM_RUNNER_CUSTOM_IDS), a \ + sealed baseline (`script_sha256` + `metrics_commitment`), and floors that only \ + tighten the pin. Fix the draft, re-sign, and re-run." + ); + } + SetupError::Seal(e) => { + return format!( + "the seal does not bind: {e}\n Nothing moved. The usual cause is a \ + `metrics_commitment` that is not the one `proof-admin topic baseline` printed \ + for this topic (it is over the measured vector, not over the file), or a \ + `custom_value` the RLM never measured. Re-read the measurement and re-sign." + ); + } + SetupError::State(proof_rlm::StateError::Illegal { from, .. }) => { + return format!( + "the topic's lifecycle is at {from:?}, not `baselining`, so there is no \ + measured baseline to seal against. Either the install never drove the RLM \ + (`--drive-rlm`) or it was sealed already. `proof-admin topic install-log \ + --topic {topic_id}` shows the install; `proof-admin topic show {topic_id}` \ + shows the current state." + ); + } + _ => "Nothing moved; the topic is still at its previous state.", + }; + format!("the seal was refused: {err}\n {guidance}") +} + +/// Turn a publish refusal into an operator instruction. +fn publish_failure(why: &str, topic_id: &str) -> String { + format!( + "the seal is recorded and the topic is **open**, but the publish failed: {why}\n The \ + published document is still the previous one, so miners cannot reach the open topic \ + yet. Nothing is wrong with the seal: re-run the same command with --publish (the seal \ + is idempotent from `open` — it publishes the document you pass) once the admin URL or \ + bearer is fixed.\n If the route refused an `open` document because the install is not \ + `applied`, finish the install first: `proof-admin topic install-log --topic \ + {topic_id}`." + ) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used)] + + use super::*; + use proof_rlm::fixtures; + use proof_task::MetricFamily; + + /// The measured baseline the RLM leaves in `proof_baseline_measurement`. + fn measured(primary: f64) -> BaselineRow { + BaselineRow { + topic_id: "tb4".into(), + rules_version: 3, + primary_value: primary, + report: proof_rlm::CustomRunReport { + schema_version: proof_rlm::RUN_REPORT_SCHEMA, + topic_id: "tb4".into(), + custom_id: "tb4-metric".into(), + submission_digest: "11".repeat(32), + artifact_digest: "22".repeat(32), + rules_version: 3, + primary_value: primary, + claim_holds: true, + sandboxed: true, + flops_used: None, + evidence: std::collections::BTreeMap::new(), + results: None, + }, + } + } + + fn document(pin: &ProofPin) -> TopicDocument { + let mut doc = TopicDocument { + id: "tb4".into(), + status: TopicStatus::Open, + ..TopicDocument::default() + }; + doc.metric.family = MetricFamily::Custom; + doc.metric.custom_id = "tb4-metric".into(); + doc.holdout_commitment = "cd".repeat(32); + doc.baseline.script_sha256 = "ee".repeat(32); + doc.baseline.metrics_commitment.clear(); + let _ = pin; + doc + } + + /// The two halves agree: the commitment `topic baseline` prints is the one + /// `mark_sealed` accepts when the document carries it, and a document that + /// seals anything else is refused. This is the wiring the live ceremony + /// depends on, checked without a database or a signature. + #[test] + fn the_commitment_we_print_is_the_one_mark_sealed_verifies() { + let pin = fixtures::pin(); + let row = measured(0.42); + let sealed = seal_measurement(&pin, &document(&pin), &row); + assert_eq!(sealed.custom_value, Some(0.42)); + assert_eq!( + sealed.eval_image_digest, pin.eval_image_digest, + "the seal binds to the pinned eval image" + ); + assert_eq!( + sealed.split_nll.len(), + HoldoutSplit::SCORED.len(), + "the vector has the topic's holdout shape" + ); + + // The document that carries what we printed verifies. + let mut open = document(&pin); + open.baseline.metrics_commitment = sealed.commitment(); + sealed + .verify(&pin, &open) + .expect("the printed commitment is the one the runtime accepts"); + + // A document sealing a different value is refused — the seal is over + // the measured vector, not over the file. + let mut wrong = open.clone(); + wrong.baseline.metrics_commitment = "00".repeat(32); + assert!(sealed.verify(&pin, &wrong).is_err()); + + // And a measurement that is not the measured primary is refused even + // when the document agrees with itself. + let other = seal_measurement(&pin, &document(&pin), &measured(0.99)); + let mut open_other = document(&pin); + open_other.baseline.metrics_commitment = other.commitment(); + assert!(other.verify(&pin, &open_other).is_ok()); + assert!( + other.verify(&pin, &open).is_err(), + "0.99 cannot verify against a document sealing 0.42" + ); + } +} diff --git a/bins/proof-admin/tests/cli.rs b/bins/proof-admin/tests/cli.rs index 26a25f897..5eaca94f6 100644 --- a/bins/proof-admin/tests/cli.rs +++ b/bins/proof-admin/tests/cli.rs @@ -1,13 +1,15 @@ -//! Process-level tests for `proof-admin` (dynamic-topics P0). +//! Process-level tests for `proof-admin`. //! //! The commands that must work end to end are `topic validate` and //! `topic install --dry-run`: both run the same acceptance checks the existing -//! `POST /v1/admin/proof/topics` route runs, and neither touches a host. The -//! stubs must fail closed with exit code 3 rather than doing something partial. +//! `POST /v1/admin/proof/topics` route runs, and neither touches a host. Every +//! command that writes — a real install, the gate (`disable` / `enable`), the +//! seal — is driven here only as far as its **refusals**: no test in this file +//! reaches a master, spends, or provisions. //! -//! A real install is deliberately **not** implemented in this slice, so the -//! test asserts it refuses rather than writing anything; the registry view -//! (`topic list` / `topic show`) is covered against Postgres in +//! A real install is deliberately **not** run here, so the test asserts it +//! refuses rather than writing anything; the registry view (`topic list` / +//! `topic show`) is covered against Postgres in //! `crates/proof-rlm-store/tests/store_contract.rs` and by one DB-gated test //! here. @@ -22,8 +24,6 @@ use std::sync::{Arc, Mutex}; const EXIT_ERROR: i32 = 1; /// Exit code for bad usage or missing configuration. const EXIT_USAGE: i32 = 2; -/// Exit code for a command a later slice owns. -const EXIT_NOT_IMPLEMENTED: i32 = 3; fn workdir(tag: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!( @@ -971,28 +971,19 @@ fn database_url_and_file_are_mutually_exclusive() { fs::remove_dir_all(&dir).ok(); } +/// `topic seal` takes the signed open document; without one it is a usage +/// error, not a guess. (The old stub exited 3 with "not implemented"; every +/// command in this CLI is implemented now, so the exit codes are 0/1/2.) #[test] -fn seal_still_fails_closed_with_exit_3() { - let args = vec!["topic", "seal", "tb4", "--value", "0.42"]; +fn seal_needs_the_open_document() { + let args = vec!["topic", "seal", "tb4"]; let out = run(&args); - assert_eq!( - code(&out), - EXIT_NOT_IMPLEMENTED, - "{args:?}: {}", - stderr(&out) - ); + assert_eq!(code(&out), EXIT_USAGE, "{args:?}: {}", stderr(&out)); let err = stderr(&out); - assert!( - err.contains("not implemented in this slice"), - "{args:?}: {err}" - ); - assert!( - err.contains("Nothing was changed"), - "a stub must say it changed nothing: {args:?}: {err}" - ); + assert!(err.contains("--document"), "{args:?}: {err}"); assert!( stdout(&out).is_empty(), - "a stub prints nothing to stdout: {args:?}" + "nothing is reported as changed: {args:?}" ); } @@ -1024,7 +1015,7 @@ fn disable_and_enable_need_the_gate_database() { } #[test] -fn help_lists_every_subcommand_and_says_what_is_not_implemented() { +fn help_lists_every_subcommand() { let out = run(&["topic", "--help"]); assert_eq!(code(&out), 0); let body = stdout(&out); @@ -1036,17 +1027,15 @@ fn help_lists_every_subcommand_and_says_what_is_not_implemented() { "show", "enable", "disable", + "baseline", "seal", ] { assert!(body.contains(sub), "missing subcommand {sub} in:\n{body}"); } - // `topic --help` lists subcommands; the flags live on `install --help`. + // Every command is implemented, so nothing advertises a stub. assert!( - !body.contains("not implemented in this slice") - || body.contains("enable") - || body.contains("disable") - || body.contains("seal"), - "the stubs must be the ones that say so:\n{body}" + !body.contains("not implemented in this slice"), + "no command is a stub any more:\n{body}" ); let out = run(&["topic", "install", "--help"]); From f703108e8ec72ab19120443faa67961643d91b2e Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:32:02 +0000 Subject: [PATCH 4/8] feat(proof): one VM per submission, and two submissions at once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps, one change: the allocator pin was recorded and never enforced, and the scorer held a **per-topic** lease for the whole run — so two submissions to one topic were a queue, and `vms_per_submission = 1` was a journal entry nothing read. Allocator pin (enforced): - The submit path reads the pin from the topic's newest install row and refuses a topic installed under anything but 1 with **503, no row, nonce unspent** — this build runs one VM per submission, and running a topic under a binding its install never recorded would make the journal a lie. A topic with no install row (staged before the journal existed) is not refused for that. - The read is folded into the operator gate (`InstallJournal::submit_gate`), so a submission costs one journal read, not two. Concurrency (the two-VM path): - `RlmScorer` leases **per submission** (`(topic_id, digest)`), not per topic: two submissions of one topic are two paid runs, each in its own Firecracker VM. The topic's shared state — the lifecycle and the best pointer — is written under a separate topic lock taken only for a write, so a paid run never blocks another run. - The lifecycle is a *phase* marker, not a per-run counter: a second submission while one is evaluating keeps `evaluating`, and the scorer's own in-flight count is what decides whether a persisted `evaluating` is stale (a dead run) or live (another submission working). `recover_stale` no longer moves the phase out from under a run that is still going. - The promotion compare-and-swap still refuses a crown whose best pointer moved, so parallel runs cannot both crown off one stale bar. Tests: two concurrent submits reach the runner as two runs (HTTP path), two submissions of an experiment topic ask for **two** experiment VMs and both are destroyed after their jobs, and a topic installed under another pin is refused. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- Cargo.lock | 1 + crates/proof-challenge/src/topic_routes.rs | 28 ++- crates/proof-http/Cargo.toml | 3 + crates/proof-http/src/lib.rs | 202 ++++++++++++--- crates/proof-rlm-scorer/src/scorer.rs | 165 +++++++++++-- crates/proof-rlm-scorer/tests/rlm_e2e.rs | 273 +++++++++++++++++---- crates/proof-rlm/src/state.rs | 68 ++++- crates/proof-rlm/src/vm.rs | 14 ++ 8 files changed, 646 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 97886e50f..0e82a5e51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3916,6 +3916,7 @@ dependencies = [ "proof-store", "proof-submit", "proof-task", + "proof-topic-install", "proof-vm-proto", "reqwest 0.12.28", "serde", diff --git a/crates/proof-challenge/src/topic_routes.rs b/crates/proof-challenge/src/topic_routes.rs index 22b012c7d..a06ec9454 100644 --- a/crates/proof-challenge/src/topic_routes.rs +++ b/crates/proof-challenge/src/topic_routes.rs @@ -121,14 +121,28 @@ impl proof_http::InstallJournal for PgInstallJournal { .map_err(|e| e.to_string()) } - async fn disabled(&self, topic_id: &str) -> Result, String> { - proof_topic_install::gate(&self.pool, topic_id) + async fn submit_gate(&self, topic_id: &str) -> Result { + let disabled = proof_topic_install::gate(&self.pool, topic_id) .await - .map(|gate| { - gate.filter(proof_topic_install::Gate::is_disabled) - .map(|g| g.reason) - }) - .map_err(|e| e.to_string()) + .map_err(|e| e.to_string())? + .filter(proof_topic_install::Gate::is_disabled) + .map(|g| g.reason); + // The allocator pin the topic's newest install recorded, read from the + // same journal the publish gate reads: a topic installed under a + // different pin is one this host cannot run. + let vms_per_submission = proof_topic_install::latest_install(&self.pool, topic_id) + .await + .map_err(|e| e.to_string())? + .and_then(|row| { + row.binding + .get("vms_per_submission") + .and_then(serde_json::Value::as_u64) + .and_then(|n| u32::try_from(n).ok()) + }); + Ok(proof_http::SubmitGate { + disabled_reason: disabled, + vms_per_submission, + }) } async fn disabled_topics(&self) -> Result, String> { diff --git a/crates/proof-http/Cargo.toml b/crates/proof-http/Cargo.toml index 5376449e5..044d46864 100644 --- a/crates/proof-http/Cargo.toml +++ b/crates/proof-http/Cargo.toml @@ -19,6 +19,9 @@ proof-score = { path = "../proof-score" } proof-store = { path = "../proof-store" } proof-submit = { path = "../proof-submit" } proof-task = { path = "../proof-task" } +# The allocator pin (`VMS_PER_SUBMISSION`) the submit path enforces: one +# source of truth with the install that records it. +proof-topic-install = { path = "../proof-topic-install" } proof-vm-proto = { path = "../proof-vm-proto" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 00f1149cc..637dcedbe 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -179,6 +179,35 @@ pub trait VmOrchestratorProbe: Send + Sync { async fn probe(&self) -> VmOrchestratorReport; } +/// What the **submit** path needs from the journal, in one read. +/// +/// Both fields are about the topic's install and the operator's switch, and +/// both are read before anything is spent, so they are one call: a submission +/// costs one gate read, not two. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SubmitGate { + /// The operator's reason, when the topic is disabled (`topic disable`). + pub disabled_reason: Option, + /// The `vms_per_submission` the topic's newest install recorded. + /// + /// `None` when the topic has no install row or the row predates the + /// field. `Some(n)` with `n != 1` is refused on the submit path: this + /// build runs **one VM per submission**, and a topic installed with a + /// different pin is one it cannot honour. + pub vms_per_submission: Option, +} + +impl SubmitGate { + /// The operator gate, as the submit path reads it. + #[must_use] + pub fn disabled(reason: impl Into) -> Self { + Self { + disabled_reason: Some(reason.into()), + ..Self::default() + } + } +} + /// Whether a topic's install reached `applied`, as the publish route reads it. /// /// A trait rather than a pool so the route can be exercised without a @@ -194,26 +223,20 @@ pub trait InstallJournal: Send + Sync { /// publish: an unreadable journal is not an installed topic. async fn applied(&self, topic_id: &str) -> Result; - /// `Ok(Some(reason))` when an operator **disabled** `topic_id`, `Ok(None)` - /// when the topic was never disabled (or was enabled again). - /// - /// The submit path reads this and refuses a disabled topic with a 403 - /// carrying the reason — before the single-use nonce is spent, so a miner - /// loses nothing to an operator's switch. It is the operator's incident - /// switch (`proof-admin topic disable`), and it needs no re-sign, no - /// restart, and no redeploy. + /// The operator gate and the allocator pin for one topic, for the submit + /// path. /// /// # Errors /// /// The reason the gate could not be read. The caller answers **503**: an /// unreadable gate is not an enabled topic. - async fn disabled(&self, topic_id: &str) -> Result, String>; + async fn submit_gate(&self, topic_id: &str) -> Result; /// Every topic currently disabled, with the operator's reason. /// /// One read for the public listing, which annotates each topic with the /// flag rather than paying a query per topic. The submit path uses - /// [`Self::disabled`] for the single topic it is admitting. + /// [`Self::submit_gate`] for the single topic it is admitting. /// /// # Errors /// @@ -1788,13 +1811,18 @@ async fn install_gate(st: &AppState, topic_id: &str) -> Result<(), String> { } } -/// The operator gate on the **submit** path: is this topic disabled? +/// The operator gate on the **submit** path: is this topic disabled, and is +/// its allocator pin one this build runs? /// /// `Ok(())` admits the submission. A disabled topic is a **403** naming the -/// operator's reason; an unreadable gate is a **503** — never an admission, -/// and never a 404 that would read as "no such topic". A host that resolved -/// no journal (no database) has no gate to read: it also has no published -/// topics, so the submission is refused by the topic lookup above it. +/// operator's reason; a topic whose install recorded a +/// `vms_per_submission` other than 1 is a **503** — this build runs exactly +/// one VM per submission, and silently running a different number would make +/// the journal a lie; an unreadable gate is a **503** too — never an +/// admission, and never a 404 that would read as "no such topic". A host that +/// resolved no journal (no database) has no gate to read: it also has no +/// published topics, so the submission is refused by the topic lookup above +/// it. /// /// This is deliberately **not cached**: a disable has to take effect on the /// next request, which is what makes it usable during an incident. @@ -1802,9 +1830,21 @@ async fn disabled_gate(st: &AppState, topic_id: &str) -> Result<(), ErrResp> { let Some(journal) = st.install_journal.as_deref() else { return Ok(()); }; - match journal.disabled(topic_id).await { - Ok(None) => Ok(()), - Ok(Some(reason)) => Err(err( + let gate = match journal.submit_gate(topic_id).await { + Ok(gate) => gate, + Err(e) => { + return Err(err( + StatusCode::SERVICE_UNAVAILABLE, + &format!( + "the topic gate could not be read for {topic_id:?}: {e}. The submission is \ + refused rather than admitted on an unread fact; fix the database and re-post \ + (the submit_nonce is unspent)." + ), + )) + } + }; + if let Some(reason) = gate.disabled_reason.as_deref() { + return Err(err( StatusCode::FORBIDDEN, &format!( "topic {topic_id:?} is disabled by the operator{}", @@ -1814,16 +1854,24 @@ async fn disabled_gate(st: &AppState, topic_id: &str) -> Result<(), ErrResp> { format!(": {}", reason.trim()) } ), - )), - Err(e) => Err(err( - StatusCode::SERVICE_UNAVAILABLE, - &format!( - "the topic gate could not be read for {topic_id:?}: {e}. The submission is \ - refused rather than admitted on an unread fact; fix the database and re-post \ - (the submit_nonce is unspent)." - ), - )), + )); + } + if let Some(n) = gate.vms_per_submission { + if n != proof_topic_install::VMS_PER_SUBMISSION { + return Err(err( + StatusCode::SERVICE_UNAVAILABLE, + &format!( + "topic {topic_id:?} was installed with vms_per_submission={n}, but this host \ + runs exactly {} VM per submission (the pin this build enforces). Re-install \ + the topic with the pin this host carries, or run the host that matches the \ + install. The submission is refused rather than run under a binding the \ + install did not record; the submit_nonce is unspent.", + proof_topic_install::VMS_PER_SUBMISSION + ), + )); + } } + Ok(()) } fn store_err(e: &proof_store::StoreError) -> (StatusCode, Json) { @@ -2972,6 +3020,63 @@ mod tests { assert_eq!(body["items"][0]["disabled"], false, "{body}"); } + /// The allocator pin, read from the same journal: a topic installed with + /// `vms_per_submission` 1 (or with no install row at all) submits, and a + /// topic installed with any other number is refused **before the nonce is + /// spent** — this build runs one VM per submission, and running a topic + /// under a binding its install never recorded would make the journal a + /// lie. + #[tokio::test] + async fn a_topic_installed_under_another_allocator_pin_is_refused() { + // The pin this build carries: the submission lands. + let app = app_with_gate(Some(Arc::new(PinnedJournal { + vms_per_submission: Some(proof_topic_install::VMS_PER_SUBMISSION), + }))); + let body = submit_body("pinned-ok", &serde_json::json!({})); + let (st, created) = json_req(app, "POST", "/v1/submissions", body, None).await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + + // No install row (a topic staged before the journal existed): the pin + // is unknown, and the topic is not refused for that. + let app = app_with_gate(Some(Arc::new(PinnedJournal { + vms_per_submission: None, + }))); + let body = submit_body("unpinned", &serde_json::json!({})); + let (st, created) = json_req(app, "POST", "/v1/submissions", body, None).await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + + // A different pin: refused, and the body is re-postable unchanged. + let app = app_with_gate(Some(Arc::new(PinnedJournal { + vms_per_submission: Some(2), + }))); + let body = submit_body("two-vms", &serde_json::json!({})); + let (st, refused) = + json_req(app.clone(), "POST", "/v1/submissions", body.clone(), None).await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{refused}"); + let error = refused["error"].as_str().unwrap_or_default(); + assert!(error.contains("vms_per_submission=2"), "{refused}"); + assert!(error.contains("unspent"), "{refused}"); + let (_, list) = json_req( + app.clone(), + "GET", + "/v1/submissions", + serde_json::json!({}), + None, + ) + .await; + assert!( + list["items"].as_array().is_some_and(Vec::is_empty), + "no row: {list}" + ); + + // Re-installed under the pin this host runs: the same body lands. + let app = app_with_gate(Some(Arc::new(PinnedJournal { + vms_per_submission: Some(proof_topic_install::VMS_PER_SUBMISSION), + }))); + let (st, created) = json_req(app, "POST", "/v1/submissions", body, None).await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + } + #[tokio::test] async fn submit_requires_a_hotkey_signature() { let app = app("op"); @@ -3647,8 +3752,8 @@ mod tests { Ok(true) } - async fn disabled(&self, _topic_id: &str) -> Result, String> { - Ok(None) + async fn submit_gate(&self, _topic_id: &str) -> Result { + Ok(SubmitGate::default()) } async fn disabled_topics(&self) -> Result, String> { @@ -3666,7 +3771,7 @@ mod tests { Err("journal unavailable".into()) } - async fn disabled(&self, _topic_id: &str) -> Result, String> { + async fn submit_gate(&self, _topic_id: &str) -> Result { Err("gate unavailable".into()) } @@ -3684,8 +3789,8 @@ mod tests { Ok(false) } - async fn disabled(&self, _topic_id: &str) -> Result, String> { - Ok(None) + async fn submit_gate(&self, _topic_id: &str) -> Result { + Ok(SubmitGate::default()) } async fn disabled_topics(&self) -> Result, String> { @@ -3722,9 +3827,13 @@ mod tests { Ok(true) } - async fn disabled(&self, topic_id: &str) -> Result, String> { + async fn submit_gate(&self, topic_id: &str) -> Result { let on = self.disabled.load(std::sync::atomic::Ordering::SeqCst); - Ok((on && topic_id == self.topic_id).then(|| self.reason.clone())) + Ok(if on && topic_id == self.topic_id { + SubmitGate::disabled(self.reason.clone()) + } else { + SubmitGate::default() + }) } async fn disabled_topics(&self) -> Result, String> { @@ -3737,6 +3846,31 @@ mod tests { } } + /// A journal whose newest install recorded a `vms_per_submission` the + /// running host does not carry: the submit path must refuse rather than + /// run the topic under a binding the install never recorded. + struct PinnedJournal { + vms_per_submission: Option, + } + + #[async_trait] + impl InstallJournal for PinnedJournal { + async fn applied(&self, _topic_id: &str) -> Result { + Ok(true) + } + + async fn submit_gate(&self, _topic_id: &str) -> Result { + Ok(SubmitGate { + disabled_reason: None, + vms_per_submission: self.vms_per_submission, + }) + } + + async fn disabled_topics(&self) -> Result, String> { + Ok(BTreeMap::new()) + } + } + /// The `app("op")` host — one open topic, Sim, submitable — with `journal` /// as the operator gate, so the submit path and the listing can be driven /// against a gate the test controls. diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs index eca3b3f34..2e1becff9 100644 --- a/crates/proof-rlm-scorer/src/scorer.rs +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -68,13 +68,33 @@ struct Decided { struct Pending { bundle: ArtefactBundle, decided: Option, - /// Topic lease held since `score` returned; dropped when the row is - /// persisted (end of `on_persisted`) or the entry is reaped. + /// The submission's lease, held since `score` returned; dropped when the + /// row is persisted (end of `on_persisted`) or the entry is reaped. lease: Option>, since: Instant, } /// Family scorer over the runner registry, the RLM store, and the artefact store. +/// +/// # Concurrency +/// +/// Two submissions of the **same topic** are two separate paid runs, each in +/// its own Firecracker VM (the topic VM is never shared, and +/// `vms_per_submission` is 1 per submission), so they must be able to be in +/// flight at once. What is shared is the topic's *state*: the lifecycle and +/// the best pointer. Those are written under [`Self::topic_lock`], taken only +/// for a write, while each run holds its own +/// [`Self::lease`] for its whole life: +/// +/// - `score` applies `SubmissionReceived` (a write, under the topic lock) and +/// then evaluates **without** holding it, so a second submission is not +/// blocked behind a paid run; +/// - the promotion decision is a compare-and-swap under the topic lock, and +/// [`Self::crown`] refuses a decision whose best pointer moved in between, +/// so parallel runs cannot both crown off one stale bar; +/// - a run that never persists releases its own lease after +/// [`RlmScorer::lease_ttl`] (or immediately, when the topic lock is +/// contended), never the whole topic's. pub struct RlmScorer { registry: Arc, store: Arc, @@ -82,6 +102,11 @@ pub struct RlmScorer { pending: Mutex>, locks: Mutex>>>, lease_ttl: Duration, + /// Runs of each topic that have taken `SubmissionReceived` and not yet + /// written a verdict. Read under the topic lock, so "0" means no run of + /// this topic is in flight — which is what makes a persisted `evaluating` + /// a *stale* phase rather than another submission's. + inflight: Mutex>, } fn unwired(custom_id: &str, detail: String) -> EvalError { @@ -158,6 +183,7 @@ impl RlmScorer { pending: Mutex::new(BTreeMap::new()), locks: Mutex::new(BTreeMap::new()), lease_ttl: DEFAULT_LEASE_TTL, + inflight: Mutex::new(BTreeMap::new()), } } @@ -191,6 +217,42 @@ impl RlmScorer { .clone() } + /// Take the **topic** lock: held only around a write to the topic's + /// shared state (the lifecycle, the best pointer), never across a paid + /// run. See the type's `Concurrency` docs. + async fn topic_guard(&self, topic_id: &str) -> OwnedMutexGuard<()> { + self.topic_lock(topic_id).lock_owned().await + } + + /// The lease one run holds from `score` until its row is persisted. + /// + /// Keyed `(topic_id, submission_digest)`, not by topic alone: two + /// submissions of the same topic are two **separate** paid runs, each in + /// its own Firecracker VM, so they must be able to be in flight at the + /// same time — `vms_per_submission` is 1, and that 1 belongs to the + /// submission, never to the topic. The topic itself is shared, and the + /// writes that touch it (the promotion compare-and-swap over the best + /// pointer, the lifecycle moves) are serialized by + /// [`Self::topic_lock`], which the write phases take for themselves: + /// + /// - two submissions of one topic evaluate **in parallel**; + /// - whichever writes first decides its promotion against the best pointer + /// as it is then, and the other against the updated one — a bar that + /// moved between a decision and its write is refused by [`Self::crown`], + /// which is what makes the ordering safe without holding a topic lock + /// across a paid run. + /// + /// A retry of the *same* digest is refused earlier still (the store's + /// per-digest claim), and would take the same lock here. + fn lease_lock(&self, topic_id: &str, submission_digest: &str) -> Arc> { + self.locks + .lock() + .unwrap_or_else(PoisonError::into_inner) + .entry(format!("{topic_id}\u{1f}{}", submission_digest.trim())) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + } + /// Drop pending runs of `topic_id` whose row never landed within the TTL, /// releasing the lease they hold. fn reap_abandoned(&self, topic_id: &str) { @@ -210,9 +272,10 @@ impl RlmScorer { } } - /// Take the topic lease, reaping abandoned holders while waiting. - async fn lease(&self, topic_id: &str) -> OwnedMutexGuard<()> { - let lock = self.topic_lock(topic_id); + /// Take the lease for one **submission**, reaping abandoned holders while + /// waiting. Runs of different submissions do not contend here. + async fn lease(&self, topic_id: &str, submission_digest: &str) -> OwnedMutexGuard<()> { + let lock = self.lease_lock(topic_id, submission_digest); loop { self.reap_abandoned(topic_id); if let Ok(guard) = tokio::time::timeout(LEASE_POLL, lock.clone().lock_owned()).await { @@ -297,10 +360,19 @@ impl RlmScorer { } } - /// Called with the topic lease held: a persisted `evaluating` / + /// Called with the topic lock held: a persisted `evaluating` / /// `promoting` means the previous run's row never landed. Close that /// phase rather than refusing the topic forever. + /// + /// **Only when nothing of this topic is in flight.** With parallel + /// submissions a persisted `evaluating` is the normal state of another + /// run that is still working, and closing it would move the lifecycle out + /// from under a live run. `inflight` is read under the same topic lock the + /// count is taken under, so a non-zero count is a real run. async fn recover_stale(&self, topic: &TopicDocument) -> Result<(), EvalError> { + if self.inflight(topic.id.as_str()) > 0 { + return Ok(()); + } match self.lifecycle(topic).await?.state { RlmState::Evaluating => { self.apply( @@ -323,6 +395,38 @@ impl RlmScorer { Ok(()) } + /// Runs of `topic_id` between `SubmissionReceived` and their verdict. + fn inflight(&self, topic_id: &str) -> usize { + self.inflight + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(topic_id) + .copied() + .unwrap_or(0) + } + + /// Record that a run of `topic_id` entered its evaluation phase. + fn enter(&self, topic_id: &str) { + *self + .inflight + .lock() + .unwrap_or_else(PoisonError::into_inner) + .entry(topic_id.to_owned()) + .or_insert(0) += 1; + } + + /// Record that a run of `topic_id` left its evaluation phase. Called + /// exactly once per [`Self::enter`], whatever the outcome. + fn leave(&self, topic_id: &str) { + let mut inflight = self.inflight.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(n) = inflight.get_mut(topic_id) { + *n = n.saturating_sub(1); + if *n == 0 { + inflight.remove(topic_id); + } + } + } + /// The topic's current rule version, seeding version 1 from the signed /// document on first use so the gate a run was ticked against is in the /// store, not only in the document. @@ -720,11 +824,21 @@ impl LiveScorer for RlmScorer { artifact_tar: Option<&[u8]>, ) -> Result { self.ready_for_topic(topic)?; - let lease = self.lease(&topic.id).await; + // The submission's own lease: a second submission of this topic is a + // second paid run, not a queue entry. The **writes** below take the + // topic lock for themselves, so nothing here holds it across the + // evaluation. + let lease = self.lease(&topic.id, frozen_digest).await; self.ensure_topic(topic).await?; - self.recover_stale(topic).await?; - self.apply(topic, RlmEvent::SubmissionReceived, frozen_digest) - .await?; + { + // The topic's shared state, written under its own lock: nothing + // else may move the lifecycle while this runs. + let _topic = self.topic_guard(&topic.id).await; + self.recover_stale(topic).await?; + self.apply(topic, RlmEvent::SubmissionReceived, frozen_digest) + .await?; + self.enter(&topic.id); + } let out = self .evaluate( pin, @@ -744,7 +858,7 @@ impl LiveScorer for RlmScorer { Ok(_) => { // The lease now belongs to the pending run: promotion is // decided and persisted under it, then it is released in - // `on_persisted`. + // `on_persisted` (which also leaves the in-flight count). self.hold(frozen_digest, lease); } Err(err) => { @@ -757,9 +871,12 @@ impl LiveScorer for RlmScorer { error = %err, "evaluate refused; no row" ); - // No row will follow a refusal, so the verdict phase is over now. + // No row will follow a refusal, so the verdict phase is over + // now — under the topic lock, since it moves shared state. + let _topic = self.topic_guard(&topic.id).await; self.apply_logged(topic, RlmEvent::VerdictRecorded, "refused; no row") .await; + self.leave(&topic.id); drop(lease); } } @@ -809,10 +926,12 @@ impl LiveScorer for RlmScorer { false } - /// Decided under the topic lease this run has held since `score` - /// returned, against the harder of the caller's bar and the store's - /// current best: no other run of this topic can be between score and - /// persist, and a bar computed before an earlier crown cannot be reused. + /// Decided under the **topic lock** (see the type's `Concurrency` docs): + /// the bar is the harder of the caller's and the store's current best, + /// and the decision records the best it was taken against, so a crown + /// whose best pointer moved in between is refused ([`Self::crown`]). + /// Parallel runs of the same topic therefore cannot both crown off one + /// stale bar. async fn auto_promote( &self, topic: &TopicDocument, @@ -829,6 +948,7 @@ impl LiveScorer for RlmScorer { if !held { return false; } + let _topic = self.topic_guard(&topic.id).await; let current = match self.store.best(&topic.id).await { Ok(b) => b, Err(e) => { @@ -881,12 +1001,16 @@ impl LiveScorer for RlmScorer { submission_id: &str, promoted: bool, ) { - // `pending` (and the topic lease inside it) lives to the end of this - // function: the artefact, the promotion row, and the best pointer - // land under the guard the decision was taken under. + // `pending` (and the submission's lease inside it) lives to the end of + // this function: the artefact, the promotion row, and the best pointer + // land under the guard the decision was taken under. The **topic** + // lock is taken here too, so the promotion compare-and-swap and the + // lifecycle moves are serialized against any other run of this topic + // that is writing at the same time. let Some(pending) = self.take(submission_digest) else { return; }; + let _topic = self.topic_guard(topic_id).await; let topic = self .store .latest_topic(topic_id) @@ -917,6 +1041,9 @@ impl LiveScorer for RlmScorer { }; self.apply_logged(t, event, submission_id).await; } + // This run's evaluation phase is over: the topic may now be recovered + // if a *later* run finds the phase stale. + self.leave(topic_id); drop(pending); } } diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index d979ea02b..2b44dbe18 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -16,9 +16,11 @@ //! declaration is a persisted reject and a missing one is a 503; //! every scored row leaves its zip, the crown leaves `best.json` + a //! promotion row, and the store holds rules v1, every checklist, and the -//! lifecycle. Runs hold their topic lease until persisted, so a worse run +//! lifecycle. Two submissions of one topic evaluate **in parallel** — each +//! holds its own lease, and the writes (the promotion compare-and-swap and +//! the lifecycle moves) are serialized under the topic lock — so a worse run //! decided against a stale bar can never displace the champion, a moved -//! best pointer refuses a stale crown, and an abandoned run releases its +//! best pointer refuses a stale crown, and an abandoned run releases its own //! lease after the TTL. Then the setup driver walks `draft → … → //! baselining` with RLM-written rules and a baseline in the store, and //! `mark_sealed` opens the topic only for the signed, valid, open document @@ -72,6 +74,14 @@ impl proof_http::InstallJournal for InstalledJournal { async fn applied(&self, _topic_id: &str) -> Result { Ok(true) } + + async fn submit_gate(&self, _topic_id: &str) -> Result { + Ok(proof_http::SubmitGate::default()) + } + + async fn disabled_topics(&self) -> Result, String> { + Ok(std::collections::BTreeMap::new()) + } } fn test_executor(pin: &ProofPin) -> EvalExecutorOffer { @@ -833,6 +843,11 @@ async fn a_refused_paid_run_is_host_logged_with_its_error() { /// Two runs decided against the same old bar: the second cannot score until /// the first is persisted, its decision then sees the new best, and a moved /// best pointer refuses a crown that was decided before it moved. +/// +/// The two runs are **not** serialized while they evaluate — that is +/// [`two_submissions_of_one_topic_run_in_parallel`] — but each decision is +/// taken under the topic lock against the store's best, which is what makes +/// the ordering of the *writes* safe. #[tokio::test] async fn a_worse_run_never_displaces_the_champion_under_the_topic_lease() { let d = direct("lease", None); @@ -843,38 +858,7 @@ async fn a_worse_run_never_displaces_the_champion_under_the_topic_lease() { assert!((doc_a.harness.custom_value.unwrap() - 0.7).abs() < 1e-12); assert_eq!(d.scorer.pending_len(), 1); - // B (0.60) is blocked on the lease, not scored against the stale world. - d.orchestrator.set_primary(0.6); - let scorer_b = d.scorer.clone(); - let (pin_b, topic_b, plan_b) = (d.pin.clone(), d.topic.clone(), d.plan.clone()); - let mut task_b = tokio::spawn(async move { - let budget = topic_b.flops_budget; - scorer_b - .score( - &pin_b, - &topic_b, - &offer(), - &plan_b, - "digest-b", - &digest("b"), - Some(&locator("b")), - budget, - &[], - "placeholder claim", - &proof_rlm::MinerEnv::new(), - None, - ) - .await - }); - assert!( - tokio::time::timeout(Duration::from_millis(300), &mut task_b) - .await - .is_err(), - "b must wait for a's row" - ); - assert_eq!(paid_runs(&d.orchestrator), 1, "b has not run"); - - // A is decided against bar 0.50 and persisted under the lease. + // A is decided against bar 0.50 and persisted under its lease. assert!( d.scorer .auto_promote(&d.topic, "digest-a", true, Some(0.7), Some(0.5)) @@ -885,9 +869,10 @@ async fn a_worse_run_never_displaces_the_champion_under_the_topic_lease() { .await; assert_eq!(d.scorer.pending_len(), 0); - // The lease is free: b scores, and its decision — even handed the stale - // bar 0.50 — is taken against the store's best 0.70. - let doc_b = task_b.await.unwrap().expect("b scores after a persisted"); + // B (0.60) scores — its own lease, not A's — and its decision, even + // handed the stale bar 0.50, is taken against the store's best 0.70. + d.orchestrator.set_primary(0.6); + let doc_b = score(&d, "b").await.expect("b scores"); assert!((doc_b.harness.custom_value.unwrap() - 0.6).abs() < 1e-12); assert!( !d.scorer @@ -971,34 +956,232 @@ async fn a_worse_run_never_displaces_the_champion_under_the_topic_lease() { let _ = std::fs::remove_dir_all(&d.root); } -/// A run whose row never lands must not hold its topic hostage: past the -/// lease TTL the next run reaps it, recovers the lifecycle, and proceeds. +/// Two concurrent submissions to **one topic** are two paid runs, so the KVM +/// host is asked for two VMs (one per submission: `vms_per_submission` is 1 +/// *per submission*, never a queue behind the topic). +/// +/// The host models that: `inspect` attaches the topic's VM (shared, and the +/// only VM for a topic with no in-guest runner), and each paid run is its own +/// job. What this pins is the control plane's half — both submissions reach +/// the runner concurrently, and neither waits for the other's row. +#[tokio::test] +async fn two_concurrent_submits_reach_the_runner_as_two_runs() { + let Stack { + app, + orchestrator, + root, + topic, + .. + } = stack(true); + let tid = topic.id.clone(); + + // Both bodies are accepted and scored on their own. `json_req` drives one + // request at a time; the two `oneshot` calls below are what makes them + // concurrent — the router is cloned, so each future owns its own service. + let a = submit_declaring(&tid, "concurrent-a", 1); + let b = submit_declaring(&tid, "concurrent-b", 1); + let (ra, rb) = tokio::join!( + json_req(app.clone(), "POST", "/v1/submissions", a), + json_req(app.clone(), "POST", "/v1/submissions", b), + ); + assert_eq!(ra.0, StatusCode::CREATED, "{:?}", ra.1); + assert_eq!(rb.0, StatusCode::CREATED, "{:?}", rb.1); + let (id_a, id_b) = ( + ra.1["id"].as_str().unwrap().to_owned(), + rb.1["id"].as_str().unwrap().to_owned(), + ); + assert_ne!(id_a, id_b, "two submissions are two rows"); + + // Both are scored: two paid runs reached the orchestrator, and neither + // was rejected for arriving while the other was in flight. + let (_, row_a) = json_req( + app.clone(), + "GET", + &format!("/v1/submissions/{id_a}"), + serde_json::json!({}), + ) + .await; + let (_, row_b) = json_req( + app.clone(), + "GET", + &format!("/v1/submissions/{id_b}"), + serde_json::json!({}), + ) + .await; + for row in [&row_a, &row_b] { + assert_ne!(row["state"], "rejected", "{row}"); + assert_eq!(row["verdict"]["agent"]["verdict"], "clean", "{row}"); + } + assert_eq!( + paid_runs(&orchestrator), + 2, + "each submission is its own paid run, not a queue entry" + ); + // Exactly one of them is the champion (both measured the same primary, so + // the second is refused by the compare-and-swap, not by a queue). + let champions = [&row_a, &row_b] + .iter() + .filter(|r| r["state"] == "champion") + .count(); + assert_eq!(champions, 1, "one crown: {row_a} / {row_b}"); + let _ = std::fs::remove_dir_all(&root); +} + +/// A topic that selects an in-guest runner gets **one dedicated experiment VM +/// per paid job**, and two submissions in flight are two jobs: two VMs, never +/// one shared. This is the allocator rule the live 2-VM check exercises, at +/// the control plane's end of it. +#[tokio::test] +async fn two_submissions_of_an_experiment_topic_ask_for_two_vms() { + let mut d = direct("parallel-experiment", None); + // The topic selects an in-guest runner, so every paid job is its own VM. + d.topic.constraints.params.insert( + proof_experiment::PARAM_RUNNER.into(), + "placeholder_in_guest_runner".into(), + ); + d.topic.constraints.params.insert( + proof_experiment::PARAM_PACK_DIGEST.into(), + format!("sha256:{}", "ee".repeat(32)), + ); + d.plan = d + .scorer + .plan(&d.pin, &d.topic, &test_executor(&d.pin)) + .expect("plan"); + let d = Arc::new(d); + + let a = tokio::spawn({ + let d = d.clone(); + async move { score(&d, "exp-a").await } + }); + let b = tokio::spawn({ + let d = d.clone(); + async move { score(&d, "exp-b").await } + }); + let (ra, rb) = (a.await.unwrap(), b.await.unwrap()); + assert!(ra.is_ok(), "a scores: {ra:?}"); + assert!(rb.is_ok(), "b scores: {rb:?}"); + assert_eq!( + d.orchestrator.experiments().len(), + 2, + "one experiment vm per paid job" + ); + assert_eq!(paid_runs(&d.orchestrator), 2); + // Each experiment VM was destroyed after its job (the fake confirms it), + // so the host is not left holding capacity for either. + assert_eq!( + d.orchestrator + .teardowns() + .iter() + .filter(|(_, policy)| *policy == proof_rlm::RetainPolicy::Destroy) + .count(), + 2, + "both experiment vms were destroyed after their jobs" + ); + let _ = std::fs::remove_dir_all(&d.root); +} + +/// A run whose row never lands must not hold its **own** lease forever: past +/// the TTL it is reaped, so its entry leaves `pending` and a re-submission of +/// the same digest is not blocked by it. +/// +/// It no longer blocks *other* submissions either — that is the point of the +/// per-submission lease ([`two_submissions_of_one_topic_run_in_parallel`]) — +/// and the lifecycle is left `evaluating` because a live run is still in +/// flight. The abandoned run is not "recovered" out from under that run: the +/// phase is closed only once nothing of this topic is running. #[tokio::test] async fn an_abandoned_run_releases_its_topic_lease_after_the_ttl() { let d = direct("ttl", Some(Duration::ZERO)); let tid = d.topic.id.clone(); score(&d, "abandoned").await.expect("scores"); assert_eq!(d.scorer.pending_len(), 1); + let next = tokio::time::timeout(Duration::from_secs(10), score(&d, "next")) .await .expect("the abandoned lease is reaped, not waited on") .expect("scores"); assert!((next.harness.custom_value.unwrap() - 0.7).abs() < 1e-12); - assert_eq!(d.scorer.pending_len(), 1, "only the live run is pending"); + assert_eq!( + d.scorer.pending_len(), + 1, + "the reaped run is gone; only the live one is pending" + ); assert!( !d.scorer .auto_promote(&d.topic, "digest-abandoned", true, Some(0.7), Some(0.5)) .await, "a reaped run cannot be promoted" ); + + // The live run persists; only then does the phase close, and the abandoned + // one's `evaluating` is recovered as stale — nothing is left in flight. + d.scorer + .on_persisted(&tid, "digest-next", "pf_0000000000000002", false) + .await; + assert_eq!(d.scorer.pending_len(), 0); let lc = d.rlm_store.lifecycle(&tid).await.unwrap().unwrap(); + assert_eq!(lc.state, RlmState::Open, "the topic is not stuck: {lc:?}"); + let _ = std::fs::remove_dir_all(&d.root); +} + +/// Two submissions of **one topic** are two runs, not a queue: each holds its +/// own lease, so the second evaluates while the first is still in flight. +/// (Its row cannot land first, though — `on_persisted` takes the topic lock — +/// which is what keeps the promotion compare-and-swap honest.) +#[tokio::test] +async fn two_submissions_of_one_topic_run_in_parallel() { + let d = direct("parallel", None); + let tid = d.topic.id.clone(); + + // A scores and holds its lease: its row has not landed yet. + score(&d, "a").await.expect("a scores"); + assert_eq!(d.scorer.pending_len(), 1); + assert_eq!(paid_runs(&d.orchestrator), 1); + + // B is a *different* submission: it evaluates now, without waiting for + // A's row. The topic lock is not held across a run, so nothing here + // blocks on A. + d.orchestrator.set_primary(0.6); + let doc_b = tokio::time::timeout(Duration::from_secs(10), score(&d, "b")) + .await + .expect("b is not blocked behind a's row") + .expect("b scores"); + assert!((doc_b.harness.custom_value.unwrap() - 0.6).abs() < 1e-12); + assert_eq!( + paid_runs(&d.orchestrator), + 2, + "each submission is its own paid run" + ); + assert_eq!(d.scorer.pending_len(), 2, "both runs await their rows"); + + // Both rows land, and the better one keeps the crown: A (0.7) was decided + // first, B (0.6) decides against A's best and does not displace it. assert!( - lc.history - .iter() - .any(|h| h.event == RlmEvent::VerdictRecorded && h.note.contains("recovered")), - "{lc:?}" + d.scorer + .auto_promote(&d.topic, "digest-a", true, Some(0.7), Some(0.5)) + .await + ); + d.scorer + .on_persisted(&tid, "digest-a", "pf_0000000000000001", true) + .await; + assert!( + !d.scorer + .auto_promote(&d.topic, "digest-b", true, Some(0.6), Some(0.5)) + .await + ); + d.scorer + .on_persisted(&tid, "digest-b", "pf_0000000000000002", false) + .await; + assert_eq!(d.scorer.pending_len(), 0); + let best = d.rlm_store.best(&tid).await.unwrap().expect("best"); + assert_eq!(best.submission_id, "pf_0000000000000001"); + assert!((best.primary_value - 0.7).abs() < 1e-12); + assert_eq!(d.rlm_store.promotions(&tid).await.unwrap().len(), 1); + assert_eq!( + d.rlm_store.lifecycle(&tid).await.unwrap().unwrap().state, + RlmState::Open, + "the topic is back to open after both runs" ); - assert_eq!(lc.state, RlmState::Evaluating, "the live run is still open"); let _ = std::fs::remove_dir_all(&d.root); } diff --git a/crates/proof-rlm/src/state.rs b/crates/proof-rlm/src/state.rs index fc0967771..fc365d467 100644 --- a/crates/proof-rlm/src/state.rs +++ b/crates/proof-rlm/src/state.rs @@ -183,6 +183,26 @@ pub enum StateError { /// The transition table. Every edge is explicit; anything else is illegal. /// +/// # More than one run at a time +/// +/// A topic takes **several submissions at once** — each is its own paid run in +/// its own Firecracker VM — so the table has the edges that state needs, and +/// they are all of the "stay where you are" shape: +/// +/// - `evaluating --submission_received--> evaluating`: a second run starts +/// while the first is still evaluating. +/// - `promoting --submission_received--> evaluating`: a new run arrives while +/// an earlier one is settling its promotion. +/// - `promoting --promotion_candidate--> promoting`: two runs both decide to +/// promote; the store's compare-and-swap decides which crown lands. +/// - `promoting --verdict_recorded--> open`: the last run out records a plain +/// verdict while a promotion was still open; the promotion row (written +/// under the same lock) is the record of what happened, not this state. +/// +/// The lifecycle is therefore a *phase* marker ("is this topic evaluating +/// something?"), not a per-run counter. The per-run count lives in the scorer, +/// which is what decides when a phase is stale. +/// /// # Errors /// /// [`StateError::Illegal`] for a pair the table does not name. @@ -194,15 +214,20 @@ pub fn transition(from: RlmState, event: RlmEvent) -> Result S::AwaitingOwnerKeys, (S::AwaitingOwnerKeys, E::OwnerKeysPresent) => S::Provisioning, (S::Provisioning, E::Provisioned) => S::Baselining, - (S::Open, E::SubmissionReceived) => S::Evaluating, - (S::Evaluating, E::PromotionCandidate) => S::Promoting, + // A submission on an open topic starts evaluating; another one while + // it is already evaluating (or settling a promotion) keeps the phase: + // the run is separate, the phase is not per-run. + (S::Open | S::Evaluating | S::Promoting, E::SubmissionReceived) => S::Evaluating, + // Two runs both deciding to promote keep the phase; the store's + // compare-and-swap decides which crown lands. + (S::Evaluating | S::Promoting, E::PromotionCandidate) => S::Promoting, // Back to draft: the owner said no, or the ceremony failed. (S::OwnerPresend, E::OwnerDeclined) | (S::Provisioning, E::ProvisionFailed) | (S::Baselining, E::BaselineFailed) => S::Draft, // Back to open: sealed, verdict recorded, or promotion settled. (S::Baselining, E::BaselineSealed) - | (S::Evaluating, E::VerdictRecorded) + | (S::Evaluating | S::Promoting, E::VerdictRecorded) | (S::Promoting, E::Promoted | E::PromotionRefused) => S::Open, (S::Closed, _) => return Err(StateError::Illegal { from, event }), (_, E::Close) => S::Closed, @@ -620,6 +645,43 @@ mod tests { ); } + /// A topic takes several submissions at once, so the table has the edges + /// parallel runs need — and they are all "the phase stays, the run is + /// separate". The scorer owns the per-run count; the lifecycle is a phase + /// marker, not a counter. + #[test] + fn a_second_run_while_one_is_in_flight_keeps_the_phase() { + assert_eq!( + transition(RlmState::Evaluating, RlmEvent::SubmissionReceived), + Ok(RlmState::Evaluating), + "a second submission does not restart the phase" + ); + assert_eq!( + transition(RlmState::Promoting, RlmEvent::SubmissionReceived), + Ok(RlmState::Evaluating), + "a new run arrives while an earlier one settles its promotion" + ); + assert_eq!( + transition(RlmState::Promoting, RlmEvent::PromotionCandidate), + Ok(RlmState::Promoting), + "two runs both decide to promote; the store's cas decides" + ); + assert_eq!( + transition(RlmState::Promoting, RlmEvent::VerdictRecorded), + Ok(RlmState::Open), + "the last run out records a plain verdict" + ); + // Still no way to skip a state, and closed is still terminal. + assert!(matches!( + transition(RlmState::Open, RlmEvent::SubmissionReceived), + Ok(RlmState::Evaluating) + )); + assert!(matches!( + transition(RlmState::Closed, RlmEvent::SubmissionReceived), + Err(StateError::Illegal { .. }) + )); + } + /// Without a hook the machine cannot leave owner_presend: nothing is /// sent on the owner's behalf. #[test] diff --git a/crates/proof-rlm/src/vm.rs b/crates/proof-rlm/src/vm.rs index 2280dbade..ef915ca3a 100644 --- a/crates/proof-rlm/src/vm.rs +++ b/crates/proof-rlm/src/vm.rs @@ -573,6 +573,20 @@ pub async fn run_paid_job( /// VM — or, for a topic whose params select an in-guest runner, inside a /// dedicated experiment VM per paid job ([`run_paid_job`]). Registering it /// under a `custom_id` is an operator action; nothing registers it by default. +/// +/// # One VM per submission, and parallel submissions +/// +/// A topic's **topic VM** is shared (one per topic: `attach` finds it, and the +/// KVM host refuses a second `create` for the same topic), and it is where the +/// cheap, non-paid jobs run (`inspect`, `propose_rules`). Every **paid** job +/// gets a VM of its own: an experiment VM for a topic that selects an in-guest +/// runner ([`run_paid_job`]), or the topic VM for one that does not — and +/// either way the *submission* is what it belongs to, not the topic. That is +/// [`proof_topic_install::VMS_PER_SUBMISSION`] (1) read as a runtime rule: +/// two submissions in flight are two runs and two VMs, never one VM shared or +/// one submission queued behind the other. The scorer holds no per-topic +/// lease across a run (see `proof-rlm-scorer`'s `RlmScorer` docs); what +/// serializes is the *write* after each run, not the run itself. pub struct VmBackedRunner { orchestrator: Arc, template: VmTemplate, From afa8c81552bd939c86642fa08344b56b1a3bebfb Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:35:23 +0000 Subject: [PATCH 5/8] feat(proof): say whether the install reached scorable, and how to finish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The install left the topic unscorable and said so only in prose that did not name the commands that finish the job. Two changes: - `InstallReport` carries the document's own status, and the install prints a plain `Scorable` / `NOT scorable yet — ` line. `--json` gains `scorable` and `remaining`, so a script can tell "run the next command" from "something is wrong" without parsing English. - The closing advice is the real path: `topic baseline ` (read the measurement and the commitment), sign the open document, `topic seal --document … --publish`. Every branch of it ends at the same two commands, because the install never makes a topic scorable on its own — whichever way it was run. `--skip-baseline` is described as what it is: a pause, not a path. The dry-run fixture README documents the same three commands, plus migration 0026 (the operator gate) in the staging-migrate section. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- bins/proof-admin/src/install.rs | 114 ++++++++++++++++-- bins/proof-admin/tests/cli.rs | 20 +++ .../tests/fixtures/README-dry-run.md | 45 ++++++- crates/proof-topic-install/src/install.rs | 8 ++ 4 files changed, 173 insertions(+), 14 deletions(-) diff --git a/bins/proof-admin/src/install.rs b/bins/proof-admin/src/install.rs index 1b82a16bd..815464d11 100644 --- a/bins/proof-admin/src/install.rs +++ b/bins/proof-admin/src/install.rs @@ -302,6 +302,11 @@ async fn run_real( "binding": report.binding, "setup": report.setup, "aliases": alias_notes, + // Whether the topic can be scored right now, and what is left. + // A machine caller needs this to decide whether to go on to the + // seal; the install never makes a topic scorable on its own. + "scorable": scorable(&report, args), + "remaining": remaining_steps(&report, args, &plan.topic_id), }))?; return Ok(()); } @@ -316,10 +321,70 @@ async fn run_real( } } println!(); + println!("{}", scorable_line(&report, args)); + println!(); println!("{}", next_steps(plan, args)); Ok(()) } +/// Whether this install left the topic **scorable**, and why not when it did +/// not. +/// +/// The install never makes a topic scorable by itself: scoring needs an +/// `open` document whose baseline is sealed, and the seal is the operator's +/// (the CLI holds no `proof` key). What this reports is which of the two +/// halves is still missing, so an operator — or a script — can tell "run the +/// next command" from "something is wrong". +fn scorable(report: &proof_topic_install::InstallReport, args: &InstallArgs<'_>) -> bool { + matches!(report.setup, SetupSummary::Baselined { .. }) + && report.document_status == proof_task::TopicStatus::Open + && !args.skip_baseline +} + +/// The one-line answer to "can it score now?". +fn scorable_line(report: &proof_topic_install::InstallReport, args: &InstallArgs<'_>) -> String { + if scorable(report, args) { + return "Scorable: the baseline is measured and the document is open. Confirm with \ + GET /v1/status (`scorable_topics`)." + .to_owned(); + } + let missing = if args.skip_baseline { + "--skip-baseline: no baseline was measured, and a topic cannot open without one" + } else if !matches!(report.setup, SetupSummary::Baselined { .. }) { + "the RLM setup was not driven: no baseline was measured" + } else { + "the document is not `open`: the signed open document has not been sealed and published" + }; + format!("NOT scorable yet — {missing}.") +} + +/// The steps still standing between this install and a scorable topic, for a +/// machine caller. Empty when the topic is already scorable. +fn remaining_steps( + report: &proof_topic_install::InstallReport, + args: &InstallArgs<'_>, + topic_id: &str, +) -> Vec { + if scorable(report, args) { + return Vec::new(); + } + let mut steps = Vec::new(); + if args.skip_baseline || !matches!(report.setup, SetupSummary::Baselined { .. }) { + steps.push(format!( + "proof-admin topic install --bundle {} --env {} --drive-rlm --owner-approved", + args.bundle.display(), + args.env + )); + } + steps.push(format!("proof-admin topic baseline {topic_id}")); + steps.push(format!( + "sign the open document sealing that commitment, then: proof-admin topic seal \ + {topic_id} --document --publish --admin-url \ + --admin-token-file " + )); + steps +} + /// Read the live judge offer the baseline's paid run needs. /// /// Read and validated here rather than inside the driver so a misconfigured @@ -438,36 +503,65 @@ fn print_install_report(report: &proof_topic_install::InstallReport) { } /// What the operator does next, which depends on where the install stopped. +/// +/// The last two steps of the ceremony are the **same** for a draft and for an +/// installed-and-measured topic, so they are named once: `topic baseline` +/// reads the measurement and prints the commitment an `open` document must +/// seal, and `topic seal --publish` records the seal and publishes the open +/// document. Between them the operator signs the open document (the CLI never +/// holds the `proof` key). That is the whole remaining path to a **scorable** +/// topic — nothing else is required, and nothing here spends. fn next_steps(plan: &TopicInstallPlan, args: &InstallArgs<'_>) -> String { + let seal = seal_steps(&plan.topic_id); if plan.document_status == proof_task::TopicStatus::Draft { return format!( "The document is a draft, so miners cannot submit to it yet. To go live:\n \ 1. Drive the RLM setup (provision, rules, baseline):\n \ proof-admin topic install --bundle {} --env {} --drive-rlm --owner-approved\n \ - 2. Seal the baseline the RLM measured, re-sign the document as `open`, and\n \ - publish it through POST /v1/admin/proof/topics.\n \ - 3. Confirm it is live: proof-admin topic show {}", + {}", args.bundle.display(), args.env, - plan.topic_id + seal ); } if args.skip_baseline { return format!( "The document is {}, but --skip-baseline was given, so no baseline was measured.\n\ - Re-run without it before the topic can score:\n \ - proof-admin topic install --bundle {} --env {} --drive-rlm --owner-approved", + Re-run without it — that is the only way to a scorable topic:\n \ + proof-admin topic install --bundle {} --env {} --drive-rlm --owner-approved\n \ + {}", crate::status_word(plan.document_status), args.bundle.display(), - args.env + args.env, + seal ); } format!( - "The document is {}. If the RLM setup was not driven, do that before miners submit:\n \ - proof-admin topic install --bundle {} --env {} --drive-rlm --owner-approved", + "The document is {}. If the RLM setup was not driven, do that first:\n \ + proof-admin topic install --bundle {} --env {} --drive-rlm --owner-approved\n \ + {}", crate::status_word(plan.document_status), args.bundle.display(), - args.env + args.env, + seal + ) +} + +/// The last two steps: read the measurement, sign the open document, seal it. +/// +/// Shared by every branch above because they all end here, and because these +/// are the commands that make the topic **scorable** — the install alone never +/// does, whichever way it was run. +fn seal_steps(topic_id: &str) -> String { + format!( + "2. Read the measured baseline and the commitment the open document must seal:\n \ + proof-admin topic baseline {topic_id}\n \ + 3. Put that `metrics_commitment` into the document, set `status: open`, sign it\n \ + (the `proof` key stays with you: `xtask proof-topic` signs a draft), then:\n \ + proof-admin topic seal {topic_id} --document --publish \\\n \ + --admin-url --admin-token-file \n \ + 4. Confirm the host scores it: GET /v1/status reports `can_score` and lists the topic\n \ + in `scorable_topics` (`ctx proof status` from a miner host)." ) } diff --git a/bins/proof-admin/tests/cli.rs b/bins/proof-admin/tests/cli.rs index 5eaca94f6..8581fac5a 100644 --- a/bins/proof-admin/tests/cli.rs +++ b/bins/proof-admin/tests/cli.rs @@ -1014,6 +1014,26 @@ fn disable_and_enable_need_the_gate_database() { } } +/// The commands that take an install to a **scorable** topic are implemented +/// and fail closed when they cannot run: `topic baseline` needs the registry +/// (it reads the measurement the RLM stored) and `topic seal` needs the signed +/// open document. An install never makes a topic scorable on its own, so this +/// is the path an operator is told to take. +#[test] +fn the_scorable_path_fails_closed_on_its_inputs() { + let out = run(&["topic", "baseline", "tb4"]); + assert_eq!(code(&out), EXIT_USAGE, "{}", stderr(&out)); + assert!( + stderr(&out).contains("BASE_DATABASE_URL"), + "{}", + stderr(&out) + ); + + let out = run(&["topic", "seal", "tb4"]); + assert_eq!(code(&out), EXIT_USAGE, "{}", stderr(&out)); + assert!(stderr(&out).contains("--document"), "{}", stderr(&out)); +} + #[test] fn help_lists_every_subcommand() { let out = run(&["topic", "--help"]); diff --git a/bins/proof-admin/tests/fixtures/README-dry-run.md b/bins/proof-admin/tests/fixtures/README-dry-run.md index 33c70642f..782df316a 100644 --- a/bins/proof-admin/tests/fixtures/README-dry-run.md +++ b/bins/proof-admin/tests/fixtures/README-dry-run.md @@ -62,6 +62,37 @@ Read the journal back with: BASE_DATABASE_URL=… proof-admin topic install-log --topic tb4 ``` +## The rest of the path to a scorable topic + +An install never makes a topic scorable on its own, whichever way it runs: the +topic needs an **`open`** document whose baseline is **sealed**, and the seal +is the operator's. `topic install` prints this, and `--json` reports it as +`"scorable": false` with the `remaining` steps. The three commands are: + +```bash +# 1. Measure the baseline (paid: a VM + a judge call). Requires the +# topic-VM orchestrator and the owner's assertions. +proof-admin topic install --bundle --env staging --drive-rlm --owner-approved \ + --admin-url --admin-token-file + +# 2. Read what was measured and the commitment the open document must seal. +proof-admin topic baseline tb4 + +# 3. Put that `metrics_commitment` into the document, set `status: open`, sign +# it with the `proof` key (`xtask proof-topic`), then seal and publish. +proof-admin topic seal tb4 --document --publish \ + --admin-url --admin-token-file +``` + +`topic seal` runs the same `mark_sealed` the runtime uses, so a document it +accepts is one the scoring path accepts; `--publish` then posts it through the +admin route (which still refuses an `open` document whose install is not +`applied`). Confirm with `GET /v1/status`: `can_score` is true and the topic is +in `scorable_topics`. + +`--skip-baseline` is a **pause**, not a path: it installs the rules and leaves +the topic unscorable until a run without the flag measures a baseline. + ### Two things the command needs **`-p proof-admin-bin`, not `-p proof-admin`.** The repo names binary packages @@ -101,8 +132,9 @@ row key and is a follow-up (see below). ## Staging migrate -`crates/db/migrations/0024_proof_topic_alias.sql` and -`crates/db/migrations/0025_proof_topic_install.sql` are the schema changes in +`crates/db/migrations/0024_proof_topic_alias.sql`, +`crates/db/migrations/0025_proof_topic_install.sql`, and +`crates/db/migrations/0026_proof_topic_gate.sql` are the schema changes in this stack. **There is no manual migration command to run.** Migrations are embedded in @@ -134,8 +166,13 @@ What they do, exactly: paths stored **relative** so a row cannot escape the topic's prefix). Both are append-only for `base_app`: a journal that could be edited in place would not be a journal, so a re-install appends. -- Neither **does** `ALTER` or `DROP` anything: the `0020` tables keep their - columns, keys, and grants. +- `0026` **adds** `proof_topic_gate` (the operator switch `topic disable` / + `topic enable` appends to: state, reason, actor). Append-only too — the + newest row per topic is the state, the rows before it are the history — and + the challenge reads it on the submit path, so a disable takes effect on the + next request with no re-sign, restart, or redeploy. +- None of them **does** `ALTER` or `DROP` anything: the `0020` tables keep + their columns, keys, and grants. ## Regenerating diff --git a/crates/proof-topic-install/src/install.rs b/crates/proof-topic-install/src/install.rs index bc1327281..09b3494db 100644 --- a/crates/proof-topic-install/src/install.rs +++ b/crates/proof-topic-install/src/install.rs @@ -203,6 +203,13 @@ pub struct InstallReport { pub setup: SetupSummary, /// The journal row this run appended. pub journal_id: i64, + /// The document's own status, as the bundle carries it. + /// + /// Reported so an operator (and `topic install --json`) can see whether + /// the install left a **scorable** topic: a `draft` is not one, whatever + /// the setup step measured, and the remaining step is the operator's seal + /// (`proof-admin topic seal --publish`). + pub document_status: TopicStatus, } /// Everything one install needs, resolved by the caller. @@ -337,6 +344,7 @@ impl Installer<'_> { binding: binding.clone(), setup, journal_id, + document_status: request.topic.status, }) } From ab1b53207e3d574543fbf6e1a08cdf63d01b4911 Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:58:31 +0000 Subject: [PATCH 6/8] refactor(proof): stay under the per-crate LOC cap by extracting what moved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workspace caps a crate at 1500 non-test lines, and this branch's changes pushed three of them over: `proof-http` (1562), `gateway` (1529) and `proof-admin` (1863). The cap is per **crate**, so moving code between modules of the same crate does not help; the repository's own answer — the reason `gateway-core` and `proof-topic-setup` exist — is to extract the subject into its own crate. This does that, with no behaviour change: - `gateway-core::admin_route` owns the one forwarded admin route and its bearer floor, with the four gates and the tests. `gateway::proxy` re-exports them and keeps only the HTTP shape of the refusal. - `gateway-core::proxy_paths` owns the read gates the proxy applies before dialing (operator-local report bodies, miner-controlled viewer paths), with their tests. - `proof-http::operator` owns the journal the submit path reads (`InstallJournal`, `SubmitGate`, the publish and submit gates) and the topic-VM orchestrator diagnostic. - `proof-submit` owns `SubmitBody` and `SubmitBody::authenticate`, beside the `SubmitFields` it has to agree with: a field added to the wire type without a matching signed field would be a value a miner sends and nobody authenticates. `is_digest_of_nothing` and `nonce_from` move with it. - `proof-topic-ops` is new: the operator procedures that reach a live host (`drive`, `baseline`, `seal`) plus `PublishTarget`. `proof-admin` keeps the argument parsing and the printing, and its exit codes still follow the same usage/error split (`OpsError`). `VmAgentHealth` is now the wire type itself (`proof_vm_proto::AgentHealth`) rather than a five-field mirror of it, so the route cannot publish a shape the agent does not send. Verified: workspace clippy clean, `loc-cap`/`consensus-lint`/`spec-check`/ `design-check`/`external-docs-check` pass, and the touched crates' tests are green. `cargo deny` advisories and two `chmod 000`-based tests fail the same way on the base commit (they rely on permission bits, which root ignores). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- Cargo.lock | 22 + bins/proof-admin/Cargo.toml | 1 + bins/proof-admin/src/install.rs | 94 +--- bins/proof-admin/src/main.rs | 465 +++++------------- bins/proof-admin/src/registry.rs | 353 +++++++++++++ bins/proof-challenge/src/main.rs | 10 +- crates/gateway-core/src/admin_route.rs | 172 +++++++ crates/gateway-core/src/lib.rs | 6 + crates/gateway-core/src/proxy_paths.rs | 72 +++ crates/gateway-core/src/topic_routes.rs | 17 + crates/gateway/src/proxy.rs | 210 +------- crates/proof-challenge/src/lib.rs | 2 +- crates/proof-http/src/lib.rs | 447 ++--------------- crates/proof-http/src/operator.rs | 279 +++++++++++ crates/proof-http/src/submit.rs | 3 +- crates/proof-results/src/lib.rs | 14 +- crates/proof-submit/Cargo.toml | 3 + crates/proof-submit/src/lib.rs | 132 +++++ crates/proof-topic-ops/Cargo.toml | 29 ++ .../proof-topic-ops}/src/drive.rs | 29 +- crates/proof-topic-ops/src/lib.rs | 83 ++++ crates/proof-topic-ops/src/publish.rs | 101 ++++ .../proof-topic-ops}/src/seal.rs | 237 ++++----- .../harness/summarize.py | 21 +- .../tests/test_summarize.py | 4 +- .../assert-harbor-runner-results-emit.sh | 7 +- 26 files changed, 1635 insertions(+), 1178 deletions(-) create mode 100644 bins/proof-admin/src/registry.rs create mode 100644 crates/gateway-core/src/admin_route.rs create mode 100644 crates/gateway-core/src/proxy_paths.rs create mode 100644 crates/proof-http/src/operator.rs create mode 100644 crates/proof-topic-ops/Cargo.toml rename {bins/proof-admin => crates/proof-topic-ops}/src/drive.rs (93%) create mode 100644 crates/proof-topic-ops/src/lib.rs create mode 100644 crates/proof-topic-ops/src/publish.rs rename {bins/proof-admin => crates/proof-topic-ops}/src/seal.rs (75%) diff --git a/Cargo.lock b/Cargo.lock index 0e82a5e51..fabfc40a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3697,6 +3697,7 @@ dependencies = [ "proof-task", "proof-topic-bundle", "proof-topic-install", + "proof-topic-ops", "proof-topic-setup", "proof-vm-fc", "reqwest 0.12.28", @@ -4033,7 +4034,10 @@ version = "0.1.0" dependencies = [ "crypto", "hex", + "proof-canon", + "proof-store", "rand_core 0.6.4", + "serde", "serde_json", "sha2 0.10.9", "thiserror 2.0.19", @@ -4089,6 +4093,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "proof-topic-ops" +version = "0.1.0" +dependencies = [ + "db", + "proof-eval", + "proof-rlm", + "proof-rlm-store", + "proof-task", + "proof-topic-bundle", + "proof-topic-install", + "proof-topic-setup", + "proof-vm-fc", + "reqwest 0.12.28", + "serde_json", + "sqlx", +] + [[package]] name = "proof-topic-setup" version = "0.1.0" diff --git a/bins/proof-admin/Cargo.toml b/bins/proof-admin/Cargo.toml index 33a31f9a5..8bc3aa183 100644 --- a/bins/proof-admin/Cargo.toml +++ b/bins/proof-admin/Cargo.toml @@ -21,6 +21,7 @@ proof-rlm-store = { path = "../../crates/proof-rlm-store" } proof-task = { path = "../../crates/proof-task" } proof-topic-bundle = { path = "../../crates/proof-topic-bundle" } proof-topic-install = { path = "../../crates/proof-topic-install" } +proof-topic-ops = { path = "../../crates/proof-topic-ops" } proof-topic-setup = { path = "../../crates/proof-topic-setup" } proof-vm-fc = { path = "../../crates/proof-vm-fc" } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } diff --git a/bins/proof-admin/src/install.rs b/bins/proof-admin/src/install.rs index 815464d11..7aa6460f6 100644 --- a/bins/proof-admin/src/install.rs +++ b/bins/proof-admin/src/install.rs @@ -52,6 +52,7 @@ use proof_topic_install::install::{InstallRequest, Installer, SetupSummary}; use proof_topic_install::InstallError; use crate::{Failure, Options}; +use proof_topic_ops::PublishTarget; /// What the operator asserted, and what the install is therefore allowed to do. /// @@ -172,7 +173,8 @@ async fn run_real( ) -> Result<(), Failure> { // The bearer and the URL are resolved before anything is written, so a // misconfiguration cannot leave a half-installed topic. - let admin = AdminTarget::resolve(args.admin_url, args.admin_token_file)?; + let admin = PublishTarget::resolve(args.admin_url, args.admin_token_file) + .map_err(crate::ops_to_failure)?; let database_url = crate::database_url(opts)?.ok_or_else(|| { Failure::Usage( "a real install writes to the topic registry, so it needs a database: set \ @@ -421,7 +423,7 @@ async fn drive_rlm( pin: &ProofPin, pool: &sqlx::PgPool, store: &PgRlmStore, -) -> Result { +) -> Result { let _ = store; let offer = if args.skip_baseline { None @@ -433,7 +435,7 @@ async fn drive_rlm( .ok() .map(|p| PathBuf::from(p.trim().to_owned())); let digest = std::env::var(RLM_VM_IMAGE_DIGEST_ENV).ok(); - crate::drive::drive( + proof_topic_ops::drive( topic, pin, PgRlmStore::new(pool.clone()), @@ -446,6 +448,7 @@ async fn drive_rlm( args.owner_key_file, ) .await + .map_err(crate::ops_to_failure) } /// Print the install report. @@ -565,91 +568,6 @@ fn seal_steps(topic_id: &str) -> String { ) } -/// Where the admin publish call goes, and the bearer it uses. -pub(crate) struct AdminTarget { - base_url: String, - token: String, -} - -impl AdminTarget { - /// Resolve the URL and bearer, refusing a half-configured pair. - pub(crate) fn resolve( - admin_url: Option<&str>, - admin_token_file: Option<&Path>, - ) -> Result { - let Some(base_url) = admin_url.map(str::trim).filter(|u| !u.is_empty()) else { - return Err(Failure::Usage( - "a real install publishes through the admin route, so it needs the master's \ - base URL: pass --admin-url (or set PROOF_ADMIN_URL), e.g. \ - --admin-url http://127.0.0.1:8100 for the challenge service directly, or the \ - gateway's address. `--dry-run` needs none." - .to_owned(), - )); - }; - let Some(path) = admin_token_file else { - return Err(Failure::Usage( - "a real install needs the operator bearer for /v1/admin/*: pass \ - --admin-token-file (or set PROOF_ADMIN_TOKEN_FILE). The file is read and never \ - logged or printed. `--dry-run` needs none." - .to_owned(), - )); - }; - let token = std::fs::read_to_string(path) - .map_err(|e| Failure::Error(format!("read {}: {e}", path.display())))?; - // A tokens file holds one bearer per line; the first non-comment line - // is the one this call uses. - let token = token - .lines() - .map(str::trim) - .find(|l| !l.is_empty() && !l.starts_with('#')) - .map(str::to_owned); - let Some(token) = token else { - return Err(Failure::Error(format!( - "{} holds no bearer (every line is blank or a comment)", - path.display() - ))); - }; - Ok(Self { - base_url: base_url.trim_end_matches('/').to_owned(), - token, - }) - } - - /// How this target is printed: the URL, never the bearer. - fn redacted(&self) -> String { - format!("{} (bearer read, never printed)", self.base_url) - } - - /// Publish the document through the existing admin route. - pub(crate) async fn publish(&self, doc: &proof_task::TopicDocument) -> Result<(), String> { - let url = format!("{}{}", self.base_url, proof_topic_bundle::PUBLISH_PATH); - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_mins(1)) - .build() - .map_err(|e| format!("http client: {e}"))?; - let response = client - .post(&url) - .header("authorization", format!("Bearer {}", self.token)) - .header("content-type", "application/json") - .body( - serde_json::to_string(doc) - .map_err(|e| format!("serialize the signed document: {e}"))?, - ) - .send() - .await - .map_err(|e| format!("POST {url}: {e}"))?; - let status = response.status(); - if status.is_success() { - return Ok(()); - } - let body = response.text().await.unwrap_or_default(); - Err(format!( - "POST {url} answered {status}: {}", - body.trim().chars().take(400).collect::() - )) - } -} - /// Turn a publish refusal into an operator instruction. /// /// The install has already run by the time this can happen, so the message diff --git a/bins/proof-admin/src/main.rs b/bins/proof-admin/src/main.rs index ffb86abec..c3d21dd81 100644 --- a/bins/proof-admin/src/main.rs +++ b/bins/proof-admin/src/main.rs @@ -36,13 +36,16 @@ use std::path::{Path, PathBuf}; use std::process::ExitCode; use clap::{Parser, Subcommand}; -use proof_rlm_store::{MemoryRlmStore, PgRlmStore, RlmStore, TopicVersionRow}; use proof_task::ProofPin; use proof_topic_bundle::{InstallEnvironment, TopicInstallBundle, TopicInstallPlan, PUBLISH_PATH}; -mod drive; mod install; -mod seal; +mod registry; + +pub(crate) use registry::print_json; +use registry::{ + cmd_gate, cmd_list, cmd_show, dash_if_empty, database_url, open_pool, open_store, status_word, +}; use install::InstallArgs; @@ -417,20 +420,18 @@ async fn run_topic(opts: &Options, cmd: &TopicCmd) -> Result<(), Failure> { admin_url, admin_token_file, } => { - seal::seal( + cmd_seal( opts, - &seal::SealArgs { - topic_id, - document, - pin, - publish: *publish, - admin_url: admin_url.as_deref(), - admin_token_file: admin_token_file.as_deref(), - }, + topic_id, + document, + pin, + *publish, + admin_url.as_deref(), + admin_token_file.as_deref(), ) .await } - TopicCmd::Baseline { topic_id, pin } => seal::baseline(opts, topic_id, pin).await, + TopicCmd::Baseline { topic_id, pin } => cmd_baseline(opts, topic_id, pin).await, } } @@ -615,6 +616,108 @@ async fn cmd_install_log(opts: &Options, topic_id: &str) -> Result<(), Failure> install::install_log(opts, topic_id).await } +/// `topic baseline`: read the measurement and print what to seal. +/// +/// The procedure is [`proof_topic_ops::baseline`]; this is the printing. +async fn cmd_baseline(opts: &Options, topic_id: &str, pin_path: &Path) -> Result<(), Failure> { + let pool = open_pool(opts).await?; + let pin = load_pin(pin_path)?; + let report = proof_topic_ops::baseline(&pool, &pin, topic_id) + .await + .map_err(ops_to_failure)?; + if opts.json { + return print_json(&serde_json::json!({ + "topic_id": report.topic_id, + "rules_version": report.rules_version, + "primary_value": report.primary_value, + "metric_primary": report.metric_primary, + "custom_id": report.custom_id, + "holdout_commitment": report.holdout_commitment, + "metrics_commitment": report.metrics_commitment, + "document_status": report.document_status, + "next": report.next_steps(), + })); + } + println!("topic {} — measured baseline", report.topic_id); + println!(" primary_value {}", report.primary_value); + println!(" metric_primary {}", report.metric_primary); + println!(" custom_id {}", dash_if_empty(&report.custom_id)); + println!(" rules_version {}", report.rules_version); + println!(" holdout {}", report.holdout_commitment); + println!( + " document_status {}", + status_word(report.document_status) + ); + println!(); + println!("An `open` document must seal this measurement. Its baseline block needs:"); + println!(" metrics_commitment {}", report.metrics_commitment); + println!(); + println!("{}", report.next_steps()); + Ok(()) +} + +/// `topic seal`: record the seal, then optionally publish. +/// +/// The procedure is [`proof_topic_ops::seal`]; this is the printing. +async fn cmd_seal( + opts: &Options, + topic_id: &str, + document: &Path, + pin_path: &Path, + publish: bool, + admin_url: Option<&str>, + admin_token_file: Option<&Path>, +) -> Result<(), Failure> { + let pool = open_pool(opts).await?; + let pin = load_pin(pin_path)?; + let outcome = proof_topic_ops::seal( + &pool, + &proof_topic_ops::SealArgs { + topic_id, + document, + pin: &pin, + publish, + admin_url, + admin_token_file, + registered_custom: registered_custom_from_env(), + }, + ) + .await + .map_err(ops_to_failure)?; + if opts.json { + return print_json(&serde_json::json!({ + "ok": true, + "topic_id": outcome.topic_id, + "state": "open", + "document_version": outcome.document_version, + "metrics_commitment": outcome.metrics_commitment, + "primary_value": outcome.primary_value, + "published": outcome.published, + })); + } + println!("topic {} sealed and opened.", outcome.topic_id); + println!(" state open"); + println!(" document_version {}", outcome.document_version); + println!(" primary_value {}", outcome.primary_value); + println!(" commitment {}", outcome.metrics_commitment); + if outcome.published { + println!(" published yes (the topic's routes and document are live)"); + } else { + println!(" published no (--publish was not given)"); + } + println!(); + println!("{}", outcome.after()); + Ok(()) +} + +/// An operator procedure's refusal, as the CLI's exit code and message. +pub(crate) fn ops_to_failure(e: proof_topic_ops::OpsError) -> Failure { + match e { + proof_topic_ops::OpsError::Usage(m) => Failure::Usage(m), + proof_topic_ops::OpsError::Error(m) => Failure::Error(m), + } +} + pub(crate) fn print_plan(plan: &TopicInstallPlan, bundle_path: &Path, pin_path: &Path) { println!("topic install plan"); println!(" topic_id {}", plan.topic_id); @@ -722,339 +825,3 @@ fn publish_block(bundle_path: &Path) -> String { fn shell_single_quote(s: &str) -> String { format!("'{}'", s.replace('\'', r"'\''")) } - -async fn cmd_list(opts: &Options) -> Result<(), Failure> { - let store = open_store(opts).await?; - let rows = store - .latest_topics() - .await - .map_err(|e| Failure::Error(format!("list topics: {e}")))?; - if opts.json { - let body: Vec = rows.iter().map(topic_json).collect(); - print_json(&body)?; - return Ok(()); - } - if rows.is_empty() { - println!("No topics installed."); - return Ok(()); - } - println!("{} topic(s) installed:", rows.len()); - for row in &rows { - println!(" {}", summarize(row)); - } - println!(); - println!("Read from proof_topic_version; the signed document is the source of truth."); - Ok(()) -} - -async fn cmd_show(opts: &Options, topic_id: &str) -> Result<(), Failure> { - let pool = open_pool(opts).await?; - let store = PgRlmStore::new(pool.clone()); - // An alias resolves to its canonical slug first, so `show tbench` finds - // `tb4`. Resolution is fail-closed in the store: an alias whose topic has - // no published version resolves to nothing rather than to an empty row. - let resolved = store - .resolve_alias(topic_id) - .await - .map_err(|e| Failure::Error(format!("resolve {topic_id}: {e}")))?; - let canonical = resolved.as_deref().unwrap_or(topic_id); - let row = store - .latest_topic(canonical) - .await - .map_err(|e| Failure::Error(format!("show {canonical}: {e}")))?; - let Some((version, document)) = row else { - return Err(Failure::Error(format!( - "no installed topic {topic_id:?}{}. Use `proof-admin topic list` to see the exact ids.", - resolved - .as_deref() - .map(|c| format!(" (alias of {c:?})")) - .unwrap_or_default() - ))); - }; - let row = TopicVersionRow { - topic_id: canonical.to_owned(), - version, - document, - }; - // The operator gate, read from the same table the challenge reads: `show` - // must not report a topic as open for work when the submit path refuses - // it. An unreadable gate is reported rather than assumed enabled. - let gate = proof_topic_install::gate(&pool, canonical) - .await - .map_err(|e| Failure::Error(format!("{canonical} gate: {e}")))?; - if let Some(canonical) = resolved.as_deref() { - if !opts.json { - println!("{topic_id} is an alias of {canonical}"); - println!(); - } - } - if opts.json { - let mut body = topic_json(&row); - if let Some(obj) = body.as_object_mut() { - obj.insert( - "disabled".to_owned(), - serde_json::Value::Bool( - gate.as_ref() - .is_some_and(proof_topic_install::Gate::is_disabled), - ), - ); - if let Some(gate) = gate.as_ref().filter(|g| g.is_disabled()) { - obj.insert( - "disabled_reason".to_owned(), - serde_json::Value::String(gate.reason.clone()), - ); - } - } - print_json(&body)?; - return Ok(()); - } - print_row(&row); - print_gate(gate.as_ref(), canonical); - Ok(()) -} - -/// The operator gate line(s) for `topic show`. -fn print_gate(gate: Option<&proof_topic_install::Gate>, topic_id: &str) { - println!(); - match gate { - None => println!("Operator gate: enabled (no `proof_topic_gate` row)."), - Some(gate) if !gate.is_disabled() => { - println!( - "Operator gate: enabled (gate row {}; the newest row is an enable).", - gate.id - ); - } - Some(gate) => { - println!("Operator gate: DISABLED (gate row {}).", gate.id); - if gate.reason.is_empty() { - println!(" reason (none given)"); - } else { - println!(" reason {}", gate.reason); - } - if !gate.actor.is_empty() { - println!(" actor {}", gate.actor); - } - println!(); - println!("Submissions to this topic are refused. Re-enable with:"); - println!(" proof-admin topic enable {topic_id}"); - } - } -} - -/// The topic registry: the existing `proof_topic_version` rows. -/// -/// A configured but unreachable database is fatal: falling back to an empty -/// in-memory view would report "nothing installed" for a host that has topics. -async fn open_store(opts: &Options) -> Result, Failure> { - let pool = open_pool(opts).await?; - // `PgRlmStore` is the production registry; the memory store exists for - // CI/local and is never selected here, so a real host never reads an - // empty view by accident. - let _ = MemoryRlmStore::new; - Ok(Box::new(PgRlmStore::new(pool))) -} - -/// A connection pool over the topic database. -/// -/// The gate commands write (`proof_topic_gate`) as well as read, so they need -/// the pool itself and not only the registry trait object. -pub(crate) async fn open_pool(opts: &Options) -> Result { - let Some(url) = database_url(opts)? else { - return Err(Failure::Usage( - "this command reads the topic registry and needs a database: set \ - BASE_DATABASE_URL (or BASE_DATABASE_URL_FILE). `topic validate` and \ - `topic install --dry-run` need no database." - .into(), - )); - }; - db::connect(&url) - .await - .map_err(|e| Failure::Error(format!("connect: {e}"))) -} - -/// `topic disable` / `topic enable`: throw the operator gate. -/// -/// The topic must be published (or be an alias of one): a typo must not -/// silently disable nothing, because the operator would then believe a topic -/// is stopped while it is still taking submissions. The write is append-only -/// — the newest row is the state, the rows before it are the history — and it -/// is visible to the challenge on the next request, which is the point of the -/// switch. -async fn cmd_gate( - opts: &Options, - topic_id: &str, - state: proof_topic_install::GateState, - reason: Option<&str>, - actor: Option<&str>, -) -> Result<(), Failure> { - let pool = open_pool(opts).await?; - let store = PgRlmStore::new(pool.clone()); - let resolved = store - .resolve_alias(topic_id) - .await - .map_err(|e| Failure::Error(format!("resolve {topic_id}: {e}")))?; - let canonical = resolved.as_deref().unwrap_or(topic_id); - let row = store - .latest_topic(canonical) - .await - .map_err(|e| Failure::Error(format!("{canonical}: {e}")))?; - if row.is_none() { - return Err(Failure::Error(format!( - "no installed topic {topic_id:?}{}. Nothing was changed — check the id with \ - `proof-admin topic list`.", - resolved - .as_deref() - .map(|c| format!(" (alias of {c:?})")) - .unwrap_or_default() - ))); - } - let reason = reason.unwrap_or_default(); - let actor = actor.unwrap_or_default(); - let gate = proof_topic_install::set(&pool, canonical, state, reason, actor) - .await - .map_err(|e| Failure::Error(format!("{canonical}: {e}")))?; - let disabled = gate.is_disabled(); - if opts.json { - print_json(&serde_json::json!({ - "ok": true, - "topic_id": canonical, - "state": gate.state.as_str(), - "disabled": disabled, - "reason": gate.reason, - "actor": gate.actor, - "gate_row": gate.id, - }))?; - return Ok(()); - } - if disabled { - println!("topic {canonical} is disabled (gate row {}).", gate.id); - if gate.reason.is_empty() { - println!(" reason (none given)"); - } else { - println!(" reason {}", gate.reason); - } - println!(); - println!( - "Submissions are refused from the next request on, with this reason. The document \ - keeps its own status, in-flight evaluations finish, and rows already scored keep \ - their verdicts. Nothing was re-signed and nothing was restarted." - ); - println!(); - println!("To let it take submissions again:"); - println!(" proof-admin topic enable {canonical}"); - } else { - println!("topic {canonical} is enabled again (gate row {}).", gate.id); - println!(); - println!( - "Submissions are admitted from the next request on, under the topic's own document \ - (`status`), which was never changed. The disable rows stay in the history." - ); - } - Ok(()) -} - -/// `BASE_DATABASE_URL` value, or the contents of `BASE_DATABASE_URL_FILE`. -/// -/// The two are mutually exclusive, matching `crates/config`: a value and a -/// file that disagree would be a silent choice between two databases. -pub(crate) fn database_url(opts: &Options) -> Result, Failure> { - let value = opts - .database_url - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - let file = opts.database_url_file.as_deref(); - match (value, file) { - (Some(_), Some(_)) => Err(Failure::Usage( - "set BASE_DATABASE_URL or BASE_DATABASE_URL_FILE, not both".into(), - )), - (Some(url), None) => Ok(Some(url.to_owned())), - (None, Some(path)) => { - let raw = std::fs::read_to_string(path) - .map_err(|e| Failure::Error(format!("read {}: {e}", path.display())))?; - let trimmed = raw.trim(); - if trimmed.is_empty() { - return Err(Failure::Usage(format!("{} is empty", path.display()))); - } - Ok(Some(trimmed.to_owned())) - } - (None, None) => Ok(None), - } -} - -fn print_row(row: &TopicVersionRow) { - let doc = &row.document; - println!("topic {}", row.topic_id); - println!(" version {}", row.version); - println!(" status {}", status_word(doc.status)); - println!(" metric_family {}", doc.metric.family.as_str()); - println!( - " custom_id {}", - dash_if_empty(&doc.metric.custom_id) - ); - println!(" payout_mode {}", doc.payout_mode.as_str()); - println!(" valid_from_epoch {}", doc.valid_from_epoch); - println!( - " valid_until_epoch {}", - doc.valid_until_epoch - .map_or_else(|| "-".to_owned(), |e| e.to_string()) - ); - println!(" baseline_sealed {}", doc.baseline.is_sealed()); - println!( - " signature {}…", - doc.signature.get(..16).unwrap_or(doc.signature.as_str()) - ); - println!(); - println!("The signed document is the source of truth; this view reads it verbatim."); -} - -/// One-line summary for `topic list`. -fn summarize(row: &TopicVersionRow) -> String { - let doc = &row.document; - format!( - "{:<24} v{:<3} {:<10} {:<10} custom_id={}", - row.topic_id, - row.version, - status_word(doc.status), - doc.metric.family.as_str(), - dash_if_empty(&doc.metric.custom_id) - ) -} - -/// Lifecycle word, matching the wire spelling the document uses. -pub(crate) fn status_word(status: proof_task::TopicStatus) -> &'static str { - match status { - proof_task::TopicStatus::Draft => "draft", - proof_task::TopicStatus::Open => "open", - proof_task::TopicStatus::Closed => "closed", - } -} - -fn topic_json(row: &TopicVersionRow) -> serde_json::Value { - serde_json::json!({ - "topic_id": row.topic_id, - "version": row.version, - "status": row.document.status, - "metric_family": row.document.metric.family, - "custom_id": row.document.metric.custom_id, - "payout_mode": row.document.payout_mode.as_str(), - "valid_from_epoch": row.document.valid_from_epoch, - "valid_until_epoch": row.document.valid_until_epoch, - "baseline_sealed": row.document.baseline.is_sealed(), - "document": row.document, - }) -} - -pub(crate) fn print_json(value: &T) -> Result<(), Failure> { - let body = serde_json::to_string_pretty(value).map_err(|e| Failure::Error(e.to_string()))?; - println!("{body}"); - Ok(()) -} - -pub(crate) fn dash_if_empty(s: &str) -> String { - if s.trim().is_empty() { - "-".to_owned() - } else { - s.to_owned() - } -} diff --git a/bins/proof-admin/src/registry.rs b/bins/proof-admin/src/registry.rs new file mode 100644 index 000000000..27c3c5a43 --- /dev/null +++ b/bins/proof-admin/src/registry.rs @@ -0,0 +1,353 @@ +//! The **registry** half of the CLI: reading what is installed, and the +//! operator gate. +//! +//! `topic list` / `show` / `install-log` and `topic disable` / `enable` are +//! all one shape — open the topic database, read or append one row, print it — +//! so they live together here rather than in `main.rs`, which is at the +//! repository's per-crate LOC cap. The *decision* logic they depend on (what +//! an install journal means, what a gate row is) stays in +//! `proof_topic_install`; this module is the CLI's reading and writing of it. +//! +//! Everything here is **fail-closed on the database**: a configured but +//! unreachable database is fatal, because falling back to an empty in-memory +//! view would report "nothing installed" for a host that has topics. + +use proof_rlm_store::{MemoryRlmStore, PgRlmStore, RlmStore, TopicVersionRow}; + +use crate::{Failure, Options}; + +pub(crate) async fn cmd_list(opts: &Options) -> Result<(), Failure> { + let store = open_store(opts).await?; + let rows = store + .latest_topics() + .await + .map_err(|e| Failure::Error(format!("list topics: {e}")))?; + if opts.json { + let body: Vec = rows.iter().map(topic_json).collect(); + print_json(&body)?; + return Ok(()); + } + if rows.is_empty() { + println!("No topics installed."); + return Ok(()); + } + println!("{} topic(s) installed:", rows.len()); + for row in &rows { + println!(" {}", summarize(row)); + } + println!(); + println!("Read from proof_topic_version; the signed document is the source of truth."); + Ok(()) +} + +pub(crate) async fn cmd_show(opts: &Options, topic_id: &str) -> Result<(), Failure> { + let pool = open_pool(opts).await?; + let store = PgRlmStore::new(pool.clone()); + // An alias resolves to its canonical slug first, so `show tbench` finds + // `tb4`. Resolution is fail-closed in the store: an alias whose topic has + // no published version resolves to nothing rather than to an empty row. + let resolved = store + .resolve_alias(topic_id) + .await + .map_err(|e| Failure::Error(format!("resolve {topic_id}: {e}")))?; + let canonical = resolved.as_deref().unwrap_or(topic_id); + let row = store + .latest_topic(canonical) + .await + .map_err(|e| Failure::Error(format!("show {canonical}: {e}")))?; + let Some((version, document)) = row else { + return Err(Failure::Error(format!( + "no installed topic {topic_id:?}{}. Use `proof-admin topic list` to see the exact ids.", + resolved + .as_deref() + .map(|c| format!(" (alias of {c:?})")) + .unwrap_or_default() + ))); + }; + let row = TopicVersionRow { + topic_id: canonical.to_owned(), + version, + document, + }; + // The operator gate, read from the same table the challenge reads: `show` + // must not report a topic as open for work when the submit path refuses + // it. An unreadable gate is reported rather than assumed enabled. + let gate = proof_topic_install::gate(&pool, canonical) + .await + .map_err(|e| Failure::Error(format!("{canonical} gate: {e}")))?; + if let Some(canonical) = resolved.as_deref() { + if !opts.json { + println!("{topic_id} is an alias of {canonical}"); + println!(); + } + } + if opts.json { + let mut body = topic_json(&row); + if let Some(obj) = body.as_object_mut() { + obj.insert( + "disabled".to_owned(), + serde_json::Value::Bool( + gate.as_ref() + .is_some_and(proof_topic_install::Gate::is_disabled), + ), + ); + if let Some(gate) = gate.as_ref().filter(|g| g.is_disabled()) { + obj.insert( + "disabled_reason".to_owned(), + serde_json::Value::String(gate.reason.clone()), + ); + } + } + print_json(&body)?; + return Ok(()); + } + print_row(&row); + print_gate(gate.as_ref(), canonical); + Ok(()) +} + +/// The operator gate line(s) for `topic show`. +pub(crate) fn print_gate(gate: Option<&proof_topic_install::Gate>, topic_id: &str) { + println!(); + match gate { + None => println!("Operator gate: enabled (no `proof_topic_gate` row)."), + Some(gate) if !gate.is_disabled() => { + println!( + "Operator gate: enabled (gate row {}; the newest row is an enable).", + gate.id + ); + } + Some(gate) => { + println!("Operator gate: DISABLED (gate row {}).", gate.id); + if gate.reason.is_empty() { + println!(" reason (none given)"); + } else { + println!(" reason {}", gate.reason); + } + if !gate.actor.is_empty() { + println!(" actor {}", gate.actor); + } + println!(); + println!("Submissions to this topic are refused. Re-enable with:"); + println!(" proof-admin topic enable {topic_id}"); + } + } +} + +/// The topic registry: the existing `proof_topic_version` rows. +/// +/// A configured but unreachable database is fatal: falling back to an empty +/// in-memory view would report "nothing installed" for a host that has topics. +pub(crate) async fn open_store(opts: &Options) -> Result, Failure> { + let pool = open_pool(opts).await?; + // `PgRlmStore` is the production registry; the memory store exists for + // CI/local and is never selected here, so a real host never reads an + // empty view by accident. + let _ = MemoryRlmStore::new; + Ok(Box::new(PgRlmStore::new(pool))) +} + +/// A connection pool over the topic database. +/// +/// The gate commands write (`proof_topic_gate`) as well as read, so they need +/// the pool itself and not only the registry trait object. +pub(crate) async fn open_pool(opts: &Options) -> Result { + let Some(url) = database_url(opts)? else { + return Err(Failure::Usage( + "this command reads the topic registry and needs a database: set \ + BASE_DATABASE_URL (or BASE_DATABASE_URL_FILE). `topic validate` and \ + `topic install --dry-run` need no database." + .into(), + )); + }; + db::connect(&url) + .await + .map_err(|e| Failure::Error(format!("connect: {e}"))) +} + +/// `topic disable` / `topic enable`: throw the operator gate. +/// +/// The topic must be published (or be an alias of one): a typo must not +/// silently disable nothing, because the operator would then believe a topic +/// is stopped while it is still taking submissions. The write is append-only +/// — the newest row is the state, the rows before it are the history — and it +/// is visible to the challenge on the next request, which is the point of the +/// switch. +pub(crate) async fn cmd_gate( + opts: &Options, + topic_id: &str, + state: proof_topic_install::GateState, + reason: Option<&str>, + actor: Option<&str>, +) -> Result<(), Failure> { + let pool = open_pool(opts).await?; + let store = PgRlmStore::new(pool.clone()); + let resolved = store + .resolve_alias(topic_id) + .await + .map_err(|e| Failure::Error(format!("resolve {topic_id}: {e}")))?; + let canonical = resolved.as_deref().unwrap_or(topic_id); + let row = store + .latest_topic(canonical) + .await + .map_err(|e| Failure::Error(format!("{canonical}: {e}")))?; + if row.is_none() { + return Err(Failure::Error(format!( + "no installed topic {topic_id:?}{}. Nothing was changed — check the id with \ + `proof-admin topic list`.", + resolved + .as_deref() + .map(|c| format!(" (alias of {c:?})")) + .unwrap_or_default() + ))); + } + let reason = reason.unwrap_or_default(); + let actor = actor.unwrap_or_default(); + let gate = proof_topic_install::set(&pool, canonical, state, reason, actor) + .await + .map_err(|e| Failure::Error(format!("{canonical}: {e}")))?; + let disabled = gate.is_disabled(); + if opts.json { + print_json(&serde_json::json!({ + "ok": true, + "topic_id": canonical, + "state": gate.state.as_str(), + "disabled": disabled, + "reason": gate.reason, + "actor": gate.actor, + "gate_row": gate.id, + }))?; + return Ok(()); + } + if disabled { + println!("topic {canonical} is disabled (gate row {}).", gate.id); + if gate.reason.is_empty() { + println!(" reason (none given)"); + } else { + println!(" reason {}", gate.reason); + } + println!(); + println!( + "Submissions are refused from the next request on, with this reason. The document \ + keeps its own status, in-flight evaluations finish, and rows already scored keep \ + their verdicts. Nothing was re-signed and nothing was restarted." + ); + println!(); + println!("To let it take submissions again:"); + println!(" proof-admin topic enable {canonical}"); + } else { + println!("topic {canonical} is enabled again (gate row {}).", gate.id); + println!(); + println!( + "Submissions are admitted from the next request on, under the topic's own document \ + (`status`), which was never changed. The disable rows stay in the history." + ); + } + Ok(()) +} + +/// `BASE_DATABASE_URL` value, or the contents of `BASE_DATABASE_URL_FILE`. +/// +/// The two are mutually exclusive, matching `crates/config`: a value and a +/// file that disagree would be a silent choice between two databases. +pub(crate) fn database_url(opts: &Options) -> Result, Failure> { + let value = opts + .database_url + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()); + let file = opts.database_url_file.as_deref(); + match (value, file) { + (Some(_), Some(_)) => Err(Failure::Usage( + "set BASE_DATABASE_URL or BASE_DATABASE_URL_FILE, not both".into(), + )), + (Some(url), None) => Ok(Some(url.to_owned())), + (None, Some(path)) => { + let raw = std::fs::read_to_string(path) + .map_err(|e| Failure::Error(format!("read {}: {e}", path.display())))?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err(Failure::Usage(format!("{} is empty", path.display()))); + } + Ok(Some(trimmed.to_owned())) + } + (None, None) => Ok(None), + } +} + +pub(crate) fn print_row(row: &TopicVersionRow) { + let doc = &row.document; + println!("topic {}", row.topic_id); + println!(" version {}", row.version); + println!(" status {}", status_word(doc.status)); + println!(" metric_family {}", doc.metric.family.as_str()); + println!( + " custom_id {}", + dash_if_empty(&doc.metric.custom_id) + ); + println!(" payout_mode {}", doc.payout_mode.as_str()); + println!(" valid_from_epoch {}", doc.valid_from_epoch); + println!( + " valid_until_epoch {}", + doc.valid_until_epoch + .map_or_else(|| "-".to_owned(), |e| e.to_string()) + ); + println!(" baseline_sealed {}", doc.baseline.is_sealed()); + println!( + " signature {}…", + doc.signature.get(..16).unwrap_or(doc.signature.as_str()) + ); + println!(); + println!("The signed document is the source of truth; this view reads it verbatim."); +} + +/// One-line summary for `topic list`. +pub(crate) fn summarize(row: &TopicVersionRow) -> String { + let doc = &row.document; + format!( + "{:<24} v{:<3} {:<10} {:<10} custom_id={}", + row.topic_id, + row.version, + status_word(doc.status), + doc.metric.family.as_str(), + dash_if_empty(&doc.metric.custom_id) + ) +} + +/// Lifecycle word, matching the wire spelling the document uses. +pub(crate) fn status_word(status: proof_task::TopicStatus) -> &'static str { + match status { + proof_task::TopicStatus::Draft => "draft", + proof_task::TopicStatus::Open => "open", + proof_task::TopicStatus::Closed => "closed", + } +} + +pub(crate) fn topic_json(row: &TopicVersionRow) -> serde_json::Value { + serde_json::json!({ + "topic_id": row.topic_id, + "version": row.version, + "status": row.document.status, + "metric_family": row.document.metric.family, + "custom_id": row.document.metric.custom_id, + "payout_mode": row.document.payout_mode.as_str(), + "valid_from_epoch": row.document.valid_from_epoch, + "valid_until_epoch": row.document.valid_until_epoch, + "baseline_sealed": row.document.baseline.is_sealed(), + "document": row.document, + }) +} + +pub(crate) fn print_json(value: &T) -> Result<(), Failure> { + let body = serde_json::to_string_pretty(value).map_err(|e| Failure::Error(e.to_string()))?; + println!("{body}"); + Ok(()) +} + +pub(crate) fn dash_if_empty(s: &str) -> String { + if s.trim().is_empty() { + "-".to_owned() + } else { + s.to_owned() + } +} diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index 62648e8c2..df44b40ce 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -26,7 +26,7 @@ use proof_challenge::{ challenge_router, executor_slot, hash_admin_token, parse_holdout_file, AppState, ArtefactVault, BaselineMeasurement, EvalBackend, EvalExecutorOffer, GatewayClient, GatewayClientConfig, HarvestOverrides, InferenceOffer, LiveScorer, MemoryStore, MinerEnvVault, ProofEmitter, - ProofPin, TopicDocument, VmAgentHealth, VmOrchestratorProbe, VmOrchestratorReport, + ProofPin, TopicDocument, VmOrchestratorProbe, VmOrchestratorReport, ARTEFACT_STAGING_DIR_ENV, CHALLENGE_ID, DEFAULT_EMIT_POLL_SECS, MINER_BYOK_DIR_ENV, SCORING_VERSION, }; @@ -622,13 +622,7 @@ impl VmOrchestratorProbe for TopicVm { image_digest: template.image_digest.clone(), vcpus: template.vcpus, mem_mib: template.mem_mib, - agent: health.as_ref().ok().map(|h| VmAgentHealth { - api_version: h.api_version, - ready: h.ready, - reason: h.reason.clone(), - hypervisor: h.hypervisor.clone(), - vms: h.vms, - }), + agent: health.as_ref().ok().cloned(), agent_error: health.err().map(|e| e.to_string()), live_harvest_wired: false, custom_family_wired: false, diff --git a/crates/gateway-core/src/admin_route.rs b/crates/gateway-core/src/admin_route.rs new file mode 100644 index 000000000..dd54e5ab9 --- /dev/null +++ b/crates/gateway-core/src/admin_route.rs @@ -0,0 +1,172 @@ +//! The one admin route the gateway **forwards**, and the bearer floor it holds. +//! +//! Every `/v1/admin/*` path is master-local: the gateway refuses it with a 403 +//! so an operator surface is never on the public miner path. The operator +//! publish is the exception, because it is the route the install CLI calls +//! from wherever the operator runs it, and the alternative was an ephemeral +//! rewrite proxy in front of the gateway — a hop that had to be stood up, kept +//! alive, and trusted, to reach one route. +//! +//! The rule lives here rather than in `gateway::proxy` for the reason the rest +//! of this crate does: `proxy.rs` is at the repository's per-crate LOC cap, and +//! a rule this security-relevant is better off in a module whose whole subject +//! is that rule — with its own tests — than squeezed into a proxy that also +//! handles round-robin, ejection, and viewer lockdown. +//! +//! # What still holds +//! +//! Forwarding one route does **not** open the admin surface: +//! +//! - the challenge id must be the **Proof challenge**, so +//! `/challenge/{topic_id}/v1/admin/…` keeps its 403 (a topic's own routes +//! are never a way in); +//! - the path must be the publish route **exactly**, after the same +//! normalization the 403 gate uses, so `v1/admin/proof/queue/drain` and +//! every future admin route stay master-local until one is named here; +//! - the method is pinned to `POST` (a read of the publish route has no +//! meaning — the document is in `proof_topic_version`); +//! - a bearer must be **present**. The gateway never holds the operator token +//! and must not learn it: the challenge compares the hash. This is the cheap +//! floor that keeps an anonymous `POST` from even costing a hop. + +use axum::http::{header, HeaderMap, Method}; + +/// The challenge whose topics publish their own routes, and whose operator +/// surface carries the one forwarded route. +pub const PROOF_CHALLENGE_ID: &str = "proof"; + +/// The publish route, relative to the challenge (no leading slash): the path +/// `proof_topic_bundle::PUBLISH_PATH` carries after the challenge prefix. +pub const PUBLISH_ADMIN_PATH: &str = "v1/admin/proof/topics"; + +/// Whether `path` is an admin path, after normalization. +/// +/// Match after path normalization: raw `v1/./admin/…` must not bypass the gate +/// when the HTTP client collapses `.` before dialing the challenge upstream. +#[must_use] +pub fn is_admin_path(path: &str) -> bool { + let n = crate::proxy_detach::normalize_proxy_path(path); + n.starts_with("v1/admin/") || n == "v1/admin" +} + +/// The **one** admin route the gateway forwards: the operator publish. +/// +/// See the module docs for the four gates that still hold. +#[must_use] +pub fn is_forwardable_admin_route(method: &Method, challenge_id: &str, rest: &str) -> bool { + *method == Method::POST + && challenge_id == PROOF_CHALLENGE_ID + && crate::proxy_detach::normalize_proxy_path(rest) == PUBLISH_ADMIN_PATH +} + +/// Whether the request carries an operator bearer at all. +/// +/// Presence only: the gateway does not hold the operator token and must not +/// learn it. The challenge compares the hash; this is the cheap floor that +/// keeps an anonymous `POST` from reaching the admin route. +#[must_use] +pub fn has_operator_bearer(headers: &HeaderMap) -> bool { + headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|raw| raw.strip_prefix("Bearer ").or(Some(raw))) + .is_some_and(|token| !token.trim().is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn admin_paths_are_recognized_after_normalization() { + assert!(is_admin_path("v1/admin/proof/topics")); + assert!(is_admin_path("v1/admin")); + assert!(is_admin_path("v1/./admin/rounds/1/winners")); + assert!(is_admin_path("v1//admin/rounds/1/candidates")); + assert!(is_admin_path("./v1/admin/rounds/1/winners")); + assert!(is_admin_path("v1/admin/../admin/rounds/1/winners")); + assert!(is_admin_path("foo/../v1/admin/rounds/1/winners")); + assert!(!is_admin_path("v1/./harness")); + assert!(!is_admin_path("v1/not-admin/rounds/1/winners")); + } + + /// Exactly one admin route is forwarded, and only for the Proof + /// challenge: the operator publish the install CLI calls. + #[test] + fn only_the_operator_publish_route_is_forwarded() { + assert!(is_forwardable_admin_route( + &Method::POST, + "proof", + "v1/admin/proof/topics" + )); + // Normalized the same way the 403 gate is, so a client that collapses + // `.` cannot slip a different route past the allowlist. + assert!(is_forwardable_admin_route( + &Method::POST, + "proof", + "v1/./admin/proof/topics" + )); + assert!(is_forwardable_admin_route( + &Method::POST, + "proof", + "v1/admin/../admin/proof/topics" + )); + + // A topic id is never a way to reach the admin surface. + assert!(!is_forwardable_admin_route( + &Method::POST, + "tb4", + "v1/admin/proof/topics" + )); + // Another challenge's admin surface is not this route. + assert!(!is_forwardable_admin_route( + &Method::POST, + "bounty", + "v1/admin/proof/topics" + )); + // Every other admin route stays master-local. + for rest in [ + "v1/admin", + "v1/admin/proof/executor", + "v1/admin/proof/queue/drain", + "v1/admin/proof/vm-orchestrator", + "v1/admin/proof/submissions/pf/score", + "v1/admin/proof/topics/extra", + ] { + assert!( + !is_forwardable_admin_route(&Method::POST, "proof", rest), + "{rest:?} must stay master-local" + ); + } + // The publish route is a POST; a read of it is not forwarded either. + assert!(!is_forwardable_admin_route( + &Method::GET, + "proof", + "v1/admin/proof/topics" + )); + } + + /// The forwarded route still needs a bearer: the gateway never holds the + /// operator token, it only refuses an anonymous call before the hop. + #[test] + fn the_operator_bearer_floor_is_presence_only() { + use axum::http::HeaderValue; + let mut headers = HeaderMap::new(); + assert!(!has_operator_bearer(&headers)); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("")); + assert!(!has_operator_bearer(&headers)); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer ")); + assert!(!has_operator_bearer(&headers)); + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static("Bearer operator-token"), + ); + assert!(has_operator_bearer(&headers)); + // A raw token (no scheme) is what `admin_ok` accepts too. + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static("operator-token"), + ); + assert!(has_operator_bearer(&headers)); + } +} diff --git a/crates/gateway-core/src/lib.rs b/crates/gateway-core/src/lib.rs index 27319e4b1..dc593ddbe 100644 --- a/crates/gateway-core/src/lib.rs +++ b/crates/gateway-core/src/lib.rs @@ -3,9 +3,13 @@ //! callers should keep importing `gateway::*`. //! //! - [`admin_auth`]: Bearer gate for `/v1/admin/*`. +//! - [`admin_route`]: the one admin route the gateway forwards (the operator +//! publish) plus its bearer floor; every other admin path stays master-local. //! - [`admin_attest`]: master-only owner credit for non-TEE runtimes. //! - [`weights_store`]: raw-weight leaf row + in-memory store + ingress errors. //! - [`proxy_detach`]: Proof-only disconnect-survive hop + path normalize. +//! - [`proxy_paths`]: the read gates the proxy applies before dialing — +//! operator-local report bodies, and the miner-controlled viewer paths. //! - [`topic_routes`]: the `/challenge/{topic_id}/…` rule — a topic id the //! registry does not know is forwarded to the Proof challenge, which //! resolves it against `proof_topic_api`. @@ -14,6 +18,8 @@ pub mod admin_attest; pub mod admin_auth; +pub mod admin_route; pub mod proxy_detach; +pub mod proxy_paths; pub mod topic_routes; pub mod weights_store; diff --git a/crates/gateway-core/src/proxy_paths.rs b/crates/gateway-core/src/proxy_paths.rs new file mode 100644 index 000000000..c43cfbeae --- /dev/null +++ b/crates/gateway-core/src/proxy_paths.rs @@ -0,0 +1,72 @@ +//! Path predicates the proxy decides on before it dials an upstream. +//! +//! These are the gateway's **read** gates: which paths are operator-local +//! (report bodies, and the admin surface in [`crate::admin_route`]), and +//! which carry miner-controlled HTML the gateway must lock down before it +//! forwards a response. +//! +//! They live here rather than in `gateway::proxy` because that file is at the +//! repository's per-crate LOC cap, and because every one of them is a +//! decision made **before** a request leaves the host — the kind of rule that +//! deserves its own tests rather than being folded in with round-robin and +//! ejection. + +use axum::http::Method; + +use crate::proxy_detach::normalize_proxy_path; + +/// Report bodies are operator-local. POST submit stays on the miner path. +/// +/// HEAD is a read: Axum would otherwise map it onto the GET handler and leak +/// status/headers (and whether a row exists) through the public gateway. +#[must_use] +pub fn is_blocked_report_read(method: &Method, rest: &str) -> bool { + let n = normalize_proxy_path(rest); + *method != Method::POST && (n == "v1/reports" || n.starts_with("v1/reports/")) +} + +/// Miner-controlled viewer paths (`/challenge/{id}/v1/view/{run}/{page}`). +#[must_use] +pub fn is_view_path(rest: &str) -> bool { + normalize_proxy_path(rest).starts_with("v1/view/") +} + +/// Captured PNG screenshot under `/v1/view/{run}/{page}.png`. +#[must_use] +pub fn is_view_png_path(path: &str) -> bool { + is_view_path(path) + && std::path::Path::new(path.trim_start_matches('/')) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("png")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn report_reads_are_blocked_but_submit_is_not() { + assert!(is_blocked_report_read(&Method::GET, "v1/reports")); + assert!(is_blocked_report_read(&Method::HEAD, "v1/reports")); + assert!(is_blocked_report_read(&Method::HEAD, "v1/reports/by_1")); + assert!(is_blocked_report_read(&Method::GET, "v1/reports/by_1")); + assert!(is_blocked_report_read(&Method::GET, "v1/./reports/by_1")); + assert!(is_blocked_report_read(&Method::OPTIONS, "v1/reports")); + assert!(!is_blocked_report_read(&Method::POST, "v1/reports")); + assert!(!is_blocked_report_read(&Method::GET, "v1/status")); + assert!(!is_blocked_report_read(&Method::HEAD, "v1/status")); + assert!(!is_blocked_report_read(&Method::GET, "v1/pair")); + } + + #[test] + fn view_paths_detected() { + assert!(is_view_path("v1/view/abc/index.html")); + assert!(is_view_path("/v1/view/abc/pricing.html")); + assert!(is_view_path("v1/./view/abc/index.html")); + assert!(!is_view_path("v1/runs/abc")); + assert!(!is_view_path("v1/viewx/abc")); + assert!(!is_view_path("v1/admin/view")); + assert!(is_view_png_path("v1/view/abc/index.png")); + assert!(!is_view_png_path("v1/view/abc/index.html")); + } +} diff --git a/crates/gateway-core/src/topic_routes.rs b/crates/gateway-core/src/topic_routes.rs index de7c7e04d..590559526 100644 --- a/crates/gateway-core/src/topic_routes.rs +++ b/crates/gateway-core/src/topic_routes.rs @@ -137,4 +137,21 @@ mod tests { assert_eq!(topic_route_path("tb4", ""), "challenge/tb4"); assert_eq!(topic_route_path("tb4", "/"), "challenge/tb4"); } + + /// The routing rule is **topic-agnostic**: no topic id is compiled in, so + /// a new topic needs no code change here. The names in this file's tests + /// are fixtures; the source above them must not mention any. + #[test] + fn no_topic_id_is_compiled_into_the_router() { + let src = include_str!("topic_routes.rs"); + let non_test = src.split("#[cfg(test)]").next().unwrap_or(""); + let lower = non_test.to_ascii_lowercase(); + for forbidden in ["tb4", "tbench", "terminal-bench", "harbor"] { + assert!( + !lower.contains(forbidden), + "{forbidden:?} is compiled into the gateway router: the challenge a topic is \ + routed to is `PROOF_CHALLENGE_ID`, and which topics exist is a database fact" + ); + } + } } diff --git a/crates/gateway/src/proxy.rs b/crates/gateway/src/proxy.rs index 3c85a28ea..51b79e636 100644 --- a/crates/gateway/src/proxy.rs +++ b/crates/gateway/src/proxy.rs @@ -182,11 +182,8 @@ async fn proxy_inner( /// The admin gate: `None` when the request may proceed, a refusal otherwise. /// -/// Every `/v1/admin/*` path is master-local except the one operator route -/// [`is_forwardable_admin_route`] names (the publish the install CLI calls), -/// which additionally needs a bearer **at the gateway**: the challenge -/// compares the token hash, but an anonymous `POST` should not even cost a -/// hop. A topic id never reaches the admin surface either way. +/// The rule itself lives in [`gateway_core::admin_route`] (which owns its +/// tests); this is only the HTTP shape of the refusal. fn admin_gate( method: &Method, challenge_id: &str, @@ -297,89 +294,18 @@ async fn forward( ForwardResult::Ok(response) } -/// Collapse `.` / empty / `..` segments the same way `url`/`reqwest` will before -/// the upstream request — used so gateway gates cannot be skipped via `v1/./admin`. -pub use gateway_core::proxy_detach::normalize_proxy_path; +/// Operator admin surfaces are master-local only, with **one** forwarded +/// exception (the operator publish): the rule, its four gates, and its tests +/// live in [`gateway_core::admin_route`]. Re-exported so callers of this crate +/// keep one import. +pub use gateway_core::admin_route::{ + has_operator_bearer, is_admin_path, is_forwardable_admin_route, +}; -/// Operator admin surfaces are master-local only (not on the public miner path). -/// -/// Match after path normalization: raw `v1/./admin/…` must not bypass the gate -/// when the HTTP client collapses `.` before dialing the challenge upstream. -#[must_use] -pub fn is_admin_path(rest: &str) -> bool { - let n = normalize_proxy_path(rest); - n.starts_with("v1/admin/") || n == "v1/admin" -} - -/// The **one** admin route the gateway forwards: the operator publish. -/// -/// `proof-admin topic install` publishes a signed document with -/// `POST /challenge/proof/v1/admin/proof/topics` (`proof_topic_bundle:: -/// PUBLISH_PATH`). The route is operator-authenticated at the challenge -/// (`admin_hashes`, the same bearer the master-local routes use), so the -/// gateway forwards it instead of refusing it — otherwise an operator on -/// staging needs a rewrite proxy in front of the gateway, which is exactly -/// the hop this replaces. Two gates still hold here: -/// -/// - the challenge id must be the **Proof challenge**, not a topic id, so -/// `/challenge/{topic_id}/v1/admin/…` keeps its 403 (a topic's own routes -/// are never a way to reach the admin surface); and -/// - the path must be the publish route **exactly**, after the same -/// normalization [`is_admin_path`] uses, so `v1/admin/proof/queue/drain` -/// and every future admin route stay master-local until one is named here. -/// -/// The method is pinned to `POST`: a read of the publish route has no -/// meaning (the document is in `proof_topic_version`), and a `GET` stays a -/// 403 like the rest of the admin surface. -#[must_use] -pub fn is_forwardable_admin_route(method: &Method, challenge_id: &str, rest: &str) -> bool { - *method == Method::POST - && challenge_id == gateway_core::topic_routes::PROOF_CHALLENGE_ID - && normalize_proxy_path(rest) == PUBLISH_ADMIN_PATH -} - -/// The publish route, relative to the challenge (no leading slash): the path -/// `proof_topic_bundle::PUBLISH_PATH` carries after the challenge prefix. -pub const PUBLISH_ADMIN_PATH: &str = "v1/admin/proof/topics"; - -/// Whether the request carries an operator bearer at all. -/// -/// Presence only: the gateway does not hold the operator token and must not -/// learn it. The challenge compares the hash; this is the cheap floor that -/// keeps an anonymous `POST` from reaching the admin route. -#[must_use] -pub fn has_operator_bearer(headers: &HeaderMap) -> bool { - headers - .get(header::AUTHORIZATION) - .and_then(|v| v.to_str().ok()) - .and_then(|raw| raw.strip_prefix("Bearer ").or(Some(raw))) - .is_some_and(|token| !token.trim().is_empty()) -} - -/// Report bodies are operator-local. POST submit stays on the miner path. -/// -/// HEAD is a read: Axum would otherwise map it onto the GET handler and leak -/// status/headers (and whether a row exists) through the public gateway. -#[must_use] -pub fn is_blocked_report_read(method: &Method, rest: &str) -> bool { - let n = normalize_proxy_path(rest); - *method != Method::POST && (n == "v1/reports" || n.starts_with("v1/reports/")) -} - -/// Miner-controlled viewer paths (`/challenge/{id}/v1/view/{run}/{page}`). -#[must_use] -pub fn is_view_path(rest: &str) -> bool { - normalize_proxy_path(rest).starts_with("v1/view/") -} - -/// Captured PNG screenshot under `/v1/view/{run}/{page}.png`. -#[must_use] -pub fn is_view_png_path(path: &str) -> bool { - is_view_path(path) - && std::path::Path::new(path.trim_start_matches('/')) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("png")) -} +/// The proxy's read gates — operator-local report bodies, and the +/// miner-controlled viewer paths — live in [`gateway_core::proxy_paths`] with +/// their tests. Re-exported so callers of this crate keep one import. +pub use gateway_core::proxy_paths::{is_blocked_report_read, is_view_path, is_view_png_path}; /// Re-apply the viewer header floor at the last serving layer (defense in /// depth). Non-PNG paths get the full HTML lockdown (CSP `sandbox`, CORP @@ -443,131 +369,31 @@ mod tests { )); } + /// The admin predicates moved to `gateway-core::admin_route` with their + /// tests (the module owns the rule). This pins that the re-export is the + /// same rule the gate calls. #[test] - fn admin_paths_blocked_from_gateway() { + fn the_admin_rule_is_re_exported_not_re_implemented() { assert!(is_admin_path("v1/admin/rounds/1/winners")); - assert!(is_admin_path("/v1/admin/rounds/1/candidates")); - assert!(!is_admin_path("v1/harness")); - assert!(!is_admin_path("v1/runs/abc")); - } - - #[test] - fn report_reads_are_blocked_from_gateway_but_submit_is_not() { - assert!(is_blocked_report_read(&Method::GET, "v1/reports")); - assert!(is_blocked_report_read(&Method::HEAD, "v1/reports")); - assert!(is_blocked_report_read(&Method::HEAD, "v1/reports/by_1")); - assert!(is_blocked_report_read(&Method::GET, "v1/reports/by_1")); - assert!(is_blocked_report_read(&Method::GET, "v1/./reports/by_1")); - assert!(is_blocked_report_read(&Method::OPTIONS, "v1/reports")); - assert!(!is_blocked_report_read(&Method::POST, "v1/reports")); - assert!(!is_blocked_report_read(&Method::GET, "v1/status")); - assert!(!is_blocked_report_read(&Method::HEAD, "v1/status")); - assert!(!is_blocked_report_read(&Method::GET, "v1/pair")); - } - - #[test] - fn admin_paths_blocked_despite_dot_segment_confusion() { - // axum preserves `./` in `{*rest}`; reqwest then collapses to /v1/admin/… assert!(is_admin_path("v1/./admin/rounds/1/winners")); - assert!(is_admin_path("v1//admin/rounds/1/candidates")); - assert!(is_admin_path("./v1/admin/rounds/1/winners")); - assert!(is_admin_path("v1/admin/../admin/rounds/1/winners")); - assert!(is_admin_path("foo/../v1/admin/rounds/1/winners")); - assert!(!is_admin_path("v1/./harness")); - assert!(!is_admin_path("v1/not-admin/rounds/1/winners")); - } - - /// Exactly one admin route is forwarded, and only for the Proof - /// challenge: the operator publish the install CLI calls. - #[test] - fn only_the_operator_publish_route_is_forwarded() { - // The CLI's path, as the gateway sees it (the challenge id is - // stripped, so this is the challenge-relative form). - assert!(is_forwardable_admin_route( - &Method::POST, - "proof", - "v1/admin/proof/topics" - )); - // Normalized the same way the 403 gate is, so a client that collapses - // `.` cannot slip a different route past the allowlist. - assert!(is_forwardable_admin_route( - &Method::POST, - "proof", - "v1/./admin/proof/topics" - )); + assert!(!is_admin_path("v1/harness")); assert!(is_forwardable_admin_route( &Method::POST, "proof", - "v1/admin/../admin/proof/topics" - )); - - // A topic id is never a way to reach the admin surface. - assert!(!is_forwardable_admin_route( - &Method::POST, - "tb4", - "v1/admin/proof/topics" - )); - // Another challenge's admin surface is not this route. - assert!(!is_forwardable_admin_route( - &Method::POST, - "bounty", "v1/admin/proof/topics" )); - // Every other admin route stays master-local. - for rest in [ - "v1/admin", - "v1/admin/proof/executor", - "v1/admin/proof/queue/drain", - "v1/admin/proof/vm-orchestrator", - "v1/admin/proof/submissions/pf/score", - "v1/admin/proof/topics/extra", - ] { - assert!( - !is_forwardable_admin_route(&Method::POST, "proof", rest), - "{rest:?} must stay master-local" - ); - } - // The publish route is a POST; a read of it is not forwarded either. assert!(!is_forwardable_admin_route( &Method::GET, "proof", "v1/admin/proof/topics" )); - } - - /// The forwarded route still needs a bearer: the gateway never holds the - /// operator token, it only refuses an anonymous call before the hop. - #[test] - fn the_operator_bearer_floor_is_presence_only() { let mut headers = HeaderMap::new(); assert!(!has_operator_bearer(&headers)); - headers.insert(header::AUTHORIZATION, HeaderValue::from_static("")); - assert!(!has_operator_bearer(&headers)); - headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer ")); - assert!(!has_operator_bearer(&headers)); headers.insert( header::AUTHORIZATION, HeaderValue::from_static("Bearer operator-token"), ); assert!(has_operator_bearer(&headers)); - // A raw token (no scheme) is what `admin_ok` accepts too. - headers.insert( - header::AUTHORIZATION, - HeaderValue::from_static("operator-token"), - ); - assert!(has_operator_bearer(&headers)); - } - - #[test] - fn view_paths_detected() { - assert!(is_view_path("v1/view/abc/index.html")); - assert!(is_view_path("/v1/view/abc/pricing.html")); - assert!(is_view_path("v1/./view/abc/index.html")); - assert!(!is_view_path("v1/runs/abc")); - assert!(!is_view_path("v1/viewx/abc")); - assert!(!is_view_path("v1/admin/view")); - assert!(is_view_png_path("v1/view/abc/index.png")); - assert!(!is_view_png_path("v1/view/abc/index.html")); } #[test] diff --git a/crates/proof-challenge/src/lib.rs b/crates/proof-challenge/src/lib.rs index e11b90640..027ed984a 100644 --- a/crates/proof-challenge/src/lib.rs +++ b/crates/proof-challenge/src/lib.rs @@ -30,7 +30,7 @@ pub use proof_executor::{ }; pub use proof_http::{ executor_slot, hash_admin_token, proof_router, AppState, ExecutorSlot, InstallJournal, - InstallJournalSlot, VmAgentHealth, VmOrchestratorProbe, VmOrchestratorReport, + InstallJournalSlot, VmOrchestratorProbe, VmOrchestratorReport, }; pub use proof_store::{ ArtefactVault, ArtifactManifest, MemoryStore, MinerEnvVault, ARTEFACT_STAGING_DIR_ENV, diff --git a/crates/proof-http/src/lib.rs b/crates/proof-http/src/lib.rs index 637dcedbe..20962113a 100644 --- a/crates/proof-http/src/lib.rs +++ b/crates/proof-http/src/lib.rs @@ -45,20 +45,25 @@ clippy::too_many_arguments )] -use std::collections::BTreeMap; use std::future::Future; use std::sync::{Arc, PoisonError, RwLock}; -use async_trait::async_trait; use axum::extract::{DefaultBodyLimit, Path, Query, Request, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; use axum::routing::{get, post}; use axum::{Json, Router}; +mod operator; mod submit; -use proof_canon::MinerEnv; +pub use operator::{ + InstallJournal, InstallJournalSlot, SubmitGate, VmAgentHealth, VmOrchestratorProbe, + VmOrchestratorReport, +}; + +use operator::{disabled_gate, disabled_topics, install_gate}; + use proof_eval::{ contamination_evidence, custom_ids_ref, eval_after_freeze, force_sim, registered_custom, scoring_readiness, secret_backed_base_url, EvalBackend, EvalError, LiveScorer, @@ -69,13 +74,10 @@ use proof_score::{ MinerTopicRun, ProofKind, ProofVerdict, }; use proof_store::{ - freeze_submission_digest, is_staged_artefact_uri, staged_artefact_uri, ArtifactManifest, - Enqueued, LiveEval, MemoryStore, StoreError, Submission, SubmissionState, MAX_ARTEFACT_BYTES, -}; -use proof_submit::{ - is_lowercase_hex, parse_hotkey_hex, parse_signature_hex, parse_submit_nonce_hex, verify_submit, - SubmitFields, + freeze_submission_digest, is_staged_artefact_uri, staged_artefact_uri, Enqueued, LiveEval, + MemoryStore, StoreError, Submission, SubmissionState, MAX_ARTEFACT_BYTES, }; +use proof_submit::{is_digest_of_nothing, nonce_from, SubmitSigError}; use proof_task::{ resolve_inference, InferenceOffer, MetricFamily, OfferError, ProofPin, TopicDocument, TopicError, TopicStatus, CHALLENGE_ID, SCORE_MAX, SCORING_VERSION, @@ -93,166 +95,6 @@ pub fn executor_slot(offer: Option) -> ExecutorSlot { Arc::new(RwLock::new(offer)) } -/// The KVM-host agent's health as the control plane saw it on one -/// `GET /v1/health` (mirrors `proof_vm_proto::AgentHealth` field for field). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct VmAgentHealth { - /// Wire version the agent speaks. - pub api_version: u32, - /// Whether the hypervisor could boot a VM right now. - pub ready: bool, - /// Why not (empty when ready). Never a secret. - pub reason: String, - /// Backend name (`firecracker`; `fake` only in tests). - pub hypervisor: String, - /// VMs currently bound on the host. - pub vms: usize, -} - -/// `GET /v1/admin/proof/vm-orchestrator` body: what this host resolved for -/// the topic-VM orchestrator and whether its agent answers. Operator data -/// behind the admin bearer — it may name env vars and container paths, never -/// the bearer, a key, or an origin the RLM could reach. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct VmOrchestratorReport { - /// `firecracker` (live client resolved at boot) or `unwired`. - pub orchestrator: String, - /// The client's own `ready()`: bearer file present and non-empty, RLM - /// image digest pinned. Checked per request, so a fix needs no restart. - pub ready: bool, - /// Why not ready (empty when ready). Names the env var to fix. - pub reason: String, - /// `sha256:` pin of the RLM VM image the client asks the agent to boot - /// (empty = unpinned = nothing ever boots). - pub image_digest: String, - /// RLM VM vCPUs (locked default 4). - pub vcpus: u32, - /// RLM VM memory in MiB (locked default 8192). - pub mem_mib: u32, - /// The agent's answer to one health call, when it answered. - pub agent: Option, - /// Why the agent did not answer: unreachable, bearer refused, not wired. - pub agent_error: Option, - /// Filled by the host: the digest-pinned Lium harvest (`nll` / - /// `throughput`) is wired. Lium only — informational for the custom - /// family, which is wired from the topic-VM env on its own. - #[serde(default)] - pub live_harvest_wired: bool, - /// Filled by the host: at least one custom id has a registered runner - /// (the custom family is routed, harvest or not). - #[serde(default)] - pub custom_family_wired: bool, - /// Filled by the host: custom ids with a registered runner. - #[serde(default)] - pub registered_custom: Vec, -} - -impl VmOrchestratorReport { - /// Report for a host that keeps `UnwiredVmOrchestrator`; `reason` names - /// the env vars a live one reads. `image_digest` is whatever pin the env - /// carries so "pinned but URL unset" is visible. - pub fn unwired(reason: &str, image_digest: &str) -> Self { - Self { - orchestrator: "unwired".into(), - ready: false, - reason: reason.trim().to_owned(), - image_digest: image_digest.trim().to_owned(), - vcpus: 0, - mem_mib: 0, - agent: None, - agent_error: None, - live_harvest_wired: false, - custom_family_wired: false, - registered_custom: Vec::new(), - } - } -} - -/// Operator diagnostic over the topic-VM orchestrator this host resolved at -/// boot. The binary implements it over the live `FirecrackerOrchestrator` -/// (its `ready()` plus one agent health call) or the unwired stand-in; the -/// route only adds what the host knows (harvest wired, registered ids). It -/// changes nothing and spends nothing. -#[async_trait] -pub trait VmOrchestratorProbe: Send + Sync { - /// Snapshot as of now (bearer file and pin re-read; one agent round trip). - async fn probe(&self) -> VmOrchestratorReport; -} - -/// What the **submit** path needs from the journal, in one read. -/// -/// Both fields are about the topic's install and the operator's switch, and -/// both are read before anything is spent, so they are one call: a submission -/// costs one gate read, not two. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct SubmitGate { - /// The operator's reason, when the topic is disabled (`topic disable`). - pub disabled_reason: Option, - /// The `vms_per_submission` the topic's newest install recorded. - /// - /// `None` when the topic has no install row or the row predates the - /// field. `Some(n)` with `n != 1` is refused on the submit path: this - /// build runs **one VM per submission**, and a topic installed with a - /// different pin is one it cannot honour. - pub vms_per_submission: Option, -} - -impl SubmitGate { - /// The operator gate, as the submit path reads it. - #[must_use] - pub fn disabled(reason: impl Into) -> Self { - Self { - disabled_reason: Some(reason.into()), - ..Self::default() - } - } -} - -/// Whether a topic's install reached `applied`, as the publish route reads it. -/// -/// A trait rather than a pool so the route can be exercised without a -/// database, and so a host that resolved no install journal can say so instead -/// of answering from a table it never read. -#[async_trait] -pub trait InstallJournal: Send + Sync { - /// `Ok(true)` when the newest install row for `topic_id` is `applied`. - /// - /// # Errors - /// - /// The reason the journal could not be read. The caller refuses the - /// publish: an unreadable journal is not an installed topic. - async fn applied(&self, topic_id: &str) -> Result; - - /// The operator gate and the allocator pin for one topic, for the submit - /// path. - /// - /// # Errors - /// - /// The reason the gate could not be read. The caller answers **503**: an - /// unreadable gate is not an enabled topic. - async fn submit_gate(&self, topic_id: &str) -> Result; - - /// Every topic currently disabled, with the operator's reason. - /// - /// One read for the public listing, which annotates each topic with the - /// flag rather than paying a query per topic. The submit path uses - /// [`Self::submit_gate`] for the single topic it is admitting. - /// - /// # Errors - /// - /// The reason the gate could not be read. The caller answers **503**. - async fn disabled_topics(&self) -> Result, String>; -} - -/// The journal read behind the publish gate, or `None` on a host that -/// resolved none (no database). -/// -/// `None` is **fail-closed**: [`install_applied`] refuses, so an `open` -/// document cannot be published on a host that cannot prove the install ran. -/// That is the same rule the gate enforces when the journal is unreadable — -/// the only difference is which sentence the operator reads. -pub type InstallJournalSlot = Option>; - /// Shared HTTP state. #[derive(Clone)] pub struct AppState { @@ -527,26 +369,6 @@ fn annotate_gate(doc: &TopicDocument, disabled_reason: Option<&str>) -> serde_js value } -/// The disabled set the listing annotates from, or a 503. -async fn disabled_topics( - st: &AppState, -) -> Result, (StatusCode, Json)> { - let Some(journal) = st.install_journal.as_deref() else { - // No gate on this host: no database, so no operator switch was ever - // thrown (and no topic was published either). - return Ok(BTreeMap::new()); - }; - journal.disabled_topics().await.map_err(|e| { - err( - StatusCode::SERVICE_UNAVAILABLE, - &format!( - "the topic gate could not be read: {e}. The listing is refused rather than \ - served without the operator's switch; fix the database and re-read." - ), - ) - }) -} - /// Public executor contract: the live offer (every field is public), whether /// it can rent right now, and the pin ceilings it is bound by. Always 200 — /// a missing offer is `eval_executor: null` with the reason, never a 404. @@ -561,39 +383,6 @@ async fn get_executor(State(st): State) -> impl IntoResponse { })) } -#[derive(Debug, Deserialize)] -struct SubmitBody { - miner_hotkey: String, - /// sr25519 signature over [`SubmitFields::signing_payload`] (128 lowercase hex). - #[serde(default)] - hotkey_signature: Option, - /// Client anti-replay nonce (64 lowercase hex), bound into the signature - /// and accepted once per hotkey. - #[serde(default)] - submit_nonce: Option, - artifact_digest: String, - artifact_uri: Option, - #[serde(default)] - claim: String, - #[serde(default)] - declared_flops: u64, - #[serde(default)] - topic_id: String, - /// Optional miner label. Not compared to an HF id (that check is retired). - #[serde(default)] - architecture: String, - #[serde(default)] - manifest: ArtifactManifest, - /// Miner BYOK: `{"": ""}` for the variables this topic's - /// signed document declares (`constraints.params.miner_byok` / - /// `miner_env_allowlist`). Never signed (v1 of the submit payload is - /// unchanged), never persisted on the row, never echoed back. A name the - /// topic does not declare is a **400** before the signature is checked, - /// so a mistake here never burns the single-use `submit_nonce`. - #[serde(default)] - env: MinerEnv, -} - /// What a submit — or a drain of one row — answers. #[derive(Debug, Clone, Serialize)] pub struct SubmitResp { @@ -642,91 +431,6 @@ type ErrResp = (StatusCode, Json); /// signed form: the host never normalises a hex field before verifying it, /// so a value a miner did not sign byte for byte is refused as their /// request, not silently rewritten into a signature mismatch. -fn parse_hex64(s: &str, field: &str) -> Result)> { - if !is_lowercase_hex(s, 64) { - return Err(err( - StatusCode::BAD_REQUEST, - &format!("invalid {field}: exactly 64 lowercase hex, no 0x"), - )); - } - Ok(s.to_owned()) -} - -/// A digest of **nothing** is not an artefact digest: the sha256 of zero -/// bytes and of an empty tar archive (`tar cf - -T /dev/null`, 10240 zero -/// bytes). Staging's happy path once matched exactly such a digest because -/// the RLM guest could not fetch the artefact and fell back to an empty -/// tree; refusing it at submit keeps that stub out of every row and rent. -fn is_digest_of_nothing(hex64: &str) -> bool { - let empty_input = hex::encode(Sha256::digest(b"")); - let empty_tar = hex::encode(Sha256::digest([0u8; 10_240])); - hex64.eq_ignore_ascii_case(&empty_input) || hex64.eq_ignore_ascii_case(&empty_tar) -} - -fn parse_artifact_digest(s: &str) -> Result)> { - // Named before the encoding check so a pasted empty-tar digest gets the - // useful answer whatever its case. - if is_digest_of_nothing(&proof_submit::canonical_hex(s)) { - return Err(err( - StatusCode::BAD_REQUEST, - "artifact_digest is the sha256 of empty input (or of an empty tar archive): hash the recipe bytes you upload (or ship at artifact_uri)", - )); - } - parse_hex64(s, "artifact_digest") -} - -/// Miner identity for one submit: `hotkey_signature` over every gate input -/// (topic, artefact, FLOPs, claim, manifest) plus the client `submit_nonce`. -/// Verified over the strings exactly as posted (`hotkey` and `artifact` are -/// already in their only accepted form). Returns the nonce the caller must -/// reserve before any row or rent. -fn authenticate_submit( - hotkey: &str, - topic_id: &str, - artifact: &str, - body: &SubmitBody, -) -> Result)> { - let sig_hex = body.hotkey_signature.as_deref().filter(|s| !s.is_empty()); - let Some(sig_hex) = sig_hex else { - return Err(err(StatusCode::UNAUTHORIZED, "hotkey_signature required")); - }; - let sig = parse_signature_hex(sig_hex) - .map_err(|_| err(StatusCode::UNAUTHORIZED, "hotkey_signature invalid"))?; - let nonce = body.submit_nonce.as_deref().filter(|s| !s.is_empty()); - let Some(nonce) = nonce else { - return Err(err(StatusCode::UNAUTHORIZED, "submit_nonce required")); - }; - parse_submit_nonce_hex(nonce) - .map_err(|_| err(StatusCode::UNAUTHORIZED, "submit_nonce invalid"))?; - let pk = parse_hotkey_hex(hotkey) - .map_err(|_| err(StatusCode::BAD_REQUEST, "invalid miner_hotkey"))?; - verify_submit( - &pk, - &SubmitFields { - hotkey_hex: hotkey, - topic_id, - artifact_digest: artifact, - declared_flops: body.declared_flops, - claim: &body.claim, - train_content_hashes: &body.manifest.train_content_hashes, - train_dataset_ids: &body.manifest.train_dataset_ids, - submit_nonce_hex: nonce, - }, - &sig, - ) - .map_err(|_| err(StatusCode::UNAUTHORIZED, "hotkey_signature invalid"))?; - Ok(nonce.to_owned()) -} - -fn nonce_from(hotkey: &str, topic_id: &str, digest: &str) -> String { - let mut h = Sha256::new(); - h.update(b"proof-nonce-v1"); - h.update(hotkey.as_bytes()); - h.update(topic_id.as_bytes()); - h.update(digest.as_bytes()); - hex::encode(h.finalize()) -} - async fn submit( State(st): State, headers: HeaderMap, @@ -780,7 +484,9 @@ async fn submit( if topic.metric.family == MetricFamily::Custom && uploaded.is_none() && miner_uri.is_none() { return Err(err(StatusCode::BAD_REQUEST, "artifact required")); } - let submit_nonce = authenticate_submit(&hotkey, &body.topic_id, &artifact, &body)?; + let submit_nonce = body + .authenticate(&hotkey, &artifact) + .map_err(|e| submit_sig_err(&e))?; // A verified request is single-use, whatever happens to it next: a // replay must never reach evaluation or a second row. if !st @@ -1772,106 +1478,42 @@ fn err(code: StatusCode, msg: &str) -> (StatusCode, Json) { (code, Json(serde_json::json!({ "error": msg }))) } -/// Whether the topic's install reached `applied`, as the publish gate reads it. -/// -/// **Fail-closed on every doubt**: no journal slot, an unreadable journal, and -/// a topic with no install row all refuse, so an `open` document is never -/// admitted on a fact the host cannot prove. -/// -/// The refusal says **which** of those it was, because the operator has to -/// tell "not installed yet" from "the journal could not be read" — one is a -/// step to finish, the other is a host to fix. It is returned rather than -/// logged: `proof-http` has no logging dependency, and this reason belongs in -/// the response the operator is already reading. -/// -/// # Errors -/// -/// The reason the `open` document cannot be published. -async fn install_gate(st: &AppState, topic_id: &str) -> Result<(), String> { - let Some(journal) = st.install_journal.as_deref() else { - return Err(format!( - "this host resolved no install journal (no database), so it cannot prove that topic \ - {topic_id:?} was installed. Publish the document as `draft`, or wire \ - BASE_DATABASE_URL and restart." +/// Exactly 64 lowercase hex, no `0x`, no whitespace. The wire form **is** the +/// signed form: the host never normalises a hex field before verifying it, so +/// a value a miner did not sign byte for byte is refused as their request, +/// not silently rewritten into a signature mismatch. +fn parse_hex64(s: &str, field: &str) -> Result { + if !proof_submit::is_lowercase_hex(s, 64) { + return Err(err( + StatusCode::BAD_REQUEST, + &format!("invalid {field}: exactly 64 lowercase hex, no 0x"), )); - }; - match journal.applied(topic_id).await { - Ok(true) => Ok(()), - Ok(false) => Err(format!( - "topic {topic_id:?} has no `applied` install row: run `proof-admin topic install` to \ - completion first (the journal is `proof_topic_install`; read it with `proof-admin \ - topic install-log --topic {topic_id}`). A `pending` or `failed` row means the \ - migrations, routes, or rules are not in place, and an `open` document is submitable \ - the moment it is published." - )), - Err(e) => Err(format!( - "the install journal could not be read for topic {topic_id:?}: {e}. The publish is \ - refused rather than admitted on an unread fact; fix the database and re-publish." - )), } + Ok(s.to_owned()) } -/// The operator gate on the **submit** path: is this topic disabled, and is -/// its allocator pin one this build runs? +/// The artefact digest, refused when it is the sha256 of **nothing**. /// -/// `Ok(())` admits the submission. A disabled topic is a **403** naming the -/// operator's reason; a topic whose install recorded a -/// `vms_per_submission` other than 1 is a **503** — this build runs exactly -/// one VM per submission, and silently running a different number would make -/// the journal a lie; an unreadable gate is a **503** too — never an -/// admission, and never a 404 that would read as "no such topic". A host that -/// resolved no journal (no database) has no gate to read: it also has no -/// published topics, so the submission is refused by the topic lookup above -/// it. -/// -/// This is deliberately **not cached**: a disable has to take effect on the -/// next request, which is what makes it usable during an incident. -async fn disabled_gate(st: &AppState, topic_id: &str) -> Result<(), ErrResp> { - let Some(journal) = st.install_journal.as_deref() else { - return Ok(()); - }; - let gate = match journal.submit_gate(topic_id).await { - Ok(gate) => gate, - Err(e) => { - return Err(err( - StatusCode::SERVICE_UNAVAILABLE, - &format!( - "the topic gate could not be read for {topic_id:?}: {e}. The submission is \ - refused rather than admitted on an unread fact; fix the database and re-post \ - (the submit_nonce is unspent)." - ), - )) - } - }; - if let Some(reason) = gate.disabled_reason.as_deref() { +/// Named before the encoding check so a pasted empty-tar digest gets the +/// useful answer whatever its case. +fn parse_artifact_digest(s: &str) -> Result { + if is_digest_of_nothing(&proof_submit::canonical_hex(s)) { return Err(err( - StatusCode::FORBIDDEN, - &format!( - "topic {topic_id:?} is disabled by the operator{}", - if reason.trim().is_empty() { - String::new() - } else { - format!(": {}", reason.trim()) - } - ), + StatusCode::BAD_REQUEST, + "artifact_digest is the sha256 of empty input (or of an empty tar archive): hash the recipe bytes you upload (or ship at artifact_uri)", )); } - if let Some(n) = gate.vms_per_submission { - if n != proof_topic_install::VMS_PER_SUBMISSION { - return Err(err( - StatusCode::SERVICE_UNAVAILABLE, - &format!( - "topic {topic_id:?} was installed with vms_per_submission={n}, but this host \ - runs exactly {} VM per submission (the pin this build enforces). Re-install \ - the topic with the pin this host carries, or run the host that matches the \ - install. The submission is refused rather than run under a binding the \ - install did not record; the submit_nonce is unspent.", - proof_topic_install::VMS_PER_SUBMISSION - ), - )); - } - } - Ok(()) + parse_hex64(s, "artifact_digest") +} + +/// A submit-signature refusal, as the miner reads it: 401 for identity, 400 +/// for a hotkey that is not a hotkey. +fn submit_sig_err(e: &SubmitSigError) -> (StatusCode, Json) { + let code = match e { + SubmitSigError::InvalidHotkey => StatusCode::BAD_REQUEST, + _ => StatusCode::UNAUTHORIZED, + }; + err(code, &e.to_string()) } fn store_err(e: &proof_store::StoreError) -> (StatusCode, Json) { @@ -1926,6 +1568,9 @@ mod tests { use tower::ServiceExt; use super::*; + use proof_canon::MinerEnv; + use proof_store::ArtifactManifest; + use std::collections::BTreeMap; fn digest(label: &str) -> String { let mut h = Sha256::new(); @@ -2711,6 +2356,8 @@ mod tests { reason: String::new(), hypervisor: "firecracker".into(), vms: 2, + experiment_vms: 1, + max_experiment_vms: 2, }); // The probe's own view of the host gates is overwritten by the route. wired.live_harvest_wired = false; diff --git a/crates/proof-http/src/operator.rs b/crates/proof-http/src/operator.rs new file mode 100644 index 000000000..6002b058e --- /dev/null +++ b/crates/proof-http/src/operator.rs @@ -0,0 +1,279 @@ +//! The operator-facing surface of the Proof HTTP API: the journal the submit +//! path reads, and the topic-VM orchestrator diagnostic. +//! +//! These are the types a **host** implements (the challenge binary reads the +//! shared database; the VM client answers the orchestrator probe) and the +//! routes consume. They live in their own module rather than in the router +//! file because the router is at the repository's per-crate LOC cap, and +//! because their subject is a boundary: what the control plane is allowed to +//! *ask* about a topic's install and its VM host. + +use std::sync::Arc; + +use axum::http::StatusCode; +use serde::{Deserialize, Serialize}; + +use crate::{err, AppState, ErrResp}; + +/// The KVM-host agent's health, as the agent's own `GET /v1/health` reports it. +/// +/// Re-exported from `proof-vm-proto` rather than mirrored: the route publishes +/// what the agent said, so a second struct here could only drift from the wire +/// type it copies. +pub use proof_vm_proto::AgentHealth as VmAgentHealth; + +/// `GET /v1/admin/proof/vm-orchestrator` body: what this host resolved for +/// the topic-VM orchestrator and whether its agent answers. Operator data +/// behind the admin bearer — it may name env vars and container paths, never +/// the bearer, a key, or an origin the RLM could reach. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VmOrchestratorReport { + /// `firecracker` (live client resolved at boot) or `unwired`. + pub orchestrator: String, + /// The client's own `ready()`: bearer file present and non-empty, RLM + /// image digest pinned. Checked per request, so a fix needs no restart. + pub ready: bool, + /// Why not ready (empty when ready). Names the env var to fix. + pub reason: String, + /// `sha256:` pin of the RLM VM image the client asks the agent to boot + /// (empty = unpinned = nothing ever boots). + pub image_digest: String, + /// RLM VM vCPUs (locked default 4). + pub vcpus: u32, + /// RLM VM memory in MiB (locked default 8192). + pub mem_mib: u32, + /// The agent's answer to one health call, when it answered. + pub agent: Option, + /// Why the agent did not answer: unreachable, bearer refused, not wired. + pub agent_error: Option, + /// Filled by the host: the digest-pinned Lium harvest (`nll` / + /// `throughput`) is wired. Lium only — informational for the custom + /// family, which is wired from the topic-VM env on its own. + #[serde(default)] + pub live_harvest_wired: bool, + /// Filled by the host: at least one custom id has a registered runner + /// (the custom family is routed, harvest or not). + #[serde(default)] + pub custom_family_wired: bool, + /// Filled by the host: custom ids with a registered runner. + #[serde(default)] + pub registered_custom: Vec, +} + +impl VmOrchestratorReport { + /// Report for a host that keeps `UnwiredVmOrchestrator`; `reason` names + /// the env vars a live one reads. `image_digest` is whatever pin the env + /// carries so "pinned but URL unset" is visible. + #[must_use] + pub fn unwired(reason: &str, image_digest: &str) -> Self { + Self { + orchestrator: "unwired".into(), + ready: false, + reason: reason.trim().to_owned(), + image_digest: image_digest.trim().to_owned(), + vcpus: 0, + mem_mib: 0, + agent: None, + agent_error: None, + live_harvest_wired: false, + custom_family_wired: false, + registered_custom: Vec::new(), + } + } +} + +/// Operator diagnostic over the topic-VM orchestrator this host resolved at +/// boot. The binary implements it over the live `FirecrackerOrchestrator` +/// (its `ready()` plus one agent health call) or the unwired stand-in; the +/// route only adds what the host knows (harvest wired, registered ids). It +/// changes nothing and spends nothing. +#[async_trait::async_trait] +pub trait VmOrchestratorProbe: Send + Sync { + /// Snapshot as of now (bearer file and pin re-read; one agent round trip). + async fn probe(&self) -> VmOrchestratorReport; +} + +/// What the **submit** path needs from the journal, in one read. +/// +/// Both fields are about the topic's install and the operator's switch, and +/// both are read before anything is spent, so they are one call: a submission +/// costs one gate read, not two. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SubmitGate { + /// The operator's reason, when the topic is disabled (`topic disable`). + pub disabled_reason: Option, + /// The `vms_per_submission` the topic's newest install recorded. + /// + /// `None` when the topic has no install row or the row predates the + /// field. `Some(n)` with `n != 1` is refused on the submit path: this + /// build runs **one VM per submission**, and a topic installed with a + /// different pin is one it cannot honour. + pub vms_per_submission: Option, +} + +impl SubmitGate { + /// The operator gate, as the submit path reads it. + #[must_use] + pub fn disabled(reason: impl Into) -> Self { + Self { + disabled_reason: Some(reason.into()), + ..Self::default() + } + } +} + +/// Whether a topic's install reached `applied`, as the publish route reads it. +/// +/// A trait rather than a pool so the route can be exercised without a +/// database, and so a host that resolved no install journal can say so instead +/// of answering from a table it never read. +#[async_trait::async_trait] +pub trait InstallJournal: Send + Sync { + /// `Ok(true)` when the newest install row for `topic_id` is `applied`. + /// + /// # Errors + /// + /// The reason the journal could not be read. The caller refuses the + /// publish: an unreadable journal is not an installed topic. + async fn applied(&self, topic_id: &str) -> Result; + + /// The operator gate and the allocator pin for one topic, for the submit + /// path. + /// + /// # Errors + /// + /// The reason the gate could not be read. The caller answers **503**: an + /// unreadable gate is not an enabled topic. + async fn submit_gate(&self, topic_id: &str) -> Result; + + /// Every topic currently disabled, with the operator's reason. + /// + /// One read for the public listing, which annotates each topic with the + /// flag rather than paying a query per topic. The submit path uses + /// [`Self::submit_gate`] for the single topic it is admitting. + /// + /// # Errors + /// + /// The reason the gate could not be read. The caller answers **503**. + async fn disabled_topics(&self) -> Result, String>; +} + +/// The journal read behind the publish gate, or `None` on a host that +/// resolved none (no database). +/// +/// `None` is **fail-closed**: [`install_gate`] refuses, so an `open` +/// document cannot be published on a host that cannot prove the install ran. +/// That is the same rule the gate enforces when the journal is unreadable — +/// the only difference is which sentence the operator reads. +pub type InstallJournalSlot = Option>; + +/// Whether the topic's install reached `applied`, as the publish gate reads it. +/// +/// **Fail-closed on every doubt**: no journal slot, an unreadable journal, and +/// a topic with no install row all refuse, so an `open` document is never +/// published before its migrations, routes, and rules are in place. +pub(crate) async fn install_gate(st: &AppState, topic_id: &str) -> Result<(), String> { + let Some(journal) = st.install_journal.as_deref() else { + return Err(format!( + "this host resolved no install journal (no database), so it cannot prove that topic \ + {topic_id:?} was installed. Publish the document as `draft`, or wire \ + BASE_DATABASE_URL and restart." + )); + }; + match journal.applied(topic_id).await { + Ok(true) => Ok(()), + Ok(false) => Err(format!( + "topic {topic_id:?} has no `applied` install row: run `proof-admin topic install` to \ + completion first (the journal is `proof_topic_install`; read it with `proof-admin \ + topic install-log --topic {topic_id}`). A `pending` or `failed` row means the \ + migrations, routes, or rules are not in place, and an `open` document is submitable \ + the moment it is published." + )), + Err(e) => Err(format!( + "the install journal could not be read for topic {topic_id:?}: {e}. The publish is \ + refused rather than admitted on an unread fact; fix the database and re-publish." + )), + } +} + +/// The operator gate on the **submit** path: is this topic disabled, and is +/// its allocator pin one this build runs? +/// +/// `Ok(())` admits the submission. A disabled topic is a **403** naming the +/// operator's reason; a topic whose install recorded a `vms_per_submission` +/// other than 1 is a **503** — this build runs exactly one VM per submission, +/// and silently running a different number would make the journal a lie; an +/// unreadable gate is a **503** too — never an admission, and never a 404 that +/// would read as "no such topic". A host that resolved no journal (no +/// database) has no gate to read: it also has no published topics, so the +/// submission is refused by the topic lookup above it. +/// +/// This is deliberately **not cached**: a disable has to take effect on the +/// next request, which is what makes it usable during an incident. +pub(crate) async fn disabled_gate(st: &AppState, topic_id: &str) -> Result<(), ErrResp> { + let Some(journal) = st.install_journal.as_deref() else { + return Ok(()); + }; + let gate = match journal.submit_gate(topic_id).await { + Ok(gate) => gate, + Err(e) => { + return Err(err( + StatusCode::SERVICE_UNAVAILABLE, + &format!( + "the topic gate could not be read for {topic_id:?}: {e}. The submission is \ + refused rather than admitted on an unread fact; fix the database and re-post \ + (the submit_nonce is unspent)." + ), + )) + } + }; + if let Some(reason) = gate.disabled_reason.as_deref() { + return Err(err( + StatusCode::FORBIDDEN, + &format!( + "topic {topic_id:?} is disabled by the operator{}", + if reason.trim().is_empty() { + String::new() + } else { + format!(": {}", reason.trim()) + } + ), + )); + } + if let Some(n) = gate.vms_per_submission { + if n != proof_topic_install::VMS_PER_SUBMISSION { + return Err(err( + StatusCode::SERVICE_UNAVAILABLE, + &format!( + "topic {topic_id:?} was installed with vms_per_submission={n}, but this host \ + runs exactly {} VM per submission (the pin this build enforces). Re-install \ + the topic with the pin this host carries, or run the host that matches the \ + install. The submission is refused rather than run under a binding the \ + install did not record; the submit_nonce is unspent.", + proof_topic_install::VMS_PER_SUBMISSION + ), + )); + } + } + Ok(()) +} + +/// The disabled set the listing annotates from, or a 503. +pub(crate) async fn disabled_topics( + st: &AppState, +) -> Result, ErrResp> { + let Some(journal) = st.install_journal.as_deref() else { + // No gate on this host: no database, so no operator switch was ever + // thrown (and no topic was published either). + return Ok(std::collections::BTreeMap::new()); + }; + journal.disabled_topics().await.map_err(|e| { + err( + StatusCode::SERVICE_UNAVAILABLE, + &format!( + "the topic gate could not be read: {e}. The listing is refused rather than \ + served without the operator's switch; fix the database and re-read." + ), + ) + }) +} diff --git a/crates/proof-http/src/submit.rs b/crates/proof-http/src/submit.rs index fcda0923a..ff2497f07 100644 --- a/crates/proof-http/src/submit.rs +++ b/crates/proof-http/src/submit.rs @@ -11,7 +11,8 @@ use proof_store::MAX_ARTEFACT_BYTES; use proof_vm_proto::tar::{verify_artifact, TarError}; use sha2::{Digest, Sha256}; -use super::{err, ErrResp, SubmitBody}; +use super::{err, ErrResp}; +use proof_submit::SubmitBody; /// Multipart overhead budget on top of the 5 MiB artefact cap. pub const SUBMIT_BODY_LIMIT: usize = MAX_ARTEFACT_BYTES + 512 * 1024; diff --git a/crates/proof-results/src/lib.rs b/crates/proof-results/src/lib.rs index b420fd8fc..ee1ffcc41 100644 --- a/crates/proof-results/src/lib.rs +++ b/crates/proof-results/src/lib.rs @@ -78,10 +78,15 @@ pub const PARAM_RESULTS_PATH: &str = "results_path"; /// Envelope-only contract every custom evaluate may satisfy. pub const CONTRACT_GENERIC: &str = "generic-custom-v1"; -/// Harbor trial-summary contract (tbench and any Harbor-scored topic). +/// Harbor trial-summary contract: the generic id every Harbor-scored topic +/// pins. pub const CONTRACT_HARBOR_TRIALS: &str = "harbor-trials-v1"; -/// Alias a tbench topic may pin; same shape as [`CONTRACT_HARBOR_TRIALS`]. +/// Legacy alias of [`CONTRACT_HARBOR_TRIALS`], kept because it is a **wire +/// value**: a topic signed before the generic id existed pins this in its +/// `constraints.params.results_contract`, and a signed document cannot be +/// edited. New topics pin [`CONTRACT_HARBOR_TRIALS`]; the guest harness +/// accepts both, and nothing branches on a topic. pub const CONTRACT_TBENCH_HARBOR: &str = "tbench-harbor-v1"; /// Harbor trial that produced a verifier reward. @@ -179,11 +184,12 @@ impl<'a> ReportBind<'a> { pub enum Contract { /// Envelope + a non-empty `display` object. Generic, - /// Harbor / tbench trial table. + /// Harbor trial table. HarborTrials, } -/// Known contract id → family. [`CONTRACT_TBENCH_HARBOR`] aliases Harbor. +/// Known contract id → family. [`CONTRACT_TBENCH_HARBOR`] is the legacy +/// spelling of [`CONTRACT_HARBOR_TRIALS`]. #[must_use] pub fn known_contract(id: &str) -> Option { match id.trim() { diff --git a/crates/proof-submit/Cargo.toml b/crates/proof-submit/Cargo.toml index 82f9a89bf..0ee7f37f3 100644 --- a/crates/proof-submit/Cargo.toml +++ b/crates/proof-submit/Cargo.toml @@ -10,8 +10,11 @@ publish = false [dependencies] crypto = { path = "../crypto" } +proof-canon = { path = "../proof-canon" } +proof-store = { path = "../proof-store" } hex = "0.4" rand_core = { version = "0.6", features = ["getrandom"] } +serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" thiserror = "2" diff --git a/crates/proof-submit/src/lib.rs b/crates/proof-submit/src/lib.rs index ce6c4e515..3e965672b 100644 --- a/crates/proof-submit/src/lib.rs +++ b/crates/proof-submit/src/lib.rs @@ -58,6 +58,12 @@ pub fn sha256_hex(bytes: &[u8]) -> String { /// Why a submit signature cannot be built or checked. #[derive(Debug, Error, Clone, PartialEq, Eq)] pub enum SubmitSigError { + /// `hotkey_signature` was absent or empty. + #[error("hotkey_signature required")] + MissingSignature, + /// `submit_nonce` was absent or empty. + #[error("submit_nonce required")] + MissingNonce, /// `hotkey_signature` was not exactly 128 lowercase hex characters. #[error("hotkey_signature invalid")] InvalidSignature, @@ -668,3 +674,129 @@ mod tests { ); } } + +/// The Proof submit body, as it arrives on the wire. +/// +/// This is the **wire contract** — the shape `POST /v1/submissions` parses and +/// the shape a miner signs over. It lives beside [`SubmitFields`] (the exact +/// bytes signed) rather than in the HTTP crate, because the two have to agree: +/// a field added here without a matching `SubmitFields` entry would be a value +/// a miner sends but nobody authenticates. +/// +/// The `manifest` and `env` types are the store's and the canonical-JSON +/// crate's, so the wire shape cannot drift from the gate that reads it. +#[derive(Debug, Clone, serde::Deserialize)] +pub struct SubmitBody { + /// 64 hex sr25519 public key. + pub miner_hotkey: String, + /// sr25519 signature over [`SubmitFields::signing_payload`] (128 lowercase hex). + #[serde(default)] + pub hotkey_signature: Option, + /// Client anti-replay nonce (64 lowercase hex), bound into the signature + /// and accepted once per hotkey. + #[serde(default)] + pub submit_nonce: Option, + /// 64 lowercase hex sha256 of the artefact. + pub artifact_digest: String, + /// Where the bytes live, when the miner did not upload them. + #[serde(default)] + pub artifact_uri: Option, + /// Claim text, signed. + #[serde(default)] + pub claim: String, + /// Declared FLOPs, signed. + #[serde(default)] + pub declared_flops: u64, + /// Topic submitted to. + #[serde(default)] + pub topic_id: String, + /// Optional miner label. Not compared to an HF id (that check is retired). + #[serde(default)] + pub architecture: String, + /// Training manifest the contamination gate reads. + #[serde(default)] + pub manifest: proof_store::ArtifactManifest, + /// Miner BYOK: `{"": ""}` for the variables this topic's + /// signed document declares (`constraints.params.miner_byok` / + /// `miner_env_allowlist`). Never signed (v1 of the submit payload is + /// unchanged), never persisted on the row, never echoed back. A name the + /// topic does not declare is a **400** before the signature is checked, + /// so a mistake here never burns the single-use `submit_nonce`. + #[serde(default)] + pub env: proof_canon::MinerEnv, +} + +impl SubmitBody { + /// Miner identity for one submit: `hotkey_signature` over every gate input + /// (topic, artefact, FLOPs, claim, manifest) plus the client + /// `submit_nonce`, verified over the strings exactly as posted. + /// + /// Returns the nonce the caller must reserve before any row or rent, or + /// the [`SubmitSigError`] naming which field was wrong. + /// + /// # Errors + /// + /// [`SubmitSigError`] — a missing or malformed signature, nonce, or + /// hotkey, or a signature that does not verify. + pub fn authenticate(&self, hotkey: &str, artifact: &str) -> Result { + let sig_hex = self + .hotkey_signature + .as_deref() + .filter(|s| !s.is_empty()) + .ok_or(SubmitSigError::MissingSignature)?; + let sig = parse_signature_hex(sig_hex)?; + let nonce = self + .submit_nonce + .as_deref() + .filter(|s| !s.is_empty()) + .ok_or(SubmitSigError::MissingNonce)?; + parse_submit_nonce_hex(nonce)?; + let pk = parse_hotkey_hex(hotkey)?; + // A signature that does not verify is the **wire** refusal + // (`hotkey_signature invalid`), not the crypto layer's own words: the + // miner reads this string, and it is the same one every other + // malformed-signature case answers. + verify_submit( + &pk, + &SubmitFields { + hotkey_hex: hotkey, + topic_id: &self.topic_id, + artifact_digest: artifact, + declared_flops: self.declared_flops, + claim: &self.claim, + train_content_hashes: &self.manifest.train_content_hashes, + train_dataset_ids: &self.manifest.train_dataset_ids, + submit_nonce_hex: nonce, + }, + &sig, + ) + .map_err(|_| SubmitSigError::InvalidSignature)?; + Ok(nonce.to_owned()) + } +} + +/// A digest of **nothing** is not an artefact digest. +/// +/// The sha256 of zero bytes and of an empty tar archive (`tar cf - -T +/// /dev/null`, 10240 zero bytes). Staging's happy path once matched exactly +/// such a digest because the RLM guest could not fetch the artefact and fell +/// back to an empty tree; refusing it at submit keeps that stub out of every +/// row and rent. +#[must_use] +pub fn is_digest_of_nothing(hex64: &str) -> bool { + let empty_input = sha256_hex(b""); + let empty_tar = sha256_hex(&[0u8; 10_240]); + hex64.eq_ignore_ascii_case(&empty_input) || hex64.eq_ignore_ascii_case(&empty_tar) +} + +/// The row nonce for `(hotkey, topic, digest)`: the frozen digest's own input, +/// so a row can be re-derived from the wire fields that produced it. +#[must_use] +pub fn nonce_from(hotkey: &str, topic_id: &str, digest: &str) -> String { + let mut h = Sha256::new(); + h.update(b"proof-nonce-v1"); + h.update(hotkey.as_bytes()); + h.update(topic_id.as_bytes()); + h.update(digest.as_bytes()); + hex::encode(h.finalize()) +} diff --git a/crates/proof-topic-ops/Cargo.toml b/crates/proof-topic-ops/Cargo.toml new file mode 100644 index 000000000..bfb866de4 --- /dev/null +++ b/crates/proof-topic-ops/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "proof-topic-ops" +description = "Proof operator procedures that reach a live host: drive the RLM setup, seal the measured baseline, and publish through the admin route" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +db = { path = "../db" } +proof-eval = { path = "../proof-eval" } +proof-rlm = { path = "../proof-rlm" } +proof-rlm-store = { path = "../proof-rlm-store" } +proof-task = { path = "../proof-task" } +proof-topic-bundle = { path = "../proof-topic-bundle" } +proof-topic-install = { path = "../proof-topic-install" } +proof-topic-setup = { path = "../proof-topic-setup" } +proof-vm-fc = { path = "../proof-vm-fc" } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } +serde_json = "1" +sqlx = { version = "0.8", features = ["runtime-tokio", "postgres", "json"] } + +[dev-dependencies] +proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } + +[lints] +workspace = true diff --git a/bins/proof-admin/src/drive.rs b/crates/proof-topic-ops/src/drive.rs similarity index 93% rename from bins/proof-admin/src/drive.rs rename to crates/proof-topic-ops/src/drive.rs index 4e149ebee..04bb38831 100644 --- a/bins/proof-admin/src/drive.rs +++ b/crates/proof-topic-ops/src/drive.rs @@ -31,7 +31,7 @@ use proof_rlm_store::{PgRlmStore, RlmStore}; use proof_task::{InferenceOffer, ProofPin, TopicDocument}; use proof_topic_setup::{SetupError, SetupOutcome, TopicSetup}; -use crate::Failure; +use crate::OpsError; /// What driving the RLM produced, for the install report and the operator. pub struct DriveOutcome { @@ -71,8 +71,9 @@ impl DriveOutcome { /// /// # Errors /// -/// [`Failure::Usage`] for a missing piece of host configuration (naming the -/// env var), [`Failure::Error`] for a refusal from the orchestrator, the +/// [`OpsError::usage`](crate::OpsError::usage) for a missing piece of host +/// configuration (naming the env var), [`OpsError::error`](crate::OpsError::error) +/// for a refusal from the orchestrator, the /// lifecycle, or the store. #[allow(clippy::too_many_arguments)] pub async fn drive( @@ -86,9 +87,9 @@ pub async fn drive( rlm_image_digest: Option<&str>, offer: Option, owner_key_file: Option<&Path>, -) -> Result { +) -> Result { if !owner_approved { - return Err(Failure::Usage( + return Err(OpsError::usage( "driving the RLM provisions a topic VM and runs a paid baseline, so it requires \ --owner-approved." .to_owned(), @@ -102,7 +103,7 @@ pub async fn drive( // A baseline is a paid run: without an offer there is nothing to measure // against. `--skip-baseline` is the path that does not need one. if offer.is_none() && !skip_baseline { - return Err(Failure::Usage( + return Err(OpsError::usage( "the RLM's baseline is a paid run that needs a live judge offer: set \ PROOF_INFERENCE_OFFER_FILE to an open InferenceOffer (or pass --skip-baseline to \ install the rules without measuring one). Without a baseline the topic cannot open \ @@ -122,7 +123,7 @@ pub async fn drive( store: Arc::new(store) as Arc, template: proof_rlm::VmTemplate::from_env(), experiments: proof_rlm::ExperimentPolicy::from_env().map_err(|e| { - Failure::Usage(format!( + OpsError::usage(format!( "the per-experiment VM policy is malformed: {e}. Fix the \ PROOF_EXPERIMENT_VM_* env before driving the RLM." )) @@ -138,7 +139,7 @@ pub async fn drive( let outcome = setup .run(topic, pin, offer.as_ref()) .await - .map_err(|e| Failure::Error(drive_failure(&e)))?; + .map_err(|e| OpsError::error(drive_failure(&e)))?; Ok(outcome_summary(outcome)) } @@ -147,12 +148,12 @@ fn resolve_orchestrator( url: Option<&str>, token_file: Option<&Path>, image_digest: Option<&str>, -) -> Result, Failure> { +) -> Result, OpsError> { // Presence only: `FirecrackerOrchestrator::from_env` reads the env itself, // so the CLI checks that each piece *is* set (and names the missing one) // without duplicating the client's parsing and validation. if url.map(str::trim).is_none_or(str::is_empty) { - return Err(Failure::Usage(format!( + return Err(OpsError::usage(format!( "driving the RLM needs the topic-VM orchestrator: set {VM_ORCHESTRATOR_URL_ENV} \ (https, the KVM host agent) plus {VM_ORCHESTRATOR_TOKEN_FILE_ENV} and \ {RLM_VM_IMAGE_DIGEST_ENV}. Nothing is driven on the control-plane host — that is the \ @@ -160,24 +161,24 @@ fn resolve_orchestrator( ))); } if token_file.is_none() { - return Err(Failure::Usage(format!( + return Err(OpsError::usage(format!( "driving the RLM needs {VM_ORCHESTRATOR_TOKEN_FILE_ENV}: a file holding the bearer \ for the topic-VM orchestrator. It is re-read per request and never logged." ))); } if image_digest.map(str::trim).is_none_or(str::is_empty) { - return Err(Failure::Usage(format!( + return Err(OpsError::usage(format!( "driving the RLM needs {RLM_VM_IMAGE_DIGEST_ENV}: the sha256 digest of the RLM VM \ image the orchestrator boots. A digest is never invented." ))); } match proof_vm_fc::FirecrackerOrchestrator::from_env() { Ok(Some(fc)) => Ok(Arc::new(fc)), - Ok(None) => Err(Failure::Usage(format!( + Ok(None) => Err(OpsError::usage(format!( "no topic-VM orchestrator resolved from {VM_ORCHESTRATOR_URL_ENV} / \ {VM_ORCHESTRATOR_TOKEN_FILE_ENV} / {RLM_VM_IMAGE_DIGEST_ENV}" ))), - Err(e) => Err(Failure::Usage(format!( + Err(e) => Err(OpsError::usage(format!( "the topic-VM orchestrator configuration was refused: {e}" ))), } diff --git a/crates/proof-topic-ops/src/lib.rs b/crates/proof-topic-ops/src/lib.rs new file mode 100644 index 000000000..f339bf92e --- /dev/null +++ b/crates/proof-topic-ops/src/lib.rs @@ -0,0 +1,83 @@ +//! The Proof operator **procedures** that reach a live host: drive the RLM +//! setup, read the baseline it measured, seal it, and publish the open +//! document. +//! +//! `proof-admin` is the CLI; this crate is what its subcommands *do*. The +//! split exists because the binary is at the repository's per-crate LOC cap, +//! and because these procedures have a different subject from argument +//! parsing: every one of them either provisions something (a VM, a paid +//! baseline), writes durable state (the seal, the lifecycle move), or calls a +//! master over the network — the things an operator needs to be able to read +//! in one place. +//! +//! | Procedure | What it reaches | +//! |-----------|-----------------| +//! | [`drive`] | the topic-VM orchestrator: provision → `propose_rules` → baseline | +//! | [`baseline`] | the shared database: the measurement the RLM stored | +//! | [`seal`] | `TopicSetup::mark_sealed`, then the admin publish route | +//! +//! Nothing here holds a signing key: the `proof` topic key stays with the +//! operator, and `xtask proof-topic` is what signs a document. Nothing here +//! falls back to the control-plane host: an unwired orchestrator is a +//! refusal, and a missing owner assertion is a refusal, both named. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::module_name_repetitions +)] + +pub mod drive; +pub mod publish; +pub mod seal; + +pub use drive::{drive, DriveOutcome}; +pub use publish::PublishTarget; +pub use seal::{baseline, seal, BaselineReport, SealArgs, SealOutcome}; + +/// Why an operator procedure refused. +/// +/// Two kinds, because the CLI's exit codes depend on the difference: a +/// **usage** error is a flag or an environment variable the operator can fix +/// before re-running, and anything else is a refusal from a host or the +/// database. Keeping them distinct here is what lets the binary stay a thin +/// adapter. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OpsError { + /// Bad usage or missing configuration: the operator fixes a flag or an + /// env var. The CLI exits 2. + Usage(String), + /// A refusal from a host, the database, or a document. The CLI exits 1. + Error(String), +} + +impl std::fmt::Display for OpsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Usage(m) | Self::Error(m) => f.write_str(m), + } + } +} + +impl std::error::Error for OpsError {} + +impl OpsError { + /// A usage refusal. + #[must_use] + pub fn usage(message: impl Into) -> Self { + Self::Usage(message.into()) + } + + /// Any other refusal. + #[must_use] + pub fn error(message: impl Into) -> Self { + Self::Error(message.into()) + } + + /// Whether this is the operator's usage to fix. + #[must_use] + pub const fn is_usage(&self) -> bool { + matches!(self, Self::Usage(_)) + } +} diff --git a/crates/proof-topic-ops/src/publish.rs b/crates/proof-topic-ops/src/publish.rs new file mode 100644 index 000000000..42014aea2 --- /dev/null +++ b/crates/proof-topic-ops/src/publish.rs @@ -0,0 +1,101 @@ +//! Publishing a signed document through the existing admin route. +//! +//! `proof-admin topic install` and `topic seal` both end here: the document is +//! already validated and signed, and this is the one HTTP call that makes it +//! reachable. The route is `POST /v1/admin/proof/topics` +//! (`proof_topic_bundle::PUBLISH_PATH`), and the bearer is read from a file — +//! never printed, never logged, never an argument. +//! +//! # Why the bearer is a file +//! +//! A token on a command line is a token in the shell history and in `ps`. The +//! file is read here and held in this struct; [`PublishTarget::redacted`] is +//! what an operator sees when the call is reported. + +use std::path::Path; + +/// Where the admin publish call goes, and the bearer it uses. +pub struct PublishTarget { + base_url: String, + token: String, +} + +impl PublishTarget { + /// Resolve the URL and bearer, refusing a half-configured pair. + pub fn resolve( + admin_url: Option<&str>, + admin_token_file: Option<&Path>, + ) -> Result { + let Some(base_url) = admin_url.map(str::trim).filter(|u| !u.is_empty()) else { + return Err(crate::OpsError::usage( + "a real install publishes through the admin route, so it needs the master's \ + base URL: pass --admin-url (or set PROOF_ADMIN_URL), e.g. \ + --admin-url http://127.0.0.1:8100 for the challenge service directly, or the \ + gateway's address. `--dry-run` needs none." + .to_owned(), + )); + }; + let Some(path) = admin_token_file else { + return Err(crate::OpsError::usage( + "a real install needs the operator bearer for /v1/admin/*: pass \ + --admin-token-file (or set PROOF_ADMIN_TOKEN_FILE). The file is read and never \ + logged or printed. `--dry-run` needs none." + .to_owned(), + )); + }; + let token = std::fs::read_to_string(path) + .map_err(|e| crate::OpsError::error(format!("read {}: {e}", path.display())))?; + // A tokens file holds one bearer per line; the first non-comment line + // is the one this call uses. + let token = token + .lines() + .map(str::trim) + .find(|l| !l.is_empty() && !l.starts_with('#')) + .map(str::to_owned); + let Some(token) = token else { + return Err(crate::OpsError::error(format!( + "{} holds no bearer (every line is blank or a comment)", + path.display() + ))); + }; + Ok(Self { + base_url: base_url.trim_end_matches('/').to_owned(), + token, + }) + } + + /// How this target is printed: the URL, never the bearer. + #[must_use] + pub fn redacted(&self) -> String { + format!("{} (bearer read, never printed)", self.base_url) + } + + /// Publish the document through the existing admin route. + pub async fn publish(&self, doc: &proof_task::TopicDocument) -> Result<(), String> { + let url = format!("{}{}", self.base_url, proof_topic_bundle::PUBLISH_PATH); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_mins(1)) + .build() + .map_err(|e| format!("http client: {e}"))?; + let response = client + .post(&url) + .header("authorization", format!("Bearer {}", self.token)) + .header("content-type", "application/json") + .body( + serde_json::to_string(doc) + .map_err(|e| format!("serialize the signed document: {e}"))?, + ) + .send() + .await + .map_err(|e| format!("POST {url}: {e}"))?; + let status = response.status(); + if status.is_success() { + return Ok(()); + } + let body = response.text().await.unwrap_or_default(); + Err(format!( + "POST {url} answered {status}: {}", + body.trim().chars().take(400).collect::() + )) + } +} diff --git a/bins/proof-admin/src/seal.rs b/crates/proof-topic-ops/src/seal.rs similarity index 75% rename from bins/proof-admin/src/seal.rs rename to crates/proof-topic-ops/src/seal.rs index e0e2dbb79..3344de5c7 100644 --- a/bins/proof-admin/src/seal.rs +++ b/crates/proof-topic-ops/src/seal.rs @@ -18,7 +18,7 @@ //! //! [`baseline`] is the read half: the measured primary, the rule version it //! was measured under, and the `metrics_commitment` an `open` document must -//! carry — the value that goes into the draft before it is signed (the CLI +//! carry — the value that goes into the draft before it is signed (this crate //! never signs: the `proof` key stays with the operator, and `xtask //! proof-topic` is what signs a draft). //! @@ -48,14 +48,15 @@ //! against the document. use std::path::Path; +use std::sync::Arc; use proof_eval::BaselineMeasurement; use proof_rlm_store::{BaselineRow, PgRlmStore, RlmStore}; use proof_task::{HoldoutSplit, ProofPin, TopicDocument, TopicStatus}; use proof_topic_setup::TopicSetup; -use crate::install::AdminTarget; -use crate::{Failure, Options}; +use crate::publish::PublishTarget; +use crate::OpsError; /// Everything `topic seal` was asked to do. pub struct SealArgs<'a> { @@ -64,97 +65,122 @@ pub struct SealArgs<'a> { /// The signed `status: open` document. pub document: &'a Path, /// Pin the document is checked against. - pub pin: &'a Path, + pub pin: &'a ProofPin, /// Publish the sealed document through the admin route. pub publish: bool, /// Master base URL for the publish call (with `--publish`). pub admin_url: Option<&'a str>, /// File holding the operator bearer (with `--publish`). pub admin_token_file: Option<&'a Path>, + /// Custom ids this host registers for scoring + /// (`PROOF_VM_RUNNER_CUSTOM_IDS`): an open custom topic needs one. + pub registered_custom: Vec, +} + +/// What the RLM measured, and the commitment an `open` document must seal. +#[derive(Debug, Clone, PartialEq)] +pub struct BaselineReport { + /// Canonical topic slug (the alias resolved). + pub topic_id: String, + /// Rule version the baseline was measured under. + pub rules_version: u32, + /// The measured primary — what the document's `custom_value` seals. + pub primary_value: f64, + /// The document's metric primary name. + pub metric_primary: String, + /// The document's custom id. + pub custom_id: String, + /// The topic's holdout commitment. + pub holdout_commitment: String, + /// The `baseline.metrics_commitment` an open document must carry. + pub metrics_commitment: String, + /// The document's current status (a draft is not scorable). + pub document_status: TopicStatus, +} + +impl BaselineReport { + /// The steps that turn this measurement into a scorable topic. + #[must_use] + pub fn next_steps(&self) -> String { + next_seal_steps(&self.topic_id, &self.metrics_commitment) + } +} + +/// What sealing produced. +#[derive(Debug, Clone, PartialEq)] +pub struct SealOutcome { + /// Canonical topic slug. + pub topic_id: String, + /// New document version the seal stored. + pub document_version: u32, + /// The primary the RLM measured and the document sealed. + pub primary_value: f64, + /// The commitment the document carries. + pub metrics_commitment: String, + /// Whether the open document was published through the admin route. + pub published: bool, +} + +impl SealOutcome { + /// What to do next, for the operator. + #[must_use] + pub fn after(&self) -> String { + after_seal(&self.topic_id, self.published) + } } /// `topic baseline`: what the RLM measured, and what to seal. /// /// # Errors /// -/// [`Failure::Usage`] without a database, [`Failure::Error`] when the topic -/// or its measurement cannot be read. -pub async fn baseline(opts: &Options, topic_id: &str, pin_path: &Path) -> Result<(), Failure> { - let pool = crate::open_pool(opts).await?; +/// [`OpsError::error`] when the topic or its measurement cannot be read. +pub async fn baseline( + pool: &sqlx::PgPool, + pin: &ProofPin, + topic_id: &str, +) -> Result { let store = PgRlmStore::new(pool.clone()); - let pin = crate::load_pin(pin_path)?; let (canonical, _, document) = resolve_topic(&store, topic_id).await?; let Some(measured) = store .baseline(&canonical) .await - .map_err(|e| Failure::Error(format!("{canonical} baseline: {e}")))? + .map_err(|e| OpsError::error(format!("{canonical} baseline: {e}")))? else { - return Err(Failure::Error(format!( + return Err(OpsError::error(format!( "no baseline measured for topic {canonical:?}. It is written by the RLM's baseline \ job: run `proof-admin topic install --bundle --env --drive-rlm \ --owner-approved` (without --skip-baseline) first. Nothing to seal yet." ))); }; - let commitment = seal_measurement(&pin, &document, &measured).commitment(); - if opts.json { - crate::print_json(&serde_json::json!({ - "topic_id": canonical, - "rules_version": measured.rules_version, - "primary_value": measured.primary_value, - "metric_primary": document.metric.primary, - "custom_id": document.metric.custom_id, - "holdout_commitment": document.holdout_commitment, - "metrics_commitment": commitment, - "document_status": document.status, - "document_version_signature": document.signature, - "next": next_seal_steps(&canonical, &commitment), - }))?; - return Ok(()); - } - println!("topic {canonical} — measured baseline"); - println!(" primary_value {}", measured.primary_value); - println!(" metric_primary {}", document.metric.primary); - println!( - " custom_id {}", - crate::dash_if_empty(&document.metric.custom_id) - ); - println!(" rules_version {}", measured.rules_version); - println!(" holdout {}", document.holdout_commitment); - println!( - " document_status {}", - crate::status_word(document.status) - ); - println!(); - println!("An `open` document must seal this measurement. Its baseline block needs:"); - println!(" metrics_commitment {commitment}"); - println!( - " script_sha256 {}", - crate::dash_if_empty(&document.baseline.script_sha256) - ); - println!(); - println!("{}", next_seal_steps(&canonical, &commitment)); - Ok(()) + let commitment = seal_measurement(pin, &document, &measured).commitment(); + Ok(BaselineReport { + topic_id: canonical, + rules_version: measured.rules_version, + primary_value: measured.primary_value, + metric_primary: document.metric.primary.clone(), + custom_id: document.metric.custom_id.clone(), + holdout_commitment: document.holdout_commitment.clone(), + metrics_commitment: commitment, + document_status: document.status, + }) } /// `topic seal`: record the operator's seal and open the topic. /// /// # Errors /// -/// [`Failure::Usage`] without a database or without a publish target when -/// `--publish` was given, [`Failure::Error`] for a refused document, a -/// missing measurement, a lifecycle that is not at `baselining`, or a -/// refused publish. -pub async fn seal(opts: &Options, args: &SealArgs<'_>) -> Result<(), Failure> { - let pool = crate::open_pool(opts).await?; +/// [`OpsError::usage`] for a document that is not this topic's or is not +/// `open`, [`OpsError::error`] for a missing measurement, a lifecycle that is +/// not at `baselining`, a refused document, or a refused publish. +pub async fn seal(pool: &sqlx::PgPool, args: &SealArgs<'_>) -> Result { let store = PgRlmStore::new(pool.clone()); let (canonical, version, _) = resolve_topic(&store, args.topic_id).await?; - let pin = crate::load_pin(args.pin)?; let body = std::fs::read_to_string(args.document) - .map_err(|e| Failure::Error(format!("read {}: {e}", args.document.display())))?; + .map_err(|e| OpsError::error(format!("read {}: {e}", args.document.display())))?; let document: TopicDocument = serde_json::from_str(&body) - .map_err(|e| Failure::Error(format!("{}: {e}", args.document.display())))?; + .map_err(|e| OpsError::error(format!("{}: {e}", args.document.display())))?; if document.id != canonical { - return Err(Failure::Usage(format!( + return Err(OpsError::usage(format!( "{} carries topic {:?}, but this command is sealing {canonical:?}{}. Nothing was \ changed.", args.document.display(), @@ -163,69 +189,50 @@ pub async fn seal(opts: &Options, args: &SealArgs<'_>) -> Result<(), Failure> { ))); } if document.status != TopicStatus::Open { - return Err(Failure::Usage(format!( + return Err(OpsError::usage(format!( "{} is `{}`, not `open`. Sealing opens a topic, so the document has to be the open \ one: set `status: open`, seal `baseline.metrics_commitment` from `proof-admin topic \ baseline`, sign it, and re-run. Nothing was changed.", args.document.display(), - crate::status_word(document.status) + status_word(document.status) ))); } let Some(measured) = store .baseline(&canonical) .await - .map_err(|e| Failure::Error(format!("{canonical} baseline: {e}")))? + .map_err(|e| OpsError::error(format!("{canonical} baseline: {e}")))? else { - return Err(Failure::Error(format!( + return Err(OpsError::error(format!( "no baseline measured for topic {canonical:?}, so there is nothing to seal. Run the \ install with --drive-rlm (without --skip-baseline) first. Nothing was changed." ))); }; - let sealed = seal_measurement(&pin, &document, &measured); + let sealed = seal_measurement(args.pin, &document, &measured); // The one call that decides: the same `mark_sealed` the runtime's own // tests drive, so a document this command accepts is one the scoring path // would accept. - let registered = crate::registered_custom_from_env(); - let registered: Vec<&str> = registered.iter().map(String::as_str).collect(); + let registered: Vec<&str> = args.registered_custom.iter().map(String::as_str).collect(); let setup = seal_setup(store); - let state = setup - .mark_sealed(&document, &pin, ®istered, &sealed) + setup + .mark_sealed(&document, args.pin, ®istered, &sealed) .await - .map_err(|e| Failure::Error(seal_failure(&e, &canonical)))?; + .map_err(|e| OpsError::error(seal_failure(&e, &canonical)))?; let commitment = sealed.commitment(); if args.publish { - let admin = AdminTarget::resolve(args.admin_url, args.admin_token_file)?; + let admin = PublishTarget::resolve(args.admin_url, args.admin_token_file)?; admin .publish(&document) .await - .map_err(|e| Failure::Error(publish_failure(&e, &canonical)))?; - } - if opts.json { - crate::print_json(&serde_json::json!({ - "ok": true, - "topic_id": canonical, - "state": format!("{state:?}").to_lowercase(), - "document_version": version + 1, - "metrics_commitment": commitment, - "primary_value": measured.primary_value, - "published": args.publish, - }))?; - return Ok(()); - } - println!("topic {canonical} sealed and opened."); - println!(" state open"); - println!(" document_version {}", version + 1); - println!(" primary_value {}", measured.primary_value); - println!(" commitment {commitment}"); - if args.publish { - println!(" published yes (the topic's routes and document are live)"); - } else { - println!(" published no (--publish was not given)"); + .map_err(|e| OpsError::error(publish_failure(&e, &canonical)))?; } - println!(); - println!("{}", after_seal(&canonical, args.publish)); - Ok(()) + Ok(SealOutcome { + topic_id: canonical, + document_version: version + 1, + primary_value: measured.primary_value, + metrics_commitment: commitment, + published: args.publish, + }) } /// The `TopicSetup` `mark_sealed` needs: the store, and a VM boundary that is @@ -237,14 +244,14 @@ pub async fn seal(opts: &Options, args: &SealArgs<'_>) -> Result<(), Failure> { /// here instead of quietly running on the control-plane host. fn seal_setup(store: PgRlmStore) -> TopicSetup { TopicSetup { - orchestrator: std::sync::Arc::new(proof_rlm::UnwiredVmOrchestrator), - store: std::sync::Arc::new(store) as std::sync::Arc, + orchestrator: Arc::new(proof_rlm::UnwiredVmOrchestrator), + store: Arc::new(store) as Arc, template: proof_rlm::VmTemplate::from_env(), experiments: proof_rlm::ExperimentPolicy::default(), - owner: std::sync::Arc::new(proof_rlm::StaticOwnerHook( + owner: Arc::new(proof_rlm::StaticOwnerHook( proof_rlm::OwnerDecision::Approve, )), - keys: std::sync::Arc::new(SealKeys), + keys: Arc::new(SealKeys), spend_cap_usd: None, skip_baseline: false, } @@ -295,18 +302,18 @@ fn seal_measurement( async fn resolve_topic( store: &PgRlmStore, topic_id: &str, -) -> Result<(String, u32, TopicDocument), Failure> { +) -> Result<(String, u32, TopicDocument), OpsError> { let resolved = store .resolve_alias(topic_id) .await - .map_err(|e| Failure::Error(format!("resolve {topic_id}: {e}")))?; + .map_err(|e| OpsError::error(format!("resolve {topic_id}: {e}")))?; let canonical = resolved.as_deref().unwrap_or(topic_id); let row = store .latest_topic(canonical) .await - .map_err(|e| Failure::Error(format!("{canonical}: {e}")))?; + .map_err(|e| OpsError::error(format!("{canonical}: {e}")))?; let Some((version, document)) = row else { - return Err(Failure::Error(format!( + return Err(OpsError::error(format!( "no installed topic {topic_id:?}{}. Use `proof-admin topic list` to see the exact \ ids.", alias_note(topic_id, canonical) @@ -323,6 +330,15 @@ fn alias_note(topic_id: &str, canonical: &str) -> String { } } +/// The lifecycle word, matching the wire spelling the document uses. +fn status_word(status: TopicStatus) -> &'static str { + match status { + TopicStatus::Draft => "draft", + TopicStatus::Open => "open", + TopicStatus::Closed => "closed", + } +} + /// What the operator does with the commitment `topic baseline` printed. fn next_seal_steps(topic_id: &str, commitment: &str) -> String { format!( @@ -435,7 +451,7 @@ mod tests { } } - fn document(pin: &ProofPin) -> TopicDocument { + fn document() -> TopicDocument { let mut doc = TopicDocument { id: "tb4".into(), status: TopicStatus::Open, @@ -446,7 +462,6 @@ mod tests { doc.holdout_commitment = "cd".repeat(32); doc.baseline.script_sha256 = "ee".repeat(32); doc.baseline.metrics_commitment.clear(); - let _ = pin; doc } @@ -458,7 +473,7 @@ mod tests { fn the_commitment_we_print_is_the_one_mark_sealed_verifies() { let pin = fixtures::pin(); let row = measured(0.42); - let sealed = seal_measurement(&pin, &document(&pin), &row); + let sealed = seal_measurement(&pin, &document(), &row); assert_eq!(sealed.custom_value, Some(0.42)); assert_eq!( sealed.eval_image_digest, pin.eval_image_digest, @@ -471,7 +486,7 @@ mod tests { ); // The document that carries what we printed verifies. - let mut open = document(&pin); + let mut open = document(); open.baseline.metrics_commitment = sealed.commitment(); sealed .verify(&pin, &open) @@ -485,8 +500,8 @@ mod tests { // And a measurement that is not the measured primary is refused even // when the document agrees with itself. - let other = seal_measurement(&pin, &document(&pin), &measured(0.99)); - let mut open_other = document(&pin); + let other = seal_measurement(&pin, &document(), &measured(0.99)); + let mut open_other = document(); open_other.baseline.metrics_commitment = other.commitment(); assert!(other.verify(&pin, &open_other).is_ok()); assert!( diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py index 1aea38881..5717c36f8 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py @@ -76,9 +76,13 @@ POLICY_FAIL = "fail" POLICY_ZERO = "zero" EXCEPTION_POLICIES = (POLICY_FAIL, POLICY_ZERO) -CONTRACT_TBENCH = "tbench-harbor-v1" CONTRACT_HARBOR_TRIALS = "harbor-trials-v1" -HARBOR_CONTRACTS = (CONTRACT_TBENCH, CONTRACT_HARBOR_TRIALS) +# Legacy alias for the same family, kept because it is a **wire value**: a +# topic signed before the generic id existed pins this in its +# `constraints.params.results_contract`, and a signed document cannot be +# edited. New topics pin `harbor-trials-v1`; nothing here branches on a topic. +CONTRACT_TBENCH = "tbench-harbor-v1" +HARBOR_CONTRACTS = (CONTRACT_HARBOR_TRIALS, CONTRACT_TBENCH) def _fail(msg: str, code: int = 2) -> None: @@ -110,12 +114,17 @@ def results_file_name(pin: str) -> str: def results_contract(pin: str) -> str: - """Harbor / tbench contract id. Unknown pin is fail-closed, never generic.""" - name = (pin or "").strip() or CONTRACT_TBENCH + """Harbor contract id the topic pinned, or the generic one when it pinned none. + + The pin is the topic's (`constraints.params.results_contract`); an absent + pin gets the **generic** Harbor id, never a topic-specific one. An unknown + pin is fail-closed, never silently generic. + """ + name = (pin or "").strip() or CONTRACT_HARBOR_TRIALS if name not in HARBOR_CONTRACTS: _fail( f"results_contract {name!r} is not a Harbor trial contract " - f"({CONTRACT_TBENCH} / {CONTRACT_HARBOR_TRIALS})" + f"({CONTRACT_HARBOR_TRIALS} / {CONTRACT_TBENCH})" ) return name @@ -622,7 +631,7 @@ def build_results( report: dict[str, Any], trials: list[dict[str, Any]], log_tail: str, - contract: str = CONTRACT_TBENCH, + contract: str = CONTRACT_HARBOR_TRIALS, ) -> dict[str, Any]: """Complete Harbor display document. Trials are never truncated here.""" ev = report["evidence"] diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py index 990b81939..3555dc19d 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py @@ -695,7 +695,7 @@ def test_results_json_is_complete_harbor_contract(self) -> None: os.environ.update(saved) self.assertEqual(rc, 0) results = json.loads((root / "results.json").read_text(encoding="utf-8")) - self.assertEqual(results["contract"], "tbench-harbor-v1") + self.assertEqual(results["contract"], "harbor-trials-v1") self.assertEqual(results["topic_id"], "tbench-x0032") self.assertEqual(results["n_scored"], 3) self.assertEqual(len(results["trials"]), 3) @@ -943,7 +943,7 @@ def test_emit_results_from_report_repairs_overlay_that_wrote_report_only(self) - os.environ.update(saved) self.assertEqual(rc, 0) results = json.loads((root / "results.json").read_text(encoding="utf-8")) - self.assertEqual(results["contract"], "tbench-harbor-v1") + self.assertEqual(results["contract"], "harbor-trials-v1") self.assertEqual(results["n_scored"], 2) self.assertEqual(len(results["trials"]), 2) self.assertAlmostEqual(results["primary_value"], 0.0) diff --git a/deploy/scripts/assert-harbor-runner-results-emit.sh b/deploy/scripts/assert-harbor-runner-results-emit.sh index 292eead36..b6d825f7a 100755 --- a/deploy/scripts/assert-harbor-runner-results-emit.sh +++ b/deploy/scripts/assert-harbor-runner-results-emit.sh @@ -29,8 +29,11 @@ need() { } need "$SUMMARIZE" 'def write_results_next_to_report' -need "$SUMMARIZE" 'CONTRACT_TBENCH = "tbench-harbor-v1"' +# The generic contract id is what an un-pinned topic gets; the tbench-named id +# is a legacy wire value (a signed topic may still pin it) and stays accepted. need "$SUMMARIZE" 'CONTRACT_HARBOR_TRIALS = "harbor-trials-v1"' +need "$SUMMARIZE" 'CONTRACT_TBENCH = "tbench-harbor-v1"' +need "$SUMMARIZE" 'name = (pin or "").strip() or CONTRACT_HARBOR_TRIALS' need "$SUMMARIZE" 'results.json first' need "$SUMMARIZE" 'write_results_next_to_report(out, report, trials, log_tail, secrets)' need "$SUMMARIZE" 'atomic_write(out, dumped + "\n")' @@ -64,4 +67,4 @@ if "def write_results_next_to_report" not in src: print("harbor runner tree: summarize writes results.json before report.json") PY -echo "harbor runner tree on tip emits results.json (tbench-harbor-v1)" +echo "harbor runner tree on tip emits results.json (harbor-trials-v1)" From 9515ad3f94fb97039ffa80925be331fdc905bcb5 Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:00:37 +0000 Subject: [PATCH 7/8] fix(gateway): require the Bearer scheme on the forwarded publish route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile (P1): `has_operator_bearer` accepted a bare `Authorization: operator-token`, so the newly forwarded publish route would pass through a header form the contract does not allow. The gateway is the public edge and the challenge's own `admin_ok` accepts a bare token on a **master-local** call, so the two differ on purpose — but the gateway must not forward a form it never documented. `Authorization: Bearer ` with a non-empty token after trimming is now required; a bare value, another scheme, and a lowercase scheme are all a 401 naming what is expected. Tests pin each of those, and the proxy integration test drives the bare-header case against the real router. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- bins/proof-challenge/src/main.rs | 5 ++-- crates/gateway-core/src/admin_route.rs | 36 +++++++++++++++++++++----- crates/gateway/tests/proxy_rr.rs | 19 ++++++++++++++ 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index df44b40ce..72d5404a7 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -26,9 +26,8 @@ use proof_challenge::{ challenge_router, executor_slot, hash_admin_token, parse_holdout_file, AppState, ArtefactVault, BaselineMeasurement, EvalBackend, EvalExecutorOffer, GatewayClient, GatewayClientConfig, HarvestOverrides, InferenceOffer, LiveScorer, MemoryStore, MinerEnvVault, ProofEmitter, - ProofPin, TopicDocument, VmOrchestratorProbe, VmOrchestratorReport, - ARTEFACT_STAGING_DIR_ENV, CHALLENGE_ID, DEFAULT_EMIT_POLL_SECS, MINER_BYOK_DIR_ENV, - SCORING_VERSION, + ProofPin, TopicDocument, VmOrchestratorProbe, VmOrchestratorReport, ARTEFACT_STAGING_DIR_ENV, + CHALLENGE_ID, DEFAULT_EMIT_POLL_SECS, MINER_BYOK_DIR_ENV, SCORING_VERSION, }; use proof_challenge::{InstallJournalSlot, PgInstallJournal}; use proof_eval::{custom_ids_ref, registered_custom, FamilyMux}; diff --git a/crates/gateway-core/src/admin_route.rs b/crates/gateway-core/src/admin_route.rs index dd54e5ab9..501f76c58 100644 --- a/crates/gateway-core/src/admin_route.rs +++ b/crates/gateway-core/src/admin_route.rs @@ -59,17 +59,24 @@ pub fn is_forwardable_admin_route(method: &Method, challenge_id: &str, rest: &st && crate::proxy_detach::normalize_proxy_path(rest) == PUBLISH_ADMIN_PATH } -/// Whether the request carries an operator bearer at all. +/// Whether the request carries a well-formed operator bearer. /// /// Presence only: the gateway does not hold the operator token and must not /// learn it. The challenge compares the hash; this is the cheap floor that /// keeps an anonymous `POST` from reaching the admin route. +/// +/// The scheme is **required**: `Authorization: Bearer `, with a +/// non-empty token after trimming. A bare value (`Authorization: `) is +/// refused here even though the challenge's own `admin_ok` would accept it on +/// a master-local call — the gateway is the public edge, and the one header +/// form it forwards is the documented one. A client that sent a bare token +/// gets a 401 naming the scheme, not a silent pass-through. #[must_use] pub fn has_operator_bearer(headers: &HeaderMap) -> bool { headers .get(header::AUTHORIZATION) .and_then(|v| v.to_str().ok()) - .and_then(|raw| raw.strip_prefix("Bearer ").or(Some(raw))) + .and_then(|raw| raw.strip_prefix("Bearer ")) .is_some_and(|token| !token.trim().is_empty()) } @@ -146,10 +153,11 @@ mod tests { )); } - /// The forwarded route still needs a bearer: the gateway never holds the - /// operator token, it only refuses an anonymous call before the hop. + /// The forwarded route needs a **well-formed** bearer: the gateway never + /// holds the operator token, it only refuses a call that could not be one + /// before the hop. #[test] - fn the_operator_bearer_floor_is_presence_only() { + fn the_operator_bearer_floor_requires_the_scheme() { use axum::http::HeaderValue; let mut headers = HeaderMap::new(); assert!(!has_operator_bearer(&headers)); @@ -162,11 +170,25 @@ mod tests { HeaderValue::from_static("Bearer operator-token"), ); assert!(has_operator_bearer(&headers)); - // A raw token (no scheme) is what `admin_ok` accepts too. + // A bare value is **not** a bearer here, even though the challenge's + // own `admin_ok` accepts one on a master-local call: the gateway is + // the public edge and forwards the documented form only. headers.insert( header::AUTHORIZATION, HeaderValue::from_static("operator-token"), ); - assert!(has_operator_bearer(&headers)); + assert!(!has_operator_bearer(&headers)); + // A different scheme is not this one either. + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static("Basic b3BlcmF0b3I="), + ); + assert!(!has_operator_bearer(&headers)); + // Lowercase scheme is refused too: the contract names one spelling. + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static("bearer operator-token"), + ); + assert!(!has_operator_bearer(&headers)); } } diff --git a/crates/gateway/tests/proxy_rr.rs b/crates/gateway/tests/proxy_rr.rs index ab0c778a0..8c8dc712e 100644 --- a/crates/gateway/tests/proxy_rr.rs +++ b/crates/gateway/tests/proxy_rr.rs @@ -792,6 +792,25 @@ async fn the_operator_publish_route_is_forwarded_with_a_bearer() { .expect("proxy"); assert_eq!(resp.status().as_u16(), 401); + // A bare value (no `Bearer ` scheme) is refused too: the gateway is the + // public edge and forwards the documented form only. The challenge's own + // `admin_ok` accepts a bare token on a master-local call, so this is the + // gateway's floor and not a pass-through of whatever arrived. + let resp = client + .post(format!( + "http://{addr}/challenge/proof/v1/admin/proof/topics" + )) + .header("authorization", "operator-token") + .json(&serde_json::json!({"id": "tb4", "status": "draft"})) + .send() + .await + .expect("proxy"); + assert_eq!( + resp.status().as_u16(), + 401, + "a bare Authorization value must not reach the admin route" + ); + // Every other admin route stays master-local, bearer or not. for rest in [ "v1/admin/proof/executor", From 37fa0920610c03bd9dfd2d5e5453937652cf72d3 Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:55:27 +0000 Subject: [PATCH 8/8] fix(proof): three concurrency/retry defects Greptile found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three were real; each is fixed with a test that fails without the fix. **Reaping a run leaked its in-flight count.** `reap_abandoned` removed a stale pending entry without balancing the `enter` that preceded it, and `on_persisted` could not repair it (the entry it would have used was the one reaped). The topic then stayed at "something is in flight" forever, so `recover_stale` skipped the recovery that unsticks a persisted `evaluating`. Reaping now releases the count with the lease; the new test asserts 1 (not 2) after a reap, then 0, then that a fresh run is admitted. **A failed publish could not be retried.** `topic seal` called `mark_sealed` (moving `baselining → open`) *before* the HTTP publish, so the retry the operator is told to run was refused by the lifecycle the previous attempt had already moved. `seal` now detects "already open under this document" and skips `mark_sealed` for that case only — a draft, a different signature, or a lifecycle anywhere else still goes through it and still gets its refusal. **Two paid submissions shared one VM.** A topic that selects no in-guest runner has a single VM (shared with the cheap jobs) and the KVM host refuses a second concurrent job on one VM, so two submissions arriving at once had one of them fail rather than wait. `RlmScorer` now serializes that path on a per-topic lock held from the first job of the run (the anti-cheat `inspect`, which is also a job on that VM) to the last. An experiment topic takes no lock: each paid job is its own VM, which is what keeps two submissions genuinely parallel there. The test fake now models the host's one-job-per-VM rule (and can be told to slow a job) — without that it could not observe the race at all: the test failed with `vm vm-0 is running a job` when the lock was removed, and passes with it. That check is the point of the fixture, not a convenience. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- bins/proof-admin/src/main.rs | 18 ++-- crates/proof-rlm-scorer/src/scorer.rs | 102 +++++++++++++++++++---- crates/proof-rlm-scorer/tests/rlm_e2e.rs | 101 ++++++++++++++++++++++ crates/proof-rlm/Cargo.toml | 6 +- crates/proof-rlm/src/fixtures_tests.rs | 35 ++++++++ crates/proof-topic-ops/src/seal.rs | 68 +++++++++++++-- crates/proof-topic-setup/src/lib.rs | 2 + 7 files changed, 303 insertions(+), 29 deletions(-) diff --git a/bins/proof-admin/src/main.rs b/bins/proof-admin/src/main.rs index c3d21dd81..798bcf08f 100644 --- a/bins/proof-admin/src/main.rs +++ b/bins/proof-admin/src/main.rs @@ -693,13 +693,21 @@ async fn cmd_seal( "metrics_commitment": outcome.metrics_commitment, "primary_value": outcome.primary_value, "published": outcome.published, + "already_sealed": outcome.already_sealed, })); } - println!("topic {} sealed and opened.", outcome.topic_id); - println!(" state open"); - println!(" document_version {}", outcome.document_version); - println!(" primary_value {}", outcome.primary_value); - println!(" commitment {}", outcome.metrics_commitment); + if outcome.already_sealed { + println!( + "topic {} was already sealed (document version {}); this run only published.", + outcome.topic_id, outcome.document_version + ); + } else { + println!("topic {} sealed and opened.", outcome.topic_id); + println!(" state open"); + println!(" document_version {}", outcome.document_version); + println!(" primary_value {}", outcome.primary_value); + println!(" commitment {}", outcome.metrics_commitment); + } if outcome.published { println!(" published yes (the topic's routes and document are live)"); } else { diff --git a/crates/proof-rlm-scorer/src/scorer.rs b/crates/proof-rlm-scorer/src/scorer.rs index 2e1becff9..c63f68308 100644 --- a/crates/proof-rlm-scorer/src/scorer.rs +++ b/crates/proof-rlm-scorer/src/scorer.rs @@ -107,6 +107,9 @@ pub struct RlmScorer { /// this topic is in flight — which is what makes a persisted `evaluating` /// a *stale* phase rather than another submission's. inflight: Mutex>, + /// Per-topic locks for paid runs that share their topic's **one** VM (a + /// topic with no in-guest runner). See [`Self::shared_vm`]. + shared_vms: Mutex>>>, } fn unwired(custom_id: &str, detail: String) -> EvalError { @@ -184,6 +187,7 @@ impl RlmScorer { locks: Mutex::new(BTreeMap::new()), lease_ttl: DEFAULT_LEASE_TTL, inflight: Mutex::new(BTreeMap::new()), + shared_vms: Mutex::new(BTreeMap::new()), } } @@ -208,6 +212,17 @@ impl RlmScorer { self.pending.lock().map_or(0, |m| m.len()) } + /// Runs of `topic_id` between `SubmissionReceived` and their verdict. + /// + /// `0` means nothing of this topic is in flight, which is what lets the + /// next run treat a persisted `evaluating` / `promoting` as a dead run's + /// phase and recover it. Exposed so a test can pin that a reaped run + /// releases its count (a phantom count would skip that recovery). + #[must_use] + pub fn inflight_len(&self, topic_id: &str) -> usize { + self.inflight(topic_id) + } + fn topic_lock(&self, topic_id: &str) -> Arc> { self.locks .lock() @@ -217,6 +232,24 @@ impl RlmScorer { .clone() } + /// The lock that serializes the paid runs of `topic_id` on its **shared** + /// VM. + /// + /// Taken only by a run whose topic selects no in-guest runner (see + /// [`Self::evaluate`]): such a topic has one VM and no second one to ask + /// for, and the KVM host refuses a second concurrent job on one VM. A + /// topic that selects a runner takes no lock here — each paid job is its + /// own experiment VM, which is what makes two submissions of such a topic + /// genuinely parallel. + fn shared_vm(&self, topic_id: &str) -> Arc> { + self.shared_vms + .lock() + .unwrap_or_else(PoisonError::into_inner) + .entry(topic_id.to_owned()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + } + /// Take the **topic** lock: held only around a write to the topic's /// shared state (the lifecycle, the best pointer), never across a paid /// run. See the type's `Concurrency` docs. @@ -254,21 +287,40 @@ impl RlmScorer { } /// Drop pending runs of `topic_id` whose row never landed within the TTL, - /// releasing the lease they hold. + /// releasing the lease they hold **and their evaluation-phase count**. + /// + /// The count matters as much as the lease: `recover_stale` reads it to + /// decide whether a persisted `evaluating` is a dead run or a live one, so + /// a reaped run that left its count behind would pin the topic at + /// "something is in flight" forever — and no later callback could repair + /// it, because the entry this reaps is the one that would have. Exactly + /// one `enter` is balanced per reaped entry, and `on_persisted` cannot + /// double-count it: `take` and this `remove` race for the same entry, and + /// only the winner calls `leave`. fn reap_abandoned(&self, topic_id: &str) { - let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner); - let stale: Vec = pending - .iter() - .filter(|(_, p)| p.bundle.topic_id == topic_id && p.since.elapsed() >= self.lease_ttl) - .map(|(digest, _)| digest.clone()) - .collect(); - for digest in stale { - pending.remove(&digest); - tracing::warn!( - topic_id, - submission_digest = %digest, - "scored run never persisted within the lease ttl; lease released" - ); + let reaped = { + let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner); + let stale: Vec = pending + .iter() + .filter(|(_, p)| { + p.bundle.topic_id == topic_id && p.since.elapsed() >= self.lease_ttl + }) + .map(|(digest, _)| digest.clone()) + .collect(); + for digest in &stale { + pending.remove(digest); + tracing::warn!( + topic_id, + submission_digest = %digest, + "scored run never persisted within the lease ttl; lease released" + ); + } + stale.len() + }; + // The inflight lock is taken **after** the pending lock is released, + // so the two are never held at once and no ordering rule is needed. + for _ in 0..reaped { + self.leave(topic_id); } } @@ -610,6 +662,28 @@ impl RlmScorer { if let Some(bytes) = artifact_tar { req = req.with_artifact_tar(bytes); } + // The topic's **shared** VM, held for the whole paid run — inspect + // included. + // + // A topic whose signed params select an in-guest runner gets one + // experiment VM per paid job, so two submissions run in parallel by + // construction and take no lock. A topic that selects none has a + // single VM shared with the cheap jobs, and the KVM host refuses a + // second concurrent job on one VM — so this run takes `shared_vm` + // from its first job (the anti-cheat `inspect`, which is *also* a job + // on that VM) to its last, and the second submission waits here + // rather than racing into that refusal. That is what makes a + // submission parallel: either it has its own VM, or it takes its + // turn. + let shared_vm_lock = match req.experiment() { + Ok(Some(_)) => None, + Ok(None) => Some(self.shared_vm(&req.topic_id)), + Err(e) => return Err(EvalError::Backend(format!("experiment binding: {e}"))), + }; + let _shared_vm = match &shared_vm_lock { + Some(lock) => Some(lock.lock().await), + None => None, + }; let inspected = runner .inspect(&req, &rules) .await diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index 2b44dbe18..19985901b 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -1124,6 +1124,107 @@ async fn an_abandoned_run_releases_its_topic_lease_after_the_ttl() { let _ = std::fs::remove_dir_all(&d.root); } +/// A topic whose signed params select **no** in-guest runner has one VM and +/// no second one to ask for, so its paid runs take turns on that VM: the host +/// refuses a second concurrent job on one VM (`409 Busy`), and two +/// submissions arriving at once must wait rather than race into that refusal. +/// +/// This is the other half of "one VM per submission": a submission either has +/// a VM of its own (an experiment topic) or it takes its turn. +/// +/// **Multi-threaded on purpose.** The default `#[tokio::test]` runtime is +/// single-threaded, so two spawned submissions cannot actually interleave and +/// the race this pins would not be observable — the fake's one-job-per-VM +/// check would never see a second entrant. This needs real parallelism. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_topic_without_an_experiment_runner_serializes_its_paid_runs() { + let d = Arc::new(direct("shared-vm", None)); + assert!( + !d.topic + .constraints + .params + .contains_key(proof_experiment::PARAM_RUNNER), + "this topic selects no in-guest runner" + ); + // Long enough that the two submissions genuinely overlap without the + // lock: the fake holds the VM busy for the job's duration. + d.orchestrator.set_job_delay(Duration::from_millis(100)); + + // Two different submissions, both driven at once. Each holds its own + // lease (so neither waits on the other's *row*), but the shared VM's lock + // serializes their **evaluations**. + let a = tokio::spawn({ + let d = d.clone(); + async move { score(&d, "shared-a").await } + }); + let b = tokio::spawn({ + let d = d.clone(); + async move { score(&d, "shared-b").await } + }); + let (ra, rb) = (a.await.unwrap(), b.await.unwrap()); + assert!(ra.is_ok(), "a scores: {ra:?}"); + assert!(rb.is_ok(), "b scores: {rb:?}"); + assert_eq!(paid_runs(&d.orchestrator), 2, "both paid runs happened"); + assert_eq!( + d.orchestrator.created(), + 1, + "a topic with no in-guest runner has exactly one VM" + ); + // The lock is released after each run, so a third submission is admitted + // rather than deadlocked behind them. + let c = tokio::time::timeout(Duration::from_secs(10), score(&d, "shared-c")) + .await + .expect("the shared-VM lock is released after each run") + .expect("c scores"); + assert!((c.harness.custom_value.unwrap() - 0.7).abs() < 1e-12); + let _ = std::fs::remove_dir_all(&d.root); +} + +/// A run whose row never lands must not hold its **own** lease forever, and +/// reaping it must release **both** halves of its state: the lease and the +/// evaluation-phase count. A reaped run that left its count behind would pin +/// the topic at "something is in flight" forever, and `recover_stale` would +/// skip the recovery that unsticks it. +#[tokio::test] +async fn a_reaped_run_releases_its_inflight_count_too() { + let d = direct("ttl-inflight", Some(Duration::ZERO)); + let tid = d.topic.id.clone(); + score(&d, "abandoned").await.expect("scores"); + assert_eq!(d.scorer.pending_len(), 1); + + // The next run reaps the abandoned one on its way in. Both runs' counts + // must be balanced once each has left: the reaped one by the reap, the + // live one by its own persist. + let next = tokio::time::timeout(Duration::from_secs(10), score(&d, "next")) + .await + .expect("the abandoned lease is reaped, not waited on") + .expect("scores"); + assert!((next.harness.custom_value.unwrap() - 0.7).abs() < 1e-12); + assert_eq!(d.scorer.pending_len(), 1, "only the live run is pending"); + assert_eq!( + d.scorer.inflight_len(&tid), + 1, + "the reaped run's count is gone; only the live run's remains" + ); + + d.scorer + .on_persisted(&tid, "digest-next", "pf_0000000000000002", false) + .await; + assert_eq!( + d.scorer.inflight_len(&tid), + 0, + "nothing is in flight, so a later run may recover the phase" + ); + + // And the topic is not stuck: a fresh run is admitted, which is what the + // skipped recovery would have prevented. + let after = tokio::time::timeout(Duration::from_secs(10), score(&d, "after")) + .await + .expect("a topic with nothing in flight is not stuck") + .expect("scores"); + assert!((after.harness.custom_value.unwrap() - 0.7).abs() < 1e-12); + let _ = std::fs::remove_dir_all(&d.root); +} /// Two submissions of **one topic** are two runs, not a queue: each holds its /// own lease, so the second evaluates while the first is still in flight. /// (Its row cannot land first, though — `on_persisted` takes the topic lock — diff --git a/crates/proof-rlm/Cargo.toml b/crates/proof-rlm/Cargo.toml index 3a92c187c..985a6ceed 100644 --- a/crates/proof-rlm/Cargo.toml +++ b/crates/proof-rlm/Cargo.toml @@ -10,7 +10,7 @@ publish = false [features] # Exposes `proof_rlm::fixtures` (fake orchestrator, canned report) to sibling crates' tests. -test-fixtures = [] +test-fixtures = ["dep:tokio"] [dependencies] async-trait = "0.1" @@ -25,6 +25,10 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" thiserror = "2" +# Optional: the fixture orchestrator sleeps between jobs so a concurrency +# test can observe the host's one-job-per-VM rule. The library itself needs +# no runtime. +tokio = { version = "1", default-features = false, features = ["time"], optional = true } tracing = "0.1" [dev-dependencies] diff --git a/crates/proof-rlm/src/fixtures_tests.rs b/crates/proof-rlm/src/fixtures_tests.rs index 6023bed83..9aa494a11 100644 --- a/crates/proof-rlm/src/fixtures_tests.rs +++ b/crates/proof-rlm/src/fixtures_tests.rs @@ -207,6 +207,12 @@ pub struct FakeOrchestrator { experiments: Mutex>, /// Every job with the VM it ran on. runs: Mutex>, + /// VMs with a job running right now, like the host's per-VM job lock. + busy: Mutex>, + /// How long a paid job takes. `ZERO` (the default) runs the whole job + /// without suspending, so no second job can overlap it — a test that + /// needs the one-job-per-VM rule to be *observable* has to set this. + job_delay: Mutex, teardowns: Mutex>, proposed: Mutex>, } @@ -224,6 +230,8 @@ impl FakeOrchestrator { vms: Mutex::new(Vec::new()), experiments: Mutex::new(Vec::new()), runs: Mutex::new(Vec::new()), + busy: Mutex::new(Vec::new()), + job_delay: Mutex::new(std::time::Duration::ZERO), teardowns: Mutex::new(Vec::new()), proposed: Mutex::new(vec![ChecklistRule { id: "rlm_rule".into(), @@ -232,6 +240,11 @@ impl FakeOrchestrator { }) } + /// How long each job takes, so a test can make concurrent jobs overlap. + pub fn set_job_delay(&self, d: std::time::Duration) { + *self.job_delay.lock().unwrap() = d; + } + /// Every job fails inside the guest (`Backend`) until cleared. pub fn set_fail_run(&self, v: bool) { self.fail_run.store(v, Ordering::SeqCst); @@ -359,10 +372,32 @@ impl TopicVmOrchestrator for FakeOrchestrator { self.vms.lock().unwrap().contains(handle), "job on an unknown vm" ); + // The host's own rule: one job per VM at a time. A second concurrent + // job on the same VM is a `Busy`, exactly as `proof-vm-agent` answers + // it — which is what makes a shared-VM paid run need its caller to + // serialize rather than race. + { + let mut busy = self.busy.lock().unwrap(); + if busy.contains(&handle.vm_id) { + return Err(VmError::Backend(format!( + "vm {} is running a job", + handle.vm_id + ))); + } + busy.push(handle.vm_id.clone()); + } self.runs .lock() .unwrap() .push((handle.vm_id.clone(), job.clone())); + // Hold the slot for the job's duration, so a concurrent job on the + // same VM really does see it busy (a test that needs the rule to be + // observable sets `job_delay`; the default is zero). + let delay = *self.job_delay.lock().unwrap(); + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + self.busy.lock().unwrap().retain(|id| id != &handle.vm_id); if self.fail_run.load(Ordering::SeqCst) { return Err(VmError::Backend("guest: injected run failure".into())); } diff --git a/crates/proof-topic-ops/src/seal.rs b/crates/proof-topic-ops/src/seal.rs index 3344de5c7..883959567 100644 --- a/crates/proof-topic-ops/src/seal.rs +++ b/crates/proof-topic-ops/src/seal.rs @@ -111,7 +111,7 @@ impl BaselineReport { pub struct SealOutcome { /// Canonical topic slug. pub topic_id: String, - /// New document version the seal stored. + /// The document version now stored (unchanged on a retry). pub document_version: u32, /// The primary the RLM measured and the document sealed. pub primary_value: f64, @@ -119,6 +119,12 @@ pub struct SealOutcome { pub metrics_commitment: String, /// Whether the open document was published through the admin route. pub published: bool, + /// The seal was already recorded (this run only published). + /// + /// The retry path: the previous run sealed the topic and the publish + /// failed, so the operator re-runs the same command and this one skips + /// `mark_sealed` rather than being refused by the lifecycle it moved. + pub already_sealed: bool, } impl SealOutcome { @@ -208,16 +214,30 @@ pub async fn seal(pool: &sqlx::PgPool, args: &SealArgs<'_>) -> Result = args.registered_custom.iter().map(String::as_str).collect(); - let setup = seal_setup(store); - setup - .mark_sealed(&document, args.pin, ®istered, &sealed) - .await - .map_err(|e| OpsError::error(seal_failure(&e, &canonical)))?; - let commitment = sealed.commitment(); + // + // **Unless the seal already landed.** `mark_sealed` moves the lifecycle + // `baselining → open`, and the publish is a *remote* call that can fail + // after it: the retry an operator is told to run (`--publish` again) would + // otherwise be refused by the lifecycle it already moved. So a topic that + // is already open under *this* document is not re-sealed — the seal is + // recorded, and what remains is the publish. Anything else (an open topic + // under a different document, or a lifecycle that never reached + // `baselining`) still goes through `mark_sealed`, which is what refuses + // it with the reason. + let already_open = already_sealed(&store, &canonical, &document).await?; + if !already_open { + let registered: Vec<&str> = args.registered_custom.iter().map(String::as_str).collect(); + let setup = seal_setup(store); + setup + .mark_sealed(&document, args.pin, ®istered, &sealed) + .await + .map_err(|e| OpsError::error(seal_failure(&e, &canonical)))?; + } + let document_version = if already_open { version } else { version + 1 }; if args.publish { let admin = PublishTarget::resolve(args.admin_url, args.admin_token_file)?; @@ -228,13 +248,43 @@ pub async fn seal(pool: &sqlx::PgPool, args: &SealArgs<'_>) -> Result Result { + let Some((_, latest)) = store + .latest_topic(canonical) + .await + .map_err(|e| OpsError::error(format!("{canonical}: {e}")))? + else { + return Ok(false); + }; + if latest.signature != document.signature || latest.status != TopicStatus::Open { + return Ok(false); + } + let lifecycle = store + .lifecycle(canonical) + .await + .map_err(|e| OpsError::error(format!("{canonical} lifecycle: {e}")))?; + Ok(lifecycle.is_some_and(|lc| lc.state == proof_rlm::RlmState::Open)) +} + /// The `TopicSetup` `mark_sealed` needs: the store, and a VM boundary that is /// deliberately **unwired**. /// diff --git a/crates/proof-topic-setup/src/lib.rs b/crates/proof-topic-setup/src/lib.rs index 133dee778..ec5787318 100644 --- a/crates/proof-topic-setup/src/lib.rs +++ b/crates/proof-topic-setup/src/lib.rs @@ -302,6 +302,8 @@ impl TopicSetup { let job = VmJob::Baseline { request: request.clone(), }; + // The baseline is the only paid job the setup driver runs, so it + // never contends for a topic's shared-VM lock: it passes `None`. let ran = run_paid_job( self.orchestrator.as_ref(), &self.experiments,