diff --git a/Cargo.lock b/Cargo.lock index ed54368c3..8275e4ed4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -438,9 +438,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -448,9 +448,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.43.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -679,10 +679,13 @@ dependencies = [ "chain", "challenge-common", "hex", + "http-body-util", "reqwest 0.12.28", "serde_json", "thiserror 2.0.19", "tokio", + "toml", + "tower", "tracing", "trustroot", ] @@ -4614,9 +4617,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -4679,9 +4682,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -5531,7 +5534,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/bins/bounty-challenge/src/main.rs b/bins/bounty-challenge/src/main.rs index 5b2a28de1..f639e958a 100644 --- a/bins/bounty-challenge/src/main.rs +++ b/bins/bounty-challenge/src/main.rs @@ -21,8 +21,8 @@ use std::time::Duration; use bounty_challenge::{ bounty_router, hash_admin_token, legacy_sim_opt_in_present, resolve_scoring_backend, AppState, - BountyEmitter, BountyStore, GatewayClient, GatewayClientConfig, ScoringBackend, CHALLENGE_ID, - DEFAULT_EMIT_POLL_SECS, SCORING_VERSION, + BountyEmitter, BountyStore, EmitterStatus, GatewayClient, GatewayClientConfig, ScoringBackend, + CHALLENGE_ID, DEFAULT_EMIT_POLL_SECS, SCORING_VERSION, }; use challenge_keys::load_challenge_secret; use clap::Parser; @@ -124,17 +124,26 @@ fn run(cli: &Cli) -> Result<(), String> { 503 and the emitter will pay nobody (the challenge share burns to uid 0) until then" ), } + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + // The emitter owns the read side published on GET /v1/status, so build it + // first and hand the same handle to the HTTP state. A host that wires no + // emitter (missing challenge key) publishes `emitter_wired: false`, which + // is the one condition that also 409s the seal. + let emitter = build_emitter(cli, scoring, sk)?; + let status = emitter + .as_ref() + .map_or_else(|| Arc::new(EmitterStatus::new(false)), |e| e.status()); let state = AppState { store: BountyStore::new(), session_secret: Arc::new(session_secret), scoring, admin_hashes: Arc::new(admin_hashes), + emitter: Some(status), }; - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .map_err(|e| e.to_string())?; - if let Some(emitter) = build_emitter(cli, scoring, sk)? { + if let Some(emitter) = emitter { let poll = Duration::from_secs(cli.emit_poll_secs.max(1)); rt.spawn(emitter.run(poll)); } diff --git a/bins/ctx/src/bounty.rs b/bins/ctx/src/bounty.rs index 8bbc457ca..0435f9fca 100644 --- a/bins/ctx/src/bounty.rs +++ b/bins/ctx/src/bounty.rs @@ -230,6 +230,35 @@ pub async fn status(client: &Client, json_out: bool) -> Result<(), String> { if let Some(v) = reply.body.get("champion_hotkey") { println!("champion_hotkey: {}", compact(v)); } + // `can_score` says this host *may* pay; the emitter says whether it *is*. + // A reachable feed with nothing adjudicated pays nobody, and a miner + // deserves to see which of those they are looking at. + if let Some(v) = reply.body.get("emitter_wired") { + println!("emitter_wired: {}", compact(v)); + } + if let Some(e) = reply.body.get("emitter").filter(|v| v.is_object()) { + println!("last_outcome: {}", compact(&e["last_outcome"])); + println!("last_feed_read: {}", compact(&e["last_feed_read"])); + println!("last_paid: {}", compact(&e["last_paid"])); + if let Some(reason) = e["last_reason"].as_str().filter(|s| !s.is_empty()) { + println!("last_reason: {reason}"); + } + match e["last_outcome"].as_str().unwrap_or("never") { + "unpaid" => println!( + " note: the feed answered and nothing was payable — reports become weight \ + once an operator adjudicates them valid with a severity." + ), + "burned" => println!( + " note: the feed could not be read, so this epoch pays nobody. Not a fault \ + on your side; check again later." + ), + "error" => println!( + " note: the emitter could not post leaves at all (chain, signing, or \ + gateway). An operator has to look at this." + ), + _ => {} + } + } if let Some(q) = reply.body.get("quotas") { println!("quotas: {q}"); } else { diff --git a/crates/bounty-challenge-task/src/lib.rs b/crates/bounty-challenge-task/src/lib.rs index 83aad2071..f89a55489 100644 --- a/crates/bounty-challenge-task/src/lib.rs +++ b/crates/bounty-challenge-task/src/lib.rs @@ -16,6 +16,7 @@ use keystore::{ss58_decode, ss58_encode, BITTENSOR_SS58_PREFIX, KEY_LEN}; use schnorrkel::{signing_context, ExpansionMode, MiniSecretKey, PublicKey, Signature}; use serde::{Deserialize, Serialize}; +use std::sync::Mutex; use thiserror::Error; /// Normative challenge id (trust-root / leaf `challenge_id` string). @@ -206,6 +207,218 @@ pub fn resolve_scoring_backend() -> ScoringBackend { } } +// --- Emitter observability --------------------------------------------------- +// +// The reward linkage is a chain of two halves that fail independently: the +// backend publishes adjudications, and this host turns them into signed +// leaves. `/v1/status` already says whether a feed is *configured*; that is not +// the same as whether weight is being produced. A readable feed that publishes +// nothing payable, or a crowned hotkey that is not in `E`, is an empty payout +// that no local store reading can see. These types are the surface that makes +// it visible instead of leaving it to a log grep. + +/// What one completed emitter tick did. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EmitterOutcomeKind { + /// No tick has completed yet. + Never, + /// The feed published payable rows and they became signed scores. + Scored, + /// The feed was read, but nothing payable became weight this tick. + Unpaid, + /// The feed could not be read; `E` was covered with `ChallengeInternal`. + Burned, + /// A scored epoch was left in place through a feed outage. + Held, + /// The tick could not emit at all (chain, signing, or gateway). + Error, +} + +/// One completed tick, as recorded by [`EmitterStatus::record`]. +#[derive(Debug, Clone, Copy)] +pub struct EmitterTick<'a> { + /// What the tick did. + pub kind: EmitterOutcomeKind, + /// Subnet epoch the leaves were signed for (0 when the tick never got there). + pub epoch: u64, + /// Block `E` was pinned at (0 when unknown). + pub pin_block: u64, + /// Size of `E`. + pub participants: usize, + /// Hotkeys that received a positive score. + pub paid: usize, + /// Whether the backend public feed answered this tick. + pub feed_read: bool, + /// Highest epoch this process has scored (0 = none). + pub scored_epoch: u64, + /// Why nobody was paid, or why the epoch was held. + pub reason: Option<&'a str>, + /// Why the tick emitted nothing at all. + pub error: Option<&'a str>, +} + +/// Read-side view of the emitter, published on `GET /v1/status`. +/// +/// `last_outcome: "unpaid"` with `last_feed_read: true` is the one an operator +/// has to notice: the feed is reachable, the challenge is running, and the +/// epoch still pays nobody. `last_reason` says which half is missing. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EmitterStatusView { + /// Whether this host wired an emitter at all (a missing challenge key + /// leaves `false`, which is the one case that also 409s the seal). + pub wired: bool, + /// Completed ticks since boot. + pub ticks: u64, + /// Outcome of the most recent tick. + pub last_outcome: EmitterOutcomeKind, + /// Epoch of the most recent tick (0 = none yet). + pub last_epoch: u64, + /// Pin block of the most recent tick (0 = none yet). + pub last_pin_block: u64, + /// Size of `E` on the most recent tick. + pub last_participants: u64, + /// Positive leaves on the most recent tick. + pub last_paid: u64, + /// Whether the feed answered on the most recent tick. + pub last_feed_read: bool, + /// Highest epoch this process scored (0 = none). In-process: a restart + /// inside an outage can still burn an epoch that had scores, and the next + /// successful tick supersedes it back. + pub scored_epoch: u64, + /// Why nobody was paid, or why the epoch was held. + pub last_reason: String, + /// Why the tick emitted nothing at all. + pub last_error: String, +} + +/// Shared emitter state. The emitter writes it; `/v1/status` reads it. +/// +/// Everything a reader sees is published under **one** lock, counters +/// included. The per-tick fields describe a single tick and are read as a set +/// (`unpaid` with a feed that was read, `scored` with a positive paid count), +/// so publishing them one atomic at a time would let a concurrent reader pair +/// the new outcome with the previous tick's paid count or reason — a +/// combination no tick ever produced, and the CLI and operators read these +/// fields together to decide which half of the reward path is missing. +/// +/// The counters are in the same lock rather than beside it for the same +/// reason: a reader that saw `ticks: 1` next to the pre-tick `Never` would be +/// looking at two different states, and would have to know that pairing is +/// legal to read the snapshot correctly. One lock makes the whole view a +/// value that some tick actually produced. +#[derive(Debug)] +pub struct EmitterStatus { + wired: bool, + published: Mutex, +} + +/// The complete published state: monotonic counters plus the last tick. +#[derive(Debug, Default)] +struct Published { + ticks: u64, + scored_epoch: u64, + last: LastTick, +} + +/// The fields of one completed tick, published and read as one value. +#[derive(Debug, Clone)] +struct LastTick { + kind: EmitterOutcomeKind, + epoch: u64, + pin_block: u64, + participants: u64, + paid: u64, + feed_read: bool, + reason: String, + error: String, +} + +impl Default for LastTick { + fn default() -> Self { + Self { + kind: EmitterOutcomeKind::Never, + epoch: 0, + pin_block: 0, + participants: 0, + paid: 0, + feed_read: false, + reason: String::new(), + error: String::new(), + } + } +} + +impl EmitterStatus { + /// Fresh status for a host that did or did not wire an emitter. + #[must_use] + pub fn new(wired: bool) -> Self { + Self { + wired, + published: Mutex::new(Published::default()), + } + } + + /// Whether this host wired an emitter. + #[must_use] + pub fn wired(&self) -> bool { + self.wired + } + + /// Record one completed tick as a single replacement, so no reader can + /// observe a mix of this tick and the previous one. + pub fn record(&self, tick: EmitterTick<'_>) { + let mut published = lock(&self.published); + published.ticks = published.ticks.saturating_add(1); + published.scored_epoch = published.scored_epoch.max(tick.scored_epoch); + published.last = LastTick { + kind: tick.kind, + epoch: tick.epoch, + pin_block: tick.pin_block, + participants: count_u64(tick.participants), + paid: count_u64(tick.paid), + feed_read: tick.feed_read, + reason: tick.reason.unwrap_or_default().to_owned(), + error: tick.error.unwrap_or_default().to_owned(), + }; + } + + /// Read-side snapshot for `/v1/status`. + /// + /// The whole snapshot is read under one lock, so the counters and the + /// last-tick fields always describe the same state. Reading them + /// separately would let a reader see `ticks: 1` beside the pre-tick + /// `Never`, which no observer should have to reason about. + #[must_use] + pub fn view(&self) -> EmitterStatusView { + let published = lock(&self.published); + let last = &published.last; + EmitterStatusView { + wired: self.wired, + ticks: published.ticks, + last_outcome: last.kind, + last_epoch: last.epoch, + last_pin_block: last.pin_block, + last_participants: last.participants, + last_paid: last.paid, + last_feed_read: last.feed_read, + scored_epoch: published.scored_epoch, + last_reason: last.reason.clone(), + last_error: last.error.clone(), + } + } +} + +/// A poisoned lock here is a status snapshot, not a consensus input: recover +/// the guard rather than take the challenge down over a display field. +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn count_u64(n: usize) -> u64 { + u64::try_from(n).unwrap_or(u64::MAX) +} + /// Most reports one hotkey may leave awaiting adjudication. /// /// Adjudication is the scarce resource in this challenge: every pending report diff --git a/crates/bounty-challenge/Cargo.toml b/crates/bounty-challenge/Cargo.toml index 9e0628f4f..b479bae3c 100644 --- a/crates/bounty-challenge/Cargo.toml +++ b/crates/bounty-challenge/Cargo.toml @@ -26,8 +26,11 @@ trustroot = { path = "../trustroot" } [dev-dependencies] axum = "0.8" +http-body-util = "0.1" serde_json = "1" tokio = { version = "1", features = ["macros", "net", "rt", "rt-multi-thread", "time"] } +toml = "0.8" +tower = { version = "0.5", features = ["util"] } [lints] workspace = true diff --git a/crates/bounty-challenge/src/backend.rs b/crates/bounty-challenge/src/backend.rs index 2bce52ded..112c44f6b 100644 --- a/crates/bounty-challenge/src/backend.rs +++ b/crates/bounty-challenge/src/backend.rs @@ -48,6 +48,16 @@ pub enum BackendError { /// `/reports` is stable under re-read and must still be refused. #[error("backend public leaderboard and reports do not agree")] Mismatched, + /// The feed answered, and every row it published maps to no payable + /// weight. Not an outage: the backend is reachable and simply has no + /// crowned miner, so this epoch pays nobody and the share burns. + /// + /// It is an error rather than a skip for the same reason a failed read is: + /// an emitter that treats "nothing payable" as a successful tick signs + /// `NotAttempted` for every participant, which claims the challenge chose + /// not to invoke them, and reports a healthy score while paying nobody. + #[error("backend public feed has no payable rows: {0}")] + NoPayableRows(String), } /// How many times one call re-reads the pair of routes looking for two diff --git a/crates/bounty-challenge/src/emit.rs b/crates/bounty-challenge/src/emit.rs index ff2c8f720..581801a11 100644 --- a/crates/bounty-challenge/src/emit.rs +++ b/crates/bounty-challenge/src/emit.rs @@ -16,7 +16,19 @@ //! - **It must still cover `E`.** A paid challenge with no leaves fails D24 //! completeness, so `POST /v1/admin/seal` answers 409 and the epoch seals //! for *no* challenge. Silence here would make an unconfigured bounty host -//! take down relearn's weights too. +//! take down proof's weights too. +//! +//! Reading the feed and finding nothing payable is a *different* failure from +//! not reading it, and it is treated as one. A feed that answers with zero +//! adjudicated rows is reachable, but it still produces no weight, and the +//! honest leaf for that epoch is the same `ChallengeInternal` cover. This case +//! has to be separated out because treating it as a "score" emitted +//! `NotAttempted` for every participant, which is a claim that the challenge +//! *chose* not to invoke them. That claim is false, and it is the worse of the +//! two: `NotAttempted` is not the burn cover, so it seals as a +//! legitimate-looking unpaid epoch while `/v1/status` reports a successful +//! score. An operator has to be able to tell "backend is down" from "backend +//! is up and has crowned nobody". //! //! A failed tick also tries not to overwrite good leaves: once *this process* //! has scored an epoch, a later feed outage inside that same epoch holds @@ -32,6 +44,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; +use bounty_challenge_task::{EmitterOutcomeKind, EmitterStatus, EmitterTick}; use bundle::{NoScoreReasonCode, ScoreOrAbsence}; use chain::{gather_schedule_state, ChainClient}; use challenge_common::{ @@ -46,6 +59,10 @@ use crate::{emission_from_public_snapshot, CHALLENGE_ID_BYTES}; /// Default seconds between emitter ticks. pub const DEFAULT_EMIT_POLL_SECS: u64 = 120; +/// Why a readable feed still paid nobody this tick. +const NO_PAYABLE_ROWS_REASON: &str = + "backend public feed published no payable adjudication (nobody crowned)"; + /// Why a tick could not emit anything at all. #[derive(Debug, Error)] pub enum EmitError { @@ -77,24 +94,45 @@ pub enum EmitOutcome { /// Hotkeys that received a positive score. paid: usize, }, + /// The feed answered but nothing payable became weight, so `E` was covered + /// with `NoScore(ChallengeInternal)`: nobody is paid, the share burns to + /// uid 0, and the bundle can still seal. + /// + /// Distinct from [`Self::Burned`] because the cause is not an outage: the + /// feed is up and still has no crowned hotkey. + Unpaid { + /// Subnet epoch the cover was signed for. + epoch: u64, + /// Block `E` was pinned at. A cover is still signed against a specific + /// participant snapshot, so the pin is as meaningful here as it is on + /// [`Self::Scored`]. + pin_block: u64, + /// Size of `E`. + participants: usize, + /// Why nobody was paid. + reason: String, + }, /// The feed could not be read, so `E` was covered with /// `NoScore(ChallengeInternal)`: nobody is paid, the share burns to uid 0, /// and the bundle can still seal. Burned { /// Subnet epoch the burn set was signed for. epoch: u64, + /// Block `E` was pinned at (see [`Self::Unpaid`]). + pin_block: u64, /// Size of `E`. participants: usize, /// Why the feed was unreadable. reason: String, }, - /// The feed could not be read, but a scored set already stands for this - /// epoch. Overwriting it with a burn would take back a score the backend - /// really did publish. + /// The feed produced no weight, but a scored set already stands for this + /// epoch. Overwriting it with a cover would take back a score the backend + /// really did publish — whether the feed went down or stayed up and + /// stopped publishing the crowned hotkey. Held { /// Epoch whose scored leaves were left in place. epoch: u64, - /// Why the feed was unreadable. + /// Why this tick would have covered the epoch. reason: String, }, } @@ -107,6 +145,7 @@ pub struct BountyEmitter { netuid: u16, backend_base: Option, scored_epoch: AtomicU64, + status: Arc, } impl BountyEmitter { @@ -131,9 +170,23 @@ impl BountyEmitter { .map(|s| s.trim().to_owned()) .filter(|s| !s.is_empty()), scored_epoch: AtomicU64::new(0), + status: Arc::new(EmitterStatus::new(true)), } } + /// Share the read-side status published on `GET /v1/status`. + #[must_use] + pub fn with_status(mut self, status: Arc) -> Self { + self.status = status; + self + } + + /// Read-side status handle (shared with the HTTP state). + #[must_use] + pub fn status(&self) -> Arc { + Arc::clone(&self.status) + } + /// Highest epoch this process scored from the feed (0 = none yet). pub fn scored_epoch(&self) -> u64 { self.scored_epoch.load(Ordering::Relaxed) @@ -144,20 +197,52 @@ impl BountyEmitter { /// # Errors /// See [`EmitError`] — those are the failures that leave `E` uncovered. /// A missing or broken feed is not among them; it is an - /// [`EmitOutcome::Burned`] (or [`EmitOutcome::Held`]) instead. + /// [`EmitOutcome::Burned`] (or [`EmitOutcome::Held`]) instead. A feed that + /// answers without payable rows is [`EmitOutcome::Unpaid`]. pub async fn tick(&self) -> Result { + // Whether the feed answered is tracked here rather than inferred from + // the outcome: a tick that read the feed and then failed at the + // gateway is an error, and reporting `last_feed_read: false` for it + // would send an operator to the backend for a fault that is not there. + let mut feed_read = false; + let outcome = self.tick_inner(&mut feed_read).await; + self.record(&outcome, feed_read); + outcome + } + + /// One tick, before its outcome is published to [`EmitterStatus`]. + async fn tick_inner(&self, feed_read: &mut bool) -> Result { let feed = fetch_public_snapshot(self.backend_base.as_deref()).await; + *feed_read = feed.is_ok(); let pinned = self.expected_set_at_last_epoch()?; let (epoch, pin_block, hotkeys) = pinned; let snapshot = match feed { Ok(s) => s, - Err(e) => return self.cover_without_a_feed(epoch, &hotkeys, &e).await, + Err(e) => { + return self + .cover_without_a_feed(epoch, pin_block, &hotkeys, &e) + .await + } }; let (_plan, leaf_scores) = emission_from_public_snapshot(&hotkeys, &snapshot); let paid = leaf_scores .values() .filter(|s| matches!(s, ScoreOrAbsence::Score { value } if *value > 0)) .count(); + // A readable feed that crowns nobody must not be signed as a scored + // epoch. `NotAttempted` claims the challenge chose not to invoke the + // miner; nobody was crowned, so the honest cover is + // `ChallengeInternal` and the share burns. + if paid == 0 { + return self + .cover_without_a_feed( + epoch, + pin_block, + &hotkeys, + &BackendError::NoPayableRows(NO_PAYABLE_ROWS_REASON.to_owned()), + ) + .await; + } self.submit(epoch, &hotkeys, &leaf_scores).await?; self.scored_epoch.fetch_max(epoch, Ordering::Relaxed); Ok(EmitOutcome::Scored { @@ -168,6 +253,92 @@ impl BountyEmitter { }) } + /// Publish one completed tick to `/v1/status`. + /// + /// `feed_read` is the tick's own record of whether the feed answered, not + /// something inferred from the outcome: an error after a successful read + /// is a gateway problem, and saying otherwise would point an operator at + /// the wrong service. + fn record(&self, outcome: &Result, feed_read: bool) { + let scored_epoch = self.scored_epoch(); + let tick = match outcome { + Ok(EmitOutcome::Scored { + epoch, + pin_block, + participants, + paid, + }) => EmitterTick { + kind: EmitterOutcomeKind::Scored, + epoch: *epoch, + pin_block: *pin_block, + participants: *participants, + paid: *paid, + feed_read, + scored_epoch, + reason: None, + error: None, + }, + Ok(EmitOutcome::Unpaid { + epoch, + pin_block, + participants, + reason, + }) => EmitterTick { + kind: EmitterOutcomeKind::Unpaid, + epoch: *epoch, + pin_block: *pin_block, + participants: *participants, + paid: 0, + feed_read, + scored_epoch, + reason: Some(reason), + error: None, + }, + Ok(EmitOutcome::Burned { + epoch, + pin_block, + participants, + reason, + }) => EmitterTick { + kind: EmitterOutcomeKind::Burned, + epoch: *epoch, + pin_block: *pin_block, + participants: *participants, + paid: 0, + feed_read, + scored_epoch, + reason: Some(reason), + error: None, + }, + // A hold leaves an earlier tick's leaves standing, so it has no + // pin of its own to publish: `last_pin_block` stays 0 rather than + // claiming a block this tick never derived `E` at. + Ok(EmitOutcome::Held { epoch, reason }) => EmitterTick { + kind: EmitterOutcomeKind::Held, + epoch: *epoch, + pin_block: 0, + participants: 0, + paid: 0, + feed_read, + scored_epoch, + reason: Some(reason), + error: None, + }, + Err(e) => EmitterTick { + kind: EmitterOutcomeKind::Error, + epoch: 0, + pin_block: 0, + participants: 0, + paid: 0, + feed_read, + scored_epoch, + reason: None, + error: Some(&e.to_string()), + }, + }; + self.status.record(tick); + } + /// Tick forever. A failed tick is logged and retried; it never falls back /// to a local verdict. pub async fn run(self: Arc, poll: Duration) { @@ -190,12 +361,27 @@ impl BountyEmitter { paid, "bounty leaf set submitted from the backend public feed" ), + Ok(EmitOutcome::Unpaid { + epoch, + pin_block, + participants, + reason, + }) => tracing::warn!( + epoch, + pin_block, + participants, + %reason, + "bounty read the feed and found nothing payable: covered E with \ + ChallengeInternal, so the challenge share burns to uid 0" + ), Ok(EmitOutcome::Burned { epoch, + pin_block, participants, reason, }) => tracing::warn!( epoch, + pin_block, participants, %reason, "bounty could not read the feed: covered E with ChallengeInternal, \ @@ -204,7 +390,7 @@ impl BountyEmitter { Ok(EmitOutcome::Held { epoch, reason }) => tracing::warn!( epoch, %reason, - "bounty could not read the feed; keeping this epoch's scored leaves" + "bounty produced no weight this tick; keeping this epoch's scored leaves" ), Err(e) => tracing::warn!( error = %e, @@ -238,11 +424,18 @@ impl BountyEmitter { Ok((epoch, pin_block, expected.hotkeys())) } - /// Cover `E` when the feed is unreadable: burn, or hold an already-scored - /// epoch. + /// Cover `E` when no weight could be produced: burn, or hold an + /// already-scored epoch. + /// + /// `cause` is either an unreadable feed or [`BackendError::NoPayableRows`] + /// (the feed answered and crowned nobody). Both pay nobody, but only the + /// outage is reported as [`EmitOutcome::Burned`] — an operator reading + /// `/v1/status` has to be able to tell "the backend is down" from "the + /// backend is up and there is nothing to pay". async fn cover_without_a_feed( &self, epoch: u64, + pin_block: u64, hotkeys: &BTreeSet, cause: &BackendError, ) -> Result { @@ -262,9 +455,19 @@ impl BountyEmitter { }) .collect(); self.submit(epoch, hotkeys, &burn).await?; + let participants = hotkeys.len(); + if matches!(cause, BackendError::NoPayableRows(_)) { + return Ok(EmitOutcome::Unpaid { + epoch, + pin_block, + participants, + reason, + }); + } Ok(EmitOutcome::Burned { epoch, - participants: hotkeys.len(), + pin_block, + participants, reason, }) } diff --git a/crates/bounty-challenge/src/lib.rs b/crates/bounty-challenge/src/lib.rs index c53e312af..bc0cd1ac1 100644 --- a/crates/bounty-challenge/src/lib.rs +++ b/crates/bounty-challenge/src/lib.rs @@ -24,7 +24,8 @@ use challenge_common::{emit_signed_leaf_set, Hotkey, LeafEmitError}; pub use backend::{fetch_public_snapshot, public_path, snapshot_from_json, BackendError}; pub use bounty_challenge_task::{ backend_public_url, chat_command_display, legacy_sim_opt_in_present, resolve_scoring_backend, - ScoringBackend, CHALLENGE_ID, CHALLENGE_ID_BYTES as BOUNTY_ID_BYTES, CHAT_COMMAND_PLACEHOLDER, + EmitterOutcomeKind, EmitterStatus, EmitterStatusView, EmitterTick, ScoringBackend, + CHALLENGE_ID, CHALLENGE_ID_BYTES as BOUNTY_ID_BYTES, CHAT_COMMAND_PLACEHOLDER, SCORE_MAX as BOUNTY_SCORE_MAX, SCORING_VERSION, TERMS_TEXT, }; pub use bounty_http::{bounty_router, hash_admin_token, AppState}; diff --git a/crates/bounty-challenge/tests/emit_fail_closed.rs b/crates/bounty-challenge/tests/emit_fail_closed.rs index a01bb578c..2e135d7f9 100644 --- a/crates/bounty-challenge/tests/emit_fail_closed.rs +++ b/crates/bounty-challenge/tests/emit_fail_closed.rs @@ -1,6 +1,6 @@ //! Bounty emission is only as real as the backend feed behind it. //! -//! Three properties are load-bearing and none is visible from a unit test of +//! Five properties are load-bearing and none is visible from a unit test of //! the scorer: //! //! 1. A host that *can* read `{BOUNTY_BACKEND_PUBLIC_URL}/v1/bounty/public/*` @@ -10,28 +10,33 @@ //! `NoScore(ChallengeInternal)`, so the challenge share burns to uid 0 — //! while still covering `E`, because a paid challenge with no leaves fails //! D24 and takes every other challenge's seal down with it. -//! 3. A feed outage inside an already-scored epoch does not take back the -//! scores the backend really did publish. -//! 4. The leaderboard and the reports are separate GETs, so a snapshot is only +//! 3. A feed that answers and crowns nobody is the third case, and it is not +//! either of the first two: reachable, healthy, and worth nothing. It must +//! cover `E` with the same `ChallengeInternal` cover rather than sign +//! `NotAttempted` (which claims the challenge chose not to invoke anyone) +//! or report a successful score. +//! 4. A feed outage, or a readable feed that stops paying, inside an +//! already-scored epoch does not take back the scores the backend really +//! did publish. +//! 5. The leaderboard and the reports are separate GETs, so a snapshot is only //! signed once the feed holds still across both of them. #![forbid(unsafe_code)] #![allow(clippy::unwrap_used, clippy::expect_used)] -use std::net::SocketAddr; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; - use axum::extract::State; use axum::routing::{get, post}; use axum::{Json, Router}; use bounty_challenge::{ - fetch_public_snapshot, BackendError, BountyEmitter, EmitOutcome, GatewayClient, - GatewayClientConfig, + fetch_public_snapshot, BackendError, BountyEmitter, EmitOutcome, EmitterOutcomeKind, + EmitterStatus, EmitterTick, GatewayClient, GatewayClientConfig, }; use chain::{ AxonInfo, ChainClient, ChainError, FakeChain, FakeChainConfig, Metagraph, WeightsTlockPayload, }; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; /// `FakeChain` keeps its call log in a `RefCell`; the emitter needs `Sync`. struct LockedFake(Mutex); @@ -270,6 +275,130 @@ async fn spawn_stable_torn_backend() -> String { serve(app).await } +/// A backend that answers 503 on both routes — the shape of an outage, as +/// opposed to [`spawn_empty_backend`]'s reachable-but-unpaid feed. +async fn spawn_down_backend() -> String { + let app = Router::new().fallback(|| async { axum::http::StatusCode::SERVICE_UNAVAILABLE }); + serve(app).await +} + +/// A backend that starts payable and can be switched to publishing nothing, +/// without changing its URL. That is the "backend stopped paying a hotkey it +/// had already crowned" case: still reachable, no longer payable. +async fn spawn_switchable_backend() -> (String, Arc) { + let payable = Arc::new(AtomicBool::new(true)); + let state = Arc::clone(&payable); + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(move |State(payable): State>| async move { + if payable.load(Ordering::Relaxed) { + Json(leaderboard_json()) + } else { + Json(serde_json::json!({ "items": [] })) + } + }), + ) + .route( + "/v1/bounty/public/reports", + get(move |State(payable): State>| async move { + if payable.load(Ordering::Relaxed) { + Json(reports_json()) + } else { + Json(serde_json::json!({ "items": [] })) + } + }), + ) + .with_state(state); + (serve(app).await, payable) +} + +/// A backend that answers 200 on both routes with nothing published yet: no +/// reports adjudicated, so no leaderboard row to agree with. This is the shape +/// of a backend that is *up* and has crowned nobody — a reachable feed that +/// still produces no weight, which is not the same failure as an outage. +async fn spawn_empty_backend() -> String { + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(|| async { Json(serde_json::json!({ "items": [] })) }), + ) + .route( + "/v1/bounty/public/reports", + get(|| async { Json(serde_json::json!({ "items": [] })) }), + ); + serve(app).await +} + +/// A backend that publishes reports but never adjudicates them: every row is +/// still `pending`, which is not scorable. Reachable, justified-looking, and +/// worth nothing — the trap an operator has to be able to see. +async fn spawn_all_pending_backend() -> String { + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(|| async { + Json(serde_json::json!({ + "items": [{ "hotkey": hex::encode(CHAMPION), "valid_count": 0 }] + })) + }), + ) + .route( + "/v1/bounty/public/reports", + get(|| async { + Json(serde_json::json!({ + "items": [{ + "id": "pending-0", + "hotkey": hex::encode(CHAMPION), + "status": "pending", + "problem_found": "something may be wrong", + "adjudicator": "bounty-adjudicator@cortex", + "justification": "awaiting triage", + "adjudicated_at": "2026-08-30T00:00:00Z", + "created_at": "2026-08-29T00:00:00Z", + }] + })) + }), + ); + serve(app).await +} + +/// A backend whose champion is an unpriced `valid`: adjudicated, justified, +/// and still not creditable, because nobody priced the bug. The crown gate +/// refuses it, so the feed is readable and pays nobody. +async fn spawn_unpriced_backend() -> String { + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(|| async { + Json(serde_json::json!({ + "items": [{ "hotkey": hex::encode(CHAMPION), "valid_count": 3 }] + })) + }), + ) + .route( + "/v1/bounty/public/reports", + get(|| async { + let items: Vec<_> = (0..3) + .map(|i| { + serde_json::json!({ + "id": format!("unpriced-{i}"), + "hotkey": hex::encode(CHAMPION), + "status": "valid", + "problem_found": format!("regression {i}"), + "adjudicator": "bounty-adjudicator@cortex", + "justification": "reproduced on master", + "adjudicated_at": "2026-08-30T00:00:00Z", + "created_at": "2026-08-29T00:00:00Z", + }) + }) + .collect(); + Json(serde_json::json!({ "items": items })) + }), + ); + serve(app).await +} + /// Matching `valid_count` with distinct envelope revisions: Greptile's stable /// torn case (leaderboard A, reports B) where a count-only check would pass. async fn spawn_stable_torn_revision_backend() -> String { @@ -410,10 +539,16 @@ async fn an_unset_backend_url_burns_without_paying_anyone() { match em.tick().await.expect("burn covers E") { EmitOutcome::Burned { epoch, + pin_block, participants, reason, } => { assert_eq!(epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert_eq!( + pin_block, + chain::fake_defaults::LAST_EPOCH_BLOCK, + "a cover is signed against a pinned participant snapshot" + ); assert_eq!(participants, 3); assert!(reason.contains("BOUNTY_BACKEND_PUBLIC_URL"), "{reason}"); } @@ -606,4 +741,518 @@ async fn an_outage_after_a_scored_epoch_holds_instead_of_burning_it() { > 0, "the champion's score must still stand" ); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Held); + assert!(!view.last_feed_read, "an outage is not a read feed"); +} + +// --- The feed is up and pays nobody ----------------------------------------- +// +// A reachable backend with no payable adjudication is the failure mode that +// looks healthy from every other angle: `/health` is up, `can_score` is true, +// the feed answers 200, and the epoch still pays nobody. It must not be signed +// as a scored epoch, and it must not read like an outage either. + +/// An empty feed is reachable and crowns nobody. Signing it as `Scored` would +/// emit `NotAttempted` for every participant — a claim that the challenge +/// chose not to invoke them — and report a healthy tick while paying nothing. +/// The honest leaf is the same `ChallengeInternal` cover a burn uses. +#[tokio::test] +async fn a_readable_feed_with_no_rows_covers_e_instead_of_claiming_a_score() { + let backend = spawn_empty_backend().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + match em.tick().await.expect("cover covers E") { + EmitOutcome::Unpaid { + epoch, + pin_block, + participants, + reason, + } => { + assert_eq!(epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert_eq!( + pin_block, + chain::fake_defaults::LAST_EPOCH_BLOCK, + "a cover is signed against a pinned participant snapshot" + ); + assert_eq!(participants, 3); + assert!(reason.contains("no payable rows"), "{reason}"); + } + other => panic!("a readable feed that pays nobody must not score: {other:?}"), + } + assert_eq!(accepted_count(&accepted), 3); + assert_burn_covers_e(&accepted); + assert_eq!( + em.scored_epoch(), + 0, + "covering E without paying is not a score and must not mark the epoch scored" + ); + // The whole point of the separate outcome: an operator can tell this apart + // from an outage. + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Unpaid); + assert!( + view.last_feed_read, + "the feed answered; this is not an outage" + ); + assert_eq!(view.last_paid, 0); + assert_eq!(view.last_participants, 3); +} + +/// Reports that are published but still `pending` are not adjudications. The +/// feed is up and looks populated, and it is worth exactly nothing. +#[tokio::test] +async fn pending_only_rows_are_reachable_and_unpayable() { + let backend = spawn_all_pending_backend().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("cover covers E"), + EmitOutcome::Unpaid { .. } + )); + assert_burn_covers_e(&accepted); + assert_eq!(em.scored_epoch(), 0); + let view = em.status().view(); + assert!(view.last_feed_read); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Unpaid); +} + +/// A `valid` row nobody priced is adjudicated and justified, and still not +/// creditable: the crown gate refuses it. The feed is readable, so this is an +/// unpaid epoch rather than an outage. +#[tokio::test] +async fn an_unpriced_valid_row_leaves_the_epoch_unpaid_not_burned() { + let backend = spawn_unpriced_backend().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("cover covers E"), + EmitOutcome::Unpaid { .. } + )); + assert_burn_covers_e(&accepted); + assert_eq!(em.scored_epoch(), 0); + assert!(em.status().view().last_feed_read); +} + +/// The direction that protects a real payout: once the feed has crowned +/// someone this epoch, a later readable-but-unpaid tick must not take it back +/// by covering the epoch. This is the "backend stopped paying a hotkey it had +/// already crowned" case, and it holds rather than burns. +#[tokio::test] +async fn a_readable_feed_that_stops_paying_holds_a_scored_epoch() { + let (backend, payable) = spawn_switchable_backend().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("scored"), + EmitOutcome::Scored { paid: 1, .. } + )); + let after_scored = accepted_count(&accepted); + + // The feed stays up, but it no longer publishes anything payable. + payable.store(false, Ordering::Relaxed); + match em.tick().await.expect("hold") { + EmitOutcome::Held { epoch, reason } => { + assert_eq!(epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert!(reason.contains("no payable rows"), "{reason}"); + } + other => panic!("a scored epoch must not be taken back: {other:?}"), + } + assert_eq!( + accepted_count(&accepted), + after_scored, + "holding must post nothing at all" + ); + assert!( + leaf_for(&accepted, CHAMPION)["score_or_absence"]["score"]["value"] + .as_u64() + .unwrap_or_default() + > 0, + "the champion's score must still stand" + ); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Held); + assert!(view.last_feed_read); + assert_eq!( + view.scored_epoch, + chain::fake_defaults::SUBNET_EPOCH_INDEX, + "the hold must not forget the epoch was scored" + ); +} + +/// `/v1/status` has to distinguish "can this host pay" from "is it paying". +/// Before any tick, `wired` is true and nothing has happened yet. +#[tokio::test] +async fn status_reports_a_wired_emitter_that_has_not_ticked_yet() { + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(None, &gateway); + let view = em.status().view(); + assert!(view.wired); + assert_eq!(view.ticks, 0); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Never); + assert!(!view.last_feed_read); + assert_eq!(view.last_paid, 0); + assert_eq!(view.last_pin_block, 0, "no tick has pinned a block yet"); +} + +/// A cover is signed against a pinned participant snapshot, so the status must +/// publish that block rather than report zero. Reporting `last_pin_block: 0` +/// beside a real epoch and participant count hides the block a validator would +/// have to reproduce, and 0 is documented as "no tick yet". +#[tokio::test] +async fn a_cover_publishes_the_block_it_pinned_e() { + for backend in [spawn_empty_backend().await, spawn_down_backend().await] { + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + assert!(matches!( + em.tick().await.expect("cover covers E"), + EmitOutcome::Unpaid { .. } | EmitOutcome::Burned { .. } + )); + let view = em.status().view(); + assert_eq!( + view.last_pin_block, + chain::fake_defaults::LAST_EPOCH_BLOCK, + "a cover pins E at a real block: {view:?}" + ); + assert_eq!(view.last_epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert_eq!(view.last_participants, 3); + } +} + +/// A held tick publishes no pin of its own: the leaves it left standing came +/// from an earlier tick, and claiming a block this tick never derived `E` at +/// would be a fabrication. +#[tokio::test] +async fn a_held_tick_does_not_claim_a_pin_it_did_not_use() { + let (backend, payable) = spawn_switchable_backend().await; + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("scored"), + EmitOutcome::Scored { .. } + )); + assert_eq!( + em.status().view().last_pin_block, + chain::fake_defaults::LAST_EPOCH_BLOCK + ); + + payable.store(false, Ordering::Relaxed); + assert!(matches!( + em.tick().await.expect("hold"), + EmitOutcome::Held { .. } + )); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Held); + assert_eq!(view.last_pin_block, 0, "a hold has no pin of its own"); + assert_eq!(view.last_participants, 0); +} + +/// The published fields describe one tick, never a blend of two. A reader that +/// saw `scored` beside the previous tick's `paid: 0` would be reading a state +/// no tick produced — exactly the combination an operator is told to treat as +/// a fault. +/// +/// The writer alternates between two ticks whose fields differ in *every* +/// position, so a snapshot carrying one tick's outcome with the other's counts +/// is caught. Termination does not depend on which interleaving happens: the +/// loop is bounded by a hard iteration cap and breaks when the writer finishes, +/// and the final assertion runs after `join` on a state the last write fixes. +/// A writer that finishes early therefore ends the test rather than spinning. +#[tokio::test] +async fn status_never_pairs_an_outcome_with_another_ticks_counts() { + const WRITES: u32 = 200_000; + let status = Arc::new(EmitterStatus::new(true)); + let scored = EmitterTick { + kind: EmitterOutcomeKind::Scored, + epoch: 42, + pin_block: 1_000, + participants: 3, + paid: 1, + feed_read: true, + scored_epoch: 42, + reason: None, + error: None, + }; + let unpaid = EmitterTick { + kind: EmitterOutcomeKind::Unpaid, + epoch: 43, + pin_block: 1_360, + participants: 7, + paid: 0, + feed_read: true, + scored_epoch: 43, + reason: Some("nothing payable"), + error: None, + }; + + let writer = { + let status = Arc::clone(&status); + tokio::task::spawn_blocking(move || { + for i in 0..WRITES { + status.record(if i % 2 == 0 { scored } else { unpaid }); + } + }) + }; + + // Read while the writer runs, asserting every snapshot is one whole tick. + // The cap makes this terminate even if the writer is descheduled forever. + let mut snapshots = 0u32; + for _ in 0..WRITES { + let view = status.view(); + match view.last_outcome { + EmitterOutcomeKind::Scored => { + assert_eq!( + ( + view.last_epoch, + view.last_pin_block, + view.last_participants, + view.last_paid + ), + (42, 1_000, 3, 1), + "a scored snapshot must be exactly the scored tick: {view:?}" + ); + assert!(view.last_reason.is_empty(), "{view:?}"); + } + EmitterOutcomeKind::Unpaid => { + assert_eq!( + ( + view.last_epoch, + view.last_pin_block, + view.last_participants, + view.last_paid + ), + (43, 1_360, 7, 0), + "an unpaid snapshot must be exactly the unpaid tick: {view:?}" + ); + assert_eq!(view.last_reason, "nothing payable", "{view:?}"); + } + // `Never` is the pre-tick state, and the whole view is published + // under one lock, so it can only be observed with `ticks: 0` — + // never beside a counter that has already moved. + EmitterOutcomeKind::Never => { + assert_eq!(view.ticks, 0, "Never is only before the first tick"); + assert_eq!(view.last_participants, 0); + assert_eq!(view.last_paid, 0); + assert_eq!(view.last_epoch, 0); + assert_eq!(view.last_pin_block, 0); + assert!(view.last_reason.is_empty()); + } + other => panic!("no other outcome was ever recorded: {other:?}"), + } + snapshots += 1; + if writer.is_finished() { + break; + } + tokio::task::yield_now().await; + } + writer.await.expect("writer"); + assert!(snapshots > 0, "the reader must have observed the status"); + + // Deterministic final state: the last write is an odd index, so it is the + // `unpaid` tick, and the counters must agree with it. + let final_view = status.view(); + assert_eq!(final_view.last_outcome, EmitterOutcomeKind::Unpaid); + assert_eq!(final_view.ticks, u64::from(WRITES)); + assert_eq!(final_view.scored_epoch, 43); + assert_eq!(final_view.last_epoch, 43); + assert_eq!(final_view.last_pin_block, 1_360); + assert_eq!(final_view.last_participants, 7); + assert_eq!(final_view.last_paid, 0); +} + +/// The exact field set of each outcome, read back deterministically. +/// +/// The concurrency test above proves snapshots are consistent; this one proves +/// they are *correct*, which is what a split-publish regression would break +/// without necessarily losing a race on the machine running the tests. +#[tokio::test] +async fn each_outcome_publishes_its_own_fields_and_nothing_else() { + let status = EmitterStatus::new(true); + + // Before any tick: the documented initial state, counters included. + let initial = status.view(); + assert_eq!(initial.last_outcome, EmitterOutcomeKind::Never); + assert_eq!(initial.ticks, 0); + assert_eq!(initial.scored_epoch, 0); + assert_eq!(initial.last_epoch, 0); + assert_eq!(initial.last_pin_block, 0); + assert_eq!(initial.last_participants, 0); + assert_eq!(initial.last_paid, 0); + assert!(!initial.last_feed_read); + + status.record(EmitterTick { + kind: EmitterOutcomeKind::Scored, + epoch: 42, + pin_block: 1_000, + participants: 3, + paid: 2, + feed_read: true, + scored_epoch: 42, + reason: None, + error: None, + }); + let scored = status.view(); + assert_eq!(scored.last_outcome, EmitterOutcomeKind::Scored); + assert_eq!(scored.ticks, 1); + assert_eq!(scored.last_epoch, 42); + assert_eq!(scored.last_pin_block, 1_000); + assert_eq!(scored.last_participants, 3); + assert_eq!(scored.last_paid, 2); + assert!(scored.last_feed_read); + assert_eq!(scored.scored_epoch, 42); + assert!(scored.last_reason.is_empty()); + assert!(scored.last_error.is_empty()); + + // A later cover replaces every field rather than leaving the scored + // tick's counts behind. + status.record(EmitterTick { + kind: EmitterOutcomeKind::Unpaid, + epoch: 43, + pin_block: 1_360, + participants: 9, + paid: 0, + feed_read: true, + scored_epoch: 43, + reason: Some("nothing payable"), + error: None, + }); + let unpaid = status.view(); + assert_eq!(unpaid.last_outcome, EmitterOutcomeKind::Unpaid); + assert_eq!(unpaid.ticks, 2); + assert_eq!(unpaid.last_epoch, 43); + assert_eq!(unpaid.last_pin_block, 1_360); + assert_eq!(unpaid.last_participants, 9); + assert_eq!(unpaid.last_paid, 0, "a cover pays nobody"); + assert_eq!(unpaid.last_reason, "nothing payable"); + assert_eq!(unpaid.scored_epoch, 43); + + // An error tick carries no reason and names the failure. + status.record(EmitterTick { + kind: EmitterOutcomeKind::Error, + epoch: 0, + pin_block: 0, + participants: 0, + paid: 0, + feed_read: false, + scored_epoch: 43, + reason: None, + error: Some("chain: block_hash@1: timeout"), + }); + let errored = status.view(); + assert_eq!(errored.last_outcome, EmitterOutcomeKind::Error); + assert_eq!(errored.ticks, 3); + assert!(errored.last_reason.is_empty()); + assert!(errored.last_error.contains("timeout")); + assert_eq!( + errored.scored_epoch, 43, + "an error does not un-score an epoch" + ); +} + +/// A scored tick publishes what was actually paid, so an operator can see the +/// reward linkage land rather than infer it from a log line. +#[tokio::test] +async fn a_scored_tick_publishes_what_it_paid() { + let backend = spawn_backend().await; + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("scored"), + EmitOutcome::Scored { paid: 1, .. } + )); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Scored); + assert!(view.last_feed_read); + assert_eq!(view.ticks, 1); + assert_eq!(view.last_paid, 1); + assert_eq!(view.last_participants, 3); + assert_eq!(view.last_epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert_eq!(view.scored_epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert!(view.last_reason.is_empty()); + assert!(view.last_error.is_empty()); +} + +/// A tick that could not emit at all must say so on the status surface: this +/// is the case that leaves `E` uncovered and 409s the seal. +#[tokio::test] +async fn a_chain_failure_is_reported_as_an_error_not_a_payout() { + let (gateway, accepted) = spawn_gateway().await; + // Epoch 0 is the "subnet has not run an epoch yet" refusal. + let chain = LockedFake(Mutex::new(FakeChain::new(FakeChainConfig { + netuid: NETUID, + subnet_epoch_index: 0, + ..FakeChainConfig::default() + }))); + let gateway_client = Arc::new( + GatewayClient::new(GatewayClientConfig { + base_url: gateway.clone(), + ..GatewayClientConfig::default() + }) + .expect("gateway client"), + ); + let em = BountyEmitter::new(chain, gateway_client, [7u8; 32], NETUID, None); + + let err = em.tick().await.expect_err("epoch 0 cannot emit"); + assert!( + matches!(err, bounty_challenge::EmitError::EpochZero), + "{err}" + ); + assert_eq!(accepted_count(&accepted), 0, "nothing was posted"); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Error); + assert!(view.last_error.contains("epoch 0"), "{:?}", view.last_error); + assert_eq!(view.ticks, 1); +} + +/// A tick that read the feed and then failed at the gateway is an error, but +/// it is *not* a backend outage. Reporting `last_feed_read: false` there would +/// send an operator to the wrong service — the live check that found this +/// showed exactly that shape against a healthy stand-in feed. +#[tokio::test] +async fn a_gateway_failure_after_a_read_still_reports_the_feed_as_read() { + let backend = spawn_backend().await; + // Port 1 is closed: the feed is fine, the gateway is not. + let em = emitter(Some(backend), "http://127.0.0.1:1"); + + let err = em + .tick() + .await + .expect_err("a dead gateway cannot accept leaves"); + assert!( + matches!(err, bounty_challenge::EmitError::Submit(_)), + "the failure must be the submit, not the read: {err}" + ); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Error); + assert!( + view.last_feed_read, + "the feed answered; blaming the backend would be wrong" + ); + assert!( + view.last_error.contains("gateway"), + "the error must name the gateway: {:?}", + view.last_error + ); + assert_eq!(view.last_paid, 0); +} + +/// The mirror image: a tick that never reached the feed must not claim it did. +#[tokio::test] +async fn a_failed_read_is_not_reported_as_a_read_feed() { + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(Some("http://127.0.0.1:1".to_owned()), &gateway); + + assert!(matches!( + em.tick().await.expect("cover E"), + EmitOutcome::Burned { .. } + )); + let view = em.status().view(); + assert!(!view.last_feed_read); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Burned); } diff --git a/crates/bounty-challenge/tests/rewards_linkage.rs b/crates/bounty-challenge/tests/rewards_linkage.rs new file mode 100644 index 000000000..818932b5a --- /dev/null +++ b/crates/bounty-challenge/tests/rewards_linkage.rs @@ -0,0 +1,775 @@ +//! The full Bounty reward linkage: a paired miner's report becomes weight. +//! +//! Every other test in this crate stops at one link. This one walks the whole +//! chain, because each link can pass on its own while the chain pays nobody: +//! +//! ```text +//! miner hotkey ──pair──▶ Chat account ──report──▶ operator adjudicate +//! │ │ +//! │ CortexLM/backend publishes +//! │ │ +//! └──metagraph `E`──◀── leaf emitter ◀──public feed +//! │ +//! POST /v1/weights/raw +//! │ +//! admin seal ──▶ GET /v1/weights/latest (sealed) +//! ``` +//! +//! Two properties are the point of the file: +//! +//! 1. **The pairing is not decorative.** The hotkey that signs the pairing +//! challenge is the hotkey the published row credits, and it is the hotkey +//! that gets the positive leaf. A miner who cannot pair, or whose report +//! lands under a different key, is never paid — so the test signs a real +//! sr25519 challenge and drives the real HTTP routes. +//! 2. **Bounty cannot take the subnet down with it.** Bounty holds a paid +//! trust-root row, so an epoch where it produces no weight must still cover +//! `E`; otherwise D24 fails and `POST /v1/admin/seal` 409s for *every* +//! challenge, including the one that did score. The seal at the end of this +//! file proves both directions in one bundle: bounty pays its champion, and +//! a challenge that read nothing still seals. +//! +//! Nothing here reads the real CortexLM/backend. The feed is a stand-in that +//! serves the two public routes; that is exactly what the operator points +//! `BOUNTY_BACKEND_PUBLIC_URL` at, and the DTO is the published contract. + +#![forbid(unsafe_code)] +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; + +use axum::body::Body; +use axum::extract::State; +use axum::http::{Request, StatusCode}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use bounty_challenge::{ + bounty_router, hash_admin_token, AppState, BountyEmitter, BountyStore, EmitOutcome, + EmitterOutcomeKind, GatewayClient, GatewayClientConfig, ScoringBackend, +}; +use bounty_challenge_task::{ + hotkey_hex, hotkey_ss58, pairing_code, public_from_mini_secret, sign_pair_challenge, + PairChallenge, +}; +use bundle::{ + build_sealed_bundle, make_signed_leaf, verify_bundle, LeafV1, LocalTrustRoot, + NoScoreReasonCode, ScoreOrAbsence, SealParams, +}; +use chain::{ChainClient, ChainError, FakeChain, FakeChainConfig, Metagraph}; +use challenge_common::{emit_signed_leaf_set, public_key_from_secret, verify_leaf_sig}; +use http_body_util::BodyExt; +use tokio::net::TcpListener; +use tower::ServiceExt; +use trustroot::{ + measurements_digest, ChallengeEntry, ChallengesBody, MeasurementsBody, ParticipantPolicy, + BPS_DENOM, +}; + +/// Netuid the fake chain serves. +const NETUID: u16 = 541; + +/// The miner's hotkey mini-secret. The *hotkey* is its sr25519 public key, so +/// the pairing signature, the published row, and the metagraph entry are all +/// the same key — which is the property this file exists to check. +const MINER_SECRET: [u8; 32] = [0xA1; 32]; + +/// A metagraph hotkey that filed nothing. It still has to appear in `E`. +const SILENT: [u8; 32] = [0xC3; 32]; + +/// The miner's public hotkey, as the metagraph and the pairing both see it. +fn miner_hotkey() -> [u8; 32] { + public_from_mini_secret(&MINER_SECRET).expect("miner pk") +} + +/// Challenge leaf-signing keys (throwaway, in-test only). +const BOUNTY_SK: [u8; 32] = [0x11; 32]; +const PROOF_SK: [u8; 32] = [0x22; 32]; +const GATEWAY_SK: [u8; 32] = [0x33; 32]; + +/// `FakeChain` keeps its call log in a `RefCell`; the emitter needs `Sync`. +struct LockedFake(Mutex); + +macro_rules! delegate { + (fn $name:ident(&self) -> $ret:ty) => { + fn $name(&self) -> $ret { + self.0.lock().expect("lock").$name() + } + }; + (fn $name:ident(&self, $($arg:ident : $t:ty),*) -> $ret:ty) => { + fn $name(&self, $($arg: $t),*) -> $ret { + self.0.lock().expect("lock").$name($($arg),*) + } + }; +} + +impl ChainClient for LockedFake { + delegate!(fn current_block(&self) -> Result); + delegate!(fn block_hash(&self, n: u64) -> Result<[u8; 32], ChainError>); + delegate!(fn metagraph_at(&self, block_hash: &[u8; 32]) -> Result); + delegate!(fn subnet_owner_hotkey(&self, netuid: u16) -> Result, ChainError>); + delegate!(fn axon(&self, netuid: u16, hotkey: &[u8]) -> Result, ChainError>); + delegate!(fn axons(&self, netuid: u16) -> Result, chain::AxonInfo)>, ChainError>); + delegate!(fn commit_reveal_enabled(&self, netuid: u16) -> Result); + delegate!(fn commit_reveal_version(&self, netuid: u16) -> Result); + delegate!(fn tempo(&self, netuid: u16) -> Result); + delegate!(fn reveal_period_epochs(&self, netuid: u16) -> Result); + delegate!(fn block_time(&self) -> Result); + delegate!(fn last_epoch_block(&self, netuid: u16) -> Result); + delegate!(fn pending_epoch_at(&self, netuid: u16) -> Result); + delegate!(fn subnet_epoch_index(&self, netuid: u16) -> Result); + delegate!(fn blocks_since_last_step(&self, netuid: u16) -> Result); + delegate!(fn submit_timelocked_weights( + &self, + mecid: u8, + payload: chain::WeightsTlockPayload, + reveal_round: u64 + ) -> Result<(), ChainError>); + delegate!(fn set_weights( + &self, + netuid: u16, + uids: Vec, + values: Vec, + version_key: u64 + ) -> Result<(), ChainError>); +} + +fn fake_chain() -> FakeChain { + FakeChain::new(FakeChainConfig { + netuid: NETUID, + hotkeys: vec![miner_hotkey().to_vec(), SILENT.to_vec()], + ..FakeChainConfig::default() + }) +} + +// --- the miner side: pair, report, adjudicate (real HTTP routes) ------------- + +/// A pairing payload signed by the miner's own hotkey, over the canonical +/// challenge string. `ctx bounty pair` builds exactly this shape. +fn signed_pair_payload(secret: &[u8; 32], account_id: &str) -> serde_json::Value { + let pk = public_from_mini_secret(secret).expect("pk"); + let challenge = PairChallenge { + account_id: account_id.into(), + nonce: "0123456789abcdef".into(), + exp: 2_000_000_000, + }; + let encoded = challenge.encode().expect("encode"); + let sig = sign_pair_challenge(secret, &encoded).expect("sign"); + // The code a miner pastes into Chat carries the same signature. + let code = pairing_code(&encoded, &hex::encode(sig), &hotkey_ss58(&pk)); + assert!(code.contains(&hotkey_ss58(&pk))); + serde_json::json!({ + "account_id": challenge.account_id, + "hotkey": hotkey_ss58(&pk), + "nonce": challenge.nonce, + "exp": challenge.exp, + "signature": hex::encode(sig), + "terms_accepted": true, + }) +} + +/// A report body that clears the substance floor. +fn report_body(session: &str) -> serde_json::Value { + serde_json::json!({ + "session": session, + "title": "seal returns 500 on an empty bundle", + "body": "POST /v1/admin/seal answers 500 when the bundle has no leaves, \ + instead of the documented 400. Observed on master at commit tip.", + "repro_steps": "curl the seal route with no leaves posted and watch it 500", + }) +} + +async fn json_req( + app: &Router, + method: &str, + uri: &str, + body: serde_json::Value, + auth: Option<&str>, +) -> (StatusCode, serde_json::Value) { + let mut b = Request::builder().method(method).uri(uri); + if let Some(a) = auth { + b = b.header(axum::http::header::AUTHORIZATION, format!("Bearer {a}")); + } + let req = b + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("req"); + let resp = app.clone().oneshot(req).await.expect("resp"); + let status = resp.status(); + let bytes = resp.into_body().collect().await.expect("body").to_bytes(); + let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap_or(serde_json::json!({})); + (status, v) +} + +/// Drive the real challenge routes: pair a hotkey, file a report, adjudicate it +/// `valid` at `severity`. Returns the adjudicated report body. +async fn pair_report_adjudicate(operator_token: &str) -> serde_json::Value { + let app = bounty_router(AppState { + store: BountyStore::new(), + session_secret: Arc::new(b"test-session-secret".to_vec()), + scoring: ScoringBackend::BackendPublic, + admin_hashes: Arc::new(vec![hash_admin_token(operator_token)]), + emitter: None, + }); + + let (st, paired) = json_req( + &app, + "POST", + "/v1/pair", + signed_pair_payload(&MINER_SECRET, "acct-miner-rewards"), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "pair: {paired}"); + // The binding is the miner's own hotkey, not something the host invented. + assert_eq!(paired["miner_hotkey"], hotkey_hex(&miner_hotkey())); + let session = paired["session"].as_str().expect("session"); + + let (st, created) = json_req(&app, "POST", "/v1/reports", report_body(session), None).await; + assert_eq!(st, StatusCode::CREATED, "report: {created}"); + assert_eq!(created["state"], "pending"); + + let (st, adjudicated) = json_req( + &app, + "POST", + "/v1/admin/adjudicate", + serde_json::json!({ + "report_id": created["id"], + "verdict": "valid", + "severity": "major", + }), + Some(operator_token), + ) + .await; + assert_eq!(st, StatusCode::OK, "adjudicate: {adjudicated}"); + assert_eq!(adjudicated["adjudication"], "valid"); + assert_eq!(adjudicated["severity"], "major"); + adjudicated +} + +// --- the backend side: the published feed ------------------------------------ + +/// The backend's published view of the adjudicated report above. +/// +/// This is the handoff the whole challenge depends on: Cortex adjudicates +/// internally, CortexLM/backend publishes, and this host reads the published +/// rows. If the published row names a different hotkey than the one that +/// paired, the miner is paid nothing — which is why the assertion at the end +/// of the test is on `MINER`'s leaf rather than on "a leaf scored". +fn published_row(severity: &str) -> serde_json::Value { + serde_json::json!({ + "id": "report-1", + "hotkey": hotkey_ss58(&miner_hotkey()), + "status": "valid", + "severity": severity, + "problem_found": "seal returns 500 on an empty bundle", + "adjudicator": "bounty-adjudicator@cortex", + "justification": "reproduced on master at commit tip", + "adjudicated_at": "2026-09-01T00:00:00Z", + "created_at": "2026-09-01T00:00:00Z", + }) +} + +/// One justified, priced finding so the miner clears `MIN_HOLDOUT_DECIDED`. +fn payable_rows() -> Vec { + (1..=3) + .map(|i| { + let mut row = published_row("major"); + row["id"] = serde_json::json!(format!("report-{i}")); + row["problem_found"] = serde_json::json!(format!("regression {i} on the seal path")); + row + }) + .collect() +} + +/// Stand-in for `{BOUNTY_BACKEND_PUBLIC_URL}`: the two public routes. +/// +/// The leaderboard row carries the real `valid_count` for the published +/// reports. A count that did not match would be refused as a torn pair before +/// it could ever score, so the stand-in has to publish one coherent revision — +/// which is also what the live backend does. +async fn spawn_public_feed(rows: Vec) -> String { + let rows = Arc::new(rows); + let leaderboard = Arc::clone(&rows); + let reports = Arc::clone(&rows); + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(move || { + let rows = Arc::clone(&leaderboard); + async move { + let valid = rows + .iter() + .filter(|r| r["status"] == serde_json::Value::String("valid".to_owned())) + .count(); + let items = if rows.is_empty() { + Vec::new() + } else { + vec![serde_json::json!({ + "hotkey": rows[0]["hotkey"], + "valid_count": valid, + })] + }; + Json(serde_json::json!({ "items": items })) + } + }), + ) + .route( + "/v1/bounty/public/reports", + get(move || { + let rows = Arc::clone(&reports); + async move { Json(serde_json::json!({ "items": *rows })) } + }), + ); + serve(app).await +} + +/// A feed that answers 503 on both routes — the shape of a backend outage. +async fn spawn_down_feed() -> String { + let app = Router::new().fallback(|| async { StatusCode::SERVICE_UNAVAILABLE }); + serve(app).await +} + +async fn serve(app: Router) -> String { + let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("bind"); + let addr = listener.local_addr().expect("addr"); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + format!("http://{addr}") +} + +/// Leaves the mock gateway accepted, in arrival order. +type Accepted = Arc>>; + +/// Stand-in for the master gateway's `POST /v1/weights/raw`. +async fn spawn_gateway() -> (String, Accepted) { + let accepted: Accepted = Arc::new(Mutex::new(Vec::new())); + let app = Router::new() + .route( + "/v1/weights/raw", + post( + |State(seen): State, Json(body): Json| async move { + seen.lock().expect("lock").push(body); + StatusCode::ACCEPTED + }, + ), + ) + .with_state(Arc::clone(&accepted)); + (serve(app).await, accepted) +} + +/// Rebuild the `LeafV1` the gateway received, so the seal below operates on +/// the exact leaves that were posted rather than a parallel re-derivation. +fn leaf_from_accepted(v: &serde_json::Value) -> LeafV1 { + let soa = match &v["score_or_absence"] { + s if s.get("score").is_some() => ScoreOrAbsence::Score { + value: s["score"]["value"].as_u64().expect("score value"), + }, + s => { + let reason = u8::try_from(s["no_score"]["reason"].as_u64().expect("reason")) + .expect("reason fits u8"); + let reason = match reason { + 0 => NoScoreReasonCode::NotAttempted, + 1 => NoScoreReasonCode::Timeout, + 2 => NoScoreReasonCode::InvalidResponse, + 3 => NoScoreReasonCode::AttestationNotVerified, + 4 => NoScoreReasonCode::MinerError, + 5 => NoScoreReasonCode::RateLimited, + 6 => NoScoreReasonCode::ChallengeInternal, + other => panic!("unexpected absence reason {other}"), + }; + ScoreOrAbsence::NoScore { reason } + } + }; + let mut miner_hotkey = [0u8; 32]; + miner_hotkey.copy_from_slice( + &hex::decode(v["miner_hotkey"].as_str().expect("miner_hotkey")).expect("hex"), + ); + let mut challenge_sig = [0u8; 64]; + challenge_sig.copy_from_slice( + &hex::decode(v["challenge_sig"].as_str().expect("challenge_sig")).expect("hex"), + ); + LeafV1 { + challenge_id: v["challenge_id"].as_str().expect("challenge_id").into(), + miner_hotkey, + epoch: v["epoch"].as_u64().expect("epoch"), + score_or_absence: soa, + challenge_sig, + } +} + +fn emitter(backend: Option, gateway_url: &str) -> BountyEmitter { + let gateway = Arc::new( + GatewayClient::new(GatewayClientConfig { + base_url: gateway_url.to_owned(), + ..GatewayClientConfig::default() + }) + .expect("gateway client"), + ); + BountyEmitter::new( + LockedFake(Mutex::new(fake_chain())), + gateway, + BOUNTY_SK, + NETUID, + backend, + ) +} + +/// The trust root a validator loads from disk: bounty 2000, proof 8000. +fn trust_root() -> LocalTrustRoot { + LocalTrustRoot { + challenges: ChallengesBody { + challenges: vec![ + ChallengeEntry { + id: b"bounty".to_vec(), + public_key: public_key_from_secret(&BOUNTY_SK).expect("bounty pk"), + emission_share_bps: 2000, + policy: ParticipantPolicy::AllMetagraphHotkeys, + }, + ChallengeEntry { + id: b"proof".to_vec(), + public_key: public_key_from_secret(&PROOF_SK).expect("proof pk"), + emission_share_bps: 8000, + policy: ParticipantPolicy::AllMetagraphHotkeys, + }, + ], + }, + measurements_digest: measurements_digest(&MeasurementsBody::default()), + } +} + +/// Cover `E` for a challenge that produced no weight, exactly as that +/// challenge's own emitter does when it cannot score. +fn cover_with_noscore( + sk: &[u8; 32], + challenge_id: &[u8], + epoch: u64, + hotkeys: &BTreeSet<[u8; 32]>, + reason: NoScoreReasonCode, +) -> BTreeMap<[u8; 32], LeafV1> { + let scores: BTreeMap<[u8; 32], ScoreOrAbsence> = hotkeys + .iter() + .map(|h| (*h, ScoreOrAbsence::NoScore { reason })) + .collect(); + emit_signed_leaf_set(sk, challenge_id, epoch, hotkeys, &scores).expect("cover E") +} + +fn expected_set() -> BTreeSet<[u8; 32]> { + let mut e = BTreeSet::new(); + e.insert(miner_hotkey()); + e.insert(SILENT); + e +} + +// --- the tests --------------------------------------------------------------- + +/// The happy path, end to end: pair → report → adjudicate → publish → leaves. +/// +/// The assertion that matters is that `MINER` — the hotkey that signed the +/// pairing challenge — is the one holding a positive leaf. Every intermediate +/// step could be green while the credit lands on a different key, and this is +/// the only place that catches it. +#[tokio::test] +async fn a_paired_miners_adjudicated_report_becomes_a_scored_leaf() { + // 1. The miner pairs and files; an operator adjudicates. + let adjudicated = pair_report_adjudicate("op-token").await; + assert_eq!(adjudicated["miner_hotkey"], hotkey_hex(&miner_hotkey())); + + // 2. The backend publishes that adjudication on the public feed. + let feed = spawn_public_feed(payable_rows()).await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(feed), &gateway); + + // 3. The emitter turns the published rows into signed leaves for `E`. + let epoch = match em.tick().await.expect("tick") { + EmitOutcome::Scored { + epoch, + participants, + paid, + .. + } => { + assert_eq!(participants, 2, "every hotkey in E needs a leaf"); + assert_eq!(paid, 1, "exactly the adjudicated miner was paid"); + epoch + } + other => panic!("a readable, payable feed must score: {other:?}"), + }; + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Scored); + assert_eq!(view.last_paid, 1); + assert!(view.last_feed_read); + + // 4. The leaf the gateway received verifies under the trust-root key, and + // it is `MINER`'s — the hotkey that signed the pairing challenge. + let bounty_pk = public_key_from_secret(&BOUNTY_SK).expect("pk"); + let rows = accepted.lock().expect("lock").clone(); + assert_eq!(rows.len(), 2); + let miner_leaf = rows + .iter() + .find(|v| v["miner_hotkey"] == serde_json::Value::String(hotkey_hex(&miner_hotkey()))) + .expect("a leaf for the paired miner"); + let leaf = leaf_from_accepted(miner_leaf); + verify_leaf_sig(&leaf, &bounty_pk).expect("the sealed leaf must verify under the trust root"); + assert_eq!(leaf.challenge_id, b"bounty"); + assert_eq!(leaf.epoch, epoch); + assert!( + matches!(leaf.score_or_absence, ScoreOrAbsence::Score { value } if value > 0), + "the adjudicated report must pay: {leaf:?}" + ); + + // 5. The hotkey that filed nothing is explicit, never silently omitted — + // an omission here is what fails D24 at the seal. + let silent_leaf = rows + .iter() + .find(|v| v["miner_hotkey"] == serde_json::Value::String(hotkey_hex(&SILENT))) + .expect("a leaf for the silent hotkey"); + assert_eq!( + silent_leaf["score_or_absence"]["no_score"]["reason"], 0, + "NotAttempted: the challenge did invoke E, this hotkey just has no rows" + ); +} + +/// The bundle a validator actually fetches: bounty's paid leaf and proof's +/// no-score cover seal together. +/// +/// This is the cross-challenge property. Bounty holds a paid trust-root row, so +/// if its emitter ever left `E` uncovered the seal would 409 for *proof* too. +/// The test asserts both halves at once — the paid vector and the fact that a +/// challenge with nothing to pay is not a reason to fail the epoch. +#[tokio::test] +async fn a_paid_bounty_seals_alongside_a_challenge_that_scored_nothing() { + let feed = spawn_public_feed(payable_rows()).await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(feed), &gateway); + let epoch = match em.tick().await.expect("tick") { + EmitOutcome::Scored { epoch, .. } => epoch, + other => panic!("expected a score: {other:?}"), + }; + + let e = expected_set(); + let mut leaves: Vec = accepted + .lock() + .expect("lock") + .iter() + .map(leaf_from_accepted) + .collect(); + // Proof had nothing to score, so it covers `E` the same way bounty does on + // an outage. Bounty's leaves are already in the set. + leaves.extend( + cover_with_noscore( + &PROOF_SK, + b"proof", + epoch, + &e, + NoScoreReasonCode::ChallengeInternal, + ) + .into_values(), + ); + + let trust = trust_root(); + let chain = fake_chain(); + let block_b = chain::fake_defaults::LAST_EPOCH_BLOCK; + let bundle = build_sealed_bundle( + &chain, + &trust, + leaves, + &SealParams { + epoch, + netuid: NETUID, + block_b, + gateway_secret: GATEWAY_SK, + }, + ) + .expect("D24 completeness holds: both paid challenges covered E"); + + // A validator's own verification path, against the same trust root. + verify_bundle(&bundle, &chain, &trust).expect("the bundle a validator fetches must verify"); + + // Emission shares are the two live challenges, at the configured split. + let shares: Vec<(String, u16)> = bundle + .body + .emission_shares + .iter() + .map(|(id, bps)| (String::from_utf8_lossy(id).into_owned(), *bps)) + .collect(); + assert_eq!( + shares, + vec![("bounty".to_owned(), 2000), ("proof".to_owned(), 8000)], + "the sealed split is the trust root's, not a default" + ); + + // Bounty's 2000 bps went to the miner who filed the report; proof's 8000 + // bps burned to uid 0 because it had nothing payable. The point is that the + // vector exists at all: an uncovered `E` would have 409'd before this. + let miner_uid = bundle + .body + .uid_map + .iter() + .find(|(h, _)| *h == miner_hotkey()) + .map(|(_, uid)| *uid) + .expect("the miner is in the sealed metagraph"); + let miner_weight = bundle + .body + .final_vector + .iter() + .find(|(uid, _)| *uid == miner_uid) + .map(|(_, w)| *w) + .expect("the miner has a weight"); + assert!( + miner_weight > 0, + "the adjudicated miner must hold weight: {:?}", + bundle.body.final_vector + ); + let silent_uid = bundle + .body + .uid_map + .iter() + .find(|(h, _)| *h == SILENT) + .map(|(_, uid)| *uid) + .expect("the silent hotkey is in the sealed metagraph"); + assert!( + !bundle + .body + .final_vector + .iter() + .any(|(uid, w)| *uid == silent_uid && *w > 0), + "a hotkey that filed nothing must not be paid: {:?}", + bundle.body.final_vector + ); +} + +/// The fail-closed direction, at the seal: a bounty host that cannot read the +/// feed still seals the epoch, and still pays nobody. +/// +/// Both halves are required. Paying nobody is the honest outcome; covering `E` +/// is what keeps the 409 off proof's seal. A host that did neither would take +/// the whole subnet's weights down with it. +#[tokio::test] +async fn an_unreadable_feed_pays_nobody_without_breaking_the_seal() { + let down = spawn_down_feed().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(down), &gateway); + + let epoch = match em.tick().await.expect("cover E") { + EmitOutcome::Burned { + epoch, + pin_block, + participants, + reason, + } => { + assert_eq!(participants, 2); + assert_eq!(pin_block, chain::fake_defaults::LAST_EPOCH_BLOCK); + assert!(reason.contains("503"), "{reason}"); + epoch + } + other => panic!("a down feed must burn, not score: {other:?}"), + }; + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Burned); + assert!(!view.last_feed_read, "an outage is not a read feed"); + assert_eq!(view.last_paid, 0); + assert_eq!(em.scored_epoch(), 0); + + // Every leaf is the ChallengeInternal cover: nobody is paid. + let posted = accepted.lock().expect("lock").clone(); + assert_eq!(posted.len(), 2); + for leaf in &posted { + assert_eq!( + leaf["score_or_absence"]["no_score"]["reason"], 6, + "an unreadable feed must cover E with ChallengeInternal: {leaf}" + ); + } + + let e = expected_set(); + let mut leaves: Vec = posted.iter().map(leaf_from_accepted).collect(); + leaves.extend( + cover_with_noscore( + &PROOF_SK, + b"proof", + epoch, + &e, + NoScoreReasonCode::ChallengeInternal, + ) + .into_values(), + ); + + let trust = trust_root(); + let chain = fake_chain(); + let bundle = build_sealed_bundle( + &chain, + &trust, + leaves, + &SealParams { + epoch, + netuid: NETUID, + block_b: chain::fake_defaults::LAST_EPOCH_BLOCK, + gateway_secret: GATEWAY_SK, + }, + ) + .expect("a burn still covers E, so the epoch seals for every challenge"); + verify_bundle(&bundle, &chain, &trust).expect("verify"); + + assert!( + bundle.body.final_vector.iter().all(|(uid, _)| *uid == 0), + "a burn epoch must sink to uid 0: {:?}", + bundle.body.final_vector + ); + let weights: u32 = bundle + .body + .final_vector + .iter() + .map(|(_, w)| u32::from(*w)) + .sum(); + assert!(weights > 0, "the burn vector must still be sealable"); +} + +/// A leaf signed by the wrong key does not verify. This is the guard on the +/// assertion above: `verify_leaf_sig` passing is evidence, not a tautology. +#[tokio::test] +async fn a_leaf_from_another_challenge_key_does_not_verify() { + let trust = trust_root(); + let bounty_pk = trust + .challenges + .get(b"bounty") + .expect("bounty row") + .public_key; + let proof_pk = trust + .challenges + .get(b"proof") + .expect("proof row") + .public_key; + assert_ne!(bounty_pk, proof_pk, "the two rows carry distinct keys"); + + let leaf = make_signed_leaf( + &PROOF_SK, + b"bounty", + miner_hotkey(), + 1, + ScoreOrAbsence::Score { value: 1 }, + ) + .expect("leaf"); + assert!( + verify_leaf_sig(&leaf, &bounty_pk).is_err(), + "a proof-key signature must not verify as a bounty leaf" + ); + verify_leaf_sig(&leaf, &proof_pk).expect("it verifies under its own key"); +} + +/// `BPS_DENOM` is what `ChallengesBody::validate` enforces; the test trust root +/// above must satisfy it or the fixture is not the shape validators load. +#[test] +fn the_test_trust_root_is_a_valid_split() { + let trust = trust_root(); + trust.challenges.validate().expect("valid split"); + let total: u32 = trust + .challenges + .challenges + .iter() + .map(|c| u32::from(c.emission_share_bps)) + .sum(); + assert_eq!(total, u32::from(BPS_DENOM)); +} diff --git a/crates/bounty-challenge/tests/trust_root_linkage.rs b/crates/bounty-challenge/tests/trust_root_linkage.rs new file mode 100644 index 000000000..19b799b68 --- /dev/null +++ b/crates/bounty-challenge/tests/trust_root_linkage.rs @@ -0,0 +1,211 @@ +//! The committed trust root must keep bounty live and payable. +//! +//! Bounty's reward linkage has a precondition that lives outside its own code: +//! the challenge has to be in the owner-signed trust root with a nonzero +//! emission share, and the keys the emitter signs with have to be the keys that +//! root names. Nothing in the emitter can check either one — a host whose +//! `bounty_sk` does not match the root's `bounty` row signs leaves that every +//! validator rejects, and a host whose root has no bounty row emits into a +//! challenge that does not exist. +//! +//! These tests read the files an operator actually deploys +//! (`config/challenges.toml` and the staging override) and verify them the way +//! a validator does: owner signature, then the row the reward path depends on. +//! The point is not to restate the numbers — it is that changing them is a +//! decision, not a drift. + +#![forbid(unsafe_code)] +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::path::{Path, PathBuf}; + +use bounty_challenge_task::{CHALLENGE_ID, SCORING_VERSION}; +use trustroot::{load_challenges_file, ChallengesBody, ChallengesToml, BPS_DENOM}; + +/// Bounty's committed share. Proof holds the remainder; the pair must sum to +/// `BPS_DENOM` or `ChallengesBody::validate` refuses the document outright. +const BOUNTY_BPS: u16 = 2000; + +/// Proof's committed share (20/80 with bounty). +const PROOF_BPS: u16 = 8000; + +fn repo_config_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../config") + .canonicalize() + .expect("config dir") +} + +fn owner_public() -> [u8; 32] { + trustroot::load_owner_public_key(&repo_config_dir().join("owner.pubkey")).expect("owner pubkey") +} + +/// Load a root the way a validator does: verify the owner signature first, so +/// a hand-edited row without a re-sign fails here rather than in production. +fn verified(body_path: &Path) -> ChallengesBody { + load_challenges_file(body_path, &owner_public()) + .unwrap_or_else(|e| { + panic!( + "{} must verify under the committed owner: {e}", + body_path.display() + ) + }) + .body +} + +fn row<'a>(body: &'a ChallengesBody, id: &str) -> &'a trustroot::ChallengeEntry { + body.get(id.as_bytes()) + .unwrap_or_else(|| panic!("{id} row missing from the trust root")) +} + +fn files() -> Vec<(&'static str, PathBuf)> { + let dir = repo_config_dir(); + vec![ + ("prod", dir.join("challenges.toml")), + ("staging", dir.join("challenges.staging.toml")), + ] +} + +/// Bounty is a live, payable challenge in every committed root. +/// +/// An emission share of 0 is not a degraded bounty — it is a challenge that +/// cannot pay anyone while still being asked for leaves, and `assert_ +/// participant_completeness` skips 0-bps rows entirely, so the emitter would be +/// posting into a set nothing seals. +#[test] +fn bounty_is_live_and_payable_in_every_committed_root() { + for (label, path) in files() { + let body = verified(&path); + body.validate() + .unwrap_or_else(|e| panic!("{label}: committed root must validate: {e}")); + + let bounty = row(&body, CHALLENGE_ID); + assert_eq!( + bounty.emission_share_bps, BOUNTY_BPS, + "{label}: bounty share drifted from the committed {BOUNTY_BPS} bps" + ); + assert!( + bounty.emission_share_bps > 0, + "{label}: a 0-bps bounty is not payable" + ); + assert_eq!( + bounty.policy, + trustroot::ParticipantPolicy::AllMetagraphHotkeys, + "{label}: bounty derives E from the whole metagraph" + ); + + // Proof is the other live row; the two shares are the whole budget. + let proof = row(&body, "proof"); + assert_eq!(proof.emission_share_bps, PROOF_BPS, "{label}: proof share"); + + let total: u32 = body + .challenges + .iter() + .map(|c| u32::from(c.emission_share_bps)) + .sum(); + assert_eq!( + total, + u32::from(BPS_DENOM), + "{label}: shares must sum to {BPS_DENOM}" + ); + assert_eq!(body.challenges.len(), 2, "{label}: two live challenges"); + } +} + +/// The challenge id the emitter signs under is the id the root publishes. +/// +/// `CHALLENGE_ID_BYTES` is what `emit_signed_leaf_set` puts in every leaf, and +/// `ChallengesBody::get` is what the gateway and validators look it up by. A +/// mismatch would make every leaf unattributable — and it would look like a +/// signature failure, which is the wrong place to go looking. +#[test] +fn the_emitters_challenge_id_is_the_trust_roots_id() { + assert_eq!(CHALLENGE_ID, "bounty"); + for (label, path) in files() { + let body = verified(&path); + let ids: Vec = body + .challenges + .iter() + .map(|c| String::from_utf8_lossy(&c.id).into_owned()) + .collect(); + assert!( + ids.iter().any(|id| id == CHALLENGE_ID), + "{label}: no row for the id the emitter signs under: {ids:?}" + ); + } +} + +/// The retired products stay absent. A leftover row would restore an emission +/// the owner removed, and `relearn` / `design` / `prism` have no code left to +/// serve it. +#[test] +fn retired_products_have_no_row_in_any_committed_root() { + for (label, path) in files() { + let body = verified(&path); + for off in [ + "relearn", + "relearn-image", + "relearn-agent", + "relearn-mm", + "design", + "prism", + ] { + assert!( + body.get(off.as_bytes()).is_none(), + "{label}: {off} must not have a trust-root row" + ); + } + } +} + +/// Staging mirrors production's split. +/// +/// Staging exists to prove the production path, so a split that diverges there +/// would let the staging soak pass while prod seals a different vector. The +/// validator compares emission shares against its local root (D23), so a +/// divergence also shows up as a staging-only failure. +#[test] +fn staging_mirrors_prod_shares_and_keys() { + let dir = repo_config_dir(); + let prod = verified(&dir.join("challenges.toml")); + let staging = verified(&dir.join("challenges.staging.toml")); + + for id in [CHALLENGE_ID, "proof"] { + let p = row(&prod, id); + let s = row(&staging, id); + assert_eq!( + p.emission_share_bps, s.emission_share_bps, + "{id}: staging share must mirror prod" + ); + assert_eq!( + p.public_key, s.public_key, + "{id}: staging key must mirror prod, or a staging leaf cannot be \ + verified against the same trust root shape" + ); + } +} + +/// The committed roots parse as the deployed TOML shape, and the parsed body +/// re-derives the same rows. This catches a file that verifies but carries a +/// field the loader drops. +#[test] +fn committed_roots_round_trip_through_the_toml_shape() { + for (label, path) in files() { + let text = std::fs::read_to_string(&path).expect("read"); + let doc: ChallengesToml = toml::from_str(&text).expect("toml shape"); + let body = doc.to_body().expect("to_body"); + assert_eq!( + body, + verified(&path), + "{label}: the parsed document must equal the verified body" + ); + assert_eq!(doc.version, 1, "{label}: document version"); + assert_eq!(doc.introduced_epoch, 0, "{label}: introduced_epoch"); + } +} + +/// The scoring version is part of the published identity a miner checks. +#[test] +fn the_scoring_version_is_pinned() { + assert_eq!(SCORING_VERSION, 1); +} diff --git a/crates/bounty-http/src/lib.rs b/crates/bounty-http/src/lib.rs index 7ede5a9c3..7e30c471b 100644 --- a/crates/bounty-http/src/lib.rs +++ b/crates/bounty-http/src/lib.rs @@ -39,7 +39,7 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use bounty_challenge_task::{ backend_public_url, hotkey_hex, parse_hotkey, parse_signature, verify_pair_signature, - PairChallenge, ScoringBackend, CHALLENGE_ID, MAX_PENDING_REPORTS_PER_HOTKEY, + EmitterStatus, PairChallenge, ScoringBackend, CHALLENGE_ID, MAX_PENDING_REPORTS_PER_HOTKEY, MIN_REPORT_BODY_CHARS, MIN_REPORT_INTERVAL_SECS, MIN_REPRO_CHARS, MIN_UNIQUE_BODY_TOKENS, SCORE_MAX, SCORING_VERSION, TERMS_TEXT, }; @@ -61,6 +61,9 @@ pub struct AppState { pub scoring: ScoringBackend, /// Operator bearer hashes (sha256 hex). Empty → admin 503. pub admin_hashes: Arc>, + /// Emitter read side. `None` (or an unwired status) means this host signs + /// no leaves at all, which is the only condition that also 409s the seal. + pub emitter: Option>, } impl AppState { @@ -68,6 +71,11 @@ impl AppState { fn can_score(&self) -> bool { self.scoring != ScoringBackend::Unconfigured } + + /// Whether this host wired a leaf emitter. + fn emitter_wired(&self) -> bool { + self.emitter.as_ref().is_some_and(|e| e.wired()) + } } /// Build the router. @@ -92,6 +100,7 @@ async fn health() -> impl IntoResponse { async fn status(State(st): State) -> impl IntoResponse { let champ = st.store.champion_hotkey().ok().flatten(); + let emitter = st.emitter.as_ref().map(|e| e.view()); Json(serde_json::json!({ "challenge_id": CHALLENGE_ID, "scoring_version": SCORING_VERSION, @@ -102,6 +111,14 @@ async fn status(State(st): State) -> impl IntoResponse { "scoring_backend": st.scoring, "can_score": st.can_score(), "backend_public_configured": backend_public_url().is_some(), + // Whether that scorer has actually turned into weight. `can_score` says + // this host *may* pay; `emitter_wired` plus `emitter` say whether it + // *is*. A readable feed that crowns nobody shows up here as + // `last_outcome: "unpaid"` with `last_feed_read: true` rather than as a + // successful score, because a silent empty payout is the failure this + // whole path exists to make visible. + "emitter_wired": st.emitter_wired(), + "emitter": emitter, // Published so miners can see what is being measured — and what is // not: `triage_noise` never enters the score they are paid on. "scoring": { @@ -437,6 +454,7 @@ mod tests { session_secret: Arc::new(b"test-session-secret".to_vec()), scoring, admin_hashes: Arc::new(vec![hash_admin_token(token)]), + emitter: None, }); (router, token.to_owned()) } @@ -728,6 +746,7 @@ mod tests { session_secret: Arc::new(b"test-session-secret".to_vec()), scoring: ScoringBackend::BackendPublic, admin_hashes: Arc::new(vec![]), + emitter: None, }); let exp = unix_now().saturating_add(600); let (st, paired) = json_req(app.clone(), "POST", "/v1/pair", pair_payload(exp), None).await; diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 49feea1e8..b17a058bc 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -48,7 +48,7 @@ Migrations (`crates/db/migrations`) run on boot in gateway when `BASE_DATABASE_U Bounty scores **only** from the CortexLM/backend public feed. Set `BOUNTY_BACKEND_PUBLIC_URL` on the host (`deploy/env/bounty-challenge.env`; never bake a hostname into git). The service fetches `/v1/bounty/public/leaderboard` + `/reports`, signs an exact-`E` leaf set every `BOUNTY_EMIT_POLL_SECS` (default 120), and posts to the master gateway — validators only ever verify the sealed bundle. -With no readable feed the host answers **503** on `POST /v1/reports` and pays nobody: it covers `E` with `NoScore(ChallengeInternal)`, so the 2000 bps burns to uid 0 while D24 still holds (leaving `E` uncovered would 409 the seal for *every* challenge, since bounty holds a paid trust-root row). `BOUNTY_FORCE_SIM` is retired and ignored; `assert-compose-matrix.sh` fails if any compose file reintroduces it. Verify with `./deploy/scripts/local-e2e.sh --smoke` (it POSTs ingest and asserts 503 without a feed, 401 with one) or by hand: `GET /challenge/bounty/v1/status` → `scoring_backend`, `can_score`. Details: [`docs/BOUNTY.md`](../docs/BOUNTY.md). +With no readable feed the host answers **503** on `POST /v1/reports` and pays nobody: it covers `E` with `NoScore(ChallengeInternal)`, so the 2000 bps burns to uid 0 while D24 still holds (leaving `E` uncovered would 409 the seal for *every* challenge, since bounty holds a paid trust-root row). A feed that **answers** and crowns nobody is a separate outcome — `unpaid`, same cover, same burn — and must not be read as a healthy score: adjudication is what turns a published row into weight, so a `valid` row without a `severity` pays nothing. `BOUNTY_FORCE_SIM` is retired and ignored; `assert-compose-matrix.sh` fails if any compose file reintroduces it. Verify with `./deploy/scripts/local-e2e.sh --smoke` (it POSTs ingest and asserts 503 without a feed, 401 with one, plus the emitter's outcome/feed/paid fields) or by hand: `GET /challenge/bounty/v1/status` → `scoring_backend`, `can_score`, `emitter_wired`, `emitter.last_outcome` / `last_feed_read` / `last_paid`. Details: [`docs/BOUNTY.md`](../docs/BOUNTY.md). ## Proof harvest (Lium) diff --git a/deploy/compose/env-staging.yml b/deploy/compose/env-staging.yml index 2d964aaab..88181647f 100644 --- a/deploy/compose/env-staging.yml +++ b/deploy/compose/env-staging.yml @@ -45,7 +45,7 @@ services: - ./deploy/secrets/wallets:/run/base/wallets:ro - ./deploy/secrets/gateway_admin_token:/run/secrets/gateway_admin_token:ro # The sealer must sign with the same staging trust root the validator - # verifies against (bounty 7000 + proof 3000), or /v1/weights/latest + # verifies against (bounty 2000 + proof 8000), or /v1/weights/latest # never shows challenge weight and the validator flags emission share mismatch (D23). - ./config/challenges.staging.toml:/etc/base/config/challenges.toml:ro - ./config/challenges.staging.toml.sig:/etc/base/config/challenges.toml.sig:ro diff --git a/deploy/env/bounty-challenge.env.example b/deploy/env/bounty-challenge.env.example index b7114e603..b8a2d3e4a 100644 --- a/deploy/env/bounty-challenge.env.example +++ b/deploy/env/bounty-challenge.env.example @@ -17,14 +17,31 @@ BOUNTY_CHAT_COMMAND= # Empty (or unreachable) and this host cannot score: POST /v1/reports answers # 503 rather than collecting real bug-hunting work it could never pay for, and # the emitter pays nobody — it covers the expected set with an explicit -# no-score, so the 7000 bps burns to uid 0. There is no offline scorer to fall +# no-score, so the 2000 bps burns to uid 0. There is no offline scorer to fall # back to; BOUNTY_FORCE_SIM is retired and ignored. # Check GET /v1/status → scoring_backend, can_score. +# +# A reachable feed is not the same as a paying one. Adjudication is the +# dependency: CortexLM/backend must publish a row that is `valid`, carries a +# `severity`, and is justified, or the crown gate refuses it and the epoch pays +# nobody. That case shows up on /v1/status as `emitter.last_outcome: "unpaid"` +# with `last_feed_read: true` — the feed answered, and there was nothing to +# pay. Do not read that as an outage. BOUNTY_BACKEND_PUBLIC_URL= # Seconds between emitter ticks (fetch feed → sign exact-E leaf set → gateway). BOUNTY_EMIT_POLL_SECS=120 +# What the emitter last did, published on GET /v1/status: +# emitter_wired false ⇒ no challenge key; no leaves, and the seal 409s +# emitter.last_outcome scored | unpaid | burned | held | error +# emitter.last_feed_read whether the feed answered on the last tick +# emitter.last_paid positive leaves on the last tick +# `unpaid` + `last_feed_read: true` is the one to watch: the challenge is +# running and the epoch still pays nobody. +# Validators never read this feed. They verify the sealed bundle, so the only +# way a backend adjudication becomes weight is this emitter posting leaves. + # Chain endpoint for the expected set E. The env overlays (env-staging.yml / # env-prod.yml) set the ordered failover list; this is the single-endpoint # fallback for a bare stack. diff --git a/deploy/scripts/local-e2e.sh b/deploy/scripts/local-e2e.sh index 2786992bb..8f17d78f2 100755 --- a/deploy/scripts/local-e2e.sh +++ b/deploy/scripts/local-e2e.sh @@ -426,10 +426,11 @@ PY import pathlib, sys # Mirrors config/challenges.toml. Shares must sum to 10000 or the validator -# flags an emission-share mismatch (D23). Bounty 7000 + proof 3000. +# flags an emission-share mismatch (D23). Bounty 2000 + proof 8000 — the live +# 20/80 split, so a local seal exercises the same vector production seals. rows = [ - ("bounty", sys.argv[2], 7000), - ("proof", sys.argv[3], 3000), + ("bounty", sys.argv[2], 2000), + ("proof", sys.argv[3], 8000), ] text = "version = 1\nintroduced_epoch = 0\n" for cid, pk, bps in rows: @@ -641,6 +642,11 @@ probe_weights_latest() { # # no BOUNTY_BACKEND_PUBLIC_URL → 503 (fail-closed; there is no offline scorer) # feed configured → 401 invalid_session (ingest open, gate off) +# +# Both cases also assert the emitter read side, because ingest being reachable +# says nothing about weight being produced. The failure this catches is a host +# that answers ingest, reports `can_score: true`, and still pays nobody every +# epoch — which is invisible without the emitter fields. probe_bounty_fail_closed() { local base="http://127.0.0.1:${BOUNTY_HOST_PORT}" local status code body @@ -650,6 +656,17 @@ probe_bounty_fail_closed() { return 0 fi log "bounty status: $status" + # A host with a challenge key must wire the emitter even with no feed: bounty + # holds a paid trust-root row, so an uncovered E 409s the seal for every + # challenge, including proof's. + if [[ "$(echo "$status" | jq -r '.emitter_wired // "missing"')" == "false" ]]; then + log "warning: bounty emitter not wired (no BASE_CHALLENGE_SK_FILE); the seal will 409 until it is" + else + echo "$status" | jq -e '.emitter | objects' >/dev/null \ + || die "bounty /v1/status has no emitter block; weight production is not observable" + echo "$status" | jq -e '.emitter | has("last_outcome") and has("last_feed_read") and has("last_paid")' >/dev/null \ + || die "bounty emitter block is missing outcome/feed/paid fields" + fi body='{"session":"not-a-session","title":"probe","body":"probe","repro_steps":"probe"}' code="$(curl -sS -m 5 -o /tmp/local-e2e-bounty-report.json -w '%{http_code}' \ -X POST -H 'content-type: application/json' -d "$body" "${base}/v1/reports" || true)" @@ -661,6 +678,27 @@ probe_bounty_fail_closed() { else [[ "$code" == "401" ]] || die "bounty ingest answered HTTP $code with a feed configured (expected 401 invalid_session)" log "bounty ingest OK: feed configured → POST /v1/reports reaches session auth" + # Feed configured but nothing adjudicated is the quiet empty-payout case: + # the emitter covers E with ChallengeInternal and reports `unpaid`, which + # must not read as a healthy score. + local outcome paid + outcome="$(echo "$status" | jq -r '.emitter.last_outcome // "never"')" + paid="$(echo "$status" | jq -r '.emitter.last_paid // 0')" + case "$outcome" in + scored) + [[ "$paid" -gt 0 ]] || die "bounty reports a scored tick that paid nobody" + log "bounty reward linkage OK: last tick scored $paid hotkey(s)" + ;; + unpaid) + log "bounty emitter unpaid: feed read, nothing adjudicated yet (share burns to uid 0)" + ;; + burned|held|error|never) + log "bounty emitter last_outcome=$outcome (waiting on the backend feed)" + ;; + *) + die "bounty emitter reported an unknown outcome: $outcome" + ;; + esac fi } diff --git a/docs/BOUNTY.md b/docs/BOUNTY.md index 494ea4347..7c8ef194e 100644 --- a/docs/BOUNTY.md +++ b/docs/BOUNTY.md @@ -28,6 +28,48 @@ board. | `challenge_scoring_version` | `1` | | Port | `8096` (local host `28096`) | | Emission | `2000` bps (20%; Proof has the other 80%) | +| Trust-root row | `bounty` @ 2000 bps, `all_metagraph_hotkeys` | + +The row is committed in [`../config/challenges.toml`](../config/challenges.toml) +and mirrored by [`../config/challenges.staging.toml`](../config/challenges.staging.toml) +for staging. Both are owner-signed: changing a share without re-signing fails +at load, and `crates/bounty-challenge/tests/trust_root_linkage.rs` asserts both +files stay live, payable, and in step. Do not edit them by hand — see +[`../config/CEREMONY.md`](../config/CEREMONY.md) and +[`runbooks/trust-root-rotation.md`](runbooks/trust-root-rotation.md). + +## Operator environment + +Set on the **master** host (never baked into git; see +[`../deploy/env/bounty-challenge.env.example`](../deploy/env/bounty-challenge.env.example)): + +| Variable | Role | +|----------|------| +| `BOUNTY_BACKEND_PUBLIC_URL` | Base URL of the CortexLM/backend public API. **The only scorer.** Unset → ingest 503s and every leaf is a cover. | +| `BOUNTY_EMIT_POLL_SECS` | Seconds between emitter ticks (default 120). | +| `BASE_CHALLENGE_SK_FILE` | Bounty leaf mini-secret. Its public key **must** match the trust-root `bounty` row, or every leaf is rejected. | +| `BASE_CHALLENGE_GATEWAY_ENDPOINT` | Master gateway for `POST /v1/weights/raw` (default `http://gateway:8080`). | +| `BASE_NETUID` | Subnet `E` is derived from. | +| `BASE_CHAIN_ENDPOINT(S)` | Chain for `E`; `BASE_CHAIN_ENDPOINTS` is the ordered failover list and wins over the singular. | +| `BOUNTY_ADMIN_TOKENS_FILE` | Operator bearer for `POST /v1/admin/adjudicate` and the report reads. Empty → admin **503**. | +| `BOUNTY_SESSION_SECRET_FILE` | Pairing session HMAC. Empty → ephemeral (pairings do not survive restart). | +| `BOUNTY_CHAT_COMMAND` | Chat inject command. Env-only; docs use the placeholder. | + +`BOUNTY_FORCE_SIM` is retired: it is ignored, warned about at boot, and +`deploy/scripts/assert-compose-matrix.sh` fails if any compose file sets it. + +### What validators do and do not verify + +Validators **never** read the bounty feed and **never** re-run a report. They +verify the sealed bundle: the gateway signature, the merkle root, D24 +completeness against the local owner-signed trust root, and the recomputed +weight vector. So the feed → leaf → seal path is entirely on the challenge +host, and a bug there is invisible to consensus until someone reads +`/v1/status` or the sealed vector. + +That is why the emitter publishes its outcome: the reward linkage has two +halves that fail independently — the backend publishing adjudications, and this +host turning them into signed leaves. `can_score` covers only the first. ## Why bug reports need different evaluation @@ -90,19 +132,50 @@ scorer. Each tick the challenge service: 2. derives `E` from the metagraph at `last_epoch_block` (`AllMetagraphHotkeys`) 3. maps published rows onto one leaf per hotkey in `E` for the current subnet epoch (champion → `Score`, net-malicious → `InvalidResponse`, everyone else - → `NotAttempted`); an unreadable feed pays nobody but still covers `E` with - `ChallengeInternal` + → `NotAttempted`) 4. `POST /v1/weights/raw` on the gateway, which seals what validators fetch +**A reachable feed is not a paying one**, and the difference is a separate +outcome. When the feed answers and no row maps to payable weight — nothing +adjudicated, everything still `pending`, or a `valid` row the operator never +priced — the tick is `unpaid`: `E` is covered with +`NoScore(ChallengeInternal)`, the share burns to uid 0, and `/v1/status` +reports `last_outcome: "unpaid"` with `last_feed_read: true`. It is not signed +as a scored epoch, because `NotAttempted` claims the challenge *chose* not to +invoke the miner, which is false here and would report a healthy tick while +paying nobody. + +Adjudication is therefore a hard dependency of the reward path, not a +reporting one. A published row only becomes weight when it is `valid`, carries +a `severity`, and is justified; anything short of that pays nothing that epoch. + Operator knobs: `BASE_CHALLENGE_GATEWAY_ENDPOINT`, `BASE_NETUID`, `BASE_CHAIN_ENDPOINT(S)`, `BOUNTY_EMIT_POLL_SECS` (default 120s). Re-emitting the current epoch is normal: the gateway supersedes on a changed digest and 409s an identical one. +### Reading `/v1/status` + +| Field | Meaning | +|-------|---------| +| `scoring_backend` | `backend_public` or `unconfigured`. | +| `can_score` | Whether this host *may* turn a report into weight. | +| `emitter_wired` | Whether a leaf emitter was wired. `false` is the only condition that also 409s the seal (missing `BASE_CHALLENGE_SK_FILE`). | +| `emitter.last_outcome` | `never` / `scored` / `unpaid` / `burned` / `held` / `error`. | +| `emitter.last_feed_read` | Whether the feed answered on the last tick. | +| `emitter.last_paid` | Positive leaves on the last tick. | +| `emitter.last_reason` | Why nobody was paid, or why the epoch was held. | +| `emitter.scored_epoch` | Highest epoch this process scored (in-process). | + +`can_score` alone does not mean anyone is being paid. `unpaid` with +`last_feed_read: true` is the combination to watch: the backend is up and the +epoch still pays nobody, which means adjudication is behind. + ## Fail-closed ingest and emission `GET /v1/status` publishes `scoring_backend` (`backend_public` | -`unconfigured`), `backend_public_configured`, and `can_score`. +`unconfigured`), `backend_public_configured`, `can_score`, `emitter_wired`, +and the `emitter` block. Without `BOUNTY_BACKEND_PUBLIC_URL` — or when the feed is unreachable, 5xx, unparseable, or moving under the read — the host cannot turn a report into @@ -114,6 +187,16 @@ weight. Two things follow, and neither is a degraded mode: `NoScore(ChallengeInternal)` (`BUNDLE_SPEC` §3.3.1 — "challenge-side fault; still must cover the participant"), so the 2000 bps burns to uid 0. +A feed that answers and crowns nobody is the third case, and it is treated as +a cover rather than a score. `NotAttempted` on every leaf would claim the +challenge chose not to invoke the miners, which is false, and it would seal as +a legitimate-looking unpaid epoch while `/v1/status` reported success. The +tick is `unpaid` instead: same `ChallengeInternal` cover, same burn, and the +status says which half is missing. The three outcomes are deliberately +distinct — `scored`, `unpaid` (feed read, nothing payable), `burned` (feed +unreadable) — because an operator needs to know whether to wait on the backend +or go look at adjudication. + Covering `E` is not a hedge, it is the difference between bounty failing and the subnet failing. Bounty holds a **paid** trust-root row, and D24 requires a leaf per participant for every paid challenge: leave `E` uncovered and @@ -137,18 +220,22 @@ and the paragraphs above apply. No backend change is needed for this; a published revision or ETag on both routes would let the moving-feed check collapse to a single round. -A failed tick also tries not to take back a score. Once the process has scored -an epoch, a feed outage inside that same epoch **holds** instead of superseding -a champion's leaf with a burn; a backend hiccup does not get to decide the -epoch. The watermark is in-process (the gateway has no read side for raw -leaves), so a restart during an outage can still burn an epoch that had -scores — the next successful tick supersedes it back. The bias is deliberate: -burning pays nobody who was not already paid, while staying silent would 409 -the seal for every challenge. +A tick that produces no weight also tries not to take back a score. Once the +process has scored an epoch, a later unproductive tick inside that same epoch +**holds** instead of superseding a champion's leaf with a cover — whether the +feed went down or stayed up and simply stopped publishing the crowned hotkey. +A backend hiccup, or a publish that reverts, does not get to decide the epoch. +The watermark is in-process (the gateway has no read side for raw leaves), so a +restart during an outage can still burn an epoch that had scores — the next +successful tick supersedes it back. The bias is deliberate: burning pays nobody +who was not already paid, while staying silent would 409 the seal for every +challenge. Independently, the gateway refuses a `ChallengeInternal` cover from +replacing a positive leaf for the same key (409, original kept), so a lost +watermark cannot reseal a paid allocation into a uid-0 burn. Only a missing `BASE_CHALLENGE_SK_FILE` stops emission entirely — a leaf the trust root rejects is not weight — and that case is logged as the 409 it will -cause. +cause. It is also the only case that publishes `emitter_wired: false`. **There is no offline scorer.** `BOUNTY_FORCE_SIM` is retired: it is ignored, warned about at boot, and `deploy/scripts/assert-compose-matrix.sh` fails if diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index bbf3aa997..daabb9749 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -62,7 +62,8 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Component | Status | Notes | |-----------|--------|-------| | Crates (`crates/bounty-*`) | **done** | task (pairing), score (precision × severity, triage-noise canary off the lattice), store, http (fail-closed ingest + quotas), challenge (backend public **consumer** + fail-closed leaf emitter; the two public routes are re-read until they agree, and `/leaderboard` `valid_count` must match the `valid` reports, so a mid-publish pair or a stable A+B mix is never signed as one snapshot). | -| Binary (`bins/bounty-challenge`) | **done** | Internal HTTP on `:8096` plus the emitter (backend feed → exact-`E` leaves → gateway `POST /v1/weights/raw`, `BOUNTY_EMIT_POLL_SECS`). Does **not** serve `/v1/public/*`. No feed (or an unreadable one) pays nobody: `E` is covered with `NoScore(ChallengeInternal)` so D24 holds and the share burns to uid 0. A scored epoch is never downgraded to a burn mid-epoch. | +| Binary (`bins/bounty-challenge`) | **done** | Internal HTTP on `:8096` plus the emitter (backend feed → exact-`E` leaves → gateway `POST /v1/weights/raw`, `BOUNTY_EMIT_POLL_SECS`). Does **not** serve `/v1/public/*`. No feed (or an unreadable one) pays nobody: `E` is covered with `NoScore(ChallengeInternal)` so D24 holds and the share burns to uid 0. A feed that answers and crowns nobody is the separate `unpaid` outcome — same cover, same burn — rather than a scored tick, because `NotAttempted` would claim the challenge chose not to invoke anyone. A scored epoch is never downgraded to a cover mid-epoch. `/v1/status` publishes `emitter_wired` plus the emitter's last outcome / feed-read / paid counts, so a reachable feed that pays nothing is visible instead of looking healthy. | +| Reward linkage tests | **done** | `crates/bounty-challenge/tests/rewards_linkage.rs` walks pair → report → adjudicate → published feed → signed leaves → sealed bundle (leaves verify under the trust root, the paired hotkey is the one paid, a silent hotkey is explicit, and a paid bounty seals beside a challenge that scored nothing). `tests/trust_root_linkage.rs` asserts the committed prod **and** staging roots keep bounty live at 2000 bps with shares summing to 10000, mirroring each other, under the owner signature. `tests/emit_fail_closed.rs` covers the unpaid / burned / held split. | | Miner CLI (`bins/ctx`) | **done** | `ctx bounty pair|report|show|status`. `bins/cortex-bounty` deprecates to `ctx bounty pair`. | | Compose / images | **done** | Default compose + `images.yml` target `bounty-challenge`. | | Emission | **2000 bps** | Payable share (20%). Sum `10000`. | diff --git a/docs/external-miner/bounty.md b/docs/external-miner/bounty.md index d9939eb32..e0a91a6dc 100644 --- a/docs/external-miner/bounty.md +++ b/docs/external-miner/bounty.md @@ -163,6 +163,24 @@ offline stand-in, so a 503 here is the honest answer rather than a temporary degradation you can submit through. Check `scoring_backend` and `can_score` before you go hunting. +`can_score` is not the whole story, and it is worth knowing which half is +missing when an epoch pays nobody. The feed being *reachable* is not the same +as it *paying*: adjudication is what turns your report into weight, so a +backend that is up with nothing adjudicated produces no weight either. `ctx +bounty status` prints the emitter's last outcome alongside the scorer: + +| Outcome | Meaning | +|---------|---------| +| `scored` | The feed published payable rows and they became signed weight. | +| `unpaid` | The feed answered, and nothing was payable. Wait on adjudication — the backend is up. | +| `burned` | The feed could not be read. This one is a backend or network problem. | +| `held` | The feed produced no weight, but this epoch already had a score, so the score stands. | +| `error` | The tick could not emit at all (chain, signing, or gateway). | + +`unpaid` with the feed read is the one to expect right after you file: your +report is real, and it becomes weight when an operator adjudicates it `valid` +with a severity. Nothing about that is a fault on your side. + ## Scoring (precision × severity, not volume) | Outcome | Result |