From a27952b4b4eddc8e6498d67b19c4a0f450b71353 Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Wed, 16 Sep 2026 13:48:15 +0000 Subject: [PATCH 1/4] fix(bounty): never sign an unpaid epoch as scored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The emitter turned a readable-but-unpayable feed into a "scored" tick. With zero adjudicated rows it emitted `NotAttempted` for every participant and reported success, so an epoch that paid nobody sealed as if the challenge had deliberately skipped the miners. That is the one outcome the reward path must not produce silently: the 2000 bps still burned to uid 0, and `can_score: true` plus a green tick said nothing was wrong. Readable-and-unpaid is now its own outcome. It covers `E` with the same `NoScore(ChallengeInternal)` a burn uses, so D24 still holds and the share still burns, and it is reported as `unpaid` rather than `scored`. The three cases stay distinct — `scored`, `unpaid` (feed read, nothing payable), `burned` (feed unreadable) — because an operator needs to know whether to wait on the backend or go look at adjudication. A hold now covers the same ground: a scored epoch is kept when the feed stops paying, not only when it stops answering. `GET /v1/status` publishes the emitter's read side (`emitter_wired` plus last outcome / feed-read / paid counts), and `ctx bounty status` prints it. `last_feed_read` is the tick's own record, not inferred from the outcome: a gateway failure after a successful read reports the feed as read, which a live check against a stand-in feed showed the inferred version got wrong. Tests walk the whole linkage rather than one link. `rewards_linkage.rs` drives pair → report → adjudicate → published feed → signed leaves, asserts the paired hotkey is the one paid, and seals a bundle where a paid bounty sits beside a challenge that scored nothing. `trust_root_linkage.rs` holds the committed prod and staging roots to bounty at 2000 bps with shares summing to 10000 under the owner signature, so the config side of the linkage cannot drift either. The emitter tests were mutation-checked against the old behaviour. Also bumps rustls to 0.23.45 for RUSTSEC-2026-0285, which landed after the last green CI run and now fails `cargo deny` on every branch. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- Cargo.lock | 21 +- bins/bounty-challenge/src/main.rs | 23 +- bins/ctx/src/bounty.rs | 29 + crates/bounty-challenge-task/src/lib.rs | 204 +++++ crates/bounty-challenge/Cargo.toml | 3 + crates/bounty-challenge/src/backend.rs | 10 + crates/bounty-challenge/src/emit.rs | 200 ++++- crates/bounty-challenge/src/lib.rs | 3 +- .../tests/emit_fail_closed.rs | 387 ++++++++- .../bounty-challenge/tests/rewards_linkage.rs | 773 ++++++++++++++++++ .../tests/trust_root_linkage.rs | 211 +++++ crates/bounty-http/src/lib.rs | 21 +- deploy/AGENTS.md | 2 +- deploy/compose/env-staging.yml | 2 +- deploy/env/bounty-challenge.env.example | 19 +- deploy/scripts/local-e2e.sh | 44 +- docs/BOUNTY.md | 111 ++- docs/COMPLETENESS.md | 3 +- docs/external-miner/bounty.md | 18 + 19 files changed, 2031 insertions(+), 53 deletions(-) create mode 100644 crates/bounty-challenge/tests/rewards_linkage.rs create mode 100644 crates/bounty-challenge/tests/trust_root_linkage.rs diff --git a/Cargo.lock b/Cargo.lock index ed54368c3..8275e4ed4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -438,9 +438,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -448,9 +448,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.43.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -679,10 +679,13 @@ dependencies = [ "chain", "challenge-common", "hex", + "http-body-util", "reqwest 0.12.28", "serde_json", "thiserror 2.0.19", "tokio", + "toml", + "tower", "tracing", "trustroot", ] @@ -4614,9 +4617,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -4679,9 +4682,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -5531,7 +5534,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/bins/bounty-challenge/src/main.rs b/bins/bounty-challenge/src/main.rs index 5b2a28de1..f639e958a 100644 --- a/bins/bounty-challenge/src/main.rs +++ b/bins/bounty-challenge/src/main.rs @@ -21,8 +21,8 @@ use std::time::Duration; use bounty_challenge::{ bounty_router, hash_admin_token, legacy_sim_opt_in_present, resolve_scoring_backend, AppState, - BountyEmitter, BountyStore, GatewayClient, GatewayClientConfig, ScoringBackend, CHALLENGE_ID, - DEFAULT_EMIT_POLL_SECS, SCORING_VERSION, + BountyEmitter, BountyStore, EmitterStatus, GatewayClient, GatewayClientConfig, ScoringBackend, + CHALLENGE_ID, DEFAULT_EMIT_POLL_SECS, SCORING_VERSION, }; use challenge_keys::load_challenge_secret; use clap::Parser; @@ -124,17 +124,26 @@ fn run(cli: &Cli) -> Result<(), String> { 503 and the emitter will pay nobody (the challenge share burns to uid 0) until then" ), } + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + // The emitter owns the read side published on GET /v1/status, so build it + // first and hand the same handle to the HTTP state. A host that wires no + // emitter (missing challenge key) publishes `emitter_wired: false`, which + // is the one condition that also 409s the seal. + let emitter = build_emitter(cli, scoring, sk)?; + let status = emitter + .as_ref() + .map_or_else(|| Arc::new(EmitterStatus::new(false)), |e| e.status()); let state = AppState { store: BountyStore::new(), session_secret: Arc::new(session_secret), scoring, admin_hashes: Arc::new(admin_hashes), + emitter: Some(status), }; - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .map_err(|e| e.to_string())?; - if let Some(emitter) = build_emitter(cli, scoring, sk)? { + if let Some(emitter) = emitter { let poll = Duration::from_secs(cli.emit_poll_secs.max(1)); rt.spawn(emitter.run(poll)); } diff --git a/bins/ctx/src/bounty.rs b/bins/ctx/src/bounty.rs index 8bbc457ca..0435f9fca 100644 --- a/bins/ctx/src/bounty.rs +++ b/bins/ctx/src/bounty.rs @@ -230,6 +230,35 @@ pub async fn status(client: &Client, json_out: bool) -> Result<(), String> { if let Some(v) = reply.body.get("champion_hotkey") { println!("champion_hotkey: {}", compact(v)); } + // `can_score` says this host *may* pay; the emitter says whether it *is*. + // A reachable feed with nothing adjudicated pays nobody, and a miner + // deserves to see which of those they are looking at. + if let Some(v) = reply.body.get("emitter_wired") { + println!("emitter_wired: {}", compact(v)); + } + if let Some(e) = reply.body.get("emitter").filter(|v| v.is_object()) { + println!("last_outcome: {}", compact(&e["last_outcome"])); + println!("last_feed_read: {}", compact(&e["last_feed_read"])); + println!("last_paid: {}", compact(&e["last_paid"])); + if let Some(reason) = e["last_reason"].as_str().filter(|s| !s.is_empty()) { + println!("last_reason: {reason}"); + } + match e["last_outcome"].as_str().unwrap_or("never") { + "unpaid" => println!( + " note: the feed answered and nothing was payable — reports become weight \ + once an operator adjudicates them valid with a severity." + ), + "burned" => println!( + " note: the feed could not be read, so this epoch pays nobody. Not a fault \ + on your side; check again later." + ), + "error" => println!( + " note: the emitter could not post leaves at all (chain, signing, or \ + gateway). An operator has to look at this." + ), + _ => {} + } + } if let Some(q) = reply.body.get("quotas") { println!("quotas: {q}"); } else { diff --git a/crates/bounty-challenge-task/src/lib.rs b/crates/bounty-challenge-task/src/lib.rs index 83aad2071..fb776d0ca 100644 --- a/crates/bounty-challenge-task/src/lib.rs +++ b/crates/bounty-challenge-task/src/lib.rs @@ -16,6 +16,8 @@ use keystore::{ss58_decode, ss58_encode, BITTENSOR_SS58_PREFIX, KEY_LEN}; use schnorrkel::{signing_context, ExpansionMode, MiniSecretKey, PublicKey, Signature}; use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}; +use std::sync::Mutex; use thiserror::Error; /// Normative challenge id (trust-root / leaf `challenge_id` string). @@ -206,6 +208,208 @@ pub fn resolve_scoring_backend() -> ScoringBackend { } } +// --- Emitter observability --------------------------------------------------- +// +// The reward linkage is a chain of two halves that fail independently: the +// backend publishes adjudications, and this host turns them into signed +// leaves. `/v1/status` already says whether a feed is *configured*; that is not +// the same as whether weight is being produced. A readable feed that publishes +// nothing payable, or a crowned hotkey that is not in `E`, is an empty payout +// that no local store reading can see. These types are the surface that makes +// it visible instead of leaving it to a log grep. + +/// What one completed emitter tick did. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EmitterOutcomeKind { + /// No tick has completed yet. + Never, + /// The feed published payable rows and they became signed scores. + Scored, + /// The feed was read, but nothing payable became weight this tick. + Unpaid, + /// The feed could not be read; `E` was covered with `ChallengeInternal`. + Burned, + /// A scored epoch was left in place through a feed outage. + Held, + /// The tick could not emit at all (chain, signing, or gateway). + Error, +} + +impl EmitterOutcomeKind { + /// Stable code for the atomic slot. + const fn code(self) -> u8 { + match self { + Self::Never => 0, + Self::Scored => 1, + Self::Unpaid => 2, + Self::Burned => 3, + Self::Held => 4, + Self::Error => 5, + } + } + + /// Inverse of [`Self::code`]; unknown codes read back as [`Self::Never`]. + const fn from_code(code: u8) -> Self { + match code { + 1 => Self::Scored, + 2 => Self::Unpaid, + 3 => Self::Burned, + 4 => Self::Held, + 5 => Self::Error, + _ => Self::Never, + } + } +} + +/// One completed tick, as recorded by [`EmitterStatus::record`]. +#[derive(Debug, Clone, Copy)] +pub struct EmitterTick<'a> { + /// What the tick did. + pub kind: EmitterOutcomeKind, + /// Subnet epoch the leaves were signed for (0 when the tick never got there). + pub epoch: u64, + /// Block `E` was pinned at (0 when unknown). + pub pin_block: u64, + /// Size of `E`. + pub participants: usize, + /// Hotkeys that received a positive score. + pub paid: usize, + /// Whether the backend public feed answered this tick. + pub feed_read: bool, + /// Highest epoch this process has scored (0 = none). + pub scored_epoch: u64, + /// Why nobody was paid, or why the epoch was held. + pub reason: Option<&'a str>, + /// Why the tick emitted nothing at all. + pub error: Option<&'a str>, +} + +/// Read-side view of the emitter, published on `GET /v1/status`. +/// +/// `last_outcome: "unpaid"` with `last_feed_read: true` is the one an operator +/// has to notice: the feed is reachable, the challenge is running, and the +/// epoch still pays nobody. `last_reason` says which half is missing. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EmitterStatusView { + /// Whether this host wired an emitter at all (a missing challenge key + /// leaves `false`, which is the one case that also 409s the seal). + pub wired: bool, + /// Completed ticks since boot. + pub ticks: u64, + /// Outcome of the most recent tick. + pub last_outcome: EmitterOutcomeKind, + /// Epoch of the most recent tick (0 = none yet). + pub last_epoch: u64, + /// Pin block of the most recent tick (0 = none yet). + pub last_pin_block: u64, + /// Size of `E` on the most recent tick. + pub last_participants: u64, + /// Positive leaves on the most recent tick. + pub last_paid: u64, + /// Whether the feed answered on the most recent tick. + pub last_feed_read: bool, + /// Highest epoch this process scored (0 = none). In-process: a restart + /// inside an outage can still burn an epoch that had scores, and the next + /// successful tick supersedes it back. + pub scored_epoch: u64, + /// Why nobody was paid, or why the epoch was held. + pub last_reason: String, + /// Why the tick emitted nothing at all. + pub last_error: String, +} + +/// Shared emitter state. The emitter writes it; `/v1/status` reads it. +#[derive(Debug)] +pub struct EmitterStatus { + wired: bool, + ticks: AtomicU64, + last_kind: AtomicU8, + last_epoch: AtomicU64, + last_pin_block: AtomicU64, + last_participants: AtomicU64, + last_paid: AtomicU64, + last_feed_read: AtomicBool, + scored_epoch: AtomicU64, + last_reason: Mutex, + last_error: Mutex, +} + +impl EmitterStatus { + /// Fresh status for a host that did or did not wire an emitter. + #[must_use] + pub fn new(wired: bool) -> Self { + Self { + wired, + ticks: AtomicU64::new(0), + last_kind: AtomicU8::new(EmitterOutcomeKind::Never.code()), + last_epoch: AtomicU64::new(0), + last_pin_block: AtomicU64::new(0), + last_participants: AtomicU64::new(0), + last_paid: AtomicU64::new(0), + last_feed_read: AtomicBool::new(false), + scored_epoch: AtomicU64::new(0), + last_reason: Mutex::new(String::new()), + last_error: Mutex::new(String::new()), + } + } + + /// Whether this host wired an emitter. + #[must_use] + pub fn wired(&self) -> bool { + self.wired + } + + /// Record one completed tick. + pub fn record(&self, tick: EmitterTick<'_>) { + self.ticks.fetch_add(1, Ordering::Relaxed); + self.last_kind.store(tick.kind.code(), Ordering::Relaxed); + self.last_epoch.store(tick.epoch, Ordering::Relaxed); + self.last_pin_block.store(tick.pin_block, Ordering::Relaxed); + self.last_participants + .store(count_u64(tick.participants), Ordering::Relaxed); + self.last_paid + .store(count_u64(tick.paid), Ordering::Relaxed); + self.last_feed_read.store(tick.feed_read, Ordering::Relaxed); + self.scored_epoch + .fetch_max(tick.scored_epoch, Ordering::Relaxed); + tick.reason + .unwrap_or_default() + .clone_into(&mut lock(&self.last_reason)); + tick.error + .unwrap_or_default() + .clone_into(&mut lock(&self.last_error)); + } + + /// Read-side snapshot for `/v1/status`. + #[must_use] + pub fn view(&self) -> EmitterStatusView { + EmitterStatusView { + wired: self.wired, + ticks: self.ticks.load(Ordering::Relaxed), + last_outcome: EmitterOutcomeKind::from_code(self.last_kind.load(Ordering::Relaxed)), + last_epoch: self.last_epoch.load(Ordering::Relaxed), + last_pin_block: self.last_pin_block.load(Ordering::Relaxed), + last_participants: self.last_participants.load(Ordering::Relaxed), + last_paid: self.last_paid.load(Ordering::Relaxed), + last_feed_read: self.last_feed_read.load(Ordering::Relaxed), + scored_epoch: self.scored_epoch.load(Ordering::Relaxed), + last_reason: lock(&self.last_reason).clone(), + last_error: lock(&self.last_error).clone(), + } + } +} + +/// A poisoned lock here is a status string, not a consensus input: recover the +/// guard rather than take the challenge down over a display field. +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, String> { + m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn count_u64(n: usize) -> u64 { + u64::try_from(n).unwrap_or(u64::MAX) +} + /// Most reports one hotkey may leave awaiting adjudication. /// /// Adjudication is the scarce resource in this challenge: every pending report diff --git a/crates/bounty-challenge/Cargo.toml b/crates/bounty-challenge/Cargo.toml index 9e0628f4f..b479bae3c 100644 --- a/crates/bounty-challenge/Cargo.toml +++ b/crates/bounty-challenge/Cargo.toml @@ -26,8 +26,11 @@ trustroot = { path = "../trustroot" } [dev-dependencies] axum = "0.8" +http-body-util = "0.1" serde_json = "1" tokio = { version = "1", features = ["macros", "net", "rt", "rt-multi-thread", "time"] } +toml = "0.8" +tower = { version = "0.5", features = ["util"] } [lints] workspace = true diff --git a/crates/bounty-challenge/src/backend.rs b/crates/bounty-challenge/src/backend.rs index 2bce52ded..112c44f6b 100644 --- a/crates/bounty-challenge/src/backend.rs +++ b/crates/bounty-challenge/src/backend.rs @@ -48,6 +48,16 @@ pub enum BackendError { /// `/reports` is stable under re-read and must still be refused. #[error("backend public leaderboard and reports do not agree")] Mismatched, + /// The feed answered, and every row it published maps to no payable + /// weight. Not an outage: the backend is reachable and simply has no + /// crowned miner, so this epoch pays nobody and the share burns. + /// + /// It is an error rather than a skip for the same reason a failed read is: + /// an emitter that treats "nothing payable" as a successful tick signs + /// `NotAttempted` for every participant, which claims the challenge chose + /// not to invoke them, and reports a healthy score while paying nobody. + #[error("backend public feed has no payable rows: {0}")] + NoPayableRows(String), } /// How many times one call re-reads the pair of routes looking for two diff --git a/crates/bounty-challenge/src/emit.rs b/crates/bounty-challenge/src/emit.rs index ff2c8f720..8c8a4b6fe 100644 --- a/crates/bounty-challenge/src/emit.rs +++ b/crates/bounty-challenge/src/emit.rs @@ -16,7 +16,19 @@ //! - **It must still cover `E`.** A paid challenge with no leaves fails D24 //! completeness, so `POST /v1/admin/seal` answers 409 and the epoch seals //! for *no* challenge. Silence here would make an unconfigured bounty host -//! take down relearn's weights too. +//! take down proof's weights too. +//! +//! Reading the feed and finding nothing payable is a *different* failure from +//! not reading it, and it is treated as one. A feed that answers with zero +//! adjudicated rows is reachable, but it still produces no weight, and the +//! honest leaf for that epoch is the same `ChallengeInternal` cover. This case +//! has to be separated out because treating it as a "score" emitted +//! `NotAttempted` for every participant, which is a claim that the challenge +//! *chose* not to invoke them. That claim is false, and it is the worse of the +//! two: `NotAttempted` is not the burn cover, so it seals as a +//! legitimate-looking unpaid epoch while `/v1/status` reports a successful +//! score. An operator has to be able to tell "backend is down" from "backend +//! is up and has crowned nobody". //! //! A failed tick also tries not to overwrite good leaves: once *this process* //! has scored an epoch, a later feed outage inside that same epoch holds @@ -32,6 +44,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; +use bounty_challenge_task::{EmitterOutcomeKind, EmitterStatus, EmitterTick}; use bundle::{NoScoreReasonCode, ScoreOrAbsence}; use chain::{gather_schedule_state, ChainClient}; use challenge_common::{ @@ -46,6 +59,10 @@ use crate::{emission_from_public_snapshot, CHALLENGE_ID_BYTES}; /// Default seconds between emitter ticks. pub const DEFAULT_EMIT_POLL_SECS: u64 = 120; +/// Why a readable feed still paid nobody this tick. +const NO_PAYABLE_ROWS_REASON: &str = + "backend public feed published no payable adjudication (nobody crowned)"; + /// Why a tick could not emit anything at all. #[derive(Debug, Error)] pub enum EmitError { @@ -77,6 +94,20 @@ pub enum EmitOutcome { /// Hotkeys that received a positive score. paid: usize, }, + /// The feed answered but nothing payable became weight, so `E` was covered + /// with `NoScore(ChallengeInternal)`: nobody is paid, the share burns to + /// uid 0, and the bundle can still seal. + /// + /// Distinct from [`Self::Burned`] because the cause is not an outage: the + /// feed is up and still has no crowned hotkey. + Unpaid { + /// Subnet epoch the cover was signed for. + epoch: u64, + /// Size of `E`. + participants: usize, + /// Why nobody was paid. + reason: String, + }, /// The feed could not be read, so `E` was covered with /// `NoScore(ChallengeInternal)`: nobody is paid, the share burns to uid 0, /// and the bundle can still seal. @@ -88,13 +119,14 @@ pub enum EmitOutcome { /// Why the feed was unreadable. reason: String, }, - /// The feed could not be read, but a scored set already stands for this - /// epoch. Overwriting it with a burn would take back a score the backend - /// really did publish. + /// The feed produced no weight, but a scored set already stands for this + /// epoch. Overwriting it with a cover would take back a score the backend + /// really did publish — whether the feed went down or stayed up and + /// stopped publishing the crowned hotkey. Held { /// Epoch whose scored leaves were left in place. epoch: u64, - /// Why the feed was unreadable. + /// Why this tick would have covered the epoch. reason: String, }, } @@ -107,6 +139,7 @@ pub struct BountyEmitter { netuid: u16, backend_base: Option, scored_epoch: AtomicU64, + status: Arc, } impl BountyEmitter { @@ -131,9 +164,23 @@ impl BountyEmitter { .map(|s| s.trim().to_owned()) .filter(|s| !s.is_empty()), scored_epoch: AtomicU64::new(0), + status: Arc::new(EmitterStatus::new(true)), } } + /// Share the read-side status published on `GET /v1/status`. + #[must_use] + pub fn with_status(mut self, status: Arc) -> Self { + self.status = status; + self + } + + /// Read-side status handle (shared with the HTTP state). + #[must_use] + pub fn status(&self) -> Arc { + Arc::clone(&self.status) + } + /// Highest epoch this process scored from the feed (0 = none yet). pub fn scored_epoch(&self) -> u64 { self.scored_epoch.load(Ordering::Relaxed) @@ -144,9 +191,23 @@ impl BountyEmitter { /// # Errors /// See [`EmitError`] — those are the failures that leave `E` uncovered. /// A missing or broken feed is not among them; it is an - /// [`EmitOutcome::Burned`] (or [`EmitOutcome::Held`]) instead. + /// [`EmitOutcome::Burned`] (or [`EmitOutcome::Held`]) instead. A feed that + /// answers without payable rows is [`EmitOutcome::Unpaid`]. pub async fn tick(&self) -> Result { + // Whether the feed answered is tracked here rather than inferred from + // the outcome: a tick that read the feed and then failed at the + // gateway is an error, and reporting `last_feed_read: false` for it + // would send an operator to the backend for a fault that is not there. + let mut feed_read = false; + let outcome = self.tick_inner(&mut feed_read).await; + self.record(&outcome, feed_read); + outcome + } + + /// One tick, before its outcome is published to [`EmitterStatus`]. + async fn tick_inner(&self, feed_read: &mut bool) -> Result { let feed = fetch_public_snapshot(self.backend_base.as_deref()).await; + *feed_read = feed.is_ok(); let pinned = self.expected_set_at_last_epoch()?; let (epoch, pin_block, hotkeys) = pinned; let snapshot = match feed { @@ -158,6 +219,19 @@ impl BountyEmitter { .values() .filter(|s| matches!(s, ScoreOrAbsence::Score { value } if *value > 0)) .count(); + // A readable feed that crowns nobody must not be signed as a scored + // epoch. `NotAttempted` claims the challenge chose not to invoke the + // miner; nobody was crowned, so the honest cover is + // `ChallengeInternal` and the share burns. + if paid == 0 { + return self + .cover_without_a_feed( + epoch, + &hotkeys, + &BackendError::NoPayableRows(NO_PAYABLE_ROWS_REASON.to_owned()), + ) + .await; + } self.submit(epoch, &hotkeys, &leaf_scores).await?; self.scored_epoch.fetch_max(epoch, Ordering::Relaxed); Ok(EmitOutcome::Scored { @@ -168,6 +242,87 @@ impl BountyEmitter { }) } + /// Publish one completed tick to `/v1/status`. + /// + /// `feed_read` is the tick's own record of whether the feed answered, not + /// something inferred from the outcome: an error after a successful read + /// is a gateway problem, and saying otherwise would point an operator at + /// the wrong service. + fn record(&self, outcome: &Result, feed_read: bool) { + let scored_epoch = self.scored_epoch(); + let tick = match outcome { + Ok(EmitOutcome::Scored { + epoch, + pin_block, + participants, + paid, + }) => EmitterTick { + kind: EmitterOutcomeKind::Scored, + epoch: *epoch, + pin_block: *pin_block, + participants: *participants, + paid: *paid, + feed_read, + scored_epoch, + reason: None, + error: None, + }, + Ok(EmitOutcome::Unpaid { + epoch, + participants, + reason, + }) => EmitterTick { + kind: EmitterOutcomeKind::Unpaid, + epoch: *epoch, + pin_block: 0, + participants: *participants, + paid: 0, + feed_read, + scored_epoch, + reason: Some(reason), + error: None, + }, + Ok(EmitOutcome::Burned { + epoch, + participants, + reason, + }) => EmitterTick { + kind: EmitterOutcomeKind::Burned, + epoch: *epoch, + pin_block: 0, + participants: *participants, + paid: 0, + feed_read, + scored_epoch, + reason: Some(reason), + error: None, + }, + Ok(EmitOutcome::Held { epoch, reason, .. }) => EmitterTick { + kind: EmitterOutcomeKind::Held, + epoch: *epoch, + pin_block: 0, + participants: 0, + paid: 0, + feed_read, + scored_epoch, + reason: Some(reason), + error: None, + }, + Err(e) => EmitterTick { + kind: EmitterOutcomeKind::Error, + epoch: 0, + pin_block: 0, + participants: 0, + paid: 0, + feed_read, + scored_epoch, + reason: None, + error: Some(&e.to_string()), + }, + }; + self.status.record(tick); + } + /// Tick forever. A failed tick is logged and retried; it never falls back /// to a local verdict. pub async fn run(self: Arc, poll: Duration) { @@ -190,6 +345,17 @@ impl BountyEmitter { paid, "bounty leaf set submitted from the backend public feed" ), + Ok(EmitOutcome::Unpaid { + epoch, + participants, + reason, + }) => tracing::warn!( + epoch, + participants, + %reason, + "bounty read the feed and found nothing payable: covered E with \ + ChallengeInternal, so the challenge share burns to uid 0" + ), Ok(EmitOutcome::Burned { epoch, participants, @@ -204,7 +370,7 @@ impl BountyEmitter { Ok(EmitOutcome::Held { epoch, reason }) => tracing::warn!( epoch, %reason, - "bounty could not read the feed; keeping this epoch's scored leaves" + "bounty produced no weight this tick; keeping this epoch's scored leaves" ), Err(e) => tracing::warn!( error = %e, @@ -238,8 +404,14 @@ impl BountyEmitter { Ok((epoch, pin_block, expected.hotkeys())) } - /// Cover `E` when the feed is unreadable: burn, or hold an already-scored - /// epoch. + /// Cover `E` when no weight could be produced: burn, or hold an + /// already-scored epoch. + /// + /// `cause` is either an unreadable feed or [`BackendError::NoPayableRows`] + /// (the feed answered and crowned nobody). Both pay nobody, but only the + /// outage is reported as [`EmitOutcome::Burned`] — an operator reading + /// `/v1/status` has to be able to tell "the backend is down" from "the + /// backend is up and there is nothing to pay". async fn cover_without_a_feed( &self, epoch: u64, @@ -262,9 +434,17 @@ impl BountyEmitter { }) .collect(); self.submit(epoch, hotkeys, &burn).await?; + let participants = hotkeys.len(); + if matches!(cause, BackendError::NoPayableRows(_)) { + return Ok(EmitOutcome::Unpaid { + epoch, + participants, + reason, + }); + } Ok(EmitOutcome::Burned { epoch, - participants: hotkeys.len(), + participants, reason, }) } diff --git a/crates/bounty-challenge/src/lib.rs b/crates/bounty-challenge/src/lib.rs index c53e312af..bc0cd1ac1 100644 --- a/crates/bounty-challenge/src/lib.rs +++ b/crates/bounty-challenge/src/lib.rs @@ -24,7 +24,8 @@ use challenge_common::{emit_signed_leaf_set, Hotkey, LeafEmitError}; pub use backend::{fetch_public_snapshot, public_path, snapshot_from_json, BackendError}; pub use bounty_challenge_task::{ backend_public_url, chat_command_display, legacy_sim_opt_in_present, resolve_scoring_backend, - ScoringBackend, CHALLENGE_ID, CHALLENGE_ID_BYTES as BOUNTY_ID_BYTES, CHAT_COMMAND_PLACEHOLDER, + EmitterOutcomeKind, EmitterStatus, EmitterStatusView, EmitterTick, ScoringBackend, + CHALLENGE_ID, CHALLENGE_ID_BYTES as BOUNTY_ID_BYTES, CHAT_COMMAND_PLACEHOLDER, SCORE_MAX as BOUNTY_SCORE_MAX, SCORING_VERSION, TERMS_TEXT, }; pub use bounty_http::{bounty_router, hash_admin_token, AppState}; diff --git a/crates/bounty-challenge/tests/emit_fail_closed.rs b/crates/bounty-challenge/tests/emit_fail_closed.rs index a01bb578c..fd910e0e0 100644 --- a/crates/bounty-challenge/tests/emit_fail_closed.rs +++ b/crates/bounty-challenge/tests/emit_fail_closed.rs @@ -1,6 +1,6 @@ //! Bounty emission is only as real as the backend feed behind it. //! -//! Three properties are load-bearing and none is visible from a unit test of +//! Five properties are load-bearing and none is visible from a unit test of //! the scorer: //! //! 1. A host that *can* read `{BOUNTY_BACKEND_PUBLIC_URL}/v1/bounty/public/*` @@ -10,9 +10,15 @@ //! `NoScore(ChallengeInternal)`, so the challenge share burns to uid 0 — //! while still covering `E`, because a paid challenge with no leaves fails //! D24 and takes every other challenge's seal down with it. -//! 3. A feed outage inside an already-scored epoch does not take back the -//! scores the backend really did publish. -//! 4. The leaderboard and the reports are separate GETs, so a snapshot is only +//! 3. A feed that answers and crowns nobody is the third case, and it is not +//! either of the first two: reachable, healthy, and worth nothing. It must +//! cover `E` with the same `ChallengeInternal` cover rather than sign +//! `NotAttempted` (which claims the challenge chose not to invoke anyone) +//! or report a successful score. +//! 4. A feed outage, or a readable feed that stops paying, inside an +//! already-scored epoch does not take back the scores the backend really +//! did publish. +//! 5. The leaderboard and the reports are separate GETs, so a snapshot is only //! signed once the feed holds still across both of them. #![forbid(unsafe_code)] @@ -26,8 +32,8 @@ use axum::extract::State; use axum::routing::{get, post}; use axum::{Json, Router}; use bounty_challenge::{ - fetch_public_snapshot, BackendError, BountyEmitter, EmitOutcome, GatewayClient, - GatewayClientConfig, + fetch_public_snapshot, BackendError, BountyEmitter, EmitOutcome, EmitterOutcomeKind, + GatewayClient, GatewayClientConfig, }; use chain::{ AxonInfo, ChainClient, ChainError, FakeChain, FakeChainConfig, Metagraph, WeightsTlockPayload, @@ -270,6 +276,123 @@ async fn spawn_stable_torn_backend() -> String { serve(app).await } +/// A backend that starts payable and can be switched to publishing nothing, +/// without changing its URL. That is the "backend stopped paying a hotkey it +/// had already crowned" case: still reachable, no longer payable. +async fn spawn_switchable_backend() -> (String, Arc) { + let payable = Arc::new(AtomicBool::new(true)); + let state = Arc::clone(&payable); + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(move |State(payable): State>| async move { + if payable.load(Ordering::Relaxed) { + Json(leaderboard_json()) + } else { + Json(serde_json::json!({ "items": [] })) + } + }), + ) + .route( + "/v1/bounty/public/reports", + get(move |State(payable): State>| async move { + if payable.load(Ordering::Relaxed) { + Json(reports_json()) + } else { + Json(serde_json::json!({ "items": [] })) + } + }), + ) + .with_state(state); + (serve(app).await, payable) +} + +/// A backend that answers 200 on both routes with nothing published yet: no +/// reports adjudicated, so no leaderboard row to agree with. This is the shape +/// of a backend that is *up* and has crowned nobody — a reachable feed that +/// still produces no weight, which is not the same failure as an outage. +async fn spawn_empty_backend() -> String { + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(|| async { Json(serde_json::json!({ "items": [] })) }), + ) + .route( + "/v1/bounty/public/reports", + get(|| async { Json(serde_json::json!({ "items": [] })) }), + ); + serve(app).await +} + +/// A backend that publishes reports but never adjudicates them: every row is +/// still `pending`, which is not scorable. Reachable, justified-looking, and +/// worth nothing — the trap an operator has to be able to see. +async fn spawn_all_pending_backend() -> String { + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(|| async { + Json(serde_json::json!({ + "items": [{ "hotkey": hex::encode(CHAMPION), "valid_count": 0 }] + })) + }), + ) + .route( + "/v1/bounty/public/reports", + get(|| async { + Json(serde_json::json!({ + "items": [{ + "id": "pending-0", + "hotkey": hex::encode(CHAMPION), + "status": "pending", + "problem_found": "something may be wrong", + "adjudicator": "bounty-adjudicator@cortex", + "justification": "awaiting triage", + "adjudicated_at": "2026-08-30T00:00:00Z", + "created_at": "2026-08-29T00:00:00Z", + }] + })) + }), + ); + serve(app).await +} + +/// A backend whose champion is an unpriced `valid`: adjudicated, justified, +/// and still not creditable, because nobody priced the bug. The crown gate +/// refuses it, so the feed is readable and pays nobody. +async fn spawn_unpriced_backend() -> String { + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(|| async { + Json(serde_json::json!({ + "items": [{ "hotkey": hex::encode(CHAMPION), "valid_count": 3 }] + })) + }), + ) + .route( + "/v1/bounty/public/reports", + get(|| async { + let items: Vec<_> = (0..3) + .map(|i| { + serde_json::json!({ + "id": format!("unpriced-{i}"), + "hotkey": hex::encode(CHAMPION), + "status": "valid", + "problem_found": format!("regression {i}"), + "adjudicator": "bounty-adjudicator@cortex", + "justification": "reproduced on master", + "adjudicated_at": "2026-08-30T00:00:00Z", + "created_at": "2026-08-29T00:00:00Z", + }) + }) + .collect(); + Json(serde_json::json!({ "items": items })) + }), + ); + serve(app).await +} + /// Matching `valid_count` with distinct envelope revisions: Greptile's stable /// torn case (leaderboard A, reports B) where a count-only check would pass. async fn spawn_stable_torn_revision_backend() -> String { @@ -606,4 +729,256 @@ async fn an_outage_after_a_scored_epoch_holds_instead_of_burning_it() { > 0, "the champion's score must still stand" ); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Held); + assert!(!view.last_feed_read, "an outage is not a read feed"); +} + +// --- The feed is up and pays nobody ----------------------------------------- +// +// A reachable backend with no payable adjudication is the failure mode that +// looks healthy from every other angle: `/health` is up, `can_score` is true, +// the feed answers 200, and the epoch still pays nobody. It must not be signed +// as a scored epoch, and it must not read like an outage either. + +/// An empty feed is reachable and crowns nobody. Signing it as `Scored` would +/// emit `NotAttempted` for every participant — a claim that the challenge +/// chose not to invoke them — and report a healthy tick while paying nothing. +/// The honest leaf is the same `ChallengeInternal` cover a burn uses. +#[tokio::test] +async fn a_readable_feed_with_no_rows_covers_e_instead_of_claiming_a_score() { + let backend = spawn_empty_backend().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + match em.tick().await.expect("cover covers E") { + EmitOutcome::Unpaid { + epoch, + participants, + reason, + } => { + assert_eq!(epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert_eq!(participants, 3); + assert!(reason.contains("no payable rows"), "{reason}"); + } + other => panic!("a readable feed that pays nobody must not score: {other:?}"), + } + assert_eq!(accepted_count(&accepted), 3); + assert_burn_covers_e(&accepted); + assert_eq!( + em.scored_epoch(), + 0, + "covering E without paying is not a score and must not mark the epoch scored" + ); + // The whole point of the separate outcome: an operator can tell this apart + // from an outage. + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Unpaid); + assert!( + view.last_feed_read, + "the feed answered; this is not an outage" + ); + assert_eq!(view.last_paid, 0); + assert_eq!(view.last_participants, 3); +} + +/// Reports that are published but still `pending` are not adjudications. The +/// feed is up and looks populated, and it is worth exactly nothing. +#[tokio::test] +async fn pending_only_rows_are_reachable_and_unpayable() { + let backend = spawn_all_pending_backend().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("cover covers E"), + EmitOutcome::Unpaid { .. } + )); + assert_burn_covers_e(&accepted); + assert_eq!(em.scored_epoch(), 0); + let view = em.status().view(); + assert!(view.last_feed_read); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Unpaid); +} + +/// A `valid` row nobody priced is adjudicated and justified, and still not +/// creditable: the crown gate refuses it. The feed is readable, so this is an +/// unpaid epoch rather than an outage. +#[tokio::test] +async fn an_unpriced_valid_row_leaves_the_epoch_unpaid_not_burned() { + let backend = spawn_unpriced_backend().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("cover covers E"), + EmitOutcome::Unpaid { .. } + )); + assert_burn_covers_e(&accepted); + assert_eq!(em.scored_epoch(), 0); + assert!(em.status().view().last_feed_read); +} + +/// The direction that protects a real payout: once the feed has crowned +/// someone this epoch, a later readable-but-unpaid tick must not take it back +/// by covering the epoch. This is the "backend stopped paying a hotkey it had +/// already crowned" case, and it holds rather than burns. +#[tokio::test] +async fn a_readable_feed_that_stops_paying_holds_a_scored_epoch() { + let (backend, payable) = spawn_switchable_backend().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("scored"), + EmitOutcome::Scored { paid: 1, .. } + )); + let after_scored = accepted_count(&accepted); + + // The feed stays up, but it no longer publishes anything payable. + payable.store(false, Ordering::Relaxed); + match em.tick().await.expect("hold") { + EmitOutcome::Held { epoch, reason } => { + assert_eq!(epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert!(reason.contains("no payable rows"), "{reason}"); + } + other => panic!("a scored epoch must not be taken back: {other:?}"), + } + assert_eq!( + accepted_count(&accepted), + after_scored, + "holding must post nothing at all" + ); + assert!( + leaf_for(&accepted, CHAMPION)["score_or_absence"]["score"]["value"] + .as_u64() + .unwrap_or_default() + > 0, + "the champion's score must still stand" + ); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Held); + assert!(view.last_feed_read); + assert_eq!( + view.scored_epoch, + chain::fake_defaults::SUBNET_EPOCH_INDEX, + "the hold must not forget the epoch was scored" + ); +} + +/// `/v1/status` has to distinguish "can this host pay" from "is it paying". +/// Before any tick, `wired` is true and nothing has happened yet. +#[tokio::test] +async fn status_reports_a_wired_emitter_that_has_not_ticked_yet() { + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(None, &gateway); + let view = em.status().view(); + assert!(view.wired); + assert_eq!(view.ticks, 0); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Never); + assert!(!view.last_feed_read); + assert_eq!(view.last_paid, 0); +} + +/// A scored tick publishes what was actually paid, so an operator can see the +/// reward linkage land rather than infer it from a log line. +#[tokio::test] +async fn a_scored_tick_publishes_what_it_paid() { + let backend = spawn_backend().await; + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("scored"), + EmitOutcome::Scored { paid: 1, .. } + )); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Scored); + assert!(view.last_feed_read); + assert_eq!(view.ticks, 1); + assert_eq!(view.last_paid, 1); + assert_eq!(view.last_participants, 3); + assert_eq!(view.last_epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert_eq!(view.scored_epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert!(view.last_reason.is_empty()); + assert!(view.last_error.is_empty()); +} + +/// A tick that could not emit at all must say so on the status surface: this +/// is the case that leaves `E` uncovered and 409s the seal. +#[tokio::test] +async fn a_chain_failure_is_reported_as_an_error_not_a_payout() { + let (gateway, accepted) = spawn_gateway().await; + // Epoch 0 is the "subnet has not run an epoch yet" refusal. + let chain = LockedFake(Mutex::new(FakeChain::new(FakeChainConfig { + netuid: NETUID, + subnet_epoch_index: 0, + ..FakeChainConfig::default() + }))); + let gateway_client = Arc::new( + GatewayClient::new(GatewayClientConfig { + base_url: gateway.clone(), + ..GatewayClientConfig::default() + }) + .expect("gateway client"), + ); + let em = BountyEmitter::new(chain, gateway_client, [7u8; 32], NETUID, None); + + let err = em.tick().await.expect_err("epoch 0 cannot emit"); + assert!( + matches!(err, bounty_challenge::EmitError::EpochZero), + "{err}" + ); + assert_eq!(accepted_count(&accepted), 0, "nothing was posted"); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Error); + assert!(view.last_error.contains("epoch 0"), "{:?}", view.last_error); + assert_eq!(view.ticks, 1); +} + +/// A tick that read the feed and then failed at the gateway is an error, but +/// it is *not* a backend outage. Reporting `last_feed_read: false` there would +/// send an operator to the wrong service — the live check that found this +/// showed exactly that shape against a healthy stand-in feed. +#[tokio::test] +async fn a_gateway_failure_after_a_read_still_reports_the_feed_as_read() { + let backend = spawn_backend().await; + // Port 1 is closed: the feed is fine, the gateway is not. + let em = emitter(Some(backend), "http://127.0.0.1:1"); + + let err = em + .tick() + .await + .expect_err("a dead gateway cannot accept leaves"); + assert!( + matches!(err, bounty_challenge::EmitError::Submit(_)), + "the failure must be the submit, not the read: {err}" + ); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Error); + assert!( + view.last_feed_read, + "the feed answered; blaming the backend would be wrong" + ); + assert!( + view.last_error.contains("gateway"), + "the error must name the gateway: {:?}", + view.last_error + ); + assert_eq!(view.last_paid, 0); +} + +/// The mirror image: a tick that never reached the feed must not claim it did. +#[tokio::test] +async fn a_failed_read_is_not_reported_as_a_read_feed() { + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(Some("http://127.0.0.1:1".to_owned()), &gateway); + + assert!(matches!( + em.tick().await.expect("cover E"), + EmitOutcome::Burned { .. } + )); + let view = em.status().view(); + assert!(!view.last_feed_read); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Burned); } diff --git a/crates/bounty-challenge/tests/rewards_linkage.rs b/crates/bounty-challenge/tests/rewards_linkage.rs new file mode 100644 index 000000000..61bb3def4 --- /dev/null +++ b/crates/bounty-challenge/tests/rewards_linkage.rs @@ -0,0 +1,773 @@ +//! The full Bounty reward linkage: a paired miner's report becomes weight. +//! +//! Every other test in this crate stops at one link. This one walks the whole +//! chain, because each link can pass on its own while the chain pays nobody: +//! +//! ```text +//! miner hotkey ──pair──▶ Chat account ──report──▶ operator adjudicate +//! │ │ +//! │ CortexLM/backend publishes +//! │ │ +//! └──metagraph `E`──◀── leaf emitter ◀──public feed +//! │ +//! POST /v1/weights/raw +//! │ +//! admin seal ──▶ GET /v1/weights/latest (sealed) +//! ``` +//! +//! Two properties are the point of the file: +//! +//! 1. **The pairing is not decorative.** The hotkey that signs the pairing +//! challenge is the hotkey the published row credits, and it is the hotkey +//! that gets the positive leaf. A miner who cannot pair, or whose report +//! lands under a different key, is never paid — so the test signs a real +//! sr25519 challenge and drives the real HTTP routes. +//! 2. **Bounty cannot take the subnet down with it.** Bounty holds a paid +//! trust-root row, so an epoch where it produces no weight must still cover +//! `E`; otherwise D24 fails and `POST /v1/admin/seal` 409s for *every* +//! challenge, including the one that did score. The seal at the end of this +//! file proves both directions in one bundle: bounty pays its champion, and +//! a challenge that read nothing still seals. +//! +//! Nothing here reads the real CortexLM/backend. The feed is a stand-in that +//! serves the two public routes; that is exactly what the operator points +//! `BOUNTY_BACKEND_PUBLIC_URL` at, and the DTO is the published contract. + +#![forbid(unsafe_code)] +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; + +use axum::body::Body; +use axum::extract::State; +use axum::http::{Request, StatusCode}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use bounty_challenge::{ + bounty_router, hash_admin_token, AppState, BountyEmitter, BountyStore, EmitOutcome, + EmitterOutcomeKind, GatewayClient, GatewayClientConfig, ScoringBackend, +}; +use bounty_challenge_task::{ + hotkey_hex, hotkey_ss58, pairing_code, public_from_mini_secret, sign_pair_challenge, + PairChallenge, +}; +use bundle::{ + build_sealed_bundle, make_signed_leaf, verify_bundle, LeafV1, LocalTrustRoot, + NoScoreReasonCode, ScoreOrAbsence, SealParams, +}; +use chain::{ChainClient, ChainError, FakeChain, FakeChainConfig, Metagraph}; +use challenge_common::{emit_signed_leaf_set, public_key_from_secret, verify_leaf_sig}; +use http_body_util::BodyExt; +use tokio::net::TcpListener; +use tower::ServiceExt; +use trustroot::{ + measurements_digest, ChallengeEntry, ChallengesBody, MeasurementsBody, ParticipantPolicy, + BPS_DENOM, +}; + +/// Netuid the fake chain serves. +const NETUID: u16 = 541; + +/// The miner's hotkey mini-secret. The *hotkey* is its sr25519 public key, so +/// the pairing signature, the published row, and the metagraph entry are all +/// the same key — which is the property this file exists to check. +const MINER_SECRET: [u8; 32] = [0xA1; 32]; + +/// A metagraph hotkey that filed nothing. It still has to appear in `E`. +const SILENT: [u8; 32] = [0xC3; 32]; + +/// The miner's public hotkey, as the metagraph and the pairing both see it. +fn miner_hotkey() -> [u8; 32] { + public_from_mini_secret(&MINER_SECRET).expect("miner pk") +} + +/// Challenge leaf-signing keys (throwaway, in-test only). +const BOUNTY_SK: [u8; 32] = [0x11; 32]; +const PROOF_SK: [u8; 32] = [0x22; 32]; +const GATEWAY_SK: [u8; 32] = [0x33; 32]; + +/// `FakeChain` keeps its call log in a `RefCell`; the emitter needs `Sync`. +struct LockedFake(Mutex); + +macro_rules! delegate { + (fn $name:ident(&self) -> $ret:ty) => { + fn $name(&self) -> $ret { + self.0.lock().expect("lock").$name() + } + }; + (fn $name:ident(&self, $($arg:ident : $t:ty),*) -> $ret:ty) => { + fn $name(&self, $($arg: $t),*) -> $ret { + self.0.lock().expect("lock").$name($($arg),*) + } + }; +} + +impl ChainClient for LockedFake { + delegate!(fn current_block(&self) -> Result); + delegate!(fn block_hash(&self, n: u64) -> Result<[u8; 32], ChainError>); + delegate!(fn metagraph_at(&self, block_hash: &[u8; 32]) -> Result); + delegate!(fn subnet_owner_hotkey(&self, netuid: u16) -> Result, ChainError>); + delegate!(fn axon(&self, netuid: u16, hotkey: &[u8]) -> Result, ChainError>); + delegate!(fn axons(&self, netuid: u16) -> Result, chain::AxonInfo)>, ChainError>); + delegate!(fn commit_reveal_enabled(&self, netuid: u16) -> Result); + delegate!(fn commit_reveal_version(&self, netuid: u16) -> Result); + delegate!(fn tempo(&self, netuid: u16) -> Result); + delegate!(fn reveal_period_epochs(&self, netuid: u16) -> Result); + delegate!(fn block_time(&self) -> Result); + delegate!(fn last_epoch_block(&self, netuid: u16) -> Result); + delegate!(fn pending_epoch_at(&self, netuid: u16) -> Result); + delegate!(fn subnet_epoch_index(&self, netuid: u16) -> Result); + delegate!(fn blocks_since_last_step(&self, netuid: u16) -> Result); + delegate!(fn submit_timelocked_weights( + &self, + mecid: u8, + payload: chain::WeightsTlockPayload, + reveal_round: u64 + ) -> Result<(), ChainError>); + delegate!(fn set_weights( + &self, + netuid: u16, + uids: Vec, + values: Vec, + version_key: u64 + ) -> Result<(), ChainError>); +} + +fn fake_chain() -> FakeChain { + FakeChain::new(FakeChainConfig { + netuid: NETUID, + hotkeys: vec![miner_hotkey().to_vec(), SILENT.to_vec()], + ..FakeChainConfig::default() + }) +} + +// --- the miner side: pair, report, adjudicate (real HTTP routes) ------------- + +/// A pairing payload signed by the miner's own hotkey, over the canonical +/// challenge string. `ctx bounty pair` builds exactly this shape. +fn signed_pair_payload(secret: &[u8; 32], account_id: &str) -> serde_json::Value { + let pk = public_from_mini_secret(secret).expect("pk"); + let challenge = PairChallenge { + account_id: account_id.into(), + nonce: "0123456789abcdef".into(), + exp: 2_000_000_000, + }; + let encoded = challenge.encode().expect("encode"); + let sig = sign_pair_challenge(secret, &encoded).expect("sign"); + // The code a miner pastes into Chat carries the same signature. + let code = pairing_code(&encoded, &hex::encode(sig), &hotkey_ss58(&pk)); + assert!(code.contains(&hotkey_ss58(&pk))); + serde_json::json!({ + "account_id": challenge.account_id, + "hotkey": hotkey_ss58(&pk), + "nonce": challenge.nonce, + "exp": challenge.exp, + "signature": hex::encode(sig), + "terms_accepted": true, + }) +} + +/// A report body that clears the substance floor. +fn report_body(session: &str) -> serde_json::Value { + serde_json::json!({ + "session": session, + "title": "seal returns 500 on an empty bundle", + "body": "POST /v1/admin/seal answers 500 when the bundle has no leaves, \ + instead of the documented 400. Observed on master at commit tip.", + "repro_steps": "curl the seal route with no leaves posted and watch it 500", + }) +} + +async fn json_req( + app: &Router, + method: &str, + uri: &str, + body: serde_json::Value, + auth: Option<&str>, +) -> (StatusCode, serde_json::Value) { + let mut b = Request::builder().method(method).uri(uri); + if let Some(a) = auth { + b = b.header(axum::http::header::AUTHORIZATION, format!("Bearer {a}")); + } + let req = b + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("req"); + let resp = app.clone().oneshot(req).await.expect("resp"); + let status = resp.status(); + let bytes = resp.into_body().collect().await.expect("body").to_bytes(); + let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap_or(serde_json::json!({})); + (status, v) +} + +/// Drive the real challenge routes: pair a hotkey, file a report, adjudicate it +/// `valid` at `severity`. Returns the adjudicated report body. +async fn pair_report_adjudicate(operator_token: &str) -> serde_json::Value { + let app = bounty_router(AppState { + store: BountyStore::new(), + session_secret: Arc::new(b"test-session-secret".to_vec()), + scoring: ScoringBackend::BackendPublic, + admin_hashes: Arc::new(vec![hash_admin_token(operator_token)]), + emitter: None, + }); + + let (st, paired) = json_req( + &app, + "POST", + "/v1/pair", + signed_pair_payload(&MINER_SECRET, "acct-miner-rewards"), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "pair: {paired}"); + // The binding is the miner's own hotkey, not something the host invented. + assert_eq!(paired["miner_hotkey"], hotkey_hex(&miner_hotkey())); + let session = paired["session"].as_str().expect("session"); + + let (st, created) = json_req(&app, "POST", "/v1/reports", report_body(session), None).await; + assert_eq!(st, StatusCode::CREATED, "report: {created}"); + assert_eq!(created["state"], "pending"); + + let (st, adjudicated) = json_req( + &app, + "POST", + "/v1/admin/adjudicate", + serde_json::json!({ + "report_id": created["id"], + "verdict": "valid", + "severity": "major", + }), + Some(operator_token), + ) + .await; + assert_eq!(st, StatusCode::OK, "adjudicate: {adjudicated}"); + assert_eq!(adjudicated["adjudication"], "valid"); + assert_eq!(adjudicated["severity"], "major"); + adjudicated +} + +// --- the backend side: the published feed ------------------------------------ + +/// The backend's published view of the adjudicated report above. +/// +/// This is the handoff the whole challenge depends on: Cortex adjudicates +/// internally, CortexLM/backend publishes, and this host reads the published +/// rows. If the published row names a different hotkey than the one that +/// paired, the miner is paid nothing — which is why the assertion at the end +/// of the test is on `MINER`'s leaf rather than on "a leaf scored". +fn published_row(severity: &str) -> serde_json::Value { + serde_json::json!({ + "id": "report-1", + "hotkey": hotkey_ss58(&miner_hotkey()), + "status": "valid", + "severity": severity, + "problem_found": "seal returns 500 on an empty bundle", + "adjudicator": "bounty-adjudicator@cortex", + "justification": "reproduced on master at commit tip", + "adjudicated_at": "2026-09-01T00:00:00Z", + "created_at": "2026-09-01T00:00:00Z", + }) +} + +/// One justified, priced finding so the miner clears `MIN_HOLDOUT_DECIDED`. +fn payable_rows() -> Vec { + (1..=3) + .map(|i| { + let mut row = published_row("major"); + row["id"] = serde_json::json!(format!("report-{i}")); + row["problem_found"] = serde_json::json!(format!("regression {i} on the seal path")); + row + }) + .collect() +} + +/// Stand-in for `{BOUNTY_BACKEND_PUBLIC_URL}`: the two public routes. +/// +/// The leaderboard row carries the real `valid_count` for the published +/// reports. A count that did not match would be refused as a torn pair before +/// it could ever score, so the stand-in has to publish one coherent revision — +/// which is also what the live backend does. +async fn spawn_public_feed(rows: Vec) -> String { + let rows = Arc::new(rows); + let leaderboard = Arc::clone(&rows); + let reports = Arc::clone(&rows); + let app = Router::new() + .route( + "/v1/bounty/public/leaderboard", + get(move || { + let rows = Arc::clone(&leaderboard); + async move { + let valid = rows + .iter() + .filter(|r| r["status"] == serde_json::Value::String("valid".to_owned())) + .count(); + let items = if rows.is_empty() { + Vec::new() + } else { + vec![serde_json::json!({ + "hotkey": rows[0]["hotkey"], + "valid_count": valid, + })] + }; + Json(serde_json::json!({ "items": items })) + } + }), + ) + .route( + "/v1/bounty/public/reports", + get(move || { + let rows = Arc::clone(&reports); + async move { Json(serde_json::json!({ "items": *rows })) } + }), + ); + serve(app).await +} + +/// A feed that answers 503 on both routes — the shape of a backend outage. +async fn spawn_down_feed() -> String { + let app = Router::new().fallback(|| async { StatusCode::SERVICE_UNAVAILABLE }); + serve(app).await +} + +async fn serve(app: Router) -> String { + let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("bind"); + let addr = listener.local_addr().expect("addr"); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + format!("http://{addr}") +} + +/// Leaves the mock gateway accepted, in arrival order. +type Accepted = Arc>>; + +/// Stand-in for the master gateway's `POST /v1/weights/raw`. +async fn spawn_gateway() -> (String, Accepted) { + let accepted: Accepted = Arc::new(Mutex::new(Vec::new())); + let app = Router::new() + .route( + "/v1/weights/raw", + post( + |State(seen): State, Json(body): Json| async move { + seen.lock().expect("lock").push(body); + StatusCode::ACCEPTED + }, + ), + ) + .with_state(Arc::clone(&accepted)); + (serve(app).await, accepted) +} + +/// Rebuild the `LeafV1` the gateway received, so the seal below operates on +/// the exact leaves that were posted rather than a parallel re-derivation. +fn leaf_from_accepted(v: &serde_json::Value) -> LeafV1 { + let soa = match &v["score_or_absence"] { + s if s.get("score").is_some() => ScoreOrAbsence::Score { + value: s["score"]["value"].as_u64().expect("score value"), + }, + s => { + let reason = u8::try_from(s["no_score"]["reason"].as_u64().expect("reason")) + .expect("reason fits u8"); + let reason = match reason { + 0 => NoScoreReasonCode::NotAttempted, + 1 => NoScoreReasonCode::Timeout, + 2 => NoScoreReasonCode::InvalidResponse, + 3 => NoScoreReasonCode::AttestationNotVerified, + 4 => NoScoreReasonCode::MinerError, + 5 => NoScoreReasonCode::RateLimited, + 6 => NoScoreReasonCode::ChallengeInternal, + other => panic!("unexpected absence reason {other}"), + }; + ScoreOrAbsence::NoScore { reason } + } + }; + let mut miner_hotkey = [0u8; 32]; + miner_hotkey.copy_from_slice( + &hex::decode(v["miner_hotkey"].as_str().expect("miner_hotkey")).expect("hex"), + ); + let mut challenge_sig = [0u8; 64]; + challenge_sig.copy_from_slice( + &hex::decode(v["challenge_sig"].as_str().expect("challenge_sig")).expect("hex"), + ); + LeafV1 { + challenge_id: v["challenge_id"].as_str().expect("challenge_id").into(), + miner_hotkey, + epoch: v["epoch"].as_u64().expect("epoch"), + score_or_absence: soa, + challenge_sig, + } +} + +fn emitter(backend: Option, gateway_url: &str) -> BountyEmitter { + let gateway = Arc::new( + GatewayClient::new(GatewayClientConfig { + base_url: gateway_url.to_owned(), + ..GatewayClientConfig::default() + }) + .expect("gateway client"), + ); + BountyEmitter::new( + LockedFake(Mutex::new(fake_chain())), + gateway, + BOUNTY_SK, + NETUID, + backend, + ) +} + +/// The trust root a validator loads from disk: bounty 2000, proof 8000. +fn trust_root() -> LocalTrustRoot { + LocalTrustRoot { + challenges: ChallengesBody { + challenges: vec![ + ChallengeEntry { + id: b"bounty".to_vec(), + public_key: public_key_from_secret(&BOUNTY_SK).expect("bounty pk"), + emission_share_bps: 2000, + policy: ParticipantPolicy::AllMetagraphHotkeys, + }, + ChallengeEntry { + id: b"proof".to_vec(), + public_key: public_key_from_secret(&PROOF_SK).expect("proof pk"), + emission_share_bps: 8000, + policy: ParticipantPolicy::AllMetagraphHotkeys, + }, + ], + }, + measurements_digest: measurements_digest(&MeasurementsBody::default()), + } +} + +/// Cover `E` for a challenge that produced no weight, exactly as that +/// challenge's own emitter does when it cannot score. +fn cover_with_noscore( + sk: &[u8; 32], + challenge_id: &[u8], + epoch: u64, + hotkeys: &BTreeSet<[u8; 32]>, + reason: NoScoreReasonCode, +) -> BTreeMap<[u8; 32], LeafV1> { + let scores: BTreeMap<[u8; 32], ScoreOrAbsence> = hotkeys + .iter() + .map(|h| (*h, ScoreOrAbsence::NoScore { reason })) + .collect(); + emit_signed_leaf_set(sk, challenge_id, epoch, hotkeys, &scores).expect("cover E") +} + +fn expected_set() -> BTreeSet<[u8; 32]> { + let mut e = BTreeSet::new(); + e.insert(miner_hotkey()); + e.insert(SILENT); + e +} + +// --- the tests --------------------------------------------------------------- + +/// The happy path, end to end: pair → report → adjudicate → publish → leaves. +/// +/// The assertion that matters is that `MINER` — the hotkey that signed the +/// pairing challenge — is the one holding a positive leaf. Every intermediate +/// step could be green while the credit lands on a different key, and this is +/// the only place that catches it. +#[tokio::test] +async fn a_paired_miners_adjudicated_report_becomes_a_scored_leaf() { + // 1. The miner pairs and files; an operator adjudicates. + let adjudicated = pair_report_adjudicate("op-token").await; + assert_eq!(adjudicated["miner_hotkey"], hotkey_hex(&miner_hotkey())); + + // 2. The backend publishes that adjudication on the public feed. + let feed = spawn_public_feed(payable_rows()).await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(feed), &gateway); + + // 3. The emitter turns the published rows into signed leaves for `E`. + let epoch = match em.tick().await.expect("tick") { + EmitOutcome::Scored { + epoch, + participants, + paid, + .. + } => { + assert_eq!(participants, 2, "every hotkey in E needs a leaf"); + assert_eq!(paid, 1, "exactly the adjudicated miner was paid"); + epoch + } + other => panic!("a readable, payable feed must score: {other:?}"), + }; + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Scored); + assert_eq!(view.last_paid, 1); + assert!(view.last_feed_read); + + // 4. The leaf the gateway received verifies under the trust-root key, and + // it is `MINER`'s — the hotkey that signed the pairing challenge. + let bounty_pk = public_key_from_secret(&BOUNTY_SK).expect("pk"); + let rows = accepted.lock().expect("lock").clone(); + assert_eq!(rows.len(), 2); + let miner_leaf = rows + .iter() + .find(|v| v["miner_hotkey"] == serde_json::Value::String(hotkey_hex(&miner_hotkey()))) + .expect("a leaf for the paired miner"); + let leaf = leaf_from_accepted(miner_leaf); + verify_leaf_sig(&leaf, &bounty_pk).expect("the sealed leaf must verify under the trust root"); + assert_eq!(leaf.challenge_id, b"bounty"); + assert_eq!(leaf.epoch, epoch); + assert!( + matches!(leaf.score_or_absence, ScoreOrAbsence::Score { value } if value > 0), + "the adjudicated report must pay: {leaf:?}" + ); + + // 5. The hotkey that filed nothing is explicit, never silently omitted — + // an omission here is what fails D24 at the seal. + let silent_leaf = rows + .iter() + .find(|v| v["miner_hotkey"] == serde_json::Value::String(hotkey_hex(&SILENT))) + .expect("a leaf for the silent hotkey"); + assert_eq!( + silent_leaf["score_or_absence"]["no_score"]["reason"], 0, + "NotAttempted: the challenge did invoke E, this hotkey just has no rows" + ); +} + +/// The bundle a validator actually fetches: bounty's paid leaf and proof's +/// no-score cover seal together. +/// +/// This is the cross-challenge property. Bounty holds a paid trust-root row, so +/// if its emitter ever left `E` uncovered the seal would 409 for *proof* too. +/// The test asserts both halves at once — the paid vector and the fact that a +/// challenge with nothing to pay is not a reason to fail the epoch. +#[tokio::test] +async fn a_paid_bounty_seals_alongside_a_challenge_that_scored_nothing() { + let feed = spawn_public_feed(payable_rows()).await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(feed), &gateway); + let epoch = match em.tick().await.expect("tick") { + EmitOutcome::Scored { epoch, .. } => epoch, + other => panic!("expected a score: {other:?}"), + }; + + let e = expected_set(); + let mut leaves: Vec = accepted + .lock() + .expect("lock") + .iter() + .map(leaf_from_accepted) + .collect(); + // Proof had nothing to score, so it covers `E` the same way bounty does on + // an outage. Bounty's leaves are already in the set. + leaves.extend( + cover_with_noscore( + &PROOF_SK, + b"proof", + epoch, + &e, + NoScoreReasonCode::ChallengeInternal, + ) + .into_values(), + ); + + let trust = trust_root(); + let chain = fake_chain(); + let block_b = chain::fake_defaults::LAST_EPOCH_BLOCK; + let bundle = build_sealed_bundle( + &chain, + &trust, + leaves, + &SealParams { + epoch, + netuid: NETUID, + block_b, + gateway_secret: GATEWAY_SK, + }, + ) + .expect("D24 completeness holds: both paid challenges covered E"); + + // A validator's own verification path, against the same trust root. + verify_bundle(&bundle, &chain, &trust).expect("the bundle a validator fetches must verify"); + + // Emission shares are the two live challenges, at the configured split. + let shares: Vec<(String, u16)> = bundle + .body + .emission_shares + .iter() + .map(|(id, bps)| (String::from_utf8_lossy(id).into_owned(), *bps)) + .collect(); + assert_eq!( + shares, + vec![("bounty".to_owned(), 2000), ("proof".to_owned(), 8000)], + "the sealed split is the trust root's, not a default" + ); + + // Bounty's 2000 bps went to the miner who filed the report; proof's 8000 + // bps burned to uid 0 because it had nothing payable. The point is that the + // vector exists at all: an uncovered `E` would have 409'd before this. + let miner_uid = bundle + .body + .uid_map + .iter() + .find(|(h, _)| *h == miner_hotkey()) + .map(|(_, uid)| *uid) + .expect("the miner is in the sealed metagraph"); + let miner_weight = bundle + .body + .final_vector + .iter() + .find(|(uid, _)| *uid == miner_uid) + .map(|(_, w)| *w) + .expect("the miner has a weight"); + assert!( + miner_weight > 0, + "the adjudicated miner must hold weight: {:?}", + bundle.body.final_vector + ); + let silent_uid = bundle + .body + .uid_map + .iter() + .find(|(h, _)| *h == SILENT) + .map(|(_, uid)| *uid) + .expect("the silent hotkey is in the sealed metagraph"); + assert!( + !bundle + .body + .final_vector + .iter() + .any(|(uid, w)| *uid == silent_uid && *w > 0), + "a hotkey that filed nothing must not be paid: {:?}", + bundle.body.final_vector + ); +} + +/// The fail-closed direction, at the seal: a bounty host that cannot read the +/// feed still seals the epoch, and still pays nobody. +/// +/// Both halves are required. Paying nobody is the honest outcome; covering `E` +/// is what keeps the 409 off proof's seal. A host that did neither would take +/// the whole subnet's weights down with it. +#[tokio::test] +async fn an_unreadable_feed_pays_nobody_without_breaking_the_seal() { + let down = spawn_down_feed().await; + let (gateway, accepted) = spawn_gateway().await; + let em = emitter(Some(down), &gateway); + + let epoch = match em.tick().await.expect("cover E") { + EmitOutcome::Burned { + epoch, + participants, + reason, + } => { + assert_eq!(participants, 2); + assert!(reason.contains("503"), "{reason}"); + epoch + } + other => panic!("a down feed must burn, not score: {other:?}"), + }; + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Burned); + assert!(!view.last_feed_read, "an outage is not a read feed"); + assert_eq!(view.last_paid, 0); + assert_eq!(em.scored_epoch(), 0); + + // Every leaf is the ChallengeInternal cover: nobody is paid. + let posted = accepted.lock().expect("lock").clone(); + assert_eq!(posted.len(), 2); + for leaf in &posted { + assert_eq!( + leaf["score_or_absence"]["no_score"]["reason"], 6, + "an unreadable feed must cover E with ChallengeInternal: {leaf}" + ); + } + + let e = expected_set(); + let mut leaves: Vec = posted.iter().map(leaf_from_accepted).collect(); + leaves.extend( + cover_with_noscore( + &PROOF_SK, + b"proof", + epoch, + &e, + NoScoreReasonCode::ChallengeInternal, + ) + .into_values(), + ); + + let trust = trust_root(); + let chain = fake_chain(); + let bundle = build_sealed_bundle( + &chain, + &trust, + leaves, + &SealParams { + epoch, + netuid: NETUID, + block_b: chain::fake_defaults::LAST_EPOCH_BLOCK, + gateway_secret: GATEWAY_SK, + }, + ) + .expect("a burn still covers E, so the epoch seals for every challenge"); + verify_bundle(&bundle, &chain, &trust).expect("verify"); + + assert!( + bundle.body.final_vector.iter().all(|(uid, _)| *uid == 0), + "a burn epoch must sink to uid 0: {:?}", + bundle.body.final_vector + ); + let weights: u32 = bundle + .body + .final_vector + .iter() + .map(|(_, w)| u32::from(*w)) + .sum(); + assert!(weights > 0, "the burn vector must still be sealable"); +} + +/// A leaf signed by the wrong key does not verify. This is the guard on the +/// assertion above: `verify_leaf_sig` passing is evidence, not a tautology. +#[tokio::test] +async fn a_leaf_from_another_challenge_key_does_not_verify() { + let trust = trust_root(); + let bounty_pk = trust + .challenges + .get(b"bounty") + .expect("bounty row") + .public_key; + let proof_pk = trust + .challenges + .get(b"proof") + .expect("proof row") + .public_key; + assert_ne!(bounty_pk, proof_pk, "the two rows carry distinct keys"); + + let leaf = make_signed_leaf( + &PROOF_SK, + b"bounty", + miner_hotkey(), + 1, + ScoreOrAbsence::Score { value: 1 }, + ) + .expect("leaf"); + assert!( + verify_leaf_sig(&leaf, &bounty_pk).is_err(), + "a proof-key signature must not verify as a bounty leaf" + ); + verify_leaf_sig(&leaf, &proof_pk).expect("it verifies under its own key"); +} + +/// `BPS_DENOM` is what `ChallengesBody::validate` enforces; the test trust root +/// above must satisfy it or the fixture is not the shape validators load. +#[test] +fn the_test_trust_root_is_a_valid_split() { + let trust = trust_root(); + trust.challenges.validate().expect("valid split"); + let total: u32 = trust + .challenges + .challenges + .iter() + .map(|c| u32::from(c.emission_share_bps)) + .sum(); + assert_eq!(total, u32::from(BPS_DENOM)); +} diff --git a/crates/bounty-challenge/tests/trust_root_linkage.rs b/crates/bounty-challenge/tests/trust_root_linkage.rs new file mode 100644 index 000000000..19b799b68 --- /dev/null +++ b/crates/bounty-challenge/tests/trust_root_linkage.rs @@ -0,0 +1,211 @@ +//! The committed trust root must keep bounty live and payable. +//! +//! Bounty's reward linkage has a precondition that lives outside its own code: +//! the challenge has to be in the owner-signed trust root with a nonzero +//! emission share, and the keys the emitter signs with have to be the keys that +//! root names. Nothing in the emitter can check either one — a host whose +//! `bounty_sk` does not match the root's `bounty` row signs leaves that every +//! validator rejects, and a host whose root has no bounty row emits into a +//! challenge that does not exist. +//! +//! These tests read the files an operator actually deploys +//! (`config/challenges.toml` and the staging override) and verify them the way +//! a validator does: owner signature, then the row the reward path depends on. +//! The point is not to restate the numbers — it is that changing them is a +//! decision, not a drift. + +#![forbid(unsafe_code)] +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::path::{Path, PathBuf}; + +use bounty_challenge_task::{CHALLENGE_ID, SCORING_VERSION}; +use trustroot::{load_challenges_file, ChallengesBody, ChallengesToml, BPS_DENOM}; + +/// Bounty's committed share. Proof holds the remainder; the pair must sum to +/// `BPS_DENOM` or `ChallengesBody::validate` refuses the document outright. +const BOUNTY_BPS: u16 = 2000; + +/// Proof's committed share (20/80 with bounty). +const PROOF_BPS: u16 = 8000; + +fn repo_config_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../config") + .canonicalize() + .expect("config dir") +} + +fn owner_public() -> [u8; 32] { + trustroot::load_owner_public_key(&repo_config_dir().join("owner.pubkey")).expect("owner pubkey") +} + +/// Load a root the way a validator does: verify the owner signature first, so +/// a hand-edited row without a re-sign fails here rather than in production. +fn verified(body_path: &Path) -> ChallengesBody { + load_challenges_file(body_path, &owner_public()) + .unwrap_or_else(|e| { + panic!( + "{} must verify under the committed owner: {e}", + body_path.display() + ) + }) + .body +} + +fn row<'a>(body: &'a ChallengesBody, id: &str) -> &'a trustroot::ChallengeEntry { + body.get(id.as_bytes()) + .unwrap_or_else(|| panic!("{id} row missing from the trust root")) +} + +fn files() -> Vec<(&'static str, PathBuf)> { + let dir = repo_config_dir(); + vec![ + ("prod", dir.join("challenges.toml")), + ("staging", dir.join("challenges.staging.toml")), + ] +} + +/// Bounty is a live, payable challenge in every committed root. +/// +/// An emission share of 0 is not a degraded bounty — it is a challenge that +/// cannot pay anyone while still being asked for leaves, and `assert_ +/// participant_completeness` skips 0-bps rows entirely, so the emitter would be +/// posting into a set nothing seals. +#[test] +fn bounty_is_live_and_payable_in_every_committed_root() { + for (label, path) in files() { + let body = verified(&path); + body.validate() + .unwrap_or_else(|e| panic!("{label}: committed root must validate: {e}")); + + let bounty = row(&body, CHALLENGE_ID); + assert_eq!( + bounty.emission_share_bps, BOUNTY_BPS, + "{label}: bounty share drifted from the committed {BOUNTY_BPS} bps" + ); + assert!( + bounty.emission_share_bps > 0, + "{label}: a 0-bps bounty is not payable" + ); + assert_eq!( + bounty.policy, + trustroot::ParticipantPolicy::AllMetagraphHotkeys, + "{label}: bounty derives E from the whole metagraph" + ); + + // Proof is the other live row; the two shares are the whole budget. + let proof = row(&body, "proof"); + assert_eq!(proof.emission_share_bps, PROOF_BPS, "{label}: proof share"); + + let total: u32 = body + .challenges + .iter() + .map(|c| u32::from(c.emission_share_bps)) + .sum(); + assert_eq!( + total, + u32::from(BPS_DENOM), + "{label}: shares must sum to {BPS_DENOM}" + ); + assert_eq!(body.challenges.len(), 2, "{label}: two live challenges"); + } +} + +/// The challenge id the emitter signs under is the id the root publishes. +/// +/// `CHALLENGE_ID_BYTES` is what `emit_signed_leaf_set` puts in every leaf, and +/// `ChallengesBody::get` is what the gateway and validators look it up by. A +/// mismatch would make every leaf unattributable — and it would look like a +/// signature failure, which is the wrong place to go looking. +#[test] +fn the_emitters_challenge_id_is_the_trust_roots_id() { + assert_eq!(CHALLENGE_ID, "bounty"); + for (label, path) in files() { + let body = verified(&path); + let ids: Vec = body + .challenges + .iter() + .map(|c| String::from_utf8_lossy(&c.id).into_owned()) + .collect(); + assert!( + ids.iter().any(|id| id == CHALLENGE_ID), + "{label}: no row for the id the emitter signs under: {ids:?}" + ); + } +} + +/// The retired products stay absent. A leftover row would restore an emission +/// the owner removed, and `relearn` / `design` / `prism` have no code left to +/// serve it. +#[test] +fn retired_products_have_no_row_in_any_committed_root() { + for (label, path) in files() { + let body = verified(&path); + for off in [ + "relearn", + "relearn-image", + "relearn-agent", + "relearn-mm", + "design", + "prism", + ] { + assert!( + body.get(off.as_bytes()).is_none(), + "{label}: {off} must not have a trust-root row" + ); + } + } +} + +/// Staging mirrors production's split. +/// +/// Staging exists to prove the production path, so a split that diverges there +/// would let the staging soak pass while prod seals a different vector. The +/// validator compares emission shares against its local root (D23), so a +/// divergence also shows up as a staging-only failure. +#[test] +fn staging_mirrors_prod_shares_and_keys() { + let dir = repo_config_dir(); + let prod = verified(&dir.join("challenges.toml")); + let staging = verified(&dir.join("challenges.staging.toml")); + + for id in [CHALLENGE_ID, "proof"] { + let p = row(&prod, id); + let s = row(&staging, id); + assert_eq!( + p.emission_share_bps, s.emission_share_bps, + "{id}: staging share must mirror prod" + ); + assert_eq!( + p.public_key, s.public_key, + "{id}: staging key must mirror prod, or a staging leaf cannot be \ + verified against the same trust root shape" + ); + } +} + +/// The committed roots parse as the deployed TOML shape, and the parsed body +/// re-derives the same rows. This catches a file that verifies but carries a +/// field the loader drops. +#[test] +fn committed_roots_round_trip_through_the_toml_shape() { + for (label, path) in files() { + let text = std::fs::read_to_string(&path).expect("read"); + let doc: ChallengesToml = toml::from_str(&text).expect("toml shape"); + let body = doc.to_body().expect("to_body"); + assert_eq!( + body, + verified(&path), + "{label}: the parsed document must equal the verified body" + ); + assert_eq!(doc.version, 1, "{label}: document version"); + assert_eq!(doc.introduced_epoch, 0, "{label}: introduced_epoch"); + } +} + +/// The scoring version is part of the published identity a miner checks. +#[test] +fn the_scoring_version_is_pinned() { + assert_eq!(SCORING_VERSION, 1); +} diff --git a/crates/bounty-http/src/lib.rs b/crates/bounty-http/src/lib.rs index 7ede5a9c3..7e30c471b 100644 --- a/crates/bounty-http/src/lib.rs +++ b/crates/bounty-http/src/lib.rs @@ -39,7 +39,7 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use bounty_challenge_task::{ backend_public_url, hotkey_hex, parse_hotkey, parse_signature, verify_pair_signature, - PairChallenge, ScoringBackend, CHALLENGE_ID, MAX_PENDING_REPORTS_PER_HOTKEY, + EmitterStatus, PairChallenge, ScoringBackend, CHALLENGE_ID, MAX_PENDING_REPORTS_PER_HOTKEY, MIN_REPORT_BODY_CHARS, MIN_REPORT_INTERVAL_SECS, MIN_REPRO_CHARS, MIN_UNIQUE_BODY_TOKENS, SCORE_MAX, SCORING_VERSION, TERMS_TEXT, }; @@ -61,6 +61,9 @@ pub struct AppState { pub scoring: ScoringBackend, /// Operator bearer hashes (sha256 hex). Empty → admin 503. pub admin_hashes: Arc>, + /// Emitter read side. `None` (or an unwired status) means this host signs + /// no leaves at all, which is the only condition that also 409s the seal. + pub emitter: Option>, } impl AppState { @@ -68,6 +71,11 @@ impl AppState { fn can_score(&self) -> bool { self.scoring != ScoringBackend::Unconfigured } + + /// Whether this host wired a leaf emitter. + fn emitter_wired(&self) -> bool { + self.emitter.as_ref().is_some_and(|e| e.wired()) + } } /// Build the router. @@ -92,6 +100,7 @@ async fn health() -> impl IntoResponse { async fn status(State(st): State) -> impl IntoResponse { let champ = st.store.champion_hotkey().ok().flatten(); + let emitter = st.emitter.as_ref().map(|e| e.view()); Json(serde_json::json!({ "challenge_id": CHALLENGE_ID, "scoring_version": SCORING_VERSION, @@ -102,6 +111,14 @@ async fn status(State(st): State) -> impl IntoResponse { "scoring_backend": st.scoring, "can_score": st.can_score(), "backend_public_configured": backend_public_url().is_some(), + // Whether that scorer has actually turned into weight. `can_score` says + // this host *may* pay; `emitter_wired` plus `emitter` say whether it + // *is*. A readable feed that crowns nobody shows up here as + // `last_outcome: "unpaid"` with `last_feed_read: true` rather than as a + // successful score, because a silent empty payout is the failure this + // whole path exists to make visible. + "emitter_wired": st.emitter_wired(), + "emitter": emitter, // Published so miners can see what is being measured — and what is // not: `triage_noise` never enters the score they are paid on. "scoring": { @@ -437,6 +454,7 @@ mod tests { session_secret: Arc::new(b"test-session-secret".to_vec()), scoring, admin_hashes: Arc::new(vec![hash_admin_token(token)]), + emitter: None, }); (router, token.to_owned()) } @@ -728,6 +746,7 @@ mod tests { session_secret: Arc::new(b"test-session-secret".to_vec()), scoring: ScoringBackend::BackendPublic, admin_hashes: Arc::new(vec![]), + emitter: None, }); let exp = unix_now().saturating_add(600); let (st, paired) = json_req(app.clone(), "POST", "/v1/pair", pair_payload(exp), None).await; diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 49feea1e8..b17a058bc 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -48,7 +48,7 @@ Migrations (`crates/db/migrations`) run on boot in gateway when `BASE_DATABASE_U Bounty scores **only** from the CortexLM/backend public feed. Set `BOUNTY_BACKEND_PUBLIC_URL` on the host (`deploy/env/bounty-challenge.env`; never bake a hostname into git). The service fetches `/v1/bounty/public/leaderboard` + `/reports`, signs an exact-`E` leaf set every `BOUNTY_EMIT_POLL_SECS` (default 120), and posts to the master gateway — validators only ever verify the sealed bundle. -With no readable feed the host answers **503** on `POST /v1/reports` and pays nobody: it covers `E` with `NoScore(ChallengeInternal)`, so the 2000 bps burns to uid 0 while D24 still holds (leaving `E` uncovered would 409 the seal for *every* challenge, since bounty holds a paid trust-root row). `BOUNTY_FORCE_SIM` is retired and ignored; `assert-compose-matrix.sh` fails if any compose file reintroduces it. Verify with `./deploy/scripts/local-e2e.sh --smoke` (it POSTs ingest and asserts 503 without a feed, 401 with one) or by hand: `GET /challenge/bounty/v1/status` → `scoring_backend`, `can_score`. Details: [`docs/BOUNTY.md`](../docs/BOUNTY.md). +With no readable feed the host answers **503** on `POST /v1/reports` and pays nobody: it covers `E` with `NoScore(ChallengeInternal)`, so the 2000 bps burns to uid 0 while D24 still holds (leaving `E` uncovered would 409 the seal for *every* challenge, since bounty holds a paid trust-root row). A feed that **answers** and crowns nobody is a separate outcome — `unpaid`, same cover, same burn — and must not be read as a healthy score: adjudication is what turns a published row into weight, so a `valid` row without a `severity` pays nothing. `BOUNTY_FORCE_SIM` is retired and ignored; `assert-compose-matrix.sh` fails if any compose file reintroduces it. Verify with `./deploy/scripts/local-e2e.sh --smoke` (it POSTs ingest and asserts 503 without a feed, 401 with one, plus the emitter's outcome/feed/paid fields) or by hand: `GET /challenge/bounty/v1/status` → `scoring_backend`, `can_score`, `emitter_wired`, `emitter.last_outcome` / `last_feed_read` / `last_paid`. Details: [`docs/BOUNTY.md`](../docs/BOUNTY.md). ## Proof harvest (Lium) diff --git a/deploy/compose/env-staging.yml b/deploy/compose/env-staging.yml index 2d964aaab..88181647f 100644 --- a/deploy/compose/env-staging.yml +++ b/deploy/compose/env-staging.yml @@ -45,7 +45,7 @@ services: - ./deploy/secrets/wallets:/run/base/wallets:ro - ./deploy/secrets/gateway_admin_token:/run/secrets/gateway_admin_token:ro # The sealer must sign with the same staging trust root the validator - # verifies against (bounty 7000 + proof 3000), or /v1/weights/latest + # verifies against (bounty 2000 + proof 8000), or /v1/weights/latest # never shows challenge weight and the validator flags emission share mismatch (D23). - ./config/challenges.staging.toml:/etc/base/config/challenges.toml:ro - ./config/challenges.staging.toml.sig:/etc/base/config/challenges.toml.sig:ro diff --git a/deploy/env/bounty-challenge.env.example b/deploy/env/bounty-challenge.env.example index b7114e603..b8a2d3e4a 100644 --- a/deploy/env/bounty-challenge.env.example +++ b/deploy/env/bounty-challenge.env.example @@ -17,14 +17,31 @@ BOUNTY_CHAT_COMMAND= # Empty (or unreachable) and this host cannot score: POST /v1/reports answers # 503 rather than collecting real bug-hunting work it could never pay for, and # the emitter pays nobody — it covers the expected set with an explicit -# no-score, so the 7000 bps burns to uid 0. There is no offline scorer to fall +# no-score, so the 2000 bps burns to uid 0. There is no offline scorer to fall # back to; BOUNTY_FORCE_SIM is retired and ignored. # Check GET /v1/status → scoring_backend, can_score. +# +# A reachable feed is not the same as a paying one. Adjudication is the +# dependency: CortexLM/backend must publish a row that is `valid`, carries a +# `severity`, and is justified, or the crown gate refuses it and the epoch pays +# nobody. That case shows up on /v1/status as `emitter.last_outcome: "unpaid"` +# with `last_feed_read: true` — the feed answered, and there was nothing to +# pay. Do not read that as an outage. BOUNTY_BACKEND_PUBLIC_URL= # Seconds between emitter ticks (fetch feed → sign exact-E leaf set → gateway). BOUNTY_EMIT_POLL_SECS=120 +# What the emitter last did, published on GET /v1/status: +# emitter_wired false ⇒ no challenge key; no leaves, and the seal 409s +# emitter.last_outcome scored | unpaid | burned | held | error +# emitter.last_feed_read whether the feed answered on the last tick +# emitter.last_paid positive leaves on the last tick +# `unpaid` + `last_feed_read: true` is the one to watch: the challenge is +# running and the epoch still pays nobody. +# Validators never read this feed. They verify the sealed bundle, so the only +# way a backend adjudication becomes weight is this emitter posting leaves. + # Chain endpoint for the expected set E. The env overlays (env-staging.yml / # env-prod.yml) set the ordered failover list; this is the single-endpoint # fallback for a bare stack. diff --git a/deploy/scripts/local-e2e.sh b/deploy/scripts/local-e2e.sh index 2786992bb..8f17d78f2 100755 --- a/deploy/scripts/local-e2e.sh +++ b/deploy/scripts/local-e2e.sh @@ -426,10 +426,11 @@ PY import pathlib, sys # Mirrors config/challenges.toml. Shares must sum to 10000 or the validator -# flags an emission-share mismatch (D23). Bounty 7000 + proof 3000. +# flags an emission-share mismatch (D23). Bounty 2000 + proof 8000 — the live +# 20/80 split, so a local seal exercises the same vector production seals. rows = [ - ("bounty", sys.argv[2], 7000), - ("proof", sys.argv[3], 3000), + ("bounty", sys.argv[2], 2000), + ("proof", sys.argv[3], 8000), ] text = "version = 1\nintroduced_epoch = 0\n" for cid, pk, bps in rows: @@ -641,6 +642,11 @@ probe_weights_latest() { # # no BOUNTY_BACKEND_PUBLIC_URL → 503 (fail-closed; there is no offline scorer) # feed configured → 401 invalid_session (ingest open, gate off) +# +# Both cases also assert the emitter read side, because ingest being reachable +# says nothing about weight being produced. The failure this catches is a host +# that answers ingest, reports `can_score: true`, and still pays nobody every +# epoch — which is invisible without the emitter fields. probe_bounty_fail_closed() { local base="http://127.0.0.1:${BOUNTY_HOST_PORT}" local status code body @@ -650,6 +656,17 @@ probe_bounty_fail_closed() { return 0 fi log "bounty status: $status" + # A host with a challenge key must wire the emitter even with no feed: bounty + # holds a paid trust-root row, so an uncovered E 409s the seal for every + # challenge, including proof's. + if [[ "$(echo "$status" | jq -r '.emitter_wired // "missing"')" == "false" ]]; then + log "warning: bounty emitter not wired (no BASE_CHALLENGE_SK_FILE); the seal will 409 until it is" + else + echo "$status" | jq -e '.emitter | objects' >/dev/null \ + || die "bounty /v1/status has no emitter block; weight production is not observable" + echo "$status" | jq -e '.emitter | has("last_outcome") and has("last_feed_read") and has("last_paid")' >/dev/null \ + || die "bounty emitter block is missing outcome/feed/paid fields" + fi body='{"session":"not-a-session","title":"probe","body":"probe","repro_steps":"probe"}' code="$(curl -sS -m 5 -o /tmp/local-e2e-bounty-report.json -w '%{http_code}' \ -X POST -H 'content-type: application/json' -d "$body" "${base}/v1/reports" || true)" @@ -661,6 +678,27 @@ probe_bounty_fail_closed() { else [[ "$code" == "401" ]] || die "bounty ingest answered HTTP $code with a feed configured (expected 401 invalid_session)" log "bounty ingest OK: feed configured → POST /v1/reports reaches session auth" + # Feed configured but nothing adjudicated is the quiet empty-payout case: + # the emitter covers E with ChallengeInternal and reports `unpaid`, which + # must not read as a healthy score. + local outcome paid + outcome="$(echo "$status" | jq -r '.emitter.last_outcome // "never"')" + paid="$(echo "$status" | jq -r '.emitter.last_paid // 0')" + case "$outcome" in + scored) + [[ "$paid" -gt 0 ]] || die "bounty reports a scored tick that paid nobody" + log "bounty reward linkage OK: last tick scored $paid hotkey(s)" + ;; + unpaid) + log "bounty emitter unpaid: feed read, nothing adjudicated yet (share burns to uid 0)" + ;; + burned|held|error|never) + log "bounty emitter last_outcome=$outcome (waiting on the backend feed)" + ;; + *) + die "bounty emitter reported an unknown outcome: $outcome" + ;; + esac fi } diff --git a/docs/BOUNTY.md b/docs/BOUNTY.md index 494ea4347..7c8ef194e 100644 --- a/docs/BOUNTY.md +++ b/docs/BOUNTY.md @@ -28,6 +28,48 @@ board. | `challenge_scoring_version` | `1` | | Port | `8096` (local host `28096`) | | Emission | `2000` bps (20%; Proof has the other 80%) | +| Trust-root row | `bounty` @ 2000 bps, `all_metagraph_hotkeys` | + +The row is committed in [`../config/challenges.toml`](../config/challenges.toml) +and mirrored by [`../config/challenges.staging.toml`](../config/challenges.staging.toml) +for staging. Both are owner-signed: changing a share without re-signing fails +at load, and `crates/bounty-challenge/tests/trust_root_linkage.rs` asserts both +files stay live, payable, and in step. Do not edit them by hand — see +[`../config/CEREMONY.md`](../config/CEREMONY.md) and +[`runbooks/trust-root-rotation.md`](runbooks/trust-root-rotation.md). + +## Operator environment + +Set on the **master** host (never baked into git; see +[`../deploy/env/bounty-challenge.env.example`](../deploy/env/bounty-challenge.env.example)): + +| Variable | Role | +|----------|------| +| `BOUNTY_BACKEND_PUBLIC_URL` | Base URL of the CortexLM/backend public API. **The only scorer.** Unset → ingest 503s and every leaf is a cover. | +| `BOUNTY_EMIT_POLL_SECS` | Seconds between emitter ticks (default 120). | +| `BASE_CHALLENGE_SK_FILE` | Bounty leaf mini-secret. Its public key **must** match the trust-root `bounty` row, or every leaf is rejected. | +| `BASE_CHALLENGE_GATEWAY_ENDPOINT` | Master gateway for `POST /v1/weights/raw` (default `http://gateway:8080`). | +| `BASE_NETUID` | Subnet `E` is derived from. | +| `BASE_CHAIN_ENDPOINT(S)` | Chain for `E`; `BASE_CHAIN_ENDPOINTS` is the ordered failover list and wins over the singular. | +| `BOUNTY_ADMIN_TOKENS_FILE` | Operator bearer for `POST /v1/admin/adjudicate` and the report reads. Empty → admin **503**. | +| `BOUNTY_SESSION_SECRET_FILE` | Pairing session HMAC. Empty → ephemeral (pairings do not survive restart). | +| `BOUNTY_CHAT_COMMAND` | Chat inject command. Env-only; docs use the placeholder. | + +`BOUNTY_FORCE_SIM` is retired: it is ignored, warned about at boot, and +`deploy/scripts/assert-compose-matrix.sh` fails if any compose file sets it. + +### What validators do and do not verify + +Validators **never** read the bounty feed and **never** re-run a report. They +verify the sealed bundle: the gateway signature, the merkle root, D24 +completeness against the local owner-signed trust root, and the recomputed +weight vector. So the feed → leaf → seal path is entirely on the challenge +host, and a bug there is invisible to consensus until someone reads +`/v1/status` or the sealed vector. + +That is why the emitter publishes its outcome: the reward linkage has two +halves that fail independently — the backend publishing adjudications, and this +host turning them into signed leaves. `can_score` covers only the first. ## Why bug reports need different evaluation @@ -90,19 +132,50 @@ scorer. Each tick the challenge service: 2. derives `E` from the metagraph at `last_epoch_block` (`AllMetagraphHotkeys`) 3. maps published rows onto one leaf per hotkey in `E` for the current subnet epoch (champion → `Score`, net-malicious → `InvalidResponse`, everyone else - → `NotAttempted`); an unreadable feed pays nobody but still covers `E` with - `ChallengeInternal` + → `NotAttempted`) 4. `POST /v1/weights/raw` on the gateway, which seals what validators fetch +**A reachable feed is not a paying one**, and the difference is a separate +outcome. When the feed answers and no row maps to payable weight — nothing +adjudicated, everything still `pending`, or a `valid` row the operator never +priced — the tick is `unpaid`: `E` is covered with +`NoScore(ChallengeInternal)`, the share burns to uid 0, and `/v1/status` +reports `last_outcome: "unpaid"` with `last_feed_read: true`. It is not signed +as a scored epoch, because `NotAttempted` claims the challenge *chose* not to +invoke the miner, which is false here and would report a healthy tick while +paying nobody. + +Adjudication is therefore a hard dependency of the reward path, not a +reporting one. A published row only becomes weight when it is `valid`, carries +a `severity`, and is justified; anything short of that pays nothing that epoch. + Operator knobs: `BASE_CHALLENGE_GATEWAY_ENDPOINT`, `BASE_NETUID`, `BASE_CHAIN_ENDPOINT(S)`, `BOUNTY_EMIT_POLL_SECS` (default 120s). Re-emitting the current epoch is normal: the gateway supersedes on a changed digest and 409s an identical one. +### Reading `/v1/status` + +| Field | Meaning | +|-------|---------| +| `scoring_backend` | `backend_public` or `unconfigured`. | +| `can_score` | Whether this host *may* turn a report into weight. | +| `emitter_wired` | Whether a leaf emitter was wired. `false` is the only condition that also 409s the seal (missing `BASE_CHALLENGE_SK_FILE`). | +| `emitter.last_outcome` | `never` / `scored` / `unpaid` / `burned` / `held` / `error`. | +| `emitter.last_feed_read` | Whether the feed answered on the last tick. | +| `emitter.last_paid` | Positive leaves on the last tick. | +| `emitter.last_reason` | Why nobody was paid, or why the epoch was held. | +| `emitter.scored_epoch` | Highest epoch this process scored (in-process). | + +`can_score` alone does not mean anyone is being paid. `unpaid` with +`last_feed_read: true` is the combination to watch: the backend is up and the +epoch still pays nobody, which means adjudication is behind. + ## Fail-closed ingest and emission `GET /v1/status` publishes `scoring_backend` (`backend_public` | -`unconfigured`), `backend_public_configured`, and `can_score`. +`unconfigured`), `backend_public_configured`, `can_score`, `emitter_wired`, +and the `emitter` block. Without `BOUNTY_BACKEND_PUBLIC_URL` — or when the feed is unreachable, 5xx, unparseable, or moving under the read — the host cannot turn a report into @@ -114,6 +187,16 @@ weight. Two things follow, and neither is a degraded mode: `NoScore(ChallengeInternal)` (`BUNDLE_SPEC` §3.3.1 — "challenge-side fault; still must cover the participant"), so the 2000 bps burns to uid 0. +A feed that answers and crowns nobody is the third case, and it is treated as +a cover rather than a score. `NotAttempted` on every leaf would claim the +challenge chose not to invoke the miners, which is false, and it would seal as +a legitimate-looking unpaid epoch while `/v1/status` reported success. The +tick is `unpaid` instead: same `ChallengeInternal` cover, same burn, and the +status says which half is missing. The three outcomes are deliberately +distinct — `scored`, `unpaid` (feed read, nothing payable), `burned` (feed +unreadable) — because an operator needs to know whether to wait on the backend +or go look at adjudication. + Covering `E` is not a hedge, it is the difference between bounty failing and the subnet failing. Bounty holds a **paid** trust-root row, and D24 requires a leaf per participant for every paid challenge: leave `E` uncovered and @@ -137,18 +220,22 @@ and the paragraphs above apply. No backend change is needed for this; a published revision or ETag on both routes would let the moving-feed check collapse to a single round. -A failed tick also tries not to take back a score. Once the process has scored -an epoch, a feed outage inside that same epoch **holds** instead of superseding -a champion's leaf with a burn; a backend hiccup does not get to decide the -epoch. The watermark is in-process (the gateway has no read side for raw -leaves), so a restart during an outage can still burn an epoch that had -scores — the next successful tick supersedes it back. The bias is deliberate: -burning pays nobody who was not already paid, while staying silent would 409 -the seal for every challenge. +A tick that produces no weight also tries not to take back a score. Once the +process has scored an epoch, a later unproductive tick inside that same epoch +**holds** instead of superseding a champion's leaf with a cover — whether the +feed went down or stayed up and simply stopped publishing the crowned hotkey. +A backend hiccup, or a publish that reverts, does not get to decide the epoch. +The watermark is in-process (the gateway has no read side for raw leaves), so a +restart during an outage can still burn an epoch that had scores — the next +successful tick supersedes it back. The bias is deliberate: burning pays nobody +who was not already paid, while staying silent would 409 the seal for every +challenge. Independently, the gateway refuses a `ChallengeInternal` cover from +replacing a positive leaf for the same key (409, original kept), so a lost +watermark cannot reseal a paid allocation into a uid-0 burn. Only a missing `BASE_CHALLENGE_SK_FILE` stops emission entirely — a leaf the trust root rejects is not weight — and that case is logged as the 409 it will -cause. +cause. It is also the only case that publishes `emitter_wired: false`. **There is no offline scorer.** `BOUNTY_FORCE_SIM` is retired: it is ignored, warned about at boot, and `deploy/scripts/assert-compose-matrix.sh` fails if diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index bbf3aa997..daabb9749 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -62,7 +62,8 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover | Component | Status | Notes | |-----------|--------|-------| | Crates (`crates/bounty-*`) | **done** | task (pairing), score (precision × severity, triage-noise canary off the lattice), store, http (fail-closed ingest + quotas), challenge (backend public **consumer** + fail-closed leaf emitter; the two public routes are re-read until they agree, and `/leaderboard` `valid_count` must match the `valid` reports, so a mid-publish pair or a stable A+B mix is never signed as one snapshot). | -| Binary (`bins/bounty-challenge`) | **done** | Internal HTTP on `:8096` plus the emitter (backend feed → exact-`E` leaves → gateway `POST /v1/weights/raw`, `BOUNTY_EMIT_POLL_SECS`). Does **not** serve `/v1/public/*`. No feed (or an unreadable one) pays nobody: `E` is covered with `NoScore(ChallengeInternal)` so D24 holds and the share burns to uid 0. A scored epoch is never downgraded to a burn mid-epoch. | +| Binary (`bins/bounty-challenge`) | **done** | Internal HTTP on `:8096` plus the emitter (backend feed → exact-`E` leaves → gateway `POST /v1/weights/raw`, `BOUNTY_EMIT_POLL_SECS`). Does **not** serve `/v1/public/*`. No feed (or an unreadable one) pays nobody: `E` is covered with `NoScore(ChallengeInternal)` so D24 holds and the share burns to uid 0. A feed that answers and crowns nobody is the separate `unpaid` outcome — same cover, same burn — rather than a scored tick, because `NotAttempted` would claim the challenge chose not to invoke anyone. A scored epoch is never downgraded to a cover mid-epoch. `/v1/status` publishes `emitter_wired` plus the emitter's last outcome / feed-read / paid counts, so a reachable feed that pays nothing is visible instead of looking healthy. | +| Reward linkage tests | **done** | `crates/bounty-challenge/tests/rewards_linkage.rs` walks pair → report → adjudicate → published feed → signed leaves → sealed bundle (leaves verify under the trust root, the paired hotkey is the one paid, a silent hotkey is explicit, and a paid bounty seals beside a challenge that scored nothing). `tests/trust_root_linkage.rs` asserts the committed prod **and** staging roots keep bounty live at 2000 bps with shares summing to 10000, mirroring each other, under the owner signature. `tests/emit_fail_closed.rs` covers the unpaid / burned / held split. | | Miner CLI (`bins/ctx`) | **done** | `ctx bounty pair|report|show|status`. `bins/cortex-bounty` deprecates to `ctx bounty pair`. | | Compose / images | **done** | Default compose + `images.yml` target `bounty-challenge`. | | Emission | **2000 bps** | Payable share (20%). Sum `10000`. | diff --git a/docs/external-miner/bounty.md b/docs/external-miner/bounty.md index d9939eb32..e0a91a6dc 100644 --- a/docs/external-miner/bounty.md +++ b/docs/external-miner/bounty.md @@ -163,6 +163,24 @@ offline stand-in, so a 503 here is the honest answer rather than a temporary degradation you can submit through. Check `scoring_backend` and `can_score` before you go hunting. +`can_score` is not the whole story, and it is worth knowing which half is +missing when an epoch pays nobody. The feed being *reachable* is not the same +as it *paying*: adjudication is what turns your report into weight, so a +backend that is up with nothing adjudicated produces no weight either. `ctx +bounty status` prints the emitter's last outcome alongside the scorer: + +| Outcome | Meaning | +|---------|---------| +| `scored` | The feed published payable rows and they became signed weight. | +| `unpaid` | The feed answered, and nothing was payable. Wait on adjudication — the backend is up. | +| `burned` | The feed could not be read. This one is a backend or network problem. | +| `held` | The feed produced no weight, but this epoch already had a score, so the score stands. | +| `error` | The tick could not emit at all (chain, signing, or gateway). | + +`unpaid` with the feed read is the one to expect right after you file: your +report is real, and it becomes weight when an operator adjudicates it `valid` +with a severity. Nothing about that is a fault on your side. + ## Scoring (precision × severity, not volume) | Outcome | Result | From 851f7888e2feb24faf03400fdb4b4e036b29593d Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:22:14 +0000 Subject: [PATCH 2/4] fix(bounty): publish one coherent emitter snapshot per tick MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings on the emitter read side, both about the status telling an operator something the tick did not. `record` published a tick through separate relaxed atomics and two mutexes, and `view` read them independently, so a concurrent `/v1/status` could pair one tick's outcome with another's counts — `scored` beside `paid: 0`, or an `unpaid` outcome with a previous tick's reason. Those are exactly the combinations an operator is told to read as a fault, so a reader could be sent after a failure that never happened. The per-tick fields now live in one mutex and are replaced as a single value; `ticks` is bumped last so the count never claims a tick whose fields have not landed. The counters stay separate: their value does not depend on which tick is last. Cover outcomes hard-coded `last_pin_block: 0`, which is documented as "no tick yet". A cover is still signed against a pinned participant snapshot, so the status published a real epoch and participant count while hiding the block a validator would have to reproduce. `Unpaid` and `Burned` now carry the pin they derived `E` at. `Held` keeps 0 on purpose: its leaves came from an earlier tick, and claiming a block this tick never used would be a fabrication. The concurrency test alternates two ticks that differ in every field, so a torn read cannot coincidentally look coherent. Mutation-checked: splitting the write again fails it with `scored` beside `participants: 0, paid: 0`. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- crates/bounty-challenge-task/src/lib.rs | 142 +++++++-------- crates/bounty-challenge/src/emit.rs | 31 +++- .../tests/emit_fail_closed.rs | 170 +++++++++++++++++- .../bounty-challenge/tests/rewards_linkage.rs | 2 + 4 files changed, 269 insertions(+), 76 deletions(-) diff --git a/crates/bounty-challenge-task/src/lib.rs b/crates/bounty-challenge-task/src/lib.rs index fb776d0ca..5e249276d 100644 --- a/crates/bounty-challenge-task/src/lib.rs +++ b/crates/bounty-challenge-task/src/lib.rs @@ -16,7 +16,7 @@ use keystore::{ss58_decode, ss58_encode, BITTENSOR_SS58_PREFIX, KEY_LEN}; use schnorrkel::{signing_context, ExpansionMode, MiniSecretKey, PublicKey, Signature}; use serde::{Deserialize, Serialize}; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; use thiserror::Error; @@ -236,32 +236,6 @@ pub enum EmitterOutcomeKind { Error, } -impl EmitterOutcomeKind { - /// Stable code for the atomic slot. - const fn code(self) -> u8 { - match self { - Self::Never => 0, - Self::Scored => 1, - Self::Unpaid => 2, - Self::Burned => 3, - Self::Held => 4, - Self::Error => 5, - } - } - - /// Inverse of [`Self::code`]; unknown codes read back as [`Self::Never`]. - const fn from_code(code: u8) -> Self { - match code { - 1 => Self::Scored, - 2 => Self::Unpaid, - 3 => Self::Burned, - 4 => Self::Held, - 5 => Self::Error, - _ => Self::Never, - } - } -} - /// One completed tick, as recorded by [`EmitterStatus::record`]. #[derive(Debug, Clone, Copy)] pub struct EmitterTick<'a> { @@ -320,19 +294,51 @@ pub struct EmitterStatusView { } /// Shared emitter state. The emitter writes it; `/v1/status` reads it. +/// +/// The per-tick fields live in one [`Mutex`] rather than in separate atomics: +/// they describe a single tick and are read as a set (`unpaid` with a feed that +/// was read, `scored` with a positive paid count). Publishing them one atomic +/// at a time would let a concurrent reader pair the new outcome with the +/// previous tick's paid count or reason, which is worse than a stale snapshot — +/// it is a combination no tick ever produced, and the CLI and operators read +/// these fields together to decide which half of the reward path is missing. +/// +/// `ticks` and `scored_epoch` stay separate: they are monotonic counters whose +/// value does not depend on which tick is "last". #[derive(Debug)] pub struct EmitterStatus { wired: bool, ticks: AtomicU64, - last_kind: AtomicU8, - last_epoch: AtomicU64, - last_pin_block: AtomicU64, - last_participants: AtomicU64, - last_paid: AtomicU64, - last_feed_read: AtomicBool, scored_epoch: AtomicU64, - last_reason: Mutex, - last_error: Mutex, + last: Mutex, +} + +/// The fields of one completed tick, published and read as one value. +#[derive(Debug, Clone)] +struct LastTick { + kind: EmitterOutcomeKind, + epoch: u64, + pin_block: u64, + participants: u64, + paid: u64, + feed_read: bool, + reason: String, + error: String, +} + +impl Default for LastTick { + fn default() -> Self { + Self { + kind: EmitterOutcomeKind::Never, + epoch: 0, + pin_block: 0, + participants: 0, + paid: 0, + feed_read: false, + reason: String::new(), + error: String::new(), + } + } } impl EmitterStatus { @@ -342,15 +348,8 @@ impl EmitterStatus { Self { wired, ticks: AtomicU64::new(0), - last_kind: AtomicU8::new(EmitterOutcomeKind::Never.code()), - last_epoch: AtomicU64::new(0), - last_pin_block: AtomicU64::new(0), - last_participants: AtomicU64::new(0), - last_paid: AtomicU64::new(0), - last_feed_read: AtomicBool::new(false), scored_epoch: AtomicU64::new(0), - last_reason: Mutex::new(String::new()), - last_error: Mutex::new(String::new()), + last: Mutex::new(LastTick::default()), } } @@ -360,49 +359,50 @@ impl EmitterStatus { self.wired } - /// Record one completed tick. + /// Record one completed tick as a single replacement, so no reader can + /// observe a mix of this tick and the previous one. pub fn record(&self, tick: EmitterTick<'_>) { - self.ticks.fetch_add(1, Ordering::Relaxed); - self.last_kind.store(tick.kind.code(), Ordering::Relaxed); - self.last_epoch.store(tick.epoch, Ordering::Relaxed); - self.last_pin_block.store(tick.pin_block, Ordering::Relaxed); - self.last_participants - .store(count_u64(tick.participants), Ordering::Relaxed); - self.last_paid - .store(count_u64(tick.paid), Ordering::Relaxed); - self.last_feed_read.store(tick.feed_read, Ordering::Relaxed); self.scored_epoch .fetch_max(tick.scored_epoch, Ordering::Relaxed); - tick.reason - .unwrap_or_default() - .clone_into(&mut lock(&self.last_reason)); - tick.error - .unwrap_or_default() - .clone_into(&mut lock(&self.last_error)); + *lock(&self.last) = LastTick { + kind: tick.kind, + epoch: tick.epoch, + pin_block: tick.pin_block, + participants: count_u64(tick.participants), + paid: count_u64(tick.paid), + feed_read: tick.feed_read, + reason: tick.reason.unwrap_or_default().to_owned(), + error: tick.error.unwrap_or_default().to_owned(), + }; + // Bumped last: a reader that sees tick N may still see N-1's fields if + // it raced the lock, but the count never claims a tick whose fields + // have not landed. + self.ticks.fetch_add(1, Ordering::Release); } /// Read-side snapshot for `/v1/status`. #[must_use] pub fn view(&self) -> EmitterStatusView { + let last = lock(&self.last).clone(); EmitterStatusView { wired: self.wired, - ticks: self.ticks.load(Ordering::Relaxed), - last_outcome: EmitterOutcomeKind::from_code(self.last_kind.load(Ordering::Relaxed)), - last_epoch: self.last_epoch.load(Ordering::Relaxed), - last_pin_block: self.last_pin_block.load(Ordering::Relaxed), - last_participants: self.last_participants.load(Ordering::Relaxed), - last_paid: self.last_paid.load(Ordering::Relaxed), - last_feed_read: self.last_feed_read.load(Ordering::Relaxed), + ticks: self.ticks.load(Ordering::Acquire), + last_outcome: last.kind, + last_epoch: last.epoch, + last_pin_block: last.pin_block, + last_participants: last.participants, + last_paid: last.paid, + last_feed_read: last.feed_read, scored_epoch: self.scored_epoch.load(Ordering::Relaxed), - last_reason: lock(&self.last_reason).clone(), - last_error: lock(&self.last_error).clone(), + last_reason: last.reason, + last_error: last.error, } } } -/// A poisoned lock here is a status string, not a consensus input: recover the -/// guard rather than take the challenge down over a display field. -fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, String> { +/// A poisoned lock here is a status snapshot, not a consensus input: recover +/// the guard rather than take the challenge down over a display field. +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) } diff --git a/crates/bounty-challenge/src/emit.rs b/crates/bounty-challenge/src/emit.rs index 8c8a4b6fe..581801a11 100644 --- a/crates/bounty-challenge/src/emit.rs +++ b/crates/bounty-challenge/src/emit.rs @@ -103,6 +103,10 @@ pub enum EmitOutcome { Unpaid { /// Subnet epoch the cover was signed for. epoch: u64, + /// Block `E` was pinned at. A cover is still signed against a specific + /// participant snapshot, so the pin is as meaningful here as it is on + /// [`Self::Scored`]. + pin_block: u64, /// Size of `E`. participants: usize, /// Why nobody was paid. @@ -114,6 +118,8 @@ pub enum EmitOutcome { Burned { /// Subnet epoch the burn set was signed for. epoch: u64, + /// Block `E` was pinned at (see [`Self::Unpaid`]). + pin_block: u64, /// Size of `E`. participants: usize, /// Why the feed was unreadable. @@ -212,7 +218,11 @@ impl BountyEmitter { let (epoch, pin_block, hotkeys) = pinned; let snapshot = match feed { Ok(s) => s, - Err(e) => return self.cover_without_a_feed(epoch, &hotkeys, &e).await, + Err(e) => { + return self + .cover_without_a_feed(epoch, pin_block, &hotkeys, &e) + .await + } }; let (_plan, leaf_scores) = emission_from_public_snapshot(&hotkeys, &snapshot); let paid = leaf_scores @@ -227,6 +237,7 @@ impl BountyEmitter { return self .cover_without_a_feed( epoch, + pin_block, &hotkeys, &BackendError::NoPayableRows(NO_PAYABLE_ROWS_REASON.to_owned()), ) @@ -269,12 +280,13 @@ impl BountyEmitter { }, Ok(EmitOutcome::Unpaid { epoch, + pin_block, participants, reason, }) => EmitterTick { kind: EmitterOutcomeKind::Unpaid, epoch: *epoch, - pin_block: 0, + pin_block: *pin_block, participants: *participants, paid: 0, feed_read, @@ -284,12 +296,13 @@ impl BountyEmitter { }, Ok(EmitOutcome::Burned { epoch, + pin_block, participants, reason, }) => EmitterTick { kind: EmitterOutcomeKind::Burned, epoch: *epoch, - pin_block: 0, + pin_block: *pin_block, participants: *participants, paid: 0, feed_read, @@ -297,7 +310,10 @@ impl BountyEmitter { reason: Some(reason), error: None, }, - Ok(EmitOutcome::Held { epoch, reason, .. }) => EmitterTick { + // A hold leaves an earlier tick's leaves standing, so it has no + // pin of its own to publish: `last_pin_block` stays 0 rather than + // claiming a block this tick never derived `E` at. + Ok(EmitOutcome::Held { epoch, reason }) => EmitterTick { kind: EmitterOutcomeKind::Held, epoch: *epoch, pin_block: 0, @@ -347,10 +363,12 @@ impl BountyEmitter { ), Ok(EmitOutcome::Unpaid { epoch, + pin_block, participants, reason, }) => tracing::warn!( epoch, + pin_block, participants, %reason, "bounty read the feed and found nothing payable: covered E with \ @@ -358,10 +376,12 @@ impl BountyEmitter { ), Ok(EmitOutcome::Burned { epoch, + pin_block, participants, reason, }) => tracing::warn!( epoch, + pin_block, participants, %reason, "bounty could not read the feed: covered E with ChallengeInternal, \ @@ -415,6 +435,7 @@ impl BountyEmitter { async fn cover_without_a_feed( &self, epoch: u64, + pin_block: u64, hotkeys: &BTreeSet, cause: &BackendError, ) -> Result { @@ -438,12 +459,14 @@ impl BountyEmitter { if matches!(cause, BackendError::NoPayableRows(_)) { return Ok(EmitOutcome::Unpaid { epoch, + pin_block, participants, reason, }); } Ok(EmitOutcome::Burned { epoch, + pin_block, participants, reason, }) diff --git a/crates/bounty-challenge/tests/emit_fail_closed.rs b/crates/bounty-challenge/tests/emit_fail_closed.rs index fd910e0e0..00b3c1290 100644 --- a/crates/bounty-challenge/tests/emit_fail_closed.rs +++ b/crates/bounty-challenge/tests/emit_fail_closed.rs @@ -33,7 +33,7 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use bounty_challenge::{ fetch_public_snapshot, BackendError, BountyEmitter, EmitOutcome, EmitterOutcomeKind, - GatewayClient, GatewayClientConfig, + EmitterStatus, EmitterTick, GatewayClient, GatewayClientConfig, }; use chain::{ AxonInfo, ChainClient, ChainError, FakeChain, FakeChainConfig, Metagraph, WeightsTlockPayload, @@ -276,6 +276,13 @@ async fn spawn_stable_torn_backend() -> String { serve(app).await } +/// A backend that answers 503 on both routes — the shape of an outage, as +/// opposed to [`spawn_empty_backend`]'s reachable-but-unpaid feed. +async fn spawn_down_backend() -> String { + let app = Router::new().fallback(|| async { axum::http::StatusCode::SERVICE_UNAVAILABLE }); + serve(app).await +} + /// A backend that starts payable and can be switched to publishing nothing, /// without changing its URL. That is the "backend stopped paying a hotkey it /// had already crowned" case: still reachable, no longer payable. @@ -533,10 +540,16 @@ async fn an_unset_backend_url_burns_without_paying_anyone() { match em.tick().await.expect("burn covers E") { EmitOutcome::Burned { epoch, + pin_block, participants, reason, } => { assert_eq!(epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert_eq!( + pin_block, + chain::fake_defaults::LAST_EPOCH_BLOCK, + "a cover is signed against a pinned participant snapshot" + ); assert_eq!(participants, 3); assert!(reason.contains("BOUNTY_BACKEND_PUBLIC_URL"), "{reason}"); } @@ -754,10 +767,16 @@ async fn a_readable_feed_with_no_rows_covers_e_instead_of_claiming_a_score() { match em.tick().await.expect("cover covers E") { EmitOutcome::Unpaid { epoch, + pin_block, participants, reason, } => { assert_eq!(epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert_eq!( + pin_block, + chain::fake_defaults::LAST_EPOCH_BLOCK, + "a cover is signed against a pinned participant snapshot" + ); assert_eq!(participants, 3); assert!(reason.contains("no payable rows"), "{reason}"); } @@ -878,6 +897,155 @@ async fn status_reports_a_wired_emitter_that_has_not_ticked_yet() { assert_eq!(view.last_outcome, EmitterOutcomeKind::Never); assert!(!view.last_feed_read); assert_eq!(view.last_paid, 0); + assert_eq!(view.last_pin_block, 0, "no tick has pinned a block yet"); +} + +/// A cover is signed against a pinned participant snapshot, so the status must +/// publish that block rather than report zero. Reporting `last_pin_block: 0` +/// beside a real epoch and participant count hides the block a validator would +/// have to reproduce, and 0 is documented as "no tick yet". +#[tokio::test] +async fn a_cover_publishes_the_block_it_pinned_e() { + for backend in [spawn_empty_backend().await, spawn_down_backend().await] { + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + assert!(matches!( + em.tick().await.expect("cover covers E"), + EmitOutcome::Unpaid { .. } | EmitOutcome::Burned { .. } + )); + let view = em.status().view(); + assert_eq!( + view.last_pin_block, + chain::fake_defaults::LAST_EPOCH_BLOCK, + "a cover pins E at a real block: {view:?}" + ); + assert_eq!(view.last_epoch, chain::fake_defaults::SUBNET_EPOCH_INDEX); + assert_eq!(view.last_participants, 3); + } +} + +/// A held tick publishes no pin of its own: the leaves it left standing came +/// from an earlier tick, and claiming a block this tick never derived `E` at +/// would be a fabrication. +#[tokio::test] +async fn a_held_tick_does_not_claim_a_pin_it_did_not_use() { + let (backend, payable) = spawn_switchable_backend().await; + let (gateway, _accepted) = spawn_gateway().await; + let em = emitter(Some(backend), &gateway); + + assert!(matches!( + em.tick().await.expect("scored"), + EmitOutcome::Scored { .. } + )); + assert_eq!( + em.status().view().last_pin_block, + chain::fake_defaults::LAST_EPOCH_BLOCK + ); + + payable.store(false, Ordering::Relaxed); + assert!(matches!( + em.tick().await.expect("hold"), + EmitOutcome::Held { .. } + )); + let view = em.status().view(); + assert_eq!(view.last_outcome, EmitterOutcomeKind::Held); + assert_eq!(view.last_pin_block, 0, "a hold has no pin of its own"); + assert_eq!(view.last_participants, 0); +} + +/// The published fields describe one tick, never a blend of two. A reader that +/// saw `scored` beside the previous tick's `paid: 0` would be reading a state +/// no tick produced — exactly the combination an operator is told to treat as +/// a fault. +/// +/// The writer alternates between two ticks whose fields differ in *every* +/// position, so a torn write cannot coincidentally look coherent: a `scored` +/// snapshot carrying the other tick's `paid`/`participants`/`reason` is +/// detected immediately. +#[tokio::test] +async fn status_never_pairs_an_outcome_with_another_ticks_counts() { + let status = Arc::new(EmitterStatus::new(true)); + let scored = EmitterTick { + kind: EmitterOutcomeKind::Scored, + epoch: 42, + pin_block: 1_000, + participants: 3, + paid: 1, + feed_read: true, + scored_epoch: 42, + reason: None, + error: None, + }; + let unpaid = EmitterTick { + kind: EmitterOutcomeKind::Unpaid, + epoch: 43, + pin_block: 1_360, + participants: 7, + paid: 0, + feed_read: true, + scored_epoch: 43, + reason: Some("nothing payable"), + error: None, + }; + + let writer = { + let status = Arc::clone(&status); + tokio::task::spawn_blocking(move || { + for i in 0..20_000u32 { + status.record(if i % 2 == 0 { scored } else { unpaid }); + } + }) + }; + + let mut snapshots = 0u32; + while !writer.is_finished() { + let view = status.view(); + match view.last_outcome { + EmitterOutcomeKind::Scored => { + assert_eq!( + ( + view.last_epoch, + view.last_pin_block, + view.last_participants, + view.last_paid + ), + (42, 1_000, 3, 1), + "a scored snapshot must be exactly the scored tick: {view:?}" + ); + assert!(view.last_reason.is_empty(), "{view:?}"); + } + EmitterOutcomeKind::Unpaid => { + assert_eq!( + ( + view.last_epoch, + view.last_pin_block, + view.last_participants, + view.last_paid + ), + (43, 1_360, 7, 0), + "an unpaid snapshot must be exactly the unpaid tick: {view:?}" + ); + assert_eq!(view.last_reason, "nothing payable", "{view:?}"); + } + // The reader can legitimately race ahead of the writer's first + // tick; that is the documented initial state, not a torn read. + EmitterOutcomeKind::Never => { + assert_eq!(view.ticks, 0, "Never is only before the first tick"); + assert_eq!(view.last_participants, 0); + assert_eq!(view.last_paid, 0); + } + other => panic!("no other outcome was ever recorded: {other:?}"), + } + snapshots += 1; + tokio::task::yield_now().await; + } + writer.await.expect("writer"); + assert!( + snapshots > 0, + "the reader must actually have raced the writer" + ); + // The counters are monotonic and independent of which tick is last. + assert_eq!(status.view().scored_epoch, 43); } /// A scored tick publishes what was actually paid, so an operator can see the diff --git a/crates/bounty-challenge/tests/rewards_linkage.rs b/crates/bounty-challenge/tests/rewards_linkage.rs index 61bb3def4..818932b5a 100644 --- a/crates/bounty-challenge/tests/rewards_linkage.rs +++ b/crates/bounty-challenge/tests/rewards_linkage.rs @@ -657,10 +657,12 @@ async fn an_unreadable_feed_pays_nobody_without_breaking_the_seal() { let epoch = match em.tick().await.expect("cover E") { EmitOutcome::Burned { epoch, + pin_block, participants, reason, } => { assert_eq!(participants, 2); + assert_eq!(pin_block, chain::fake_defaults::LAST_EPOCH_BLOCK); assert!(reason.contains("503"), "{reason}"); epoch } From 31a03806a7ab4baa7c9b7f78f8a74e8b423b32b7 Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:49:09 +0000 Subject: [PATCH 3/4] fix(bounty): publish counters and last tick under one lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `view()` copied the locked last-tick state and then loaded `ticks` separately, so a reader could legitimately observe `ticks: 1` beside the pre-tick `Never`. That is not a torn last-tick — the fields were already consistent — but it is still two states in one snapshot, and a reader should not have to know which pairings are legal. The counters and the last tick now live in one struct under one lock, so the whole view is a state some tick actually produced. The test asserts the invariant that makes true (`Never` only with `ticks: 0`) instead of tolerating the interleaving, and requires the reader to observe both alternating states so it cannot pass by never racing the writer. Mutation-checked: publishing the counters before the fields land fails it with exactly the reported `Never` beside `ticks: 1`. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- crates/bounty-challenge-task/src/lib.rs | 65 +++++++++++-------- .../tests/emit_fail_closed.rs | 24 +++++-- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/crates/bounty-challenge-task/src/lib.rs b/crates/bounty-challenge-task/src/lib.rs index 5e249276d..f89a55489 100644 --- a/crates/bounty-challenge-task/src/lib.rs +++ b/crates/bounty-challenge-task/src/lib.rs @@ -16,7 +16,6 @@ use keystore::{ss58_decode, ss58_encode, BITTENSOR_SS58_PREFIX, KEY_LEN}; use schnorrkel::{signing_context, ExpansionMode, MiniSecretKey, PublicKey, Signature}; use serde::{Deserialize, Serialize}; -use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; use thiserror::Error; @@ -295,22 +294,31 @@ pub struct EmitterStatusView { /// Shared emitter state. The emitter writes it; `/v1/status` reads it. /// -/// The per-tick fields live in one [`Mutex`] rather than in separate atomics: -/// they describe a single tick and are read as a set (`unpaid` with a feed that -/// was read, `scored` with a positive paid count). Publishing them one atomic -/// at a time would let a concurrent reader pair the new outcome with the -/// previous tick's paid count or reason, which is worse than a stale snapshot — -/// it is a combination no tick ever produced, and the CLI and operators read -/// these fields together to decide which half of the reward path is missing. +/// Everything a reader sees is published under **one** lock, counters +/// included. The per-tick fields describe a single tick and are read as a set +/// (`unpaid` with a feed that was read, `scored` with a positive paid count), +/// so publishing them one atomic at a time would let a concurrent reader pair +/// the new outcome with the previous tick's paid count or reason — a +/// combination no tick ever produced, and the CLI and operators read these +/// fields together to decide which half of the reward path is missing. /// -/// `ticks` and `scored_epoch` stay separate: they are monotonic counters whose -/// value does not depend on which tick is "last". +/// The counters are in the same lock rather than beside it for the same +/// reason: a reader that saw `ticks: 1` next to the pre-tick `Never` would be +/// looking at two different states, and would have to know that pairing is +/// legal to read the snapshot correctly. One lock makes the whole view a +/// value that some tick actually produced. #[derive(Debug)] pub struct EmitterStatus { wired: bool, - ticks: AtomicU64, - scored_epoch: AtomicU64, - last: Mutex, + published: Mutex, +} + +/// The complete published state: monotonic counters plus the last tick. +#[derive(Debug, Default)] +struct Published { + ticks: u64, + scored_epoch: u64, + last: LastTick, } /// The fields of one completed tick, published and read as one value. @@ -347,9 +355,7 @@ impl EmitterStatus { pub fn new(wired: bool) -> Self { Self { wired, - ticks: AtomicU64::new(0), - scored_epoch: AtomicU64::new(0), - last: Mutex::new(LastTick::default()), + published: Mutex::new(Published::default()), } } @@ -362,9 +368,10 @@ impl EmitterStatus { /// Record one completed tick as a single replacement, so no reader can /// observe a mix of this tick and the previous one. pub fn record(&self, tick: EmitterTick<'_>) { - self.scored_epoch - .fetch_max(tick.scored_epoch, Ordering::Relaxed); - *lock(&self.last) = LastTick { + let mut published = lock(&self.published); + published.ticks = published.ticks.saturating_add(1); + published.scored_epoch = published.scored_epoch.max(tick.scored_epoch); + published.last = LastTick { kind: tick.kind, epoch: tick.epoch, pin_block: tick.pin_block, @@ -374,28 +381,30 @@ impl EmitterStatus { reason: tick.reason.unwrap_or_default().to_owned(), error: tick.error.unwrap_or_default().to_owned(), }; - // Bumped last: a reader that sees tick N may still see N-1's fields if - // it raced the lock, but the count never claims a tick whose fields - // have not landed. - self.ticks.fetch_add(1, Ordering::Release); } /// Read-side snapshot for `/v1/status`. + /// + /// The whole snapshot is read under one lock, so the counters and the + /// last-tick fields always describe the same state. Reading them + /// separately would let a reader see `ticks: 1` beside the pre-tick + /// `Never`, which no observer should have to reason about. #[must_use] pub fn view(&self) -> EmitterStatusView { - let last = lock(&self.last).clone(); + let published = lock(&self.published); + let last = &published.last; EmitterStatusView { wired: self.wired, - ticks: self.ticks.load(Ordering::Acquire), + ticks: published.ticks, last_outcome: last.kind, last_epoch: last.epoch, last_pin_block: last.pin_block, last_participants: last.participants, last_paid: last.paid, last_feed_read: last.feed_read, - scored_epoch: self.scored_epoch.load(Ordering::Relaxed), - last_reason: last.reason, - last_error: last.error, + scored_epoch: published.scored_epoch, + last_reason: last.reason.clone(), + last_error: last.error.clone(), } } } diff --git a/crates/bounty-challenge/tests/emit_fail_closed.rs b/crates/bounty-challenge/tests/emit_fail_closed.rs index 00b3c1290..dd8db9594 100644 --- a/crates/bounty-challenge/tests/emit_fail_closed.rs +++ b/crates/bounty-challenge/tests/emit_fail_closed.rs @@ -991,14 +991,18 @@ async fn status_never_pairs_an_outcome_with_another_ticks_counts() { let writer = { let status = Arc::clone(&status); tokio::task::spawn_blocking(move || { - for i in 0..20_000u32 { + for i in 0..200_000u32 { status.record(if i % 2 == 0 { scored } else { unpaid }); } }) }; let mut snapshots = 0u32; - while !writer.is_finished() { + let mut saw_scored = false; + let mut saw_unpaid = false; + // Keep reading while the writer is live *and* until both states have been + // observed, so the test cannot pass by never overlapping the writer. + while !(writer.is_finished() && saw_scored && saw_unpaid) { let view = status.view(); match view.last_outcome { EmitterOutcomeKind::Scored => { @@ -1013,6 +1017,7 @@ async fn status_never_pairs_an_outcome_with_another_ticks_counts() { "a scored snapshot must be exactly the scored tick: {view:?}" ); assert!(view.last_reason.is_empty(), "{view:?}"); + saw_scored = true; } EmitterOutcomeKind::Unpaid => { assert_eq!( @@ -1026,13 +1031,18 @@ async fn status_never_pairs_an_outcome_with_another_ticks_counts() { "an unpaid snapshot must be exactly the unpaid tick: {view:?}" ); assert_eq!(view.last_reason, "nothing payable", "{view:?}"); + saw_unpaid = true; } - // The reader can legitimately race ahead of the writer's first - // tick; that is the documented initial state, not a torn read. + // `Never` is the pre-tick state, and the whole view is published + // under one lock, so it can only be observed with `ticks: 0` — + // never beside a counter that has already moved. EmitterOutcomeKind::Never => { assert_eq!(view.ticks, 0, "Never is only before the first tick"); assert_eq!(view.last_participants, 0); assert_eq!(view.last_paid, 0); + assert_eq!(view.last_epoch, 0); + assert_eq!(view.last_pin_block, 0); + assert!(view.last_reason.is_empty()); } other => panic!("no other outcome was ever recorded: {other:?}"), } @@ -1041,11 +1051,11 @@ async fn status_never_pairs_an_outcome_with_another_ticks_counts() { } writer.await.expect("writer"); assert!( - snapshots > 0, - "the reader must actually have raced the writer" + snapshots > 0 && saw_scored && saw_unpaid, + "the reader must actually have raced the writer (snapshots={snapshots})" ); - // The counters are monotonic and independent of which tick is last. assert_eq!(status.view().scored_epoch, 43); + assert_eq!(status.view().ticks, 200_000); } /// A scored tick publishes what was actually paid, so an operator can see the From e7b1bc7b09f9a7a172bac36cee0301175fc9c44d Mon Sep 17 00:00:00 2001 From: DroidAgent <154886644+echobt@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:10:40 +0000 Subject: [PATCH 4/4] test(bounty): bound the status race test so it cannot spin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reader loop waited for the writer to finish *and* for both alternating states to be observed. If the writer completed first — leaving the durable final state `unpaid` — `saw_scored` could never become true, so the loop spun forever instead of failing. That is a hang in a test that guards a hang-free property, and it would have wedged local and CI runs. Termination no longer depends on which interleaving happens. The loop is bounded by a hard iteration cap and breaks when the writer finishes, and the deterministic assertions moved after the join: the last write is an odd index, so the final state is fixed and every field can be asserted outright rather than waited for. The concurrency check keeps its power — mutation-checked, a publish that lands the counters before the tick fields still fails it with `Never` beside a moved counter. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .../tests/emit_fail_closed.rs | 136 +++++++++++++++--- 1 file changed, 116 insertions(+), 20 deletions(-) diff --git a/crates/bounty-challenge/tests/emit_fail_closed.rs b/crates/bounty-challenge/tests/emit_fail_closed.rs index dd8db9594..2e135d7f9 100644 --- a/crates/bounty-challenge/tests/emit_fail_closed.rs +++ b/crates/bounty-challenge/tests/emit_fail_closed.rs @@ -24,10 +24,6 @@ #![forbid(unsafe_code)] #![allow(clippy::unwrap_used, clippy::expect_used)] -use std::net::SocketAddr; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; - use axum::extract::State; use axum::routing::{get, post}; use axum::{Json, Router}; @@ -38,6 +34,9 @@ use bounty_challenge::{ use chain::{ AxonInfo, ChainClient, ChainError, FakeChain, FakeChainConfig, Metagraph, WeightsTlockPayload, }; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; /// `FakeChain` keeps its call log in a `RefCell`; the emitter needs `Sync`. struct LockedFake(Mutex); @@ -959,11 +958,14 @@ async fn a_held_tick_does_not_claim_a_pin_it_did_not_use() { /// a fault. /// /// The writer alternates between two ticks whose fields differ in *every* -/// position, so a torn write cannot coincidentally look coherent: a `scored` -/// snapshot carrying the other tick's `paid`/`participants`/`reason` is -/// detected immediately. +/// position, so a snapshot carrying one tick's outcome with the other's counts +/// is caught. Termination does not depend on which interleaving happens: the +/// loop is bounded by a hard iteration cap and breaks when the writer finishes, +/// and the final assertion runs after `join` on a state the last write fixes. +/// A writer that finishes early therefore ends the test rather than spinning. #[tokio::test] async fn status_never_pairs_an_outcome_with_another_ticks_counts() { + const WRITES: u32 = 200_000; let status = Arc::new(EmitterStatus::new(true)); let scored = EmitterTick { kind: EmitterOutcomeKind::Scored, @@ -991,18 +993,16 @@ async fn status_never_pairs_an_outcome_with_another_ticks_counts() { let writer = { let status = Arc::clone(&status); tokio::task::spawn_blocking(move || { - for i in 0..200_000u32 { + for i in 0..WRITES { status.record(if i % 2 == 0 { scored } else { unpaid }); } }) }; + // Read while the writer runs, asserting every snapshot is one whole tick. + // The cap makes this terminate even if the writer is descheduled forever. let mut snapshots = 0u32; - let mut saw_scored = false; - let mut saw_unpaid = false; - // Keep reading while the writer is live *and* until both states have been - // observed, so the test cannot pass by never overlapping the writer. - while !(writer.is_finished() && saw_scored && saw_unpaid) { + for _ in 0..WRITES { let view = status.view(); match view.last_outcome { EmitterOutcomeKind::Scored => { @@ -1017,7 +1017,6 @@ async fn status_never_pairs_an_outcome_with_another_ticks_counts() { "a scored snapshot must be exactly the scored tick: {view:?}" ); assert!(view.last_reason.is_empty(), "{view:?}"); - saw_scored = true; } EmitterOutcomeKind::Unpaid => { assert_eq!( @@ -1031,7 +1030,6 @@ async fn status_never_pairs_an_outcome_with_another_ticks_counts() { "an unpaid snapshot must be exactly the unpaid tick: {view:?}" ); assert_eq!(view.last_reason, "nothing payable", "{view:?}"); - saw_unpaid = true; } // `Never` is the pre-tick state, and the whole view is published // under one lock, so it can only be observed with `ticks: 0` — @@ -1047,15 +1045,113 @@ async fn status_never_pairs_an_outcome_with_another_ticks_counts() { other => panic!("no other outcome was ever recorded: {other:?}"), } snapshots += 1; + if writer.is_finished() { + break; + } tokio::task::yield_now().await; } writer.await.expect("writer"); - assert!( - snapshots > 0 && saw_scored && saw_unpaid, - "the reader must actually have raced the writer (snapshots={snapshots})" + assert!(snapshots > 0, "the reader must have observed the status"); + + // Deterministic final state: the last write is an odd index, so it is the + // `unpaid` tick, and the counters must agree with it. + let final_view = status.view(); + assert_eq!(final_view.last_outcome, EmitterOutcomeKind::Unpaid); + assert_eq!(final_view.ticks, u64::from(WRITES)); + assert_eq!(final_view.scored_epoch, 43); + assert_eq!(final_view.last_epoch, 43); + assert_eq!(final_view.last_pin_block, 1_360); + assert_eq!(final_view.last_participants, 7); + assert_eq!(final_view.last_paid, 0); +} + +/// The exact field set of each outcome, read back deterministically. +/// +/// The concurrency test above proves snapshots are consistent; this one proves +/// they are *correct*, which is what a split-publish regression would break +/// without necessarily losing a race on the machine running the tests. +#[tokio::test] +async fn each_outcome_publishes_its_own_fields_and_nothing_else() { + let status = EmitterStatus::new(true); + + // Before any tick: the documented initial state, counters included. + let initial = status.view(); + assert_eq!(initial.last_outcome, EmitterOutcomeKind::Never); + assert_eq!(initial.ticks, 0); + assert_eq!(initial.scored_epoch, 0); + assert_eq!(initial.last_epoch, 0); + assert_eq!(initial.last_pin_block, 0); + assert_eq!(initial.last_participants, 0); + assert_eq!(initial.last_paid, 0); + assert!(!initial.last_feed_read); + + status.record(EmitterTick { + kind: EmitterOutcomeKind::Scored, + epoch: 42, + pin_block: 1_000, + participants: 3, + paid: 2, + feed_read: true, + scored_epoch: 42, + reason: None, + error: None, + }); + let scored = status.view(); + assert_eq!(scored.last_outcome, EmitterOutcomeKind::Scored); + assert_eq!(scored.ticks, 1); + assert_eq!(scored.last_epoch, 42); + assert_eq!(scored.last_pin_block, 1_000); + assert_eq!(scored.last_participants, 3); + assert_eq!(scored.last_paid, 2); + assert!(scored.last_feed_read); + assert_eq!(scored.scored_epoch, 42); + assert!(scored.last_reason.is_empty()); + assert!(scored.last_error.is_empty()); + + // A later cover replaces every field rather than leaving the scored + // tick's counts behind. + status.record(EmitterTick { + kind: EmitterOutcomeKind::Unpaid, + epoch: 43, + pin_block: 1_360, + participants: 9, + paid: 0, + feed_read: true, + scored_epoch: 43, + reason: Some("nothing payable"), + error: None, + }); + let unpaid = status.view(); + assert_eq!(unpaid.last_outcome, EmitterOutcomeKind::Unpaid); + assert_eq!(unpaid.ticks, 2); + assert_eq!(unpaid.last_epoch, 43); + assert_eq!(unpaid.last_pin_block, 1_360); + assert_eq!(unpaid.last_participants, 9); + assert_eq!(unpaid.last_paid, 0, "a cover pays nobody"); + assert_eq!(unpaid.last_reason, "nothing payable"); + assert_eq!(unpaid.scored_epoch, 43); + + // An error tick carries no reason and names the failure. + status.record(EmitterTick { + kind: EmitterOutcomeKind::Error, + epoch: 0, + pin_block: 0, + participants: 0, + paid: 0, + feed_read: false, + scored_epoch: 43, + reason: None, + error: Some("chain: block_hash@1: timeout"), + }); + let errored = status.view(); + assert_eq!(errored.last_outcome, EmitterOutcomeKind::Error); + assert_eq!(errored.ticks, 3); + assert!(errored.last_reason.is_empty()); + assert!(errored.last_error.contains("timeout")); + assert_eq!( + errored.scored_epoch, 43, + "an error does not un-score an epoch" ); - assert_eq!(status.view().scored_epoch, 43); - assert_eq!(status.view().ticks, 200_000); } /// A scored tick publishes what was actually paid, so an operator can see the