Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 16 additions & 7 deletions bins/bounty-challenge/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ use std::time::Duration;

use bounty_challenge::{
bounty_router, hash_admin_token, legacy_sim_opt_in_present, resolve_scoring_backend, AppState,
BountyEmitter, BountyStore, GatewayClient, GatewayClientConfig, ScoringBackend, CHALLENGE_ID,
DEFAULT_EMIT_POLL_SECS, SCORING_VERSION,
BountyEmitter, BountyStore, EmitterStatus, GatewayClient, GatewayClientConfig, ScoringBackend,
CHALLENGE_ID, DEFAULT_EMIT_POLL_SECS, SCORING_VERSION,
};
use challenge_keys::load_challenge_secret;
use clap::Parser;
Expand Down Expand Up @@ -124,17 +124,26 @@ fn run(cli: &Cli) -> Result<(), String> {
503 and the emitter will pay nobody (the challenge share burns to uid 0) until then"
),
}
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|e| e.to_string())?;
// The emitter owns the read side published on GET /v1/status, so build it
// first and hand the same handle to the HTTP state. A host that wires no
// emitter (missing challenge key) publishes `emitter_wired: false`, which
// is the one condition that also 409s the seal.
let emitter = build_emitter(cli, scoring, sk)?;
let status = emitter
.as_ref()
.map_or_else(|| Arc::new(EmitterStatus::new(false)), |e| e.status());
let state = AppState {
store: BountyStore::new(),
session_secret: Arc::new(session_secret),
scoring,
admin_hashes: Arc::new(admin_hashes),
emitter: Some(status),
};
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|e| e.to_string())?;
if let Some(emitter) = build_emitter(cli, scoring, sk)? {
if let Some(emitter) = emitter {
let poll = Duration::from_secs(cli.emit_poll_secs.max(1));
rt.spawn(emitter.run(poll));
}
Expand Down
29 changes: 29 additions & 0 deletions bins/ctx/src/bounty.rs
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,35 @@ pub async fn status(client: &Client, json_out: bool) -> Result<(), String> {
if let Some(v) = reply.body.get("champion_hotkey") {
println!("champion_hotkey: {}", compact(v));
}
// `can_score` says this host *may* pay; the emitter says whether it *is*.
// A reachable feed with nothing adjudicated pays nobody, and a miner
// deserves to see which of those they are looking at.
if let Some(v) = reply.body.get("emitter_wired") {
println!("emitter_wired: {}", compact(v));
}
if let Some(e) = reply.body.get("emitter").filter(|v| v.is_object()) {
println!("last_outcome: {}", compact(&e["last_outcome"]));
println!("last_feed_read: {}", compact(&e["last_feed_read"]));
println!("last_paid: {}", compact(&e["last_paid"]));
if let Some(reason) = e["last_reason"].as_str().filter(|s| !s.is_empty()) {
println!("last_reason: {reason}");
}
match e["last_outcome"].as_str().unwrap_or("never") {
"unpaid" => println!(
" note: the feed answered and nothing was payable — reports become weight \
once an operator adjudicates them valid with a severity."
),
"burned" => println!(
" note: the feed could not be read, so this epoch pays nobody. Not a fault \
on your side; check again later."
),
"error" => println!(
" note: the emitter could not post leaves at all (chain, signing, or \
gateway). An operator has to look at this."
),
_ => {}
}
}
if let Some(q) = reply.body.get("quotas") {
println!("quotas: {q}");
} else {
Expand Down
213 changes: 213 additions & 0 deletions crates/bounty-challenge-task/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
use keystore::{ss58_decode, ss58_encode, BITTENSOR_SS58_PREFIX, KEY_LEN};
use schnorrkel::{signing_context, ExpansionMode, MiniSecretKey, PublicKey, Signature};
use serde::{Deserialize, Serialize};
use std::sync::Mutex;
use thiserror::Error;

/// Normative challenge id (trust-root / leaf `challenge_id` string).
Expand Down Expand Up @@ -206,6 +207,218 @@ pub fn resolve_scoring_backend() -> ScoringBackend {
}
}

// --- Emitter observability ---------------------------------------------------
//
// The reward linkage is a chain of two halves that fail independently: the
// backend publishes adjudications, and this host turns them into signed
// leaves. `/v1/status` already says whether a feed is *configured*; that is not
// the same as whether weight is being produced. A readable feed that publishes
// nothing payable, or a crowned hotkey that is not in `E`, is an empty payout
// that no local store reading can see. These types are the surface that makes
// it visible instead of leaving it to a log grep.

/// What one completed emitter tick did.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum EmitterOutcomeKind {
/// No tick has completed yet.
Never,
/// The feed published payable rows and they became signed scores.
Scored,
/// The feed was read, but nothing payable became weight this tick.
Unpaid,
/// The feed could not be read; `E` was covered with `ChallengeInternal`.
Burned,
/// A scored epoch was left in place through a feed outage.
Held,
/// The tick could not emit at all (chain, signing, or gateway).
Error,
}

/// One completed tick, as recorded by [`EmitterStatus::record`].
#[derive(Debug, Clone, Copy)]
pub struct EmitterTick<'a> {
/// What the tick did.
pub kind: EmitterOutcomeKind,
/// Subnet epoch the leaves were signed for (0 when the tick never got there).
pub epoch: u64,
/// Block `E` was pinned at (0 when unknown).
pub pin_block: u64,
/// Size of `E`.
pub participants: usize,
/// Hotkeys that received a positive score.
pub paid: usize,
/// Whether the backend public feed answered this tick.
pub feed_read: bool,
/// Highest epoch this process has scored (0 = none).
pub scored_epoch: u64,
/// Why nobody was paid, or why the epoch was held.
pub reason: Option<&'a str>,
/// Why the tick emitted nothing at all.
pub error: Option<&'a str>,
}

/// Read-side view of the emitter, published on `GET /v1/status`.
///
/// `last_outcome: "unpaid"` with `last_feed_read: true` is the one an operator
/// has to notice: the feed is reachable, the challenge is running, and the
/// epoch still pays nobody. `last_reason` says which half is missing.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EmitterStatusView {
/// Whether this host wired an emitter at all (a missing challenge key
/// leaves `false`, which is the one case that also 409s the seal).
pub wired: bool,
/// Completed ticks since boot.
pub ticks: u64,
/// Outcome of the most recent tick.
pub last_outcome: EmitterOutcomeKind,
/// Epoch of the most recent tick (0 = none yet).
pub last_epoch: u64,
/// Pin block of the most recent tick (0 = none yet).
pub last_pin_block: u64,
/// Size of `E` on the most recent tick.
pub last_participants: u64,
/// Positive leaves on the most recent tick.
pub last_paid: u64,
/// Whether the feed answered on the most recent tick.
pub last_feed_read: bool,
/// Highest epoch this process scored (0 = none). In-process: a restart
/// inside an outage can still burn an epoch that had scores, and the next
/// successful tick supersedes it back.
pub scored_epoch: u64,
/// Why nobody was paid, or why the epoch was held.
pub last_reason: String,
/// Why the tick emitted nothing at all.
pub last_error: String,
}

/// Shared emitter state. The emitter writes it; `/v1/status` reads it.
///
/// Everything a reader sees is published under **one** lock, counters
/// included. The per-tick fields describe a single tick and are read as a set
/// (`unpaid` with a feed that was read, `scored` with a positive paid count),
/// so publishing them one atomic at a time would let a concurrent reader pair
/// the new outcome with the previous tick's paid count or reason — a
/// combination no tick ever produced, and the CLI and operators read these
/// fields together to decide which half of the reward path is missing.
///
/// The counters are in the same lock rather than beside it for the same
/// reason: a reader that saw `ticks: 1` next to the pre-tick `Never` would be
/// looking at two different states, and would have to know that pairing is
/// legal to read the snapshot correctly. One lock makes the whole view a
/// value that some tick actually produced.
#[derive(Debug)]
pub struct EmitterStatus {
wired: bool,
published: Mutex<Published>,
}

/// The complete published state: monotonic counters plus the last tick.
#[derive(Debug, Default)]
struct Published {
ticks: u64,
scored_epoch: u64,
last: LastTick,
}

/// The fields of one completed tick, published and read as one value.
#[derive(Debug, Clone)]
struct LastTick {
kind: EmitterOutcomeKind,
epoch: u64,
pin_block: u64,
participants: u64,
paid: u64,
feed_read: bool,
reason: String,
error: String,
}

impl Default for LastTick {
fn default() -> Self {
Self {
kind: EmitterOutcomeKind::Never,
epoch: 0,
pin_block: 0,
participants: 0,
paid: 0,
feed_read: false,
reason: String::new(),
error: String::new(),
}
}
}

impl EmitterStatus {
/// Fresh status for a host that did or did not wire an emitter.
#[must_use]
pub fn new(wired: bool) -> Self {
Self {
wired,
published: Mutex::new(Published::default()),
}
}

/// Whether this host wired an emitter.
#[must_use]
pub fn wired(&self) -> bool {
self.wired
}

/// Record one completed tick as a single replacement, so no reader can
/// observe a mix of this tick and the previous one.
pub fn record(&self, tick: EmitterTick<'_>) {
let mut published = lock(&self.published);
published.ticks = published.ticks.saturating_add(1);
published.scored_epoch = published.scored_epoch.max(tick.scored_epoch);
published.last = LastTick {
kind: tick.kind,
epoch: tick.epoch,
pin_block: tick.pin_block,
participants: count_u64(tick.participants),
paid: count_u64(tick.paid),
feed_read: tick.feed_read,
reason: tick.reason.unwrap_or_default().to_owned(),
error: tick.error.unwrap_or_default().to_owned(),
};
}

/// Read-side snapshot for `/v1/status`.
///
/// The whole snapshot is read under one lock, so the counters and the
/// last-tick fields always describe the same state. Reading them
/// separately would let a reader see `ticks: 1` beside the pre-tick
/// `Never`, which no observer should have to reason about.
#[must_use]
pub fn view(&self) -> EmitterStatusView {
let published = lock(&self.published);
let last = &published.last;
EmitterStatusView {
wired: self.wired,
ticks: published.ticks,
last_outcome: last.kind,
last_epoch: last.epoch,
last_pin_block: last.pin_block,
last_participants: last.participants,
last_paid: last.paid,
last_feed_read: last.feed_read,
scored_epoch: published.scored_epoch,
last_reason: last.reason.clone(),
last_error: last.error.clone(),
}
}
}

/// A poisoned lock here is a status snapshot, not a consensus input: recover
/// the guard rather than take the challenge down over a display field.
fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}

fn count_u64(n: usize) -> u64 {
u64::try_from(n).unwrap_or(u64::MAX)
}

/// Most reports one hotkey may leave awaiting adjudication.
///
/// Adjudication is the scarce resource in this challenge: every pending report
Expand Down
3 changes: 3 additions & 0 deletions crates/bounty-challenge/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,11 @@ trustroot = { path = "../trustroot" }

[dev-dependencies]
axum = "0.8"
http-body-util = "0.1"
serde_json = "1"
tokio = { version = "1", features = ["macros", "net", "rt", "rt-multi-thread", "time"] }
toml = "0.8"
tower = { version = "0.5", features = ["util"] }

[lints]
workspace = true
Loading
Loading