From 7836e72ffbed0f3531d56dccd84973912bc56c94 Mon Sep 17 00:00:00 2001 From: Bohdan Ohorodnii <273991985+varex83agent@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:06:12 +0200 Subject: [PATCH 1/3] feat(tracing): propagate log topic across spawns; label components (#588) `MetricsLayer` labels `app_log_{warn,error}_total` with the `topic` field from the nearest enclosing span. Span context is not carried across `tokio::spawn`, and pluto set the `topic` field in only two places, so almost every warn/error was counted under `topic=""`. Add a span-propagating spawn helper (`pluto_tracing::spawn`) that attaches `Span::current()` to the spawned future, restoring context-like topic propagation, and set a `&'static str` `topic` root span on each long-running component, reusing charon's topic names: - sched, tracker, sigagg, bcast (+recast), parsigex, vapi, qbft, p2p, peerinfo, dkg, relay, vmock, and app-start as the catch-all. Adds tests asserting the helper propagates the topic across the task boundary while a bare `tokio::spawn` does not. Co-Authored-By: Bohdan Ohorodnii <35969035+varex83@users.noreply.github.com> --- crates/app/src/node/mod.rs | 9 ++ crates/app/src/node/wire.rs | 34 ++++--- crates/cli/src/commands/relay.rs | 1 + crates/consensus/src/qbft/component.rs | 33 ++++--- crates/consensus/src/qbft/runner.rs | 3 + crates/core/src/bcast/mod.rs | 1 + crates/core/src/bcast/recast.rs | 1 + crates/core/src/scheduler.rs | 1 + crates/core/src/sigagg.rs | 1 + crates/core/src/tracker/mod.rs | 1 + crates/core/src/validatorapi/router.rs | 12 +++ crates/dkg/src/dkg.rs | 3 +- crates/p2p/src/bootnode.rs | 12 ++- crates/p2p/src/p2p.rs | 1 + crates/parsigex/src/behaviour.rs | 21 +++- crates/peerinfo/src/protocol.rs | 12 +++ crates/relay-server/src/web.rs | 2 +- .../testutil/src/validatormock/component.rs | 6 +- crates/tracing/src/lib.rs | 4 + crates/tracing/src/spawn.rs | 95 +++++++++++++++++++ 20 files changed, 212 insertions(+), 41 deletions(-) create mode 100644 crates/tracing/src/spawn.rs diff --git a/crates/app/src/node/mod.rs b/crates/app/src/node/mod.rs index 9c134c84..06ee5e53 100644 --- a/crates/app/src/node/mod.rs +++ b/crates/app/src/node/mod.rs @@ -226,6 +226,15 @@ impl App { /// Loads the cluster lock + key, builds the consensus component and P2P /// behaviours, wires the core workflow, and drives the node. +/// +/// Carries the `app-start` topic as the catch-all for log metrics not +/// attributed to a more specific component (mirrors charon's `app.Run`). +#[tracing::instrument( + name = "app-start", + level = "debug", + skip_all, + fields(topic = "app-start") +)] async fn run(config: AppConfig, ct: CancellationToken) -> Result<(), AppError> { // ---- (1) Load cluster lock + key, derive peers and this node's index ---- // diff --git a/crates/app/src/node/wire.rs b/crates/app/src/node/wire.rs index 40dab2ff..7bbe6db6 100644 --- a/crates/app/src/node/wire.rs +++ b/crates/app/src/node/wire.rs @@ -652,26 +652,32 @@ pub async fn wire_core_workflow( move |duty: Duty, value: pbcore::UnsignedDataSet| { let dutydb = Arc::clone(&dutydb); let tracker = Arc::clone(&tracker); - tokio::spawn(async move { - let core_set = - match unsigneddata::unsigned_data_set_from_proto(&duty.duty_type, &value) { + let span = tracing::debug_span!("app-start", topic = "app-start"); + tokio::spawn(tracing::Instrument::instrument( + async move { + let core_set = match unsigneddata::unsigned_data_set_from_proto( + &duty.duty_type, + &value, + ) { Ok(set) => set, Err(err) => { tracing::warn!(?err, "dutydb: decode unsigned data set"); return; } }; - let pubkeys: Vec = core_set.keys().copied().collect(); - // Logged before the error moves into the tracker's `Arc`. - let step_err = match dutydb.store(duty.clone(), core_set).await { - Ok(()) => None, - Err(err) => { - tracing::warn!(?err, "dutydb: store"); - Some(owned_step_err(err)) - } - }; - tracker.duty_db_stored(duty, &pubkeys, step_err).await; - }); + let pubkeys: Vec = core_set.keys().copied().collect(); + // Logged before the error moves into the tracker's `Arc`. + let step_err = match dutydb.store(duty.clone(), core_set).await { + Ok(()) => None, + Err(err) => { + tracing::warn!(?err, "dutydb: store"); + Some(owned_step_err(err)) + } + }; + tracker.duty_db_stored(duty, &pubkeys, step_err).await; + }, + span, + )); Ok(()) }, )); diff --git a/crates/cli/src/commands/relay.rs b/crates/cli/src/commands/relay.rs index 44e9d795..0572bdbd 100644 --- a/crates/cli/src/commands/relay.rs +++ b/crates/cli/src/commands/relay.rs @@ -293,6 +293,7 @@ pub struct RelayLokiArgs { pub loki_service: String, } +#[tracing::instrument(name = "relay", level = "debug", skip_all, fields(topic = "relay"))] pub async fn run( config: pluto_relay_server::config::Config, ct: CancellationToken, diff --git a/crates/consensus/src/qbft/component.rs b/crates/consensus/src/qbft/component.rs index 820ac25e..1db7cc82 100644 --- a/crates/consensus/src/qbft/component.rs +++ b/crates/consensus/src/qbft/component.rs @@ -12,6 +12,7 @@ use prost::{Message, Name}; use prost_types::Any; use tokio::{sync::mpsc, task::JoinHandle}; use tokio_util::sync::CancellationToken; +use tracing::Instrument as _; use crate::{ instance::InstanceIo, @@ -474,22 +475,26 @@ impl Consensus { .expect("start must be called exactly once"); let instances = Arc::clone(&self.instances); - tokio::spawn(async move { - loop { - tokio::select! { - () = ct.cancelled() => return, - duty = expired_rx.recv() => match duty { - Some(duty) => { - instances - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remove(&duty); - } - None => return, - }, + let span = tracing::debug_span!("qbft", topic = "qbft"); + tokio::spawn( + async move { + loop { + tokio::select! { + () = ct.cancelled() => return, + duty = expired_rx.recv() => match duty { + Some(duty) => { + instances + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&duty); + } + None => return, + }, + } } } - }) + .instrument(span), + ) } /// Returns existing instance I/O for `duty`, or creates an empty one. diff --git a/crates/consensus/src/qbft/runner.rs b/crates/consensus/src/qbft/runner.rs index bd05764e..bef2b4c0 100644 --- a/crates/consensus/src/qbft/runner.rs +++ b/crates/consensus/src/qbft/runner.rs @@ -123,6 +123,7 @@ pub(crate) async fn propose_priority( } /// Hashes and packs the local value, then starts or joins the duty runner. +#[tracing::instrument(name = "qbft", level = "debug", skip_all, fields(topic = "qbft"))] async fn propose( consensus: &Consensus, duty: Duty, @@ -166,6 +167,7 @@ where } /// Starts participating in a duty without a local proposal value. +#[tracing::instrument(name = "qbft", level = "debug", skip_all, fields(topic = "qbft"))] pub(crate) async fn participate( consensus: &Consensus, duty: Duty, @@ -194,6 +196,7 @@ pub(crate) async fn participate( } /// Runs one consensus instance and publishes its completion result. +#[tracing::instrument(name = "qbft", level = "debug", skip_all, fields(topic = "qbft"))] pub(crate) async fn run_instance( consensus: &Consensus, duty: Duty, diff --git a/crates/core/src/bcast/mod.rs b/crates/core/src/bcast/mod.rs index 99a3f53e..91738365 100644 --- a/crates/core/src/bcast/mod.rs +++ b/crates/core/src/bcast/mod.rs @@ -268,6 +268,7 @@ impl Broadcaster { /// success record the broadcast count and submission delay. Internal-only /// duties (randao, prepare-aggregator, prepare-sync-contribution) are /// no-ops; deprecated and unknown duty types return an error. + #[tracing::instrument(name = "bcast", level = "debug", skip_all, fields(topic = "bcast"))] pub async fn broadcast(&self, mut duty: Duty, set: SignedDataSet) -> Result<()> { match duty.duty_type { DutyType::Attester => self.broadcast_attester(&duty, &set).await?, diff --git a/crates/core/src/bcast/recast.rs b/crates/core/src/bcast/recast.rs index 4514349a..07117d85 100644 --- a/crates/core/src/bcast/recast.rs +++ b/crates/core/src/bcast/recast.rs @@ -92,6 +92,7 @@ impl Recaster { } /// Called when new slots tick. + #[tracing::instrument(name = "bcast", level = "debug", skip_all, fields(topic = "bcast"))] pub async fn slot_ticked(&self, slot: Slot) -> Result<()> { if !slot.first_in_epoch() { return Ok(()); diff --git a/crates/core/src/scheduler.rs b/crates/core/src/scheduler.rs index d29b5abe..5437d788 100644 --- a/crates/core/src/scheduler.rs +++ b/crates/core/src/scheduler.rs @@ -313,6 +313,7 @@ struct SchedulerActor { } impl SchedulerActor { + #[tracing::instrument(name = "sched", level = "debug", skip_all, fields(topic = "sched"))] async fn run( mut self, mut slot_rx: sync::mpsc::Receiver, diff --git a/crates/core/src/sigagg.rs b/crates/core/src/sigagg.rs index de1dfcdd..50b26ca5 100644 --- a/crates/core/src/sigagg.rs +++ b/crates/core/src/sigagg.rs @@ -143,6 +143,7 @@ impl Aggregator { /// /// If aggregation fails for any validator the entire call returns that /// error immediately — no partial results are emitted. + #[tracing::instrument(name = "sigagg", level = "debug", skip_all, fields(topic = "sigagg"))] pub async fn aggregate( &self, duty: &Duty, diff --git a/crates/core/src/tracker/mod.rs b/crates/core/src/tracker/mod.rs index fb027213..af8a96c8 100644 --- a/crates/core/src/tracker/mod.rs +++ b/crates/core/src/tracker/mod.rs @@ -455,6 +455,7 @@ impl TrackerService { ); } + #[tracing::instrument(name = "tracker", level = "debug", skip_all, fields(topic = "tracker"))] async fn run(mut self) { let mut events: HashMap> = HashMap::new(); diff --git a/crates/core/src/validatorapi/router.rs b/crates/core/src/validatorapi/router.rs index b675fcba..39d720c4 100644 --- a/crates/core/src/validatorapi/router.rs +++ b/crates/core/src/validatorapi/router.rs @@ -250,9 +250,21 @@ pub fn new_router( ) .route("/eth/v1/node/version", get(node_version)) .fallback(proxy_handler) + // Attach the `vapi` topic to every request so warn/error logs emitted + // while handling it are counted under `app_log_{warn,error}_total{topic="vapi"}`. + .layer(middleware::from_fn(with_vapi_topic)) .with_state(state) } +/// Middleware that runs each request handler inside a `vapi` topic span so log +/// metrics are attributed to the validator API component. +async fn with_vapi_topic(req: Request, next: Next) -> Response { + use tracing::Instrument as _; + + let span = tracing::debug_span!("vapi", topic = "vapi"); + next.run(req).instrument(span).await +} + async fn attester_duties( State(state): State>, Path(epoch): Path, diff --git a/crates/dkg/src/dkg.rs b/crates/dkg/src/dkg.rs index d5bc9d91..5c07cd76 100644 --- a/crates/dkg/src/dkg.rs +++ b/crates/dkg/src/dkg.rs @@ -377,6 +377,7 @@ fn default_tracing_config() -> TracingConfig { } /// Runs the DKG entrypoint. +#[tracing::instrument(name = "dkg", level = "debug", skip_all, fields(topic = "dkg"))] pub async fn run(conf: Config, ct: CancellationToken) -> Result<(), DkgError> { if ct.is_cancelled() { return Err(DkgError::ShutdownRequestedBeforeStartup); @@ -594,7 +595,7 @@ async fn run_inner(conf: Config, ct: CancellationToken) -> Result<(), DkgError> let sync_clients = handlers.sync.clone(); let sync_server = handlers.sync_server.clone(); let network_ct = ct.child_token(); - let network_task = tokio::spawn(drive_dkg_network(node, network_ct.clone())); + let network_task = pluto_tracing::spawn(drive_dkg_network(node, network_ct.clone())); let result = run_ceremony( &conf, diff --git a/crates/p2p/src/bootnode.rs b/crates/p2p/src/bootnode.rs index ea2d2ec3..2d0ec1ba 100644 --- a/crates/p2p/src/bootnode.rs +++ b/crates/p2p/src/bootnode.rs @@ -6,7 +6,7 @@ use backon::Retryable; use libp2p::Multiaddr; use pluto_eth2util::enr::Record; use tokio_util::sync::CancellationToken; -use tracing::{info, warn}; +use tracing::{Instrument as _, info, warn}; use url::Url; use crate::{ @@ -127,9 +127,13 @@ pub async fn new_relays( let mutable_clone = mutable.clone(); let cancel_clone = cancel.child_token(); - tokio::spawn(async move { - resolve_relay(cancel_clone, url, hash, mutable_clone).await; - }); + let span = tracing::debug_span!("relay", topic = "relay"); + tokio::spawn( + async move { + resolve_relay(cancel_clone, url, hash, mutable_clone).await; + } + .instrument(span), + ); resp.push(mutable); } diff --git a/crates/p2p/src/p2p.rs b/crates/p2p/src/p2p.rs index ceb4b580..5aebae30 100644 --- a/crates/p2p/src/p2p.rs +++ b/crates/p2p/src/p2p.rs @@ -630,6 +630,7 @@ impl Node { } /// Handles a swarm event to update metrics and logging. + #[tracing::instrument(name = "p2p", level = "debug", skip_all, fields(topic = "p2p"))] fn handle_event(&mut self, event: &SwarmEvent>) { match event { // Identify - update peer addresses in the peer store. diff --git a/crates/parsigex/src/behaviour.rs b/crates/parsigex/src/behaviour.rs index 4505150e..0d55ae04 100644 --- a/crates/parsigex/src/behaviour.rs +++ b/crates/parsigex/src/behaviour.rs @@ -21,6 +21,7 @@ use libp2p::{ }, }; use tokio::sync::{RwLock, mpsc, oneshot}; +use tracing::Instrument as _; use pluto_core::{ eth2signeddata, @@ -203,6 +204,12 @@ impl Handle { result_rx.await.map_err(|_| Error::Closed)? } + #[tracing::instrument( + name = "parsigex", + level = "debug", + skip_all, + fields(topic = "parsigex") + )] async fn enqueue( &self, duty: Duty, @@ -498,12 +505,16 @@ impl Behaviour { /// subscribers async). fn notify_subscribers(&self, duty: Duty, data_set: ParSignedDataSet) { let shared_subs = self.shared_subs.clone(); - tokio::spawn(async move { - let subs = shared_subs.subs.read().await.clone(); - for sub in &subs { - sub(duty.clone(), data_set.clone()).await; + let span = tracing::debug_span!("parsigex", topic = "parsigex"); + tokio::spawn( + async move { + let subs = shared_subs.subs.read().await.clone(); + for sub in &subs { + sub(duty.clone(), data_set.clone()).await; + } } - }); + .instrument(span), + ); } } diff --git a/crates/peerinfo/src/protocol.rs b/crates/peerinfo/src/protocol.rs index fe485f09..6bd94570 100644 --- a/crates/peerinfo/src/protocol.rs +++ b/crates/peerinfo/src/protocol.rs @@ -281,6 +281,12 @@ impl ProtocolState { /// Sends a peer info request and waits for a response. /// /// Returns the response `PeerInfo` on success. + #[tracing::instrument( + name = "peerinfo", + level = "debug", + skip_all, + fields(topic = "peerinfo") + )] pub async fn send_peer_info( &self, mut stream: Stream, @@ -301,6 +307,12 @@ impl ProtocolState { /// Receives a peer info request and sends a response. /// /// Returns the stream for potential reuse after successfully responding. + #[tracing::instrument( + name = "peerinfo", + level = "debug", + skip_all, + fields(topic = "peerinfo") + )] pub async fn recv_peer_info( &self, mut stream: Stream, diff --git a/crates/relay-server/src/web.rs b/crates/relay-server/src/web.rs index dcbdd6a8..d51d1730 100644 --- a/crates/relay-server/src/web.rs +++ b/crates/relay-server/src/web.rs @@ -127,7 +127,7 @@ pub async fn enr_server( let resolver_handle = state.p2p_config.external_host.clone().map(|external_host| { let state = state.clone(); let ct = ct.child_token(); - tokio::spawn(resolve_external_host_periodically(state, external_host, ct)) + pluto_tracing::spawn(resolve_external_host_periodically(state, external_host, ct)) }); info!( diff --git a/crates/testutil/src/validatormock/component.rs b/crates/testutil/src/validatormock/component.rs index c7ea50c8..432062d2 100644 --- a/crates/testutil/src/validatormock/component.rs +++ b/crates/testutil/src/validatormock/component.rs @@ -138,6 +138,7 @@ impl Component { } /// Called externally each slot. Mirrors Go's `Component.SlotTicked`. + #[tracing::instrument(name = "vmock", level = "debug", skip_all, fields(topic = "vmock"))] pub async fn slot_ticked(&self, slot: u64) -> Result<()> { if self.delay_on_startup().await { return Ok(()); @@ -270,6 +271,7 @@ impl Drop for Component { } } +#[tracing::instrument(name = "vmock", level = "debug", skip_all, fields(topic = "vmock"))] async fn run_scheduler( inner: Arc, cancel: CancellationToken, @@ -287,7 +289,7 @@ async fn run_scheduler( let Some(scheduled) = maybe else { break }; let inner_for_task = Arc::clone(&inner); let cancel_for_task = cancel.clone(); - duties.spawn(async move { + duties.spawn(tracing::Instrument::instrument(async move { let start_time = scheduled.start_time; let slot = scheduled.slot; let duty_label = scheduled.duty_type.clone(); @@ -312,7 +314,7 @@ async fn run_scheduler( } } } - }); + }, tracing::Span::current())); } // Reap finished duties to keep the JoinSet bounded. Disabled when // empty — `Some(_)` does not match `None`. diff --git a/crates/tracing/src/lib.rs b/crates/tracing/src/lib.rs index 7a4d407d..7632d6b5 100644 --- a/crates/tracing/src/lib.rs +++ b/crates/tracing/src/lib.rs @@ -16,6 +16,10 @@ pub mod layers; /// Metrics for the tracing. pub mod metrics; +/// Span-propagating task spawning. +pub mod spawn; + pub use config::{ConsoleConfig, LokiConfig, TracingConfig, TracingConfigBuilder}; pub use init::{LokiInit, init}; +pub use spawn::spawn; diff --git a/crates/tracing/src/spawn.rs b/crates/tracing/src/spawn.rs new file mode 100644 index 00000000..02ebce97 --- /dev/null +++ b/crates/tracing/src/spawn.rs @@ -0,0 +1,95 @@ +//! Span-propagating task spawning. +//! +//! [`tokio::spawn`] does **not** carry the current [`tracing`] span into the +//! spawned future: the new task starts with an empty span stack. That breaks +//! the `topic` label used by [`crate::layers::metrics::MetricsLayer`], because +//! a `warn!`/`error!` emitted from a bare spawn lands on `topic=""` even when +//! the spawning code is inside a component's `topic` span. +//! +//! Charon derives the same label from `context.Context`, which *is* propagated +//! into goroutines. To restore context-like propagation here, wrap the spawned +//! future with [`tracing::Instrument`] and attach [`tracing::Span::current`]. +//! +//! Prefer [`spawn`] over [`tokio::spawn`] in long-running components so that +//! the component's root `topic` span is inherited by its subtasks by default. + +use std::future::Future; + +use tokio::task::JoinHandle; +use tracing::Instrument as _; + +/// Like [`tokio::spawn`], but attaches the current [`tracing::Span`] to the +/// spawned future so span context (and therefore the metrics `topic` label) is +/// propagated across the task boundary. +/// +/// Use this instead of [`tokio::spawn`] when the calling code runs inside a +/// component's `topic` span and the spawned work should be attributed to the +/// same topic. +pub fn spawn(future: F) -> JoinHandle +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + tokio::spawn(future.instrument(tracing::Span::current())) +} + +#[cfg(test)] +mod tests { + use tracing_subscriber::layer::SubscriberExt as _; + + use crate::{layers::metrics::MetricsLayer, metrics::TRACING_METRICS}; + + #[tokio::test] + async fn spawn_propagates_topic_across_task_boundary() { + let topic = "spawn_helper_propagation_test"; + let subscriber = tracing_subscriber::registry().with(MetricsLayer); + + let before = TRACING_METRICS.error_total[&topic.to_owned()].get(); + + // `Instrument` captures both the span and the dispatcher at spawn time, + // so the default subscriber set below applies to the spawned task. + let guard = tracing::subscriber::set_default(subscriber); + + let span = tracing::info_span!("component", topic); + let handle = { + let _enter = span.enter(); + super::spawn(async { + tracing::error!("boom from spawned task"); + }) + }; + handle.await.unwrap(); + + drop(guard); + + let after = TRACING_METRICS.error_total[&topic.to_owned()].get(); + assert_eq!( + after, + before.saturating_add(1), + "spawned task should inherit topic" + ); + } + + #[tokio::test] + async fn bare_tokio_spawn_loses_topic() { + // Documents the behaviour the helper fixes: a bare spawn drops the + // topic and is counted under the empty label. + let topic = "spawn_helper_bare_test"; + let subscriber = tracing_subscriber::registry().with(MetricsLayer); + + let before = TRACING_METRICS.error_total[&topic.to_owned()].get(); + + let guard = tracing::subscriber::set_default(subscriber); + let span = tracing::info_span!("component", topic); + let handle = { + let _enter = span.enter(); + tokio::spawn(async { + tracing::error!("boom from bare spawned task"); + }) + }; + handle.await.unwrap(); + drop(guard); + + let after = TRACING_METRICS.error_total[&topic.to_owned()].get(); + assert_eq!(after, before, "bare spawn must not inherit topic"); + } +} From d1ef1ff449d865c1288d01727c2d2df8a592071d Mon Sep 17 00:00:00 2001 From: Bohdan Ohorodnii <273991985+varex83agent@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:16:06 +0200 Subject: [PATCH 2/3] docs(tracing): disambiguate spawn intra-doc link The rustdoc build failed with -D warnings because [`spawn`] is ambiguous between the `spawn` module and the `spawn` function. Add parentheses to link to the function. Co-Authored-By: Bohdan Ohorodnii --- crates/tracing/src/spawn.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracing/src/spawn.rs b/crates/tracing/src/spawn.rs index 02ebce97..f3342d08 100644 --- a/crates/tracing/src/spawn.rs +++ b/crates/tracing/src/spawn.rs @@ -10,7 +10,7 @@ //! into goroutines. To restore context-like propagation here, wrap the spawned //! future with [`tracing::Instrument`] and attach [`tracing::Span::current`]. //! -//! Prefer [`spawn`] over [`tokio::spawn`] in long-running components so that +//! Prefer [`spawn()`] over [`tokio::spawn`] in long-running components so that //! the component's root `topic` span is inherited by its subtasks by default. use std::future::Future; From 5af9a332af7e062e1f53ff65032c43846ff45d78 Mon Sep 17 00:00:00 2001 From: Bohdan Ohorodnii <273991985+varex83agent@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:19:50 +0200 Subject: [PATCH 3/3] style(app): wrap comment to satisfy rustfmt comment_width Co-Authored-By: Bohdan Ohorodnii --- crates/app/src/node/wire.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/app/src/node/wire.rs b/crates/app/src/node/wire.rs index 7bbe6db6..813538b4 100644 --- a/crates/app/src/node/wire.rs +++ b/crates/app/src/node/wire.rs @@ -666,7 +666,8 @@ pub async fn wire_core_workflow( } }; let pubkeys: Vec = core_set.keys().copied().collect(); - // Logged before the error moves into the tracker's `Arc`. + // Logged before the error moves into the tracker's + // `Arc`. let step_err = match dutydb.store(duty.clone(), core_set).await { Ok(()) => None, Err(err) => {