From 0534a51443487051531a382662fad2bf61a6c681 Mon Sep 17 00:00:00 2001 From: Mauro Ezequiel Moltrasio Date: Wed, 26 Aug 2026 17:16:59 +0200 Subject: [PATCH 1/3] feat(metrics): add partial scan duration and inode map size gauge Add two new prometheus metrics: * A histogram for tracking duration of partial scans. * A gauge that tracks the size of the userspace side inode map. The inode map gauge is a bit involved, all places where inodes may be added or removed from the map need to be tracked, we might need to implement a couple wrapper methods to make it easier for tracking. --- fact/src/host_scanner.rs | 51 +++++++++++++++++++++++++------- fact/src/metrics/host_scanner.rs | 23 +++++++++++++- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/fact/src/host_scanner.rs b/fact/src/host_scanner.rs index 2ef1a66c..61d48171 100644 --- a/fact/src/host_scanner.rs +++ b/fact/src/host_scanner.rs @@ -178,6 +178,7 @@ impl HostScanner { } else { let _ = self.kernel_inode_map.borrow_mut().remove(inode); self.metrics.scan_inc(ScanLabels::InodeRemoved); + self.metrics.inode_map_size.dec(); false } }); @@ -274,6 +275,7 @@ impl HostScanner { /// base path (the path up to the first glob special character) that /// matches the supplied path. fn scan_partial(&self, path: &Path) -> anyhow::Result<()> { + let start = Instant::now(); let scan_prefix_patterns = self.paths_patterns .iter() @@ -296,6 +298,10 @@ impl HostScanner { for pattern in scan_set.iter().map(|index| &self.paths_patterns[*index]) { self.scan_inner(pattern)?; } + + self.metrics + .scan_partial_duration + .observe(start.elapsed().as_secs_f64()); Ok(()) } @@ -326,6 +332,7 @@ impl HostScanner { } None => { self.metrics.scan_inc(ScanLabels::FileUpdated); + self.metrics.inode_map_size.inc(); inode_map.insert(inode, path.clone()); } }; @@ -399,6 +406,7 @@ You can increase this limit with: if self.inode_map.borrow_mut().remove(inode).is_some() { self.metrics.scan_inc(ScanLabels::InodeRemoved); + self.metrics.inode_map_size.dec(); } self.metrics.scan_inc(ScanLabels::FileRemoved); @@ -412,14 +420,22 @@ You can increase this limit with: // inode we are landing on and put the associated host path in // the old inode. let mut inode_map = self.inode_map.borrow_mut(); - let Some(path) = inode_map.remove(event.get_inode()) else { - warn!("Old path was not found for inode tracked event"); - return; + let path = match inode_map.remove(event.get_inode()) { + Some(p) => { + self.metrics.inode_map_size.dec(); + p + } + None => { + warn!("Old path was not found for inode tracked event"); + return; + } }; let Some(old_inode) = event.get_old_inode() else { unreachable!("old inode not found for rename event"); }; - inode_map.insert(*old_inode, path); + if inode_map.insert(*old_inode, path).is_none() { + self.metrics.inode_map_size.inc(); + } } monitored_t::NOT_MONITORED if event.get_old_monitored() == Some(monitored_t::MONITORED_BY_INODE) => @@ -436,6 +452,7 @@ You can increase this limit with: } let _ = self.kernel_inode_map.borrow_mut().remove(inode); + self.metrics.inode_map_size.dec(); false }); } @@ -446,11 +463,18 @@ You can increase this limit with: monitored_t::MONITORED_BY_PARENT if !event.get_inode().empty() => { // The parent for the target is monitored, but the file itself // is not. Remove the entry for the old file from the map. - self.inode_map.borrow_mut().remove( - event - .get_old_inode() - .expect("rename event did not have old inode"), - ); + if self + .inode_map + .borrow_mut() + .remove( + event + .get_old_inode() + .expect("rename event did not have old inode"), + ) + .is_some() + { + self.metrics.inode_map_size.dec(); + } } monitored_t::MONITORED_BY_PARENT if event.get_old_monitored() == Some(monitored_t::MONITORED_BY_INODE) => @@ -496,6 +520,7 @@ You can increase this limit with: if let Err(e) = self.kernel_inode_map.borrow_mut().remove(inode) { warn!("Failed to remove inode kernel entry: {e:?}"); } + self.metrics.inode_map_size.dec(); false }); } @@ -642,10 +667,14 @@ You can increase this limit with: // whether the event is ignored now that we have the // full inode context. if self.event_is_ignored(&event) { - self.inode_map.borrow_mut().remove(event.get_inode()); + if self.inode_map.borrow_mut().remove(event.get_inode()).is_some() { + self.metrics.inode_map_size.dec(); + } let _ = self.kernel_inode_map.borrow_mut().remove(event.get_inode()); if let Some(old_inode) = event.get_old_inode() { - self.inode_map.borrow_mut().remove(old_inode); + if self.inode_map.borrow_mut().remove(old_inode).is_some() { + self.metrics.inode_map_size.dec(); + } let _ = self.kernel_inode_map.borrow_mut().remove(old_inode); } self.metrics.events.ignored(); diff --git a/fact/src/metrics/host_scanner.rs b/fact/src/metrics/host_scanner.rs index 6aac0c0e..0b477cf9 100644 --- a/fact/src/metrics/host_scanner.rs +++ b/fact/src/metrics/host_scanner.rs @@ -1,6 +1,6 @@ use prometheus_client::{ encoding::{EncodeLabelSet, EncodeLabelValue}, - metrics::{counter::Counter, family::Family, histogram::Histogram}, + metrics::{counter::Counter, family::Family, gauge::Gauge, histogram::Histogram}, registry::Registry, }; @@ -33,6 +33,8 @@ pub struct HostScannerMetrics { pub events: EventCounter, pub scan: Family>, pub scan_duration: Histogram, + pub scan_partial_duration: Histogram, + pub inode_map_size: Gauge, } impl HostScannerMetrics { @@ -68,11 +70,18 @@ impl HostScannerMetrics { let scan_duration = Histogram::new([ 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, ]); + let scan_partial_duration = Histogram::new([ + 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, + ]); + + let inode_map_size = Gauge::default(); HostScannerMetrics { events, scan, scan_duration, + scan_partial_duration, + inode_map_size, } } @@ -89,6 +98,18 @@ impl HostScannerMetrics { "Histogram of scan durations from the host scanner component", self.scan_duration.clone(), ); + + reg.register( + "host_scanner_scan_partial_duration", + "Histogram of partial scan durations from the host scanner component", + self.scan_partial_duration.clone(), + ); + + reg.register( + "host_scanner_inode_map_size", + "Gauge tracking the number of elements in the inode map", + self.inode_map_size.clone(), + ); } pub fn scan_inc(&self, label: ScanLabels) { From 72d8b83e594621f199119482f441ed1432cfbea8 Mon Sep 17 00:00:00 2001 From: Mauro Ezequiel Moltrasio Date: Wed, 26 Aug 2026 17:28:35 +0200 Subject: [PATCH 2/3] cleanup(metrics): add a `remove_inode` helper --- fact/src/host_scanner.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/fact/src/host_scanner.rs b/fact/src/host_scanner.rs index 61d48171..5cf6430a 100644 --- a/fact/src/host_scanner.rs +++ b/fact/src/host_scanner.rs @@ -142,6 +142,15 @@ impl HostScanner { Ok((host_scanner, output)) } + fn remove_inode(&self, inode: &inode_key_t) -> Option { + let _ = self.kernel_inode_map.borrow_mut().remove(inode); + let res = self.inode_map.borrow_mut().remove(inode); + if res.is_some() { + self.metrics.inode_map_size.dec(); + } + res + } + fn reload_paths_config(&mut self) -> anyhow::Result<()> { let paths = self.paths.borrow(); let mut builder = GlobSetBuilder::new(); @@ -404,9 +413,8 @@ You can increase this limit with: fn handle_unlink_event(&self, event: &Event) { let inode = event.get_inode(); - if self.inode_map.borrow_mut().remove(inode).is_some() { + if self.remove_inode(inode).is_some() { self.metrics.scan_inc(ScanLabels::InodeRemoved); - self.metrics.inode_map_size.dec(); } self.metrics.scan_inc(ScanLabels::FileRemoved); @@ -667,15 +675,9 @@ You can increase this limit with: // whether the event is ignored now that we have the // full inode context. if self.event_is_ignored(&event) { - if self.inode_map.borrow_mut().remove(event.get_inode()).is_some() { - self.metrics.inode_map_size.dec(); - } - let _ = self.kernel_inode_map.borrow_mut().remove(event.get_inode()); + self.remove_inode(event.get_inode()); if let Some(old_inode) = event.get_old_inode() { - if self.inode_map.borrow_mut().remove(old_inode).is_some() { - self.metrics.inode_map_size.dec(); - } - let _ = self.kernel_inode_map.borrow_mut().remove(old_inode); + self.remove_inode(old_inode); } self.metrics.events.ignored(); continue; From 7e789299b0860af1209790afd3e08d1584c64352 Mon Sep 17 00:00:00 2001 From: Mauro Ezequiel Moltrasio Date: Thu, 27 Aug 2026 11:44:54 +0200 Subject: [PATCH 3/3] cleanup: use introspection endpoint for updating inode_map_size metric --- fact/src/endpoints.rs | 56 +++++++++++++++++++----- fact/src/host_scanner.rs | 92 ++++++++++++++++++++-------------------- fact/src/lib.rs | 4 +- 3 files changed, 95 insertions(+), 57 deletions(-) diff --git a/fact/src/endpoints.rs b/fact/src/endpoints.rs index 947076b6..34d95334 100644 --- a/fact/src/endpoints.rs +++ b/fact/src/endpoints.rs @@ -15,7 +15,11 @@ use tokio::{ task::JoinHandle, }; -use crate::{config::EndpointConfig, metrics::exporter::Exporter}; +use crate::{ + config::EndpointConfig, + host_scanner::{self, IntrospectionRequestType as HostScannerReq}, + metrics::exporter::Exporter, +}; #[derive(Clone)] pub struct Server { @@ -23,7 +27,7 @@ pub struct Server { config: watch::Receiver, running: watch::Receiver, - host_scanner_intro: mpsc::Sender>>, + host_scanner_intro: mpsc::Sender, } impl Server { @@ -31,7 +35,7 @@ impl Server { metrics: Exporter, config: watch::Receiver, running: watch::Receiver, - host_scanner_intro: mpsc::Sender>>, + host_scanner_intro: mpsc::Sender, ) -> Self { Server { metrics, @@ -124,11 +128,29 @@ impl Server { .unwrap()) } - fn handle_metrics(&self) -> Result>, anyhow::Error> { + async fn handle_metrics(&self) -> Result>, anyhow::Error> { if !self.metrics_is_active() { return Server::make_response(StatusCode::SERVICE_UNAVAILABLE, ""); } + // Trigger an update of the inode_map_size metric + let (tx, rx) = oneshot::channel(); + if let Err(e) = self + .host_scanner_intro + .send((HostScannerReq::InodeMapSize, tx)) + .await + { + return Server::make_response( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to request update inode_map_size metric: {e:?}"), + ); + } + if let Err(e) = rx.await { + return Server::make_response( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to update inode_map_size metric: {e:?}"), + ); + } self.metrics.encode().map(|buf| { let body = Full::new(Bytes::from(buf)); Response::builder() @@ -156,19 +178,33 @@ impl Server { } let (tx, rx) = oneshot::channel(); - if let Err(e) = self.host_scanner_intro.send(tx).await { + if let Err(e) = self + .host_scanner_intro + .send((HostScannerReq::InodeMap, tx)) + .await + { return Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()); } - match rx.await { - Ok(Ok(b)) => Response::builder() + let res = match rx.await { + Ok(res) => res, + Err(e) => { + return Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()); + } + }; + + use host_scanner::IntrospectionResponseType::*; + match res { + InodeMap(Ok(b)) => Response::builder() .header( hyper::header::CONTENT_TYPE, "application/json; charset=utf-8", ) .body(Full::new(Bytes::from(b))) .map_err(anyhow::Error::new), - Ok(Err(e)) => Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()), - Err(e) => Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()), + InodeMap(Err(e)) => { + Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) + } + InodeMapSize => unreachable!("received InodeMapSize response to InodeMap request"), } } } @@ -182,7 +218,7 @@ impl Service> for Server { let s = self.clone(); Box::pin(async move { match (req.method(), req.uri().path()) { - (&Method::GET, "/metrics") => s.handle_metrics(), + (&Method::GET, "/metrics") => s.handle_metrics().await, (&Method::GET, "/health_check") => s.handle_health_check(), (&Method::GET, "/inodes") => s.handle_inodes().await, _ => Server::make_response(StatusCode::NOT_FOUND, ""), diff --git a/fact/src/host_scanner.rs b/fact/src/host_scanner.rs index 5cf6430a..26ec609d 100644 --- a/fact/src/host_scanner.rs +++ b/fact/src/host_scanner.rs @@ -91,6 +91,23 @@ impl Serialize for InodeMap { } } +#[derive(Debug)] +pub enum IntrospectionRequestType { + InodeMap, + InodeMapSize, +} + +#[derive(Debug)] +pub enum IntrospectionResponseType { + InodeMap(serde_json::Result), + InodeMapSize, +} + +pub type IntrospectionRequest = ( + IntrospectionRequestType, + oneshot::Sender, +); + pub struct HostScanner { kernel_inode_map: RefCell>, inode_map: RefCell, @@ -100,7 +117,7 @@ pub struct HostScanner { rx: mpsc::Receiver, tx: mpsc::Sender, - introspection: mpsc::Receiver>>, + introspection: mpsc::Receiver, metrics: HostScannerMetrics, @@ -115,7 +132,7 @@ impl HostScanner { paths: watch::Receiver>, scan_interval: watch::Receiver, metrics: HostScannerMetrics, - introspection: mpsc::Receiver>>, + introspection: mpsc::Receiver, ) -> anyhow::Result<(Self, mpsc::Receiver)> { let kernel_inode_map = RefCell::new(bpf.take_inode_map()?); let inode_map = RefCell::new(InodeMap::new()); @@ -142,15 +159,6 @@ impl HostScanner { Ok((host_scanner, output)) } - fn remove_inode(&self, inode: &inode_key_t) -> Option { - let _ = self.kernel_inode_map.borrow_mut().remove(inode); - let res = self.inode_map.borrow_mut().remove(inode); - if res.is_some() { - self.metrics.inode_map_size.dec(); - } - res - } - fn reload_paths_config(&mut self) -> anyhow::Result<()> { let paths = self.paths.borrow(); let mut builder = GlobSetBuilder::new(); @@ -187,7 +195,6 @@ impl HostScanner { } else { let _ = self.kernel_inode_map.borrow_mut().remove(inode); self.metrics.scan_inc(ScanLabels::InodeRemoved); - self.metrics.inode_map_size.dec(); false } }); @@ -341,7 +348,6 @@ impl HostScanner { } None => { self.metrics.scan_inc(ScanLabels::FileUpdated); - self.metrics.inode_map_size.inc(); inode_map.insert(inode, path.clone()); } }; @@ -413,7 +419,7 @@ You can increase this limit with: fn handle_unlink_event(&self, event: &Event) { let inode = event.get_inode(); - if self.remove_inode(inode).is_some() { + if self.inode_map.borrow_mut().remove(inode).is_some() { self.metrics.scan_inc(ScanLabels::InodeRemoved); } @@ -428,22 +434,14 @@ You can increase this limit with: // inode we are landing on and put the associated host path in // the old inode. let mut inode_map = self.inode_map.borrow_mut(); - let path = match inode_map.remove(event.get_inode()) { - Some(p) => { - self.metrics.inode_map_size.dec(); - p - } - None => { - warn!("Old path was not found for inode tracked event"); - return; - } + let Some(path) = inode_map.remove(event.get_inode()) else { + warn!("Old path was not found for inode tracked event"); + return; }; let Some(old_inode) = event.get_old_inode() else { unreachable!("old inode not found for rename event"); }; - if inode_map.insert(*old_inode, path).is_none() { - self.metrics.inode_map_size.inc(); - } + inode_map.insert(*old_inode, path); } monitored_t::NOT_MONITORED if event.get_old_monitored() == Some(monitored_t::MONITORED_BY_INODE) => @@ -460,7 +458,6 @@ You can increase this limit with: } let _ = self.kernel_inode_map.borrow_mut().remove(inode); - self.metrics.inode_map_size.dec(); false }); } @@ -471,18 +468,11 @@ You can increase this limit with: monitored_t::MONITORED_BY_PARENT if !event.get_inode().empty() => { // The parent for the target is monitored, but the file itself // is not. Remove the entry for the old file from the map. - if self - .inode_map - .borrow_mut() - .remove( - event - .get_old_inode() - .expect("rename event did not have old inode"), - ) - .is_some() - { - self.metrics.inode_map_size.dec(); - } + self.inode_map.borrow_mut().remove( + event + .get_old_inode() + .expect("rename event did not have old inode"), + ); } monitored_t::MONITORED_BY_PARENT if event.get_old_monitored() == Some(monitored_t::MONITORED_BY_INODE) => @@ -528,7 +518,6 @@ You can increase this limit with: if let Err(e) = self.kernel_inode_map.borrow_mut().remove(inode) { warn!("Failed to remove inode kernel entry: {e:?}"); } - self.metrics.inode_map_size.dec(); false }); } @@ -675,9 +664,11 @@ You can increase this limit with: // whether the event is ignored now that we have the // full inode context. if self.event_is_ignored(&event) { - self.remove_inode(event.get_inode()); + self.inode_map.borrow_mut().remove(event.get_inode()); + let _ = self.kernel_inode_map.borrow_mut().remove(event.get_inode()); if let Some(old_inode) = event.get_old_inode() { - self.remove_inode(old_inode); + self.inode_map.borrow_mut().remove(old_inode); + let _ = self.kernel_inode_map.borrow_mut().remove(old_inode); } self.metrics.events.ignored(); continue; @@ -689,12 +680,23 @@ You can increase this limit with: } }, req = self.introspection.recv() => { - let Some(req) = req else { + let Some((req_type, ch)) = req else { continue; }; - let resp = serde_json::to_string(&*self.inode_map.borrow()); - if let Err(e) = req.send(resp) { + use IntrospectionRequestType::*; + let resp = match req_type { + InodeMap => { + let resp = serde_json::to_string(&*self.inode_map.borrow()); + IntrospectionResponseType::InodeMap(resp) + } + InodeMapSize => { + let len = self.inode_map.borrow().len(); + self.metrics.inode_map_size.set(len as i64); + IntrospectionResponseType::InodeMapSize + } + }; + if let Err(e) = ch.send(resp) { warn!("Failed to reply introspection query: {e:?}"); } } diff --git a/fact/src/lib.rs b/fact/src/lib.rs index 0969615b..2c0d335a 100644 --- a/fact/src/lib.rs +++ b/fact/src/lib.rs @@ -9,7 +9,7 @@ use metrics::exporter::Exporter; use rate_limiter::RateLimiter; use tokio::{ signal::unix::{SignalKind, signal}, - sync::{mpsc, oneshot, watch}, + sync::{mpsc, watch}, task::JoinSet, time::timeout, }; @@ -102,7 +102,7 @@ struct SetupArgs<'a> { // BPF mode bpf_config: BpfConfig, - host_scanner_intro: mpsc::Receiver>>, + host_scanner_intro: mpsc::Receiver, } pub async fn run(config: FactConfig) -> anyhow::Result<()> {