diff --git a/fact/src/endpoints.rs b/fact/src/endpoints.rs index 947076b6..34d95334 100644 --- a/fact/src/endpoints.rs +++ b/fact/src/endpoints.rs @@ -15,7 +15,11 @@ use tokio::{ task::JoinHandle, }; -use crate::{config::EndpointConfig, metrics::exporter::Exporter}; +use crate::{ + config::EndpointConfig, + host_scanner::{self, IntrospectionRequestType as HostScannerReq}, + metrics::exporter::Exporter, +}; #[derive(Clone)] pub struct Server { @@ -23,7 +27,7 @@ pub struct Server { config: watch::Receiver, running: watch::Receiver, - host_scanner_intro: mpsc::Sender>>, + host_scanner_intro: mpsc::Sender, } impl Server { @@ -31,7 +35,7 @@ impl Server { metrics: Exporter, config: watch::Receiver, running: watch::Receiver, - host_scanner_intro: mpsc::Sender>>, + host_scanner_intro: mpsc::Sender, ) -> Self { Server { metrics, @@ -124,11 +128,29 @@ impl Server { .unwrap()) } - fn handle_metrics(&self) -> Result>, anyhow::Error> { + async fn handle_metrics(&self) -> Result>, anyhow::Error> { if !self.metrics_is_active() { return Server::make_response(StatusCode::SERVICE_UNAVAILABLE, ""); } + // Trigger an update of the inode_map_size metric + let (tx, rx) = oneshot::channel(); + if let Err(e) = self + .host_scanner_intro + .send((HostScannerReq::InodeMapSize, tx)) + .await + { + return Server::make_response( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to request update inode_map_size metric: {e:?}"), + ); + } + if let Err(e) = rx.await { + return Server::make_response( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to update inode_map_size metric: {e:?}"), + ); + } self.metrics.encode().map(|buf| { let body = Full::new(Bytes::from(buf)); Response::builder() @@ -156,19 +178,33 @@ impl Server { } let (tx, rx) = oneshot::channel(); - if let Err(e) = self.host_scanner_intro.send(tx).await { + if let Err(e) = self + .host_scanner_intro + .send((HostScannerReq::InodeMap, tx)) + .await + { return Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()); } - match rx.await { - Ok(Ok(b)) => Response::builder() + let res = match rx.await { + Ok(res) => res, + Err(e) => { + return Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()); + } + }; + + use host_scanner::IntrospectionResponseType::*; + match res { + InodeMap(Ok(b)) => Response::builder() .header( hyper::header::CONTENT_TYPE, "application/json; charset=utf-8", ) .body(Full::new(Bytes::from(b))) .map_err(anyhow::Error::new), - Ok(Err(e)) => Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()), - Err(e) => Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()), + InodeMap(Err(e)) => { + Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) + } + InodeMapSize => unreachable!("received InodeMapSize response to InodeMap request"), } } } @@ -182,7 +218,7 @@ impl Service> for Server { let s = self.clone(); Box::pin(async move { match (req.method(), req.uri().path()) { - (&Method::GET, "/metrics") => s.handle_metrics(), + (&Method::GET, "/metrics") => s.handle_metrics().await, (&Method::GET, "/health_check") => s.handle_health_check(), (&Method::GET, "/inodes") => s.handle_inodes().await, _ => Server::make_response(StatusCode::NOT_FOUND, ""), diff --git a/fact/src/host_scanner.rs b/fact/src/host_scanner.rs index 2ef1a66c..26ec609d 100644 --- a/fact/src/host_scanner.rs +++ b/fact/src/host_scanner.rs @@ -91,6 +91,23 @@ impl Serialize for InodeMap { } } +#[derive(Debug)] +pub enum IntrospectionRequestType { + InodeMap, + InodeMapSize, +} + +#[derive(Debug)] +pub enum IntrospectionResponseType { + InodeMap(serde_json::Result), + InodeMapSize, +} + +pub type IntrospectionRequest = ( + IntrospectionRequestType, + oneshot::Sender, +); + pub struct HostScanner { kernel_inode_map: RefCell>, inode_map: RefCell, @@ -100,7 +117,7 @@ pub struct HostScanner { rx: mpsc::Receiver, tx: mpsc::Sender, - introspection: mpsc::Receiver>>, + introspection: mpsc::Receiver, metrics: HostScannerMetrics, @@ -115,7 +132,7 @@ impl HostScanner { paths: watch::Receiver>, scan_interval: watch::Receiver, metrics: HostScannerMetrics, - introspection: mpsc::Receiver>>, + introspection: mpsc::Receiver, ) -> anyhow::Result<(Self, mpsc::Receiver)> { let kernel_inode_map = RefCell::new(bpf.take_inode_map()?); let inode_map = RefCell::new(InodeMap::new()); @@ -274,6 +291,7 @@ impl HostScanner { /// base path (the path up to the first glob special character) that /// matches the supplied path. fn scan_partial(&self, path: &Path) -> anyhow::Result<()> { + let start = Instant::now(); let scan_prefix_patterns = self.paths_patterns .iter() @@ -296,6 +314,10 @@ impl HostScanner { for pattern in scan_set.iter().map(|index| &self.paths_patterns[*index]) { self.scan_inner(pattern)?; } + + self.metrics + .scan_partial_duration + .observe(start.elapsed().as_secs_f64()); Ok(()) } @@ -658,12 +680,23 @@ You can increase this limit with: } }, req = self.introspection.recv() => { - let Some(req) = req else { + let Some((req_type, ch)) = req else { continue; }; - let resp = serde_json::to_string(&*self.inode_map.borrow()); - if let Err(e) = req.send(resp) { + use IntrospectionRequestType::*; + let resp = match req_type { + InodeMap => { + let resp = serde_json::to_string(&*self.inode_map.borrow()); + IntrospectionResponseType::InodeMap(resp) + } + InodeMapSize => { + let len = self.inode_map.borrow().len(); + self.metrics.inode_map_size.set(len as i64); + IntrospectionResponseType::InodeMapSize + } + }; + if let Err(e) = ch.send(resp) { warn!("Failed to reply introspection query: {e:?}"); } } diff --git a/fact/src/lib.rs b/fact/src/lib.rs index 0969615b..2c0d335a 100644 --- a/fact/src/lib.rs +++ b/fact/src/lib.rs @@ -9,7 +9,7 @@ use metrics::exporter::Exporter; use rate_limiter::RateLimiter; use tokio::{ signal::unix::{SignalKind, signal}, - sync::{mpsc, oneshot, watch}, + sync::{mpsc, watch}, task::JoinSet, time::timeout, }; @@ -102,7 +102,7 @@ struct SetupArgs<'a> { // BPF mode bpf_config: BpfConfig, - host_scanner_intro: mpsc::Receiver>>, + host_scanner_intro: mpsc::Receiver, } pub async fn run(config: FactConfig) -> anyhow::Result<()> { diff --git a/fact/src/metrics/host_scanner.rs b/fact/src/metrics/host_scanner.rs index 6aac0c0e..0b477cf9 100644 --- a/fact/src/metrics/host_scanner.rs +++ b/fact/src/metrics/host_scanner.rs @@ -1,6 +1,6 @@ use prometheus_client::{ encoding::{EncodeLabelSet, EncodeLabelValue}, - metrics::{counter::Counter, family::Family, histogram::Histogram}, + metrics::{counter::Counter, family::Family, gauge::Gauge, histogram::Histogram}, registry::Registry, }; @@ -33,6 +33,8 @@ pub struct HostScannerMetrics { pub events: EventCounter, pub scan: Family>, pub scan_duration: Histogram, + pub scan_partial_duration: Histogram, + pub inode_map_size: Gauge, } impl HostScannerMetrics { @@ -68,11 +70,18 @@ impl HostScannerMetrics { let scan_duration = Histogram::new([ 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, ]); + let scan_partial_duration = Histogram::new([ + 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, + ]); + + let inode_map_size = Gauge::default(); HostScannerMetrics { events, scan, scan_duration, + scan_partial_duration, + inode_map_size, } } @@ -89,6 +98,18 @@ impl HostScannerMetrics { "Histogram of scan durations from the host scanner component", self.scan_duration.clone(), ); + + reg.register( + "host_scanner_scan_partial_duration", + "Histogram of partial scan durations from the host scanner component", + self.scan_partial_duration.clone(), + ); + + reg.register( + "host_scanner_inode_map_size", + "Gauge tracking the number of elements in the inode map", + self.inode_map_size.clone(), + ); } pub fn scan_inc(&self, label: ScanLabels) {