//! Background task that drains the pending-alert retry queue. use tokio::task::JoinHandle; use tracing::info; use pom::alerts::Alerter; use pom::config::Config; use pom::db; use pom::types::HealthStatus; use super::CheckInterval; pub(crate) fn spawn_meta_alert_task( config: &Config, pool: &sqlx::SqlitePool, default_interval: u64, cancel: &tokio_util::sync::CancellationToken, alerter: Option<&Alerter>, ) -> Option> { let health_target_names: Vec = config .target_names() .into_iter() .filter(|n| config.get_target(n).is_some_and(|t| t.health.is_some())) .collect(); if health_target_names.len() < 2 { return None; } let alerter = alerter.cloned()?; let pool = pool.clone(); let cancel = cancel.clone(); let meta_interval_secs = default_interval * 2; info!( "Meta-alert: monitoring-offline check every {meta_interval_secs}s ({} targets)", health_target_names.len() ); Some(tokio::spawn(async move { let mut ticks = CheckInterval::new(meta_interval_secs, cancel); // Seed from the ledger so a restart while monitoring is already-offline // doesn't re-fire the alert: was-down iff the latest monitoring alert is // the offline (not the recovery) one. let mut was_all_down = db::get_latest_alert_matching(&pool, "monitoring:self", "monitoring_%") .await .ok() .flatten() .is_some_and(|a| a.alert_type == "monitoring_offline"); // A snapshot older than this is treated as unreachable, not its stale // status. Without it a probe task that has died (stopped inserting rows) // leaves its last row reading `operational` forever, so this "is PoM // blind?" net reads permanently-green and never fires, the one net meant // to catch a dead probe is itself blind to it (fuzz-2026-07-06 #3). Three // meta-intervals (= 6 base check intervals) of silence means the probe is // not running, not that the target is healthy. let staleness_threshold = chrono::Duration::seconds((meta_interval_secs * 3) as i64); while ticks.next().await { let now = chrono::Utc::now(); let mut all_down = true; for name in &health_target_names { if let Ok(Some(snap)) = db::get_latest_health(&pool, name).await && snapshot_is_live(snap.status, &snap.checked_at, now, staleness_threshold) { all_down = false; break; } } if all_down && !was_all_down { alerter .send_monitoring_offline_alert(health_target_names.len()) .await; } else if !all_down && was_all_down { alerter.send_monitoring_recovery().await; } was_all_down = all_down; } })) } /// Whether a health snapshot counts as "target is live" for the monitoring-offline /// net: it must be a healthy status AND recent. A stale or unparseable timestamp /// is treated as not-live so a dead probe's frozen `operational` row cannot mask /// an outage (fuzz-2026-07-06 #3). fn snapshot_is_live( status: HealthStatus, checked_at: &str, now: chrono::DateTime, staleness_threshold: chrono::Duration, ) -> bool { let healthy = matches!(status, HealthStatus::Operational | HealthStatus::Degraded); if !healthy { return false; } chrono::DateTime::parse_from_rfc3339(checked_at).is_ok_and(|dt| { now.signed_duration_since(dt.with_timezone(&chrono::Utc)) < staleness_threshold }) } #[cfg(test)] mod tests { use super::*; #[test] fn fresh_operational_is_live() { let now = chrono::Utc::now(); let recent = (now - chrono::Duration::seconds(10)).to_rfc3339(); assert!(snapshot_is_live( HealthStatus::Operational, &recent, now, chrono::Duration::seconds(60) )); } #[test] fn stale_operational_is_not_live() { // A dead probe: last row says operational but is far older than the window. let now = chrono::Utc::now(); let old = (now - chrono::Duration::seconds(600)).to_rfc3339(); assert!(!snapshot_is_live( HealthStatus::Operational, &old, now, chrono::Duration::seconds(60) )); } #[test] fn unparseable_timestamp_is_not_live() { let now = chrono::Utc::now(); assert!(!snapshot_is_live( HealthStatus::Operational, "not-a-date", now, chrono::Duration::seconds(60) )); } #[test] fn error_status_is_not_live_even_if_fresh() { let now = chrono::Utc::now(); let recent = now.to_rfc3339(); assert!(!snapshot_is_live( HealthStatus::Error, &recent, now, chrono::Duration::seconds(60) )); } }