Skip to main content

max / makenotwork

5.0 KB · 156 lines History Blame Raw
1 //! Background task that drains the pending-alert retry queue.
2
3 use tokio::task::JoinHandle;
4 use tracing::info;
5
6 use pom::alerts::Alerter;
7 use pom::config::Config;
8 use pom::db;
9 use pom::types::HealthStatus;
10
11 use super::CheckInterval;
12
13 pub(crate) fn spawn_meta_alert_task(
14 config: &Config,
15 pool: &sqlx::SqlitePool,
16 default_interval: u64,
17 cancel: &tokio_util::sync::CancellationToken,
18 alerter: Option<&Alerter>,
19 ) -> Option<JoinHandle<()>> {
20 let health_target_names: Vec<String> = config
21 .target_names()
22 .into_iter()
23 .filter(|n| config.get_target(n).is_some_and(|t| t.health.is_some()))
24 .collect();
25
26 if health_target_names.len() < 2 {
27 return None;
28 }
29 let alerter = alerter.cloned()?;
30
31 let pool = pool.clone();
32 let cancel = cancel.clone();
33 let meta_interval_secs = default_interval * 2;
34
35 info!(
36 "Meta-alert: monitoring-offline check every {meta_interval_secs}s ({} targets)",
37 health_target_names.len()
38 );
39
40 Some(tokio::spawn(async move {
41 let mut ticks = CheckInterval::new(meta_interval_secs, cancel);
42 // Seed from the ledger so a restart while monitoring is already-offline
43 // doesn't re-fire the alert: was-down iff the latest monitoring alert is
44 // the offline (not the recovery) one.
45 let mut was_all_down =
46 db::get_latest_alert_matching(&pool, "monitoring:self", "monitoring_%")
47 .await
48 .ok()
49 .flatten()
50 .is_some_and(|a| a.alert_type == "monitoring_offline");
51
52 // A snapshot older than this is treated as unreachable, not its stale
53 // status. Without it a probe task that has died (stopped inserting rows)
54 // leaves its last row reading `operational` forever, so this "is PoM
55 // blind?" net reads permanently-green and never fires, the one net meant
56 // to catch a dead probe is itself blind to it (fuzz-2026-07-06 #3). Three
57 // meta-intervals (= 6 base check intervals) of silence means the probe is
58 // not running, not that the target is healthy.
59 let staleness_threshold = chrono::Duration::seconds((meta_interval_secs * 3) as i64);
60
61 while ticks.next().await {
62 let now = chrono::Utc::now();
63 let mut all_down = true;
64 for name in &health_target_names {
65 if let Ok(Some(snap)) = db::get_latest_health(&pool, name).await
66 && snapshot_is_live(snap.status, &snap.checked_at, now, staleness_threshold)
67 {
68 all_down = false;
69 break;
70 }
71 }
72
73 if all_down && !was_all_down {
74 alerter
75 .send_monitoring_offline_alert(health_target_names.len())
76 .await;
77 } else if !all_down && was_all_down {
78 alerter.send_monitoring_recovery().await;
79 }
80 was_all_down = all_down;
81 }
82 }))
83 }
84
85 /// Whether a health snapshot counts as "target is live" for the monitoring-offline
86 /// net: it must be a healthy status AND recent. A stale or unparseable timestamp
87 /// is treated as not-live so a dead probe's frozen `operational` row cannot mask
88 /// an outage (fuzz-2026-07-06 #3).
89 fn snapshot_is_live(
90 status: HealthStatus,
91 checked_at: &str,
92 now: chrono::DateTime<chrono::Utc>,
93 staleness_threshold: chrono::Duration,
94 ) -> bool {
95 let healthy = matches!(status, HealthStatus::Operational | HealthStatus::Degraded);
96 if !healthy {
97 return false;
98 }
99 chrono::DateTime::parse_from_rfc3339(checked_at).is_ok_and(|dt| {
100 now.signed_duration_since(dt.with_timezone(&chrono::Utc)) < staleness_threshold
101 })
102 }
103
104 #[cfg(test)]
105 mod tests {
106 use super::*;
107
108 #[test]
109 fn fresh_operational_is_live() {
110 let now = chrono::Utc::now();
111 let recent = (now - chrono::Duration::seconds(10)).to_rfc3339();
112 assert!(snapshot_is_live(
113 HealthStatus::Operational,
114 &recent,
115 now,
116 chrono::Duration::seconds(60)
117 ));
118 }
119
120 #[test]
121 fn stale_operational_is_not_live() {
122 // A dead probe: last row says operational but is far older than the window.
123 let now = chrono::Utc::now();
124 let old = (now - chrono::Duration::seconds(600)).to_rfc3339();
125 assert!(!snapshot_is_live(
126 HealthStatus::Operational,
127 &old,
128 now,
129 chrono::Duration::seconds(60)
130 ));
131 }
132
133 #[test]
134 fn unparseable_timestamp_is_not_live() {
135 let now = chrono::Utc::now();
136 assert!(!snapshot_is_live(
137 HealthStatus::Operational,
138 "not-a-date",
139 now,
140 chrono::Duration::seconds(60)
141 ));
142 }
143
144 #[test]
145 fn error_status_is_not_live_even_if_fresh() {
146 let now = chrono::Utc::now();
147 let recent = now.to_rfc3339();
148 assert!(!snapshot_is_live(
149 HealthStatus::Error,
150 &recent,
151 now,
152 chrono::Duration::seconds(60)
153 ));
154 }
155 }
156