max / makenotwork

pom: treat a stale health snapshot as unreachable in the offline net The monitoring-offline meta-alert decided 'are all targets down?' from get_latest_health with no freshness check. A probe task that dies (stops inserting rows) leaves its last row reading operational forever, so the one net designed to catch 'PoM itself is blind' read permanently-green and never fired (fuzz-2026-07-06 #3). A snapshot older than three meta-intervals (or with an unparseable timestamp) now counts as not-live, so a dead probe can't mask an outage. Logic extracted to a pure snapshot_is_live for unit coverage. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: Max Johnson <me@maxj.phd> · 2026-07-07 16:07 UTC

Commit: 7335fec41ba7015aa3e2c321862f9a0c16423259

Parent: 0f24243

1 file changed, +62 insertions, -2 deletions

M pom/src/cli/tasks/meta_alert.rs +62 -2

			@@ -36,17 +36,26 @@ pub(crate) fn spawn_meta_alert_task(
36	36		interval.tick().await; // consume immediate first tick
37	37		let mut was_all_down = false;
38	38
	39	+	// A snapshot older than this is treated as unreachable, not its stale
	40	+	// status. Without it a probe task that has died (stopped inserting rows)
	41	+	// leaves its last row reading `operational` forever, so this "is PoM
	42	+	// blind?" net reads permanently-green and never fires — the one net meant
	43	+	// to catch a dead probe is itself blind to it (fuzz-2026-07-06 #3). Three
	44	+	// meta-intervals (= 6 base check intervals) of silence means the probe is
	45	+	// not running, not that the target is healthy.
	46	+	let staleness_threshold = chrono::Duration::seconds((meta_interval_secs * 3) as i64);
	47	+
39	48		loop {
40	49		tokio::select! {
41	50		_ = cancel.cancelled() => break,
42	51		_ = interval.tick() => {}
43	52		}
44	53
	54	+	let now = chrono::Utc::now();
45	55		let mut all_down = true;
46	56		for name in &health_target_names {
47	57		if let Ok(Some(snap)) = db::get_latest_health(&pool, name).await
48		-	&& (snap.status == HealthStatus::Operational
49		-	\|\| snap.status == HealthStatus::Degraded)
	58	+	&& snapshot_is_live(snap.status, &snap.checked_at, now, staleness_threshold)
50	59		{
51	60		all_down = false;
52	61		break;
			@@ -62,3 +71,54 @@ pub(crate) fn spawn_meta_alert_task(
62	71		}
63	72		}))
64	73		}
	74	+
	75	+	/// Whether a health snapshot counts as "target is live" for the monitoring-offline
	76	+	/// net: it must be a healthy status AND recent. A stale or unparseable timestamp
	77	+	/// is treated as not-live so a dead probe's frozen `operational` row cannot mask
	78	+	/// an outage (fuzz-2026-07-06 #3).
	79	+	fn snapshot_is_live(
	80	+	status: HealthStatus,
	81	+	checked_at: &str,
	82	+	now: chrono::DateTime<chrono::Utc>,
	83	+	staleness_threshold: chrono::Duration,
	84	+	) -> bool {
	85	+	let healthy = matches!(status, HealthStatus::Operational \| HealthStatus::Degraded);
	86	+	if !healthy {
	87	+	return false;
	88	+	}
	89	+	chrono::DateTime::parse_from_rfc3339(checked_at)
	90	+	.is_ok_and(\|dt\| now.signed_duration_since(dt.with_timezone(&chrono::Utc)) < staleness_threshold)
	91	+	}
	92	+
	93	+	#[cfg(test)]
	94	+	mod tests {
	95	+	use super::*;
	96	+
	97	+	#[test]
	98	+	fn fresh_operational_is_live() {
	99	+	let now = chrono::Utc::now();
	100	+	let recent = (now - chrono::Duration::seconds(10)).to_rfc3339();
	101	+	assert!(snapshot_is_live(HealthStatus::Operational, &recent, now, chrono::Duration::seconds(60)));
	102	+	}
	103	+
	104	+	#[test]
	105	+	fn stale_operational_is_not_live() {
	106	+	// A dead probe: last row says operational but is far older than the window.
	107	+	let now = chrono::Utc::now();
	108	+	let old = (now - chrono::Duration::seconds(600)).to_rfc3339();
	109	+	assert!(!snapshot_is_live(HealthStatus::Operational, &old, now, chrono::Duration::seconds(60)));
	110	+	}
	111	+
	112	+	#[test]
	113	+	fn unparseable_timestamp_is_not_live() {
	114	+	let now = chrono::Utc::now();
	115	+	assert!(!snapshot_is_live(HealthStatus::Operational, "not-a-date", now, chrono::Duration::seconds(60)));
	116	+	}
	117	+
	118	+	#[test]
	119	+	fn error_status_is_not_live_even_if_fresh() {
	120	+	let now = chrono::Utc::now();
	121	+	let recent = now.to_rfc3339();
	122	+	assert!(!snapshot_is_live(HealthStatus::Error, &recent, now, chrono::Duration::seconds(60)));
	123	+	}
	124	+	}