Skip to main content

max / makenotwork

Split pom alerts.rs god-struct into per-domain modules alerts.rs was 1343 lines: one Alerter struct whose impl held both the shared alert plumbing and ~25 near-identical per-domain send_*_alert/send_*_recovery methods. Convert to an alerts/ directory. mod.rs keeps Alerter/AlertMeta, the priority helpers, all shared plumbing (cooldown, send_email, record, fire_failure/recovery, dispatch, retry, wam_ticket), and the tests. The 24 domain methods move into 12 per-domain files (health, tls, peer, route, dns, whois, cors, latency, test_duration, backup, scan, offline) as `impl Alerter` blocks. The domain modules reach the private plumbing and fields as descendants of `alerts`, so no visibility changes are needed and every send_* stays a pub method on Alerter — all callers are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-13 15:24 UTC
Commit: 425e81701af4fb8ba1c024d15137718fcdd33e58
Parent: 1ea5741
14 files changed, +1196 insertions, -500 deletions
D pom/src/alerts.rs -500
@@ -1,1343 +0,0 @@
1 - //! Email alerting via Postmark API.
2 - //!
3 - //! Sends alerts on health status transitions and peer disappearance/recovery.
4 - //! If no `postmark_token` is configured, alerts are logged to stdout instead.
5 -
6 - use sqlx::SqlitePool;
7 - use tracing::{info, instrument, warn};
8 -
9 - use crate::config::AlertConfig;
10 - use crate::db;
11 - use crate::types::AlertCategory;
12 -
13 - /// WAM ticket priority for a transition into a non-operational health status.
14 - fn health_status_priority(to_status: &str) -> &'static str {
15 - match to_status {
16 - "error" | "unreachable" => "critical",
17 - "degraded" => "high",
18 - _ => "medium",
19 - }
20 - }
21 -
22 - /// WAM ticket priority for a TLS certificate that expires in `days`.
23 - fn tls_expiry_priority(days: i64) -> &'static str {
24 - if days <= 3 { "critical" } else if days <= 7 { "high" } else { "medium" }
25 - }
26 -
27 - /// WAM ticket priority for a domain registration that expires in `days`.
28 - fn whois_expiry_priority(days: i64) -> &'static str {
29 - if days <= 7 { "critical" } else if days <= 14 { "high" } else { "medium" }
30 - }
31 -
32 - /// WAM ticket priority for a stale/missing/error backup status.
33 - fn backup_status_priority(status: &str) -> &'static str {
34 - if status == "missing" { "critical" } else { "high" }
35 - }
36 -
37 - /// Human-readable detail text for a backup status alert.
38 - fn backup_status_detail(status: &str, age_hours: Option<i64>) -> String {
39 - match (status, age_hours) {
40 - ("stale", Some(hours)) => format!("last backup is {hours}h old"),
41 - ("missing", _) => "no backup files found".to_string(),
42 - ("error", _) => "backup check failed".to_string(),
43 - _ => format!("status: {status}"),
44 - }
45 - }
46 -
47 - #[derive(Clone)]
48 - pub struct Alerter {
49 - config: AlertConfig,
50 - client: reqwest::Client,
51 - pool: SqlitePool,
52 - instance_name: String,
53 - wam_url: Option<String>,
54 - }
55 -
56 - /// The record-context an alert carries: what to write to the `alerts` ledger on
57 - /// successful delivery, and what to persist for retry on failure.
58 - struct AlertMeta<'a> {
59 - key: &'a str,
60 - category: AlertCategory,
61 - from: Option<&'a str>,
62 - to: Option<&'a str>,
63 - error: Option<&'a str>,
64 - }
65 -
66 - /// `true` if `sent_at` (rfc3339) is within `cooldown_secs` of now. An unparseable
67 - /// timestamp is treated as expired (not in cooldown) so a bad row can never wedge
68 - /// alerting shut.
69 - fn within_cooldown_secs(sent_at: &str, cooldown_secs: u64) -> bool {
70 - match chrono::DateTime::parse_from_rfc3339(sent_at) {
71 - Ok(dt) => {
72 - chrono::Utc::now().signed_duration_since(dt).num_seconds() < cooldown_secs as i64
73 - }
74 - Err(_) => false,
75 - }
76 - }
77 -
78 - impl Alerter {
79 - pub fn new(config: AlertConfig, pool: SqlitePool, instance_name: String) -> Self {
80 - let client = reqwest::Client::builder()
81 - .timeout(std::time::Duration::from_secs(10))
82 - .build()
83 - .unwrap_or_default();
84 - let wam_url = config.wam_url.clone();
85 - if wam_url.is_none() {
86 - warn!(
87 - "alerts: wam_url not configured — failure alerts will be delivered by email \
88 - instead of WAM tickets (set alerts.wam_url to route them to WAM)"
89 - );
90 - }
91 - Self { config, client, pool, instance_name, wam_url }
92 - }
93 -
94 - #[instrument(skip_all)]
95 - pub async fn send_health_alert(
96 - &self,
97 - target: &str,
98 - label: &str,
99 - from_status: &str,
100 - to_status: &str,
101 - error: Option<&str>,
102 - ) {
103 - let alert_key = format!("health:{target}");
104 - let subject = format!("[PoM] {target}: {from_status} -> {to_status}");
105 - let mut body = format!(
106 - "Target: {label} ({target})\n\
107 - Status: {from_status} -> {to_status}\n\
108 - Instance: {}\n\
109 - Time: {}\n",
110 - self.instance_name,
111 - chrono::Utc::now().to_rfc3339(),
112 - );
113 - if let Some(err) = error {
114 - body.push_str(&format!("Error: {err}\n"));
115 - }
116 - body.push_str("\n- PoM");
117 -
118 - let priority = health_status_priority(to_status);
119 - self.fire_failure(&subject, &body, priority, "pom-health", Some(target),
120 - AlertMeta { key: &alert_key, category: AlertCategory::Health, from: Some(from_status), to: Some(to_status), error }).await;
121 - }
122 -
123 - #[instrument(skip_all)]
124 - pub async fn send_health_recovery(
125 - &self,
126 - target: &str,
127 - label: &str,
128 - from_status: &str,
129 - ) {
130 - let alert_key = format!("health:{target}");
131 - let subject = format!("[PoM] {target}: recovered");
132 - let body = format!(
133 - "Target: {label} ({target})\n\
134 - Status: {from_status} -> operational\n\
135 - Instance: {}\n\
136 - Time: {}\n\n\
137 - - PoM",
138 - self.instance_name,
139 - chrono::Utc::now().to_rfc3339(),
140 - );
141 -
142 - self.fire_recovery(&subject, &body,
143 - AlertMeta { key: &alert_key, category: AlertCategory::Recovery, from: Some(from_status), to: Some("operational"), error: None }).await;
144 - }
145 -
146 - #[instrument(skip_all)]
147 - pub async fn send_tls_expiry_alert(
148 - &self,
149 - target: &str,
150 - host: &str,
151 - days_remaining: i64,
152 - not_after: &str,
153 - ) {
154 - let alert_key = format!("tls:{target}");
155 - let subject = format!("[PoM] {target}: TLS cert expires in {days_remaining} days");
156 - let body = format!(
157 - "Target: {target}\n\
158 - Host: {host}\n\
159 - Days remaining: {days_remaining}\n\
160 - Expires: {not_after}\n\
161 - Instance: {}\n\
162 - Time: {}\n\n\
163 - - PoM",
164 - self.instance_name,
165 - chrono::Utc::now().to_rfc3339(),
166 - );
167 -
168 - let priority = tls_expiry_priority(days_remaining);
169 - self.fire_failure(&subject, &body, priority, "pom-tls", Some(&format!("{target}:{host}")),
170 - AlertMeta { key: &alert_key, category: AlertCategory::TlsExpiry, from: None, to: None, error: None }).await;
171 - }
172 -
173 - #[instrument(skip_all)]
174 - pub async fn send_tls_error_alert(
175 - &self,
176 - target: &str,
177 - host: &str,
178 - error: &str,
179 - ) {
180 - let alert_key = format!("tls:{target}");
181 - let subject = format!("[PoM] {target}: TLS check failed");
182 - let body = format!(
183 - "Target: {target}\n\
184 - Host: {host}\n\
185 - Error: {error}\n\
186 - Instance: {}\n\
187 - Time: {}\n\n\
188 - - PoM",
189 - self.instance_name,
190 - chrono::Utc::now().to_rfc3339(),
191 - );
192 -
193 - self.fire_failure(&subject, &body, "high", "pom-tls", Some(&format!("{target}:{host}")),
194 - AlertMeta { key: &alert_key, category: AlertCategory::TlsError, from: None, to: None, error: Some(error) }).await;
195 - }
196 -
197 - #[instrument(skip_all)]
198 - pub async fn send_tls_recovery(
199 - &self,
200 - target: &str,
201 - label: &str,
202 - days_remaining: i64,
203 - ) {
204 - let alert_key = format!("tls:{target}");
205 - let subject = format!("[PoM] {target}: TLS cert renewed");
206 - let body = format!(
207 - "Target: {label} ({target})\n\
208 - Days remaining: {days_remaining}\n\
209 - Instance: {}\n\
210 - Time: {}\n\n\
211 - - PoM",
212 - self.instance_name,
213 - chrono::Utc::now().to_rfc3339(),
214 - );
215 -
216 - self.fire_recovery(&subject, &body,
217 - AlertMeta { key: &alert_key, category: AlertCategory::TlsRecovery, from: None, to: None, error: None }).await;
218 - }
219 -
220 - #[instrument(skip_all)]
221 - pub async fn send_peer_missing(
222 - &self,
223 - peer_name: &str,
224 - address: &str,
225 - consecutive_failures: u32,
226 - ) {
227 - let alert_key = format!("peer:{peer_name}");
228 - let subject = format!("[PoM] peer {peer_name}: missing");
229 - let body = format!(
230 - "Peer: {peer_name}\n\
231 - Address: {address}\n\
232 - Consecutive failures: {consecutive_failures}\n\
233 - Instance: {}\n\
234 - Time: {}\n\n\
235 - - PoM",
236 - self.instance_name,
237 - chrono::Utc::now().to_rfc3339(),
238 - );
239 -
240 - self.fire_failure(&subject, &body, "high", "pom-peer", Some(peer_name),
241 - AlertMeta { key: &alert_key, category: AlertCategory::PeerMissing, from: None, to: None, error: None }).await;
242 - }
243 -
244 - #[instrument(skip_all)]
245 - pub async fn send_peer_recovery(
246 - &self,
247 - peer_name: &str,
248 - address: &str,
249 - ) {
250 - let subject = format!("[PoM] peer {peer_name}: recovered");
251 - let body = format!(
252 - "Peer: {peer_name}\n\
253 - Address: {address}\n\
254 - Instance: {}\n\
255 - Time: {}\n\n\
256 - - PoM",
257 - self.instance_name,
258 - chrono::Utc::now().to_rfc3339(),
259 - );
260 -
261 - let alert_key = format!("peer:{peer_name}");
262 - self.fire_recovery(&subject, &body,
263 - AlertMeta { key: &alert_key, category: AlertCategory::PeerRecovery, from: None, to: None, error: None }).await;
264 - }
265 -
266 - #[instrument(skip_all)]
267 - pub async fn send_route_failure_alert(
268 - &self,
269 - target: &str,
270 - label: &str,
271 - failed_paths: &[String],
272 - ) {
273 - let alert_key = format!("route:{target}");
274 - let n = failed_paths.len();
275 - let subject = format!("[PoM] {label}: {n} route(s) failing");
276 - let body = format!(
277 - "Target: {label} ({target})\n\
278 - Failed routes:\n{}\n\
279 - Instance: {}\n\
280 - Time: {}\n\n\
281 - - PoM",
282 - failed_paths.iter().map(|p| format!(" - {p}")).collect::<Vec<_>>().join("\n"),
283 - self.instance_name,
284 - chrono::Utc::now().to_rfc3339(),
285 - );
286 -
287 - self.fire_failure(&subject, &body, "high", "pom-routes", Some(target),
288 - AlertMeta { key: &alert_key, category: AlertCategory::RouteFailure, from: None, to: None, error: None }).await;
289 - }
290 -
291 - #[instrument(skip_all)]
292 - pub async fn send_route_recovery_alert(
293 - &self,
294 - target: &str,
295 - label: &str,
296 - recovered_paths: &[String],
297 - ) {
298 - let alert_key = format!("route:{target}");
299 - let subject = format!("[PoM] {label}: routes recovered");
300 - let body = format!(
301 - "Target: {label} ({target})\n\
302 - Recovered routes:\n{}\n\
303 - Instance: {}\n\
304 - Time: {}\n\n\
305 - - PoM",
306 - recovered_paths.iter().map(|p| format!(" - {p}")).collect::<Vec<_>>().join("\n"),
307 - self.instance_name,
308 - chrono::Utc::now().to_rfc3339(),
309 - );
310 -
311 - self.fire_recovery(&subject, &body,
312 - AlertMeta { key: &alert_key, category: AlertCategory::RouteRecovery, from: None, to: None, error: None }).await;
313 - }
314 -
315 - #[instrument(skip_all)]
316 - pub async fn send_dns_mismatch_alert(
317 - &self,
318 - target: &str,
319 - label: &str,
320 - mismatches: &[crate::types::DnsCheckResult],
321 - ) {
322 - let alert_key = format!("dns:{target}");
323 - let n = mismatches.len();
324 - let subject = format!("[PoM] {label}: {n} DNS record(s) mismatched");
325 - let details: Vec<String> = mismatches
326 - .iter()
327 - .map(|m| {
328 - if let Some(ref err) = m.error {
329 - format!(" - {} {}: {err}", m.name, m.record_type)
330 - } else {
331 - format!(
332 - " - {} {}: expected {:?}, got {:?}",
333 - m.name, m.record_type, m.expected, m.actual
334 - )
335 - }
336 - })
337 - .collect();
338 - let body = format!(
339 - "Target: {label} ({target})\n\
340 - DNS mismatches:\n{}\n\
341 - Instance: {}\n\
342 - Time: {}\n\n\
343 - - PoM",
344 - details.join("\n"),
345 - self.instance_name,
346 - chrono::Utc::now().to_rfc3339(),
347 - );
348 -
349 - self.fire_failure(&subject, &body, "high", "pom-dns", Some(target),
350 - AlertMeta { key: &alert_key, category: AlertCategory::DnsMismatch, from: None, to: None, error: None }).await;
351 - }
352 -
353 - #[instrument(skip_all)]
354 - pub async fn send_dns_recovery_alert(
355 - &self,
356 - target: &str,
357 - label: &str,
358 - ) {
359 - let alert_key = format!("dns:{target}");
360 - let subject = format!("[PoM] {label}: DNS records recovered");
361 - let body = format!(
362 - "Target: {label} ({target})\n\
363 - All DNS records now match expected values.\n\
364 - Instance: {}\n\
365 - Time: {}\n\n\
366 - - PoM",
367 - self.instance_name,
368 - chrono::Utc::now().to_rfc3339(),
369 - );
370 -
371 - self.fire_recovery(&subject, &body,
372 - AlertMeta { key: &alert_key, category: AlertCategory::DnsRecovery, from: None, to: None, error: None }).await;
373 - }
374 -
375 - #[instrument(skip_all)]
376 - pub async fn send_whois_expiry_alert(
377 - &self,
378 - target: &str,
379 - label: &str,
380 - domain: &str,
381 - days_remaining: i64,
382 - ) {
383 - let alert_key = format!("whois:{target}");
384 - let subject = format!("[PoM] {label}: domain {domain} expires in {days_remaining} days");
385 - let body = format!(
386 - "Target: {label} ({target})\n\
387 - Domain: {domain}\n\
388 - Days remaining: {days_remaining}\n\
389 - Instance: {}\n\
390 - Time: {}\n\n\
391 - - PoM",
392 - self.instance_name,
393 - chrono::Utc::now().to_rfc3339(),
394 - );
395 -
396 - let priority = whois_expiry_priority(days_remaining);
397 - self.fire_failure(&subject, &body, priority, "pom-whois", Some(&format!("{target}:{domain}")),
398 - AlertMeta { key: &alert_key, category: AlertCategory::WhoisExpiry, from: None, to: None, error: None }).await;
399 - }
400 -
401 - #[instrument(skip_all)]
402 - pub async fn send_whois_error_alert(
403 - &self,
404 - target: &str,
405 - label: &str,
406 - domain: &str,
407 - error: &str,
408 - ) {
409 - let alert_key = format!("whois:{target}");
410 - let subject = format!("[PoM] {label}: WHOIS check failed for {domain}");
411 - let body = format!(
412 - "Target: {label} ({target})\n\
413 - Domain: {domain}\n\
414 - Error: {error}\n\
415 - Instance: {}\n\
416 - Time: {}\n\n\
417 - - PoM",
418 - self.instance_name,
419 - chrono::Utc::now().to_rfc3339(),
420 - );
421 -
422 - self.fire_failure(&subject, &body, "high", "pom-whois", Some(&format!("{target}:{domain}")),
423 - AlertMeta { key: &alert_key, category: AlertCategory::WhoisError, from: None, to: None, error: Some(error) }).await;
424 - }
425 -
426 - #[instrument(skip_all)]
427 - pub async fn send_cors_failure_alert(
428 - &self,
429 - target: &str,
430 - label: &str,
431 - failures: &[crate::types::CorsCheckResult],
432 - ) {
433 - let alert_key = format!("cors:{target}");
434 - let n = failures.len();
435 - let subject = format!("[PoM] {label}: {n} CORS preflight(s) failing");
436 - let details: Vec<String> = failures
437 - .iter()
438 - .map(|f| {
439 - if let Some(ref err) = f.error {
440 - format!(" - {} {} from {}: {err}", f.method, f.url, f.origin)
441 - } else {
442 - format!(" - {} {} from {}: no CORS headers", f.method, f.url, f.origin)
443 - }
444 - })
445 - .collect();
446 - let body = format!(
447 - "Target: {label} ({target})\n\
448 - CORS preflight failures:\n{}\n\
449 - Instance: {}\n\
450 - Time: {}\n\n\
451 - Browser-side uploads will silently fail without CORS.\n\n\
452 - - PoM",
453 - details.join("\n"),
454 - self.instance_name,
455 - chrono::Utc::now().to_rfc3339(),
456 - );
457 -
458 - self.fire_failure(&subject, &body, "high", "pom-cors", Some(target),
459 - AlertMeta { key: &alert_key, category: AlertCategory::CorsFailure, from: None, to: None, error: None }).await;
460 - }
461 -
462 - #[instrument(skip_all)]
463 - pub async fn send_cors_recovery_alert(
464 - &self,
465 - target: &str,
466 - label: &str,
467 - ) {
468 - let alert_key = format!("cors:{target}");
469 - let subject = format!("[PoM] {label}: CORS preflights recovered");
470 - let body = format!(
471 - "Target: {label} ({target})\n\
472 - All CORS preflight checks passing.\n\
473 - Instance: {}\n\
474 - Time: {}\n\n\
475 - - PoM",
476 - self.instance_name,
477 - chrono::Utc::now().to_rfc3339(),
478 - );
479 -
480 - self.fire_recovery(&subject, &body,
481 - AlertMeta { key: &alert_key, category: AlertCategory::CorsRecovery, from: None, to: None, error: None }).await;
482 - }
483 -
484 - #[instrument(skip_all)]
485 - pub async fn send_latency_drift_alert(
486 - &self,
487 - target: &str,
488 - label: &str,
489 - drift_message: &str,
490 - ) {
491 - let alert_key = format!("latency:{target}");
492 - let subject = format!("[PoM] {target}: latency drift detected");
493 - let body = format!(
494 - "Target: {label} ({target})\n\
495 - {drift_message}\n\
496 - Instance: {}\n\
497 - Time: {}\n\n\
498 - - PoM",
499 - self.instance_name,
500 - chrono::Utc::now().to_rfc3339(),
Lines truncated
A pom/src/alerts/backup.rs +62
@@ -0,0 +1,62 @@
1 + //! backup alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_backup_stale_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + database: &str,
15 + status: &str,
16 + age_hours: Option<i64>,
17 + ) {
18 + let alert_key = format!("backup:{target}:{database}");
19 + let detail = backup_status_detail(status, age_hours);
20 +
21 + let subject = format!("[PoM] {label}: {database} backup {status}");
22 + let body = format!(
23 + "Target: {label} ({target})\n\
24 + Database: {database}\n\
25 + Status: {status}\n\
26 + Detail: {detail}\n\
27 + Instance: {}\n\
28 + Time: {}\n\n\
29 + - PoM",
30 + self.instance_name,
31 + chrono::Utc::now().to_rfc3339(),
32 + );
33 +
34 + let priority = backup_status_priority(status);
35 + self.fire_failure(&subject, &body, priority, "pom-backup", Some(&format!("{target}:{database}")),
36 + AlertMeta { key: &alert_key, category: AlertCategory::BackupStale, from: None, to: Some(status), error: None }).await;
37 + }
38 +
39 + #[instrument(skip_all)]
40 + pub async fn send_backup_recovery(
41 + &self,
42 + target: &str,
43 + label: &str,
44 + database: &str,
45 + ) {
46 + let alert_key = format!("backup:{target}:{database}");
47 + let subject = format!("[PoM] {label}: {database} backup recovered");
48 + let body = format!(
49 + "Target: {label} ({target})\n\
50 + Database: {database}\n\
51 + Backup is now current.\n\
52 + Instance: {}\n\
53 + Time: {}\n\n\
54 + - PoM",
55 + self.instance_name,
56 + chrono::Utc::now().to_rfc3339(),
57 + );
58 +
59 + self.fire_recovery(&subject, &body,
60 + AlertMeta { key: &alert_key, category: AlertCategory::BackupRecovery, from: None, to: Some("ok"), error: None }).await;
61 + }
62 + }
A pom/src/alerts/cors.rs +66
@@ -0,0 +1,66 @@
1 + //! cors alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_cors_failure_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + failures: &[crate::types::CorsCheckResult],
15 + ) {
16 + let alert_key = format!("cors:{target}");
17 + let n = failures.len();
18 + let subject = format!("[PoM] {label}: {n} CORS preflight(s) failing");
19 + let details: Vec<String> = failures
20 + .iter()
21 + .map(|f| {
22 + if let Some(ref err) = f.error {
23 + format!(" - {} {} from {}: {err}", f.method, f.url, f.origin)
24 + } else {
25 + format!(" - {} {} from {}: no CORS headers", f.method, f.url, f.origin)
26 + }
27 + })
28 + .collect();
29 + let body = format!(
30 + "Target: {label} ({target})\n\
31 + CORS preflight failures:\n{}\n\
32 + Instance: {}\n\
33 + Time: {}\n\n\
34 + Browser-side uploads will silently fail without CORS.\n\n\
35 + - PoM",
36 + details.join("\n"),
37 + self.instance_name,
38 + chrono::Utc::now().to_rfc3339(),
39 + );
40 +
41 + self.fire_failure(&subject, &body, "high", "pom-cors", Some(target),
42 + AlertMeta { key: &alert_key, category: AlertCategory::CorsFailure, from: None, to: None, error: None }).await;
43 + }
44 +
45 + #[instrument(skip_all)]
46 + pub async fn send_cors_recovery_alert(
47 + &self,
48 + target: &str,
49 + label: &str,
50 + ) {
51 + let alert_key = format!("cors:{target}");
52 + let subject = format!("[PoM] {label}: CORS preflights recovered");
53 + let body = format!(
54 + "Target: {label} ({target})\n\
55 + All CORS preflight checks passing.\n\
56 + Instance: {}\n\
57 + Time: {}\n\n\
58 + - PoM",
59 + self.instance_name,
60 + chrono::Utc::now().to_rfc3339(),
61 + );
62 +
63 + self.fire_recovery(&subject, &body,
64 + AlertMeta { key: &alert_key, category: AlertCategory::CorsRecovery, from: None, to: None, error: None }).await;
65 + }
66 + }
A pom/src/alerts/dns.rs +68
@@ -0,0 +1,68 @@
1 + //! dns alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_dns_mismatch_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + mismatches: &[crate::types::DnsCheckResult],
15 + ) {
16 + let alert_key = format!("dns:{target}");
17 + let n = mismatches.len();
18 + let subject = format!("[PoM] {label}: {n} DNS record(s) mismatched");
19 + let details: Vec<String> = mismatches
20 + .iter()
21 + .map(|m| {
22 + if let Some(ref err) = m.error {
23 + format!(" - {} {}: {err}", m.name, m.record_type)
24 + } else {
25 + format!(
26 + " - {} {}: expected {:?}, got {:?}",
27 + m.name, m.record_type, m.expected, m.actual
28 + )
29 + }
30 + })
31 + .collect();
32 + let body = format!(
33 + "Target: {label} ({target})\n\
34 + DNS mismatches:\n{}\n\
35 + Instance: {}\n\
36 + Time: {}\n\n\
37 + - PoM",
38 + details.join("\n"),
39 + self.instance_name,
40 + chrono::Utc::now().to_rfc3339(),
41 + );
42 +
43 + self.fire_failure(&subject, &body, "high", "pom-dns", Some(target),
44 + AlertMeta { key: &alert_key, category: AlertCategory::DnsMismatch, from: None, to: None, error: None }).await;
45 + }
46 +
47 + #[instrument(skip_all)]
48 + pub async fn send_dns_recovery_alert(
49 + &self,
50 + target: &str,
51 + label: &str,
52 + ) {
53 + let alert_key = format!("dns:{target}");
54 + let subject = format!("[PoM] {label}: DNS records recovered");
55 + let body = format!(
56 + "Target: {label} ({target})\n\
57 + All DNS records now match expected values.\n\
58 + Instance: {}\n\
59 + Time: {}\n\n\
60 + - PoM",
61 + self.instance_name,
62 + chrono::Utc::now().to_rfc3339(),
63 + );
64 +
65 + self.fire_recovery(&subject, &body,
66 + AlertMeta { key: &alert_key, category: AlertCategory::DnsRecovery, from: None, to: None, error: None }).await;
67 + }
68 + }
A pom/src/alerts/health.rs +60
@@ -0,0 +1,60 @@
1 + //! health alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_health_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + from_status: &str,
15 + to_status: &str,
16 + error: Option<&str>,
17 + ) {
18 + let alert_key = format!("health:{target}");
19 + let subject = format!("[PoM] {target}: {from_status} -> {to_status}");
20 + let mut body = format!(
21 + "Target: {label} ({target})\n\
22 + Status: {from_status} -> {to_status}\n\
23 + Instance: {}\n\
24 + Time: {}\n",
25 + self.instance_name,
26 + chrono::Utc::now().to_rfc3339(),
27 + );
28 + if let Some(err) = error {
29 + body.push_str(&format!("Error: {err}\n"));
30 + }
31 + body.push_str("\n- PoM");
32 +
33 + let priority = health_status_priority(to_status);
34 + self.fire_failure(&subject, &body, priority, "pom-health", Some(target),
35 + AlertMeta { key: &alert_key, category: AlertCategory::Health, from: Some(from_status), to: Some(to_status), error }).await;
36 + }
37 +
38 + #[instrument(skip_all)]
39 + pub async fn send_health_recovery(
40 + &self,
41 + target: &str,
42 + label: &str,
43 + from_status: &str,
44 + ) {
45 + let alert_key = format!("health:{target}");
46 + let subject = format!("[PoM] {target}: recovered");
47 + let body = format!(
48 + "Target: {label} ({target})\n\
49 + Status: {from_status} -> operational\n\
50 + Instance: {}\n\
51 + Time: {}\n\n\
52 + - PoM",
53 + self.instance_name,
54 + chrono::Utc::now().to_rfc3339(),
55 + );
56 +
57 + self.fire_recovery(&subject, &body,
58 + AlertMeta { key: &alert_key, category: AlertCategory::Recovery, from: Some(from_status), to: Some("operational"), error: None }).await;
59 + }
60 + }
A pom/src/alerts/latency.rs +53
@@ -0,0 +1,53 @@
1 + //! latency alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_latency_drift_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + drift_message: &str,
15 + ) {
16 + let alert_key = format!("latency:{target}");
17 + let subject = format!("[PoM] {target}: latency drift detected");
18 + let body = format!(
19 + "Target: {label} ({target})\n\
20 + {drift_message}\n\
21 + Instance: {}\n\
22 + Time: {}\n\n\
23 + - PoM",
24 + self.instance_name,
25 + chrono::Utc::now().to_rfc3339(),
26 + );
27 +
28 + self.fire_failure(&subject, &body, "medium", "pom-latency", Some(target),
29 + AlertMeta { key: &alert_key, category: AlertCategory::LatencyDrift, from: None, to: None, error: Some(drift_message) }).await;
30 + }
31 +
32 + #[instrument(skip_all)]
33 + pub async fn send_latency_recovery(
34 + &self,
35 + target: &str,
36 + label: &str,
37 + ) {
38 + let alert_key = format!("latency:{target}");
39 + let subject = format!("[PoM] {target}: latency recovered");
40 + let body = format!(
41 + "Target: {label} ({target})\n\
42 + Latency returned to normal.\n\
43 + Instance: {}\n\
44 + Time: {}\n\n\
45 + - PoM",
46 + self.instance_name,
47 + chrono::Utc::now().to_rfc3339(),
48 + );
49 +
50 + self.fire_recovery(&subject, &body,
51 + AlertMeta { key: &alert_key, category: AlertCategory::LatencyRecovery, from: None, to: None, error: None }).await;
52 + }
53 + }
A pom/src/alerts/mod.rs +500
@@ -0,0 +1,756 @@
1 + //! Email alerting via Postmark API.
2 + //!
3 + //! Sends alerts on health status transitions and peer disappearance/recovery.
4 + //! If no `postmark_token` is configured, alerts are logged to stdout instead.
5 +
6 + use sqlx::SqlitePool;
7 + use tracing::{info, warn};
8 +
9 + use crate::config::AlertConfig;
10 + use crate::db;
11 + use crate::types::AlertCategory;
12 +
13 + mod health;
14 + mod tls;
15 + mod peer;
16 + mod route;
17 + mod dns;
18 + mod whois;
19 + mod cors;
20 + mod latency;
21 + mod test_duration;
22 + mod backup;
23 + mod scan;
24 + mod offline;
25 +
26 + /// WAM ticket priority for a transition into a non-operational health status.
27 + fn health_status_priority(to_status: &str) -> &'static str {
28 + match to_status {
29 + "error" | "unreachable" => "critical",
30 + "degraded" => "high",
31 + _ => "medium",
32 + }
33 + }
34 +
35 + /// WAM ticket priority for a TLS certificate that expires in `days`.
36 + fn tls_expiry_priority(days: i64) -> &'static str {
37 + if days <= 3 { "critical" } else if days <= 7 { "high" } else { "medium" }
38 + }
39 +
40 + /// WAM ticket priority for a domain registration that expires in `days`.
41 + fn whois_expiry_priority(days: i64) -> &'static str {
42 + if days <= 7 { "critical" } else if days <= 14 { "high" } else { "medium" }
43 + }
44 +
45 + /// WAM ticket priority for a stale/missing/error backup status.
46 + fn backup_status_priority(status: &str) -> &'static str {
47 + if status == "missing" { "critical" } else { "high" }
48 + }
49 +
50 + /// Human-readable detail text for a backup status alert.
51 + fn backup_status_detail(status: &str, age_hours: Option<i64>) -> String {
52 + match (status, age_hours) {
53 + ("stale", Some(hours)) => format!("last backup is {hours}h old"),
54 + ("missing", _) => "no backup files found".to_string(),
55 + ("error", _) => "backup check failed".to_string(),
56 + _ => format!("status: {status}"),
57 + }
58 + }
59 +
60 + #[derive(Clone)]
61 + pub struct Alerter {
62 + config: AlertConfig,
63 + client: reqwest::Client,
64 + pool: SqlitePool,
65 + instance_name: String,
66 + wam_url: Option<String>,
67 + }
68 +
69 + /// The record-context an alert carries: what to write to the `alerts` ledger on
70 + /// successful delivery, and what to persist for retry on failure.
71 + struct AlertMeta<'a> {
72 + key: &'a str,
73 + category: AlertCategory,
74 + from: Option<&'a str>,
75 + to: Option<&'a str>,
76 + error: Option<&'a str>,
77 + }
78 +
79 + /// `true` if `sent_at` (rfc3339) is within `cooldown_secs` of now. An unparseable
80 + /// timestamp is treated as expired (not in cooldown) so a bad row can never wedge
81 + /// alerting shut.
82 + fn within_cooldown_secs(sent_at: &str, cooldown_secs: u64) -> bool {
83 + match chrono::DateTime::parse_from_rfc3339(sent_at) {
84 + Ok(dt) => {
85 + chrono::Utc::now().signed_duration_since(dt).num_seconds() < cooldown_secs as i64
86 + }
87 + Err(_) => false,
88 + }
89 + }
90 +
91 + impl Alerter {
92 + pub fn new(config: AlertConfig, pool: SqlitePool, instance_name: String) -> Self {
93 + let client = reqwest::Client::builder()
94 + .timeout(std::time::Duration::from_secs(10))
95 + .build()
96 + .unwrap_or_default();
97 + let wam_url = config.wam_url.clone();
98 + if wam_url.is_none() {
99 + warn!(
100 + "alerts: wam_url not configured — failure alerts will be delivered by email \
101 + instead of WAM tickets (set alerts.wam_url to route them to WAM)"
102 + );
103 + }
104 + Self { config, client, pool, instance_name, wam_url }
105 + }
106 +
107 + /// Whether a FAILURE alert for `target` is within its cooldown.
108 + ///
109 + /// A recorded recovery *newer* than the last failure clears the cooldown: the
110 + /// target came back, so a fresh failure is a genuine new outage and must not
111 + /// be suppressed by the prior failure's window (fuzz-2026-07-06 fail→recover→
112 + /// fail suppression).
113 + async fn is_within_cooldown(&self, target: &str) -> bool {
114 + let Ok(Some(latest_fail)) = db::get_latest_alert_for_target(&self.pool, target).await else {
115 + return false; // never alerted → not in cooldown
116 + };
117 + // A recovery after the last failure voids the cooldown.
118 + if let Ok(Some(rec)) =
119 + db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
120 + && rec.id > latest_fail.id
121 + {
122 + return false;
123 + }
124 + within_cooldown_secs(&latest_fail.sent_at, self.config.cooldown_secs)
125 + }
126 +
127 + /// Whether a RECOVERY for `target` is within its own cooldown — throttles a
128 + /// flapping target so it can't emit one recovery email per flap
129 + /// (fuzz-2026-07-06 un-throttled recoveries). A failure newer than the last
130 + /// recovery clears it, so a genuine recover-after-a-new-outage still sends.
131 + async fn is_recovery_within_cooldown(&self, target: &str) -> bool {
132 + let Ok(Some(latest_rec)) =
133 + db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
134 + else {
135 + return false;
136 + };
137 + if let Ok(Some(fail)) = db::get_latest_alert_for_target(&self.pool, target).await
138 + && fail.id > latest_rec.id
139 + {
140 + return false;
141 + }
142 + within_cooldown_secs(&latest_rec.sent_at, self.config.cooldown_secs)
143 + }
144 +
145 + /// Send an email via Postmark. Returns `true` if the message was accepted
146 + /// (or in dev mode where there is no token — the operator sees it logged).
147 + async fn send_email(&self, subject: &str, body: &str) -> bool {
148 + let Some(ref token) = self.config.postmark_token else {
149 + info!("[dev] alert: {subject}");
150 + info!("[dev] {body}");
151 + return true;
152 + };
153 +
154 + let payload = serde_json::json!({
155 + "From": self.config.from,
156 + "To": self.config.to,
157 + "Subject": subject,
158 + "TextBody": body,
159 + });
160 +
161 + let send_fut = self.client
162 + .post("https://api.postmarkapp.com/email")
163 + .header("X-Postmark-Server-Token", token)
164 + .header("Content-Type", "application/json")
165 + .header("Accept", "application/json")
166 + .json(&payload)
167 + .send();
168 +
169 + // Wrap in a 30-second timeout to prevent Postmark latency from blocking
170 + // the alert task. The reqwest client has its own 10s timeout, but this
171 + // guards against DNS resolution stalls and connection pool exhaustion.
172 + match tokio::time::timeout(std::time::Duration::from_secs(30), send_fut).await {
173 + Ok(Ok(resp)) if resp.status().is_success() => {
174 + info!("alert sent: {subject}");
175 + true
176 + }
177 + Ok(Ok(resp)) => {
178 + let status = resp.status();
179 + let text = resp.text().await.unwrap_or_default();
180 + warn!("postmark error ({status}): {text}");
181 + false
182 + }
183 + Ok(Err(e)) => {
184 + warn!("failed to send alert: {e}");
185 + false
186 + }
187 + Err(_) => {
188 + warn!("alert send timed out after 30s: {subject}");
189 + false
190 + }
191 + }
192 + }
193 +
194 + async fn record_alert(
195 + &self,
196 + target: &str,
197 + alert_type: AlertCategory,
198 + from_status: Option<&str>,
199 + to_status: Option<&str>,
200 + error: Option<&str>,
201 + ) {
202 + self.record_alert_str(target, &alert_type.to_string(), from_status, to_status, error).await;
203 + }
204 +
205 + /// String-typed variant so the retry task can record a queued alert straight
206 + /// from its stored category without round-tripping through the enum.
207 + async fn record_alert_str(
208 + &self,
209 + target: &str,
210 + alert_type: &str,
211 + from_status: Option<&str>,
212 + to_status: Option<&str>,
213 + error: Option<&str>,
214 + ) {
215 + if let Err(e) = db::insert_alert(&self.pool, target, alert_type, from_status, to_status, error).await {
216 + warn!("failed to record alert: {e}");
217 + }
218 + }
219 +
220 + /// Fire a failure alert: cooldown-gate, then dispatch. The single entry point
221 + /// every `send_*_alert` funnels through, so the cooldown check + dispatch is
222 + /// written once rather than copy-pasted into each of the ~13 methods.
223 + async fn fire_failure(
224 + &self,
225 + subject: &str,
226 + body: &str,
227 + priority: &str,
228 + source: &str,
229 + source_ref: Option<&str>,
230 + meta: AlertMeta<'_>,
231 + ) {
232 + if self.is_within_cooldown(meta.key).await {
233 + info!("alert cooldown active for {}, skipping", meta.key);
234 + return;
235 + }
236 + self.dispatch_wam(subject, body, priority, source, source_ref, meta).await;
237 + }
238 +
239 + /// Fire a recovery: recovery-cooldown-gate, then dispatch by email. The single
240 + /// entry point every `send_*_recovery` funnels through.
241 + async fn fire_recovery(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
242 + if self.is_recovery_within_cooldown(meta.key).await {
243 + info!("recovery cooldown active for {}, skipping", meta.key);
244 + return;
245 + }
246 + self.dispatch_email(subject, body, meta).await;
247 + }
248 +
249 + /// Dispatch a WAM failure alert: attempt delivery (WAM, falling back to
250 + /// email), record it on success, or persist it for retry on failure. Gating
251 + /// the ledger write on delivery — and queueing the miss — is what stops a
252 + /// transient send failure at a status transition from silencing the whole
253 + /// outage: the transition fires once, but the queued alert is retried until it
254 + /// lands (fuzz-2026-07-06 CRITICAL #1).
255 + async fn dispatch_wam(
256 + &self,
257 + subject: &str,
258 + body: &str,
259 + priority: &str,
260 + source: &str,
261 + source_ref: Option<&str>,
262 + meta: AlertMeta<'_>,
263 + ) {
264 + if self.wam_ticket(subject, body, priority, source, source_ref).await {
265 + self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error).await;
266 + } else {
267 + self.enqueue_undelivered(subject, body, "wam", Some(priority), Some(source), source_ref, &meta).await;
268 + }
269 + }
270 +
271 + /// Dispatch an email alert (recoveries, and failure alerts that already chose
272 + /// email): record on success, persist for retry on failure.
273 + async fn dispatch_email(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
274 + if self.send_email(subject, body).await {
275 + self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error).await;
276 + } else {
277 + self.enqueue_undelivered(subject, body, "email", None, None, None, &meta).await;
278 + }
279 + }
280 +
281 + #[allow(clippy::too_many_arguments)]
282 + async fn enqueue_undelivered(
283 + &self,
284 + subject: &str,
285 + body: &str,
286 + channel: &str,
287 + priority: Option<&str>,
288 + source: Option<&str>,
289 + source_ref: Option<&str>,
290 + meta: &AlertMeta<'_>,
291 + ) {
292 + let category = meta.category.to_string();
293 + let pending = db::NewPendingAlert {
294 + alert_key: meta.key,
295 + category: &category,
296 + channel,
297 + subject,
298 + body,
299 + priority,
300 + source,
301 + source_ref,
302 + from_status: meta.from,
303 + to_status: meta.to,
304 + error: meta.error,
305 + };
306 + if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await {
307 + warn!("failed to enqueue undelivered alert for retry: {e}");
308 + } else {
309 + warn!("alert to {} undelivered; queued for retry", meta.key);
310 + }
311 + }
312 +
313 + /// Re-attempt one queued alert. On delivery, record it in the ledger and
314 + /// signal the caller to delete the row; otherwise leave it for a later tick.
315 + /// Called by the retry task in `serve`.
316 + pub async fn retry_pending(&self, p: &db::PendingAlertRow) -> bool {
317 + let delivered = match p.channel.as_str() {
318 + "wam" => {
319 + self.wam_ticket(
320 + &p.subject,
321 + &p.body,
322 + p.priority.as_deref().unwrap_or("high"),
323 + p.source.as_deref().unwrap_or("pom"),
324 + p.source_ref.as_deref(),
325 + )
326 + .await
327 + }
328 + _ => self.send_email(&p.subject, &p.body).await,
329 + };
330 + if delivered {
331 + self.record_alert_str(
332 + &p.alert_key,
333 + &p.category,
334 + p.from_status.as_deref(),
335 + p.to_status.as_deref(),
336 + p.error.as_deref(),
337 + )
338 + .await;
339 + }
340 + delivered
341 + }
342 +
343 + /// Create a WAM ticket for a failure alert. Returns `true` if the alert was
344 + /// delivered by *some* channel.
345 + ///
346 + /// If `wam_url` is unset the failure alert would otherwise be dropped on the
347 + /// floor — failures route to WAM, recoveries to email, so a missing `wam_url`
348 + /// silently disabled the entire down-alert channel while recovery emails kept
349 + /// firing (fuzz-2026-07-06 CRITICAL #2). Fall back to email instead: a
350 + /// monitoring tool must never silently swallow a down-alert. WAM delivery
351 + /// failures also fall back to email so a WAM outage can't blind the operator.
352 + async fn wam_ticket(
353 + &self,
354 + title: &str,
355 + body: &str,
356 + priority: &str,
357 + source: &str,
358 + source_ref: Option<&str>,
359 + ) -> bool {
360 + let Some(ref base_url) = self.wam_url else {
361 + // No WAM configured: deliver the failure alert by email instead of
362 + // dropping it.
363 + return self.send_email(title, body).await;
364 + };
365 + let url = format!("{base_url}/tickets");
366 +
367 + let mut payload = serde_json::json!({
368 + "title": title,
369 + "body": body,
370 + "priority": priority,
371 + "source": source,
372 + });
373 + if let Some(r) = source_ref {
374 + payload["source_ref"] = serde_json::json!(r);
375 + }
376 +
377 + match self.client.post(&url).json(&payload).send().await {
378 + Ok(resp) if resp.status().is_success() => {
379 + info!("WAM ticket created: {title}");
380 + true
381 + }
382 + Ok(resp) => {
383 + warn!("WAM ticket creation returned {}: {title}; falling back to email", resp.status());
384 + self.send_email(title, body).await
385 + }
386 + Err(e) => {
387 + warn!("WAM unreachable: {e}; falling back to email");
388 + self.send_email(title, body).await
389 + }
390 + }
391 + }
392 + }
393 +
394 + #[cfg(test)]
395 + mod tests {
396 + use super::*;
397 +
398 + fn test_alerter(pool: SqlitePool) -> Alerter {
399 + let config = AlertConfig {
400 + postmark_token: None, // dev mode
401 + to: "test@example.com".to_string(),
402 + from: "PoM Alerts <pom-alerts@makenot.work>".to_string(),
403 + cooldown_secs: 300,
404 + wam_url: None,
405 + };
406 + Alerter::new(config, pool, "test-instance".to_string())
407 + }
408 +
409 + #[tokio::test]
410 + async fn cooldown_prevents_duplicate_alerts() {
411 + let pool = db::connect_in_memory().await.unwrap();
412 + let alerter = test_alerter(pool.clone());
413 +
414 + // First alert — not in cooldown
415 + assert!(!alerter.is_within_cooldown("health:mnw").await);
416 +
417 + // Record an alert
418 + db::insert_alert(&pool, "health:mnw", "health", Some("operational"), Some("error"), None)
419 + .await
420 + .unwrap();
421 +
422 + // Now should be in cooldown
423 + assert!(alerter.is_within_cooldown("health:mnw").await);
424 + }
425 +
426 + #[tokio::test]
427 + async fn wam_ticket_falls_back_to_email_when_wam_url_unset() {
428 + // CRITICAL #2: with no wam_url, a failure alert must NOT be silently
429 + // dropped — it falls back to email. test_alerter has wam_url: None and
430 + // postmark_token: None (dev mode), so the email path reports delivered.
431 + let pool = db::connect_in_memory().await.unwrap();
432 + let alerter = test_alerter(pool);
433 + assert!(
434 + alerter.wam_ticket("subj", "body", "high", "pom-test", None).await,
435 + "a failure alert with no wam_url must be delivered by email, not dropped"
436 + );
437 + }
438 +
439 + #[tokio::test]
440 + async fn cooldown_does_not_affect_other_targets() {
441 + let pool = db::connect_in_memory().await.unwrap();
442 + let alerter = test_alerter(pool.clone());
443 +
444 + db::insert_alert(&pool, "health:mnw", "health", None, None, None)
445 + .await
446 + .unwrap();
447 +
448 + // Different target should not be in cooldown
449 + assert!(!alerter.is_within_cooldown("health:other").await);
450 + }
451 +
452 + #[tokio::test]
453 + async fn dev_mode_does_not_send_http() {
454 + let pool = db::connect_in_memory().await.unwrap();
455 + let alerter = test_alerter(pool.clone());
456 +
457 + // This should log instead of making HTTP calls (no panic, no error)
458 + alerter.send_health_alert("mnw", "MakeNotWork", "operational", "error", None).await;
459 +
460 + // Verify alert was recorded in DB with the prefixed key (health:mnw),
461 + // matching the cooldown lookup key format.
462 + let latest = db::get_latest_alert_for_target(&pool, "health:mnw").await.unwrap();
463 + assert!(latest.is_some());
464 + let row = latest.unwrap();
465 + assert_eq!(row.alert_type, "health");
466 + assert_eq!(row.from_status.as_deref(), Some("operational"));
467 + assert_eq!(row.to_status.as_deref(), Some("error"));
468 + }
469 +
470 + #[tokio::test]
471 + async fn route_alert_cooldown_key() {
472 + let pool = db::connect_in_memory().await.unwrap();
473 + let alerter = test_alerter(pool.clone());
474 +
475 + assert!(!alerter.is_within_cooldown("route:mnw").await);
476 +
477 + alerter.send_route_failure_alert("mnw", "MakeNotWork", &["/docs/faq".to_string()]).await;
478 +
479 + assert!(alerter.is_within_cooldown("route:mnw").await);
480 + assert!(!alerter.is_within_cooldown("route:mt").await);
481 + }
482 +
483 + #[tokio::test]
484 + async fn recovery_clears_failure_cooldown_so_next_failure_fires() {
485 + // Regression for the fail→recover→fail suppression the fuzz flagged (this
486 + // test previously enshrined the bug: it asserted the 2nd failure stays
487 + // suppressed). A recovery means the target came back, so a fresh failure
488 + // is a genuine NEW outage and must not be muffled by the prior failure's
489 + // cooldown.
490 + let pool = db::connect_in_memory().await.unwrap();
491 + let alerter = test_alerter(pool.clone());
492 +
493 + // Fail → cooldown active.
494 + alerter.send_health_alert("mnw", "MakeNotWork", "operational", "error", None).await;
495 + assert!(alerter.is_within_cooldown("health:mnw").await);
496 +
497 + // Recover → the failure cooldown is now cleared.
498 + alerter.send_health_recovery("mnw", "MakeNotWork", "error").await;
499 + assert!(
500 + !alerter.is_within_cooldown("health:mnw").await,
Lines truncated
A pom/src/alerts/offline.rs +46
@@ -0,0 +1,46 @@
1 + //! offline alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + /// All monitored targets are unreachable — likely a network issue with PoM itself.
10 + #[instrument(skip_all)]
11 + pub async fn send_monitoring_offline_alert(&self, target_count: usize) {
12 + let alert_key = "monitoring:self";
13 + let subject = format!("[PoM] all {target_count} targets unreachable");
14 + let body = format!(
15 + "All {target_count} monitored targets are non-operational.\n\
16 + This likely indicates a network issue with the PoM instance itself,\n\
17 + not an actual outage of all targets.\n\n\
18 + Instance: {}\n\
19 + Time: {}\n\n\
20 + - PoM",
21 + self.instance_name,
22 + chrono::Utc::now().to_rfc3339(),
23 + );
24 +
25 + self.fire_failure(&subject, &body, "critical", "pom-monitoring", Some("self"),
26 + AlertMeta { key: alert_key, category: AlertCategory::MonitoringOffline, from: None, to: None, error: None }).await;
27 + }
28 +
29 + /// At least one target is reachable again after a monitoring-offline event.
30 + #[instrument(skip_all)]
31 + pub async fn send_monitoring_recovery(&self) {
32 + let alert_key = "monitoring:self";
33 + let subject = "[PoM] monitoring recovered".to_string();
34 + let body = format!(
35 + "At least one target is reachable again.\n\
36 + Instance: {}\n\
37 + Time: {}\n\n\
38 + - PoM",
39 + self.instance_name,
40 + chrono::Utc::now().to_rfc3339(),
41 + );
42 +
43 + self.fire_recovery(&subject, &body,
44 + AlertMeta { key: alert_key, category: AlertCategory::MonitoringRecovery, from: None, to: None, error: None }).await;
45 + }
46 + }
A pom/src/alerts/peer.rs +54
@@ -0,0 +1,54 @@
1 + //! peer alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_peer_missing(
11 + &self,
12 + peer_name: &str,
13 + address: &str,
14 + consecutive_failures: u32,
15 + ) {
16 + let alert_key = format!("peer:{peer_name}");
17 + let subject = format!("[PoM] peer {peer_name}: missing");
18 + let body = format!(
19 + "Peer: {peer_name}\n\
20 + Address: {address}\n\
21 + Consecutive failures: {consecutive_failures}\n\
22 + Instance: {}\n\
23 + Time: {}\n\n\
24 + - PoM",
25 + self.instance_name,
26 + chrono::Utc::now().to_rfc3339(),
27 + );
28 +
29 + self.fire_failure(&subject, &body, "high", "pom-peer", Some(peer_name),
30 + AlertMeta { key: &alert_key, category: AlertCategory::PeerMissing, from: None, to: None, error: None }).await;
31 + }
32 +
33 + #[instrument(skip_all)]
34 + pub async fn send_peer_recovery(
35 + &self,
36 + peer_name: &str,
37 + address: &str,
38 + ) {
39 + let subject = format!("[PoM] peer {peer_name}: recovered");
40 + let body = format!(
41 + "Peer: {peer_name}\n\
42 + Address: {address}\n\
43 + Instance: {}\n\
44 + Time: {}\n\n\
45 + - PoM",
46 + self.instance_name,
47 + chrono::Utc::now().to_rfc3339(),
48 + );
49 +
50 + let alert_key = format!("peer:{peer_name}");
51 + self.fire_recovery(&subject, &body,
52 + AlertMeta { key: &alert_key, category: AlertCategory::PeerRecovery, from: None, to: None, error: None }).await;
53 + }
54 + }
A pom/src/alerts/route.rs +57
@@ -0,0 +1,57 @@
1 + //! route alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_route_failure_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + failed_paths: &[String],
15 + ) {
16 + let alert_key = format!("route:{target}");
17 + let n = failed_paths.len();
18 + let subject = format!("[PoM] {label}: {n} route(s) failing");
19 + let body = format!(
20 + "Target: {label} ({target})\n\
21 + Failed routes:\n{}\n\
22 + Instance: {}\n\
23 + Time: {}\n\n\
24 + - PoM",
25 + failed_paths.iter().map(|p| format!(" - {p}")).collect::<Vec<_>>().join("\n"),
26 + self.instance_name,
27 + chrono::Utc::now().to_rfc3339(),
28 + );
29 +
30 + self.fire_failure(&subject, &body, "high", "pom-routes", Some(target),
31 + AlertMeta { key: &alert_key, category: AlertCategory::RouteFailure, from: None, to: None, error: None }).await;
32 + }
33 +
34 + #[instrument(skip_all)]
35 + pub async fn send_route_recovery_alert(
36 + &self,
37 + target: &str,
38 + label: &str,
39 + recovered_paths: &[String],
40 + ) {
41 + let alert_key = format!("route:{target}");
42 + let subject = format!("[PoM] {label}: routes recovered");
43 + let body = format!(
44 + "Target: {label} ({target})\n\
45 + Recovered routes:\n{}\n\
46 + Instance: {}\n\
47 + Time: {}\n\n\
48 + - PoM",
49 + recovered_paths.iter().map(|p| format!(" - {p}")).collect::<Vec<_>>().join("\n"),
50 + self.instance_name,
51 + chrono::Utc::now().to_rfc3339(),
52 + );
53 +
54 + self.fire_recovery(&subject, &body,
55 + AlertMeta { key: &alert_key, category: AlertCategory::RouteRecovery, from: None, to: None, error: None }).await;
56 + }
57 + }
A pom/src/alerts/scan.rs +58
@@ -0,0 +1,58 @@
1 + //! scan alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + /// Fire when the scan pipeline transitions from operational → degraded or
10 + /// unreachable. Includes the audit-doc threshold issues that fired.
11 + #[instrument(skip_all)]
12 + pub async fn send_scan_pipeline_alert(
13 + &self,
14 + target: &str,
15 + label: &str,
16 + status: &str,
17 + issues: &[String],
18 + ) {
19 + let alert_key = format!("scan_pipeline:{target}");
20 + let subject = format!("[PoM] {label}: scan pipeline {status}");
21 + let body = format!(
22 + "Target: {label} ({target})\n\
23 + Status: {status}\n\
24 + Issues:\n{}\n\
25 + Instance: {}\n\
26 + Time: {}\n\n\
27 + Dashboard: <https://{}/admin/uploads>\n\n\
28 + - PoM",
29 + issues.iter().map(|i| format!(" - {i}")).collect::<Vec<_>>().join("\n"),
30 + self.instance_name,
31 + chrono::Utc::now().to_rfc3339(),
32 + target,
33 + );
34 +
35 + let priority = if status == "unreachable" { "high" } else { "medium" };
36 + self.fire_failure(&subject, &body, priority, "pom-scan-pipeline", Some(target),
37 + AlertMeta { key: &alert_key, category: AlertCategory::ScanPipelineDegraded, from: None, to: Some(status), error: None }).await;
38 + }
39 +
40 + /// Fire on recovery from a degraded / unreachable scan-pipeline state.
41 + #[instrument(skip_all)]
42 + pub async fn send_scan_pipeline_recovery(&self, target: &str, label: &str) {
43 + let alert_key = format!("scan_pipeline:{target}");
44 + let subject = format!("[PoM] {label}: scan pipeline recovered");
45 + let body = format!(
46 + "Target: {label} ({target})\n\
47 + Scan pipeline is operational.\n\
48 + Instance: {}\n\
49 + Time: {}\n\n\
50 + - PoM",
51 + self.instance_name,
52 + chrono::Utc::now().to_rfc3339(),
53 + );
54 +
55 + self.fire_recovery(&subject, &body,
56 + AlertMeta { key: &alert_key, category: AlertCategory::ScanPipelineRecovery, from: None, to: Some("operational"), error: None }).await;
57 + }
58 + }
A pom/src/alerts/test_duration.rs +31
@@ -0,0 +1,31 @@
1 + //! test_duration alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_test_duration_drift_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + drift_message: &str,
15 + ) {
16 + let alert_key = format!("test_duration:{target}");
17 + let subject = format!("[PoM] {target}: test duration drift detected");
18 + let body = format!(
19 + "Target: {label} ({target})\n\
20 + {drift_message}\n\
21 + Instance: {}\n\
22 + Time: {}\n\n\
23 + - PoM",
24 + self.instance_name,
25 + chrono::Utc::now().to_rfc3339(),
26 + );
27 +
28 + self.fire_failure(&subject, &body, "medium", "pom-test-duration", Some(target),
29 + AlertMeta { key: &alert_key, category: AlertCategory::TestDurationDrift, from: None, to: None, error: Some(drift_message) }).await;
30 + }
31 + }
A pom/src/alerts/tls.rs +82
@@ -0,0 +1,82 @@
1 + //! tls alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_tls_expiry_alert(
11 + &self,
12 + target: &str,
13 + host: &str,
14 + days_remaining: i64,
15 + not_after: &str,
16 + ) {
17 + let alert_key = format!("tls:{target}");
18 + let subject = format!("[PoM] {target}: TLS cert expires in {days_remaining} days");
19 + let body = format!(
20 + "Target: {target}\n\
21 + Host: {host}\n\
22 + Days remaining: {days_remaining}\n\
23 + Expires: {not_after}\n\
24 + Instance: {}\n\
25 + Time: {}\n\n\
26 + - PoM",
27 + self.instance_name,
28 + chrono::Utc::now().to_rfc3339(),
29 + );
30 +
31 + let priority = tls_expiry_priority(days_remaining);
32 + self.fire_failure(&subject, &body, priority, "pom-tls", Some(&format!("{target}:{host}")),
33 + AlertMeta { key: &alert_key, category: AlertCategory::TlsExpiry, from: None, to: None, error: None }).await;
34 + }
35 +
36 + #[instrument(skip_all)]
37 + pub async fn send_tls_error_alert(
38 + &self,
39 + target: &str,
40 + host: &str,
41 + error: &str,
42 + ) {
43 + let alert_key = format!("tls:{target}");
44 + let subject = format!("[PoM] {target}: TLS check failed");
45 + let body = format!(
46 + "Target: {target}\n\
47 + Host: {host}\n\
48 + Error: {error}\n\
49 + Instance: {}\n\
50 + Time: {}\n\n\
51 + - PoM",
52 + self.instance_name,
53 + chrono::Utc::now().to_rfc3339(),
54 + );
55 +
56 + self.fire_failure(&subject, &body, "high", "pom-tls", Some(&format!("{target}:{host}")),
57 + AlertMeta { key: &alert_key, category: AlertCategory::TlsError, from: None, to: None, error: Some(error) }).await;
58 + }
59 +
60 + #[instrument(skip_all)]
61 + pub async fn send_tls_recovery(
62 + &self,
63 + target: &str,
64 + label: &str,
65 + days_remaining: i64,
66 + ) {
67 + let alert_key = format!("tls:{target}");
68 + let subject = format!("[PoM] {target}: TLS cert renewed");
69 + let body = format!(
70 + "Target: {label} ({target})\n\
71 + Days remaining: {days_remaining}\n\
72 + Instance: {}\n\
73 + Time: {}\n\n\
74 + - PoM",
75 + self.instance_name,
76 + chrono::Utc::now().to_rfc3339(),
77 + );
78 +
79 + self.fire_recovery(&subject, &body,
80 + AlertMeta { key: &alert_key, category: AlertCategory::TlsRecovery, from: None, to: None, error: None }).await;
81 + }
82 + }
A pom/src/alerts/whois.rs +59
@@ -0,0 +1,59 @@
1 + //! whois alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_whois_expiry_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + domain: &str,
15 + days_remaining: i64,
16 + ) {
17 + let alert_key = format!("whois:{target}");
18 + let subject = format!("[PoM] {label}: domain {domain} expires in {days_remaining} days");
19 + let body = format!(
20 + "Target: {label} ({target})\n\
21 + Domain: {domain}\n\
22 + Days remaining: {days_remaining}\n\
23 + Instance: {}\n\
24 + Time: {}\n\n\
25 + - PoM",
26 + self.instance_name,
27 + chrono::Utc::now().to_rfc3339(),
28 + );
29 +
30 + let priority = whois_expiry_priority(days_remaining);
31 + self.fire_failure(&subject, &body, priority, "pom-whois", Some(&format!("{target}:{domain}")),
32 + AlertMeta { key: &alert_key, category: AlertCategory::WhoisExpiry, from: None, to: None, error: None }).await;
33 + }
34 +
35 + #[instrument(skip_all)]
36 + pub async fn send_whois_error_alert(
37 + &self,
38 + target: &str,
39 + label: &str,
40 + domain: &str,
41 + error: &str,
42 + ) {
43 + let alert_key = format!("whois:{target}");
44 + let subject = format!("[PoM] {label}: WHOIS check failed for {domain}");
45 + let body = format!(
46 + "Target: {label} ({target})\n\
47 + Domain: {domain}\n\
48 + Error: {error}\n\
49 + Instance: {}\n\
50 + Time: {}\n\n\
51 + - PoM",
52 + self.instance_name,
53 + chrono::Utc::now().to_rfc3339(),
54 + );
55 +
56 + self.fire_failure(&subject, &body, "high", "pom-whois", Some(&format!("{target}:{domain}")),
57 + AlertMeta { key: &alert_key, category: AlertCategory::WhoisError, from: None, to: None, error: Some(error) }).await;
58 + }
59 + }