Skip to main content

max / makenotwork

Split pom alerts.rs god-struct into per-domain modules alerts.rs was 1343 lines: one Alerter struct whose impl held both the shared alert plumbing and ~25 near-identical per-domain send_*_alert/send_*_recovery methods. Convert to an alerts/ directory. mod.rs keeps Alerter/AlertMeta, the priority helpers, all shared plumbing (cooldown, send_email, record, fire_failure/recovery, dispatch, retry, wam_ticket), and the tests. The 24 domain methods move into 12 per-domain files (health, tls, peer, route, dns, whois, cors, latency, test_duration, backup, scan, offline) as `impl Alerter` blocks. The domain modules reach the private plumbing and fields as descendants of `alerts`, so no visibility changes are needed and every send_* stays a pub method on Alerter — all callers are unchanged.
Co-Authored-By
Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-13 15:24 UTC
Signed with PGP, not checked
Commit: 425e81701af4fb8ba1c024d15137718fcdd33e58
Parent: 1ea5741
13 files changed, +710 insertions, -472 deletions
R pom/src/alerts.rs → pom/src/alerts/mod.rs +14 -472
@@ -4,12 +4,25 @@
4 4 //! If no `postmark_token` is configured, alerts are logged to stdout instead.
5 5
6 6 use sqlx::SqlitePool;
7 - use tracing::{info, instrument, warn};
7 + use tracing::{info, warn};
8 8
9 9 use crate::config::AlertConfig;
10 10 use crate::db;
11 11 use crate::types::AlertCategory;
12 12
13 + mod health;
14 + mod tls;
15 + mod peer;
16 + mod route;
17 + mod dns;
18 + mod whois;
19 + mod cors;
20 + mod latency;
21 + mod test_duration;
22 + mod backup;
23 + mod scan;
24 + mod offline;
25 +
13 26 /// WAM ticket priority for a transition into a non-operational health status.
14 27 fn health_status_priority(to_status: &str) -> &'static str {
15 28 match to_status {
@@ -91,606 +104,6 @@
91 104 Self { config, client, pool, instance_name, wam_url }
92 105 }
93 106
94 - #[instrument(skip_all)]
95 - pub async fn send_health_alert(
96 - &self,
97 - target: &str,
98 - label: &str,
99 - from_status: &str,
100 - to_status: &str,
101 - error: Option<&str>,
102 - ) {
103 - let alert_key = format!("health:{target}");
104 - let subject = format!("[PoM] {target}: {from_status} -> {to_status}");
105 - let mut body = format!(
106 - "Target: {label} ({target})\n\
107 - Status: {from_status} -> {to_status}\n\
108 - Instance: {}\n\
109 - Time: {}\n",
110 - self.instance_name,
111 - chrono::Utc::now().to_rfc3339(),
112 - );
113 - if let Some(err) = error {
114 - body.push_str(&format!("Error: {err}\n"));
115 - }
116 - body.push_str("\n- PoM");
117 -
118 - let priority = health_status_priority(to_status);
119 - self.fire_failure(&subject, &body, priority, "pom-health", Some(target),
120 - AlertMeta { key: &alert_key, category: AlertCategory::Health, from: Some(from_status), to: Some(to_status), error }).await;
121 - }
122 -
123 - #[instrument(skip_all)]
124 - pub async fn send_health_recovery(
125 - &self,
126 - target: &str,
127 - label: &str,
128 - from_status: &str,
129 - ) {
130 - let alert_key = format!("health:{target}");
131 - let subject = format!("[PoM] {target}: recovered");
132 - let body = format!(
133 - "Target: {label} ({target})\n\
134 - Status: {from_status} -> operational\n\
135 - Instance: {}\n\
136 - Time: {}\n\n\
137 - - PoM",
138 - self.instance_name,
139 - chrono::Utc::now().to_rfc3339(),
140 - );
141 -
142 - self.fire_recovery(&subject, &body,
143 - AlertMeta { key: &alert_key, category: AlertCategory::Recovery, from: Some(from_status), to: Some("operational"), error: None }).await;
144 - }
145 -
146 - #[instrument(skip_all)]
147 - pub async fn send_tls_expiry_alert(
148 - &self,
149 - target: &str,
150 - host: &str,
151 - days_remaining: i64,
152 - not_after: &str,
153 - ) {
154 - let alert_key = format!("tls:{target}");
155 - let subject = format!("[PoM] {target}: TLS cert expires in {days_remaining} days");
156 - let body = format!(
157 - "Target: {target}\n\
158 - Host: {host}\n\
159 - Days remaining: {days_remaining}\n\
160 - Expires: {not_after}\n\
161 - Instance: {}\n\
162 - Time: {}\n\n\
163 - - PoM",
164 - self.instance_name,
165 - chrono::Utc::now().to_rfc3339(),
166 - );
167 -
168 - let priority = tls_expiry_priority(days_remaining);
169 - self.fire_failure(&subject, &body, priority, "pom-tls", Some(&format!("{target}:{host}")),
170 - AlertMeta { key: &alert_key, category: AlertCategory::TlsExpiry, from: None, to: None, error: None }).await;
171 - }
172 -
173 - #[instrument(skip_all)]
174 - pub async fn send_tls_error_alert(
175 - &self,
176 - target: &str,
177 - host: &str,
178 - error: &str,
179 - ) {
180 - let alert_key = format!("tls:{target}");
181 - let subject = format!("[PoM] {target}: TLS check failed");
182 - let body = format!(
183 - "Target: {target}\n\
184 - Host: {host}\n\
185 - Error: {error}\n\
186 - Instance: {}\n\
187 - Time: {}\n\n\
188 - - PoM",
189 - self.instance_name,
190 - chrono::Utc::now().to_rfc3339(),
191 - );
192 -
193 - self.fire_failure(&subject, &body, "high", "pom-tls", Some(&format!("{target}:{host}")),
194 - AlertMeta { key: &alert_key, category: AlertCategory::TlsError, from: None, to: None, error: Some(error) }).await;
195 - }
196 -
197 - #[instrument(skip_all)]
198 - pub async fn send_tls_recovery(
199 - &self,
200 - target: &str,
201 - label: &str,
202 - days_remaining: i64,
203 - ) {
204 - let alert_key = format!("tls:{target}");
205 - let subject = format!("[PoM] {target}: TLS cert renewed");
206 - let body = format!(
207 - "Target: {label} ({target})\n\
208 - Days remaining: {days_remaining}\n\
209 - Instance: {}\n\
210 - Time: {}\n\n\
211 - - PoM",
212 - self.instance_name,
213 - chrono::Utc::now().to_rfc3339(),
214 - );
215 -
216 - self.fire_recovery(&subject, &body,
217 - AlertMeta { key: &alert_key, category: AlertCategory::TlsRecovery, from: None, to: None, error: None }).await;
218 - }
219 -
220 - #[instrument(skip_all)]
221 - pub async fn send_peer_missing(
222 - &self,
223 - peer_name: &str,
224 - address: &str,
225 - consecutive_failures: u32,
226 - ) {
227 - let alert_key = format!("peer:{peer_name}");
228 - let subject = format!("[PoM] peer {peer_name}: missing");
229 - let body = format!(
230 - "Peer: {peer_name}\n\
231 - Address: {address}\n\
232 - Consecutive failures: {consecutive_failures}\n\
233 - Instance: {}\n\
234 - Time: {}\n\n\
235 - - PoM",
236 - self.instance_name,
237 - chrono::Utc::now().to_rfc3339(),
238 - );
239 -
240 - self.fire_failure(&subject, &body, "high", "pom-peer", Some(peer_name),
241 - AlertMeta { key: &alert_key, category: AlertCategory::PeerMissing, from: None, to: None, error: None }).await;
242 - }
243 -
244 - #[instrument(skip_all)]
245 - pub async fn send_peer_recovery(
246 - &self,
247 - peer_name: &str,
248 - address: &str,
249 - ) {
250 - let subject = format!("[PoM] peer {peer_name}: recovered");
251 - let body = format!(
252 - "Peer: {peer_name}\n\
253 - Address: {address}\n\
254 - Instance: {}\n\
255 - Time: {}\n\n\
256 - - PoM",
257 - self.instance_name,
258 - chrono::Utc::now().to_rfc3339(),
259 - );
260 -
261 - let alert_key = format!("peer:{peer_name}");
262 - self.fire_recovery(&subject, &body,
263 - AlertMeta { key: &alert_key, category: AlertCategory::PeerRecovery, from: None, to: None, error: None }).await;
264 - }
265 -
266 - #[instrument(skip_all)]
267 - pub async fn send_route_failure_alert(
268 - &self,
269 - target: &str,
270 - label: &str,
271 - failed_paths: &[String],
272 - ) {
273 - let alert_key = format!("route:{target}");
274 - let n = failed_paths.len();
275 - let subject = format!("[PoM] {label}: {n} route(s) failing");
276 - let body = format!(
277 - "Target: {label} ({target})\n\
278 - Failed routes:\n{}\n\
279 - Instance: {}\n\
280 - Time: {}\n\n\
281 - - PoM",
282 - failed_paths.iter().map(|p| format!(" - {p}")).collect::<Vec<_>>().join("\n"),
283 - self.instance_name,
284 - chrono::Utc::now().to_rfc3339(),
285 - );
286 -
287 - self.fire_failure(&subject, &body, "high", "pom-routes", Some(target),
288 - AlertMeta { key: &alert_key, category: AlertCategory::RouteFailure, from: None, to: None, error: None }).await;
289 - }
290 -
291 - #[instrument(skip_all)]
292 - pub async fn send_route_recovery_alert(
293 - &self,
294 - target: &str,
295 - label: &str,
296 - recovered_paths: &[String],
297 - ) {
298 - let alert_key = format!("route:{target}");
299 - let subject = format!("[PoM] {label}: routes recovered");
300 - let body = format!(
301 - "Target: {label} ({target})\n\
302 - Recovered routes:\n{}\n\
303 - Instance: {}\n\
304 - Time: {}\n\n\
305 - - PoM",
306 - recovered_paths.iter().map(|p| format!(" - {p}")).collect::<Vec<_>>().join("\n"),
307 - self.instance_name,
308 - chrono::Utc::now().to_rfc3339(),
309 - );
310 -
311 - self.fire_recovery(&subject, &body,
312 - AlertMeta { key: &alert_key, category: AlertCategory::RouteRecovery, from: None, to: None, error: None }).await;
313 - }
314 -
315 - #[instrument(skip_all)]
316 - pub async fn send_dns_mismatch_alert(
317 - &self,
318 - target: &str,
319 - label: &str,
320 - mismatches: &[crate::types::DnsCheckResult],
321 - ) {
322 - let alert_key = format!("dns:{target}");
323 - let n = mismatches.len();
324 - let subject = format!("[PoM] {label}: {n} DNS record(s) mismatched");
325 - let details: Vec<String> = mismatches
326 - .iter()
327 - .map(|m| {
328 - if let Some(ref err) = m.error {
329 - format!(" - {} {}: {err}", m.name, m.record_type)
330 - } else {
331 - format!(
332 - " - {} {}: expected {:?}, got {:?}",
333 - m.name, m.record_type, m.expected, m.actual
334 - )
335 - }
336 - })
337 - .collect();
338 - let body = format!(
339 - "Target: {label} ({target})\n\
340 - DNS mismatches:\n{}\n\
341 - Instance: {}\n\
342 - Time: {}\n\n\
343 - - PoM",
344 - details.join("\n"),
345 - self.instance_name,
346 - chrono::Utc::now().to_rfc3339(),
347 - );
348 -
349 - self.fire_failure(&subject, &body, "high", "pom-dns", Some(target),
350 - AlertMeta { key: &alert_key, category: AlertCategory::DnsMismatch, from: None, to: None, error: None }).await;
351 - }
352 -
353 - #[instrument(skip_all)]
354 - pub async fn send_dns_recovery_alert(
355 - &self,
356 - target: &str,
357 - label: &str,
358 - ) {
359 - let alert_key = format!("dns:{target}");
360 - let subject = format!("[PoM] {label}: DNS records recovered");
361 - let body = format!(
362 - "Target: {label} ({target})\n\
363 - All DNS records now match expected values.\n\
364 - Instance: {}\n\
365 - Time: {}\n\n\
366 - - PoM",
367 - self.instance_name,
368 - chrono::Utc::now().to_rfc3339(),
369 - );
370 -
371 - self.fire_recovery(&subject, &body,
372 - AlertMeta { key: &alert_key, category: AlertCategory::DnsRecovery, from: None, to: None, error: None }).await;
373 - }
374 -
375 - #[instrument(skip_all)]
376 - pub async fn send_whois_expiry_alert(
377 - &self,
378 - target: &str,
379 - label: &str,
380 - domain: &str,
381 - days_remaining: i64,
382 - ) {
383 - let alert_key = format!("whois:{target}");
384 - let subject = format!("[PoM] {label}: domain {domain} expires in {days_remaining} days");
385 - let body = format!(
386 - "Target: {label} ({target})\n\
387 - Domain: {domain}\n\
388 - Days remaining: {days_remaining}\n\
389 - Instance: {}\n\
390 - Time: {}\n\n\
391 - - PoM",
392 - self.instance_name,
393 - chrono::Utc::now().to_rfc3339(),
394 - );
395 -
396 - let priority = whois_expiry_priority(days_remaining);
397 - self.fire_failure(&subject, &body, priority, "pom-whois", Some(&format!("{target}:{domain}")),
398 - AlertMeta { key: &alert_key, category: AlertCategory::WhoisExpiry, from: None, to: None, error: None }).await;
399 - }
400 -
401 - #[instrument(skip_all)]
402 - pub async fn send_whois_error_alert(
403 - &self,
404 - target: &str,
405 - label: &str,
406 - domain: &str,
407 - error: &str,
408 - ) {
409 - let alert_key = format!("whois:{target}");
410 - let subject = format!("[PoM] {label}: WHOIS check failed for {domain}");
411 - let body = format!(
412 - "Target: {label} ({target})\n\
413 - Domain: {domain}\n\
414 - Error: {error}\n\
415 - Instance: {}\n\
416 - Time: {}\n\n\
417 - - PoM",
418 - self.instance_name,
419 - chrono::Utc::now().to_rfc3339(),
420 - );
421 -
422 - self.fire_failure(&subject, &body, "high", "pom-whois", Some(&format!("{target}:{domain}")),
423 - AlertMeta { key: &alert_key, category: AlertCategory::WhoisError, from: None, to: None, error: Some(error) }).await;
424 - }
425 -
426 - #[instrument(skip_all)]
427 - pub async fn send_cors_failure_alert(
428 - &self,
429 - target: &str,
430 - label: &str,
431 - failures: &[crate::types::CorsCheckResult],
432 - ) {
433 - let alert_key = format!("cors:{target}");
434 - let n = failures.len();
435 - let subject = format!("[PoM] {label}: {n} CORS preflight(s) failing");
436 - let details: Vec<String> = failures
437 - .iter()
438 - .map(|f| {
439 - if let Some(ref err) = f.error {
440 - format!(" - {} {} from {}: {err}", f.method, f.url, f.origin)
441 - } else {
442 - format!(" - {} {} from {}: no CORS headers", f.method, f.url, f.origin)
443 - }
444 - })
445 - .collect();
446 - let body = format!(
447 - "Target: {label} ({target})\n\
448 - CORS preflight failures:\n{}\n\
449 - Instance: {}\n\
450 - Time: {}\n\n\
451 - Browser-side uploads will silently fail without CORS.\n\n\
452 - - PoM",
453 - details.join("\n"),
454 - self.instance_name,
455 - chrono::Utc::now().to_rfc3339(),
456 - );
457 -
458 - self.fire_failure(&subject, &body, "high", "pom-cors", Some(target),
459 - AlertMeta { key: &alert_key, category: AlertCategory::CorsFailure, from: None, to: None, error: None }).await;
460 - }
461 -
462 - #[instrument(skip_all)]
463 - pub async fn send_cors_recovery_alert(
464 - &self,
465 - target: &str,
466 - label: &str,
467 - ) {
468 - let alert_key = format!("cors:{target}");
469 - let subject = format!("[PoM] {label}: CORS preflights recovered");
470 - let body = format!(
471 - "Target: {label} ({target})\n\
472 - All CORS preflight checks passing.\n\
473 - Instance: {}\n\
474 - Time: {}\n\n\
475 - - PoM",
476 - self.instance_name,
477 - chrono::Utc::now().to_rfc3339(),
478 - );
479 -
480 - self.fire_recovery(&subject, &body,
481 - AlertMeta { key: &alert_key, category: AlertCategory::CorsRecovery, from: None, to: None, error: None }).await;
482 - }
483 -
484 - #[instrument(skip_all)]
485 - pub async fn send_latency_drift_alert(
486 - &self,
487 - target: &str,
488 - label: &str,
489 - drift_message: &str,
490 - ) {
491 - let alert_key = format!("latency:{target}");
492 - let subject = format!("[PoM] {target}: latency drift detected");
493 - let body = format!(
494 - "Target: {label} ({target})\n\
495 - {drift_message}\n\
496 - Instance: {}\n\
497 - Time: {}\n\n\
498 - - PoM",
499 - self.instance_name,
500 - chrono::Utc::now().to_rfc3339(),
501 - );
502 -
503 - self.fire_failure(&subject, &body, "medium", "pom-latency", Some(target),
504 - AlertMeta { key: &alert_key, category: AlertCategory::LatencyDrift, from: None, to: None, error: Some(drift_message) }).await;
505 - }
506 -
507 - #[instrument(skip_all)]
508 - pub async fn send_latency_recovery(
509 - &self,
510 - target: &str,
511 - label: &str,
512 - ) {
513 - let alert_key = format!("latency:{target}");
514 - let subject = format!("[PoM] {target}: latency recovered");
515 - let body = format!(
516 - "Target: {label} ({target})\n\
517 - Latency returned to normal.\n\
518 - Instance: {}\n\
519 - Time: {}\n\n\
520 - - PoM",
521 - self.instance_name,
522 - chrono::Utc::now().to_rfc3339(),
523 - );
524 -
525 - self.fire_recovery(&subject, &body,
526 - AlertMeta { key: &alert_key, category: AlertCategory::LatencyRecovery, from: None, to: None, error: None }).await;
527 - }
528 -
529 - #[instrument(skip_all)]
530 - pub async fn send_test_duration_drift_alert(
531 - &self,
532 - target: &str,
533 - label: &str,
534 - drift_message: &str,
535 - ) {
536 - let alert_key = format!("test_duration:{target}");
537 - let subject = format!("[PoM] {target}: test duration drift detected");
538 - let body = format!(
539 - "Target: {label} ({target})\n\
540 - {drift_message}\n\
541 - Instance: {}\n\
542 - Time: {}\n\n\
543 - - PoM",
544 - self.instance_name,
545 - chrono::Utc::now().to_rfc3339(),
546 - );
547 -
548 - self.fire_failure(&subject, &body, "medium", "pom-test-duration", Some(target),
549 - AlertMeta { key: &alert_key, category: AlertCategory::TestDurationDrift, from: None, to: None, error: Some(drift_message) }).await;
550 - }
551 -
552 - #[instrument(skip_all)]
553 - pub async fn send_backup_stale_alert(
554 - &self,
555 - target: &str,
556 - label: &str,
557 - database: &str,
558 - status: &str,
559 - age_hours: Option<i64>,
560 - ) {
561 - let alert_key = format!("backup:{target}:{database}");
562 - let detail = backup_status_detail(status, age_hours);
563 -
564 - let subject = format!("[PoM] {label}: {database} backup {status}");
Lines truncated
A pom/src/alerts/backup.rs +62
@@ -1,0 +1,62 @@
1 + //! backup alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_backup_stale_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + database: &str,
15 + status: &str,
16 + age_hours: Option<i64>,
17 + ) {
18 + let alert_key = format!("backup:{target}:{database}");
19 + let detail = backup_status_detail(status, age_hours);
20 +
21 + let subject = format!("[PoM] {label}: {database} backup {status}");
22 + let body = format!(
23 + "Target: {label} ({target})\n\
24 + Database: {database}\n\
25 + Status: {status}\n\
26 + Detail: {detail}\n\
27 + Instance: {}\n\
28 + Time: {}\n\n\
29 + - PoM",
30 + self.instance_name,
31 + chrono::Utc::now().to_rfc3339(),
32 + );
33 +
34 + let priority = backup_status_priority(status);
35 + self.fire_failure(&subject, &body, priority, "pom-backup", Some(&format!("{target}:{database}")),
36 + AlertMeta { key: &alert_key, category: AlertCategory::BackupStale, from: None, to: Some(status), error: None }).await;
37 + }
38 +
39 + #[instrument(skip_all)]
40 + pub async fn send_backup_recovery(
41 + &self,
42 + target: &str,
43 + label: &str,
44 + database: &str,
45 + ) {
46 + let alert_key = format!("backup:{target}:{database}");
47 + let subject = format!("[PoM] {label}: {database} backup recovered");
48 + let body = format!(
49 + "Target: {label} ({target})\n\
50 + Database: {database}\n\
51 + Backup is now current.\n\
52 + Instance: {}\n\
53 + Time: {}\n\n\
54 + - PoM",
55 + self.instance_name,
56 + chrono::Utc::now().to_rfc3339(),
57 + );
58 +
59 + self.fire_recovery(&subject, &body,
60 + AlertMeta { key: &alert_key, category: AlertCategory::BackupRecovery, from: None, to: Some("ok"), error: None }).await;
61 + }
62 + }
A pom/src/alerts/cors.rs +66
@@ -1,0 +1,66 @@
1 + //! cors alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_cors_failure_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + failures: &[crate::types::CorsCheckResult],
15 + ) {
16 + let alert_key = format!("cors:{target}");
17 + let n = failures.len();
18 + let subject = format!("[PoM] {label}: {n} CORS preflight(s) failing");
19 + let details: Vec<String> = failures
20 + .iter()
21 + .map(|f| {
22 + if let Some(ref err) = f.error {
23 + format!(" - {} {} from {}: {err}", f.method, f.url, f.origin)
24 + } else {
25 + format!(" - {} {} from {}: no CORS headers", f.method, f.url, f.origin)
26 + }
27 + })
28 + .collect();
29 + let body = format!(
30 + "Target: {label} ({target})\n\
31 + CORS preflight failures:\n{}\n\
32 + Instance: {}\n\
33 + Time: {}\n\n\
34 + Browser-side uploads will silently fail without CORS.\n\n\
35 + - PoM",
36 + details.join("\n"),
37 + self.instance_name,
38 + chrono::Utc::now().to_rfc3339(),
39 + );
40 +
41 + self.fire_failure(&subject, &body, "high", "pom-cors", Some(target),
42 + AlertMeta { key: &alert_key, category: AlertCategory::CorsFailure, from: None, to: None, error: None }).await;
43 + }
44 +
45 + #[instrument(skip_all)]
46 + pub async fn send_cors_recovery_alert(
47 + &self,
48 + target: &str,
49 + label: &str,
50 + ) {
51 + let alert_key = format!("cors:{target}");
52 + let subject = format!("[PoM] {label}: CORS preflights recovered");
53 + let body = format!(
54 + "Target: {label} ({target})\n\
55 + All CORS preflight checks passing.\n\
56 + Instance: {}\n\
57 + Time: {}\n\n\
58 + - PoM",
59 + self.instance_name,
60 + chrono::Utc::now().to_rfc3339(),
61 + );
62 +
63 + self.fire_recovery(&subject, &body,
64 + AlertMeta { key: &alert_key, category: AlertCategory::CorsRecovery, from: None, to: None, error: None }).await;
65 + }
66 + }
A pom/src/alerts/dns.rs +68
@@ -1,0 +1,68 @@
1 + //! dns alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_dns_mismatch_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + mismatches: &[crate::types::DnsCheckResult],
15 + ) {
16 + let alert_key = format!("dns:{target}");
17 + let n = mismatches.len();
18 + let subject = format!("[PoM] {label}: {n} DNS record(s) mismatched");
19 + let details: Vec<String> = mismatches
20 + .iter()
21 + .map(|m| {
22 + if let Some(ref err) = m.error {
23 + format!(" - {} {}: {err}", m.name, m.record_type)
24 + } else {
25 + format!(
26 + " - {} {}: expected {:?}, got {:?}",
27 + m.name, m.record_type, m.expected, m.actual
28 + )
29 + }
30 + })
31 + .collect();
32 + let body = format!(
33 + "Target: {label} ({target})\n\
34 + DNS mismatches:\n{}\n\
35 + Instance: {}\n\
36 + Time: {}\n\n\
37 + - PoM",
38 + details.join("\n"),
39 + self.instance_name,
40 + chrono::Utc::now().to_rfc3339(),
41 + );
42 +
43 + self.fire_failure(&subject, &body, "high", "pom-dns", Some(target),
44 + AlertMeta { key: &alert_key, category: AlertCategory::DnsMismatch, from: None, to: None, error: None }).await;
45 + }
46 +
47 + #[instrument(skip_all)]
48 + pub async fn send_dns_recovery_alert(
49 + &self,
50 + target: &str,
51 + label: &str,
52 + ) {
53 + let alert_key = format!("dns:{target}");
54 + let subject = format!("[PoM] {label}: DNS records recovered");
55 + let body = format!(
56 + "Target: {label} ({target})\n\
57 + All DNS records now match expected values.\n\
58 + Instance: {}\n\
59 + Time: {}\n\n\
60 + - PoM",
61 + self.instance_name,
62 + chrono::Utc::now().to_rfc3339(),
63 + );
64 +
65 + self.fire_recovery(&subject, &body,
66 + AlertMeta { key: &alert_key, category: AlertCategory::DnsRecovery, from: None, to: None, error: None }).await;
67 + }
68 + }
A pom/src/alerts/health.rs +60
@@ -1,0 +1,60 @@
1 + //! health alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_health_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + from_status: &str,
15 + to_status: &str,
16 + error: Option<&str>,
17 + ) {
18 + let alert_key = format!("health:{target}");
19 + let subject = format!("[PoM] {target}: {from_status} -> {to_status}");
20 + let mut body = format!(
21 + "Target: {label} ({target})\n\
22 + Status: {from_status} -> {to_status}\n\
23 + Instance: {}\n\
24 + Time: {}\n",
25 + self.instance_name,
26 + chrono::Utc::now().to_rfc3339(),
27 + );
28 + if let Some(err) = error {
29 + body.push_str(&format!("Error: {err}\n"));
30 + }
31 + body.push_str("\n- PoM");
32 +
33 + let priority = health_status_priority(to_status);
34 + self.fire_failure(&subject, &body, priority, "pom-health", Some(target),
35 + AlertMeta { key: &alert_key, category: AlertCategory::Health, from: Some(from_status), to: Some(to_status), error }).await;
36 + }
37 +
38 + #[instrument(skip_all)]
39 + pub async fn send_health_recovery(
40 + &self,
41 + target: &str,
42 + label: &str,
43 + from_status: &str,
44 + ) {
45 + let alert_key = format!("health:{target}");
46 + let subject = format!("[PoM] {target}: recovered");
47 + let body = format!(
48 + "Target: {label} ({target})\n\
49 + Status: {from_status} -> operational\n\
50 + Instance: {}\n\
51 + Time: {}\n\n\
52 + - PoM",
53 + self.instance_name,
54 + chrono::Utc::now().to_rfc3339(),
55 + );
56 +
57 + self.fire_recovery(&subject, &body,
58 + AlertMeta { key: &alert_key, category: AlertCategory::Recovery, from: Some(from_status), to: Some("operational"), error: None }).await;
59 + }
60 + }
A pom/src/alerts/latency.rs +53
@@ -1,0 +1,53 @@
1 + //! latency alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_latency_drift_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + drift_message: &str,
15 + ) {
16 + let alert_key = format!("latency:{target}");
17 + let subject = format!("[PoM] {target}: latency drift detected");
18 + let body = format!(
19 + "Target: {label} ({target})\n\
20 + {drift_message}\n\
21 + Instance: {}\n\
22 + Time: {}\n\n\
23 + - PoM",
24 + self.instance_name,
25 + chrono::Utc::now().to_rfc3339(),
26 + );
27 +
28 + self.fire_failure(&subject, &body, "medium", "pom-latency", Some(target),
29 + AlertMeta { key: &alert_key, category: AlertCategory::LatencyDrift, from: None, to: None, error: Some(drift_message) }).await;
30 + }
31 +
32 + #[instrument(skip_all)]
33 + pub async fn send_latency_recovery(
34 + &self,
35 + target: &str,
36 + label: &str,
37 + ) {
38 + let alert_key = format!("latency:{target}");
39 + let subject = format!("[PoM] {target}: latency recovered");
40 + let body = format!(
41 + "Target: {label} ({target})\n\
42 + Latency returned to normal.\n\
43 + Instance: {}\n\
44 + Time: {}\n\n\
45 + - PoM",
46 + self.instance_name,
47 + chrono::Utc::now().to_rfc3339(),
48 + );
49 +
50 + self.fire_recovery(&subject, &body,
51 + AlertMeta { key: &alert_key, category: AlertCategory::LatencyRecovery, from: None, to: None, error: None }).await;
52 + }
53 + }
A pom/src/alerts/offline.rs +46
@@ -1,0 +1,46 @@
1 + //! offline alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + /// All monitored targets are unreachable — likely a network issue with PoM itself.
10 + #[instrument(skip_all)]
11 + pub async fn send_monitoring_offline_alert(&self, target_count: usize) {
12 + let alert_key = "monitoring:self";
13 + let subject = format!("[PoM] all {target_count} targets unreachable");
14 + let body = format!(
15 + "All {target_count} monitored targets are non-operational.\n\
16 + This likely indicates a network issue with the PoM instance itself,\n\
17 + not an actual outage of all targets.\n\n\
18 + Instance: {}\n\
19 + Time: {}\n\n\
20 + - PoM",
21 + self.instance_name,
22 + chrono::Utc::now().to_rfc3339(),
23 + );
24 +
25 + self.fire_failure(&subject, &body, "critical", "pom-monitoring", Some("self"),
26 + AlertMeta { key: alert_key, category: AlertCategory::MonitoringOffline, from: None, to: None, error: None }).await;
27 + }
28 +
29 + /// At least one target is reachable again after a monitoring-offline event.
30 + #[instrument(skip_all)]
31 + pub async fn send_monitoring_recovery(&self) {
32 + let alert_key = "monitoring:self";
33 + let subject = "[PoM] monitoring recovered".to_string();
34 + let body = format!(
35 + "At least one target is reachable again.\n\
36 + Instance: {}\n\
37 + Time: {}\n\n\
38 + - PoM",
39 + self.instance_name,
40 + chrono::Utc::now().to_rfc3339(),
41 + );
42 +
43 + self.fire_recovery(&subject, &body,
44 + AlertMeta { key: alert_key, category: AlertCategory::MonitoringRecovery, from: None, to: None, error: None }).await;
45 + }
46 + }
A pom/src/alerts/peer.rs +54
@@ -1,0 +1,54 @@
1 + //! peer alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_peer_missing(
11 + &self,
12 + peer_name: &str,
13 + address: &str,
14 + consecutive_failures: u32,
15 + ) {
16 + let alert_key = format!("peer:{peer_name}");
17 + let subject = format!("[PoM] peer {peer_name}: missing");
18 + let body = format!(
19 + "Peer: {peer_name}\n\
20 + Address: {address}\n\
21 + Consecutive failures: {consecutive_failures}\n\
22 + Instance: {}\n\
23 + Time: {}\n\n\
24 + - PoM",
25 + self.instance_name,
26 + chrono::Utc::now().to_rfc3339(),
27 + );
28 +
29 + self.fire_failure(&subject, &body, "high", "pom-peer", Some(peer_name),
30 + AlertMeta { key: &alert_key, category: AlertCategory::PeerMissing, from: None, to: None, error: None }).await;
31 + }
32 +
33 + #[instrument(skip_all)]
34 + pub async fn send_peer_recovery(
35 + &self,
36 + peer_name: &str,
37 + address: &str,
38 + ) {
39 + let subject = format!("[PoM] peer {peer_name}: recovered");
40 + let body = format!(
41 + "Peer: {peer_name}\n\
42 + Address: {address}\n\
43 + Instance: {}\n\
44 + Time: {}\n\n\
45 + - PoM",
46 + self.instance_name,
47 + chrono::Utc::now().to_rfc3339(),
48 + );
49 +
50 + let alert_key = format!("peer:{peer_name}");
51 + self.fire_recovery(&subject, &body,
52 + AlertMeta { key: &alert_key, category: AlertCategory::PeerRecovery, from: None, to: None, error: None }).await;
53 + }
54 + }
A pom/src/alerts/route.rs +57
@@ -1,0 +1,57 @@
1 + //! route alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_route_failure_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + failed_paths: &[String],
15 + ) {
16 + let alert_key = format!("route:{target}");
17 + let n = failed_paths.len();
18 + let subject = format!("[PoM] {label}: {n} route(s) failing");
19 + let body = format!(
20 + "Target: {label} ({target})\n\
21 + Failed routes:\n{}\n\
22 + Instance: {}\n\
23 + Time: {}\n\n\
24 + - PoM",
25 + failed_paths.iter().map(|p| format!(" - {p}")).collect::<Vec<_>>().join("\n"),
26 + self.instance_name,
27 + chrono::Utc::now().to_rfc3339(),
28 + );
29 +
30 + self.fire_failure(&subject, &body, "high", "pom-routes", Some(target),
31 + AlertMeta { key: &alert_key, category: AlertCategory::RouteFailure, from: None, to: None, error: None }).await;
32 + }
33 +
34 + #[instrument(skip_all)]
35 + pub async fn send_route_recovery_alert(
36 + &self,
37 + target: &str,
38 + label: &str,
39 + recovered_paths: &[String],
40 + ) {
41 + let alert_key = format!("route:{target}");
42 + let subject = format!("[PoM] {label}: routes recovered");
43 + let body = format!(
44 + "Target: {label} ({target})\n\
45 + Recovered routes:\n{}\n\
46 + Instance: {}\n\
47 + Time: {}\n\n\
48 + - PoM",
49 + recovered_paths.iter().map(|p| format!(" - {p}")).collect::<Vec<_>>().join("\n"),
50 + self.instance_name,
51 + chrono::Utc::now().to_rfc3339(),
52 + );
53 +
54 + self.fire_recovery(&subject, &body,
55 + AlertMeta { key: &alert_key, category: AlertCategory::RouteRecovery, from: None, to: None, error: None }).await;
56 + }
57 + }
A pom/src/alerts/scan.rs +58
@@ -1,0 +1,58 @@
1 + //! scan alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + /// Fire when the scan pipeline transitions from operational → degraded or
10 + /// unreachable. Includes the audit-doc threshold issues that fired.
11 + #[instrument(skip_all)]
12 + pub async fn send_scan_pipeline_alert(
13 + &self,
14 + target: &str,
15 + label: &str,
16 + status: &str,
17 + issues: &[String],
18 + ) {
19 + let alert_key = format!("scan_pipeline:{target}");
20 + let subject = format!("[PoM] {label}: scan pipeline {status}");
21 + let body = format!(
22 + "Target: {label} ({target})\n\
23 + Status: {status}\n\
24 + Issues:\n{}\n\
25 + Instance: {}\n\
26 + Time: {}\n\n\
27 + Dashboard: <https://{}/admin/uploads>\n\n\
28 + - PoM",
29 + issues.iter().map(|i| format!(" - {i}")).collect::<Vec<_>>().join("\n"),
30 + self.instance_name,
31 + chrono::Utc::now().to_rfc3339(),
32 + target,
33 + );
34 +
35 + let priority = if status == "unreachable" { "high" } else { "medium" };
36 + self.fire_failure(&subject, &body, priority, "pom-scan-pipeline", Some(target),
37 + AlertMeta { key: &alert_key, category: AlertCategory::ScanPipelineDegraded, from: None, to: Some(status), error: None }).await;
38 + }
39 +
40 + /// Fire on recovery from a degraded / unreachable scan-pipeline state.
41 + #[instrument(skip_all)]
42 + pub async fn send_scan_pipeline_recovery(&self, target: &str, label: &str) {
43 + let alert_key = format!("scan_pipeline:{target}");
44 + let subject = format!("[PoM] {label}: scan pipeline recovered");
45 + let body = format!(
46 + "Target: {label} ({target})\n\
47 + Scan pipeline is operational.\n\
48 + Instance: {}\n\
49 + Time: {}\n\n\
50 + - PoM",
51 + self.instance_name,
52 + chrono::Utc::now().to_rfc3339(),
53 + );
54 +
55 + self.fire_recovery(&subject, &body,
56 + AlertMeta { key: &alert_key, category: AlertCategory::ScanPipelineRecovery, from: None, to: Some("operational"), error: None }).await;
57 + }
58 + }
A pom/src/alerts/test_duration.rs +31
@@ -1,0 +1,31 @@
1 + //! test_duration alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_test_duration_drift_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + drift_message: &str,
15 + ) {
16 + let alert_key = format!("test_duration:{target}");
17 + let subject = format!("[PoM] {target}: test duration drift detected");
18 + let body = format!(
19 + "Target: {label} ({target})\n\
20 + {drift_message}\n\
21 + Instance: {}\n\
22 + Time: {}\n\n\
23 + - PoM",
24 + self.instance_name,
25 + chrono::Utc::now().to_rfc3339(),
26 + );
27 +
28 + self.fire_failure(&subject, &body, "medium", "pom-test-duration", Some(target),
29 + AlertMeta { key: &alert_key, category: AlertCategory::TestDurationDrift, from: None, to: None, error: Some(drift_message) }).await;
30 + }
31 + }
A pom/src/alerts/tls.rs +82
@@ -1,0 +1,82 @@
1 + //! tls alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_tls_expiry_alert(
11 + &self,
12 + target: &str,
13 + host: &str,
14 + days_remaining: i64,
15 + not_after: &str,
16 + ) {
17 + let alert_key = format!("tls:{target}");
18 + let subject = format!("[PoM] {target}: TLS cert expires in {days_remaining} days");
19 + let body = format!(
20 + "Target: {target}\n\
21 + Host: {host}\n\
22 + Days remaining: {days_remaining}\n\
23 + Expires: {not_after}\n\
24 + Instance: {}\n\
25 + Time: {}\n\n\
26 + - PoM",
27 + self.instance_name,
28 + chrono::Utc::now().to_rfc3339(),
29 + );
30 +
31 + let priority = tls_expiry_priority(days_remaining);
32 + self.fire_failure(&subject, &body, priority, "pom-tls", Some(&format!("{target}:{host}")),
33 + AlertMeta { key: &alert_key, category: AlertCategory::TlsExpiry, from: None, to: None, error: None }).await;
34 + }
35 +
36 + #[instrument(skip_all)]
37 + pub async fn send_tls_error_alert(
38 + &self,
39 + target: &str,
40 + host: &str,
41 + error: &str,
42 + ) {
43 + let alert_key = format!("tls:{target}");
44 + let subject = format!("[PoM] {target}: TLS check failed");
45 + let body = format!(
46 + "Target: {target}\n\
47 + Host: {host}\n\
48 + Error: {error}\n\
49 + Instance: {}\n\
50 + Time: {}\n\n\
51 + - PoM",
52 + self.instance_name,
53 + chrono::Utc::now().to_rfc3339(),
54 + );
55 +
56 + self.fire_failure(&subject, &body, "high", "pom-tls", Some(&format!("{target}:{host}")),
57 + AlertMeta { key: &alert_key, category: AlertCategory::TlsError, from: None, to: None, error: Some(error) }).await;
58 + }
59 +
60 + #[instrument(skip_all)]
61 + pub async fn send_tls_recovery(
62 + &self,
63 + target: &str,
64 + label: &str,
65 + days_remaining: i64,
66 + ) {
67 + let alert_key = format!("tls:{target}");
68 + let subject = format!("[PoM] {target}: TLS cert renewed");
69 + let body = format!(
70 + "Target: {label} ({target})\n\
71 + Days remaining: {days_remaining}\n\
72 + Instance: {}\n\
73 + Time: {}\n\n\
74 + - PoM",
75 + self.instance_name,
76 + chrono::Utc::now().to_rfc3339(),
77 + );
78 +
79 + self.fire_recovery(&subject, &body,
80 + AlertMeta { key: &alert_key, category: AlertCategory::TlsRecovery, from: None, to: None, error: None }).await;
81 + }
82 + }
A pom/src/alerts/whois.rs +59
@@ -1,0 +1,59 @@
1 + //! whois alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 +
4 + use tracing::instrument;
5 +
6 + use super::*;
7 +
8 + impl Alerter {
9 + #[instrument(skip_all)]
10 + pub async fn send_whois_expiry_alert(
11 + &self,
12 + target: &str,
13 + label: &str,
14 + domain: &str,
15 + days_remaining: i64,
16 + ) {
17 + let alert_key = format!("whois:{target}");
18 + let subject = format!("[PoM] {label}: domain {domain} expires in {days_remaining} days");
19 + let body = format!(
20 + "Target: {label} ({target})\n\
21 + Domain: {domain}\n\
22 + Days remaining: {days_remaining}\n\
23 + Instance: {}\n\
24 + Time: {}\n\n\
25 + - PoM",
26 + self.instance_name,
27 + chrono::Utc::now().to_rfc3339(),
28 + );
29 +
30 + let priority = whois_expiry_priority(days_remaining);
31 + self.fire_failure(&subject, &body, priority, "pom-whois", Some(&format!("{target}:{domain}")),
32 + AlertMeta { key: &alert_key, category: AlertCategory::WhoisExpiry, from: None, to: None, error: None }).await;
33 + }
34 +
35 + #[instrument(skip_all)]
36 + pub async fn send_whois_error_alert(
37 + &self,
38 + target: &str,
39 + label: &str,
40 + domain: &str,
41 + error: &str,
42 + ) {
43 + let alert_key = format!("whois:{target}");
44 + let subject = format!("[PoM] {label}: WHOIS check failed for {domain}");
45 + let body = format!(
46 + "Target: {label} ({target})\n\
47 + Domain: {domain}\n\
48 + Error: {error}\n\
49 + Instance: {}\n\
50 + Time: {}\n\n\
51 + - PoM",
52 + self.instance_name,
53 + chrono::Utc::now().to_rfc3339(),
54 + );
55 +
56 + self.fire_failure(&subject, &body, "high", "pom-whois", Some(&format!("{target}:{domain}")),
57 + AlertMeta { key: &alert_key, category: AlertCategory::WhoisError, from: None, to: None, error: Some(error) }).await;
58 + }
59 + }