Skip to main content

max / makenotwork

40.8 KB · 1140 lines History Blame Raw
1 //! Email alerting via Postmark API.
2 //!
3 //! Sends alerts on health status transitions and peer disappearance/recovery.
4 //! If no `postmark_token` is configured, alerts are logged to stdout instead.
5
6 use sqlx::SqlitePool;
7 use tracing::{info, warn};
8
9 use crate::config::AlertConfig;
10 use crate::db;
11 use crate::types::AlertCategory;
12
13 mod backup;
14 mod cors;
15 mod dns;
16 mod health;
17 mod latency;
18 mod offline;
19 mod peer;
20 mod route;
21 mod scan;
22 mod test_duration;
23 mod tls;
24 mod whois;
25
26 /// WAM ticket priority for a transition into a non-operational health status.
27 fn health_status_priority(to_status: &str) -> &'static str {
28 match to_status {
29 "error" | "unreachable" => "critical",
30 "degraded" => "high",
31 _ => "medium",
32 }
33 }
34
35 /// WAM ticket priority for a TLS certificate that expires in `days`.
36 fn tls_expiry_priority(days: i64) -> &'static str {
37 if days <= 3 {
38 "critical"
39 } else if days <= 7 {
40 "high"
41 } else {
42 "medium"
43 }
44 }
45
46 /// WAM ticket priority for a domain registration that expires in `days`.
47 fn whois_expiry_priority(days: i64) -> &'static str {
48 if days <= 7 {
49 "critical"
50 } else if days <= 14 {
51 "high"
52 } else {
53 "medium"
54 }
55 }
56
57 /// WAM ticket priority for a stale/missing/error backup status.
58 fn backup_status_priority(status: &str) -> &'static str {
59 if status == "missing" {
60 "critical"
61 } else {
62 "high"
63 }
64 }
65
66 /// Human-readable detail text for a backup status alert.
67 fn backup_status_detail(status: &str, age_hours: Option<i64>) -> String {
68 match (status, age_hours) {
69 ("stale", Some(hours)) => format!("last backup is {hours}h old"),
70 ("missing", _) => "no backup files found".to_string(),
71 ("error", _) => "backup check failed".to_string(),
72 _ => format!("status: {status}"),
73 }
74 }
75
76 /// Map PoM's fine-grained [`AlertCategory`] onto MNW's domain-level alert kind
77 /// (the `AlertKind` enum in the MNW server's `db/admin_alerts.rs`). Recoveries
78 /// and sub-conditions fold onto their domain; the failure/recovery distinction
79 /// rides on severity, not kind. A new PoM category needs a line here, and, if
80 /// it introduces a new domain, a matching MNW `AlertKind` variant (else MNW
81 /// rejects the push with 422).
82 fn mnw_kind(category: AlertCategory) -> &'static str {
83 use AlertCategory::{
84 BackupRecovery, BackupStale, CorsFailure, CorsRecovery, DnsMismatch, DnsRecovery, Health,
85 LatencyDrift, LatencyRecovery, MonitoringOffline, MonitoringRecovery, PeerMissing,
86 PeerRecovery, Recovery, RouteFailure, RouteRecovery, ScanPipelineDegraded,
87 ScanPipelineRecovery, TestDurationDrift, TlsError, TlsExpiry, TlsRecovery, WhoisError,
88 WhoisExpiry,
89 };
90 match category {
91 Health | Recovery => "health",
92 TlsExpiry | TlsError | TlsRecovery => "tls",
93 DnsMismatch | DnsRecovery => "dns",
94 WhoisExpiry | WhoisError => "whois",
95 LatencyDrift | LatencyRecovery | TestDurationDrift => "latency",
96 CorsFailure | CorsRecovery => "cors",
97 BackupStale | BackupRecovery => "backup",
98 PeerMissing | PeerRecovery => "peer",
99 RouteFailure | RouteRecovery => "route",
100 ScanPipelineDegraded | ScanPipelineRecovery => "scan",
101 MonitoringOffline | MonitoringRecovery => "monitoring",
102 }
103 }
104
105 /// Map PoM's WAM priority string onto MNW's three-level severity. Recoveries do
106 /// not carry a priority; the caller passes `"info"` for those directly.
107 fn mnw_severity(priority: &str) -> &'static str {
108 match priority {
109 "critical" => "critical",
110 "high" | "medium" => "warning",
111 _ => "info",
112 }
113 }
114
115 /// Truncate `s` to at most `max` bytes on a char boundary. MNW caps title at
116 /// 200 and body at 5000; over-long input would be a 422, so clamp instead.
117 fn truncate(s: &str, max: usize) -> String {
118 if s.len() <= max {
119 return s.to_string();
120 }
121 let mut end = max;
122 while !s.is_char_boundary(end) {
123 end -= 1;
124 }
125 s[..end].to_string()
126 }
127
128 #[derive(Clone)]
129 pub struct Alerter {
130 config: AlertConfig,
131 client: reqwest::Client,
132 pool: SqlitePool,
133 instance_name: String,
134 wam_url: Option<String>,
135 }
136
137 /// The record-context an alert carries: what to write to the `alerts` ledger on
138 /// successful delivery, and what to persist for retry on failure.
139 struct AlertMeta<'a> {
140 key: &'a str,
141 category: AlertCategory,
142 from: Option<&'a str>,
143 to: Option<&'a str>,
144 error: Option<&'a str>,
145 }
146
147 /// `true` if `sent_at` (rfc3339) is within `cooldown_secs` of now. An unparseable
148 /// timestamp is treated as expired (not in cooldown) so a bad row can never wedge
149 /// alerting shut.
150 fn within_cooldown_secs(sent_at: &str, cooldown_secs: u64) -> bool {
151 match chrono::DateTime::parse_from_rfc3339(sent_at) {
152 Ok(dt) => chrono::Utc::now().signed_duration_since(dt).num_seconds() < cooldown_secs as i64,
153 Err(_) => false,
154 }
155 }
156
157 impl Alerter {
158 pub fn new(
159 config: AlertConfig,
160 pool: SqlitePool,
161 instance_name: String,
162 ) -> Result<Self, reqwest::Error> {
163 // reqwest 0.13's builder can fail (platform trust-store load) and
164 // `Client::new()`/`unwrap_or_default()` panic on that same failure. Return
165 // the error so the caller can disable alerting rather than crash at startup.
166 let client = crate::tls::https_client_builder()
167 .timeout(std::time::Duration::from_secs(10))
168 .build()?;
169 let wam_url = config.wam_url.clone();
170 if wam_url.is_none() {
171 warn!(
172 "alerts: wam_url not configured, failure alerts will be delivered by email \
173 instead of WAM tickets (set alerts.wam_url to route them to WAM)"
174 );
175 }
176 match (&config.mnw_url, &config.alerts_ingest_token) {
177 (Some(_), Some(_)) => {
178 info!("alerts: MNW sink enabled, alerts also push to the MNW operator log");
179 }
180 (Some(_), None) => warn!(
181 "alerts: mnw_url set but alerts_ingest_token missing, MNW sink disabled \
182 (set alerts.alerts_ingest_token or POM_ALERTS_INGEST_TOKEN)"
183 ),
184 _ => {}
185 }
186 Ok(Self {
187 config,
188 client,
189 pool,
190 instance_name,
191 wam_url,
192 })
193 }
194
195 /// Push one alert to MNW's operator log (`POST /api/internal/alerts`).
196 /// Returns `true` iff MNW accepted it. Returns `false`: never enqueues,
197 /// when the sink is unconfigured; callers gate on config before treating a
198 /// `false` as a delivery miss worth queuing. Bounded by a 10s timeout so MNW
199 /// latency can't stall the probe loop. The server dedups on `dedup_key`
200 /// (`{kind}:{target}`), so a retried or repeated push of one condition
201 /// collapses rather than duplicating.
202 async fn deliver_mnw(
203 &self,
204 kind: &str,
205 key: &str,
206 severity: &str,
207 subject: &str,
208 body: &str,
209 ) -> bool {
210 let (Some(base), Some(token)) = (
211 self.config.mnw_url.as_deref(),
212 self.config.alerts_ingest_token.as_deref(),
213 ) else {
214 return false;
215 };
216 let payload = serde_json::json!({
217 "source": "pom",
218 "kind": kind,
219 "severity": severity,
220 "title": truncate(subject, 200),
221 "body": truncate(body, 5000),
222 "dedup_key": truncate(&format!("{kind}:{key}"), 200),
223 "details": { "instance": self.instance_name, "target": key },
224 });
225 let url = format!("{}/api/internal/alerts", base.trim_end_matches('/'));
226 let send_fut = self
227 .client
228 .post(&url)
229 .bearer_auth(token)
230 .json(&payload)
231 .send();
232 match tokio::time::timeout(std::time::Duration::from_secs(10), send_fut).await {
233 Ok(Ok(resp)) if resp.status().is_success() => {
234 info!("mnw alert pushed: {kind}/{severity} {key}");
235 true
236 }
237 Ok(Ok(resp)) => {
238 let status = resp.status();
239 let text = resp.text().await.unwrap_or_default();
240 warn!("mnw alert push failed ({status}): {text}");
241 false
242 }
243 Ok(Err(e)) => {
244 warn!("mnw alert push error: {e}");
245 false
246 }
247 Err(_) => {
248 warn!("mnw alert push timed out: {kind}/{key}");
249 false
250 }
251 }
252 }
253
254 /// Durable push to the MNW operator log: deliver now, and on failure enqueue
255 /// for retry through the same `pending_alerts` queue WAM/email use. No-op
256 /// unless both `mnw_url` and `alerts_ingest_token` are set.
257 ///
258 /// MNW is a *secondary* sink: the primary channel (WAM/email) already
259 /// recorded this alert in the ledger, so a queued MNW row carries
260 /// `channel = "mnw"` and, unlike WAM/email retries, does not re-record on
261 /// delivery (see [`retry_pending`]), it only needs to eventually land.
262 /// `severity` derives from `priority` exactly as the retry path recomputes
263 /// it (`None` → recovery → `"info"`), so live and retried pushes match.
264 async fn push_mnw(
265 &self,
266 category: AlertCategory,
267 key: &str,
268 priority: Option<&str>,
269 subject: &str,
270 body: &str,
271 ) {
272 if self.config.mnw_url.is_none() || self.config.alerts_ingest_token.is_none() {
273 return;
274 }
275 let kind = mnw_kind(category);
276 let severity = priority.map_or("info", mnw_severity);
277 if self.deliver_mnw(kind, key, severity, subject, body).await {
278 return;
279 }
280 let category = category.to_string();
281 let pending = db::NewPendingAlert {
282 alert_key: key,
283 category: &category,
284 channel: "mnw",
285 subject,
286 body,
287 priority,
288 source: Some("pom"),
289 source_ref: None,
290 from_status: None,
291 to_status: None,
292 error: None,
293 };
294 if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await {
295 warn!("failed to enqueue undelivered mnw alert for retry: {e}");
296 } else {
297 warn!("mnw alert to {key} undelivered; queued for retry");
298 }
299 }
300
301 /// Whether a FAILURE alert for `target` is within its cooldown.
302 ///
303 /// A recorded recovery *newer* than the last failure clears the cooldown: the
304 /// target came back, so a fresh failure is a genuine new outage and must not
305 /// be suppressed by the prior failure's window (fuzz-2026-07-06 fail→recover→
306 /// fail suppression).
307 async fn is_within_cooldown(&self, target: &str) -> bool {
308 let Ok(Some(latest_fail)) = db::get_latest_alert_for_target(&self.pool, target).await
309 else {
310 return false; // never alerted → not in cooldown
311 };
312 // A recovery after the last failure voids the cooldown.
313 if let Ok(Some(rec)) = db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
314 && rec.id > latest_fail.id
315 {
316 return false;
317 }
318 within_cooldown_secs(&latest_fail.sent_at, self.config.cooldown_secs)
319 }
320
321 /// Whether a RECOVERY for `target` is within its own cooldown, throttles a
322 /// flapping target so it can't emit one recovery email per flap
323 /// (fuzz-2026-07-06 un-throttled recoveries). A failure newer than the last
324 /// recovery clears it, so a genuine recover-after-a-new-outage still sends.
325 async fn is_recovery_within_cooldown(&self, target: &str) -> bool {
326 let Ok(Some(latest_rec)) =
327 db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
328 else {
329 return false;
330 };
331 if let Ok(Some(fail)) = db::get_latest_alert_for_target(&self.pool, target).await
332 && fail.id > latest_rec.id
333 {
334 return false;
335 }
336 within_cooldown_secs(&latest_rec.sent_at, self.config.cooldown_secs)
337 }
338
339 /// Send an email via Postmark. Returns `true` if the message was accepted
340 /// (or in dev mode where there is no token, the operator sees it logged).
341 async fn send_email(&self, subject: &str, body: &str) -> bool {
342 let Some(ref token) = self.config.postmark_token else {
343 info!("[dev] alert: {subject}");
344 info!("[dev] {body}");
345 return true;
346 };
347
348 let payload = serde_json::json!({
349 "From": self.config.from,
350 "To": self.config.to,
351 "Subject": subject,
352 "TextBody": body,
353 });
354
355 let send_fut = self
356 .client
357 .post("https://api.postmarkapp.com/email")
358 .header("X-Postmark-Server-Token", token)
359 .header("Content-Type", "application/json")
360 .header("Accept", "application/json")
361 .json(&payload)
362 .send();
363
364 // Wrap in a 30-second timeout to prevent Postmark latency from blocking
365 // the alert task. The reqwest client has its own 10s timeout, but this
366 // guards against DNS resolution stalls and connection pool exhaustion.
367 match tokio::time::timeout(std::time::Duration::from_secs(30), send_fut).await {
368 Ok(Ok(resp)) if resp.status().is_success() => {
369 info!("alert sent: {subject}");
370 true
371 }
372 Ok(Ok(resp)) => {
373 let status = resp.status();
374 let text = resp.text().await.unwrap_or_default();
375 warn!("postmark error ({status}): {text}");
376 false
377 }
378 Ok(Err(e)) => {
379 warn!("failed to send alert: {e}");
380 false
381 }
382 Err(_) => {
383 warn!("alert send timed out after 30s: {subject}");
384 false
385 }
386 }
387 }
388
389 async fn record_alert(
390 &self,
391 target: &str,
392 alert_type: AlertCategory,
393 from_status: Option<&str>,
394 to_status: Option<&str>,
395 error: Option<&str>,
396 ) {
397 self.record_alert_str(
398 target,
399 &alert_type.to_string(),
400 from_status,
401 to_status,
402 error,
403 )
404 .await;
405 }
406
407 /// String-typed variant so the retry task can record a queued alert straight
408 /// from its stored category without round-tripping through the enum.
409 async fn record_alert_str(
410 &self,
411 target: &str,
412 alert_type: &str,
413 from_status: Option<&str>,
414 to_status: Option<&str>,
415 error: Option<&str>,
416 ) {
417 if let Err(e) = db::insert_alert(
418 &self.pool,
419 target,
420 alert_type,
421 from_status,
422 to_status,
423 error,
424 )
425 .await
426 {
427 warn!("failed to record alert: {e}");
428 }
429 }
430
431 /// Fire a failure alert: cooldown-gate, then dispatch. The single entry point
432 /// every `send_*_alert` funnels through, so the cooldown check + dispatch is
433 /// written once rather than copy-pasted into each of the ~13 methods.
434 async fn fire_failure(
435 &self,
436 subject: &str,
437 body: &str,
438 priority: &str,
439 source: &str,
440 source_ref: Option<&str>,
441 meta: AlertMeta<'_>,
442 ) {
443 if self.is_within_cooldown(meta.key).await {
444 info!("alert cooldown active for {}, skipping", meta.key);
445 return;
446 }
447 // Primary sink (WAM) first so a slow MNW push can't delay the ticket;
448 // capture what push_mnw needs before `meta` is moved into dispatch.
449 let (category, key) = (meta.category, meta.key);
450 self.dispatch_wam(subject, body, priority, source, source_ref, meta)
451 .await;
452 self.push_mnw(category, key, Some(priority), subject, body)
453 .await;
454 }
455
456 /// Fire a recovery: recovery-cooldown-gate, then dispatch by email. The single
457 /// entry point every `send_*_recovery` funnels through.
458 async fn fire_recovery(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
459 if self.is_recovery_within_cooldown(meta.key).await {
460 info!("recovery cooldown active for {}, skipping", meta.key);
461 return;
462 }
463 let (category, key) = (meta.category, meta.key);
464 self.dispatch_email(subject, body, meta).await;
465 self.push_mnw(category, key, None, subject, body).await;
466 }
467
468 /// Dispatch a WAM failure alert: attempt delivery (WAM, falling back to
469 /// email), record it on success, or persist it for retry on failure. Gating
470 /// the ledger write on delivery, and queueing the miss, is what stops a
471 /// transient send failure at a status transition from silencing the whole
472 /// outage: the transition fires once, but the queued alert is retried until it
473 /// lands (fuzz-2026-07-06 CRITICAL #1).
474 async fn dispatch_wam(
475 &self,
476 subject: &str,
477 body: &str,
478 priority: &str,
479 source: &str,
480 source_ref: Option<&str>,
481 meta: AlertMeta<'_>,
482 ) {
483 if self
484 .wam_ticket(subject, body, priority, source, source_ref)
485 .await
486 {
487 self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error)
488 .await;
489 } else {
490 self.enqueue_undelivered(
491 subject,
492 body,
493 "wam",
494 Some(priority),
495 Some(source),
496 source_ref,
497 &meta,
498 )
499 .await;
500 }
501 }
502
503 /// Dispatch an email alert (recoveries, and failure alerts that already chose
504 /// email): record on success, persist for retry on failure.
505 async fn dispatch_email(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
506 if self.send_email(subject, body).await {
507 self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error)
508 .await;
509 } else {
510 self.enqueue_undelivered(subject, body, "email", None, None, None, &meta)
511 .await;
512 }
513 }
514
515 #[allow(clippy::too_many_arguments)]
516 async fn enqueue_undelivered(
517 &self,
518 subject: &str,
519 body: &str,
520 channel: &str,
521 priority: Option<&str>,
522 source: Option<&str>,
523 source_ref: Option<&str>,
524 meta: &AlertMeta<'_>,
525 ) {
526 let category = meta.category.to_string();
527 let pending = db::NewPendingAlert {
528 alert_key: meta.key,
529 category: &category,
530 channel,
531 subject,
532 body,
533 priority,
534 source,
535 source_ref,
536 from_status: meta.from,
537 to_status: meta.to,
538 error: meta.error,
539 };
540 if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await {
541 warn!("failed to enqueue undelivered alert for retry: {e}");
542 } else {
543 warn!("alert to {} undelivered; queued for retry", meta.key);
544 }
545 }
546
547 /// Re-attempt one queued alert. On delivery, record it in the ledger and
548 /// signal the caller to delete the row; otherwise leave it for a later tick.
549 /// Called by the retry task in `serve`.
550 pub async fn retry_pending(&self, p: &db::PendingAlertRow) -> bool {
551 // MNW is a secondary sink: the primary channel already wrote this alert
552 // to the ledger, so a successful retry only clears the queued row, it
553 // must not re-record (that would double-count and skew the cooldown).
554 // Reconstruct kind/severity the same way the live push derives them.
555 if p.channel == "mnw" {
556 let kind = p
557 .category
558 .parse::<AlertCategory>()
559 .map_or("health", mnw_kind);
560 let severity = p.priority.as_deref().map_or("info", mnw_severity);
561 return self
562 .deliver_mnw(kind, &p.alert_key, severity, &p.subject, &p.body)
563 .await;
564 }
565 let delivered = match p.channel.as_str() {
566 "wam" => {
567 self.wam_ticket(
568 &p.subject,
569 &p.body,
570 p.priority.as_deref().unwrap_or("high"),
571 p.source.as_deref().unwrap_or("pom"),
572 p.source_ref.as_deref(),
573 )
574 .await
575 }
576 _ => self.send_email(&p.subject, &p.body).await,
577 };
578 if delivered {
579 self.record_alert_str(
580 &p.alert_key,
581 &p.category,
582 p.from_status.as_deref(),
583 p.to_status.as_deref(),
584 p.error.as_deref(),
585 )
586 .await;
587 }
588 delivered
589 }
590
591 /// Create a WAM ticket for a failure alert. Returns `true` if the alert was
592 /// delivered by *some* channel.
593 ///
594 /// If `wam_url` is unset the failure alert would otherwise be dropped on the
595 /// floor, failures route to WAM, recoveries to email, so a missing `wam_url`
596 /// silently disabled the entire down-alert channel while recovery emails kept
597 /// firing (fuzz-2026-07-06 CRITICAL #2). Fall back to email instead: a
598 /// monitoring tool must never silently swallow a down-alert. WAM delivery
599 /// failures also fall back to email so a WAM outage can't blind the operator.
600 async fn wam_ticket(
601 &self,
602 title: &str,
603 body: &str,
604 priority: &str,
605 source: &str,
606 source_ref: Option<&str>,
607 ) -> bool {
608 let Some(ref base_url) = self.wam_url else {
609 // No WAM configured: deliver the failure alert by email instead of
610 // dropping it.
611 return self.send_email(title, body).await;
612 };
613 let url = format!("{base_url}/tickets");
614
615 let mut payload = serde_json::json!({
616 "title": title,
617 "body": body,
618 "priority": priority,
619 "source": source,
620 });
621 if let Some(r) = source_ref {
622 payload["source_ref"] = serde_json::json!(r);
623 }
624
625 match self.client.post(&url).json(&payload).send().await {
626 Ok(resp) if resp.status().is_success() => {
627 info!("WAM ticket created: {title}");
628 true
629 }
630 Ok(resp) => {
631 warn!(
632 "WAM ticket creation returned {}: {title}; falling back to email",
633 resp.status()
634 );
635 self.send_email(title, body).await
636 }
637 Err(e) => {
638 warn!("WAM unreachable: {e}; falling back to email");
639 self.send_email(title, body).await
640 }
641 }
642 }
643 }
644
645 #[cfg(test)]
646 mod tests {
647 use super::*;
648
649 #[tokio::test]
650 async fn mnw_retry_routes_to_sink_and_never_records() {
651 // A queued mnw row must retry via the MNW sink, not the WAM/email path,
652 // and must NOT write the ledger (the primary channel already did). With
653 // the sink unconfigured (test_alerter has no mnw_url), delivery fails and
654 // the row stays queued, and the cooldown ledger is untouched.
655 let pool = db::connect_in_memory().await.unwrap();
656 let alerter = test_alerter(pool.clone());
657 let row = db::PendingAlertRow {
658 id: 1,
659 alert_key: "makenot.work".to_string(),
660 category: "tls_expiry".to_string(),
661 channel: "mnw".to_string(),
662 subject: "TLS expiring".to_string(),
663 body: "cert expires soon".to_string(),
664 priority: Some("high".to_string()),
665 source: Some("pom".to_string()),
666 source_ref: None,
667 from_status: None,
668 to_status: None,
669 error: None,
670 attempts: 0,
671 };
672 assert!(
673 !alerter.retry_pending(&row).await,
674 "unconfigured sink cannot deliver"
675 );
676 assert!(
677 !alerter.is_within_cooldown("makenot.work").await,
678 "mnw retry must not record to the ledger"
679 );
680 }
681
682 #[test]
683 fn mnw_kind_folds_failure_and_recovery_to_same_domain() {
684 // A condition and its recovery must map to one kind so they share a
685 // dedup thread in the MNW log; the failure/recovery split rides severity.
686 assert_eq!(mnw_kind(AlertCategory::TlsExpiry), "tls");
687 assert_eq!(mnw_kind(AlertCategory::TlsError), "tls");
688 assert_eq!(mnw_kind(AlertCategory::TlsRecovery), "tls");
689 assert_eq!(mnw_kind(AlertCategory::DnsMismatch), "dns");
690 assert_eq!(mnw_kind(AlertCategory::DnsRecovery), "dns");
691 assert_eq!(mnw_kind(AlertCategory::TestDurationDrift), "latency");
692 assert_eq!(mnw_kind(AlertCategory::MonitoringOffline), "monitoring");
693 }
694
695 #[test]
696 fn mnw_severity_collapses_priorities_to_three_levels() {
697 assert_eq!(mnw_severity("critical"), "critical");
698 assert_eq!(mnw_severity("high"), "warning");
699 assert_eq!(mnw_severity("medium"), "warning");
700 assert_eq!(mnw_severity("low"), "info");
701 assert_eq!(mnw_severity("anything-else"), "info");
702 }
703
704 #[test]
705 fn truncate_respects_cap_and_char_boundaries() {
706 assert_eq!(truncate("short", 200), "short");
707 assert_eq!(truncate("abcdef", 3), "abc");
708 // Cutting mid multi-byte char steps back to the previous boundary rather
709 // than panicking on a non-boundary slice.
710 let s = "aé"; // 'é' is two bytes; byte index 2 is not a char boundary
711 assert_eq!(truncate(s, 2), "a");
712 }
713
714 fn test_alerter(pool: SqlitePool) -> Alerter {
715 let config = AlertConfig {
716 postmark_token: None, // dev mode
717 to: "test@example.com".to_string(),
718 from: "PoM Alerts <pom-alerts@makenot.work>".to_string(),
719 cooldown_secs: 300,
720 wam_url: None,
721 mnw_url: None,
722 alerts_ingest_token: None,
723 };
724 Alerter::new(config, pool, "test-instance".to_string()).unwrap()
725 }
726
727 #[tokio::test]
728 async fn cooldown_prevents_duplicate_alerts() {
729 let pool = db::connect_in_memory().await.unwrap();
730 let alerter = test_alerter(pool.clone());
731
732 // First alert, not in cooldown
733 assert!(!alerter.is_within_cooldown("health:mnw").await);
734
735 // Record an alert
736 db::insert_alert(
737 &pool,
738 "health:mnw",
739 "health",
740 Some("operational"),
741 Some("error"),
742 None,
743 )
744 .await
745 .unwrap();
746
747 // Now should be in cooldown
748 assert!(alerter.is_within_cooldown("health:mnw").await);
749 }
750
751 #[tokio::test]
752 async fn wam_ticket_falls_back_to_email_when_wam_url_unset() {
753 // CRITICAL #2: with no wam_url, a failure alert must NOT be silently
754 // dropped, it falls back to email. test_alerter has wam_url: None and
755 // postmark_token: None (dev mode), so the email path reports delivered.
756 let pool = db::connect_in_memory().await.unwrap();
757 let alerter = test_alerter(pool);
758 assert!(
759 alerter
760 .wam_ticket("subj", "body", "high", "pom-test", None)
761 .await,
762 "a failure alert with no wam_url must be delivered by email, not dropped"
763 );
764 }
765
766 #[tokio::test]
767 async fn cooldown_does_not_affect_other_targets() {
768 let pool = db::connect_in_memory().await.unwrap();
769 let alerter = test_alerter(pool.clone());
770
771 db::insert_alert(&pool, "health:mnw", "health", None, None, None)
772 .await
773 .unwrap();
774
775 // Different target should not be in cooldown
776 assert!(!alerter.is_within_cooldown("health:other").await);
777 }
778
779 #[tokio::test]
780 async fn dev_mode_does_not_send_http() {
781 let pool = db::connect_in_memory().await.unwrap();
782 let alerter = test_alerter(pool.clone());
783
784 // This should log instead of making HTTP calls (no panic, no error)
785 alerter
786 .send_health_alert("mnw", "MakeNotWork", "operational", "error", None)
787 .await;
788
789 // Verify alert was recorded in DB with the prefixed key (health:mnw),
790 // matching the cooldown lookup key format.
791 let latest = db::get_latest_alert_for_target(&pool, "health:mnw")
792 .await
793 .unwrap();
794 assert!(latest.is_some());
795 let row = latest.unwrap();
796 assert_eq!(row.alert_type, "health");
797 assert_eq!(row.from_status.as_deref(), Some("operational"));
798 assert_eq!(row.to_status.as_deref(), Some("error"));
799 }
800
801 #[tokio::test]
802 async fn route_alert_cooldown_key() {
803 let pool = db::connect_in_memory().await.unwrap();
804 let alerter = test_alerter(pool.clone());
805
806 assert!(!alerter.is_within_cooldown("route:mnw").await);
807
808 alerter
809 .send_route_failure_alert("mnw", "MakeNotWork", &["/docs/faq".to_string()])
810 .await;
811
812 assert!(alerter.is_within_cooldown("route:mnw").await);
813 assert!(!alerter.is_within_cooldown("route:mt").await);
814 }
815
816 #[tokio::test]
817 async fn recovery_clears_failure_cooldown_so_next_failure_fires() {
818 // Regression for the fail→recover→fail suppression the fuzz flagged (this
819 // test previously enshrined the bug: it asserted the 2nd failure stays
820 // suppressed). A recovery means the target came back, so a fresh failure
821 // is a genuine NEW outage and must not be muffled by the prior failure's
822 // cooldown.
823 let pool = db::connect_in_memory().await.unwrap();
824 let alerter = test_alerter(pool.clone());
825
826 // Fail → cooldown active.
827 alerter
828 .send_health_alert("mnw", "MakeNotWork", "operational", "error", None)
829 .await;
830 assert!(alerter.is_within_cooldown("health:mnw").await);
831
832 // Recover → the failure cooldown is now cleared.
833 alerter
834 .send_health_recovery("mnw", "MakeNotWork", "error")
835 .await;
836 assert!(
837 !alerter.is_within_cooldown("health:mnw").await,
838 "a recovery after the last failure must void the failure cooldown"
839 );
840 }
841
842 #[tokio::test]
843 async fn recovery_is_throttled_by_its_own_cooldown() {
844 // A flapping target must not emit one recovery per flap: after a recorded
845 // recovery, another recovery within cooldown is throttled, until a new
846 // failure intervenes.
847 let pool = db::connect_in_memory().await.unwrap();
848 let alerter = test_alerter(pool.clone());
849
850 db::insert_alert(
851 &pool,
852 "health:mnw",
853 "recovery",
854 None,
855 Some("operational"),
856 None,
857 )
858 .await
859 .unwrap();
860 assert!(alerter.is_recovery_within_cooldown("health:mnw").await);
861
862 // A failure after the recovery clears the recovery cooldown, so the NEXT
863 // genuine recovery still sends.
864 db::insert_alert(
865 &pool,
866 "health:mnw",
867 "health",
868 Some("operational"),
869 Some("error"),
870 None,
871 )
872 .await
873 .unwrap();
874 assert!(!alerter.is_recovery_within_cooldown("health:mnw").await);
875 }
876
877 #[tokio::test]
878 async fn failed_send_enqueues_for_retry_and_retry_delivers() {
879 // CRITICAL #1: a send that fails must not be lost, it is queued, and the
880 // retry path re-delivers it and records it in the ledger.
881 let pool = db::connect_in_memory().await.unwrap();
882 // Force delivery failure: WAM points at an unroutable URL and there is no
883 // postmark token, so the email fallback in dev mode... would succeed. Use a
884 // direct enqueue + retry to exercise the queue deterministically instead.
885 let pending = db::NewPendingAlert {
886 alert_key: "health:mnw",
887 category: "health",
888 channel: "email", // dev-mode email "delivers" (logs) → retry succeeds
889 subject: "[PoM] mnw: down",
890 body: "body",
891 priority: None,
892 source: None,
893 source_ref: None,
894 from_status: Some("operational"),
895 to_status: Some("error"),
896 error: None,
897 };
898 db::enqueue_pending_alert(&pool, &pending).await.unwrap();
899 let due = db::due_pending_alerts(&pool, 10).await.unwrap();
900 assert_eq!(due.len(), 1, "the undelivered alert must be queued");
901
902 let alerter = test_alerter(pool.clone());
903 assert!(
904 alerter.retry_pending(&due[0]).await,
905 "dev-mode email retry delivers"
906 );
907 db::delete_pending_alert(&pool, due[0].id).await.unwrap();
908
909 // The retry recorded the alert in the ledger (so cooldown now applies).
910 assert!(alerter.is_within_cooldown("health:mnw").await);
911 assert!(db::due_pending_alerts(&pool, 10).await.unwrap().is_empty());
912 }
913
914 #[tokio::test]
915 async fn dns_alert_cooldown_key() {
916 let pool = db::connect_in_memory().await.unwrap();
917 let alerter = test_alerter(pool.clone());
918
919 assert!(!alerter.is_within_cooldown("dns:mnw").await);
920
921 let mismatches = vec![crate::types::DnsCheckResult {
922 target: "mnw".to_string(),
923 name: "makenot.work".to_string(),
924 record_type: crate::types::DnsRecordType::A,
925 expected: vec!["1.2.3.4".to_string()],
926 actual: vec!["5.6.7.8".to_string()],
927 matches: false,
928 checked_at: chrono::Utc::now().to_rfc3339(),
929 error: None,
930 }];
931 alerter
932 .send_dns_mismatch_alert("mnw", "MakeNotWork", &mismatches)
933 .await;
934
935 assert!(alerter.is_within_cooldown("dns:mnw").await);
936 assert!(!alerter.is_within_cooldown("dns:other").await);
937 }
938
939 #[tokio::test]
940 async fn whois_alert_cooldown_key() {
941 let pool = db::connect_in_memory().await.unwrap();
942 let alerter = test_alerter(pool.clone());
943
944 assert!(!alerter.is_within_cooldown("whois:mnw").await);
945
946 alerter
947 .send_whois_expiry_alert("mnw", "MakeNotWork", "makenot.work", 15)
948 .await;
949
950 assert!(alerter.is_within_cooldown("whois:mnw").await);
951 assert!(!alerter.is_within_cooldown("whois:other").await);
952 }
953
954 #[tokio::test]
955 async fn health_alert_cooldown_key_matches_record_key() {
956 let pool = db::connect_in_memory().await.unwrap();
957 let alerter = test_alerter(pool.clone());
958
959 // Not in cooldown initially
960 assert!(!alerter.is_within_cooldown("health:example.com").await);
961
962 // Send an alert for "example.com"
963 alerter
964 .send_health_alert("example.com", "Example", "operational", "error", None)
965 .await;
966
967 // Same target should now be in cooldown
968 assert!(alerter.is_within_cooldown("health:example.com").await);
969
970 // Different target should NOT be in cooldown
971 assert!(!alerter.is_within_cooldown("health:other.com").await);
972 }
973
974 #[tokio::test]
975 async fn tls_expiry_alert_cooldown_key() {
976 let pool = db::connect_in_memory().await.unwrap();
977 let alerter = test_alerter(pool.clone());
978
979 assert!(!alerter.is_within_cooldown("tls:mnw").await);
980 alerter
981 .send_tls_expiry_alert("mnw", "makenot.work", 10, "2026-04-01T00:00:00Z")
982 .await;
983 assert!(alerter.is_within_cooldown("tls:mnw").await);
984 }
985
986 #[tokio::test]
987 async fn tls_error_alert_cooldown_key() {
988 let pool = db::connect_in_memory().await.unwrap();
989 let alerter = test_alerter(pool.clone());
990
991 assert!(!alerter.is_within_cooldown("tls:mnw").await);
992 alerter
993 .send_tls_error_alert("mnw", "makenot.work", "certificate expired")
994 .await;
995 assert!(alerter.is_within_cooldown("tls:mnw").await);
996 }
997
998 #[tokio::test]
999 async fn latency_drift_alert_cooldown_key() {
1000 let pool = db::connect_in_memory().await.unwrap();
1001 let alerter = test_alerter(pool.clone());
1002
1003 assert!(!alerter.is_within_cooldown("latency:mnw").await);
1004 alerter
1005 .send_latency_drift_alert("mnw", "MakeNotWork", "avg 500ms, baseline 100ms")
1006 .await;
1007 assert!(alerter.is_within_cooldown("latency:mnw").await);
1008 }
1009
1010 #[tokio::test]
1011 async fn test_duration_drift_alert_cooldown_key() {
1012 let pool = db::connect_in_memory().await.unwrap();
1013 let alerter = test_alerter(pool.clone());
1014
1015 assert!(!alerter.is_within_cooldown("test_duration:mnw").await);
1016 alerter
1017 .send_test_duration_drift_alert("mnw", "MakeNotWork", "drift: 120s vs 60s baseline")
1018 .await;
1019 assert!(alerter.is_within_cooldown("test_duration:mnw").await);
1020 }
1021
1022 #[tokio::test]
1023 async fn monitoring_offline_alert_cooldown_key() {
1024 let pool = db::connect_in_memory().await.unwrap();
1025 let alerter = test_alerter(pool.clone());
1026
1027 assert!(!alerter.is_within_cooldown("monitoring:self").await);
1028 alerter.send_monitoring_offline_alert(3).await;
1029 assert!(alerter.is_within_cooldown("monitoring:self").await);
1030 }
1031
1032 #[tokio::test]
1033 async fn route_recovery_does_not_start_cooldown() {
1034 let pool = db::connect_in_memory().await.unwrap();
1035 let alerter = test_alerter(pool.clone());
1036
1037 alerter
1038 .send_route_recovery_alert("mnw", "MakeNotWork", &["/health".to_string()])
1039 .await;
1040 // Recovery alerts are excluded from cooldown lookups, so sending a recovery
1041 // should NOT put the key into cooldown.
1042 assert!(!alerter.is_within_cooldown("route:mnw").await);
1043 }
1044
1045 #[tokio::test]
1046 async fn dns_recovery_does_not_start_cooldown() {
1047 let pool = db::connect_in_memory().await.unwrap();
1048 let alerter = test_alerter(pool.clone());
1049
1050 alerter.send_dns_recovery_alert("mnw", "MakeNotWork").await;
1051 assert!(!alerter.is_within_cooldown("dns:mnw").await);
1052 }
1053
1054 #[tokio::test]
1055 async fn tls_recovery_does_not_start_cooldown() {
1056 let pool = db::connect_in_memory().await.unwrap();
1057 let alerter = test_alerter(pool.clone());
1058
1059 alerter.send_tls_recovery("mnw", "MakeNotWork", 90).await;
1060 assert!(!alerter.is_within_cooldown("tls:mnw").await);
1061 }
1062
1063 // Pure priority/severity helpers (pin the <= boundaries)
1064
1065 #[test]
1066 fn tls_expiry_priority_boundaries() {
1067 // critical: days <= 3
1068 assert_eq!(
1069 tls_expiry_priority(-5),
1070 "critical",
1071 "negative days = already expired"
1072 );
1073 assert_eq!(tls_expiry_priority(0), "critical");
1074 assert_eq!(tls_expiry_priority(3), "critical");
1075 // high: 4..=7
1076 assert_eq!(tls_expiry_priority(4), "high");
1077 assert_eq!(tls_expiry_priority(7), "high");
1078 // medium: > 7
1079 assert_eq!(tls_expiry_priority(8), "medium");
1080 assert_eq!(tls_expiry_priority(90), "medium");
1081 }
1082
1083 #[test]
1084 fn whois_expiry_priority_boundaries() {
1085 // critical: days <= 7
1086 assert_eq!(whois_expiry_priority(-1), "critical");
1087 assert_eq!(whois_expiry_priority(7), "critical");
1088 // high: 8..=14
1089 assert_eq!(whois_expiry_priority(8), "high");
1090 assert_eq!(whois_expiry_priority(14), "high");
1091 // medium: > 14
1092 assert_eq!(whois_expiry_priority(15), "medium");
1093 assert_eq!(whois_expiry_priority(180), "medium");
1094 }
1095
1096 #[test]
1097 fn backup_status_priority_missing_is_critical() {
1098 assert_eq!(backup_status_priority("missing"), "critical");
1099 assert_eq!(backup_status_priority("stale"), "high");
1100 assert_eq!(backup_status_priority("error"), "high");
1101 assert_eq!(backup_status_priority("anything-else"), "high");
1102 assert_eq!(backup_status_priority(""), "high");
1103 }
1104
1105 #[test]
1106 fn backup_status_detail_arms() {
1107 assert_eq!(
1108 backup_status_detail("stale", Some(12)),
1109 "last backup is 12h old"
1110 );
1111 // `stale` with no age falls through to the default arm.
1112 assert_eq!(backup_status_detail("stale", None), "status: stale");
1113 assert_eq!(
1114 backup_status_detail("missing", None),
1115 "no backup files found"
1116 );
1117 assert_eq!(
1118 backup_status_detail("missing", Some(5)),
1119 "no backup files found"
1120 );
1121 assert_eq!(backup_status_detail("error", None), "backup check failed");
1122 assert_eq!(
1123 backup_status_detail("error", Some(99)),
1124 "backup check failed"
1125 );
1126 assert_eq!(backup_status_detail("weird", None), "status: weird");
1127 }
1128
1129 #[test]
1130 fn health_status_priority_arms() {
1131 assert_eq!(health_status_priority("error"), "critical");
1132 assert_eq!(health_status_priority("unreachable"), "critical");
1133 assert_eq!(health_status_priority("degraded"), "high");
1134 // Anything else (operational, unknown values) falls through to medium.
1135 assert_eq!(health_status_priority("operational"), "medium");
1136 assert_eq!(health_status_priority("flapping"), "medium");
1137 assert_eq!(health_status_priority(""), "medium");
1138 }
1139 }
1140