Skip to main content

max / makenotwork

41.7 KB · 1156 lines History Blame Raw
1 //! Email alerting via Postmark API.
2 //!
3 //! Sends alerts on health status transitions and peer disappearance/recovery.
4 //! If no `postmark_token` is configured, alerts are logged to stdout instead.
5
6 use sqlx::SqlitePool;
7 use tracing::{info, warn};
8
9 use crate::config::AlertConfig;
10 use crate::db;
11 use crate::types::AlertCategory;
12
13 mod backup;
14 mod cors;
15 mod dns;
16 mod health;
17 mod latency;
18 mod offline;
19 mod peer;
20 mod route;
21 mod scan;
22 mod systemd;
23 mod test_duration;
24 mod tls;
25 mod whois;
26
27 /// WAM ticket priority for a transition into a non-operational health status.
28 fn health_status_priority(to_status: &str) -> &'static str {
29 match to_status {
30 "error" | "unreachable" => "critical",
31 "degraded" => "high",
32 _ => "medium",
33 }
34 }
35
36 /// WAM ticket priority for a TLS certificate that expires in `days`.
37 fn tls_expiry_priority(days: i64) -> &'static str {
38 if days <= 3 {
39 "critical"
40 } else if days <= 7 {
41 "high"
42 } else {
43 "medium"
44 }
45 }
46
47 /// WAM ticket priority for a domain registration that expires in `days`.
48 fn whois_expiry_priority(days: i64) -> &'static str {
49 if days <= 7 {
50 "critical"
51 } else if days <= 14 {
52 "high"
53 } else {
54 "medium"
55 }
56 }
57
58 /// WAM ticket priority for a stale/missing/error backup status.
59 fn backup_status_priority(status: &str) -> &'static str {
60 if status == "missing" {
61 "critical"
62 } else {
63 "high"
64 }
65 }
66
67 /// Human-readable detail text for a backup status alert.
68 fn backup_status_detail(status: &str, age_hours: Option<i64>) -> String {
69 match (status, age_hours) {
70 ("stale", Some(hours)) => format!("last backup is {hours}h old"),
71 ("missing", _) => "no backup files found".to_string(),
72 ("error", _) => "backup check failed".to_string(),
73 _ => format!("status: {status}"),
74 }
75 }
76
77 /// Map PoM's fine-grained [`AlertCategory`] onto MNW's domain-level alert kind
78 /// (the `AlertKind` enum in the MNW server's `db/admin_alerts.rs`). Recoveries
79 /// and sub-conditions fold onto their domain; the failure/recovery distinction
80 /// rides on severity, not kind. A new PoM category needs a line here, and, if
81 /// it introduces a new domain, a matching MNW `AlertKind` variant (else MNW
82 /// rejects the push with 422).
83 fn mnw_kind(category: AlertCategory) -> &'static str {
84 use AlertCategory::{
85 BackupRecovery, BackupStale, CorsFailure, CorsRecovery, DnsMismatch, DnsRecovery, Health,
86 LatencyDrift, LatencyRecovery, MonitoringOffline, MonitoringRecovery, PeerMissing,
87 PeerRecovery, Recovery, RouteFailure, RouteRecovery, ScanPipelineDegraded,
88 ScanPipelineRecovery, SystemdFailure, SystemdRecovery, TestDurationDrift, TlsError,
89 TlsExpiry, TlsRecovery, WhoisError, WhoisExpiry,
90 };
91 match category {
92 Health | Recovery => "health",
93 TlsExpiry | TlsError | TlsRecovery => "tls",
94 DnsMismatch | DnsRecovery => "dns",
95 WhoisExpiry | WhoisError => "whois",
96 LatencyDrift | LatencyRecovery | TestDurationDrift => "latency",
97 CorsFailure | CorsRecovery => "cors",
98 BackupStale | BackupRecovery => "backup",
99 PeerMissing | PeerRecovery => "peer",
100 RouteFailure | RouteRecovery => "route",
101 ScanPipelineDegraded | ScanPipelineRecovery => "scan",
102 // Daemon liveness folds onto the "monitoring" domain rather than adding a
103 // new MNW `AlertKind` (an unknown kind is a 422 from the ingest endpoint).
104 // It sits with MonitoringOffline: both are "the ops plane itself is sick".
105 SystemdFailure | SystemdRecovery | MonitoringOffline | MonitoringRecovery => "monitoring",
106 }
107 }
108
109 /// Map PoM's WAM priority string onto MNW's three-level severity. Recoveries do
110 /// not carry a priority; the caller passes `"info"` for those directly.
111 fn mnw_severity(priority: &str) -> &'static str {
112 match priority {
113 "critical" => "critical",
114 "high" | "medium" => "warning",
115 _ => "info",
116 }
117 }
118
119 /// Truncate `s` to at most `max` bytes on a char boundary. MNW caps title at
120 /// 200 and body at 5000; over-long input would be a 422, so clamp instead.
121 fn truncate(s: &str, max: usize) -> String {
122 if s.len() <= max {
123 return s.to_string();
124 }
125 let mut end = max;
126 while !s.is_char_boundary(end) {
127 end -= 1;
128 }
129 s[..end].to_string()
130 }
131
132 #[derive(Clone)]
133 pub struct Alerter {
134 config: AlertConfig,
135 client: reqwest::Client,
136 pool: SqlitePool,
137 instance_name: String,
138 wam_url: Option<String>,
139 }
140
141 /// The record-context an alert carries: what to write to the `alerts` ledger on
142 /// successful delivery, and what to persist for retry on failure.
143 struct AlertMeta<'a> {
144 key: &'a str,
145 category: AlertCategory,
146 from: Option<&'a str>,
147 to: Option<&'a str>,
148 error: Option<&'a str>,
149 }
150
151 /// `true` if `sent_at` (rfc3339) is within `cooldown_secs` of now. An unparseable
152 /// timestamp is treated as expired (not in cooldown) so a bad row can never wedge
153 /// alerting shut.
154 fn within_cooldown_secs(sent_at: &str, cooldown_secs: u64) -> bool {
155 match chrono::DateTime::parse_from_rfc3339(sent_at) {
156 Ok(dt) => chrono::Utc::now().signed_duration_since(dt).num_seconds() < cooldown_secs as i64,
157 Err(_) => false,
158 }
159 }
160
161 impl Alerter {
162 pub fn new(
163 config: AlertConfig,
164 pool: SqlitePool,
165 instance_name: String,
166 ) -> Result<Self, reqwest::Error> {
167 // reqwest 0.13's builder can fail (platform trust-store load) and
168 // `Client::new()`/`unwrap_or_default()` panic on that same failure. Return
169 // the error so the caller can disable alerting rather than crash at startup.
170 let client = crate::tls::https_client_builder()
171 .timeout(std::time::Duration::from_secs(10))
172 .build()?;
173 let wam_url = config.wam_url.clone();
174 if wam_url.is_none() {
175 warn!(
176 "alerts: wam_url not configured, failure alerts will be delivered by email \
177 instead of WAM tickets (set alerts.wam_url to route them to WAM)"
178 );
179 } else if config.wam_token.is_none() {
180 warn!(
181 "alerts: wam_url set but wam_token missing, WAM fails closed so ticket \
182 creation will 401 and fall back to email (set alerts.wam_token or \
183 POM_WAM_TOKEN)"
184 );
185 }
186 match (&config.mnw_url, &config.alerts_ingest_token) {
187 (Some(_), Some(_)) => {
188 info!("alerts: MNW sink enabled, alerts also push to the MNW operator log");
189 }
190 (Some(_), None) => warn!(
191 "alerts: mnw_url set but alerts_ingest_token missing, MNW sink disabled \
192 (set alerts.alerts_ingest_token or POM_ALERTS_INGEST_TOKEN)"
193 ),
194 _ => {}
195 }
196 Ok(Self {
197 config,
198 client,
199 pool,
200 instance_name,
201 wam_url,
202 })
203 }
204
205 /// Push one alert to MNW's operator log (`POST /api/internal/alerts`).
206 /// Returns `true` iff MNW accepted it. Returns `false`: never enqueues,
207 /// when the sink is unconfigured; callers gate on config before treating a
208 /// `false` as a delivery miss worth queuing. Bounded by a 10s timeout so MNW
209 /// latency can't stall the probe loop. The server dedups on `dedup_key`
210 /// (`{kind}:{target}`), so a retried or repeated push of one condition
211 /// collapses rather than duplicating.
212 async fn deliver_mnw(
213 &self,
214 kind: &str,
215 key: &str,
216 severity: &str,
217 subject: &str,
218 body: &str,
219 ) -> bool {
220 let (Some(base), Some(token)) = (
221 self.config.mnw_url.as_deref(),
222 self.config.alerts_ingest_token.as_deref(),
223 ) else {
224 return false;
225 };
226 let payload = serde_json::json!({
227 "source": "pom",
228 "kind": kind,
229 "severity": severity,
230 "title": truncate(subject, 200),
231 "body": truncate(body, 5000),
232 "dedup_key": truncate(&format!("{kind}:{key}"), 200),
233 "details": { "instance": self.instance_name, "target": key },
234 });
235 let url = format!("{}/api/internal/alerts", base.trim_end_matches('/'));
236 let send_fut = self
237 .client
238 .post(&url)
239 .bearer_auth(token)
240 .json(&payload)
241 .send();
242 match tokio::time::timeout(std::time::Duration::from_secs(10), send_fut).await {
243 Ok(Ok(resp)) if resp.status().is_success() => {
244 info!("mnw alert pushed: {kind}/{severity} {key}");
245 true
246 }
247 Ok(Ok(resp)) => {
248 let status = resp.status();
249 let text = resp.text().await.unwrap_or_default();
250 warn!("mnw alert push failed ({status}): {text}");
251 false
252 }
253 Ok(Err(e)) => {
254 warn!("mnw alert push error: {e}");
255 false
256 }
257 Err(_) => {
258 warn!("mnw alert push timed out: {kind}/{key}");
259 false
260 }
261 }
262 }
263
264 /// Durable push to the MNW operator log: deliver now, and on failure enqueue
265 /// for retry through the same `pending_alerts` queue WAM/email use. No-op
266 /// unless both `mnw_url` and `alerts_ingest_token` are set.
267 ///
268 /// MNW is a *secondary* sink: the primary channel (WAM/email) already
269 /// recorded this alert in the ledger, so a queued MNW row carries
270 /// `channel = "mnw"` and, unlike WAM/email retries, does not re-record on
271 /// delivery (see [`retry_pending`]), it only needs to eventually land.
272 /// `severity` derives from `priority` exactly as the retry path recomputes
273 /// it (`None` → recovery → `"info"`), so live and retried pushes match.
274 async fn push_mnw(
275 &self,
276 category: AlertCategory,
277 key: &str,
278 priority: Option<&str>,
279 subject: &str,
280 body: &str,
281 ) {
282 if self.config.mnw_url.is_none() || self.config.alerts_ingest_token.is_none() {
283 return;
284 }
285 let kind = mnw_kind(category);
286 let severity = priority.map_or("info", mnw_severity);
287 if self.deliver_mnw(kind, key, severity, subject, body).await {
288 return;
289 }
290 let category = category.to_string();
291 let pending = db::NewPendingAlert {
292 alert_key: key,
293 category: &category,
294 channel: "mnw",
295 subject,
296 body,
297 priority,
298 source: Some("pom"),
299 source_ref: None,
300 from_status: None,
301 to_status: None,
302 error: None,
303 };
304 if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await {
305 warn!("failed to enqueue undelivered mnw alert for retry: {e}");
306 } else {
307 warn!("mnw alert to {key} undelivered; queued for retry");
308 }
309 }
310
311 /// Whether a FAILURE alert for `target` is within its cooldown.
312 ///
313 /// A recorded recovery *newer* than the last failure clears the cooldown: the
314 /// target came back, so a fresh failure is a genuine new outage and must not
315 /// be suppressed by the prior failure's window (fuzz-2026-07-06 fail→recover→
316 /// fail suppression).
317 async fn is_within_cooldown(&self, target: &str) -> bool {
318 let Ok(Some(latest_fail)) = db::get_latest_alert_for_target(&self.pool, target).await
319 else {
320 return false; // never alerted → not in cooldown
321 };
322 // A recovery after the last failure voids the cooldown.
323 if let Ok(Some(rec)) = db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
324 && rec.id > latest_fail.id
325 {
326 return false;
327 }
328 within_cooldown_secs(&latest_fail.sent_at, self.config.cooldown_secs)
329 }
330
331 /// Whether a RECOVERY for `target` is within its own cooldown, throttles a
332 /// flapping target so it can't emit one recovery email per flap
333 /// (fuzz-2026-07-06 un-throttled recoveries). A failure newer than the last
334 /// recovery clears it, so a genuine recover-after-a-new-outage still sends.
335 async fn is_recovery_within_cooldown(&self, target: &str) -> bool {
336 let Ok(Some(latest_rec)) =
337 db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
338 else {
339 return false;
340 };
341 if let Ok(Some(fail)) = db::get_latest_alert_for_target(&self.pool, target).await
342 && fail.id > latest_rec.id
343 {
344 return false;
345 }
346 within_cooldown_secs(&latest_rec.sent_at, self.config.cooldown_secs)
347 }
348
349 /// Send an email via Postmark. Returns `true` if the message was accepted
350 /// (or in dev mode where there is no token, the operator sees it logged).
351 async fn send_email(&self, subject: &str, body: &str) -> bool {
352 let Some(ref token) = self.config.postmark_token else {
353 info!("[dev] alert: {subject}");
354 info!("[dev] {body}");
355 return true;
356 };
357
358 let payload = serde_json::json!({
359 "From": self.config.from,
360 "To": self.config.to,
361 "Subject": subject,
362 "TextBody": body,
363 });
364
365 let send_fut = self
366 .client
367 .post("https://api.postmarkapp.com/email")
368 .header("X-Postmark-Server-Token", token)
369 .header("Content-Type", "application/json")
370 .header("Accept", "application/json")
371 .json(&payload)
372 .send();
373
374 // Wrap in a 30-second timeout to prevent Postmark latency from blocking
375 // the alert task. The reqwest client has its own 10s timeout, but this
376 // guards against DNS resolution stalls and connection pool exhaustion.
377 match tokio::time::timeout(std::time::Duration::from_secs(30), send_fut).await {
378 Ok(Ok(resp)) if resp.status().is_success() => {
379 info!("alert sent: {subject}");
380 true
381 }
382 Ok(Ok(resp)) => {
383 let status = resp.status();
384 let text = resp.text().await.unwrap_or_default();
385 warn!("postmark error ({status}): {text}");
386 false
387 }
388 Ok(Err(e)) => {
389 warn!("failed to send alert: {e}");
390 false
391 }
392 Err(_) => {
393 warn!("alert send timed out after 30s: {subject}");
394 false
395 }
396 }
397 }
398
399 async fn record_alert(
400 &self,
401 target: &str,
402 alert_type: AlertCategory,
403 from_status: Option<&str>,
404 to_status: Option<&str>,
405 error: Option<&str>,
406 ) {
407 self.record_alert_str(
408 target,
409 &alert_type.to_string(),
410 from_status,
411 to_status,
412 error,
413 )
414 .await;
415 }
416
417 /// String-typed variant so the retry task can record a queued alert straight
418 /// from its stored category without round-tripping through the enum.
419 async fn record_alert_str(
420 &self,
421 target: &str,
422 alert_type: &str,
423 from_status: Option<&str>,
424 to_status: Option<&str>,
425 error: Option<&str>,
426 ) {
427 if let Err(e) = db::insert_alert(
428 &self.pool,
429 target,
430 alert_type,
431 from_status,
432 to_status,
433 error,
434 )
435 .await
436 {
437 warn!("failed to record alert: {e}");
438 }
439 }
440
441 /// Fire a failure alert: cooldown-gate, then dispatch. The single entry point
442 /// every `send_*_alert` funnels through, so the cooldown check + dispatch is
443 /// written once rather than copy-pasted into each of the ~13 methods.
444 async fn fire_failure(
445 &self,
446 subject: &str,
447 body: &str,
448 priority: &str,
449 source: &str,
450 source_ref: Option<&str>,
451 meta: AlertMeta<'_>,
452 ) {
453 if self.is_within_cooldown(meta.key).await {
454 info!("alert cooldown active for {}, skipping", meta.key);
455 return;
456 }
457 // Primary sink (WAM) first so a slow MNW push can't delay the ticket;
458 // capture what push_mnw needs before `meta` is moved into dispatch.
459 let (category, key) = (meta.category, meta.key);
460 self.dispatch_wam(subject, body, priority, source, source_ref, meta)
461 .await;
462 self.push_mnw(category, key, Some(priority), subject, body)
463 .await;
464 }
465
466 /// Fire a recovery: recovery-cooldown-gate, then dispatch by email. The single
467 /// entry point every `send_*_recovery` funnels through.
468 async fn fire_recovery(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
469 if self.is_recovery_within_cooldown(meta.key).await {
470 info!("recovery cooldown active for {}, skipping", meta.key);
471 return;
472 }
473 let (category, key) = (meta.category, meta.key);
474 self.dispatch_email(subject, body, meta).await;
475 self.push_mnw(category, key, None, subject, body).await;
476 }
477
478 /// Dispatch a WAM failure alert: attempt delivery (WAM, falling back to
479 /// email), record it on success, or persist it for retry on failure. Gating
480 /// the ledger write on delivery, and queueing the miss, is what stops a
481 /// transient send failure at a status transition from silencing the whole
482 /// outage: the transition fires once, but the queued alert is retried until it
483 /// lands (fuzz-2026-07-06 CRITICAL #1).
484 async fn dispatch_wam(
485 &self,
486 subject: &str,
487 body: &str,
488 priority: &str,
489 source: &str,
490 source_ref: Option<&str>,
491 meta: AlertMeta<'_>,
492 ) {
493 if self
494 .wam_ticket(subject, body, priority, source, source_ref)
495 .await
496 {
497 self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error)
498 .await;
499 } else {
500 self.enqueue_undelivered(
501 subject,
502 body,
503 "wam",
504 Some(priority),
505 Some(source),
506 source_ref,
507 &meta,
508 )
509 .await;
510 }
511 }
512
513 /// Dispatch an email alert (recoveries, and failure alerts that already chose
514 /// email): record on success, persist for retry on failure.
515 async fn dispatch_email(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
516 if self.send_email(subject, body).await {
517 self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error)
518 .await;
519 } else {
520 self.enqueue_undelivered(subject, body, "email", None, None, None, &meta)
521 .await;
522 }
523 }
524
525 #[allow(clippy::too_many_arguments)]
526 async fn enqueue_undelivered(
527 &self,
528 subject: &str,
529 body: &str,
530 channel: &str,
531 priority: Option<&str>,
532 source: Option<&str>,
533 source_ref: Option<&str>,
534 meta: &AlertMeta<'_>,
535 ) {
536 let category = meta.category.to_string();
537 let pending = db::NewPendingAlert {
538 alert_key: meta.key,
539 category: &category,
540 channel,
541 subject,
542 body,
543 priority,
544 source,
545 source_ref,
546 from_status: meta.from,
547 to_status: meta.to,
548 error: meta.error,
549 };
550 if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await {
551 warn!("failed to enqueue undelivered alert for retry: {e}");
552 } else {
553 warn!("alert to {} undelivered; queued for retry", meta.key);
554 }
555 }
556
557 /// Re-attempt one queued alert. On delivery, record it in the ledger and
558 /// signal the caller to delete the row; otherwise leave it for a later tick.
559 /// Called by the retry task in `serve`.
560 pub async fn retry_pending(&self, p: &db::PendingAlertRow) -> bool {
561 // MNW is a secondary sink: the primary channel already wrote this alert
562 // to the ledger, so a successful retry only clears the queued row, it
563 // must not re-record (that would double-count and skew the cooldown).
564 // Reconstruct kind/severity the same way the live push derives them.
565 if p.channel == "mnw" {
566 let kind = p
567 .category
568 .parse::<AlertCategory>()
569 .map_or("health", mnw_kind);
570 let severity = p.priority.as_deref().map_or("info", mnw_severity);
571 return self
572 .deliver_mnw(kind, &p.alert_key, severity, &p.subject, &p.body)
573 .await;
574 }
575 let delivered = match p.channel.as_str() {
576 "wam" => {
577 self.wam_ticket(
578 &p.subject,
579 &p.body,
580 p.priority.as_deref().unwrap_or("high"),
581 p.source.as_deref().unwrap_or("pom"),
582 p.source_ref.as_deref(),
583 )
584 .await
585 }
586 _ => self.send_email(&p.subject, &p.body).await,
587 };
588 if delivered {
589 self.record_alert_str(
590 &p.alert_key,
591 &p.category,
592 p.from_status.as_deref(),
593 p.to_status.as_deref(),
594 p.error.as_deref(),
595 )
596 .await;
597 }
598 delivered
599 }
600
601 /// Create a WAM ticket for a failure alert. Returns `true` if the alert was
602 /// delivered by *some* channel.
603 ///
604 /// If `wam_url` is unset the failure alert would otherwise be dropped on the
605 /// floor, failures route to WAM, recoveries to email, so a missing `wam_url`
606 /// silently disabled the entire down-alert channel while recovery emails kept
607 /// firing (fuzz-2026-07-06 CRITICAL #2). Fall back to email instead: a
608 /// monitoring tool must never silently swallow a down-alert. WAM delivery
609 /// failures also fall back to email so a WAM outage can't blind the operator.
610 async fn wam_ticket(
611 &self,
612 title: &str,
613 body: &str,
614 priority: &str,
615 source: &str,
616 source_ref: Option<&str>,
617 ) -> bool {
618 let Some(ref base_url) = self.wam_url else {
619 // No WAM configured: deliver the failure alert by email instead of
620 // dropping it.
621 return self.send_email(title, body).await;
622 };
623 let url = format!("{base_url}/tickets");
624
625 let mut payload = serde_json::json!({
626 "title": title,
627 "body": body,
628 "priority": priority,
629 "source": source,
630 });
631 if let Some(r) = source_ref {
632 payload["source_ref"] = serde_json::json!(r);
633 }
634
635 // WAM fails closed: every request must carry `Authorization: Bearer
636 // <token>` or it 401s (see MNW/wam/src/api.rs). Send the token when
637 // configured; a missing token 401s and falls back to email below rather
638 // than silently dropping the down-alert.
639 let mut req = self.client.post(&url).json(&payload);
640 if let Some(token) = self.config.wam_token.as_deref() {
641 req = req.bearer_auth(token);
642 }
643
644 match req.send().await {
645 Ok(resp) if resp.status().is_success() => {
646 info!("WAM ticket created: {title}");
647 true
648 }
649 Ok(resp) => {
650 warn!(
651 "WAM ticket creation returned {}: {title}; falling back to email",
652 resp.status()
653 );
654 self.send_email(title, body).await
655 }
656 Err(e) => {
657 warn!("WAM unreachable: {e}; falling back to email");
658 self.send_email(title, body).await
659 }
660 }
661 }
662 }
663
664 #[cfg(test)]
665 mod tests {
666 use super::*;
667
668 #[tokio::test]
669 async fn mnw_retry_routes_to_sink_and_never_records() {
670 // A queued mnw row must retry via the MNW sink, not the WAM/email path,
671 // and must NOT write the ledger (the primary channel already did). With
672 // the sink unconfigured (test_alerter has no mnw_url), delivery fails and
673 // the row stays queued, and the cooldown ledger is untouched.
674 let pool = db::connect_in_memory().await.unwrap();
675 let alerter = test_alerter(pool.clone());
676 let row = db::PendingAlertRow {
677 id: 1,
678 alert_key: "makenot.work".to_string(),
679 category: "tls_expiry".to_string(),
680 channel: "mnw".to_string(),
681 subject: "TLS expiring".to_string(),
682 body: "cert expires soon".to_string(),
683 priority: Some("high".to_string()),
684 source: Some("pom".to_string()),
685 source_ref: None,
686 from_status: None,
687 to_status: None,
688 error: None,
689 attempts: 0,
690 };
691 assert!(
692 !alerter.retry_pending(&row).await,
693 "unconfigured sink cannot deliver"
694 );
695 assert!(
696 !alerter.is_within_cooldown("makenot.work").await,
697 "mnw retry must not record to the ledger"
698 );
699 }
700
701 #[test]
702 fn mnw_kind_folds_failure_and_recovery_to_same_domain() {
703 // A condition and its recovery must map to one kind so they share a
704 // dedup thread in the MNW log; the failure/recovery split rides severity.
705 assert_eq!(mnw_kind(AlertCategory::TlsExpiry), "tls");
706 assert_eq!(mnw_kind(AlertCategory::TlsError), "tls");
707 assert_eq!(mnw_kind(AlertCategory::TlsRecovery), "tls");
708 assert_eq!(mnw_kind(AlertCategory::DnsMismatch), "dns");
709 assert_eq!(mnw_kind(AlertCategory::DnsRecovery), "dns");
710 assert_eq!(mnw_kind(AlertCategory::TestDurationDrift), "latency");
711 assert_eq!(mnw_kind(AlertCategory::MonitoringOffline), "monitoring");
712 }
713
714 #[test]
715 fn mnw_severity_collapses_priorities_to_three_levels() {
716 assert_eq!(mnw_severity("critical"), "critical");
717 assert_eq!(mnw_severity("high"), "warning");
718 assert_eq!(mnw_severity("medium"), "warning");
719 assert_eq!(mnw_severity("low"), "info");
720 assert_eq!(mnw_severity("anything-else"), "info");
721 }
722
723 #[test]
724 fn truncate_respects_cap_and_char_boundaries() {
725 assert_eq!(truncate("short", 200), "short");
726 assert_eq!(truncate("abcdef", 3), "abc");
727 // Cutting mid multi-byte char steps back to the previous boundary rather
728 // than panicking on a non-boundary slice.
729 let s = "aé"; // 'é' is two bytes; byte index 2 is not a char boundary
730 assert_eq!(truncate(s, 2), "a");
731 }
732
733 fn test_alerter(pool: SqlitePool) -> Alerter {
734 let config = AlertConfig {
735 postmark_token: None, // dev mode
736 to: "test@example.com".to_string(),
737 from: "PoM Alerts <pom-alerts@makenot.work>".to_string(),
738 cooldown_secs: 300,
739 wam_url: None,
740 wam_token: None,
741 mnw_url: None,
742 alerts_ingest_token: None,
743 };
744 Alerter::new(config, pool, "test-instance".to_string()).unwrap()
745 }
746
747 #[tokio::test]
748 async fn cooldown_prevents_duplicate_alerts() {
749 let pool = db::connect_in_memory().await.unwrap();
750 let alerter = test_alerter(pool.clone());
751
752 // First alert, not in cooldown
753 assert!(!alerter.is_within_cooldown("health:mnw").await);
754
755 db::insert_alert(
756 &pool,
757 "health:mnw",
758 "health",
759 Some("operational"),
760 Some("error"),
761 None,
762 )
763 .await
764 .unwrap();
765
766 assert!(alerter.is_within_cooldown("health:mnw").await);
767 }
768
769 #[tokio::test]
770 async fn wam_ticket_falls_back_to_email_when_wam_url_unset() {
771 // CRITICAL #2: with no wam_url, a failure alert must NOT be silently
772 // dropped, it falls back to email. test_alerter has wam_url: None and
773 // postmark_token: None (dev mode), so the email path reports delivered.
774 let pool = db::connect_in_memory().await.unwrap();
775 let alerter = test_alerter(pool);
776 assert!(
777 alerter
778 .wam_ticket("subj", "body", "high", "pom-test", None)
779 .await,
780 "a failure alert with no wam_url must be delivered by email, not dropped"
781 );
782 }
783
784 #[tokio::test]
785 async fn cooldown_does_not_affect_other_targets() {
786 let pool = db::connect_in_memory().await.unwrap();
787 let alerter = test_alerter(pool.clone());
788
789 db::insert_alert(&pool, "health:mnw", "health", None, None, None)
790 .await
791 .unwrap();
792
793 // Different target should not be in cooldown
794 assert!(!alerter.is_within_cooldown("health:other").await);
795 }
796
797 #[tokio::test]
798 async fn dev_mode_does_not_send_http() {
799 let pool = db::connect_in_memory().await.unwrap();
800 let alerter = test_alerter(pool.clone());
801
802 // This should log instead of making HTTP calls (no panic, no error)
803 alerter
804 .send_health_alert("mnw", "MakeNotWork", "operational", "error", None)
805 .await;
806
807 // Verify alert was recorded in DB with the prefixed key (health:mnw),
808 // matching the cooldown lookup key format.
809 let latest = db::get_latest_alert_for_target(&pool, "health:mnw")
810 .await
811 .unwrap();
812 assert!(latest.is_some());
813 let row = latest.unwrap();
814 assert_eq!(row.alert_type, "health");
815 assert_eq!(row.from_status.as_deref(), Some("operational"));
816 assert_eq!(row.to_status.as_deref(), Some("error"));
817 }
818
819 #[tokio::test]
820 async fn route_alert_cooldown_key() {
821 let pool = db::connect_in_memory().await.unwrap();
822 let alerter = test_alerter(pool.clone());
823
824 assert!(!alerter.is_within_cooldown("route:mnw").await);
825
826 alerter
827 .send_route_failure_alert("mnw", "MakeNotWork", &["/docs/faq".to_string()])
828 .await;
829
830 assert!(alerter.is_within_cooldown("route:mnw").await);
831 assert!(!alerter.is_within_cooldown("route:mt").await);
832 }
833
834 #[tokio::test]
835 async fn recovery_clears_failure_cooldown_so_next_failure_fires() {
836 // Regression for the fail→recover→fail suppression the fuzz flagged (this
837 // test previously enshrined the bug: it asserted the 2nd failure stays
838 // suppressed). A recovery means the target came back, so a fresh failure
839 // is a genuine NEW outage and must not be muffled by the prior failure's
840 // cooldown.
841 let pool = db::connect_in_memory().await.unwrap();
842 let alerter = test_alerter(pool.clone());
843
844 // Fail → cooldown active.
845 alerter
846 .send_health_alert("mnw", "MakeNotWork", "operational", "error", None)
847 .await;
848 assert!(alerter.is_within_cooldown("health:mnw").await);
849
850 // Recover → the failure cooldown is now cleared.
851 alerter
852 .send_health_recovery("mnw", "MakeNotWork", "error")
853 .await;
854 assert!(
855 !alerter.is_within_cooldown("health:mnw").await,
856 "a recovery after the last failure must void the failure cooldown"
857 );
858 }
859
860 #[tokio::test]
861 async fn recovery_is_throttled_by_its_own_cooldown() {
862 // A flapping target must not emit one recovery per flap: after a recorded
863 // recovery, another recovery within cooldown is throttled, until a new
864 // failure intervenes.
865 let pool = db::connect_in_memory().await.unwrap();
866 let alerter = test_alerter(pool.clone());
867
868 db::insert_alert(
869 &pool,
870 "health:mnw",
871 "recovery",
872 None,
873 Some("operational"),
874 None,
875 )
876 .await
877 .unwrap();
878 assert!(alerter.is_recovery_within_cooldown("health:mnw").await);
879
880 // A failure after the recovery clears the recovery cooldown, so the NEXT
881 // genuine recovery still sends.
882 db::insert_alert(
883 &pool,
884 "health:mnw",
885 "health",
886 Some("operational"),
887 Some("error"),
888 None,
889 )
890 .await
891 .unwrap();
892 assert!(!alerter.is_recovery_within_cooldown("health:mnw").await);
893 }
894
895 #[tokio::test]
896 async fn failed_send_enqueues_for_retry_and_retry_delivers() {
897 // CRITICAL #1: a send that fails must not be lost, it is queued, and the
898 // retry path re-delivers it and records it in the ledger.
899 let pool = db::connect_in_memory().await.unwrap();
900 // Force delivery failure: WAM points at an unroutable URL and there is no
901 // postmark token, so the email fallback in dev mode... would succeed. Use a
902 // direct enqueue + retry to exercise the queue deterministically instead.
903 let pending = db::NewPendingAlert {
904 alert_key: "health:mnw",
905 category: "health",
906 channel: "email", // dev-mode email "delivers" (logs) → retry succeeds
907 subject: "[PoM] mnw: down",
908 body: "body",
909 priority: None,
910 source: None,
911 source_ref: None,
912 from_status: Some("operational"),
913 to_status: Some("error"),
914 error: None,
915 };
916 db::enqueue_pending_alert(&pool, &pending).await.unwrap();
917 let due = db::due_pending_alerts(&pool, 10).await.unwrap();
918 assert_eq!(due.len(), 1, "the undelivered alert must be queued");
919
920 let alerter = test_alerter(pool.clone());
921 assert!(
922 alerter.retry_pending(&due[0]).await,
923 "dev-mode email retry delivers"
924 );
925 db::delete_pending_alert(&pool, due[0].id).await.unwrap();
926
927 // The retry recorded the alert in the ledger (so cooldown now applies).
928 assert!(alerter.is_within_cooldown("health:mnw").await);
929 assert!(db::due_pending_alerts(&pool, 10).await.unwrap().is_empty());
930 }
931
932 #[tokio::test]
933 async fn dns_alert_cooldown_key() {
934 let pool = db::connect_in_memory().await.unwrap();
935 let alerter = test_alerter(pool.clone());
936
937 assert!(!alerter.is_within_cooldown("dns:mnw").await);
938
939 let mismatches = vec![crate::types::DnsCheckResult {
940 target: "mnw".to_string(),
941 name: "makenot.work".to_string(),
942 record_type: crate::types::DnsRecordType::A,
943 expected: vec!["1.2.3.4".to_string()],
944 actual: vec!["5.6.7.8".to_string()],
945 matches: false,
946 checked_at: chrono::Utc::now().to_rfc3339(),
947 error: None,
948 }];
949 alerter
950 .send_dns_mismatch_alert("mnw", "MakeNotWork", &mismatches)
951 .await;
952
953 assert!(alerter.is_within_cooldown("dns:mnw").await);
954 assert!(!alerter.is_within_cooldown("dns:other").await);
955 }
956
957 #[tokio::test]
958 async fn whois_alert_cooldown_key() {
959 let pool = db::connect_in_memory().await.unwrap();
960 let alerter = test_alerter(pool.clone());
961
962 assert!(!alerter.is_within_cooldown("whois:mnw").await);
963
964 alerter
965 .send_whois_expiry_alert("mnw", "MakeNotWork", "makenot.work", 15)
966 .await;
967
968 assert!(alerter.is_within_cooldown("whois:mnw").await);
969 assert!(!alerter.is_within_cooldown("whois:other").await);
970 }
971
972 #[tokio::test]
973 async fn health_alert_cooldown_key_matches_record_key() {
974 let pool = db::connect_in_memory().await.unwrap();
975 let alerter = test_alerter(pool.clone());
976
977 // Not in cooldown initially
978 assert!(!alerter.is_within_cooldown("health:example.com").await);
979
980 alerter
981 .send_health_alert("example.com", "Example", "operational", "error", None)
982 .await;
983
984 assert!(alerter.is_within_cooldown("health:example.com").await);
985
986 // Different target should NOT be in cooldown
987 assert!(!alerter.is_within_cooldown("health:other.com").await);
988 }
989
990 #[tokio::test]
991 async fn tls_expiry_alert_cooldown_key() {
992 let pool = db::connect_in_memory().await.unwrap();
993 let alerter = test_alerter(pool.clone());
994
995 assert!(!alerter.is_within_cooldown("tls:mnw").await);
996 alerter
997 .send_tls_expiry_alert("mnw", "makenot.work", 10, "2026-04-01T00:00:00Z")
998 .await;
999 assert!(alerter.is_within_cooldown("tls:mnw").await);
1000 }
1001
1002 #[tokio::test]
1003 async fn tls_error_alert_cooldown_key() {
1004 let pool = db::connect_in_memory().await.unwrap();
1005 let alerter = test_alerter(pool.clone());
1006
1007 assert!(!alerter.is_within_cooldown("tls:mnw").await);
1008 alerter
1009 .send_tls_error_alert("mnw", "makenot.work", "certificate expired")
1010 .await;
1011 assert!(alerter.is_within_cooldown("tls:mnw").await);
1012 }
1013
1014 #[tokio::test]
1015 async fn latency_drift_alert_cooldown_key() {
1016 let pool = db::connect_in_memory().await.unwrap();
1017 let alerter = test_alerter(pool.clone());
1018
1019 assert!(!alerter.is_within_cooldown("latency:mnw").await);
1020 alerter
1021 .send_latency_drift_alert("mnw", "MakeNotWork", "avg 500ms, baseline 100ms")
1022 .await;
1023 assert!(alerter.is_within_cooldown("latency:mnw").await);
1024 }
1025
1026 #[tokio::test]
1027 async fn test_duration_drift_alert_cooldown_key() {
1028 let pool = db::connect_in_memory().await.unwrap();
1029 let alerter = test_alerter(pool.clone());
1030
1031 assert!(!alerter.is_within_cooldown("test_duration:mnw").await);
1032 alerter
1033 .send_test_duration_drift_alert("mnw", "MakeNotWork", "drift: 120s vs 60s baseline")
1034 .await;
1035 assert!(alerter.is_within_cooldown("test_duration:mnw").await);
1036 }
1037
1038 #[tokio::test]
1039 async fn monitoring_offline_alert_cooldown_key() {
1040 let pool = db::connect_in_memory().await.unwrap();
1041 let alerter = test_alerter(pool.clone());
1042
1043 assert!(!alerter.is_within_cooldown("monitoring:self").await);
1044 alerter.send_monitoring_offline_alert(3).await;
1045 assert!(alerter.is_within_cooldown("monitoring:self").await);
1046 }
1047
1048 #[tokio::test]
1049 async fn route_recovery_does_not_start_cooldown() {
1050 let pool = db::connect_in_memory().await.unwrap();
1051 let alerter = test_alerter(pool.clone());
1052
1053 alerter
1054 .send_route_recovery_alert("mnw", "MakeNotWork", &["/health".to_string()])
1055 .await;
1056 // Recovery alerts are excluded from cooldown lookups, so sending a recovery
1057 // should NOT put the key into cooldown.
1058 assert!(!alerter.is_within_cooldown("route:mnw").await);
1059 }
1060
1061 #[tokio::test]
1062 async fn dns_recovery_does_not_start_cooldown() {
1063 let pool = db::connect_in_memory().await.unwrap();
1064 let alerter = test_alerter(pool.clone());
1065
1066 alerter.send_dns_recovery_alert("mnw", "MakeNotWork").await;
1067 assert!(!alerter.is_within_cooldown("dns:mnw").await);
1068 }
1069
1070 #[tokio::test]
1071 async fn tls_recovery_does_not_start_cooldown() {
1072 let pool = db::connect_in_memory().await.unwrap();
1073 let alerter = test_alerter(pool.clone());
1074
1075 alerter.send_tls_recovery("mnw", "MakeNotWork", 90).await;
1076 assert!(!alerter.is_within_cooldown("tls:mnw").await);
1077 }
1078
1079 // Pure priority/severity helpers (pin the <= boundaries)
1080
1081 #[test]
1082 fn tls_expiry_priority_boundaries() {
1083 // critical: days <= 3
1084 assert_eq!(
1085 tls_expiry_priority(-5),
1086 "critical",
1087 "negative days = already expired"
1088 );
1089 assert_eq!(tls_expiry_priority(0), "critical");
1090 assert_eq!(tls_expiry_priority(3), "critical");
1091 // high: 4..=7
1092 assert_eq!(tls_expiry_priority(4), "high");
1093 assert_eq!(tls_expiry_priority(7), "high");
1094 // medium: > 7
1095 assert_eq!(tls_expiry_priority(8), "medium");
1096 assert_eq!(tls_expiry_priority(90), "medium");
1097 }
1098
1099 #[test]
1100 fn whois_expiry_priority_boundaries() {
1101 // critical: days <= 7
1102 assert_eq!(whois_expiry_priority(-1), "critical");
1103 assert_eq!(whois_expiry_priority(7), "critical");
1104 // high: 8..=14
1105 assert_eq!(whois_expiry_priority(8), "high");
1106 assert_eq!(whois_expiry_priority(14), "high");
1107 // medium: > 14
1108 assert_eq!(whois_expiry_priority(15), "medium");
1109 assert_eq!(whois_expiry_priority(180), "medium");
1110 }
1111
1112 #[test]
1113 fn backup_status_priority_missing_is_critical() {
1114 assert_eq!(backup_status_priority("missing"), "critical");
1115 assert_eq!(backup_status_priority("stale"), "high");
1116 assert_eq!(backup_status_priority("error"), "high");
1117 assert_eq!(backup_status_priority("anything-else"), "high");
1118 assert_eq!(backup_status_priority(""), "high");
1119 }
1120
1121 #[test]
1122 fn backup_status_detail_arms() {
1123 assert_eq!(
1124 backup_status_detail("stale", Some(12)),
1125 "last backup is 12h old"
1126 );
1127 // `stale` with no age falls through to the default arm.
1128 assert_eq!(backup_status_detail("stale", None), "status: stale");
1129 assert_eq!(
1130 backup_status_detail("missing", None),
1131 "no backup files found"
1132 );
1133 assert_eq!(
1134 backup_status_detail("missing", Some(5)),
1135 "no backup files found"
1136 );
1137 assert_eq!(backup_status_detail("error", None), "backup check failed");
1138 assert_eq!(
1139 backup_status_detail("error", Some(99)),
1140 "backup check failed"
1141 );
1142 assert_eq!(backup_status_detail("weird", None), "status: weird");
1143 }
1144
1145 #[test]
1146 fn health_status_priority_arms() {
1147 assert_eq!(health_status_priority("error"), "critical");
1148 assert_eq!(health_status_priority("unreachable"), "critical");
1149 assert_eq!(health_status_priority("degraded"), "high");
1150 // Anything else (operational, unknown values) falls through to medium.
1151 assert_eq!(health_status_priority("operational"), "medium");
1152 assert_eq!(health_status_priority("flapping"), "medium");
1153 assert_eq!(health_status_priority(""), "medium");
1154 }
1155 }
1156