Skip to main content

max / makenotwork

42.1 KB · 1164 lines History Blame Raw
1 //! Email alerting via Postmark API.
2 //!
3 //! Sends alerts on health status transitions and peer disappearance/recovery.
4 //! If no `postmark_token` is configured, alerts are logged to stdout instead.
5
6 use sqlx::SqlitePool;
7 use tracing::{info, warn};
8
9 use crate::config::AlertConfig;
10 use crate::db;
11 use crate::types::AlertCategory;
12
13 mod backup;
14 mod ca_bundle;
15 mod cors;
16 mod dns;
17 mod health;
18 mod latency;
19 mod offline;
20 mod peer;
21 mod route;
22 mod scan;
23 mod systemd;
24 mod test_duration;
25 mod test_suite;
26 mod tls;
27 mod whois;
28
29 /// WAM ticket priority for a transition into a non-operational health status.
30 fn health_status_priority(to_status: &str) -> &'static str {
31 match to_status {
32 "error" | "unreachable" => "critical",
33 "degraded" => "high",
34 _ => "medium",
35 }
36 }
37
38 /// WAM ticket priority for a TLS certificate that expires in `days`.
39 fn tls_expiry_priority(days: i64) -> &'static str {
40 if days <= 3 {
41 "critical"
42 } else if days <= 7 {
43 "high"
44 } else {
45 "medium"
46 }
47 }
48
49 /// WAM ticket priority for a domain registration that expires in `days`.
50 fn whois_expiry_priority(days: i64) -> &'static str {
51 if days <= 7 {
52 "critical"
53 } else if days <= 14 {
54 "high"
55 } else {
56 "medium"
57 }
58 }
59
60 /// WAM ticket priority for a stale/missing/error backup status.
61 fn backup_status_priority(status: &str) -> &'static str {
62 if status == "missing" {
63 "critical"
64 } else {
65 "high"
66 }
67 }
68
69 /// Human-readable detail text for a backup status alert.
70 fn backup_status_detail(status: &str, age_hours: Option<i64>) -> String {
71 match (status, age_hours) {
72 ("stale", Some(hours)) => format!("last backup is {hours}h old"),
73 ("missing", _) => "no backup files found".to_string(),
74 ("error", _) => "backup check failed".to_string(),
75 _ => format!("status: {status}"),
76 }
77 }
78
79 /// Map PoM's fine-grained [`AlertCategory`] onto MNW's domain-level alert kind
80 /// (the `AlertKind` enum in the MNW server's `db/admin_alerts.rs`). Recoveries
81 /// and sub-conditions fold onto their domain; the failure/recovery distinction
82 /// rides on severity, not kind. A new PoM category needs a line here, and, if
83 /// it introduces a new domain, a matching MNW `AlertKind` variant (else MNW
84 /// rejects the push with 422).
85 fn mnw_kind(category: AlertCategory) -> &'static str {
86 use AlertCategory::{
87 BackupRecovery, BackupStale, CaBundleRecovery, CaBundleStale, CorsFailure, CorsRecovery,
88 DnsMismatch, DnsRecovery, Health, LatencyDrift, LatencyRecovery, MonitoringOffline,
89 MonitoringRecovery, PeerMissing, PeerRecovery, Recovery, RouteFailure, RouteRecovery,
90 ScanPipelineDegraded, ScanPipelineRecovery, SystemdFailure, SystemdRecovery,
91 TestDurationDrift, TestFailure, TestRecovery, TlsError, TlsExpiry, TlsRecovery, WhoisError,
92 WhoisExpiry,
93 };
94 match category {
95 // A red test suite folds onto "health": it is a statement about the
96 // target being sick, and MNW has no `test` AlertKind (an unknown kind
97 // is a 422 from the ingest endpoint). The sub-condition rides in the
98 // dedup key, the same way `Tls` already folds three PoM categories.
99 Health | Recovery | TestFailure | TestRecovery => "health",
100 // The host CA bundle folds onto the "tls" domain: it is a trust-anchor
101 // problem, and every symptom it produces is a failed handshake. Adding a
102 // domain would need a matching MNW `AlertKind` or the ingest 422s.
103 TlsExpiry | TlsError | TlsRecovery | CaBundleStale | CaBundleRecovery => "tls",
104 DnsMismatch | DnsRecovery => "dns",
105 WhoisExpiry | WhoisError => "whois",
106 LatencyDrift | LatencyRecovery | TestDurationDrift => "latency",
107 CorsFailure | CorsRecovery => "cors",
108 BackupStale | BackupRecovery => "backup",
109 PeerMissing | PeerRecovery => "peer",
110 RouteFailure | RouteRecovery => "route",
111 ScanPipelineDegraded | ScanPipelineRecovery => "scan",
112 // Daemon liveness folds onto the "monitoring" domain rather than adding a
113 // new MNW `AlertKind` (an unknown kind is a 422 from the ingest endpoint).
114 // It sits with MonitoringOffline: both are "the ops plane itself is sick".
115 SystemdFailure | SystemdRecovery | MonitoringOffline | MonitoringRecovery => "monitoring",
116 }
117 }
118
119 /// Map PoM's WAM priority string onto MNW's three-level severity. Recoveries do
120 /// not carry a priority; the caller passes `"info"` for those directly.
121 fn mnw_severity(priority: &str) -> &'static str {
122 match priority {
123 "critical" => "critical",
124 "high" | "medium" => "warning",
125 _ => "info",
126 }
127 }
128
129 /// Truncate `s` to at most `max` bytes on a char boundary. MNW caps title at
130 /// 200 and body at 5000; over-long input would be a 422, so clamp instead.
131 fn truncate(s: &str, max: usize) -> String {
132 if s.len() <= max {
133 return s.to_string();
134 }
135 let mut end = max;
136 while !s.is_char_boundary(end) {
137 end -= 1;
138 }
139 s[..end].to_string()
140 }
141
142 #[derive(Clone)]
143 pub struct Alerter {
144 config: AlertConfig,
145 client: reqwest::Client,
146 pool: SqlitePool,
147 instance_name: String,
148 wam_url: Option<String>,
149 }
150
151 /// The record-context an alert carries: what to write to the `alerts` ledger on
152 /// successful delivery, and what to persist for retry on failure.
153 struct AlertMeta<'a> {
154 key: &'a str,
155 category: AlertCategory,
156 from: Option<&'a str>,
157 to: Option<&'a str>,
158 error: Option<&'a str>,
159 }
160
161 /// `true` if `sent_at` (rfc3339) is within `cooldown_secs` of now. An unparseable
162 /// timestamp is treated as expired (not in cooldown) so a bad row can never wedge
163 /// alerting shut.
164 fn within_cooldown_secs(sent_at: &str, cooldown_secs: u64) -> bool {
165 match chrono::DateTime::parse_from_rfc3339(sent_at) {
166 Ok(dt) => chrono::Utc::now().signed_duration_since(dt).num_seconds() < cooldown_secs as i64,
167 Err(_) => false,
168 }
169 }
170
171 impl Alerter {
172 pub fn new(
173 config: AlertConfig,
174 pool: SqlitePool,
175 instance_name: String,
176 ) -> Result<Self, reqwest::Error> {
177 // reqwest 0.13's builder can fail (platform trust-store load) and
178 // `Client::new()`/`unwrap_or_default()` panic on that same failure. Return
179 // the error so the caller can disable alerting rather than crash at startup.
180 let client = crate::tls::https_client_builder()
181 .timeout(std::time::Duration::from_secs(10))
182 .build()?;
183 let wam_url = config.wam_url.clone();
184 if wam_url.is_none() {
185 warn!(
186 "alerts: wam_url not configured, failure alerts will be delivered by email \
187 instead of WAM tickets (set alerts.wam_url to route them to WAM)"
188 );
189 } else if config.wam_token.is_none() {
190 warn!(
191 "alerts: wam_url set but wam_token missing, WAM fails closed so ticket \
192 creation will 401 and fall back to email (set alerts.wam_token or \
193 POM_WAM_TOKEN)"
194 );
195 }
196 match (&config.mnw_url, &config.alerts_ingest_token) {
197 (Some(_), Some(_)) => {
198 info!("alerts: MNW sink enabled, alerts also push to the MNW operator log");
199 }
200 (Some(_), None) => warn!(
201 "alerts: mnw_url set but alerts_ingest_token missing, MNW sink disabled \
202 (set alerts.alerts_ingest_token or POM_ALERTS_INGEST_TOKEN)"
203 ),
204 _ => {}
205 }
206 Ok(Self {
207 config,
208 client,
209 pool,
210 instance_name,
211 wam_url,
212 })
213 }
214
215 /// Push one alert to MNW's operator log (`POST /api/internal/alerts`).
216 /// Returns `true` iff MNW accepted it. Returns `false`: never enqueues,
217 /// when the sink is unconfigured; callers gate on config before treating a
218 /// `false` as a delivery miss worth queuing. Bounded by a 10s timeout so MNW
219 /// latency can't stall the probe loop. The server dedups on `dedup_key`
220 /// (`{kind}:{target}`), so a retried or repeated push of one condition
221 /// collapses rather than duplicating.
222 async fn deliver_mnw(
223 &self,
224 kind: &str,
225 key: &str,
226 severity: &str,
227 subject: &str,
228 body: &str,
229 ) -> bool {
230 let (Some(base), Some(token)) = (
231 self.config.mnw_url.as_deref(),
232 self.config.alerts_ingest_token.as_deref(),
233 ) else {
234 return false;
235 };
236 let payload = serde_json::json!({
237 "source": "pom",
238 "kind": kind,
239 "severity": severity,
240 "title": truncate(subject, 200),
241 "body": truncate(body, 5000),
242 "dedup_key": truncate(&format!("{kind}:{key}"), 200),
243 "details": { "instance": self.instance_name, "target": key },
244 });
245 let url = format!("{}/api/internal/alerts", base.trim_end_matches('/'));
246 let send_fut = self
247 .client
248 .post(&url)
249 .bearer_auth(token)
250 .json(&payload)
251 .send();
252 match tokio::time::timeout(std::time::Duration::from_secs(10), send_fut).await {
253 Ok(Ok(resp)) if resp.status().is_success() => {
254 info!("mnw alert pushed: {kind}/{severity} {key}");
255 true
256 }
257 Ok(Ok(resp)) => {
258 let status = resp.status();
259 let text = resp.text().await.unwrap_or_default();
260 warn!("mnw alert push failed ({status}): {text}");
261 false
262 }
263 Ok(Err(e)) => {
264 warn!("mnw alert push error: {e}");
265 false
266 }
267 Err(_) => {
268 warn!("mnw alert push timed out: {kind}/{key}");
269 false
270 }
271 }
272 }
273
274 /// Durable push to the MNW operator log: deliver now, and on failure enqueue
275 /// for retry through the same `pending_alerts` queue WAM/email use. No-op
276 /// unless both `mnw_url` and `alerts_ingest_token` are set.
277 ///
278 /// MNW is a *secondary* sink: the primary channel (WAM/email) already
279 /// recorded this alert in the ledger, so a queued MNW row carries
280 /// `channel = "mnw"` and, unlike WAM/email retries, does not re-record on
281 /// delivery (see [`retry_pending`]), it only needs to eventually land.
282 /// `severity` derives from `priority` exactly as the retry path recomputes
283 /// it (`None` → recovery → `"info"`), so live and retried pushes match.
284 async fn push_mnw(
285 &self,
286 category: AlertCategory,
287 key: &str,
288 priority: Option<&str>,
289 subject: &str,
290 body: &str,
291 ) {
292 if self.config.mnw_url.is_none() || self.config.alerts_ingest_token.is_none() {
293 return;
294 }
295 let kind = mnw_kind(category);
296 let severity = priority.map_or("info", mnw_severity);
297 if self.deliver_mnw(kind, key, severity, subject, body).await {
298 return;
299 }
300 let category = category.to_string();
301 let pending = db::NewPendingAlert {
302 alert_key: key,
303 category: &category,
304 channel: "mnw",
305 subject,
306 body,
307 priority,
308 source: Some("pom"),
309 source_ref: None,
310 from_status: None,
311 to_status: None,
312 error: None,
313 };
314 if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await {
315 warn!("failed to enqueue undelivered mnw alert for retry: {e}");
316 } else {
317 warn!("mnw alert to {key} undelivered; queued for retry");
318 }
319 }
320
321 /// Whether a FAILURE alert for `target` is within its cooldown.
322 ///
323 /// A recorded recovery *newer* than the last failure clears the cooldown: the
324 /// target came back, so a fresh failure is a genuine new outage and must not
325 /// be suppressed by the prior failure's window.
326 async fn is_within_cooldown(&self, target: &str) -> bool {
327 let Ok(Some(latest_fail)) = db::get_latest_alert_for_target(&self.pool, target).await
328 else {
329 return false; // never alerted → not in cooldown
330 };
331 // A recovery after the last failure voids the cooldown.
332 if let Ok(Some(rec)) = db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
333 && rec.id > latest_fail.id
334 {
335 return false;
336 }
337 within_cooldown_secs(&latest_fail.sent_at, self.config.cooldown_secs)
338 }
339
340 /// Whether a RECOVERY for `target` is within its own cooldown, throttles a
341 /// flapping target so it can't emit one recovery email per flap. A failure
342 /// newer than the last
343 /// recovery clears it, so a genuine recover-after-a-new-outage still sends.
344 async fn is_recovery_within_cooldown(&self, target: &str) -> bool {
345 let Ok(Some(latest_rec)) =
346 db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
347 else {
348 return false;
349 };
350 if let Ok(Some(fail)) = db::get_latest_alert_for_target(&self.pool, target).await
351 && fail.id > latest_rec.id
352 {
353 return false;
354 }
355 within_cooldown_secs(&latest_rec.sent_at, self.config.cooldown_secs)
356 }
357
358 /// Send an email via Postmark. Returns `true` if the message was accepted
359 /// (or in dev mode where there is no token, the operator sees it logged).
360 async fn send_email(&self, subject: &str, body: &str) -> bool {
361 let Some(ref token) = self.config.postmark_token else {
362 info!("[dev] alert: {subject}");
363 info!("[dev] {body}");
364 return true;
365 };
366
367 let payload = serde_json::json!({
368 "From": self.config.from,
369 "To": self.config.to,
370 "Subject": subject,
371 "TextBody": body,
372 });
373
374 let send_fut = self
375 .client
376 .post("https://api.postmarkapp.com/email")
377 .header("X-Postmark-Server-Token", token)
378 .header("Content-Type", "application/json")
379 .header("Accept", "application/json")
380 .json(&payload)
381 .send();
382
383 // Wrap in a 30-second timeout to prevent Postmark latency from blocking
384 // the alert task. The reqwest client has its own 10s timeout, but this
385 // guards against DNS resolution stalls and connection pool exhaustion.
386 match tokio::time::timeout(std::time::Duration::from_secs(30), send_fut).await {
387 Ok(Ok(resp)) if resp.status().is_success() => {
388 info!("alert sent: {subject}");
389 true
390 }
391 Ok(Ok(resp)) => {
392 let status = resp.status();
393 let text = resp.text().await.unwrap_or_default();
394 warn!("postmark error ({status}): {text}");
395 false
396 }
397 Ok(Err(e)) => {
398 warn!("failed to send alert: {e}");
399 false
400 }
401 Err(_) => {
402 warn!("alert send timed out after 30s: {subject}");
403 false
404 }
405 }
406 }
407
408 async fn record_alert(
409 &self,
410 target: &str,
411 alert_type: AlertCategory,
412 from_status: Option<&str>,
413 to_status: Option<&str>,
414 error: Option<&str>,
415 ) {
416 self.record_alert_str(
417 target,
418 &alert_type.to_string(),
419 from_status,
420 to_status,
421 error,
422 )
423 .await;
424 }
425
426 /// String-typed variant so the retry task can record a queued alert straight
427 /// from its stored category without round-tripping through the enum.
428 async fn record_alert_str(
429 &self,
430 target: &str,
431 alert_type: &str,
432 from_status: Option<&str>,
433 to_status: Option<&str>,
434 error: Option<&str>,
435 ) {
436 if let Err(e) = db::insert_alert(
437 &self.pool,
438 target,
439 alert_type,
440 from_status,
441 to_status,
442 error,
443 )
444 .await
445 {
446 warn!("failed to record alert: {e}");
447 }
448 }
449
450 /// Fire a failure alert: cooldown-gate, then dispatch. The single entry point
451 /// every `send_*_alert` funnels through, so the cooldown check + dispatch is
452 /// written once rather than copy-pasted into each of the ~13 methods.
453 async fn fire_failure(
454 &self,
455 subject: &str,
456 body: &str,
457 priority: &str,
458 source: &str,
459 source_ref: Option<&str>,
460 meta: AlertMeta<'_>,
461 ) {
462 if self.is_within_cooldown(meta.key).await {
463 info!("alert cooldown active for {}, skipping", meta.key);
464 return;
465 }
466 // Primary sink (WAM) first so a slow MNW push can't delay the ticket;
467 // capture what push_mnw needs before `meta` is moved into dispatch.
468 let (category, key) = (meta.category, meta.key);
469 self.dispatch_wam(subject, body, priority, source, source_ref, meta)
470 .await;
471 self.push_mnw(category, key, Some(priority), subject, body)
472 .await;
473 }
474
475 /// Fire a recovery: recovery-cooldown-gate, then dispatch by email. The single
476 /// entry point every `send_*_recovery` funnels through.
477 async fn fire_recovery(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
478 if self.is_recovery_within_cooldown(meta.key).await {
479 info!("recovery cooldown active for {}, skipping", meta.key);
480 return;
481 }
482 let (category, key) = (meta.category, meta.key);
483 self.dispatch_email(subject, body, meta).await;
484 self.push_mnw(category, key, None, subject, body).await;
485 }
486
487 /// Dispatch a WAM failure alert: attempt delivery (WAM, falling back to
488 /// email), record it on success, or persist it for retry on failure. Gating
489 /// the ledger write on delivery, and queueing the miss, is what stops a
490 /// transient send failure at a status transition from silencing the whole
491 /// outage: the transition fires once, but the queued alert is retried until
492 /// it lands.
493 async fn dispatch_wam(
494 &self,
495 subject: &str,
496 body: &str,
497 priority: &str,
498 source: &str,
499 source_ref: Option<&str>,
500 meta: AlertMeta<'_>,
501 ) {
502 if self
503 .wam_ticket(subject, body, priority, source, source_ref)
504 .await
505 {
506 self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error)
507 .await;
508 } else {
509 self.enqueue_undelivered(
510 subject,
511 body,
512 "wam",
513 Some(priority),
514 Some(source),
515 source_ref,
516 &meta,
517 )
518 .await;
519 }
520 }
521
522 /// Dispatch an email alert (recoveries, and failure alerts that already chose
523 /// email): record on success, persist for retry on failure.
524 async fn dispatch_email(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
525 if self.send_email(subject, body).await {
526 self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error)
527 .await;
528 } else {
529 self.enqueue_undelivered(subject, body, "email", None, None, None, &meta)
530 .await;
531 }
532 }
533
534 #[allow(clippy::too_many_arguments)]
535 async fn enqueue_undelivered(
536 &self,
537 subject: &str,
538 body: &str,
539 channel: &str,
540 priority: Option<&str>,
541 source: Option<&str>,
542 source_ref: Option<&str>,
543 meta: &AlertMeta<'_>,
544 ) {
545 let category = meta.category.to_string();
546 let pending = db::NewPendingAlert {
547 alert_key: meta.key,
548 category: &category,
549 channel,
550 subject,
551 body,
552 priority,
553 source,
554 source_ref,
555 from_status: meta.from,
556 to_status: meta.to,
557 error: meta.error,
558 };
559 if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await {
560 warn!("failed to enqueue undelivered alert for retry: {e}");
561 } else {
562 warn!("alert to {} undelivered; queued for retry", meta.key);
563 }
564 }
565
566 /// Re-attempt one queued alert. On delivery, record it in the ledger and
567 /// signal the caller to delete the row; otherwise leave it for a later tick.
568 /// Called by the retry task in `serve`.
569 pub async fn retry_pending(&self, p: &db::PendingAlertRow) -> bool {
570 // MNW is a secondary sink: the primary channel already wrote this alert
571 // to the ledger, so a successful retry only clears the queued row, it
572 // must not re-record (that would double-count and skew the cooldown).
573 // Reconstruct kind/severity the same way the live push derives them.
574 if p.channel == "mnw" {
575 let kind = p
576 .category
577 .parse::<AlertCategory>()
578 .map_or("health", mnw_kind);
579 let severity = p.priority.as_deref().map_or("info", mnw_severity);
580 return self
581 .deliver_mnw(kind, &p.alert_key, severity, &p.subject, &p.body)
582 .await;
583 }
584 let delivered = match p.channel.as_str() {
585 "wam" => {
586 self.wam_ticket(
587 &p.subject,
588 &p.body,
589 p.priority.as_deref().unwrap_or("high"),
590 p.source.as_deref().unwrap_or("pom"),
591 p.source_ref.as_deref(),
592 )
593 .await
594 }
595 _ => self.send_email(&p.subject, &p.body).await,
596 };
597 if delivered {
598 self.record_alert_str(
599 &p.alert_key,
600 &p.category,
601 p.from_status.as_deref(),
602 p.to_status.as_deref(),
603 p.error.as_deref(),
604 )
605 .await;
606 }
607 delivered
608 }
609
610 /// Create a WAM ticket for a failure alert. Returns `true` if the alert was
611 /// delivered by *some* channel.
612 ///
613 /// Failures route to WAM and recoveries to email, so an unset `wam_url`
614 /// would disable the whole down-alert channel while recovery emails kept
615 /// firing. It falls back to email instead: a monitoring tool must never
616 /// silently swallow a down-alert. WAM delivery
617 /// failures also fall back to email so a WAM outage can't blind the operator.
618 async fn wam_ticket(
619 &self,
620 title: &str,
621 body: &str,
622 priority: &str,
623 source: &str,
624 source_ref: Option<&str>,
625 ) -> bool {
626 let Some(ref base_url) = self.wam_url else {
627 // No WAM configured: deliver the failure alert by email instead of
628 // dropping it.
629 return self.send_email(title, body).await;
630 };
631 let url = format!("{base_url}/tickets");
632
633 let mut payload = serde_json::json!({
634 "title": title,
635 "body": body,
636 "priority": priority,
637 "source": source,
638 });
639 if let Some(r) = source_ref {
640 payload["source_ref"] = serde_json::json!(r);
641 }
642
643 // WAM fails closed: every request must carry `Authorization: Bearer
644 // <token>` or it 401s (see MNW/wam/src/api.rs). Send the token when
645 // configured; a missing token 401s and falls back to email below rather
646 // than silently dropping the down-alert.
647 let mut req = self.client.post(&url).json(&payload);
648 if let Some(token) = self.config.wam_token.as_deref() {
649 req = req.bearer_auth(token);
650 }
651
652 match req.send().await {
653 Ok(resp) if resp.status().is_success() => {
654 info!("WAM ticket created: {title}");
655 true
656 }
657 Ok(resp) => {
658 warn!(
659 "WAM ticket creation returned {}: {title}; falling back to email",
660 resp.status()
661 );
662 self.send_email(title, body).await
663 }
664 Err(e) => {
665 warn!("WAM unreachable: {e}; falling back to email");
666 self.send_email(title, body).await
667 }
668 }
669 }
670 }
671
672 #[cfg(test)]
673 mod tests {
674 use super::*;
675
676 #[tokio::test]
677 async fn mnw_retry_routes_to_sink_and_never_records() {
678 // A queued mnw row must retry via the MNW sink, not the WAM/email path,
679 // and must NOT write the ledger (the primary channel already did). With
680 // the sink unconfigured (test_alerter has no mnw_url), delivery fails and
681 // the row stays queued, and the cooldown ledger is untouched.
682 let pool = db::connect_in_memory().await.unwrap();
683 let alerter = test_alerter(pool.clone());
684 let row = db::PendingAlertRow {
685 id: 1,
686 alert_key: "makenot.work".to_string(),
687 category: "tls_expiry".to_string(),
688 channel: "mnw".to_string(),
689 subject: "TLS expiring".to_string(),
690 body: "cert expires soon".to_string(),
691 priority: Some("high".to_string()),
692 source: Some("pom".to_string()),
693 source_ref: None,
694 from_status: None,
695 to_status: None,
696 error: None,
697 attempts: 0,
698 };
699 assert!(
700 !alerter.retry_pending(&row).await,
701 "unconfigured sink cannot deliver"
702 );
703 assert!(
704 !alerter.is_within_cooldown("makenot.work").await,
705 "mnw retry must not record to the ledger"
706 );
707 }
708
709 #[test]
710 fn mnw_kind_folds_failure_and_recovery_to_same_domain() {
711 // A condition and its recovery must map to one kind so they share a
712 // dedup thread in the MNW log; the failure/recovery split rides severity.
713 assert_eq!(mnw_kind(AlertCategory::TlsExpiry), "tls");
714 assert_eq!(mnw_kind(AlertCategory::TlsError), "tls");
715 assert_eq!(mnw_kind(AlertCategory::TlsRecovery), "tls");
716 assert_eq!(mnw_kind(AlertCategory::DnsMismatch), "dns");
717 assert_eq!(mnw_kind(AlertCategory::DnsRecovery), "dns");
718 assert_eq!(mnw_kind(AlertCategory::TestDurationDrift), "latency");
719 assert_eq!(mnw_kind(AlertCategory::MonitoringOffline), "monitoring");
720 }
721
722 #[test]
723 fn mnw_severity_collapses_priorities_to_three_levels() {
724 assert_eq!(mnw_severity("critical"), "critical");
725 assert_eq!(mnw_severity("high"), "warning");
726 assert_eq!(mnw_severity("medium"), "warning");
727 assert_eq!(mnw_severity("low"), "info");
728 assert_eq!(mnw_severity("anything-else"), "info");
729 }
730
731 #[test]
732 fn truncate_respects_cap_and_char_boundaries() {
733 assert_eq!(truncate("short", 200), "short");
734 assert_eq!(truncate("abcdef", 3), "abc");
735 // Cutting mid multi-byte char steps back to the previous boundary rather
736 // than panicking on a non-boundary slice.
737 let s = "aé"; // 'é' is two bytes; byte index 2 is not a char boundary
738 assert_eq!(truncate(s, 2), "a");
739 }
740
741 fn test_alerter(pool: SqlitePool) -> Alerter {
742 let config = AlertConfig {
743 postmark_token: None, // dev mode
744 to: "test@example.com".to_string(),
745 from: "PoM Alerts <pom-alerts@makenot.work>".to_string(),
746 cooldown_secs: 300,
747 wam_url: None,
748 wam_token: None,
749 mnw_url: None,
750 alerts_ingest_token: None,
751 };
752 Alerter::new(config, pool, "test-instance".to_string()).unwrap()
753 }
754
755 #[tokio::test]
756 async fn cooldown_prevents_duplicate_alerts() {
757 let pool = db::connect_in_memory().await.unwrap();
758 let alerter = test_alerter(pool.clone());
759
760 // First alert, not in cooldown
761 assert!(!alerter.is_within_cooldown("health:mnw").await);
762
763 db::insert_alert(
764 &pool,
765 "health:mnw",
766 "health",
767 Some("operational"),
768 Some("error"),
769 None,
770 )
771 .await
772 .unwrap();
773
774 assert!(alerter.is_within_cooldown("health:mnw").await);
775 }
776
777 #[tokio::test]
778 async fn wam_ticket_falls_back_to_email_when_wam_url_unset() {
779 // CRITICAL #2: with no wam_url, a failure alert must NOT be silently
780 // dropped, it falls back to email. test_alerter has wam_url: None and
781 // postmark_token: None (dev mode), so the email path reports delivered.
782 let pool = db::connect_in_memory().await.unwrap();
783 let alerter = test_alerter(pool);
784 assert!(
785 alerter
786 .wam_ticket("subj", "body", "high", "pom-test", None)
787 .await,
788 "a failure alert with no wam_url must be delivered by email, not dropped"
789 );
790 }
791
792 #[tokio::test]
793 async fn cooldown_does_not_affect_other_targets() {
794 let pool = db::connect_in_memory().await.unwrap();
795 let alerter = test_alerter(pool.clone());
796
797 db::insert_alert(&pool, "health:mnw", "health", None, None, None)
798 .await
799 .unwrap();
800
801 // Different target should not be in cooldown
802 assert!(!alerter.is_within_cooldown("health:other").await);
803 }
804
805 #[tokio::test]
806 async fn dev_mode_does_not_send_http() {
807 let pool = db::connect_in_memory().await.unwrap();
808 let alerter = test_alerter(pool.clone());
809
810 // This should log instead of making HTTP calls (no panic, no error)
811 alerter
812 .send_health_alert("mnw", "MakeNotWork", "operational", "error", None)
813 .await;
814
815 // Verify alert was recorded in DB with the prefixed key (health:mnw),
816 // matching the cooldown lookup key format.
817 let latest = db::get_latest_alert_for_target(&pool, "health:mnw")
818 .await
819 .unwrap();
820 assert!(latest.is_some());
821 let row = latest.unwrap();
822 assert_eq!(row.alert_type, "health");
823 assert_eq!(row.from_status.as_deref(), Some("operational"));
824 assert_eq!(row.to_status.as_deref(), Some("error"));
825 }
826
827 #[tokio::test]
828 async fn route_alert_cooldown_key() {
829 let pool = db::connect_in_memory().await.unwrap();
830 let alerter = test_alerter(pool.clone());
831
832 assert!(!alerter.is_within_cooldown("route:mnw").await);
833
834 alerter
835 .send_route_failure_alert("mnw", "MakeNotWork", &["/docs/faq".to_string()])
836 .await;
837
838 assert!(alerter.is_within_cooldown("route:mnw").await);
839 assert!(!alerter.is_within_cooldown("route:mt").await);
840 }
841
842 #[tokio::test]
843 async fn recovery_clears_failure_cooldown_so_next_failure_fires() {
844 // Regression for the fail→recover→fail suppression the fuzz flagged (this
845 // test previously enshrined the bug: it asserted the 2nd failure stays
846 // suppressed). A recovery means the target came back, so a fresh failure
847 // is a genuine NEW outage and must not be muffled by the prior failure's
848 // cooldown.
849 let pool = db::connect_in_memory().await.unwrap();
850 let alerter = test_alerter(pool.clone());
851
852 // Fail → cooldown active.
853 alerter
854 .send_health_alert("mnw", "MakeNotWork", "operational", "error", None)
855 .await;
856 assert!(alerter.is_within_cooldown("health:mnw").await);
857
858 // Recover → the failure cooldown is now cleared.
859 alerter
860 .send_health_recovery("mnw", "MakeNotWork", "error")
861 .await;
862 assert!(
863 !alerter.is_within_cooldown("health:mnw").await,
864 "a recovery after the last failure must void the failure cooldown"
865 );
866 }
867
868 #[tokio::test]
869 async fn recovery_is_throttled_by_its_own_cooldown() {
870 // A flapping target must not emit one recovery per flap: after a recorded
871 // recovery, another recovery within cooldown is throttled, until a new
872 // failure intervenes.
873 let pool = db::connect_in_memory().await.unwrap();
874 let alerter = test_alerter(pool.clone());
875
876 db::insert_alert(
877 &pool,
878 "health:mnw",
879 "recovery",
880 None,
881 Some("operational"),
882 None,
883 )
884 .await
885 .unwrap();
886 assert!(alerter.is_recovery_within_cooldown("health:mnw").await);
887
888 // A failure after the recovery clears the recovery cooldown, so the NEXT
889 // genuine recovery still sends.
890 db::insert_alert(
891 &pool,
892 "health:mnw",
893 "health",
894 Some("operational"),
895 Some("error"),
896 None,
897 )
898 .await
899 .unwrap();
900 assert!(!alerter.is_recovery_within_cooldown("health:mnw").await);
901 }
902
903 #[tokio::test]
904 async fn failed_send_enqueues_for_retry_and_retry_delivers() {
905 // CRITICAL #1: a send that fails must not be lost, it is queued, and the
906 // retry path re-delivers it and records it in the ledger.
907 let pool = db::connect_in_memory().await.unwrap();
908 // Force delivery failure: WAM points at an unroutable URL and there is no
909 // postmark token, so the email fallback in dev mode... would succeed. Use a
910 // direct enqueue + retry to exercise the queue deterministically instead.
911 let pending = db::NewPendingAlert {
912 alert_key: "health:mnw",
913 category: "health",
914 channel: "email", // dev-mode email "delivers" (logs) → retry succeeds
915 subject: "[PoM] mnw: down",
916 body: "body",
917 priority: None,
918 source: None,
919 source_ref: None,
920 from_status: Some("operational"),
921 to_status: Some("error"),
922 error: None,
923 };
924 db::enqueue_pending_alert(&pool, &pending).await.unwrap();
925 let due = db::due_pending_alerts(&pool, 10).await.unwrap();
926 assert_eq!(due.len(), 1, "the undelivered alert must be queued");
927
928 let alerter = test_alerter(pool.clone());
929 assert!(
930 alerter.retry_pending(&due[0]).await,
931 "dev-mode email retry delivers"
932 );
933 db::delete_pending_alert(&pool, due[0].id).await.unwrap();
934
935 // The retry recorded the alert in the ledger (so cooldown now applies).
936 assert!(alerter.is_within_cooldown("health:mnw").await);
937 assert!(db::due_pending_alerts(&pool, 10).await.unwrap().is_empty());
938 }
939
940 #[tokio::test]
941 async fn dns_alert_cooldown_key() {
942 let pool = db::connect_in_memory().await.unwrap();
943 let alerter = test_alerter(pool.clone());
944
945 assert!(!alerter.is_within_cooldown("dns:mnw").await);
946
947 let mismatches = vec![crate::types::DnsCheckResult {
948 target: "mnw".to_string(),
949 name: "makenot.work".to_string(),
950 record_type: crate::types::DnsRecordType::A,
951 expected: vec!["1.2.3.4".to_string()],
952 actual: vec!["5.6.7.8".to_string()],
953 matches: false,
954 checked_at: chrono::Utc::now().to_rfc3339(),
955 error: None,
956 }];
957 alerter
958 .send_dns_mismatch_alert("mnw", "MakeNotWork", &mismatches)
959 .await;
960
961 assert!(alerter.is_within_cooldown("dns:mnw").await);
962 assert!(!alerter.is_within_cooldown("dns:other").await);
963 }
964
965 #[tokio::test]
966 async fn whois_alert_cooldown_key() {
967 let pool = db::connect_in_memory().await.unwrap();
968 let alerter = test_alerter(pool.clone());
969
970 assert!(!alerter.is_within_cooldown("whois:mnw").await);
971
972 alerter
973 .send_whois_expiry_alert("mnw", "MakeNotWork", "makenot.work", 15)
974 .await;
975
976 assert!(alerter.is_within_cooldown("whois:mnw").await);
977 assert!(!alerter.is_within_cooldown("whois:other").await);
978 }
979
980 #[tokio::test]
981 async fn health_alert_cooldown_key_matches_record_key() {
982 let pool = db::connect_in_memory().await.unwrap();
983 let alerter = test_alerter(pool.clone());
984
985 // Not in cooldown initially
986 assert!(!alerter.is_within_cooldown("health:example.com").await);
987
988 alerter
989 .send_health_alert("example.com", "Example", "operational", "error", None)
990 .await;
991
992 assert!(alerter.is_within_cooldown("health:example.com").await);
993
994 // Different target should NOT be in cooldown
995 assert!(!alerter.is_within_cooldown("health:other.com").await);
996 }
997
998 #[tokio::test]
999 async fn tls_expiry_alert_cooldown_key() {
1000 let pool = db::connect_in_memory().await.unwrap();
1001 let alerter = test_alerter(pool.clone());
1002
1003 assert!(!alerter.is_within_cooldown("tls:mnw").await);
1004 alerter
1005 .send_tls_expiry_alert("mnw", "makenot.work", 10, "2026-04-01T00:00:00Z")
1006 .await;
1007 assert!(alerter.is_within_cooldown("tls:mnw").await);
1008 }
1009
1010 #[tokio::test]
1011 async fn tls_error_alert_cooldown_key() {
1012 let pool = db::connect_in_memory().await.unwrap();
1013 let alerter = test_alerter(pool.clone());
1014
1015 assert!(!alerter.is_within_cooldown("tls:mnw").await);
1016 alerter
1017 .send_tls_error_alert("mnw", "makenot.work", "certificate expired")
1018 .await;
1019 assert!(alerter.is_within_cooldown("tls:mnw").await);
1020 }
1021
1022 #[tokio::test]
1023 async fn latency_drift_alert_cooldown_key() {
1024 let pool = db::connect_in_memory().await.unwrap();
1025 let alerter = test_alerter(pool.clone());
1026
1027 assert!(!alerter.is_within_cooldown("latency:mnw").await);
1028 alerter
1029 .send_latency_drift_alert("mnw", "MakeNotWork", "avg 500ms, baseline 100ms")
1030 .await;
1031 assert!(alerter.is_within_cooldown("latency:mnw").await);
1032 }
1033
1034 #[tokio::test]
1035 async fn test_duration_drift_alert_cooldown_key() {
1036 let pool = db::connect_in_memory().await.unwrap();
1037 let alerter = test_alerter(pool.clone());
1038
1039 assert!(!alerter.is_within_cooldown("test_duration:mnw").await);
1040 alerter
1041 .send_test_duration_drift_alert("mnw", "MakeNotWork", "drift: 120s vs 60s baseline")
1042 .await;
1043 assert!(alerter.is_within_cooldown("test_duration:mnw").await);
1044 }
1045
1046 #[tokio::test]
1047 async fn monitoring_offline_alert_cooldown_key() {
1048 let pool = db::connect_in_memory().await.unwrap();
1049 let alerter = test_alerter(pool.clone());
1050
1051 assert!(!alerter.is_within_cooldown("monitoring:self").await);
1052 alerter.send_monitoring_offline_alert(3).await;
1053 assert!(alerter.is_within_cooldown("monitoring:self").await);
1054 }
1055
1056 #[tokio::test]
1057 async fn route_recovery_does_not_start_cooldown() {
1058 let pool = db::connect_in_memory().await.unwrap();
1059 let alerter = test_alerter(pool.clone());
1060
1061 alerter
1062 .send_route_recovery_alert("mnw", "MakeNotWork", &["/health".to_string()])
1063 .await;
1064 // Recovery alerts are excluded from cooldown lookups, so sending a recovery
1065 // should NOT put the key into cooldown.
1066 assert!(!alerter.is_within_cooldown("route:mnw").await);
1067 }
1068
1069 #[tokio::test]
1070 async fn dns_recovery_does_not_start_cooldown() {
1071 let pool = db::connect_in_memory().await.unwrap();
1072 let alerter = test_alerter(pool.clone());
1073
1074 alerter.send_dns_recovery_alert("mnw", "MakeNotWork").await;
1075 assert!(!alerter.is_within_cooldown("dns:mnw").await);
1076 }
1077
1078 #[tokio::test]
1079 async fn tls_recovery_does_not_start_cooldown() {
1080 let pool = db::connect_in_memory().await.unwrap();
1081 let alerter = test_alerter(pool.clone());
1082
1083 alerter.send_tls_recovery("mnw", "MakeNotWork", 90).await;
1084 assert!(!alerter.is_within_cooldown("tls:mnw").await);
1085 }
1086
1087 // Pure priority/severity helpers (pin the <= boundaries)
1088
1089 #[test]
1090 fn tls_expiry_priority_boundaries() {
1091 // critical: days <= 3
1092 assert_eq!(
1093 tls_expiry_priority(-5),
1094 "critical",
1095 "negative days = already expired"
1096 );
1097 assert_eq!(tls_expiry_priority(0), "critical");
1098 assert_eq!(tls_expiry_priority(3), "critical");
1099 // high: 4..=7
1100 assert_eq!(tls_expiry_priority(4), "high");
1101 assert_eq!(tls_expiry_priority(7), "high");
1102 // medium: > 7
1103 assert_eq!(tls_expiry_priority(8), "medium");
1104 assert_eq!(tls_expiry_priority(90), "medium");
1105 }
1106
1107 #[test]
1108 fn whois_expiry_priority_boundaries() {
1109 // critical: days <= 7
1110 assert_eq!(whois_expiry_priority(-1), "critical");
1111 assert_eq!(whois_expiry_priority(7), "critical");
1112 // high: 8..=14
1113 assert_eq!(whois_expiry_priority(8), "high");
1114 assert_eq!(whois_expiry_priority(14), "high");
1115 // medium: > 14
1116 assert_eq!(whois_expiry_priority(15), "medium");
1117 assert_eq!(whois_expiry_priority(180), "medium");
1118 }
1119
1120 #[test]
1121 fn backup_status_priority_missing_is_critical() {
1122 assert_eq!(backup_status_priority("missing"), "critical");
1123 assert_eq!(backup_status_priority("stale"), "high");
1124 assert_eq!(backup_status_priority("error"), "high");
1125 assert_eq!(backup_status_priority("anything-else"), "high");
1126 assert_eq!(backup_status_priority(""), "high");
1127 }
1128
1129 #[test]
1130 fn backup_status_detail_arms() {
1131 assert_eq!(
1132 backup_status_detail("stale", Some(12)),
1133 "last backup is 12h old"
1134 );
1135 // `stale` with no age falls through to the default arm.
1136 assert_eq!(backup_status_detail("stale", None), "status: stale");
1137 assert_eq!(
1138 backup_status_detail("missing", None),
1139 "no backup files found"
1140 );
1141 assert_eq!(
1142 backup_status_detail("missing", Some(5)),
1143 "no backup files found"
1144 );
1145 assert_eq!(backup_status_detail("error", None), "backup check failed");
1146 assert_eq!(
1147 backup_status_detail("error", Some(99)),
1148 "backup check failed"
1149 );
1150 assert_eq!(backup_status_detail("weird", None), "status: weird");
1151 }
1152
1153 #[test]
1154 fn health_status_priority_arms() {
1155 assert_eq!(health_status_priority("error"), "critical");
1156 assert_eq!(health_status_priority("unreachable"), "critical");
1157 assert_eq!(health_status_priority("degraded"), "high");
1158 // Anything else (operational, unknown values) falls through to medium.
1159 assert_eq!(health_status_priority("operational"), "medium");
1160 assert_eq!(health_status_priority("flapping"), "medium");
1161 assert_eq!(health_status_priority(""), "medium");
1162 }
1163 }
1164