Skip to main content

max / makenotwork

42.4 KB · 1166 lines History Blame Raw
1 //! Email alerting via Postmark API.
2 //!
3 //! Sends alerts on health status transitions and peer disappearance/recovery.
4 //! If no `postmark_token` is configured, alerts are logged to stdout instead.
5
6 use sqlx::SqlitePool;
7 use tracing::{info, warn};
8
9 use crate::config::AlertConfig;
10 use crate::db;
11 use crate::types::AlertCategory;
12
13 mod backup;
14 mod ca_bundle;
15 mod cors;
16 mod dns;
17 mod health;
18 mod latency;
19 mod offline;
20 mod peer;
21 mod route;
22 mod scan;
23 mod systemd;
24 mod test_duration;
25 mod test_suite;
26 mod tls;
27 mod whois;
28
29 /// WAM ticket priority for a transition into a non-operational health status.
30 fn health_status_priority(to_status: &str) -> &'static str {
31 match to_status {
32 "error" | "unreachable" => "critical",
33 "degraded" => "high",
34 _ => "medium",
35 }
36 }
37
38 /// WAM ticket priority for a TLS certificate that expires in `days`.
39 fn tls_expiry_priority(days: i64) -> &'static str {
40 if days <= 3 {
41 "critical"
42 } else if days <= 7 {
43 "high"
44 } else {
45 "medium"
46 }
47 }
48
49 /// WAM ticket priority for a domain registration that expires in `days`.
50 fn whois_expiry_priority(days: i64) -> &'static str {
51 if days <= 7 {
52 "critical"
53 } else if days <= 14 {
54 "high"
55 } else {
56 "medium"
57 }
58 }
59
60 /// WAM ticket priority for a stale/missing/error backup status.
61 fn backup_status_priority(status: &str) -> &'static str {
62 if status == "missing" {
63 "critical"
64 } else {
65 "high"
66 }
67 }
68
69 /// Human-readable detail text for a backup status alert.
70 fn backup_status_detail(status: &str, age_hours: Option<i64>) -> String {
71 match (status, age_hours) {
72 ("stale", Some(hours)) => format!("last backup is {hours}h old"),
73 ("missing", _) => "no backup files found".to_string(),
74 ("error", _) => "backup check failed".to_string(),
75 _ => format!("status: {status}"),
76 }
77 }
78
79 /// Map PoM's fine-grained [`AlertCategory`] onto MNW's domain-level alert kind
80 /// (the `AlertKind` enum in the MNW server's `db/admin_alerts.rs`). Recoveries
81 /// and sub-conditions fold onto their domain; the failure/recovery distinction
82 /// rides on severity, not kind. A new PoM category needs a line here, and, if
83 /// it introduces a new domain, a matching MNW `AlertKind` variant (else MNW
84 /// rejects the push with 422).
85 fn mnw_kind(category: AlertCategory) -> &'static str {
86 use AlertCategory::{
87 BackupRecovery, BackupStale, CaBundleRecovery, CaBundleStale, CorsFailure, CorsRecovery,
88 DnsMismatch, DnsRecovery, Health, LatencyDrift, LatencyRecovery, MonitoringOffline,
89 MonitoringRecovery, PeerMissing, PeerRecovery, Recovery, RouteFailure, RouteRecovery,
90 ScanPipelineDegraded, ScanPipelineRecovery, SystemdFailure, SystemdRecovery,
91 TestDurationDrift, TestFailure, TestRecovery, TlsError, TlsExpiry, TlsRecovery, WhoisError,
92 WhoisExpiry,
93 };
94 match category {
95 // A red test suite folds onto "health": it is a statement about the
96 // target being sick, and MNW has no `test` AlertKind (an unknown kind
97 // is a 422 from the ingest endpoint). The sub-condition rides in the
98 // dedup key, the same way `Tls` already folds three PoM categories.
99 Health | Recovery | TestFailure | TestRecovery => "health",
100 // The host CA bundle folds onto the "tls" domain: it is a trust-anchor
101 // problem, and every symptom it produces is a failed handshake. Adding a
102 // domain would need a matching MNW `AlertKind` or the ingest 422s.
103 TlsExpiry | TlsError | TlsRecovery | CaBundleStale | CaBundleRecovery => "tls",
104 DnsMismatch | DnsRecovery => "dns",
105 WhoisExpiry | WhoisError => "whois",
106 LatencyDrift | LatencyRecovery | TestDurationDrift => "latency",
107 CorsFailure | CorsRecovery => "cors",
108 BackupStale | BackupRecovery => "backup",
109 PeerMissing | PeerRecovery => "peer",
110 RouteFailure | RouteRecovery => "route",
111 ScanPipelineDegraded | ScanPipelineRecovery => "scan",
112 // Daemon liveness folds onto the "monitoring" domain rather than adding a
113 // new MNW `AlertKind` (an unknown kind is a 422 from the ingest endpoint).
114 // It sits with MonitoringOffline: both are "the ops plane itself is sick".
115 SystemdFailure | SystemdRecovery | MonitoringOffline | MonitoringRecovery => "monitoring",
116 }
117 }
118
119 /// Map PoM's WAM priority string onto MNW's three-level severity. Recoveries do
120 /// not carry a priority; the caller passes `"info"` for those directly.
121 fn mnw_severity(priority: &str) -> &'static str {
122 match priority {
123 "critical" => "critical",
124 "high" | "medium" => "warning",
125 _ => "info",
126 }
127 }
128
129 /// Truncate `s` to at most `max` bytes on a char boundary. MNW caps title at
130 /// 200 and body at 5000; over-long input would be a 422, so clamp instead.
131 fn truncate(s: &str, max: usize) -> String {
132 if s.len() <= max {
133 return s.to_string();
134 }
135 let mut end = max;
136 while !s.is_char_boundary(end) {
137 end -= 1;
138 }
139 s[..end].to_string()
140 }
141
142 #[derive(Clone)]
143 pub struct Alerter {
144 config: AlertConfig,
145 client: reqwest::Client,
146 pool: SqlitePool,
147 instance_name: String,
148 wam_url: Option<String>,
149 }
150
151 /// The record-context an alert carries: what to write to the `alerts` ledger on
152 /// successful delivery, and what to persist for retry on failure.
153 struct AlertMeta<'a> {
154 key: &'a str,
155 category: AlertCategory,
156 from: Option<&'a str>,
157 to: Option<&'a str>,
158 error: Option<&'a str>,
159 }
160
161 /// `true` if `sent_at` (rfc3339) is within `cooldown_secs` of now. An unparseable
162 /// timestamp is treated as expired (not in cooldown) so a bad row can never wedge
163 /// alerting shut.
164 fn within_cooldown_secs(sent_at: &str, cooldown_secs: u64) -> bool {
165 match chrono::DateTime::parse_from_rfc3339(sent_at) {
166 Ok(dt) => chrono::Utc::now().signed_duration_since(dt).num_seconds() < cooldown_secs as i64,
167 Err(_) => false,
168 }
169 }
170
171 impl Alerter {
172 pub fn new(
173 config: AlertConfig,
174 pool: SqlitePool,
175 instance_name: String,
176 ) -> Result<Self, reqwest::Error> {
177 // reqwest 0.13's builder can fail (platform trust-store load) and
178 // `Client::new()`/`unwrap_or_default()` panic on that same failure. Return
179 // the error so the caller can disable alerting rather than crash at startup.
180 let client = crate::tls::https_client_builder()
181 .timeout(std::time::Duration::from_secs(10))
182 .build()?;
183 let wam_url = config.wam_url.clone();
184 if wam_url.is_none() {
185 warn!(
186 "alerts: wam_url not configured, failure alerts will be delivered by email \
187 instead of WAM tickets (set alerts.wam_url to route them to WAM)"
188 );
189 } else if config.wam_token.is_none() {
190 warn!(
191 "alerts: wam_url set but wam_token missing, WAM fails closed so ticket \
192 creation will 401 and fall back to email (set alerts.wam_token or \
193 POM_WAM_TOKEN)"
194 );
195 }
196 match (&config.mnw_url, &config.alerts_ingest_token) {
197 (Some(_), Some(_)) => {
198 info!("alerts: MNW sink enabled, alerts also push to the MNW operator log");
199 }
200 (Some(_), None) => warn!(
201 "alerts: mnw_url set but alerts_ingest_token missing, MNW sink disabled \
202 (set alerts.alerts_ingest_token or POM_ALERTS_INGEST_TOKEN)"
203 ),
204 _ => {}
205 }
206 Ok(Self {
207 config,
208 client,
209 pool,
210 instance_name,
211 wam_url,
212 })
213 }
214
215 /// Push one alert to MNW's operator log (`POST /api/internal/alerts`).
216 /// Returns `true` iff MNW accepted it. Returns `false`: never enqueues,
217 /// when the sink is unconfigured; callers gate on config before treating a
218 /// `false` as a delivery miss worth queuing. Bounded by a 10s timeout so MNW
219 /// latency can't stall the probe loop. The server dedups on `dedup_key`
220 /// (`{kind}:{target}`), so a retried or repeated push of one condition
221 /// collapses rather than duplicating.
222 async fn deliver_mnw(
223 &self,
224 kind: &str,
225 key: &str,
226 severity: &str,
227 subject: &str,
228 body: &str,
229 ) -> bool {
230 let (Some(base), Some(token)) = (
231 self.config.mnw_url.as_deref(),
232 self.config.alerts_ingest_token.as_deref(),
233 ) else {
234 return false;
235 };
236 let payload = serde_json::json!({
237 "source": "pom",
238 "kind": kind,
239 "severity": severity,
240 "title": truncate(subject, 200),
241 "body": truncate(body, 5000),
242 "dedup_key": truncate(&format!("{kind}:{key}"), 200),
243 "details": { "instance": self.instance_name, "target": key },
244 });
245 let url = format!("{}/api/internal/alerts", base.trim_end_matches('/'));
246 let send_fut = self
247 .client
248 .post(&url)
249 .bearer_auth(token)
250 .json(&payload)
251 .send();
252 match tokio::time::timeout(std::time::Duration::from_secs(10), send_fut).await {
253 Ok(Ok(resp)) if resp.status().is_success() => {
254 info!("mnw alert pushed: {kind}/{severity} {key}");
255 true
256 }
257 Ok(Ok(resp)) => {
258 let status = resp.status();
259 let text = resp.text().await.unwrap_or_default();
260 warn!("mnw alert push failed ({status}): {text}");
261 false
262 }
263 Ok(Err(e)) => {
264 warn!("mnw alert push error: {e}");
265 false
266 }
267 Err(_) => {
268 warn!("mnw alert push timed out: {kind}/{key}");
269 false
270 }
271 }
272 }
273
274 /// Durable push to the MNW operator log: deliver now, and on failure enqueue
275 /// for retry through the same `pending_alerts` queue WAM/email use. No-op
276 /// unless both `mnw_url` and `alerts_ingest_token` are set.
277 ///
278 /// MNW is a *secondary* sink: the primary channel (WAM/email) already
279 /// recorded this alert in the ledger, so a queued MNW row carries
280 /// `channel = "mnw"` and, unlike WAM/email retries, does not re-record on
281 /// delivery (see [`retry_pending`]), it only needs to eventually land.
282 /// `severity` derives from `priority` exactly as the retry path recomputes
283 /// it (`None` → recovery → `"info"`), so live and retried pushes match.
284 async fn push_mnw(
285 &self,
286 category: AlertCategory,
287 key: &str,
288 priority: Option<&str>,
289 subject: &str,
290 body: &str,
291 ) {
292 if self.config.mnw_url.is_none() || self.config.alerts_ingest_token.is_none() {
293 return;
294 }
295 let kind = mnw_kind(category);
296 let severity = priority.map_or("info", mnw_severity);
297 if self.deliver_mnw(kind, key, severity, subject, body).await {
298 return;
299 }
300 let category = category.to_string();
301 let pending = db::NewPendingAlert {
302 alert_key: key,
303 category: &category,
304 channel: "mnw",
305 subject,
306 body,
307 priority,
308 source: Some("pom"),
309 source_ref: None,
310 from_status: None,
311 to_status: None,
312 error: None,
313 };
314 if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await {
315 warn!("failed to enqueue undelivered mnw alert for retry: {e}");
316 } else {
317 warn!("mnw alert to {key} undelivered; queued for retry");
318 }
319 }
320
321 /// Whether a FAILURE alert for `target` is within its cooldown.
322 ///
323 /// A recorded recovery *newer* than the last failure clears the cooldown: the
324 /// target came back, so a fresh failure is a genuine new outage and must not
325 /// be suppressed by the prior failure's window (fuzz-2026-07-06 fail→recover→
326 /// fail suppression).
327 async fn is_within_cooldown(&self, target: &str) -> bool {
328 let Ok(Some(latest_fail)) = db::get_latest_alert_for_target(&self.pool, target).await
329 else {
330 return false; // never alerted → not in cooldown
331 };
332 // A recovery after the last failure voids the cooldown.
333 if let Ok(Some(rec)) = db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
334 && rec.id > latest_fail.id
335 {
336 return false;
337 }
338 within_cooldown_secs(&latest_fail.sent_at, self.config.cooldown_secs)
339 }
340
341 /// Whether a RECOVERY for `target` is within its own cooldown, throttles a
342 /// flapping target so it can't emit one recovery email per flap
343 /// (fuzz-2026-07-06 un-throttled recoveries). A failure newer than the last
344 /// recovery clears it, so a genuine recover-after-a-new-outage still sends.
345 async fn is_recovery_within_cooldown(&self, target: &str) -> bool {
346 let Ok(Some(latest_rec)) =
347 db::get_latest_alert_matching(&self.pool, target, "%recovery%").await
348 else {
349 return false;
350 };
351 if let Ok(Some(fail)) = db::get_latest_alert_for_target(&self.pool, target).await
352 && fail.id > latest_rec.id
353 {
354 return false;
355 }
356 within_cooldown_secs(&latest_rec.sent_at, self.config.cooldown_secs)
357 }
358
359 /// Send an email via Postmark. Returns `true` if the message was accepted
360 /// (or in dev mode where there is no token, the operator sees it logged).
361 async fn send_email(&self, subject: &str, body: &str) -> bool {
362 let Some(ref token) = self.config.postmark_token else {
363 info!("[dev] alert: {subject}");
364 info!("[dev] {body}");
365 return true;
366 };
367
368 let payload = serde_json::json!({
369 "From": self.config.from,
370 "To": self.config.to,
371 "Subject": subject,
372 "TextBody": body,
373 });
374
375 let send_fut = self
376 .client
377 .post("https://api.postmarkapp.com/email")
378 .header("X-Postmark-Server-Token", token)
379 .header("Content-Type", "application/json")
380 .header("Accept", "application/json")
381 .json(&payload)
382 .send();
383
384 // Wrap in a 30-second timeout to prevent Postmark latency from blocking
385 // the alert task. The reqwest client has its own 10s timeout, but this
386 // guards against DNS resolution stalls and connection pool exhaustion.
387 match tokio::time::timeout(std::time::Duration::from_secs(30), send_fut).await {
388 Ok(Ok(resp)) if resp.status().is_success() => {
389 info!("alert sent: {subject}");
390 true
391 }
392 Ok(Ok(resp)) => {
393 let status = resp.status();
394 let text = resp.text().await.unwrap_or_default();
395 warn!("postmark error ({status}): {text}");
396 false
397 }
398 Ok(Err(e)) => {
399 warn!("failed to send alert: {e}");
400 false
401 }
402 Err(_) => {
403 warn!("alert send timed out after 30s: {subject}");
404 false
405 }
406 }
407 }
408
409 async fn record_alert(
410 &self,
411 target: &str,
412 alert_type: AlertCategory,
413 from_status: Option<&str>,
414 to_status: Option<&str>,
415 error: Option<&str>,
416 ) {
417 self.record_alert_str(
418 target,
419 &alert_type.to_string(),
420 from_status,
421 to_status,
422 error,
423 )
424 .await;
425 }
426
427 /// String-typed variant so the retry task can record a queued alert straight
428 /// from its stored category without round-tripping through the enum.
429 async fn record_alert_str(
430 &self,
431 target: &str,
432 alert_type: &str,
433 from_status: Option<&str>,
434 to_status: Option<&str>,
435 error: Option<&str>,
436 ) {
437 if let Err(e) = db::insert_alert(
438 &self.pool,
439 target,
440 alert_type,
441 from_status,
442 to_status,
443 error,
444 )
445 .await
446 {
447 warn!("failed to record alert: {e}");
448 }
449 }
450
451 /// Fire a failure alert: cooldown-gate, then dispatch. The single entry point
452 /// every `send_*_alert` funnels through, so the cooldown check + dispatch is
453 /// written once rather than copy-pasted into each of the ~13 methods.
454 async fn fire_failure(
455 &self,
456 subject: &str,
457 body: &str,
458 priority: &str,
459 source: &str,
460 source_ref: Option<&str>,
461 meta: AlertMeta<'_>,
462 ) {
463 if self.is_within_cooldown(meta.key).await {
464 info!("alert cooldown active for {}, skipping", meta.key);
465 return;
466 }
467 // Primary sink (WAM) first so a slow MNW push can't delay the ticket;
468 // capture what push_mnw needs before `meta` is moved into dispatch.
469 let (category, key) = (meta.category, meta.key);
470 self.dispatch_wam(subject, body, priority, source, source_ref, meta)
471 .await;
472 self.push_mnw(category, key, Some(priority), subject, body)
473 .await;
474 }
475
476 /// Fire a recovery: recovery-cooldown-gate, then dispatch by email. The single
477 /// entry point every `send_*_recovery` funnels through.
478 async fn fire_recovery(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
479 if self.is_recovery_within_cooldown(meta.key).await {
480 info!("recovery cooldown active for {}, skipping", meta.key);
481 return;
482 }
483 let (category, key) = (meta.category, meta.key);
484 self.dispatch_email(subject, body, meta).await;
485 self.push_mnw(category, key, None, subject, body).await;
486 }
487
488 /// Dispatch a WAM failure alert: attempt delivery (WAM, falling back to
489 /// email), record it on success, or persist it for retry on failure. Gating
490 /// the ledger write on delivery, and queueing the miss, is what stops a
491 /// transient send failure at a status transition from silencing the whole
492 /// outage: the transition fires once, but the queued alert is retried until it
493 /// lands (fuzz-2026-07-06 CRITICAL #1).
494 async fn dispatch_wam(
495 &self,
496 subject: &str,
497 body: &str,
498 priority: &str,
499 source: &str,
500 source_ref: Option<&str>,
501 meta: AlertMeta<'_>,
502 ) {
503 if self
504 .wam_ticket(subject, body, priority, source, source_ref)
505 .await
506 {
507 self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error)
508 .await;
509 } else {
510 self.enqueue_undelivered(
511 subject,
512 body,
513 "wam",
514 Some(priority),
515 Some(source),
516 source_ref,
517 &meta,
518 )
519 .await;
520 }
521 }
522
523 /// Dispatch an email alert (recoveries, and failure alerts that already chose
524 /// email): record on success, persist for retry on failure.
525 async fn dispatch_email(&self, subject: &str, body: &str, meta: AlertMeta<'_>) {
526 if self.send_email(subject, body).await {
527 self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error)
528 .await;
529 } else {
530 self.enqueue_undelivered(subject, body, "email", None, None, None, &meta)
531 .await;
532 }
533 }
534
535 #[allow(clippy::too_many_arguments)]
536 async fn enqueue_undelivered(
537 &self,
538 subject: &str,
539 body: &str,
540 channel: &str,
541 priority: Option<&str>,
542 source: Option<&str>,
543 source_ref: Option<&str>,
544 meta: &AlertMeta<'_>,
545 ) {
546 let category = meta.category.to_string();
547 let pending = db::NewPendingAlert {
548 alert_key: meta.key,
549 category: &category,
550 channel,
551 subject,
552 body,
553 priority,
554 source,
555 source_ref,
556 from_status: meta.from,
557 to_status: meta.to,
558 error: meta.error,
559 };
560 if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await {
561 warn!("failed to enqueue undelivered alert for retry: {e}");
562 } else {
563 warn!("alert to {} undelivered; queued for retry", meta.key);
564 }
565 }
566
567 /// Re-attempt one queued alert. On delivery, record it in the ledger and
568 /// signal the caller to delete the row; otherwise leave it for a later tick.
569 /// Called by the retry task in `serve`.
570 pub async fn retry_pending(&self, p: &db::PendingAlertRow) -> bool {
571 // MNW is a secondary sink: the primary channel already wrote this alert
572 // to the ledger, so a successful retry only clears the queued row, it
573 // must not re-record (that would double-count and skew the cooldown).
574 // Reconstruct kind/severity the same way the live push derives them.
575 if p.channel == "mnw" {
576 let kind = p
577 .category
578 .parse::<AlertCategory>()
579 .map_or("health", mnw_kind);
580 let severity = p.priority.as_deref().map_or("info", mnw_severity);
581 return self
582 .deliver_mnw(kind, &p.alert_key, severity, &p.subject, &p.body)
583 .await;
584 }
585 let delivered = match p.channel.as_str() {
586 "wam" => {
587 self.wam_ticket(
588 &p.subject,
589 &p.body,
590 p.priority.as_deref().unwrap_or("high"),
591 p.source.as_deref().unwrap_or("pom"),
592 p.source_ref.as_deref(),
593 )
594 .await
595 }
596 _ => self.send_email(&p.subject, &p.body).await,
597 };
598 if delivered {
599 self.record_alert_str(
600 &p.alert_key,
601 &p.category,
602 p.from_status.as_deref(),
603 p.to_status.as_deref(),
604 p.error.as_deref(),
605 )
606 .await;
607 }
608 delivered
609 }
610
611 /// Create a WAM ticket for a failure alert. Returns `true` if the alert was
612 /// delivered by *some* channel.
613 ///
614 /// If `wam_url` is unset the failure alert would otherwise be dropped on the
615 /// floor, failures route to WAM, recoveries to email, so a missing `wam_url`
616 /// silently disabled the entire down-alert channel while recovery emails kept
617 /// firing (fuzz-2026-07-06 CRITICAL #2). Fall back to email instead: a
618 /// monitoring tool must never silently swallow a down-alert. WAM delivery
619 /// failures also fall back to email so a WAM outage can't blind the operator.
620 async fn wam_ticket(
621 &self,
622 title: &str,
623 body: &str,
624 priority: &str,
625 source: &str,
626 source_ref: Option<&str>,
627 ) -> bool {
628 let Some(ref base_url) = self.wam_url else {
629 // No WAM configured: deliver the failure alert by email instead of
630 // dropping it.
631 return self.send_email(title, body).await;
632 };
633 let url = format!("{base_url}/tickets");
634
635 let mut payload = serde_json::json!({
636 "title": title,
637 "body": body,
638 "priority": priority,
639 "source": source,
640 });
641 if let Some(r) = source_ref {
642 payload["source_ref"] = serde_json::json!(r);
643 }
644
645 // WAM fails closed: every request must carry `Authorization: Bearer
646 // <token>` or it 401s (see MNW/wam/src/api.rs). Send the token when
647 // configured; a missing token 401s and falls back to email below rather
648 // than silently dropping the down-alert.
649 let mut req = self.client.post(&url).json(&payload);
650 if let Some(token) = self.config.wam_token.as_deref() {
651 req = req.bearer_auth(token);
652 }
653
654 match req.send().await {
655 Ok(resp) if resp.status().is_success() => {
656 info!("WAM ticket created: {title}");
657 true
658 }
659 Ok(resp) => {
660 warn!(
661 "WAM ticket creation returned {}: {title}; falling back to email",
662 resp.status()
663 );
664 self.send_email(title, body).await
665 }
666 Err(e) => {
667 warn!("WAM unreachable: {e}; falling back to email");
668 self.send_email(title, body).await
669 }
670 }
671 }
672 }
673
674 #[cfg(test)]
675 mod tests {
676 use super::*;
677
678 #[tokio::test]
679 async fn mnw_retry_routes_to_sink_and_never_records() {
680 // A queued mnw row must retry via the MNW sink, not the WAM/email path,
681 // and must NOT write the ledger (the primary channel already did). With
682 // the sink unconfigured (test_alerter has no mnw_url), delivery fails and
683 // the row stays queued, and the cooldown ledger is untouched.
684 let pool = db::connect_in_memory().await.unwrap();
685 let alerter = test_alerter(pool.clone());
686 let row = db::PendingAlertRow {
687 id: 1,
688 alert_key: "makenot.work".to_string(),
689 category: "tls_expiry".to_string(),
690 channel: "mnw".to_string(),
691 subject: "TLS expiring".to_string(),
692 body: "cert expires soon".to_string(),
693 priority: Some("high".to_string()),
694 source: Some("pom".to_string()),
695 source_ref: None,
696 from_status: None,
697 to_status: None,
698 error: None,
699 attempts: 0,
700 };
701 assert!(
702 !alerter.retry_pending(&row).await,
703 "unconfigured sink cannot deliver"
704 );
705 assert!(
706 !alerter.is_within_cooldown("makenot.work").await,
707 "mnw retry must not record to the ledger"
708 );
709 }
710
711 #[test]
712 fn mnw_kind_folds_failure_and_recovery_to_same_domain() {
713 // A condition and its recovery must map to one kind so they share a
714 // dedup thread in the MNW log; the failure/recovery split rides severity.
715 assert_eq!(mnw_kind(AlertCategory::TlsExpiry), "tls");
716 assert_eq!(mnw_kind(AlertCategory::TlsError), "tls");
717 assert_eq!(mnw_kind(AlertCategory::TlsRecovery), "tls");
718 assert_eq!(mnw_kind(AlertCategory::DnsMismatch), "dns");
719 assert_eq!(mnw_kind(AlertCategory::DnsRecovery), "dns");
720 assert_eq!(mnw_kind(AlertCategory::TestDurationDrift), "latency");
721 assert_eq!(mnw_kind(AlertCategory::MonitoringOffline), "monitoring");
722 }
723
724 #[test]
725 fn mnw_severity_collapses_priorities_to_three_levels() {
726 assert_eq!(mnw_severity("critical"), "critical");
727 assert_eq!(mnw_severity("high"), "warning");
728 assert_eq!(mnw_severity("medium"), "warning");
729 assert_eq!(mnw_severity("low"), "info");
730 assert_eq!(mnw_severity("anything-else"), "info");
731 }
732
733 #[test]
734 fn truncate_respects_cap_and_char_boundaries() {
735 assert_eq!(truncate("short", 200), "short");
736 assert_eq!(truncate("abcdef", 3), "abc");
737 // Cutting mid multi-byte char steps back to the previous boundary rather
738 // than panicking on a non-boundary slice.
739 let s = "aé"; // 'é' is two bytes; byte index 2 is not a char boundary
740 assert_eq!(truncate(s, 2), "a");
741 }
742
743 fn test_alerter(pool: SqlitePool) -> Alerter {
744 let config = AlertConfig {
745 postmark_token: None, // dev mode
746 to: "test@example.com".to_string(),
747 from: "PoM Alerts <pom-alerts@makenot.work>".to_string(),
748 cooldown_secs: 300,
749 wam_url: None,
750 wam_token: None,
751 mnw_url: None,
752 alerts_ingest_token: None,
753 };
754 Alerter::new(config, pool, "test-instance".to_string()).unwrap()
755 }
756
757 #[tokio::test]
758 async fn cooldown_prevents_duplicate_alerts() {
759 let pool = db::connect_in_memory().await.unwrap();
760 let alerter = test_alerter(pool.clone());
761
762 // First alert, not in cooldown
763 assert!(!alerter.is_within_cooldown("health:mnw").await);
764
765 db::insert_alert(
766 &pool,
767 "health:mnw",
768 "health",
769 Some("operational"),
770 Some("error"),
771 None,
772 )
773 .await
774 .unwrap();
775
776 assert!(alerter.is_within_cooldown("health:mnw").await);
777 }
778
779 #[tokio::test]
780 async fn wam_ticket_falls_back_to_email_when_wam_url_unset() {
781 // CRITICAL #2: with no wam_url, a failure alert must NOT be silently
782 // dropped, it falls back to email. test_alerter has wam_url: None and
783 // postmark_token: None (dev mode), so the email path reports delivered.
784 let pool = db::connect_in_memory().await.unwrap();
785 let alerter = test_alerter(pool);
786 assert!(
787 alerter
788 .wam_ticket("subj", "body", "high", "pom-test", None)
789 .await,
790 "a failure alert with no wam_url must be delivered by email, not dropped"
791 );
792 }
793
794 #[tokio::test]
795 async fn cooldown_does_not_affect_other_targets() {
796 let pool = db::connect_in_memory().await.unwrap();
797 let alerter = test_alerter(pool.clone());
798
799 db::insert_alert(&pool, "health:mnw", "health", None, None, None)
800 .await
801 .unwrap();
802
803 // Different target should not be in cooldown
804 assert!(!alerter.is_within_cooldown("health:other").await);
805 }
806
807 #[tokio::test]
808 async fn dev_mode_does_not_send_http() {
809 let pool = db::connect_in_memory().await.unwrap();
810 let alerter = test_alerter(pool.clone());
811
812 // This should log instead of making HTTP calls (no panic, no error)
813 alerter
814 .send_health_alert("mnw", "MakeNotWork", "operational", "error", None)
815 .await;
816
817 // Verify alert was recorded in DB with the prefixed key (health:mnw),
818 // matching the cooldown lookup key format.
819 let latest = db::get_latest_alert_for_target(&pool, "health:mnw")
820 .await
821 .unwrap();
822 assert!(latest.is_some());
823 let row = latest.unwrap();
824 assert_eq!(row.alert_type, "health");
825 assert_eq!(row.from_status.as_deref(), Some("operational"));
826 assert_eq!(row.to_status.as_deref(), Some("error"));
827 }
828
829 #[tokio::test]
830 async fn route_alert_cooldown_key() {
831 let pool = db::connect_in_memory().await.unwrap();
832 let alerter = test_alerter(pool.clone());
833
834 assert!(!alerter.is_within_cooldown("route:mnw").await);
835
836 alerter
837 .send_route_failure_alert("mnw", "MakeNotWork", &["/docs/faq".to_string()])
838 .await;
839
840 assert!(alerter.is_within_cooldown("route:mnw").await);
841 assert!(!alerter.is_within_cooldown("route:mt").await);
842 }
843
844 #[tokio::test]
845 async fn recovery_clears_failure_cooldown_so_next_failure_fires() {
846 // Regression for the fail→recover→fail suppression the fuzz flagged (this
847 // test previously enshrined the bug: it asserted the 2nd failure stays
848 // suppressed). A recovery means the target came back, so a fresh failure
849 // is a genuine NEW outage and must not be muffled by the prior failure's
850 // cooldown.
851 let pool = db::connect_in_memory().await.unwrap();
852 let alerter = test_alerter(pool.clone());
853
854 // Fail → cooldown active.
855 alerter
856 .send_health_alert("mnw", "MakeNotWork", "operational", "error", None)
857 .await;
858 assert!(alerter.is_within_cooldown("health:mnw").await);
859
860 // Recover → the failure cooldown is now cleared.
861 alerter
862 .send_health_recovery("mnw", "MakeNotWork", "error")
863 .await;
864 assert!(
865 !alerter.is_within_cooldown("health:mnw").await,
866 "a recovery after the last failure must void the failure cooldown"
867 );
868 }
869
870 #[tokio::test]
871 async fn recovery_is_throttled_by_its_own_cooldown() {
872 // A flapping target must not emit one recovery per flap: after a recorded
873 // recovery, another recovery within cooldown is throttled, until a new
874 // failure intervenes.
875 let pool = db::connect_in_memory().await.unwrap();
876 let alerter = test_alerter(pool.clone());
877
878 db::insert_alert(
879 &pool,
880 "health:mnw",
881 "recovery",
882 None,
883 Some("operational"),
884 None,
885 )
886 .await
887 .unwrap();
888 assert!(alerter.is_recovery_within_cooldown("health:mnw").await);
889
890 // A failure after the recovery clears the recovery cooldown, so the NEXT
891 // genuine recovery still sends.
892 db::insert_alert(
893 &pool,
894 "health:mnw",
895 "health",
896 Some("operational"),
897 Some("error"),
898 None,
899 )
900 .await
901 .unwrap();
902 assert!(!alerter.is_recovery_within_cooldown("health:mnw").await);
903 }
904
905 #[tokio::test]
906 async fn failed_send_enqueues_for_retry_and_retry_delivers() {
907 // CRITICAL #1: a send that fails must not be lost, it is queued, and the
908 // retry path re-delivers it and records it in the ledger.
909 let pool = db::connect_in_memory().await.unwrap();
910 // Force delivery failure: WAM points at an unroutable URL and there is no
911 // postmark token, so the email fallback in dev mode... would succeed. Use a
912 // direct enqueue + retry to exercise the queue deterministically instead.
913 let pending = db::NewPendingAlert {
914 alert_key: "health:mnw",
915 category: "health",
916 channel: "email", // dev-mode email "delivers" (logs) → retry succeeds
917 subject: "[PoM] mnw: down",
918 body: "body",
919 priority: None,
920 source: None,
921 source_ref: None,
922 from_status: Some("operational"),
923 to_status: Some("error"),
924 error: None,
925 };
926 db::enqueue_pending_alert(&pool, &pending).await.unwrap();
927 let due = db::due_pending_alerts(&pool, 10).await.unwrap();
928 assert_eq!(due.len(), 1, "the undelivered alert must be queued");
929
930 let alerter = test_alerter(pool.clone());
931 assert!(
932 alerter.retry_pending(&due[0]).await,
933 "dev-mode email retry delivers"
934 );
935 db::delete_pending_alert(&pool, due[0].id).await.unwrap();
936
937 // The retry recorded the alert in the ledger (so cooldown now applies).
938 assert!(alerter.is_within_cooldown("health:mnw").await);
939 assert!(db::due_pending_alerts(&pool, 10).await.unwrap().is_empty());
940 }
941
942 #[tokio::test]
943 async fn dns_alert_cooldown_key() {
944 let pool = db::connect_in_memory().await.unwrap();
945 let alerter = test_alerter(pool.clone());
946
947 assert!(!alerter.is_within_cooldown("dns:mnw").await);
948
949 let mismatches = vec![crate::types::DnsCheckResult {
950 target: "mnw".to_string(),
951 name: "makenot.work".to_string(),
952 record_type: crate::types::DnsRecordType::A,
953 expected: vec!["1.2.3.4".to_string()],
954 actual: vec!["5.6.7.8".to_string()],
955 matches: false,
956 checked_at: chrono::Utc::now().to_rfc3339(),
957 error: None,
958 }];
959 alerter
960 .send_dns_mismatch_alert("mnw", "MakeNotWork", &mismatches)
961 .await;
962
963 assert!(alerter.is_within_cooldown("dns:mnw").await);
964 assert!(!alerter.is_within_cooldown("dns:other").await);
965 }
966
967 #[tokio::test]
968 async fn whois_alert_cooldown_key() {
969 let pool = db::connect_in_memory().await.unwrap();
970 let alerter = test_alerter(pool.clone());
971
972 assert!(!alerter.is_within_cooldown("whois:mnw").await);
973
974 alerter
975 .send_whois_expiry_alert("mnw", "MakeNotWork", "makenot.work", 15)
976 .await;
977
978 assert!(alerter.is_within_cooldown("whois:mnw").await);
979 assert!(!alerter.is_within_cooldown("whois:other").await);
980 }
981
982 #[tokio::test]
983 async fn health_alert_cooldown_key_matches_record_key() {
984 let pool = db::connect_in_memory().await.unwrap();
985 let alerter = test_alerter(pool.clone());
986
987 // Not in cooldown initially
988 assert!(!alerter.is_within_cooldown("health:example.com").await);
989
990 alerter
991 .send_health_alert("example.com", "Example", "operational", "error", None)
992 .await;
993
994 assert!(alerter.is_within_cooldown("health:example.com").await);
995
996 // Different target should NOT be in cooldown
997 assert!(!alerter.is_within_cooldown("health:other.com").await);
998 }
999
1000 #[tokio::test]
1001 async fn tls_expiry_alert_cooldown_key() {
1002 let pool = db::connect_in_memory().await.unwrap();
1003 let alerter = test_alerter(pool.clone());
1004
1005 assert!(!alerter.is_within_cooldown("tls:mnw").await);
1006 alerter
1007 .send_tls_expiry_alert("mnw", "makenot.work", 10, "2026-04-01T00:00:00Z")
1008 .await;
1009 assert!(alerter.is_within_cooldown("tls:mnw").await);
1010 }
1011
1012 #[tokio::test]
1013 async fn tls_error_alert_cooldown_key() {
1014 let pool = db::connect_in_memory().await.unwrap();
1015 let alerter = test_alerter(pool.clone());
1016
1017 assert!(!alerter.is_within_cooldown("tls:mnw").await);
1018 alerter
1019 .send_tls_error_alert("mnw", "makenot.work", "certificate expired")
1020 .await;
1021 assert!(alerter.is_within_cooldown("tls:mnw").await);
1022 }
1023
1024 #[tokio::test]
1025 async fn latency_drift_alert_cooldown_key() {
1026 let pool = db::connect_in_memory().await.unwrap();
1027 let alerter = test_alerter(pool.clone());
1028
1029 assert!(!alerter.is_within_cooldown("latency:mnw").await);
1030 alerter
1031 .send_latency_drift_alert("mnw", "MakeNotWork", "avg 500ms, baseline 100ms")
1032 .await;
1033 assert!(alerter.is_within_cooldown("latency:mnw").await);
1034 }
1035
1036 #[tokio::test]
1037 async fn test_duration_drift_alert_cooldown_key() {
1038 let pool = db::connect_in_memory().await.unwrap();
1039 let alerter = test_alerter(pool.clone());
1040
1041 assert!(!alerter.is_within_cooldown("test_duration:mnw").await);
1042 alerter
1043 .send_test_duration_drift_alert("mnw", "MakeNotWork", "drift: 120s vs 60s baseline")
1044 .await;
1045 assert!(alerter.is_within_cooldown("test_duration:mnw").await);
1046 }
1047
1048 #[tokio::test]
1049 async fn monitoring_offline_alert_cooldown_key() {
1050 let pool = db::connect_in_memory().await.unwrap();
1051 let alerter = test_alerter(pool.clone());
1052
1053 assert!(!alerter.is_within_cooldown("monitoring:self").await);
1054 alerter.send_monitoring_offline_alert(3).await;
1055 assert!(alerter.is_within_cooldown("monitoring:self").await);
1056 }
1057
1058 #[tokio::test]
1059 async fn route_recovery_does_not_start_cooldown() {
1060 let pool = db::connect_in_memory().await.unwrap();
1061 let alerter = test_alerter(pool.clone());
1062
1063 alerter
1064 .send_route_recovery_alert("mnw", "MakeNotWork", &["/health".to_string()])
1065 .await;
1066 // Recovery alerts are excluded from cooldown lookups, so sending a recovery
1067 // should NOT put the key into cooldown.
1068 assert!(!alerter.is_within_cooldown("route:mnw").await);
1069 }
1070
1071 #[tokio::test]
1072 async fn dns_recovery_does_not_start_cooldown() {
1073 let pool = db::connect_in_memory().await.unwrap();
1074 let alerter = test_alerter(pool.clone());
1075
1076 alerter.send_dns_recovery_alert("mnw", "MakeNotWork").await;
1077 assert!(!alerter.is_within_cooldown("dns:mnw").await);
1078 }
1079
1080 #[tokio::test]
1081 async fn tls_recovery_does_not_start_cooldown() {
1082 let pool = db::connect_in_memory().await.unwrap();
1083 let alerter = test_alerter(pool.clone());
1084
1085 alerter.send_tls_recovery("mnw", "MakeNotWork", 90).await;
1086 assert!(!alerter.is_within_cooldown("tls:mnw").await);
1087 }
1088
1089 // Pure priority/severity helpers (pin the <= boundaries)
1090
1091 #[test]
1092 fn tls_expiry_priority_boundaries() {
1093 // critical: days <= 3
1094 assert_eq!(
1095 tls_expiry_priority(-5),
1096 "critical",
1097 "negative days = already expired"
1098 );
1099 assert_eq!(tls_expiry_priority(0), "critical");
1100 assert_eq!(tls_expiry_priority(3), "critical");
1101 // high: 4..=7
1102 assert_eq!(tls_expiry_priority(4), "high");
1103 assert_eq!(tls_expiry_priority(7), "high");
1104 // medium: > 7
1105 assert_eq!(tls_expiry_priority(8), "medium");
1106 assert_eq!(tls_expiry_priority(90), "medium");
1107 }
1108
1109 #[test]
1110 fn whois_expiry_priority_boundaries() {
1111 // critical: days <= 7
1112 assert_eq!(whois_expiry_priority(-1), "critical");
1113 assert_eq!(whois_expiry_priority(7), "critical");
1114 // high: 8..=14
1115 assert_eq!(whois_expiry_priority(8), "high");
1116 assert_eq!(whois_expiry_priority(14), "high");
1117 // medium: > 14
1118 assert_eq!(whois_expiry_priority(15), "medium");
1119 assert_eq!(whois_expiry_priority(180), "medium");
1120 }
1121
1122 #[test]
1123 fn backup_status_priority_missing_is_critical() {
1124 assert_eq!(backup_status_priority("missing"), "critical");
1125 assert_eq!(backup_status_priority("stale"), "high");
1126 assert_eq!(backup_status_priority("error"), "high");
1127 assert_eq!(backup_status_priority("anything-else"), "high");
1128 assert_eq!(backup_status_priority(""), "high");
1129 }
1130
1131 #[test]
1132 fn backup_status_detail_arms() {
1133 assert_eq!(
1134 backup_status_detail("stale", Some(12)),
1135 "last backup is 12h old"
1136 );
1137 // `stale` with no age falls through to the default arm.
1138 assert_eq!(backup_status_detail("stale", None), "status: stale");
1139 assert_eq!(
1140 backup_status_detail("missing", None),
1141 "no backup files found"
1142 );
1143 assert_eq!(
1144 backup_status_detail("missing", Some(5)),
1145 "no backup files found"
1146 );
1147 assert_eq!(backup_status_detail("error", None), "backup check failed");
1148 assert_eq!(
1149 backup_status_detail("error", Some(99)),
1150 "backup check failed"
1151 );
1152 assert_eq!(backup_status_detail("weird", None), "status: weird");
1153 }
1154
1155 #[test]
1156 fn health_status_priority_arms() {
1157 assert_eq!(health_status_priority("error"), "critical");
1158 assert_eq!(health_status_priority("unreachable"), "critical");
1159 assert_eq!(health_status_priority("degraded"), "high");
1160 // Anything else (operational, unknown values) falls through to medium.
1161 assert_eq!(health_status_priority("operational"), "medium");
1162 assert_eq!(health_status_priority("flapping"), "medium");
1163 assert_eq!(health_status_priority(""), "medium");
1164 }
1165 }
1166