//! Email alerting via Postmark API. //! //! Sends alerts on health status transitions and peer disappearance/recovery. //! If no `postmark_token` is configured, alerts are logged to stdout instead. use sqlx::SqlitePool; use tracing::{info, warn}; use crate::config::AlertConfig; use crate::db; use crate::types::AlertCategory; mod backup; mod cors; mod dns; mod health; mod latency; mod offline; mod peer; mod route; mod scan; mod test_duration; mod tls; mod whois; /// WAM ticket priority for a transition into a non-operational health status. fn health_status_priority(to_status: &str) -> &'static str { match to_status { "error" | "unreachable" => "critical", "degraded" => "high", _ => "medium", } } /// WAM ticket priority for a TLS certificate that expires in `days`. fn tls_expiry_priority(days: i64) -> &'static str { if days <= 3 { "critical" } else if days <= 7 { "high" } else { "medium" } } /// WAM ticket priority for a domain registration that expires in `days`. fn whois_expiry_priority(days: i64) -> &'static str { if days <= 7 { "critical" } else if days <= 14 { "high" } else { "medium" } } /// WAM ticket priority for a stale/missing/error backup status. fn backup_status_priority(status: &str) -> &'static str { if status == "missing" { "critical" } else { "high" } } /// Human-readable detail text for a backup status alert. fn backup_status_detail(status: &str, age_hours: Option) -> String { match (status, age_hours) { ("stale", Some(hours)) => format!("last backup is {hours}h old"), ("missing", _) => "no backup files found".to_string(), ("error", _) => "backup check failed".to_string(), _ => format!("status: {status}"), } } /// Map PoM's fine-grained [`AlertCategory`] onto MNW's domain-level alert kind /// (the `AlertKind` enum in the MNW server's `db/admin_alerts.rs`). Recoveries /// and sub-conditions fold onto their domain; the failure/recovery distinction /// rides on severity, not kind. A new PoM category needs a line here, and, if /// it introduces a new domain, a matching MNW `AlertKind` variant (else MNW /// rejects the push with 422). fn mnw_kind(category: AlertCategory) -> &'static str { use AlertCategory::{ BackupRecovery, BackupStale, CorsFailure, CorsRecovery, DnsMismatch, DnsRecovery, Health, LatencyDrift, LatencyRecovery, MonitoringOffline, MonitoringRecovery, PeerMissing, PeerRecovery, Recovery, RouteFailure, RouteRecovery, ScanPipelineDegraded, ScanPipelineRecovery, TestDurationDrift, TlsError, TlsExpiry, TlsRecovery, WhoisError, WhoisExpiry, }; match category { Health | Recovery => "health", TlsExpiry | TlsError | TlsRecovery => "tls", DnsMismatch | DnsRecovery => "dns", WhoisExpiry | WhoisError => "whois", LatencyDrift | LatencyRecovery | TestDurationDrift => "latency", CorsFailure | CorsRecovery => "cors", BackupStale | BackupRecovery => "backup", PeerMissing | PeerRecovery => "peer", RouteFailure | RouteRecovery => "route", ScanPipelineDegraded | ScanPipelineRecovery => "scan", MonitoringOffline | MonitoringRecovery => "monitoring", } } /// Map PoM's WAM priority string onto MNW's three-level severity. Recoveries do /// not carry a priority; the caller passes `"info"` for those directly. fn mnw_severity(priority: &str) -> &'static str { match priority { "critical" => "critical", "high" | "medium" => "warning", _ => "info", } } /// Truncate `s` to at most `max` bytes on a char boundary. MNW caps title at /// 200 and body at 5000; over-long input would be a 422, so clamp instead. fn truncate(s: &str, max: usize) -> String { if s.len() <= max { return s.to_string(); } let mut end = max; while !s.is_char_boundary(end) { end -= 1; } s[..end].to_string() } #[derive(Clone)] pub struct Alerter { config: AlertConfig, client: reqwest::Client, pool: SqlitePool, instance_name: String, wam_url: Option, } /// The record-context an alert carries: what to write to the `alerts` ledger on /// successful delivery, and what to persist for retry on failure. struct AlertMeta<'a> { key: &'a str, category: AlertCategory, from: Option<&'a str>, to: Option<&'a str>, error: Option<&'a str>, } /// `true` if `sent_at` (rfc3339) is within `cooldown_secs` of now. An unparseable /// timestamp is treated as expired (not in cooldown) so a bad row can never wedge /// alerting shut. fn within_cooldown_secs(sent_at: &str, cooldown_secs: u64) -> bool { match chrono::DateTime::parse_from_rfc3339(sent_at) { Ok(dt) => chrono::Utc::now().signed_duration_since(dt).num_seconds() < cooldown_secs as i64, Err(_) => false, } } impl Alerter { pub fn new( config: AlertConfig, pool: SqlitePool, instance_name: String, ) -> Result { // reqwest 0.13's builder can fail (platform trust-store load) and // `Client::new()`/`unwrap_or_default()` panic on that same failure. Return // the error so the caller can disable alerting rather than crash at startup. let client = crate::tls::https_client_builder() .timeout(std::time::Duration::from_secs(10)) .build()?; let wam_url = config.wam_url.clone(); if wam_url.is_none() { warn!( "alerts: wam_url not configured, failure alerts will be delivered by email \ instead of WAM tickets (set alerts.wam_url to route them to WAM)" ); } match (&config.mnw_url, &config.alerts_ingest_token) { (Some(_), Some(_)) => { info!("alerts: MNW sink enabled, alerts also push to the MNW operator log"); } (Some(_), None) => warn!( "alerts: mnw_url set but alerts_ingest_token missing, MNW sink disabled \ (set alerts.alerts_ingest_token or POM_ALERTS_INGEST_TOKEN)" ), _ => {} } Ok(Self { config, client, pool, instance_name, wam_url, }) } /// Push one alert to MNW's operator log (`POST /api/internal/alerts`). /// Returns `true` iff MNW accepted it. Returns `false`: never enqueues, /// when the sink is unconfigured; callers gate on config before treating a /// `false` as a delivery miss worth queuing. Bounded by a 10s timeout so MNW /// latency can't stall the probe loop. The server dedups on `dedup_key` /// (`{kind}:{target}`), so a retried or repeated push of one condition /// collapses rather than duplicating. async fn deliver_mnw( &self, kind: &str, key: &str, severity: &str, subject: &str, body: &str, ) -> bool { let (Some(base), Some(token)) = ( self.config.mnw_url.as_deref(), self.config.alerts_ingest_token.as_deref(), ) else { return false; }; let payload = serde_json::json!({ "source": "pom", "kind": kind, "severity": severity, "title": truncate(subject, 200), "body": truncate(body, 5000), "dedup_key": truncate(&format!("{kind}:{key}"), 200), "details": { "instance": self.instance_name, "target": key }, }); let url = format!("{}/api/internal/alerts", base.trim_end_matches('/')); let send_fut = self .client .post(&url) .bearer_auth(token) .json(&payload) .send(); match tokio::time::timeout(std::time::Duration::from_secs(10), send_fut).await { Ok(Ok(resp)) if resp.status().is_success() => { info!("mnw alert pushed: {kind}/{severity} {key}"); true } Ok(Ok(resp)) => { let status = resp.status(); let text = resp.text().await.unwrap_or_default(); warn!("mnw alert push failed ({status}): {text}"); false } Ok(Err(e)) => { warn!("mnw alert push error: {e}"); false } Err(_) => { warn!("mnw alert push timed out: {kind}/{key}"); false } } } /// Durable push to the MNW operator log: deliver now, and on failure enqueue /// for retry through the same `pending_alerts` queue WAM/email use. No-op /// unless both `mnw_url` and `alerts_ingest_token` are set. /// /// MNW is a *secondary* sink: the primary channel (WAM/email) already /// recorded this alert in the ledger, so a queued MNW row carries /// `channel = "mnw"` and, unlike WAM/email retries, does not re-record on /// delivery (see [`retry_pending`]), it only needs to eventually land. /// `severity` derives from `priority` exactly as the retry path recomputes /// it (`None` → recovery → `"info"`), so live and retried pushes match. async fn push_mnw( &self, category: AlertCategory, key: &str, priority: Option<&str>, subject: &str, body: &str, ) { if self.config.mnw_url.is_none() || self.config.alerts_ingest_token.is_none() { return; } let kind = mnw_kind(category); let severity = priority.map_or("info", mnw_severity); if self.deliver_mnw(kind, key, severity, subject, body).await { return; } let category = category.to_string(); let pending = db::NewPendingAlert { alert_key: key, category: &category, channel: "mnw", subject, body, priority, source: Some("pom"), source_ref: None, from_status: None, to_status: None, error: None, }; if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await { warn!("failed to enqueue undelivered mnw alert for retry: {e}"); } else { warn!("mnw alert to {key} undelivered; queued for retry"); } } /// Whether a FAILURE alert for `target` is within its cooldown. /// /// A recorded recovery *newer* than the last failure clears the cooldown: the /// target came back, so a fresh failure is a genuine new outage and must not /// be suppressed by the prior failure's window (fuzz-2026-07-06 fail→recover→ /// fail suppression). async fn is_within_cooldown(&self, target: &str) -> bool { let Ok(Some(latest_fail)) = db::get_latest_alert_for_target(&self.pool, target).await else { return false; // never alerted → not in cooldown }; // A recovery after the last failure voids the cooldown. if let Ok(Some(rec)) = db::get_latest_alert_matching(&self.pool, target, "%recovery%").await && rec.id > latest_fail.id { return false; } within_cooldown_secs(&latest_fail.sent_at, self.config.cooldown_secs) } /// Whether a RECOVERY for `target` is within its own cooldown, throttles a /// flapping target so it can't emit one recovery email per flap /// (fuzz-2026-07-06 un-throttled recoveries). A failure newer than the last /// recovery clears it, so a genuine recover-after-a-new-outage still sends. async fn is_recovery_within_cooldown(&self, target: &str) -> bool { let Ok(Some(latest_rec)) = db::get_latest_alert_matching(&self.pool, target, "%recovery%").await else { return false; }; if let Ok(Some(fail)) = db::get_latest_alert_for_target(&self.pool, target).await && fail.id > latest_rec.id { return false; } within_cooldown_secs(&latest_rec.sent_at, self.config.cooldown_secs) } /// Send an email via Postmark. Returns `true` if the message was accepted /// (or in dev mode where there is no token, the operator sees it logged). async fn send_email(&self, subject: &str, body: &str) -> bool { let Some(ref token) = self.config.postmark_token else { info!("[dev] alert: {subject}"); info!("[dev] {body}"); return true; }; let payload = serde_json::json!({ "From": self.config.from, "To": self.config.to, "Subject": subject, "TextBody": body, }); let send_fut = self .client .post("https://api.postmarkapp.com/email") .header("X-Postmark-Server-Token", token) .header("Content-Type", "application/json") .header("Accept", "application/json") .json(&payload) .send(); // Wrap in a 30-second timeout to prevent Postmark latency from blocking // the alert task. The reqwest client has its own 10s timeout, but this // guards against DNS resolution stalls and connection pool exhaustion. match tokio::time::timeout(std::time::Duration::from_secs(30), send_fut).await { Ok(Ok(resp)) if resp.status().is_success() => { info!("alert sent: {subject}"); true } Ok(Ok(resp)) => { let status = resp.status(); let text = resp.text().await.unwrap_or_default(); warn!("postmark error ({status}): {text}"); false } Ok(Err(e)) => { warn!("failed to send alert: {e}"); false } Err(_) => { warn!("alert send timed out after 30s: {subject}"); false } } } async fn record_alert( &self, target: &str, alert_type: AlertCategory, from_status: Option<&str>, to_status: Option<&str>, error: Option<&str>, ) { self.record_alert_str( target, &alert_type.to_string(), from_status, to_status, error, ) .await; } /// String-typed variant so the retry task can record a queued alert straight /// from its stored category without round-tripping through the enum. async fn record_alert_str( &self, target: &str, alert_type: &str, from_status: Option<&str>, to_status: Option<&str>, error: Option<&str>, ) { if let Err(e) = db::insert_alert( &self.pool, target, alert_type, from_status, to_status, error, ) .await { warn!("failed to record alert: {e}"); } } /// Fire a failure alert: cooldown-gate, then dispatch. The single entry point /// every `send_*_alert` funnels through, so the cooldown check + dispatch is /// written once rather than copy-pasted into each of the ~13 methods. async fn fire_failure( &self, subject: &str, body: &str, priority: &str, source: &str, source_ref: Option<&str>, meta: AlertMeta<'_>, ) { if self.is_within_cooldown(meta.key).await { info!("alert cooldown active for {}, skipping", meta.key); return; } // Primary sink (WAM) first so a slow MNW push can't delay the ticket; // capture what push_mnw needs before `meta` is moved into dispatch. let (category, key) = (meta.category, meta.key); self.dispatch_wam(subject, body, priority, source, source_ref, meta) .await; self.push_mnw(category, key, Some(priority), subject, body) .await; } /// Fire a recovery: recovery-cooldown-gate, then dispatch by email. The single /// entry point every `send_*_recovery` funnels through. async fn fire_recovery(&self, subject: &str, body: &str, meta: AlertMeta<'_>) { if self.is_recovery_within_cooldown(meta.key).await { info!("recovery cooldown active for {}, skipping", meta.key); return; } let (category, key) = (meta.category, meta.key); self.dispatch_email(subject, body, meta).await; self.push_mnw(category, key, None, subject, body).await; } /// Dispatch a WAM failure alert: attempt delivery (WAM, falling back to /// email), record it on success, or persist it for retry on failure. Gating /// the ledger write on delivery, and queueing the miss, is what stops a /// transient send failure at a status transition from silencing the whole /// outage: the transition fires once, but the queued alert is retried until it /// lands (fuzz-2026-07-06 CRITICAL #1). async fn dispatch_wam( &self, subject: &str, body: &str, priority: &str, source: &str, source_ref: Option<&str>, meta: AlertMeta<'_>, ) { if self .wam_ticket(subject, body, priority, source, source_ref) .await { self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error) .await; } else { self.enqueue_undelivered( subject, body, "wam", Some(priority), Some(source), source_ref, &meta, ) .await; } } /// Dispatch an email alert (recoveries, and failure alerts that already chose /// email): record on success, persist for retry on failure. async fn dispatch_email(&self, subject: &str, body: &str, meta: AlertMeta<'_>) { if self.send_email(subject, body).await { self.record_alert(meta.key, meta.category, meta.from, meta.to, meta.error) .await; } else { self.enqueue_undelivered(subject, body, "email", None, None, None, &meta) .await; } } #[allow(clippy::too_many_arguments)] async fn enqueue_undelivered( &self, subject: &str, body: &str, channel: &str, priority: Option<&str>, source: Option<&str>, source_ref: Option<&str>, meta: &AlertMeta<'_>, ) { let category = meta.category.to_string(); let pending = db::NewPendingAlert { alert_key: meta.key, category: &category, channel, subject, body, priority, source, source_ref, from_status: meta.from, to_status: meta.to, error: meta.error, }; if let Err(e) = db::enqueue_pending_alert(&self.pool, &pending).await { warn!("failed to enqueue undelivered alert for retry: {e}"); } else { warn!("alert to {} undelivered; queued for retry", meta.key); } } /// Re-attempt one queued alert. On delivery, record it in the ledger and /// signal the caller to delete the row; otherwise leave it for a later tick. /// Called by the retry task in `serve`. pub async fn retry_pending(&self, p: &db::PendingAlertRow) -> bool { // MNW is a secondary sink: the primary channel already wrote this alert // to the ledger, so a successful retry only clears the queued row, it // must not re-record (that would double-count and skew the cooldown). // Reconstruct kind/severity the same way the live push derives them. if p.channel == "mnw" { let kind = p .category .parse::() .map_or("health", mnw_kind); let severity = p.priority.as_deref().map_or("info", mnw_severity); return self .deliver_mnw(kind, &p.alert_key, severity, &p.subject, &p.body) .await; } let delivered = match p.channel.as_str() { "wam" => { self.wam_ticket( &p.subject, &p.body, p.priority.as_deref().unwrap_or("high"), p.source.as_deref().unwrap_or("pom"), p.source_ref.as_deref(), ) .await } _ => self.send_email(&p.subject, &p.body).await, }; if delivered { self.record_alert_str( &p.alert_key, &p.category, p.from_status.as_deref(), p.to_status.as_deref(), p.error.as_deref(), ) .await; } delivered } /// Create a WAM ticket for a failure alert. Returns `true` if the alert was /// delivered by *some* channel. /// /// If `wam_url` is unset the failure alert would otherwise be dropped on the /// floor, failures route to WAM, recoveries to email, so a missing `wam_url` /// silently disabled the entire down-alert channel while recovery emails kept /// firing (fuzz-2026-07-06 CRITICAL #2). Fall back to email instead: a /// monitoring tool must never silently swallow a down-alert. WAM delivery /// failures also fall back to email so a WAM outage can't blind the operator. async fn wam_ticket( &self, title: &str, body: &str, priority: &str, source: &str, source_ref: Option<&str>, ) -> bool { let Some(ref base_url) = self.wam_url else { // No WAM configured: deliver the failure alert by email instead of // dropping it. return self.send_email(title, body).await; }; let url = format!("{base_url}/tickets"); let mut payload = serde_json::json!({ "title": title, "body": body, "priority": priority, "source": source, }); if let Some(r) = source_ref { payload["source_ref"] = serde_json::json!(r); } match self.client.post(&url).json(&payload).send().await { Ok(resp) if resp.status().is_success() => { info!("WAM ticket created: {title}"); true } Ok(resp) => { warn!( "WAM ticket creation returned {}: {title}; falling back to email", resp.status() ); self.send_email(title, body).await } Err(e) => { warn!("WAM unreachable: {e}; falling back to email"); self.send_email(title, body).await } } } } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn mnw_retry_routes_to_sink_and_never_records() { // A queued mnw row must retry via the MNW sink, not the WAM/email path, // and must NOT write the ledger (the primary channel already did). With // the sink unconfigured (test_alerter has no mnw_url), delivery fails and // the row stays queued, and the cooldown ledger is untouched. let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); let row = db::PendingAlertRow { id: 1, alert_key: "makenot.work".to_string(), category: "tls_expiry".to_string(), channel: "mnw".to_string(), subject: "TLS expiring".to_string(), body: "cert expires soon".to_string(), priority: Some("high".to_string()), source: Some("pom".to_string()), source_ref: None, from_status: None, to_status: None, error: None, attempts: 0, }; assert!( !alerter.retry_pending(&row).await, "unconfigured sink cannot deliver" ); assert!( !alerter.is_within_cooldown("makenot.work").await, "mnw retry must not record to the ledger" ); } #[test] fn mnw_kind_folds_failure_and_recovery_to_same_domain() { // A condition and its recovery must map to one kind so they share a // dedup thread in the MNW log; the failure/recovery split rides severity. assert_eq!(mnw_kind(AlertCategory::TlsExpiry), "tls"); assert_eq!(mnw_kind(AlertCategory::TlsError), "tls"); assert_eq!(mnw_kind(AlertCategory::TlsRecovery), "tls"); assert_eq!(mnw_kind(AlertCategory::DnsMismatch), "dns"); assert_eq!(mnw_kind(AlertCategory::DnsRecovery), "dns"); assert_eq!(mnw_kind(AlertCategory::TestDurationDrift), "latency"); assert_eq!(mnw_kind(AlertCategory::MonitoringOffline), "monitoring"); } #[test] fn mnw_severity_collapses_priorities_to_three_levels() { assert_eq!(mnw_severity("critical"), "critical"); assert_eq!(mnw_severity("high"), "warning"); assert_eq!(mnw_severity("medium"), "warning"); assert_eq!(mnw_severity("low"), "info"); assert_eq!(mnw_severity("anything-else"), "info"); } #[test] fn truncate_respects_cap_and_char_boundaries() { assert_eq!(truncate("short", 200), "short"); assert_eq!(truncate("abcdef", 3), "abc"); // Cutting mid multi-byte char steps back to the previous boundary rather // than panicking on a non-boundary slice. let s = "aé"; // 'é' is two bytes; byte index 2 is not a char boundary assert_eq!(truncate(s, 2), "a"); } fn test_alerter(pool: SqlitePool) -> Alerter { let config = AlertConfig { postmark_token: None, // dev mode to: "test@example.com".to_string(), from: "PoM Alerts ".to_string(), cooldown_secs: 300, wam_url: None, mnw_url: None, alerts_ingest_token: None, }; Alerter::new(config, pool, "test-instance".to_string()).unwrap() } #[tokio::test] async fn cooldown_prevents_duplicate_alerts() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); // First alert, not in cooldown assert!(!alerter.is_within_cooldown("health:mnw").await); // Record an alert db::insert_alert( &pool, "health:mnw", "health", Some("operational"), Some("error"), None, ) .await .unwrap(); // Now should be in cooldown assert!(alerter.is_within_cooldown("health:mnw").await); } #[tokio::test] async fn wam_ticket_falls_back_to_email_when_wam_url_unset() { // CRITICAL #2: with no wam_url, a failure alert must NOT be silently // dropped, it falls back to email. test_alerter has wam_url: None and // postmark_token: None (dev mode), so the email path reports delivered. let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool); assert!( alerter .wam_ticket("subj", "body", "high", "pom-test", None) .await, "a failure alert with no wam_url must be delivered by email, not dropped" ); } #[tokio::test] async fn cooldown_does_not_affect_other_targets() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); db::insert_alert(&pool, "health:mnw", "health", None, None, None) .await .unwrap(); // Different target should not be in cooldown assert!(!alerter.is_within_cooldown("health:other").await); } #[tokio::test] async fn dev_mode_does_not_send_http() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); // This should log instead of making HTTP calls (no panic, no error) alerter .send_health_alert("mnw", "MakeNotWork", "operational", "error", None) .await; // Verify alert was recorded in DB with the prefixed key (health:mnw), // matching the cooldown lookup key format. let latest = db::get_latest_alert_for_target(&pool, "health:mnw") .await .unwrap(); assert!(latest.is_some()); let row = latest.unwrap(); assert_eq!(row.alert_type, "health"); assert_eq!(row.from_status.as_deref(), Some("operational")); assert_eq!(row.to_status.as_deref(), Some("error")); } #[tokio::test] async fn route_alert_cooldown_key() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); assert!(!alerter.is_within_cooldown("route:mnw").await); alerter .send_route_failure_alert("mnw", "MakeNotWork", &["/docs/faq".to_string()]) .await; assert!(alerter.is_within_cooldown("route:mnw").await); assert!(!alerter.is_within_cooldown("route:mt").await); } #[tokio::test] async fn recovery_clears_failure_cooldown_so_next_failure_fires() { // Regression for the fail→recover→fail suppression the fuzz flagged (this // test previously enshrined the bug: it asserted the 2nd failure stays // suppressed). A recovery means the target came back, so a fresh failure // is a genuine NEW outage and must not be muffled by the prior failure's // cooldown. let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); // Fail → cooldown active. alerter .send_health_alert("mnw", "MakeNotWork", "operational", "error", None) .await; assert!(alerter.is_within_cooldown("health:mnw").await); // Recover → the failure cooldown is now cleared. alerter .send_health_recovery("mnw", "MakeNotWork", "error") .await; assert!( !alerter.is_within_cooldown("health:mnw").await, "a recovery after the last failure must void the failure cooldown" ); } #[tokio::test] async fn recovery_is_throttled_by_its_own_cooldown() { // A flapping target must not emit one recovery per flap: after a recorded // recovery, another recovery within cooldown is throttled, until a new // failure intervenes. let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); db::insert_alert( &pool, "health:mnw", "recovery", None, Some("operational"), None, ) .await .unwrap(); assert!(alerter.is_recovery_within_cooldown("health:mnw").await); // A failure after the recovery clears the recovery cooldown, so the NEXT // genuine recovery still sends. db::insert_alert( &pool, "health:mnw", "health", Some("operational"), Some("error"), None, ) .await .unwrap(); assert!(!alerter.is_recovery_within_cooldown("health:mnw").await); } #[tokio::test] async fn failed_send_enqueues_for_retry_and_retry_delivers() { // CRITICAL #1: a send that fails must not be lost, it is queued, and the // retry path re-delivers it and records it in the ledger. let pool = db::connect_in_memory().await.unwrap(); // Force delivery failure: WAM points at an unroutable URL and there is no // postmark token, so the email fallback in dev mode... would succeed. Use a // direct enqueue + retry to exercise the queue deterministically instead. let pending = db::NewPendingAlert { alert_key: "health:mnw", category: "health", channel: "email", // dev-mode email "delivers" (logs) → retry succeeds subject: "[PoM] mnw: down", body: "body", priority: None, source: None, source_ref: None, from_status: Some("operational"), to_status: Some("error"), error: None, }; db::enqueue_pending_alert(&pool, &pending).await.unwrap(); let due = db::due_pending_alerts(&pool, 10).await.unwrap(); assert_eq!(due.len(), 1, "the undelivered alert must be queued"); let alerter = test_alerter(pool.clone()); assert!( alerter.retry_pending(&due[0]).await, "dev-mode email retry delivers" ); db::delete_pending_alert(&pool, due[0].id).await.unwrap(); // The retry recorded the alert in the ledger (so cooldown now applies). assert!(alerter.is_within_cooldown("health:mnw").await); assert!(db::due_pending_alerts(&pool, 10).await.unwrap().is_empty()); } #[tokio::test] async fn dns_alert_cooldown_key() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); assert!(!alerter.is_within_cooldown("dns:mnw").await); let mismatches = vec![crate::types::DnsCheckResult { target: "mnw".to_string(), name: "makenot.work".to_string(), record_type: crate::types::DnsRecordType::A, expected: vec!["1.2.3.4".to_string()], actual: vec!["5.6.7.8".to_string()], matches: false, checked_at: chrono::Utc::now().to_rfc3339(), error: None, }]; alerter .send_dns_mismatch_alert("mnw", "MakeNotWork", &mismatches) .await; assert!(alerter.is_within_cooldown("dns:mnw").await); assert!(!alerter.is_within_cooldown("dns:other").await); } #[tokio::test] async fn whois_alert_cooldown_key() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); assert!(!alerter.is_within_cooldown("whois:mnw").await); alerter .send_whois_expiry_alert("mnw", "MakeNotWork", "makenot.work", 15) .await; assert!(alerter.is_within_cooldown("whois:mnw").await); assert!(!alerter.is_within_cooldown("whois:other").await); } #[tokio::test] async fn health_alert_cooldown_key_matches_record_key() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); // Not in cooldown initially assert!(!alerter.is_within_cooldown("health:example.com").await); // Send an alert for "example.com" alerter .send_health_alert("example.com", "Example", "operational", "error", None) .await; // Same target should now be in cooldown assert!(alerter.is_within_cooldown("health:example.com").await); // Different target should NOT be in cooldown assert!(!alerter.is_within_cooldown("health:other.com").await); } #[tokio::test] async fn tls_expiry_alert_cooldown_key() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); assert!(!alerter.is_within_cooldown("tls:mnw").await); alerter .send_tls_expiry_alert("mnw", "makenot.work", 10, "2026-04-01T00:00:00Z") .await; assert!(alerter.is_within_cooldown("tls:mnw").await); } #[tokio::test] async fn tls_error_alert_cooldown_key() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); assert!(!alerter.is_within_cooldown("tls:mnw").await); alerter .send_tls_error_alert("mnw", "makenot.work", "certificate expired") .await; assert!(alerter.is_within_cooldown("tls:mnw").await); } #[tokio::test] async fn latency_drift_alert_cooldown_key() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); assert!(!alerter.is_within_cooldown("latency:mnw").await); alerter .send_latency_drift_alert("mnw", "MakeNotWork", "avg 500ms, baseline 100ms") .await; assert!(alerter.is_within_cooldown("latency:mnw").await); } #[tokio::test] async fn test_duration_drift_alert_cooldown_key() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); assert!(!alerter.is_within_cooldown("test_duration:mnw").await); alerter .send_test_duration_drift_alert("mnw", "MakeNotWork", "drift: 120s vs 60s baseline") .await; assert!(alerter.is_within_cooldown("test_duration:mnw").await); } #[tokio::test] async fn monitoring_offline_alert_cooldown_key() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); assert!(!alerter.is_within_cooldown("monitoring:self").await); alerter.send_monitoring_offline_alert(3).await; assert!(alerter.is_within_cooldown("monitoring:self").await); } #[tokio::test] async fn route_recovery_does_not_start_cooldown() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); alerter .send_route_recovery_alert("mnw", "MakeNotWork", &["/health".to_string()]) .await; // Recovery alerts are excluded from cooldown lookups, so sending a recovery // should NOT put the key into cooldown. assert!(!alerter.is_within_cooldown("route:mnw").await); } #[tokio::test] async fn dns_recovery_does_not_start_cooldown() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); alerter.send_dns_recovery_alert("mnw", "MakeNotWork").await; assert!(!alerter.is_within_cooldown("dns:mnw").await); } #[tokio::test] async fn tls_recovery_does_not_start_cooldown() { let pool = db::connect_in_memory().await.unwrap(); let alerter = test_alerter(pool.clone()); alerter.send_tls_recovery("mnw", "MakeNotWork", 90).await; assert!(!alerter.is_within_cooldown("tls:mnw").await); } // Pure priority/severity helpers (pin the <= boundaries) #[test] fn tls_expiry_priority_boundaries() { // critical: days <= 3 assert_eq!( tls_expiry_priority(-5), "critical", "negative days = already expired" ); assert_eq!(tls_expiry_priority(0), "critical"); assert_eq!(tls_expiry_priority(3), "critical"); // high: 4..=7 assert_eq!(tls_expiry_priority(4), "high"); assert_eq!(tls_expiry_priority(7), "high"); // medium: > 7 assert_eq!(tls_expiry_priority(8), "medium"); assert_eq!(tls_expiry_priority(90), "medium"); } #[test] fn whois_expiry_priority_boundaries() { // critical: days <= 7 assert_eq!(whois_expiry_priority(-1), "critical"); assert_eq!(whois_expiry_priority(7), "critical"); // high: 8..=14 assert_eq!(whois_expiry_priority(8), "high"); assert_eq!(whois_expiry_priority(14), "high"); // medium: > 14 assert_eq!(whois_expiry_priority(15), "medium"); assert_eq!(whois_expiry_priority(180), "medium"); } #[test] fn backup_status_priority_missing_is_critical() { assert_eq!(backup_status_priority("missing"), "critical"); assert_eq!(backup_status_priority("stale"), "high"); assert_eq!(backup_status_priority("error"), "high"); assert_eq!(backup_status_priority("anything-else"), "high"); assert_eq!(backup_status_priority(""), "high"); } #[test] fn backup_status_detail_arms() { assert_eq!( backup_status_detail("stale", Some(12)), "last backup is 12h old" ); // `stale` with no age falls through to the default arm. assert_eq!(backup_status_detail("stale", None), "status: stale"); assert_eq!( backup_status_detail("missing", None), "no backup files found" ); assert_eq!( backup_status_detail("missing", Some(5)), "no backup files found" ); assert_eq!(backup_status_detail("error", None), "backup check failed"); assert_eq!( backup_status_detail("error", Some(99)), "backup check failed" ); assert_eq!(backup_status_detail("weird", None), "status: weird"); } #[test] fn health_status_priority_arms() { assert_eq!(health_status_priority("error"), "critical"); assert_eq!(health_status_priority("unreachable"), "critical"); assert_eq!(health_status_priority("degraded"), "high"); // Anything else (operational, unknown values) falls through to medium. assert_eq!(health_status_priority("operational"), "medium"); assert_eq!(health_status_priority("flapping"), "medium"); assert_eq!(health_status_priority(""), "medium"); } }