max / makenotwork
1 file changed,
+53 insertions,
-148 deletions
| @@ -101,11 +101,6 @@ impl Alerter { | |||
| 101 | 101 | error: Option<&str>, | |
| 102 | 102 | ) { | |
| 103 | 103 | let alert_key = format!("health:{target}"); | |
| 104 | - | if self.is_within_cooldown(&alert_key).await { | |
| 105 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 106 | - | return; | |
| 107 | - | } | |
| 108 | - | ||
| 109 | 104 | let subject = format!("[PoM] {target}: {from_status} -> {to_status}"); | |
| 110 | 105 | let mut body = format!( | |
| 111 | 106 | "Target: {label} ({target})\n\ | |
| @@ -121,7 +116,7 @@ impl Alerter { | |||
| 121 | 116 | body.push_str("\n- PoM"); | |
| 122 | 117 | ||
| 123 | 118 | let priority = health_status_priority(to_status); | |
| 124 | - | self.dispatch_wam(&subject, &body, priority, "pom-health", Some(target), | |
| 119 | + | self.fire_failure(&subject, &body, priority, "pom-health", Some(target), | |
| 125 | 120 | AlertMeta { key: &alert_key, category: AlertCategory::Health, from: Some(from_status), to: Some(to_status), error }).await; | |
| 126 | 121 | } | |
| 127 | 122 | ||
| @@ -133,12 +128,6 @@ impl Alerter { | |||
| 133 | 128 | from_status: &str, | |
| 134 | 129 | ) { | |
| 135 | 130 | let alert_key = format!("health:{target}"); | |
| 136 | - | // Recoveries are throttled by their own cooldown so a flapping target | |
| 137 | - | // can't spam one recovery email per flap (fuzz-2026-07-06). | |
| 138 | - | if self.is_recovery_within_cooldown(&alert_key).await { | |
| 139 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 140 | - | return; | |
| 141 | - | } | |
| 142 | 131 | let subject = format!("[PoM] {target}: recovered"); | |
| 143 | 132 | let body = format!( | |
| 144 | 133 | "Target: {label} ({target})\n\ | |
| @@ -150,7 +139,7 @@ impl Alerter { | |||
| 150 | 139 | chrono::Utc::now().to_rfc3339(), | |
| 151 | 140 | ); | |
| 152 | 141 | ||
| 153 | - | self.dispatch_email(&subject, &body, | |
| 142 | + | self.fire_recovery(&subject, &body, | |
| 154 | 143 | AlertMeta { key: &alert_key, category: AlertCategory::Recovery, from: Some(from_status), to: Some("operational"), error: None }).await; | |
| 155 | 144 | } | |
| 156 | 145 | ||
| @@ -163,11 +152,6 @@ impl Alerter { | |||
| 163 | 152 | not_after: &str, | |
| 164 | 153 | ) { | |
| 165 | 154 | let alert_key = format!("tls:{target}"); | |
| 166 | - | if self.is_within_cooldown(&alert_key).await { | |
| 167 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 168 | - | return; | |
| 169 | - | } | |
| 170 | - | ||
| 171 | 155 | let subject = format!("[PoM] {target}: TLS cert expires in {days_remaining} days"); | |
| 172 | 156 | let body = format!( | |
| 173 | 157 | "Target: {target}\n\ | |
| @@ -182,7 +166,7 @@ impl Alerter { | |||
| 182 | 166 | ); | |
| 183 | 167 | ||
| 184 | 168 | let priority = tls_expiry_priority(days_remaining); | |
| 185 | - | self.dispatch_wam(&subject, &body, priority, "pom-tls", Some(&format!("{target}:{host}")), | |
| 169 | + | self.fire_failure(&subject, &body, priority, "pom-tls", Some(&format!("{target}:{host}")), | |
| 186 | 170 | AlertMeta { key: &alert_key, category: AlertCategory::TlsExpiry, from: None, to: None, error: None }).await; | |
| 187 | 171 | } | |
| 188 | 172 | ||
| @@ -194,11 +178,6 @@ impl Alerter { | |||
| 194 | 178 | error: &str, | |
| 195 | 179 | ) { | |
| 196 | 180 | let alert_key = format!("tls:{target}"); | |
| 197 | - | if self.is_within_cooldown(&alert_key).await { | |
| 198 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 199 | - | return; | |
| 200 | - | } | |
| 201 | - | ||
| 202 | 181 | let subject = format!("[PoM] {target}: TLS check failed"); | |
| 203 | 182 | let body = format!( | |
| 204 | 183 | "Target: {target}\n\ | |
| @@ -211,7 +190,7 @@ impl Alerter { | |||
| 211 | 190 | chrono::Utc::now().to_rfc3339(), | |
| 212 | 191 | ); | |
| 213 | 192 | ||
| 214 | - | self.dispatch_wam(&subject, &body, "high", "pom-tls", Some(&format!("{target}:{host}")), | |
| 193 | + | self.fire_failure(&subject, &body, "high", "pom-tls", Some(&format!("{target}:{host}")), | |
| 215 | 194 | AlertMeta { key: &alert_key, category: AlertCategory::TlsError, from: None, to: None, error: Some(error) }).await; | |
| 216 | 195 | } | |
| 217 | 196 | ||
| @@ -223,12 +202,6 @@ impl Alerter { | |||
| 223 | 202 | days_remaining: i64, | |
| 224 | 203 | ) { | |
| 225 | 204 | let alert_key = format!("tls:{target}"); | |
| 226 | - | // Recoveries are throttled by their own cooldown so a flapping target | |
| 227 | - | // can't spam one recovery email per flap (fuzz-2026-07-06). | |
| 228 | - | if self.is_recovery_within_cooldown(&alert_key).await { | |
| 229 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 230 | - | return; | |
| 231 | - | } | |
| 232 | 205 | let subject = format!("[PoM] {target}: TLS cert renewed"); | |
| 233 | 206 | let body = format!( | |
| 234 | 207 | "Target: {label} ({target})\n\ | |
| @@ -240,7 +213,7 @@ impl Alerter { | |||
| 240 | 213 | chrono::Utc::now().to_rfc3339(), | |
| 241 | 214 | ); | |
| 242 | 215 | ||
| 243 | - | self.dispatch_email(&subject, &body, | |
| 216 | + | self.fire_recovery(&subject, &body, | |
| 244 | 217 | AlertMeta { key: &alert_key, category: AlertCategory::TlsRecovery, from: None, to: None, error: None }).await; | |
| 245 | 218 | } | |
| 246 | 219 | ||
| @@ -252,11 +225,6 @@ impl Alerter { | |||
| 252 | 225 | consecutive_failures: u32, | |
| 253 | 226 | ) { | |
| 254 | 227 | let alert_key = format!("peer:{peer_name}"); | |
| 255 | - | if self.is_within_cooldown(&alert_key).await { | |
| 256 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 257 | - | return; | |
| 258 | - | } | |
| 259 | - | ||
| 260 | 228 | let subject = format!("[PoM] peer {peer_name}: missing"); | |
| 261 | 229 | let body = format!( | |
| 262 | 230 | "Peer: {peer_name}\n\ | |
| @@ -269,7 +237,7 @@ impl Alerter { | |||
| 269 | 237 | chrono::Utc::now().to_rfc3339(), | |
| 270 | 238 | ); | |
| 271 | 239 | ||
| 272 | - | self.dispatch_wam(&subject, &body, "high", "pom-peer", Some(peer_name), | |
| 240 | + | self.fire_failure(&subject, &body, "high", "pom-peer", Some(peer_name), | |
| 273 | 241 | AlertMeta { key: &alert_key, category: AlertCategory::PeerMissing, from: None, to: None, error: None }).await; | |
| 274 | 242 | } | |
| 275 | 243 | ||
| @@ -291,11 +259,7 @@ impl Alerter { | |||
| 291 | 259 | ); | |
| 292 | 260 | ||
| 293 | 261 | let alert_key = format!("peer:{peer_name}"); | |
| 294 | - | if self.is_recovery_within_cooldown(&alert_key).await { | |
| 295 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 296 | - | return; | |
| 297 | - | } | |
| 298 | - | self.dispatch_email(&subject, &body, | |
| 262 | + | self.fire_recovery(&subject, &body, | |
| 299 | 263 | AlertMeta { key: &alert_key, category: AlertCategory::PeerRecovery, from: None, to: None, error: None }).await; | |
| 300 | 264 | } | |
| 301 | 265 | ||
| @@ -307,11 +271,6 @@ impl Alerter { | |||
| 307 | 271 | failed_paths: &[String], | |
| 308 | 272 | ) { | |
| 309 | 273 | let alert_key = format!("route:{target}"); | |
| 310 | - | if self.is_within_cooldown(&alert_key).await { | |
| 311 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 312 | - | return; | |
| 313 | - | } | |
| 314 | - | ||
| 315 | 274 | let n = failed_paths.len(); | |
| 316 | 275 | let subject = format!("[PoM] {label}: {n} route(s) failing"); | |
| 317 | 276 | let body = format!( | |
| @@ -325,7 +284,7 @@ impl Alerter { | |||
| 325 | 284 | chrono::Utc::now().to_rfc3339(), | |
| 326 | 285 | ); | |
| 327 | 286 | ||
| 328 | - | self.dispatch_wam(&subject, &body, "high", "pom-routes", Some(target), | |
| 287 | + | self.fire_failure(&subject, &body, "high", "pom-routes", Some(target), | |
| 329 | 288 | AlertMeta { key: &alert_key, category: AlertCategory::RouteFailure, from: None, to: None, error: None }).await; | |
| 330 | 289 | } | |
| 331 | 290 | ||
| @@ -337,12 +296,6 @@ impl Alerter { | |||
| 337 | 296 | recovered_paths: &[String], | |
| 338 | 297 | ) { | |
| 339 | 298 | let alert_key = format!("route:{target}"); | |
| 340 | - | // Recoveries are throttled by their own cooldown so a flapping target | |
| 341 | - | // can't spam one recovery email per flap (fuzz-2026-07-06). | |
| 342 | - | if self.is_recovery_within_cooldown(&alert_key).await { | |
| 343 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 344 | - | return; | |
| 345 | - | } | |
| 346 | 299 | let subject = format!("[PoM] {label}: routes recovered"); | |
| 347 | 300 | let body = format!( | |
| 348 | 301 | "Target: {label} ({target})\n\ | |
| @@ -355,7 +308,7 @@ impl Alerter { | |||
| 355 | 308 | chrono::Utc::now().to_rfc3339(), | |
| 356 | 309 | ); | |
| 357 | 310 | ||
| 358 | - | self.dispatch_email(&subject, &body, | |
| 311 | + | self.fire_recovery(&subject, &body, | |
| 359 | 312 | AlertMeta { key: &alert_key, category: AlertCategory::RouteRecovery, from: None, to: None, error: None }).await; | |
| 360 | 313 | } | |
| 361 | 314 | ||
| @@ -367,11 +320,6 @@ impl Alerter { | |||
| 367 | 320 | mismatches: &[crate::types::DnsCheckResult], | |
| 368 | 321 | ) { | |
| 369 | 322 | let alert_key = format!("dns:{target}"); | |
| 370 | - | if self.is_within_cooldown(&alert_key).await { | |
| 371 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 372 | - | return; | |
| 373 | - | } | |
| 374 | - | ||
| 375 | 323 | let n = mismatches.len(); | |
| 376 | 324 | let subject = format!("[PoM] {label}: {n} DNS record(s) mismatched"); | |
| 377 | 325 | let details: Vec<String> = mismatches | |
| @@ -398,7 +346,7 @@ impl Alerter { | |||
| 398 | 346 | chrono::Utc::now().to_rfc3339(), | |
| 399 | 347 | ); | |
| 400 | 348 | ||
| 401 | - | self.dispatch_wam(&subject, &body, "high", "pom-dns", Some(target), | |
| 349 | + | self.fire_failure(&subject, &body, "high", "pom-dns", Some(target), | |
| 402 | 350 | AlertMeta { key: &alert_key, category: AlertCategory::DnsMismatch, from: None, to: None, error: None }).await; | |
| 403 | 351 | } | |
| 404 | 352 | ||
| @@ -409,12 +357,6 @@ impl Alerter { | |||
| 409 | 357 | label: &str, | |
| 410 | 358 | ) { | |
| 411 | 359 | let alert_key = format!("dns:{target}"); | |
| 412 | - | // Recoveries are throttled by their own cooldown so a flapping target | |
| 413 | - | // can't spam one recovery email per flap (fuzz-2026-07-06). | |
| 414 | - | if self.is_recovery_within_cooldown(&alert_key).await { | |
| 415 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 416 | - | return; | |
| 417 | - | } | |
| 418 | 360 | let subject = format!("[PoM] {label}: DNS records recovered"); | |
| 419 | 361 | let body = format!( | |
| 420 | 362 | "Target: {label} ({target})\n\ | |
| @@ -426,7 +368,7 @@ impl Alerter { | |||
| 426 | 368 | chrono::Utc::now().to_rfc3339(), | |
| 427 | 369 | ); | |
| 428 | 370 | ||
| 429 | - | self.dispatch_email(&subject, &body, | |
| 371 | + | self.fire_recovery(&subject, &body, | |
| 430 | 372 | AlertMeta { key: &alert_key, category: AlertCategory::DnsRecovery, from: None, to: None, error: None }).await; | |
| 431 | 373 | } | |
| 432 | 374 | ||
| @@ -439,11 +381,6 @@ impl Alerter { | |||
| 439 | 381 | days_remaining: i64, | |
| 440 | 382 | ) { | |
| 441 | 383 | let alert_key = format!("whois:{target}"); | |
| 442 | - | if self.is_within_cooldown(&alert_key).await { | |
| 443 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 444 | - | return; | |
| 445 | - | } | |
| 446 | - | ||
| 447 | 384 | let subject = format!("[PoM] {label}: domain {domain} expires in {days_remaining} days"); | |
| 448 | 385 | let body = format!( | |
| 449 | 386 | "Target: {label} ({target})\n\ | |
| @@ -457,7 +394,7 @@ impl Alerter { | |||
| 457 | 394 | ); | |
| 458 | 395 | ||
| 459 | 396 | let priority = whois_expiry_priority(days_remaining); | |
| 460 | - | self.dispatch_wam(&subject, &body, priority, "pom-whois", Some(&format!("{target}:{domain}")), | |
| 397 | + | self.fire_failure(&subject, &body, priority, "pom-whois", Some(&format!("{target}:{domain}")), | |
| 461 | 398 | AlertMeta { key: &alert_key, category: AlertCategory::WhoisExpiry, from: None, to: None, error: None }).await; | |
| 462 | 399 | } | |
| 463 | 400 | ||
| @@ -470,11 +407,6 @@ impl Alerter { | |||
| 470 | 407 | error: &str, | |
| 471 | 408 | ) { | |
| 472 | 409 | let alert_key = format!("whois:{target}"); | |
| 473 | - | if self.is_within_cooldown(&alert_key).await { | |
| 474 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 475 | - | return; | |
| 476 | - | } | |
| 477 | - | ||
| 478 | 410 | let subject = format!("[PoM] {label}: WHOIS check failed for {domain}"); | |
| 479 | 411 | let body = format!( | |
| 480 | 412 | "Target: {label} ({target})\n\ | |
| @@ -487,7 +419,7 @@ impl Alerter { | |||
| 487 | 419 | chrono::Utc::now().to_rfc3339(), | |
| 488 | 420 | ); | |
| 489 | 421 | ||
| 490 | - | self.dispatch_wam(&subject, &body, "high", "pom-whois", Some(&format!("{target}:{domain}")), | |
| 422 | + | self.fire_failure(&subject, &body, "high", "pom-whois", Some(&format!("{target}:{domain}")), | |
| 491 | 423 | AlertMeta { key: &alert_key, category: AlertCategory::WhoisError, from: None, to: None, error: Some(error) }).await; | |
| 492 | 424 | } | |
| 493 | 425 | ||
| @@ -499,11 +431,6 @@ impl Alerter { | |||
| 499 | 431 | failures: &[crate::types::CorsCheckResult], | |
| 500 | 432 | ) { | |
| 501 | 433 | let alert_key = format!("cors:{target}"); | |
| 502 | - | if self.is_within_cooldown(&alert_key).await { | |
| 503 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 504 | - | return; | |
| 505 | - | } | |
| 506 | - | ||
| 507 | 434 | let n = failures.len(); | |
| 508 | 435 | let subject = format!("[PoM] {label}: {n} CORS preflight(s) failing"); | |
| 509 | 436 | let details: Vec<String> = failures | |
| @@ -528,7 +455,7 @@ impl Alerter { | |||
| 528 | 455 | chrono::Utc::now().to_rfc3339(), | |
| 529 | 456 | ); | |
| 530 | 457 | ||
| 531 | - | self.dispatch_wam(&subject, &body, "high", "pom-cors", Some(target), | |
| 458 | + | self.fire_failure(&subject, &body, "high", "pom-cors", Some(target), | |
| 532 | 459 | AlertMeta { key: &alert_key, category: AlertCategory::CorsFailure, from: None, to: None, error: None }).await; | |
| 533 | 460 | } | |
| 534 | 461 | ||
| @@ -539,12 +466,6 @@ impl Alerter { | |||
| 539 | 466 | label: &str, | |
| 540 | 467 | ) { | |
| 541 | 468 | let alert_key = format!("cors:{target}"); | |
| 542 | - | // Recoveries are throttled by their own cooldown so a flapping target | |
| 543 | - | // can't spam one recovery email per flap (fuzz-2026-07-06). | |
| 544 | - | if self.is_recovery_within_cooldown(&alert_key).await { | |
| 545 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 546 | - | return; | |
| 547 | - | } | |
| 548 | 469 | let subject = format!("[PoM] {label}: CORS preflights recovered"); | |
| 549 | 470 | let body = format!( | |
| 550 | 471 | "Target: {label} ({target})\n\ | |
| @@ -556,7 +477,7 @@ impl Alerter { | |||
| 556 | 477 | chrono::Utc::now().to_rfc3339(), | |
| 557 | 478 | ); | |
| 558 | 479 | ||
| 559 | - | self.dispatch_email(&subject, &body, | |
| 480 | + | self.fire_recovery(&subject, &body, | |
| 560 | 481 | AlertMeta { key: &alert_key, category: AlertCategory::CorsRecovery, from: None, to: None, error: None }).await; | |
| 561 | 482 | } | |
| 562 | 483 | ||
| @@ -568,11 +489,6 @@ impl Alerter { | |||
| 568 | 489 | drift_message: &str, | |
| 569 | 490 | ) { | |
| 570 | 491 | let alert_key = format!("latency:{target}"); | |
| 571 | - | if self.is_within_cooldown(&alert_key).await { | |
| 572 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 573 | - | return; | |
| 574 | - | } | |
| 575 | - | ||
| 576 | 492 | let subject = format!("[PoM] {target}: latency drift detected"); | |
| 577 | 493 | let body = format!( | |
| 578 | 494 | "Target: {label} ({target})\n\ | |
| @@ -584,7 +500,7 @@ impl Alerter { | |||
| 584 | 500 | chrono::Utc::now().to_rfc3339(), | |
| 585 | 501 | ); | |
| 586 | 502 | ||
| 587 | - | self.dispatch_wam(&subject, &body, "medium", "pom-latency", Some(target), | |
| 503 | + | self.fire_failure(&subject, &body, "medium", "pom-latency", Some(target), | |
| 588 | 504 | AlertMeta { key: &alert_key, category: AlertCategory::LatencyDrift, from: None, to: None, error: Some(drift_message) }).await; | |
| 589 | 505 | } | |
| 590 | 506 | ||
| @@ -595,12 +511,6 @@ impl Alerter { | |||
| 595 | 511 | label: &str, | |
| 596 | 512 | ) { | |
| 597 | 513 | let alert_key = format!("latency:{target}"); | |
| 598 | - | // Recoveries are throttled by their own cooldown so a flapping target | |
| 599 | - | // can't spam one recovery email per flap (fuzz-2026-07-06). | |
| 600 | - | if self.is_recovery_within_cooldown(&alert_key).await { | |
| 601 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 602 | - | return; | |
| 603 | - | } | |
| 604 | 514 | let subject = format!("[PoM] {target}: latency recovered"); | |
| 605 | 515 | let body = format!( | |
| 606 | 516 | "Target: {label} ({target})\n\ | |
| @@ -612,7 +522,7 @@ impl Alerter { | |||
| 612 | 522 | chrono::Utc::now().to_rfc3339(), | |
| 613 | 523 | ); | |
| 614 | 524 | ||
| 615 | - | self.dispatch_email(&subject, &body, | |
| 525 | + | self.fire_recovery(&subject, &body, | |
| 616 | 526 | AlertMeta { key: &alert_key, category: AlertCategory::LatencyRecovery, from: None, to: None, error: None }).await; | |
| 617 | 527 | } | |
| 618 | 528 | ||
| @@ -624,11 +534,6 @@ impl Alerter { | |||
| 624 | 534 | drift_message: &str, | |
| 625 | 535 | ) { | |
| 626 | 536 | let alert_key = format!("test_duration:{target}"); | |
| 627 | - | if self.is_within_cooldown(&alert_key).await { | |
| 628 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 629 | - | return; | |
| 630 | - | } | |
| 631 | - | ||
| 632 | 537 | let subject = format!("[PoM] {target}: test duration drift detected"); | |
| 633 | 538 | let body = format!( | |
| 634 | 539 | "Target: {label} ({target})\n\ | |
| @@ -640,7 +545,7 @@ impl Alerter { | |||
| 640 | 545 | chrono::Utc::now().to_rfc3339(), | |
| 641 | 546 | ); | |
| 642 | 547 | ||
| 643 | - | self.dispatch_wam(&subject, &body, "medium", "pom-test-duration", Some(target), | |
| 548 | + | self.fire_failure(&subject, &body, "medium", "pom-test-duration", Some(target), | |
| 644 | 549 | AlertMeta { key: &alert_key, category: AlertCategory::TestDurationDrift, from: None, to: None, error: Some(drift_message) }).await; | |
| 645 | 550 | } | |
| 646 | 551 | ||
| @@ -654,11 +559,6 @@ impl Alerter { | |||
| 654 | 559 | age_hours: Option<i64>, | |
| 655 | 560 | ) { | |
| 656 | 561 | let alert_key = format!("backup:{target}:{database}"); | |
| 657 | - | if self.is_within_cooldown(&alert_key).await { | |
| 658 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 659 | - | return; | |
| 660 | - | } | |
| 661 | - | ||
| 662 | 562 | let detail = backup_status_detail(status, age_hours); | |
| 663 | 563 | ||
| 664 | 564 | let subject = format!("[PoM] {label}: {database} backup {status}"); | |
| @@ -675,7 +575,7 @@ impl Alerter { | |||
| 675 | 575 | ); | |
| 676 | 576 | ||
| 677 | 577 | let priority = backup_status_priority(status); | |
| 678 | - | self.dispatch_wam(&subject, &body, priority, "pom-backup", Some(&format!("{target}:{database}")), | |
| 578 | + | self.fire_failure(&subject, &body, priority, "pom-backup", Some(&format!("{target}:{database}")), | |
| 679 | 579 | AlertMeta { key: &alert_key, category: AlertCategory::BackupStale, from: None, to: Some(status), error: None }).await; | |
| 680 | 580 | } | |
| 681 | 581 | ||
| @@ -687,12 +587,6 @@ impl Alerter { | |||
| 687 | 587 | database: &str, | |
| 688 | 588 | ) { | |
| 689 | 589 | let alert_key = format!("backup:{target}:{database}"); | |
| 690 | - | // Recoveries are throttled by their own cooldown so a flapping target | |
| 691 | - | // can't spam one recovery email per flap (fuzz-2026-07-06). | |
| 692 | - | if self.is_recovery_within_cooldown(&alert_key).await { | |
| 693 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 694 | - | return; | |
| 695 | - | } | |
| 696 | 590 | let subject = format!("[PoM] {label}: {database} backup recovered"); | |
| 697 | 591 | let body = format!( | |
| 698 | 592 | "Target: {label} ({target})\n\ | |
| @@ -705,7 +599,7 @@ impl Alerter { | |||
| 705 | 599 | chrono::Utc::now().to_rfc3339(), | |
| 706 | 600 | ); | |
| 707 | 601 | ||
| 708 | - | self.dispatch_email(&subject, &body, | |
| 602 | + | self.fire_recovery(&subject, &body, | |
| 709 | 603 | AlertMeta { key: &alert_key, category: AlertCategory::BackupRecovery, from: None, to: Some("ok"), error: None }).await; | |
| 710 | 604 | } | |
| 711 | 605 | ||
| @@ -720,11 +614,6 @@ impl Alerter { | |||
| 720 | 614 | issues: &[String], | |
| 721 | 615 | ) { | |
| 722 | 616 | let alert_key = format!("scan_pipeline:{target}"); | |
| 723 | - | if self.is_within_cooldown(&alert_key).await { | |
| 724 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 725 | - | return; | |
| 726 | - | } | |
| 727 | - | ||
| 728 | 617 | let subject = format!("[PoM] {label}: scan pipeline {status}"); | |
| 729 | 618 | let body = format!( | |
| 730 | 619 | "Target: {label} ({target})\n\ | |
| @@ -741,7 +630,7 @@ impl Alerter { | |||
| 741 | 630 | ); | |
| 742 | 631 | ||
| 743 | 632 | let priority = if status == "unreachable" { "high" } else { "medium" }; | |
| 744 | - | self.dispatch_wam(&subject, &body, priority, "pom-scan-pipeline", Some(target), | |
| 633 | + | self.fire_failure(&subject, &body, priority, "pom-scan-pipeline", Some(target), | |
| 745 | 634 | AlertMeta { key: &alert_key, category: AlertCategory::ScanPipelineDegraded, from: None, to: Some(status), error: None }).await; | |
| 746 | 635 | } | |
| 747 | 636 | ||
| @@ -749,10 +638,6 @@ impl Alerter { | |||
| 749 | 638 | #[instrument(skip_all)] | |
| 750 | 639 | pub async fn send_scan_pipeline_recovery(&self, target: &str, label: &str) { | |
| 751 | 640 | let alert_key = format!("scan_pipeline:{target}"); | |
| 752 | - | if self.is_recovery_within_cooldown(&alert_key).await { | |
| 753 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 754 | - | return; | |
| 755 | - | } | |
| 756 | 641 | let subject = format!("[PoM] {label}: scan pipeline recovered"); | |
| 757 | 642 | let body = format!( | |
| 758 | 643 | "Target: {label} ({target})\n\ | |
| @@ -764,7 +649,7 @@ impl Alerter { | |||
| 764 | 649 | chrono::Utc::now().to_rfc3339(), | |
| 765 | 650 | ); | |
| 766 | 651 | ||
| 767 | - | self.dispatch_email(&subject, &body, | |
| 652 | + | self.fire_recovery(&subject, &body, | |
| 768 | 653 | AlertMeta { key: &alert_key, category: AlertCategory::ScanPipelineRecovery, from: None, to: Some("operational"), error: None }).await; | |
| 769 | 654 | } | |
| 770 | 655 | ||
| @@ -772,11 +657,6 @@ impl Alerter { | |||
| 772 | 657 | #[instrument(skip_all)] | |
| 773 | 658 | pub async fn send_monitoring_offline_alert(&self, target_count: usize) { | |
| 774 | 659 | let alert_key = "monitoring:self"; | |
| 775 | - | if self.is_within_cooldown(alert_key).await { | |
| 776 | - | info!("alert cooldown active for {alert_key}, skipping"); | |
| 777 | - | return; | |
| 778 | - | } | |
| 779 | - | ||
| 780 | 660 | let subject = format!("[PoM] all {target_count} targets unreachable"); | |
| 781 | 661 | let body = format!( | |
| 782 | 662 | "All {target_count} monitored targets are non-operational.\n\ | |
| @@ -789,7 +669,7 @@ impl Alerter { | |||
| 789 | 669 | chrono::Utc::now().to_rfc3339(), | |
| 790 | 670 | ); | |
| 791 | 671 | ||
| 792 | - | self.dispatch_wam(&subject, &body, "critical", "pom-monitoring", Some("self"), | |
| 672 | + | self.fire_failure(&subject, &body, "critical", "pom-monitoring", Some("self"), | |
| 793 | 673 | AlertMeta { key: alert_key, category: AlertCategory::MonitoringOffline, from: None, to: None, error: None }).await; | |
| 794 | 674 | } | |
| 795 | 675 | ||
| @@ -797,10 +677,6 @@ impl Alerter { | |||
| 797 | 677 | #[instrument(skip_all)] | |
| 798 | 678 | pub async fn send_monitoring_recovery(&self) { | |
| 799 | 679 | let alert_key = "monitoring:self"; | |
| 800 | - | if self.is_recovery_within_cooldown(alert_key).await { | |
| 801 | - | info!("recovery cooldown active for {alert_key}, skipping"); | |
| 802 | - | return; | |
| 803 | - | } | |
| 804 | 680 | let subject = "[PoM] monitoring recovered".to_string(); | |
| 805 | 681 | let body = format!( | |
| 806 | 682 | "At least one target is reachable again.\n\ | |
| @@ -811,7 +687,7 @@ impl Alerter { | |||
| 811 | 687 | chrono::Utc::now().to_rfc3339(), | |
| 812 | 688 | ); | |
| 813 | 689 | ||
| 814 | - | self.dispatch_email(&subject, &body, | |
| 690 | + | self.fire_recovery(&subject, &body, | |
| 815 | 691 | AlertMeta { key: alert_key, category: AlertCategory::MonitoringRecovery, from: None, to: None, error: None }).await; | |
| 816 | 692 | } | |
| 817 | 693 | ||
| @@ -928,6 +804,35 @@ impl Alerter { | |||
| 928 | 804 | } | |
| 929 | 805 | } | |
| 930 | 806 | ||
| 807 | + | /// Fire a failure alert: cooldown-gate, then dispatch. The single entry point | |
| 808 | + | /// every `send_*_alert` funnels through, so the cooldown check + dispatch is | |
| 809 | + | /// written once rather than copy-pasted into each of the ~13 methods. | |
| 810 | + | async fn fire_failure( | |
| 811 | + | &self, | |
| 812 | + | subject: &str, | |
| 813 | + | body: &str, | |
| 814 | + | priority: &str, | |
| 815 | + | source: &str, | |
| 816 | + | source_ref: Option<&str>, | |
| 817 | + | meta: AlertMeta<'_>, | |
| 818 | + | ) { | |
| 819 | + | if self.is_within_cooldown(meta.key).await { | |
| 820 | + | info!("alert cooldown active for {}, skipping", meta.key); | |
| 821 | + | return; | |
| 822 | + | } | |
| 823 | + | self.dispatch_wam(subject, body, priority, source, source_ref, meta).await; | |
| 824 | + | } | |
| 825 | + | ||
| 826 | + | /// Fire a recovery: recovery-cooldown-gate, then dispatch by email. The single | |
| 827 | + | /// entry point every `send_*_recovery` funnels through. | |
| 828 | + | async fn fire_recovery(&self, subject: &str, body: &str, meta: AlertMeta<'_>) { | |
| 829 | + | if self.is_recovery_within_cooldown(meta.key).await { | |
| 830 | + | info!("recovery cooldown active for {}, skipping", meta.key); | |
| 831 | + | return; | |
| 832 | + | } | |
| 833 | + | self.dispatch_email(subject, body, meta).await; | |
| 834 | + | } | |
| 835 | + | ||
| 931 | 836 | /// Dispatch a WAM failure alert: attempt delivery (WAM, falling back to | |
| 932 | 837 | /// email), record it on success, or persist it for retry on failure. Gating | |
| 933 | 838 | /// the ledger write on delivery — and queueing the miss — is what stops a |