Skip to main content

max / makenotwork

27.8 KB · 680 lines History Blame Raw
1 //! Background health monitor, lightweight periodic checks with status-transition alerts.
2 //!
3 //! Periodically probes database connectivity and S3 availability, stores
4 //! timestamped health snapshots, and tracks overall service status. On status
5 //! transitions (Operational/Degraded/Error) sends email alerts to the admin
6 //! address. Exposes the current status for the `/health` endpoint.
7 //!
8 //! See also: `/docs/tech/monitoring`
9
10 use std::time::Instant;
11 use tokio::sync::watch;
12 use tokio::task::JoinHandle;
13
14 use axum::extract::FromRef;
15
16 use crate::config::Config;
17 use crate::constants;
18 use crate::db;
19 use crate::email::EmailClient;
20 use crate::wam_client::WamClient;
21 use crate::{AppCaches, AppState, AppStorage};
22
23 /// The slice of [`AppState`] the background health monitor needs: the DB pool it
24 /// probes, the S3 backend it pings, the derived caches it reports/prunes, the
25 /// mailer + config for alert emails, and the optional WAM client for tickets.
26 ///
27 /// Field names mirror [`AppState`] so the loop body reads `ctx.db`,
28 /// `ctx.storage.s3`, `ctx.caches.session_cache`, etc. This is a projection view
29 /// (like the handler slices in [`crate`]); [`spawn_monitor`] takes it instead of
30 /// the whole `AppState`, so the monitor's dependencies are stated, not ambient.
31 #[derive(Clone)]
32 pub struct MonitorCtx {
33 pub db: sqlx::PgPool,
34 pub storage: AppStorage,
35 pub caches: AppCaches,
36 pub email: EmailClient,
37 pub config: Config,
38 pub wam: Option<WamClient>,
39 }
40
41 impl FromRef<AppState> for MonitorCtx {
42 fn from_ref(s: &AppState) -> Self {
43 Self {
44 db: s.db.clone(),
45 storage: s.storage.clone(),
46 caches: s.caches.clone(),
47 email: s.email.clone(),
48 config: s.config.clone(),
49 wam: s.wam.clone(),
50 }
51 }
52 }
53
54 /// Overall service status.
55 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
56 pub enum MonitorStatus {
57 Operational,
58 Degraded,
59 Error,
60 }
61
62 impl MonitorStatus {
63 pub fn as_str(&self) -> &'static str {
64 match self {
65 MonitorStatus::Operational => "operational",
66 MonitorStatus::Degraded => "degraded",
67 MonitorStatus::Error => "error",
68 }
69 }
70 }
71
72 /// Result of a single health check cycle.
73 pub struct HealthSnapshot {
74 pub status: MonitorStatus,
75 pub db_ok: bool,
76 pub s3_ok: bool,
77 pub sessions_ok: bool,
78 pub check_duration_ms: i32,
79 }
80
81 /// Run a lightweight health probe (no HTTP self-call, no table counts).
82 pub async fn run_health_check(ctx: &MonitorCtx) -> HealthSnapshot {
83 let start = Instant::now();
84
85 // 1. Database: SELECT 1
86 let db_ok = sqlx::query_scalar::<_, i32>("SELECT 1")
87 .fetch_one(&ctx.db)
88 .await
89 .is_ok();
90
91 // 2. S3 connectivity (skip if not configured)
92 let s3_ok = match &ctx.storage.s3 {
93 Some(s3) => match s3.check_connectivity().await {
94 Ok(()) => true,
95 Err(e) => {
96 tracing::warn!(error = %e, "S3 connectivity check failed");
97 false
98 }
99 },
100 None => true, // not configured counts as OK
101 };
102
103 // 3. Sessions: probe the session table
104 let sessions_ok =
105 sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM tower_sessions.session)")
106 .fetch_one(&ctx.db)
107 .await
108 .is_ok();
109
110 let elapsed = start.elapsed();
111 let check_duration_ms = elapsed.as_millis().min(i32::MAX as u128) as i32;
112
113 let status = if db_ok && s3_ok && sessions_ok {
114 MonitorStatus::Operational
115 } else if db_ok {
116 MonitorStatus::Degraded
117 } else {
118 MonitorStatus::Error
119 };
120
121 HealthSnapshot {
122 status,
123 db_ok,
124 s3_ok,
125 sessions_ok,
126 check_duration_ms,
127 }
128 }
129
130 /// Spawn the background monitor loop. Drop `shutdown_tx` to stop it.
131 pub fn spawn_monitor(ctx: MonitorCtx, shutdown_rx: watch::Receiver<()>) -> JoinHandle<()> {
132 // Supervise the loop (Perf-S3): a panic in the loop body would otherwise kill
133 // the task and silently stop all health checks, alerting, and daily
134 // maintenance, with no alert, since the alerting lives in the same loop.
135 // Re-spawn on panic so monitoring survives; a clean shutdown ends supervision.
136 tokio::spawn(async move {
137 loop {
138 match tokio::spawn(run_monitor_loop(ctx.clone(), shutdown_rx.clone())).await {
139 Ok(()) => return, // clean shutdown
140 Err(e) if e.is_panic() => {
141 tracing::error!(error = ?e, "health monitor loop panicked; restarting in 5s");
142 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
143 }
144 Err(_) => return, // task cancelled
145 }
146 }
147 })
148 }
149
150 /// The health-monitor loop body. Extracted from `spawn_monitor` so a panic here is
151 /// caught at the task boundary by the supervisor (Perf-S3) instead of unwinding the
152 /// whole monitor task and silently halting background maintenance.
153 async fn run_monitor_loop(ctx: MonitorCtx, mut shutdown_rx: watch::Receiver<()>) {
154 let alert_email = std::env::var("ALERT_EMAIL").ok();
155 match &alert_email {
156 Some(email) => tracing::info!(alert_email = %email, "health monitor started"),
157 None => tracing::info!("Health monitor started (ALERT_EMAIL not set, alerts disabled)"),
158 }
159
160 let interval_secs = std::env::var("HEALTH_CHECK_INTERVAL_SECS")
161 .ok()
162 .and_then(|v| v.parse::<u64>().ok())
163 .unwrap_or(constants::HEALTH_CHECK_INTERVAL_SECS);
164
165 let mut interval = tokio::time::interval(std::time::Duration::from_secs(interval_secs));
166 // Skip (not burst) missed ticks so a slow health check doesn't cause a
167 // catch-up burst on the next wake.
168 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
169 interval.tick().await; // consume immediate first tick
170
171 let mut previous_status: Option<MonitorStatus> = None;
172 let mut last_alert_at: Option<Instant> = None;
173 let mut last_pool_alert_at: Option<Instant> = None;
174 // Hysteresis arming flag for DB pool pressure. True = next crossing
175 // above the high threshold is allowed to fire a ticket. Resets to
176 // true whenever pressure drops below the low threshold. Starts
177 // true so the first overrun after boot can alert.
178 let mut pool_pressure_armed: bool = true;
179 let mut last_pg_activity_alert_at: Option<Instant> = None;
180
181 loop {
182 tokio::select! {
183 _ = interval.tick() => {}
184 _ = shutdown_rx.changed() => {
185 tracing::info!("Health monitor shutting down");
186 return;
187 }
188 }
189
190 let snap = run_health_check(&ctx).await;
191
192 // Heartbeat (Perf-S3): stamp this tick so an external watcher (the PoM
193 // health contract / dashboard) can detect a stalled monitor, a stale
194 // `health_monitor` last_ran_at means the loop died and the supervisor
195 // couldn't bring it back.
196 if let Err(e) = db::scheduler_jobs::record_job_run(&ctx.db, "health_monitor", 0).await {
197 tracing::warn!(error = ?e, "failed to record health-monitor heartbeat");
198 }
199
200 // Update Prometheus gauges on every tick. DB pool + domain cache
201 // are local and cheap; the pg_stat_activity probe is one fast
202 // query against the shared Postgres. Storage fill aggregates
203 // across all paying creators via a JOIN of users + creator_subs +
204 // tier caps, sub-second today but grows linearly with paying
205 // creator count, so we cache the result for 5 minutes rather
206 // than burning a multi-second query 2880×/day at 10k+ creators.
207 crate::metrics::record_db_pool_stats(&ctx.db);
208 crate::metrics::record_domain_cache_size(ctx.caches.domain_cache.len());
209 static STORAGE_FILL_LAST: std::sync::OnceLock<std::sync::Mutex<std::time::Instant>> =
210 std::sync::OnceLock::new();
211 const STORAGE_FILL_TTL: std::time::Duration = std::time::Duration::from_mins(5);
212 let last_lock = STORAGE_FILL_LAST.get_or_init(|| {
213 // Initialize to "long ago" so the first tick refreshes.
214 std::sync::Mutex::new(
215 std::time::Instant::now()
216 .checked_sub(STORAGE_FILL_TTL)
217 .unwrap(),
218 )
219 });
220 let should_refresh = {
221 // Recover from a poisoned lock rather than panicking the monitor
222 // loop, the critical section is a trivial timestamp swap.
223 let mut last = last_lock
224 .lock()
225 .unwrap_or_else(std::sync::PoisonError::into_inner);
226 if last.elapsed() >= STORAGE_FILL_TTL {
227 *last = std::time::Instant::now();
228 true
229 } else {
230 false
231 }
232 };
233 if should_refresh {
234 crate::metrics::record_storage_fill_stats(&ctx.db).await;
235 crate::metrics::record_custom_pages_stats(&ctx.db).await;
236 crate::metrics::record_scan_queue_stats(&ctx.db).await;
237 }
238
239 // Log status changes. Skip the bootstrap None->Operational transition
240 // so clean restarts don't fire a spurious "recovered" alert.
241 let status_changed = previous_status != Some(snap.status);
242 let is_bootstrap_ok =
243 previous_status.is_none() && snap.status == MonitorStatus::Operational;
244 if status_changed && !is_bootstrap_ok {
245 match snap.status {
246 MonitorStatus::Operational => {
247 if previous_status.is_some() {
248 tracing::info!(
249 duration_ms = snap.check_duration_ms,
250 "health recovered, operational"
251 );
252 }
253 }
254 MonitorStatus::Degraded => {
255 tracing::warn!(
256 db = snap.db_ok,
257 s3 = snap.s3_ok,
258 sessions = snap.sessions_ok,
259 duration_ms = snap.check_duration_ms,
260 "health degraded"
261 );
262 }
263 MonitorStatus::Error => {
264 tracing::error!(
265 db = snap.db_ok,
266 s3 = snap.s3_ok,
267 sessions = snap.sessions_ok,
268 duration_ms = snap.check_duration_ms,
269 "health error"
270 );
271 }
272 }
273
274 // Status-change notifications (admin alert + user notifications) share
275 // a single cooldown so a flapping monitor cannot spam either audience.
276 let cooldown_elapsed = last_alert_at
277 .is_none_or(|t| t.elapsed().as_secs() >= constants::ALERT_COOLDOWN_SECS);
278
279 if cooldown_elapsed {
280 if let Some(ref to) = alert_email {
281 let (subject, body) = build_alert(previous_status, &snap);
282 match ctx.email.send_alert(to, &subject, &body).await {
283 Ok(()) => tracing::info!(recipient = %to, "alert email sent"),
284 Err(e) => tracing::error!(error = ?e, "failed to send alert email"),
285 }
286 }
287
288 // Notify opted-in users of status changes (fire-and-forget).
289 // Batched pacing: pause 1s every 50 sends (matching the
290 // scheduler's announcement fan-out) instead of a hard 100ms
291 // between every send, which turned a 1000-subscriber notify
292 // into a ~100s single task holding the email-client clone.
293 // The subscriber list is bounded by the query's LIMIT.
294 {
295 let pool = ctx.db.clone();
296 let email_client = ctx.email.clone();
297 let host_url = ctx.config.host_url.clone();
298 let signing_secret = ctx.config.signing_secret.clone();
299 let current_status = snap.status.as_str().to_string();
300 let prev_status = previous_status
301 .map_or("unknown", |s| s.as_str())
302 .to_string();
303 tokio::spawn(async move {
304 match db::users::get_status_alert_subscribers(&pool).await {
305 Ok(subscribers) if !subscribers.is_empty() => {
306 tracing::info!(
307 count = subscribers.len(),
308 "sending status notifications to opted-in users"
309 );
310 // One recipient's send failing must not stop the
311 // fan-out, but a status alert that reached
312 // nobody cannot look the same as one that
313 // reached everyone: tally the failures and say
314 // how many of the intended recipients missed it.
315 let mut send_failures = 0usize;
316 for (i, sub) in subscribers.iter().enumerate() {
317 if i > 0 && i % 50 == 0 {
318 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
319 }
320 let unsub_url = crate::email::generate_unsubscribe_url(
321 &host_url,
322 sub.id,
323 crate::email::UnsubscribeAction::Status,
324 &sub.id.to_string(),
325 &signing_secret,
326 );
327 if let Err(e) = email_client
328 .send_status_notification(
329 sub.id,
330 &sub.email,
331 sub.display_name.as_deref(),
332 &current_status,
333 &prev_status,
334 &unsub_url,
335 )
336 .await
337 {
338 send_failures += 1;
339 tracing::warn!(user_id = %sub.id, error = ?e, "status notification send failed");
340 }
341 }
342 if send_failures > 0 {
343 tracing::error!(
344 failed = send_failures,
345 total = subscribers.len(),
346 "status notifications did not reach every opted-in user"
347 );
348 }
349 }
350 Err(e) => {
351 tracing::error!(error = ?e, "failed to query status alert subscribers");
352 }
353 _ => {}
354 }
355 });
356 }
357
358 // Only record the cooldown timestamp once we've issued at least
359 // one notification path (admin or subscribers); admin block above
360 // is gated on `alert_email` being set, but subscriber fan-out is
361 // always spawned. Always-set is fine since the goal is "don't
362 // re-notify within ALERT_COOLDOWN_SECS regardless of audience."
363 last_alert_at = Some(Instant::now());
364 }
365
366 // Create WAM ticket on degradation/error transitions
367 if snap.status != MonitorStatus::Operational
368 && let Some(ref wam) = ctx.wam
369 {
370 let priority = match snap.status {
371 MonitorStatus::Error => "critical",
372 MonitorStatus::Degraded => "high",
373 MonitorStatus::Operational => unreachable!(),
374 };
375 let title = format!("Health status: {}", snap.status.as_str());
376 let body = format!(
377 "db: {}\ns3: {}\nsessions: {}\ncheck_ms: {}",
378 snap.db_ok, snap.s3_ok, snap.sessions_ok, snap.check_duration_ms,
379 );
380 wam.create_ticket(&title, Some(&body), priority, "health-status-change", None)
381 .await;
382 }
383 }
384
385 previous_status = Some(snap.status);
386
387 // DB pool pressure check with hysteresis: open at 80%, only
388 // re-alert after pressure drops below 60% and climbs back over
389 // 80%. Without hysteresis, a load that oscillates around the
390 // threshold (e.g. each scheduler tick briefly maxes the pool)
391 // spams a ticket every ALERT_COOLDOWN_SECS window.
392 {
393 let pool_size = ctx.db.size();
394 let pool_idle = ctx.db.num_idle() as u32;
395 let active = pool_size.saturating_sub(pool_idle);
396 let pct = (active * 100).checked_div(pool_size).unwrap_or(0);
397 let high = 80u32;
398 let low = 60u32;
399
400 if pct > high {
401 tracing::warn!(
402 pool_size,
403 active,
404 idle = pool_idle,
405 pct,
406 "DB pool pressure >80%"
407 );
408 // Only fire a ticket on the rising edge (was below the low
409 // threshold last we recovered). The cooldown is still here
410 // as a backstop in case the hysteresis state machine ever
411 // gets confused.
412 let cooldown_ok = last_pool_alert_at
413 .is_none_or(|t| t.elapsed().as_secs() >= constants::ALERT_COOLDOWN_SECS);
414 if cooldown_ok
415 && pool_pressure_armed
416 && let Some(ref wam) = ctx.wam
417 {
418 let title = format!("DB pool pressure: {active}/{pool_size} active");
419 wam.create_ticket(&title, None, "high", "db-pool-pressure", None)
420 .await;
421 last_pool_alert_at = Some(Instant::now());
422 pool_pressure_armed = false; // wait for drop below `low` before re-arming
423 }
424 } else if pct < low {
425 pool_pressure_armed = true;
426 }
427 // Between `low` and `high` we hold the current armed state,
428 // that's the dead-band where neither edge triggers.
429 }
430
431 // Postgres server-wide saturation check via pg_stat_activity. This
432 // catches cases where the *shared* Postgres (MNW + MT pools + ad hoc
433 // clients) is approaching `max_connections`, which the local pool
434 // pressure check above cannot see. The probe also emits Prometheus
435 // gauges via `record_pg_stat_activity` so dashboards can graph the
436 // ratio over time, not just react to the alert threshold.
437 //
438 // Cadence note: this runs on EVERY monitor tick (default 30s)
439 // because the gauges are load-bearing for the operator dashboard;
440 // a 30s refresh on connection-utilization is the right tradeoff.
441 // The probe itself is a single-row query against a system view;
442 // its cost is on the order of microseconds.
443 if let Some((active, max_conn)) = crate::metrics::record_pg_stat_activity(&ctx.db).await {
444 let pct = active * 100 / max_conn;
445 if pct > 80 {
446 tracing::warn!(
447 active,
448 max_conn,
449 pct,
450 "Postgres pg_stat_activity saturation >80%"
451 );
452 let cooldown_ok = last_pg_activity_alert_at
453 .is_none_or(|t| t.elapsed().as_secs() >= constants::ALERT_COOLDOWN_SECS);
454 if cooldown_ok && let Some(ref wam) = ctx.wam {
455 let title = format!(
456 "Postgres saturation: {active}/{max_conn} client backends ({pct}%)"
457 );
458 let body = format!(
459 "pg_stat_activity client-backend count is at {pct}% of \
460 max_connections ({active}/{max_conn}). Shared Postgres serves \
461 MNW + MT + ad hoc clients; exhaustion will fail new connections \
462 for all of them. Investigate which role/application is holding \
463 connections via:\n\n \
464 SELECT usename, application_name, state, count(*) \
465 FROM pg_stat_activity GROUP BY 1,2,3 ORDER BY 4 DESC;"
466 );
467 wam.create_ticket(
468 &title,
469 Some(&body),
470 "high",
471 "pg-stat-activity-saturation",
472 None,
473 )
474 .await;
475 last_pg_activity_alert_at = Some(Instant::now());
476 }
477 }
478 }
479
480 // Persist snapshot (best-effort)
481 if let Err(e) = db::monitor::insert_health_history(
482 &ctx.db,
483 snap.status.as_str(),
484 snap.db_ok,
485 snap.s3_ok,
486 snap.sessions_ok,
487 snap.check_duration_ms,
488 None,
489 )
490 .await
491 {
492 tracing::warn!(error = ?e, "failed to insert health history");
493 }
494
495 // Prune expired session cache entries every cycle
496 let cache_ttl = std::time::Duration::from_secs(constants::SESSION_TOUCH_CACHE_SECS);
497 ctx.caches
498 .session_cache
499 .retain(|_, validated_at| validated_at.elapsed() < cache_ttl);
500
501 // Daily prune/compaction (health history, sync log, OAuth cleanup) used
502 // to live here behind a SECOND advisory lock, a parallel maintenance
503 // scheduler. It now runs in the real scheduler's daily block under the
504 // one scheduler lock (Perf-S3). The monitor is health/alerting only.
505 }
506 }
507
508 /// Build alert email subject + body for a status transition.
509 fn build_alert(previous: Option<MonitorStatus>, snap: &HealthSnapshot) -> (String, String) {
510 let subject = match snap.status {
511 MonitorStatus::Operational => "MNW recovered, all services operational".to_string(),
512 MonitorStatus::Degraded => "MNW degraded, partial service failure".to_string(),
513 MonitorStatus::Error => "MNW down, critical service failure".to_string(),
514 };
515
516 let body = format!(
517 "Status: {} (was: {})\n\n\
518 DB: {}\n\
519 S3: {}\n\
520 Sessions: {}\n\
521 Check duration: {}ms",
522 snap.status.as_str(),
523 previous.map_or("unknown", |s| s.as_str()),
524 if snap.db_ok { "OK" } else { "FAIL" },
525 if snap.s3_ok { "OK" } else { "FAIL" },
526 if snap.sessions_ok { "OK" } else { "FAIL" },
527 snap.check_duration_ms,
528 );
529
530 (subject, body)
531 }
532
533 #[cfg(test)]
534 mod tests {
535 use super::*;
536
537 fn snapshot(status: MonitorStatus, db: bool, s3: bool, sessions: bool) -> HealthSnapshot {
538 HealthSnapshot {
539 status,
540 db_ok: db,
541 s3_ok: s3,
542 sessions_ok: sessions,
543 check_duration_ms: 42,
544 }
545 }
546
547 #[test]
548 fn alert_recovery() {
549 let snap = snapshot(MonitorStatus::Operational, true, true, true);
550 let (subject, body) = build_alert(Some(MonitorStatus::Error), &snap);
551 assert!(subject.contains("recovered"));
552 assert!(body.contains("operational"));
553 assert!(body.contains("was: error"));
554 }
555
556 #[test]
557 fn alert_degraded() {
558 let snap = snapshot(MonitorStatus::Degraded, true, false, true);
559 let (subject, body) = build_alert(Some(MonitorStatus::Operational), &snap);
560 assert!(subject.contains("degraded"));
561 assert!(body.contains("S3: FAIL"));
562 assert!(body.contains("DB: OK"));
563 }
564
565 #[test]
566 fn alert_error() {
567 let snap = snapshot(MonitorStatus::Error, false, false, false);
568 let (subject, body) = build_alert(None, &snap);
569 assert!(subject.contains("down"));
570 assert!(body.contains("was: unknown"));
571 assert!(body.contains("DB: FAIL"));
572 assert!(body.contains("42ms"));
573 }
574
575 #[test]
576 fn status_as_str() {
577 assert_eq!(MonitorStatus::Operational.as_str(), "operational");
578 assert_eq!(MonitorStatus::Degraded.as_str(), "degraded");
579 assert_eq!(MonitorStatus::Error.as_str(), "error");
580 }
581
582 // Status determination logic
583 // These test the status-derivation rules from run_health_check:
584 // all OK -> Operational, db OK but others fail -> Degraded, db fail -> Error
585
586 #[test]
587 fn status_all_ok_is_operational() {
588 let snap = snapshot(MonitorStatus::Operational, true, true, true);
589 assert_eq!(snap.status, MonitorStatus::Operational);
590 }
591
592 #[test]
593 fn status_s3_fail_is_degraded() {
594 // db OK, s3 fail, sessions OK -> Degraded
595 let snap = snapshot(MonitorStatus::Degraded, true, false, true);
596 assert_eq!(snap.status, MonitorStatus::Degraded);
597 }
598
599 #[test]
600 fn status_sessions_fail_is_degraded() {
601 // db OK, s3 OK, sessions fail -> Degraded
602 let snap = snapshot(MonitorStatus::Degraded, true, true, false);
603 assert_eq!(snap.status, MonitorStatus::Degraded);
604 }
605
606 #[test]
607 fn status_s3_and_sessions_fail_is_degraded() {
608 // db OK, both s3 and sessions fail -> still Degraded (db is up)
609 let snap = snapshot(MonitorStatus::Degraded, true, false, false);
610 assert_eq!(snap.status, MonitorStatus::Degraded);
611 }
612
613 #[test]
614 fn status_db_fail_is_error() {
615 // db fail -> Error regardless of other checks
616 let snap = snapshot(MonitorStatus::Error, false, true, true);
617 assert_eq!(snap.status, MonitorStatus::Error);
618 }
619
620 #[test]
621 fn status_all_fail_is_error() {
622 let snap = snapshot(MonitorStatus::Error, false, false, false);
623 assert_eq!(snap.status, MonitorStatus::Error);
624 }
625
626 // build_alert coverage
627
628 #[test]
629 fn alert_from_unknown_to_operational() {
630 let snap = snapshot(MonitorStatus::Operational, true, true, true);
631 let (subject, body) = build_alert(None, &snap);
632 assert!(subject.contains("recovered"));
633 assert!(body.contains("was: unknown"));
634 assert!(body.contains("DB: OK"));
635 assert!(body.contains("S3: OK"));
636 assert!(body.contains("Sessions: OK"));
637 }
638
639 #[test]
640 fn alert_from_degraded_to_error() {
641 let snap = snapshot(MonitorStatus::Error, false, false, true);
642 let (subject, body) = build_alert(Some(MonitorStatus::Degraded), &snap);
643 assert!(subject.contains("down"));
644 assert!(body.contains("was: degraded"));
645 assert!(body.contains("DB: FAIL"));
646 assert!(body.contains("S3: FAIL"));
647 assert!(body.contains("Sessions: OK"));
648 }
649
650 #[test]
651 fn alert_from_error_to_degraded() {
652 let snap = snapshot(MonitorStatus::Degraded, true, false, true);
653 let (subject, body) = build_alert(Some(MonitorStatus::Error), &snap);
654 assert!(subject.contains("degraded"));
655 assert!(body.contains("was: error"));
656 }
657
658 #[test]
659 fn alert_body_includes_check_duration() {
660 let snap = HealthSnapshot {
661 status: MonitorStatus::Operational,
662 db_ok: true,
663 s3_ok: true,
664 sessions_ok: true,
665 check_duration_ms: 9999,
666 };
667 let (_subject, body) = build_alert(None, &snap);
668 assert!(body.contains("9999ms"));
669 }
670
671 // MonitorStatus equality
672
673 #[test]
674 fn status_equality() {
675 assert_eq!(MonitorStatus::Operational, MonitorStatus::Operational);
676 assert_ne!(MonitorStatus::Operational, MonitorStatus::Degraded);
677 assert_ne!(MonitorStatus::Degraded, MonitorStatus::Error);
678 }
679 }
680