Skip to main content

max / makenotwork

27.7 KB · 679 lines History Blame Raw
1 //! Background health monitor, lightweight periodic checks with status-transition alerts.
2 //!
3 //! Periodically probes database connectivity and S3 availability, stores
4 //! timestamped health snapshots, and tracks overall service status. On status
5 //! transitions (Operational/Degraded/Error) sends email alerts to the admin
6 //! address. Exposes the current status for the `/health` endpoint.
7 //!
8 //! See also: `/docs/tech/monitoring`
9
10 use std::time::Instant;
11 use tokio::sync::watch;
12 use tokio::task::JoinHandle;
13
14 use axum::extract::FromRef;
15
16 use crate::config::Config;
17 use crate::constants;
18 use crate::db;
19 use crate::email::EmailClient;
20 use crate::wam_client::WamClient;
21 use crate::{AppCaches, AppState, AppStorage};
22
23 /// The slice of [`AppState`] the background health monitor needs: the DB pool it
24 /// probes, the S3 backend it pings, the derived caches it reports/prunes, the
25 /// mailer + config for alert emails, and the optional WAM client for tickets.
26 ///
27 /// Field names mirror [`AppState`] so the loop body reads `ctx.db`,
28 /// `ctx.storage.s3`, `ctx.caches.session_cache`, etc. This is a projection view
29 /// (like the handler slices in [`crate`]); [`spawn_monitor`] takes it instead of
30 /// the whole `AppState`, so the monitor's dependencies are stated, not ambient.
31 #[derive(Clone)]
32 pub struct MonitorCtx {
33 pub db: sqlx::PgPool,
34 pub storage: AppStorage,
35 pub caches: AppCaches,
36 pub email: EmailClient,
37 pub config: Config,
38 pub wam: Option<WamClient>,
39 }
40
41 impl FromRef<AppState> for MonitorCtx {
42 fn from_ref(s: &AppState) -> Self {
43 Self {
44 db: s.db.clone(),
45 storage: s.storage.clone(),
46 caches: s.caches.clone(),
47 email: s.email.clone(),
48 config: s.config.clone(),
49 wam: s.wam.clone(),
50 }
51 }
52 }
53
54 /// Overall service status.
55 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
56 pub enum MonitorStatus {
57 Operational,
58 Degraded,
59 Error,
60 }
61
62 impl MonitorStatus {
63 pub fn as_str(&self) -> &'static str {
64 match self {
65 MonitorStatus::Operational => "operational",
66 MonitorStatus::Degraded => "degraded",
67 MonitorStatus::Error => "error",
68 }
69 }
70 }
71
72 /// Result of a single health check cycle.
73 pub struct HealthSnapshot {
74 pub status: MonitorStatus,
75 pub db_ok: bool,
76 pub s3_ok: bool,
77 pub sessions_ok: bool,
78 pub check_duration_ms: i32,
79 }
80
81 /// Run a lightweight health probe (no HTTP self-call, no table counts).
82 pub async fn run_health_check(ctx: &MonitorCtx) -> HealthSnapshot {
83 let start = Instant::now();
84
85 // 1. Database: SELECT 1
86 let db_ok = sqlx::query_scalar::<_, i32>("SELECT 1")
87 .fetch_one(&ctx.db)
88 .await
89 .is_ok();
90
91 // 2. S3 connectivity (skip if not configured)
92 let s3_ok = match &ctx.storage.s3 {
93 Some(s3) => match s3.check_connectivity().await {
94 Ok(()) => true,
95 Err(e) => {
96 tracing::warn!(error = %e, "S3 connectivity check failed");
97 false
98 }
99 },
100 None => true, // not configured counts as OK
101 };
102
103 // 3. Sessions: probe the session table
104 let sessions_ok =
105 sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM tower_sessions.session)")
106 .fetch_one(&ctx.db)
107 .await
108 .is_ok();
109
110 let elapsed = start.elapsed();
111 let check_duration_ms = elapsed.as_millis().min(i32::MAX as u128) as i32;
112
113 let status = if db_ok && s3_ok && sessions_ok {
114 MonitorStatus::Operational
115 } else if db_ok {
116 MonitorStatus::Degraded
117 } else {
118 MonitorStatus::Error
119 };
120
121 HealthSnapshot {
122 status,
123 db_ok,
124 s3_ok,
125 sessions_ok,
126 check_duration_ms,
127 }
128 }
129
130 /// Spawn the background monitor loop. Drop `shutdown_tx` to stop it.
131 pub fn spawn_monitor(ctx: MonitorCtx, shutdown_rx: watch::Receiver<()>) -> JoinHandle<()> {
132 // Supervise the loop (Perf-S3): a panic in the loop body would otherwise kill
133 // the task and silently stop all health checks, alerting, and daily
134 // maintenance, with no alert, since the alerting lives in the same loop.
135 // Re-spawn on panic so monitoring survives; a clean shutdown ends supervision.
136 tokio::spawn(async move {
137 loop {
138 match tokio::spawn(run_monitor_loop(ctx.clone(), shutdown_rx.clone())).await {
139 Ok(()) => return, // clean shutdown
140 Err(e) if e.is_panic() => {
141 tracing::error!(error = ?e, "health monitor loop panicked; restarting in 5s");
142 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
143 }
144 Err(_) => return, // task cancelled
145 }
146 }
147 })
148 }
149
150 /// The health-monitor loop body. Extracted from `spawn_monitor` so a panic here is
151 /// caught at the task boundary by the supervisor (Perf-S3) instead of unwinding the
152 /// whole monitor task and silently halting background maintenance.
153 async fn run_monitor_loop(ctx: MonitorCtx, mut shutdown_rx: watch::Receiver<()>) {
154 let alert_email = std::env::var("ALERT_EMAIL").ok();
155 match &alert_email {
156 Some(email) => tracing::info!(alert_email = %email, "health monitor started"),
157 None => tracing::info!("Health monitor started (ALERT_EMAIL not set, alerts disabled)"),
158 }
159
160 let interval_secs = std::env::var("HEALTH_CHECK_INTERVAL_SECS")
161 .ok()
162 .and_then(|v| v.parse::<u64>().ok())
163 .unwrap_or(constants::HEALTH_CHECK_INTERVAL_SECS);
164
165 let mut interval = tokio::time::interval(std::time::Duration::from_secs(interval_secs));
166 // Skip (not burst) missed ticks so a slow health check doesn't cause a
167 // catch-up burst on the next wake.
168 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
169 interval.tick().await; // consume immediate first tick
170
171 let mut previous_status: Option<MonitorStatus> = None;
172 let mut last_alert_at: Option<Instant> = None;
173 let mut last_pool_alert_at: Option<Instant> = None;
174 // Hysteresis arming flag for DB pool pressure. True = next crossing
175 // above the high threshold is allowed to fire a ticket. Resets to
176 // true whenever pressure drops below the low threshold. Starts
177 // true so the first overrun after boot can alert.
178 let mut pool_pressure_armed: bool = true;
179 let mut last_pg_activity_alert_at: Option<Instant> = None;
180
181 loop {
182 tokio::select! {
183 _ = interval.tick() => {}
184 _ = shutdown_rx.changed() => {
185 tracing::info!("Health monitor shutting down");
186 return;
187 }
188 }
189
190 let snap = run_health_check(&ctx).await;
191
192 // Heartbeat (Perf-S3): stamp this tick so an external watcher (the PoM
193 // health contract / dashboard) can detect a stalled monitor, a stale
194 // `health_monitor` last_ran_at means the loop died and the supervisor
195 // couldn't bring it back.
196 if let Err(e) = db::scheduler_jobs::record_job_run(&ctx.db, "health_monitor", 0).await {
197 tracing::warn!(error = ?e, "failed to record health-monitor heartbeat");
198 }
199
200 // Update Prometheus gauges on every tick. DB pool + domain cache
201 // are local and cheap; the pg_stat_activity probe is one fast
202 // query against the shared Postgres. Storage fill aggregates
203 // across all paying creators via a JOIN of users + creator_subs +
204 // tier caps, sub-second today but grows linearly with paying
205 // creator count, so we cache the result for 5 minutes rather
206 // than burning a multi-second query 2880×/day at 10k+ creators.
207 crate::metrics::record_db_pool_stats(&ctx.db);
208 crate::metrics::record_domain_cache_size(ctx.caches.domain_cache.len());
209 static STORAGE_FILL_LAST: std::sync::OnceLock<std::sync::Mutex<std::time::Instant>> =
210 std::sync::OnceLock::new();
211 const STORAGE_FILL_TTL: std::time::Duration = std::time::Duration::from_mins(5);
212 let last_lock = STORAGE_FILL_LAST.get_or_init(|| {
213 // Initialize to "long ago" so the first tick refreshes.
214 std::sync::Mutex::new(
215 std::time::Instant::now()
216 .checked_sub(STORAGE_FILL_TTL)
217 .unwrap(),
218 )
219 });
220 let should_refresh = {
221 // Recover from a poisoned lock rather than panicking the monitor
222 // loop, the critical section is a trivial timestamp swap.
223 let mut last = last_lock
224 .lock()
225 .unwrap_or_else(std::sync::PoisonError::into_inner);
226 if last.elapsed() >= STORAGE_FILL_TTL {
227 *last = std::time::Instant::now();
228 true
229 } else {
230 false
231 }
232 };
233 if should_refresh {
234 crate::metrics::record_storage_fill_stats(&ctx.db).await;
235 crate::metrics::record_custom_pages_stats(&ctx.db).await;
236 crate::metrics::record_scan_queue_stats(&ctx.db).await;
237 }
238
239 // Log status changes. Skip the bootstrap None->Operational transition
240 // so clean restarts don't fire a spurious "recovered" alert.
241 let status_changed = previous_status != Some(snap.status);
242 let is_bootstrap_ok =
243 previous_status.is_none() && snap.status == MonitorStatus::Operational;
244 if status_changed && !is_bootstrap_ok {
245 match snap.status {
246 MonitorStatus::Operational => {
247 if previous_status.is_some() {
248 tracing::info!(
249 duration_ms = snap.check_duration_ms,
250 "health recovered, operational"
251 );
252 }
253 }
254 MonitorStatus::Degraded => {
255 tracing::warn!(
256 db = snap.db_ok,
257 s3 = snap.s3_ok,
258 sessions = snap.sessions_ok,
259 duration_ms = snap.check_duration_ms,
260 "health degraded"
261 );
262 }
263 MonitorStatus::Error => {
264 tracing::error!(
265 db = snap.db_ok,
266 s3 = snap.s3_ok,
267 sessions = snap.sessions_ok,
268 duration_ms = snap.check_duration_ms,
269 "health error"
270 );
271 }
272 }
273
274 // Status-change notifications (admin alert + user notifications) share
275 // a single cooldown so a flapping monitor cannot spam either audience.
276 let cooldown_elapsed = last_alert_at
277 .is_none_or(|t| t.elapsed().as_secs() >= constants::ALERT_COOLDOWN_SECS);
278
279 if cooldown_elapsed {
280 if let Some(ref to) = alert_email {
281 let (subject, body) = build_alert(previous_status, &snap);
282 match ctx.email.send_alert(to, &subject, &body).await {
283 Ok(()) => tracing::info!(recipient = %to, "alert email sent"),
284 Err(e) => tracing::error!(error = ?e, "failed to send alert email"),
285 }
286 }
287
288 // Notify opted-in users of status changes (fire-and-forget).
289 // Batched pacing: pause 1s every 50 sends (matching the
290 // scheduler's announcement fan-out) instead of a hard 100ms
291 // between every send, which turned a 1000-subscriber notify
292 // into a ~100s single task holding the email-client clone.
293 // The subscriber list is bounded by the query's LIMIT.
294 {
295 let pool = ctx.db.clone();
296 let email_client = ctx.email.clone();
297 let host_url = ctx.config.host_url.clone();
298 let signing_secret = ctx.config.signing_secret.clone();
299 let current_status = snap.status.as_str().to_string();
300 let prev_status = previous_status
301 .map_or("unknown", |s| s.as_str())
302 .to_string();
303 tokio::spawn(async move {
304 match db::users::get_status_alert_subscribers(&pool).await {
305 Ok(subscribers) if !subscribers.is_empty() => {
306 tracing::info!(
307 count = subscribers.len(),
308 "sending status notifications to opted-in users"
309 );
310 // One recipient's send failing must not stop the
311 // fan-out, but a status alert that reached
312 // nobody cannot look the same as one that
313 // reached everyone: tally the failures and say
314 // how many of the intended recipients missed it.
315 let mut send_failures = 0usize;
316 for (i, sub) in subscribers.iter().enumerate() {
317 if i > 0 && i % 50 == 0 {
318 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
319 }
320 let unsub_url = crate::email::generate_unsubscribe_url(
321 &host_url,
322 sub.id,
323 crate::email::UnsubscribeAction::Status,
324 &sub.id.to_string(),
325 &signing_secret,
326 );
327 if let Err(e) = email_client
328 .send_status_notification(
329 &sub.email,
330 sub.display_name.as_deref(),
331 &current_status,
332 &prev_status,
333 &unsub_url,
334 )
335 .await
336 {
337 send_failures += 1;
338 tracing::warn!(user_id = %sub.id, error = ?e, "status notification send failed");
339 }
340 }
341 if send_failures > 0 {
342 tracing::error!(
343 failed = send_failures,
344 total = subscribers.len(),
345 "status notifications did not reach every opted-in user"
346 );
347 }
348 }
349 Err(e) => {
350 tracing::error!(error = ?e, "failed to query status alert subscribers");
351 }
352 _ => {}
353 }
354 });
355 }
356
357 // Only record the cooldown timestamp once we've issued at least
358 // one notification path (admin or subscribers); admin block above
359 // is gated on `alert_email` being set, but subscriber fan-out is
360 // always spawned. Always-set is fine since the goal is "don't
361 // re-notify within ALERT_COOLDOWN_SECS regardless of audience."
362 last_alert_at = Some(Instant::now());
363 }
364
365 // Create WAM ticket on degradation/error transitions
366 if snap.status != MonitorStatus::Operational
367 && let Some(ref wam) = ctx.wam
368 {
369 let priority = match snap.status {
370 MonitorStatus::Error => "critical",
371 MonitorStatus::Degraded => "high",
372 MonitorStatus::Operational => unreachable!(),
373 };
374 let title = format!("Health status: {}", snap.status.as_str());
375 let body = format!(
376 "db: {}\ns3: {}\nsessions: {}\ncheck_ms: {}",
377 snap.db_ok, snap.s3_ok, snap.sessions_ok, snap.check_duration_ms,
378 );
379 wam.create_ticket(&title, Some(&body), priority, "health-status-change", None)
380 .await;
381 }
382 }
383
384 previous_status = Some(snap.status);
385
386 // DB pool pressure check with hysteresis: open at 80%, only
387 // re-alert after pressure drops below 60% and climbs back over
388 // 80%. Without hysteresis, a load that oscillates around the
389 // threshold (e.g. each scheduler tick briefly maxes the pool)
390 // spams a ticket every ALERT_COOLDOWN_SECS window.
391 {
392 let pool_size = ctx.db.size();
393 let pool_idle = ctx.db.num_idle() as u32;
394 let active = pool_size.saturating_sub(pool_idle);
395 let pct = (active * 100).checked_div(pool_size).unwrap_or(0);
396 let high = 80u32;
397 let low = 60u32;
398
399 if pct > high {
400 tracing::warn!(
401 pool_size,
402 active,
403 idle = pool_idle,
404 pct,
405 "DB pool pressure >80%"
406 );
407 // Only fire a ticket on the rising edge (was below the low
408 // threshold last we recovered). The cooldown is still here
409 // as a backstop in case the hysteresis state machine ever
410 // gets confused.
411 let cooldown_ok = last_pool_alert_at
412 .is_none_or(|t| t.elapsed().as_secs() >= constants::ALERT_COOLDOWN_SECS);
413 if cooldown_ok
414 && pool_pressure_armed
415 && let Some(ref wam) = ctx.wam
416 {
417 let title = format!("DB pool pressure: {active}/{pool_size} active");
418 wam.create_ticket(&title, None, "high", "db-pool-pressure", None)
419 .await;
420 last_pool_alert_at = Some(Instant::now());
421 pool_pressure_armed = false; // wait for drop below `low` before re-arming
422 }
423 } else if pct < low {
424 pool_pressure_armed = true;
425 }
426 // Between `low` and `high` we hold the current armed state,
427 // that's the dead-band where neither edge triggers.
428 }
429
430 // Postgres server-wide saturation check via pg_stat_activity. This
431 // catches cases where the *shared* Postgres (MNW + MT pools + ad hoc
432 // clients) is approaching `max_connections`, which the local pool
433 // pressure check above cannot see. The probe also emits Prometheus
434 // gauges via `record_pg_stat_activity` so dashboards can graph the
435 // ratio over time, not just react to the alert threshold.
436 //
437 // Cadence note: this runs on EVERY monitor tick (default 30s)
438 // because the gauges are load-bearing for the operator dashboard;
439 // a 30s refresh on connection-utilization is the right tradeoff.
440 // The probe itself is a single-row query against a system view;
441 // its cost is on the order of microseconds.
442 if let Some((active, max_conn)) = crate::metrics::record_pg_stat_activity(&ctx.db).await {
443 let pct = active * 100 / max_conn;
444 if pct > 80 {
445 tracing::warn!(
446 active,
447 max_conn,
448 pct,
449 "Postgres pg_stat_activity saturation >80%"
450 );
451 let cooldown_ok = last_pg_activity_alert_at
452 .is_none_or(|t| t.elapsed().as_secs() >= constants::ALERT_COOLDOWN_SECS);
453 if cooldown_ok && let Some(ref wam) = ctx.wam {
454 let title = format!(
455 "Postgres saturation: {active}/{max_conn} client backends ({pct}%)"
456 );
457 let body = format!(
458 "pg_stat_activity client-backend count is at {pct}% of \
459 max_connections ({active}/{max_conn}). Shared Postgres serves \
460 MNW + MT + ad hoc clients; exhaustion will fail new connections \
461 for all of them. Investigate which role/application is holding \
462 connections via:\n\n \
463 SELECT usename, application_name, state, count(*) \
464 FROM pg_stat_activity GROUP BY 1,2,3 ORDER BY 4 DESC;"
465 );
466 wam.create_ticket(
467 &title,
468 Some(&body),
469 "high",
470 "pg-stat-activity-saturation",
471 None,
472 )
473 .await;
474 last_pg_activity_alert_at = Some(Instant::now());
475 }
476 }
477 }
478
479 // Persist snapshot (best-effort)
480 if let Err(e) = db::monitor::insert_health_history(
481 &ctx.db,
482 snap.status.as_str(),
483 snap.db_ok,
484 snap.s3_ok,
485 snap.sessions_ok,
486 snap.check_duration_ms,
487 None,
488 )
489 .await
490 {
491 tracing::warn!(error = ?e, "failed to insert health history");
492 }
493
494 // Prune expired session cache entries every cycle
495 let cache_ttl = std::time::Duration::from_secs(constants::SESSION_TOUCH_CACHE_SECS);
496 ctx.caches
497 .session_cache
498 .retain(|_, validated_at| validated_at.elapsed() < cache_ttl);
499
500 // Daily prune/compaction (health history, sync log, OAuth cleanup) used
501 // to live here behind a SECOND advisory lock, a parallel maintenance
502 // scheduler. It now runs in the real scheduler's daily block under the
503 // one scheduler lock (Perf-S3). The monitor is health/alerting only.
504 }
505 }
506
507 /// Build alert email subject + body for a status transition.
508 fn build_alert(previous: Option<MonitorStatus>, snap: &HealthSnapshot) -> (String, String) {
509 let subject = match snap.status {
510 MonitorStatus::Operational => "MNW recovered, all services operational".to_string(),
511 MonitorStatus::Degraded => "MNW degraded, partial service failure".to_string(),
512 MonitorStatus::Error => "MNW down, critical service failure".to_string(),
513 };
514
515 let body = format!(
516 "Status: {} (was: {})\n\n\
517 DB: {}\n\
518 S3: {}\n\
519 Sessions: {}\n\
520 Check duration: {}ms",
521 snap.status.as_str(),
522 previous.map_or("unknown", |s| s.as_str()),
523 if snap.db_ok { "OK" } else { "FAIL" },
524 if snap.s3_ok { "OK" } else { "FAIL" },
525 if snap.sessions_ok { "OK" } else { "FAIL" },
526 snap.check_duration_ms,
527 );
528
529 (subject, body)
530 }
531
532 #[cfg(test)]
533 mod tests {
534 use super::*;
535
536 fn snapshot(status: MonitorStatus, db: bool, s3: bool, sessions: bool) -> HealthSnapshot {
537 HealthSnapshot {
538 status,
539 db_ok: db,
540 s3_ok: s3,
541 sessions_ok: sessions,
542 check_duration_ms: 42,
543 }
544 }
545
546 #[test]
547 fn alert_recovery() {
548 let snap = snapshot(MonitorStatus::Operational, true, true, true);
549 let (subject, body) = build_alert(Some(MonitorStatus::Error), &snap);
550 assert!(subject.contains("recovered"));
551 assert!(body.contains("operational"));
552 assert!(body.contains("was: error"));
553 }
554
555 #[test]
556 fn alert_degraded() {
557 let snap = snapshot(MonitorStatus::Degraded, true, false, true);
558 let (subject, body) = build_alert(Some(MonitorStatus::Operational), &snap);
559 assert!(subject.contains("degraded"));
560 assert!(body.contains("S3: FAIL"));
561 assert!(body.contains("DB: OK"));
562 }
563
564 #[test]
565 fn alert_error() {
566 let snap = snapshot(MonitorStatus::Error, false, false, false);
567 let (subject, body) = build_alert(None, &snap);
568 assert!(subject.contains("down"));
569 assert!(body.contains("was: unknown"));
570 assert!(body.contains("DB: FAIL"));
571 assert!(body.contains("42ms"));
572 }
573
574 #[test]
575 fn status_as_str() {
576 assert_eq!(MonitorStatus::Operational.as_str(), "operational");
577 assert_eq!(MonitorStatus::Degraded.as_str(), "degraded");
578 assert_eq!(MonitorStatus::Error.as_str(), "error");
579 }
580
581 // Status determination logic
582 // These test the status-derivation rules from run_health_check:
583 // all OK -> Operational, db OK but others fail -> Degraded, db fail -> Error
584
585 #[test]
586 fn status_all_ok_is_operational() {
587 let snap = snapshot(MonitorStatus::Operational, true, true, true);
588 assert_eq!(snap.status, MonitorStatus::Operational);
589 }
590
591 #[test]
592 fn status_s3_fail_is_degraded() {
593 // db OK, s3 fail, sessions OK -> Degraded
594 let snap = snapshot(MonitorStatus::Degraded, true, false, true);
595 assert_eq!(snap.status, MonitorStatus::Degraded);
596 }
597
598 #[test]
599 fn status_sessions_fail_is_degraded() {
600 // db OK, s3 OK, sessions fail -> Degraded
601 let snap = snapshot(MonitorStatus::Degraded, true, true, false);
602 assert_eq!(snap.status, MonitorStatus::Degraded);
603 }
604
605 #[test]
606 fn status_s3_and_sessions_fail_is_degraded() {
607 // db OK, both s3 and sessions fail -> still Degraded (db is up)
608 let snap = snapshot(MonitorStatus::Degraded, true, false, false);
609 assert_eq!(snap.status, MonitorStatus::Degraded);
610 }
611
612 #[test]
613 fn status_db_fail_is_error() {
614 // db fail -> Error regardless of other checks
615 let snap = snapshot(MonitorStatus::Error, false, true, true);
616 assert_eq!(snap.status, MonitorStatus::Error);
617 }
618
619 #[test]
620 fn status_all_fail_is_error() {
621 let snap = snapshot(MonitorStatus::Error, false, false, false);
622 assert_eq!(snap.status, MonitorStatus::Error);
623 }
624
625 // build_alert coverage
626
627 #[test]
628 fn alert_from_unknown_to_operational() {
629 let snap = snapshot(MonitorStatus::Operational, true, true, true);
630 let (subject, body) = build_alert(None, &snap);
631 assert!(subject.contains("recovered"));
632 assert!(body.contains("was: unknown"));
633 assert!(body.contains("DB: OK"));
634 assert!(body.contains("S3: OK"));
635 assert!(body.contains("Sessions: OK"));
636 }
637
638 #[test]
639 fn alert_from_degraded_to_error() {
640 let snap = snapshot(MonitorStatus::Error, false, false, true);
641 let (subject, body) = build_alert(Some(MonitorStatus::Degraded), &snap);
642 assert!(subject.contains("down"));
643 assert!(body.contains("was: degraded"));
644 assert!(body.contains("DB: FAIL"));
645 assert!(body.contains("S3: FAIL"));
646 assert!(body.contains("Sessions: OK"));
647 }
648
649 #[test]
650 fn alert_from_error_to_degraded() {
651 let snap = snapshot(MonitorStatus::Degraded, true, false, true);
652 let (subject, body) = build_alert(Some(MonitorStatus::Error), &snap);
653 assert!(subject.contains("degraded"));
654 assert!(body.contains("was: error"));
655 }
656
657 #[test]
658 fn alert_body_includes_check_duration() {
659 let snap = HealthSnapshot {
660 status: MonitorStatus::Operational,
661 db_ok: true,
662 s3_ok: true,
663 sessions_ok: true,
664 check_duration_ms: 9999,
665 };
666 let (_subject, body) = build_alert(None, &snap);
667 assert!(body.contains("9999ms"));
668 }
669
670 // MonitorStatus equality
671
672 #[test]
673 fn status_equality() {
674 assert_eq!(MonitorStatus::Operational, MonitorStatus::Operational);
675 assert_ne!(MonitorStatus::Operational, MonitorStatus::Degraded);
676 assert_ne!(MonitorStatus::Degraded, MonitorStatus::Error);
677 }
678 }
679