Skip to main content

max / makenotwork

33.4 KB · 981 lines History Blame Raw
1 //! System health dashboard and JSON monitoring endpoint.
2 //!
3 //! Two layers:
4 //! - `GET /health` (HTML); the full live fan-out (DB queries, S3 probe, endpoint
5 //! self-tests) is admin-only. Anonymous/non-admin callers get a minimal page
6 //! whose status is read from the cached monitor snapshot, the admin gate runs
7 //! before the fan-out so an unauthenticated request can't trigger it.
8 //! - `GET /api/health` (JSON); reads cached results from the background monitor's database.
9 //! Fast (<10ms), no live probes. This is what PoM and other external services should poll.
10
11 mod pom;
12 use pom::{
13 PomIncidentJson, PomSnapshotJson, fetch_pom_status, format_incident_duration,
14 format_pom_timestamp,
15 };
16
17 use std::sync::Arc;
18
19 use axum::Json;
20 use axum::extract::State;
21 use axum::http::StatusCode;
22 use axum::response::IntoResponse;
23 use tower_sessions::Session;
24
25 use sqlx::PgPool;
26
27 use crate::{
28 AppStorage, Billing, Ops,
29 config::Config,
30 db,
31 error::Result,
32 helpers::get_csrf_token,
33 templates::{
34 HealthSnapshotDisplay, HealthTemplate, HealthTest, PomIncidentDisplay, PomSnapshotDisplay,
35 PrivacyJobDisplay,
36 },
37 };
38
39 /// Per-subsystem ceiling for the outbound probes in the admin health fan-out,
40 /// so one wedged dependency can't stall the dashboard on a longer client-level
41 /// timeout.
42 const HEALTH_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
43
44 /// Format a [`std::time::Duration`] as a human-readable uptime string.
45 fn format_uptime(d: std::time::Duration) -> String {
46 let total_secs = d.as_secs();
47 let days = total_secs / 86400;
48 let hours = (total_secs % 86400) / 3600;
49 let minutes = (total_secs % 3600) / 60;
50 if days > 0 {
51 format!("{days}d {hours}h {minutes}m")
52 } else if hours > 0 {
53 format!("{hours}h {minutes}m")
54 } else {
55 format!("{minutes}m")
56 }
57 }
58
59 /// Tri-state overall status derived from service checks.
60 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
61 enum OverallStatus {
62 Operational,
63 Degraded,
64 Error,
65 }
66
67 impl OverallStatus {
68 fn label(self) -> &'static str {
69 match self {
70 Self::Operational => "All systems operational",
71 Self::Degraded => "Degraded performance",
72 Self::Error => "Issues detected",
73 }
74 }
75
76 fn css_class(self) -> &'static str {
77 match self {
78 Self::Operational => "status-ok",
79 Self::Degraded => "status-warn",
80 Self::Error => "status-error",
81 }
82 }
83
84 fn api_label(self) -> &'static str {
85 match self {
86 Self::Operational => "operational",
87 Self::Degraded => "degraded",
88 Self::Error => "error",
89 }
90 }
91 }
92
93 /// All computed health data, used by the HTML dashboard handler.
94 #[allow(dead_code)] // fields read transitively via status strings
95 struct HealthData {
96 // Overall
97 overall: OverallStatus,
98 uptime: String,
99 version: String,
100 check_duration_ms: u64,
101
102 // Database
103 db_ok: bool,
104 db_status: &'static str,
105 db_status_class: &'static str,
106 db_pool_active: u32,
107 db_pool_max: u32,
108 stats: db::health::DbHealthStats,
109
110 // Sessions
111 session_ok: bool,
112 session_status: &'static str,
113 session_status_class: &'static str,
114
115 // Storage
116 storage_configured: bool,
117 s3_reachable: bool,
118 storage_status: &'static str,
119 storage_status_class: &'static str,
120 storage_bucket: String,
121 storage_region: String,
122
123 // Stripe
124 stripe_configured: bool,
125 stripe_status: &'static str,
126 stripe_status_class: &'static str,
127 stripe_mode: &'static str,
128
129 // Email
130 #[allow(dead_code)] // used only by the HTML template via other fields
131 email_configured: bool,
132 email_status: &'static str,
133 email_status_class: &'static str,
134 email_provider: &'static str,
135
136 // SyncKit
137 synckit_configured: bool,
138 synckit_status: &'static str,
139 synckit_status_class: &'static str,
140
141 // Security & Monitoring
142 admin_configured: bool,
143
144 // Background monitor
145 monitor_enabled: bool,
146 monitor_interval_secs: u64,
147 alerts_configured: bool,
148 uptime_24h: Option<f64>,
149 uptime_7d: Option<f64>,
150 last_incident: Option<String>,
151 recent_snapshots: Vec<db::monitor::DbHealthSnapshot>,
152
153 // Server
154 environment: &'static str,
155 host: Arc<str>,
156 started_at: String,
157
158 // Tests
159 public_tests: Vec<HealthTest>,
160 db_tests: Vec<HealthTest>,
161
162 // External monitoring (PoM)
163 pom_available: bool,
164 pom_status: Option<String>,
165 pom_status_class: Option<String>,
166 pom_response_time_ms: Option<i64>,
167 pom_checked_at: Option<String>,
168 pom_uptime_24h: Option<f64>,
169 pom_uptime_7d: Option<f64>,
170 pom_recent: Vec<PomSnapshotJson>,
171 pom_incident: Option<PomIncidentJson>,
172 pom_recent_incidents: Vec<PomIncidentJson>,
173 pom_avg_latency: Option<String>,
174 pom_p95_latency: Option<String>,
175 pom_routes_total: usize,
176 pom_routes_ok: usize,
177 pom_routes_failed: Vec<String>,
178
179 // Privacy & Compliance
180 privacy_jobs: Vec<db::scheduler_jobs::SchedulerJobRun>,
181 }
182
183 /// Format scheduler job runs into display structs for the health template.
184 fn format_privacy_jobs(
185 jobs: &[db::scheduler_jobs::SchedulerJobRun],
186 now: chrono::DateTime<chrono::Utc>,
187 ) -> Vec<PrivacyJobDisplay> {
188 // Jobs we want to display, with human names and expected frequency in hours
189 let job_meta: &[(&str, &str, i64)] = &[
190 ("ip_scrub", "Session IP scrub (30-day)", 26),
191 ("session_prune", "Session prune (90-day)", 26),
192 (
193 "terminated_account_cleanup",
194 "Terminated account cleanup (30-day)",
195 26,
196 ),
197 (
198 "content_removal_cleanup",
199 "Content removal cleanup (90-day)",
200 26,
201 ),
202 ];
203
204 job_meta
205 .iter()
206 .map(|(key, description, max_hours)| {
207 let run = jobs.iter().find(|j| j.job_name == *key);
208 match run {
209 Some(r) => {
210 let age = now.signed_duration_since(r.last_ran_at);
211 let last_ran = if age.num_hours() < 1 {
212 format!("{}m ago", age.num_minutes().max(0))
213 } else if age.num_hours() < 48 {
214 format!("{}h ago", age.num_hours())
215 } else {
216 format!("{}d ago", age.num_days())
217 };
218 let status_class = if age.num_hours() <= *max_hours {
219 "status-ok"
220 } else {
221 "status-warn"
222 };
223 PrivacyJobDisplay {
224 name: key.to_string(),
225 description: description.to_string(),
226 last_ran,
227 rows_affected: r.rows_affected.to_string(),
228 status_class: status_class.to_string(),
229 }
230 }
231 None => PrivacyJobDisplay {
232 name: key.to_string(),
233 description: description.to_string(),
234 last_ran: "never".to_string(),
235 rows_affected: "-".to_string(),
236 status_class: "status-unknown".to_string(),
237 },
238 }
239 })
240 .collect()
241 }
242
243 /// Run all health checks and return computed data.
244 /// Used by the HTML dashboard; runs live probes (DB, S3, HTTP self-tests).
245 async fn collect_health(
246 db: &PgPool,
247 storage: &AppStorage,
248 payments: &Billing,
249 ops: &Ops,
250 config: &Config,
251 ) -> HealthData {
252 use std::time::Instant;
253
254 let check_start = Instant::now();
255
256 // Helper to run a timed test
257 async fn run_test<F, Fut>(name: &str, f: F) -> HealthTest
258 where
259 F: FnOnce() -> Fut,
260 Fut: std::future::Future<Output = bool>,
261 {
262 let start = Instant::now();
263 let passed = f().await;
264 HealthTest {
265 name: name.to_string(),
266 passed,
267 latency_ms: start.elapsed().as_millis() as u64,
268 }
269 }
270
271 // Database health probes. Run concurrently (each still records its own
272 // latency) instead of serially, so the dashboard's worst-case wait is the
273 // slowest single probe, not their sum (ultra-fuzz Run 4 Perf).
274 let (db_test_users, db_test_projects, db_test_items, db_test_transactions) = tokio::join!(
275 run_test("Count users", || async {
276 sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM users")
277 .fetch_one(db)
278 .await
279 .is_ok()
280 }),
281 run_test("Count projects", || async {
282 sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM projects")
283 .fetch_one(db)
284 .await
285 .is_ok()
286 }),
287 run_test("Count items", || async {
288 sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM items")
289 .fetch_one(db)
290 .await
291 .is_ok()
292 }),
293 run_test("Count transactions", || async {
294 sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM transactions")
295 .fetch_one(db)
296 .await
297 .is_ok()
298 }),
299 );
300
301 // Get actual counts
302 // On failure the counts render as zeros, but that must not be SILENT, log it
303 // so a stats-query failure is visible in the ops logs (the DB probes above are
304 // the authoritative liveness signal; a real DB outage turns them red too).
305 let stats = db::health::get_health_stats(db).await.unwrap_or_else(|e| {
306 tracing::error!(error = ?e, "health: DB stats query failed; rendering zero counts (see DB probes for liveness)");
307 db::health::DbHealthStats {
308 user_count: 0,
309 project_count: 0,
310 item_count: 0,
311 active_session_count: 0,
312 active_creator_count: 0,
313 transaction_count: 0,
314 blog_post_count: 0,
315 sync_app_count: 0,
316 sync_device_count: 0,
317 sync_log_entries: 0,
318 }
319 });
320
321 // Database status
322 let db_ok = db_test_users.passed && db_test_projects.passed;
323 let db_status = if db_ok { "Connected" } else { "Error" };
324 let db_status_class = if db_ok { "status-ok" } else { "status-error" };
325
326 // Pool info
327 let pool_max = db.size();
328 let pool_idle = db.num_idle();
329 let pool_active = pool_max.saturating_sub(pool_idle as u32);
330
331 // S3 storage status with connectivity check. Bound the one outbound network
332 // probe in this fan-out so a wedged S3 endpoint degrades to "unreachable" in
333 // a few seconds rather than stalling the admin dashboard on the client-level
334 // 60s operation timeout (audit Run 22). The DB probes are already bounded by
335 // the pooled-connection `statement_timeout`, and the PoM fetch has its own.
336 let storage_configured = storage.s3.is_some();
337 let s3_reachable = if let Some(ref s3) = storage.s3 {
338 matches!(
339 tokio::time::timeout(HEALTH_PROBE_TIMEOUT, s3.check_connectivity()).await,
340 Ok(res) if res.is_ok()
341 )
342 } else {
343 false
344 };
345 let (storage_status, storage_status_class) = if storage_configured && s3_reachable {
346 ("Connected", "status-ok")
347 } else if storage_configured {
348 ("Configured (unreachable)", "status-warn")
349 } else {
350 ("Not configured", "status-warn")
351 };
352 let (storage_bucket, storage_region) = if let Some(ref storage) = config.storage {
353 (storage.bucket.clone(), storage.region.clone())
354 } else {
355 (String::new(), String::new())
356 };
357
358 // Stripe status
359 let stripe_configured = payments.stripe.is_some();
360 let stripe_status = if stripe_configured {
361 "Configured"
362 } else {
363 "Not configured"
364 };
365 let stripe_status_class = if stripe_configured {
366 "status-ok"
367 } else {
368 "status-warn"
369 };
370 let stripe_mode = if stripe_configured {
371 if config
372 .stripe
373 .as_ref()
374 .is_some_and(|s| s.secret_key.starts_with("sk_live"))
375 {
376 "Live"
377 } else {
378 "Test"
379 }
380 } else {
381 "-"
382 };
383
384 // Email status
385 let email_configured = std::env::var("POSTMARK_TOKEN").is_ok();
386 let email_status = if email_configured {
387 "Configured"
388 } else {
389 "Dev mode (logging)"
390 };
391 let email_status_class = if email_configured {
392 "status-ok"
393 } else {
394 "status-warn"
395 };
396 let email_provider = if email_configured {
397 "Postmark"
398 } else {
399 "Console"
400 };
401
402 // Session status: the signal is whether the session store is queryable, not
403 // how many sessions happen to be active. A quiet or freshly-swept instance
404 // has zero active sessions but is perfectly healthy, the old
405 // `active_session_count > 0` check false-alarmed the whole dashboard to
406 // "Issues detected" on an idle instance (Run 15). Matches monitor.rs.
407 let session_ok =
408 sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM tower_sessions.session)")
409 .fetch_one(db)
410 .await
411 .is_ok();
412 let session_status = if session_ok { "Active" } else { "Error" };
413 let session_status_class = if session_ok {
414 "status-ok"
415 } else {
416 "status-error"
417 };
418
419 // SyncKit status
420 let synckit_configured = config.synckit_jwt_secret.is_some();
421 let synckit_status = if synckit_configured {
422 "Configured"
423 } else {
424 "Not configured"
425 };
426 let synckit_status_class = if synckit_configured {
427 "status-ok"
428 } else {
429 "status-warn"
430 };
431
432 // Security & Monitoring
433 let admin_configured = config.admin_user_id.is_some();
434
435 // Overall tri-state status
436 let overall = if !db_ok || !session_ok {
437 OverallStatus::Error
438 } else if storage_configured && !s3_reachable {
439 OverallStatus::Degraded
440 } else {
441 OverallStatus::Operational
442 };
443
444 let environment = if cfg!(debug_assertions) {
445 "Development"
446 } else {
447 "Production"
448 };
449 let host = config.host_url.clone();
450
451 // Real uptime from AppState
452 let uptime = format_uptime(ops.start_instant.elapsed());
453 let started_at = ops.started_at.format("%Y-%m-%d %H:%M:%S UTC").to_string();
454
455 let version = match option_env!("GIT_HASH") {
456 Some(hash) if !hash.is_empty() => format!("{} ({})", env!("CARGO_PKG_VERSION"), hash),
457 _ => env!("CARGO_PKG_VERSION").to_string(),
458 };
459
460 // Internal route-existence checks (no HTTP self-call, avoids pool contention)
461 let public_tests: Vec<HealthTest> = vec![];
462
463 let db_tests = vec![
464 db_test_users,
465 db_test_projects,
466 db_test_items,
467 db_test_transactions,
468 ];
469
470 let check_duration_ms = check_start.elapsed().as_millis() as u64;
471
472 // External monitoring (PoM), best-effort, don't delay the page
473 let pom = fetch_pom_status().await;
474 let (
475 pom_available,
476 pom_status,
477 pom_status_class,
478 pom_response_time_ms,
479 pom_checked_at,
480 pom_uptime_24h,
481 pom_uptime_7d,
482 pom_recent,
483 pom_incident,
484 pom_recent_incidents,
485 pom_avg_latency,
486 pom_p95_latency,
487 pom_routes_total,
488 pom_routes_ok,
489 pom_routes_failed,
490 ) = if let Some(ref pom) = pom {
491 let latest = pom.latest.as_ref();
492 let status = latest.map(|s| s.status.clone());
493 let status_class = status.as_deref().map(|s| match s {
494 "operational" => "status-ok".to_string(),
495 "degraded" => "status-warn".to_string(),
496 _ => "status-error".to_string(),
497 });
498 let avg_latency = pom
499 .latency_24h
500 .as_ref()
501 .map(|l| format!("{:.0}ms", l.avg_ms));
502 let p95_latency = pom.latency_24h.as_ref().map(|l| format!("{}ms", l.p95_ms));
503 let routes_total = pom.route_status.len();
504 let routes_ok = pom.route_status.iter().filter(|r| r.ok).count();
505 let routes_failed: Vec<String> = pom
506 .route_status
507 .iter()
508 .filter(|r| !r.ok)
509 .map(|r| r.path.clone())
510 .collect();
511 (
512 true,
513 status,
514 status_class,
515 latest.map(|s| s.response_time_ms),
516 latest.map(|s| format_pom_timestamp(&s.checked_at)),
517 pom.uptime_24h,
518 pom.uptime_7d,
519 pom.recent.clone(),
520 pom.current_incident.clone(),
521 pom.incidents
522 .iter()
523 .filter(|i| i.ended_at.is_some())
524 .cloned()
525 .collect(),
526 avg_latency,
527 p95_latency,
528 routes_total,
529 routes_ok,
530 routes_failed,
531 )
532 } else {
533 (
534 false,
535 None,
536 None,
537 None,
538 None,
539 None,
540 None,
541 Vec::new(),
542 None,
543 Vec::new(),
544 None,
545 None,
546 0,
547 0,
548 Vec::new(),
549 )
550 };
551
552 // Background monitor data (best-effort, don't fail the page if queries fail)
553 let monitor_interval_secs = std::env::var("HEALTH_CHECK_INTERVAL_SECS")
554 .ok()
555 .and_then(|v| v.parse::<u64>().ok())
556 .unwrap_or(crate::constants::HEALTH_CHECK_INTERVAL_SECS);
557
558 let alerts_configured = std::env::var("ALERT_EMAIL").is_ok();
559
560 let uptime_24h = db::monitor::get_health_uptime_percent(db, 24)
561 .await
562 .unwrap_or(None);
563 let uptime_7d = db::monitor::get_health_uptime_percent(db, 168)
564 .await
565 .unwrap_or(None);
566 let last_incident = db::monitor::get_last_incident(db)
567 .await
568 .unwrap_or(None)
569 .map(|dt| dt.format("%Y-%m-%d %H:%M UTC").to_string());
570 let recent_snapshots = db::monitor::get_recent_health_history(db, 10)
571 .await
572 .unwrap_or_default();
573
574 HealthData {
575 overall,
576 uptime,
577 version,
578 check_duration_ms,
579 db_ok,
580 db_status,
581 db_status_class,
582 db_pool_active: pool_active,
583 db_pool_max: pool_max,
584 stats,
585 session_ok,
586 session_status,
587 session_status_class,
588 storage_configured,
589 s3_reachable,
590 storage_status,
591 storage_status_class,
592 storage_bucket,
593 storage_region,
594 stripe_configured,
595 stripe_status,
596 stripe_status_class,
597 stripe_mode,
598 email_configured,
599 email_status,
600 email_status_class,
601 email_provider,
602 synckit_configured,
603 synckit_status,
604 synckit_status_class,
605 admin_configured,
606 monitor_enabled: true,
607 monitor_interval_secs,
608 alerts_configured,
609 uptime_24h,
610 uptime_7d,
611 last_incident,
612 recent_snapshots,
613 environment,
614 host,
615 started_at,
616 public_tests,
617 db_tests,
618 pom_available,
619 pom_status,
620 pom_status_class,
621 pom_response_time_ms,
622 pom_checked_at,
623 pom_uptime_24h,
624 pom_uptime_7d,
625 pom_recent,
626 pom_incident,
627 pom_recent_incidents,
628 pom_avg_latency,
629 pom_p95_latency,
630 pom_routes_total,
631 pom_routes_ok,
632 pom_routes_failed,
633 privacy_jobs: db::scheduler_jobs::get_job_runs(db)
634 .await
635 .unwrap_or_default(),
636 }
637 }
638
639 /// Render the system health dashboard with database, storage, and service status.
640 ///
641 /// The detailed dashboard, infra config, DB pool internals, business counts,
642 /// Stripe mode, S3 bucket/region, is admin-only and runs the live subsystem
643 /// fan-out (`collect_health`). Anonymous and non-admin callers get a minimal,
644 /// reachable status page (HTTP 200) whose status comes from the background
645 /// monitor's cached snapshot, NOT a live probe.
646 ///
647 /// The admin gate runs BEFORE `collect_health` so an unauthenticated request
648 /// cannot trigger the ~10-op sweep (DB counts, S3 connectivity, session probe,
649 /// outbound PoM fetch), that was an anon-reachable amplifier (audit Run 22).
650 /// Uptime monitors (including PoM) should poll `GET /api/health`, the cheap
651 /// cached JSON probe; this HTML page is a human-facing status view, and its
652 /// non-admin branch now reads the same cached snapshot `/api/health` does.
653 #[tracing::instrument(skip_all, name = "health::health")]
654 pub(super) async fn health(
655 State(db): State<PgPool>,
656 State(storage): State<AppStorage>,
657 State(payments): State<Billing>,
658 State(ops): State<Ops>,
659 State(config): State<Config>,
660 session: Session,
661 crate::auth::MaybeUserVerified(maybe_user): crate::auth::MaybeUserVerified,
662 ) -> Result<axum::response::Response> {
663 let is_admin = maybe_user.as_ref().is_some_and(|u| u.is_admin);
664 if !is_admin {
665 // Cheap cached status only, no live fan-out for unauthenticated callers.
666 let (overall, _db_ok) = cached_health_status(&db).await;
667 // `label()` is a controlled &'static str from OverallStatus, no user input.
668 let body = axum::response::Html(format!(
669 "<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\">\
670 <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\
671 <title>MakeNotWork status</title></head>\
672 <body style=\"font-family:system-ui,sans-serif;max-width:32rem;margin:4rem auto;padding:0 1rem\">\
673 <h1>MakeNotWork</h1><p>{}</p></body></html>",
674 overall.label()
675 ));
676 return Ok(body.into_response());
677 }
678
679 let data = collect_health(&db, &storage, &payments, &ops, &config).await;
680
681 let now = chrono::Utc::now();
682
683 let pool_utilization = if data.db_pool_max > 0 {
684 format!(
685 "{}%",
686 (data.db_pool_active as f64 / data.db_pool_max as f64 * 100.0) as u32
687 )
688 } else {
689 "0%".to_string()
690 };
691
692 Ok(HealthTemplate {
693 csrf_token: get_csrf_token(&session).await,
694 session_user: None,
695 overall_status: data.overall.label().to_string(),
696 overall_status_class: data.overall.css_class().to_string(),
697 uptime: data.uptime,
698 version: data.version,
699 check_duration_ms: data.check_duration_ms,
700 db_status: data.db_status.to_string(),
701 db_status_class: data.db_status_class.to_string(),
702 db_pool_size: data.db_pool_max.to_string(),
703 db_pool_max: data.db_pool_max.to_string(),
704 db_pool_utilization: pool_utilization,
705 db_active_connections: data.db_pool_active.to_string(),
706 user_count: data.stats.user_count.to_string(),
707 project_count: data.stats.project_count.to_string(),
708 item_count: data.stats.item_count.to_string(),
709 transaction_count: data.stats.transaction_count.to_string(),
710 blog_post_count: data.stats.blog_post_count.to_string(),
711 session_status: data.session_status.to_string(),
712 session_status_class: data.session_status_class.to_string(),
713 active_sessions: data.stats.active_session_count.to_string(),
714 storage_status: data.storage_status.to_string(),
715 storage_status_class: data.storage_status_class.to_string(),
716 storage_configured: data.storage_configured,
717 storage_bucket: data.storage_bucket,
718 storage_region: data.storage_region,
719 stripe_status: data.stripe_status.to_string(),
720 stripe_status_class: data.stripe_status_class.to_string(),
721 stripe_configured: data.stripe_configured,
722 stripe_mode: data.stripe_mode.to_string(),
723 connected_creators: data.stats.active_creator_count.to_string(),
724 email_status: data.email_status.to_string(),
725 email_status_class: data.email_status_class.to_string(),
726 email_provider: data.email_provider.to_string(),
727 synckit_status: data.synckit_status.to_string(),
728 synckit_status_class: data.synckit_status_class.to_string(),
729 synckit_configured: data.synckit_configured,
730 synckit_app_count: data.stats.sync_app_count.to_string(),
731 synckit_device_count: data.stats.sync_device_count.to_string(),
732 synckit_log_entries: data.stats.sync_log_entries.to_string(),
733 admin_status: if data.admin_configured {
734 "Configured".to_string()
735 } else {
736 "Not configured".to_string()
737 },
738 monitor_enabled: data.monitor_enabled,
739 monitor_interval_secs: data.monitor_interval_secs,
740 alerts_configured: data.alerts_configured,
741 uptime_24h: data.uptime_24h.map(|v| format!("{v:.1}")),
742 uptime_7d: data.uptime_7d.map(|v| format!("{v:.1}")),
743 last_incident: data.last_incident,
744 recent_snapshots: data
745 .recent_snapshots
746 .into_iter()
747 .map(|s| {
748 let status_class = match s.status.as_str() {
749 "operational" => "status-ok".to_string(),
750 "degraded" => "status-warn".to_string(),
751 _ => "status-error".to_string(),
752 };
753 HealthSnapshotDisplay {
754 checked_at: s.checked_at.format("%H:%M:%S UTC").to_string(),
755 status: s.status,
756 status_class,
757 duration_ms: s.check_duration_ms,
758 }
759 })
760 .collect(),
761 environment: data.environment.to_string(),
762 host: data.host,
763 started_at: data.started_at,
764 public_tests: data.public_tests,
765 db_tests: data.db_tests,
766 generated_at: now.format("%Y-%m-%d %H:%M:%S UTC").to_string(),
767 pom_available: data.pom_available,
768 pom_status: data.pom_status,
769 pom_status_class: data.pom_status_class,
770 pom_response_time_ms: data.pom_response_time_ms,
771 pom_checked_at: data.pom_checked_at,
772 pom_uptime_24h: data.pom_uptime_24h.map(|v| format!("{v:.1}")),
773 pom_uptime_7d: data.pom_uptime_7d.map(|v| format!("{v:.1}")),
774 pom_recent: data
775 .pom_recent
776 .into_iter()
777 .map(|s| {
778 let status_class = match s.status.as_str() {
779 "operational" => "status-ok".to_string(),
780 "degraded" => "status-warn".to_string(),
781 _ => "status-error".to_string(),
782 };
783 PomSnapshotDisplay {
784 checked_at: format_pom_timestamp(&s.checked_at),
785 status: s.status,
786 status_class,
787 response_time_ms: s.response_time_ms,
788 }
789 })
790 .collect(),
791 pom_avg_latency: data.pom_avg_latency,
792 pom_p95_latency: data.pom_p95_latency,
793 pom_incident_active: data.pom_incident.is_some(),
794 pom_incident_status: data.pom_incident.as_ref().map(|i| i.to_status.clone()),
795 pom_incident_since: data
796 .pom_incident
797 .as_ref()
798 .map(|i| format_pom_timestamp(&i.started_at)),
799 pom_recent_incidents: data
800 .pom_recent_incidents
801 .into_iter()
802 .map(|i| PomIncidentDisplay {
803 started_at: format_pom_timestamp(&i.started_at),
804 duration: i
805 .duration_secs
806 .map_or_else(|| "-".to_string(), format_incident_duration),
807 to_status: i.to_status,
808 })
809 .collect(),
810 pom_routes_total: data.pom_routes_total,
811 pom_routes_ok: data.pom_routes_ok,
812 pom_routes_failed: data.pom_routes_failed,
813 privacy_jobs: format_privacy_jobs(&data.privacy_jobs, now),
814 }
815 .into_response())
816 }
817
818 // JSON Health Endpoint (fast, reads from background monitor cache)
819
820 // Note: the HTML /health page has full diagnostics. The JSON endpoint is kept
821 // minimal: status, version, and the build's git sha, enough to answer "which
822 // commit is live?" (the question a same-semver hot-swap like the carousel can't
823 // answer from version alone), but still no uptime or service configuration. The
824 // short git sha is a public commit ref (the server source is git-public), not a
825 // secret.
826
827 /// Overall status from the background monitor's most recent snapshot, with a
828 /// single `SELECT 1` liveness fallback before the first monitor tick has landed.
829 ///
830 /// This is the cheap read shared by `GET /api/health` and the anonymous
831 /// `/health` HTML page, so both agree on status without either running the live
832 /// `collect_health` subsystem fan-out. Returns `(overall, db_ok)`.
833 async fn cached_health_status(db: &PgPool) -> (OverallStatus, bool) {
834 let latest = db::monitor::get_recent_health_history(db, 1)
835 .await
836 .unwrap_or_default();
837 if let Some(snap) = latest.first() {
838 let status = match snap.status.as_str() {
839 "operational" => OverallStatus::Operational,
840 "degraded" => OverallStatus::Degraded,
841 _ => OverallStatus::Error,
842 };
843 let db_ok = status != OverallStatus::Error;
844 (status, db_ok)
845 } else {
846 // No monitor data yet, single minimal probe.
847 let db_ok = sqlx::query_scalar::<_, i32>("SELECT 1")
848 .fetch_one(db)
849 .await
850 .is_ok();
851 let status = if db_ok {
852 OverallStatus::Operational
853 } else {
854 OverallStatus::Error
855 };
856 (status, db_ok)
857 }
858 }
859
860 /// `GET /api/health`: fast JSON health endpoint.
861 ///
862 /// Reads the latest snapshot from the background monitor's database instead of
863 /// running live probes. Returns 200 if operational or degraded, 503 if error.
864 #[tracing::instrument(skip_all, name = "health::health_json")]
865 pub(super) async fn health_json(State(db): State<PgPool>) -> impl IntoResponse {
866 // Cached monitor data only, no live probes (fast <10ms as documented),
867 // with a single-`SELECT 1` fallback before the first monitor tick.
868 let (overall, db_ok) = cached_health_status(&db).await;
869
870 let http_status = if overall == OverallStatus::Error {
871 StatusCode::SERVICE_UNAVAILABLE
872 } else {
873 StatusCode::OK
874 };
875
876 (http_status, Json(health_json_body(overall, db_ok)))
877 }
878
879 /// Build the JSON body for the `/api/health` response.
880 ///
881 /// Kept as a pure function (no AppState, no DB) so the schema-drift guard
882 /// test in this module can exercise it directly. PoM polls this endpoint
883 /// and runs key-by-key assertions from `pom/deploy/pom-hetzner.toml`; the
884 /// guard test validates that every asserted path still resolves here.
885 fn health_json_body(overall: OverallStatus, db_ok: bool) -> serde_json::Value {
886 serde_json::json!({
887 "status": overall.api_label(),
888 "version": env!("CARGO_PKG_VERSION"),
889 // The commit this binary was built from (short sha, set by build.rs).
890 // `null` on a build without git metadata. Lets a deploy driver verify
891 // the live commit when the version alone can't (same-semver swap).
892 "git_sha": option_env!("GIT_HASH").filter(|h| !h.is_empty()),
893 "checks": {
894 "database": db_ok,
895 },
896 })
897 }
898
899 #[cfg(test)]
900 mod tests {
901 use super::*;
902
903 #[test]
904 fn format_uptime_minutes_only() {
905 assert_eq!(format_uptime(std::time::Duration::from_secs(0)), "0m");
906 assert_eq!(format_uptime(std::time::Duration::from_secs(59)), "0m");
907 assert_eq!(format_uptime(std::time::Duration::from_mins(1)), "1m");
908 assert_eq!(format_uptime(std::time::Duration::from_mins(5)), "5m");
909 }
910
911 #[test]
912 fn format_uptime_hours_and_minutes() {
913 assert_eq!(format_uptime(std::time::Duration::from_hours(1)), "1h 0m");
914 assert_eq!(format_uptime(std::time::Duration::from_mins(61)), "1h 1m");
915 assert_eq!(format_uptime(std::time::Duration::from_hours(2)), "2h 0m");
916 }
917
918 #[test]
919 fn format_uptime_days() {
920 assert_eq!(
921 format_uptime(std::time::Duration::from_hours(24)),
922 "1d 0h 0m"
923 );
924 assert_eq!(
925 format_uptime(std::time::Duration::from_secs(90061)),
926 "1d 1h 1m"
927 );
928 assert_eq!(
929 format_uptime(std::time::Duration::from_hours(72)),
930 "3d 0h 0m"
931 );
932 }
933
934 #[test]
935 fn overall_status_labels() {
936 assert_eq!(
937 OverallStatus::Operational.label(),
938 "All systems operational"
939 );
940 assert_eq!(OverallStatus::Degraded.label(), "Degraded performance");
941 assert_eq!(OverallStatus::Error.label(), "Issues detected");
942 }
943
944 #[test]
945 fn overall_status_css_classes() {
946 assert_eq!(OverallStatus::Operational.css_class(), "status-ok");
947 assert_eq!(OverallStatus::Degraded.css_class(), "status-warn");
948 assert_eq!(OverallStatus::Error.css_class(), "status-error");
949 }
950
951 #[test]
952 fn overall_status_api_labels() {
953 assert_eq!(OverallStatus::Operational.api_label(), "operational");
954 assert_eq!(OverallStatus::Degraded.api_label(), "degraded");
955 assert_eq!(OverallStatus::Error.api_label(), "error");
956 }
957
958 #[test]
959 fn health_json_body_carries_version_and_git_sha_keys() {
960 // A deploy driver answers "which commit is live?" from these keys; lock
961 // their presence so the JSON endpoint can't silently drop them.
962 let body = health_json_body(OverallStatus::Operational, true);
963 assert_eq!(body["version"], env!("CARGO_PKG_VERSION"));
964 assert!(
965 body.get("git_sha").is_some(),
966 "git_sha key must be present (null is fine)"
967 );
968 }
969
970 /// Schema-drift guard for the `mnw` target. See `shared/pom-contract/`.
971 #[test]
972 fn pom_hetzner_health_expectations_resolve() {
973 let body = health_json_body(OverallStatus::Operational, true);
974 pom_contract::assert_health_expectations_resolve(
975 "../pom/deploy/pom-hetzner.toml",
976 "mnw",
977 &body,
978 );
979 }
980 }
981