Skip to main content

max / makenotwork

pom: serve GET /status.json for the release viewer PoM becomes the third source over the shared ops-status contract. Each monitored target is one node (kind = "target"); health, open incidents, TLS and domain expiry, backup staleness, and the scan pipeline map to conditions carrying the why, with version/uptime/latency/last-checked as typed fields. A target's status is the worst of its conditions: unlike a Sando promotion gate, an expiring cert or stale backup is a real problem and must color the target even when the health check is green. The scan pipeline was the one check that only alerted and never persisted, so it gains a scan_pipeline_checks table (migration 11) written by its task and read back for the payload. PoM declares no actions — it observes, and there is no recheck route. Served in the authenticated router, gated by the same bearer token as /api/*; the viewer example ships the pom source with POM_API_TOKEN. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-22 23:41 UTC
Commit: 51541c42fb891a4e6be90d106e614bdb2efdcf2e
Parent: e6e7bd4
11 files changed, +764 insertions, -20 deletions
M pom/Cargo.lock +10
@@ -1649,6 +1649,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1649 1649 checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
1650 1650
1651 1651 [[package]]
1652 + name = "ops-status"
1653 + version = "0.1.0"
1654 + dependencies = [
1655 + "chrono",
1656 + "serde",
1657 + "serde_json",
1658 + ]
1659 +
1660 + [[package]]
1652 1661 name = "option-ext"
1653 1662 version = "0.2.0"
1654 1663 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1734,6 +1743,7 @@ dependencies = [
1734 1743 "hickory-resolver",
1735 1744 "hostname",
1736 1745 "http-body-util",
1746 + "ops-status",
1737 1747 "rcgen",
1738 1748 "reqwest",
1739 1749 "rmcp",
M pom/Cargo.toml +3
@@ -36,6 +36,9 @@ serde = { version = "1", features = ["derive"] }
36 36 serde_json = "1"
37 37 schemars = "1.2"
38 38
39 + # Shared operator status contract (GET /status.json for the release viewer)
40 + ops-status = { path = "../shared/ops-status" }
41 +
39 42 # Config
40 43 toml = "1.1"
41 44
M pom/src/api.rs +124
@@ -197,6 +197,7 @@ pub fn router(pool: sqlx::SqlitePool, config: Config, mesh: Option<SharedMeshSta
197 197 // rate_limit, so an unauthenticated request is rejected before it can consume
198 198 // any rate-limit budget (fuzz-2026-07-06 SERIOUS #5).
199 199 let authenticated = Router::new()
200 + .route("/status.json", get(status_json))
200 201 .route("/api/status", get(status_all))
201 202 .route("/api/status/{target}", get(status_target))
202 203 .route("/api/trends/{target}", get(trends))
@@ -507,6 +508,129 @@ async fn status_target(
507 508 Ok(Json(status))
508 509 }
509 510
511 + /// `GET /status.json` — every monitored target restated in the shared
512 + /// cross-service payload the release viewer renders. See `crate::status`.
513 + #[instrument(skip_all)]
514 + async fn status_json(AxumState(state): AxumState<ApiState>) -> impl IntoResponse {
515 + let targets = build_status_view(&state).await;
516 + Json(crate::status::payload(&targets, chrono::Utc::now()))
517 + }
518 +
519 + /// Read each target's current signals out of the database into the pure view the
520 + /// payload mapping consumes. The DB lives here; `crate::status::payload` stays a
521 + /// pure function of `(targets, now)`.
522 + async fn build_status_view(state: &ApiState) -> Vec<crate::status::TargetView> {
523 + let mut targets = Vec::new();
524 +
525 + for name in state.config.target_names() {
526 + let Some(target_config) = state.config.get_target(&name) else {
527 + continue;
528 + };
529 +
530 + let health = db::get_latest_health(&state.pool, &name)
531 + .await
532 + .ok()
533 + .flatten()
534 + .map(|s| crate::status::HealthView {
535 + status: s.status,
536 + checked_at: s.checked_at,
537 + version: s.details.and_then(|d| d.version),
538 + error: s.error,
539 + });
540 +
541 + let uptime_24h = db::get_uptime_percent(&state.pool, &name, 24)
542 + .await
543 + .unwrap_or(None);
544 +
545 + let latency_avg_ms = {
546 + let cutoff = (chrono::Utc::now() - chrono::Duration::hours(24)).to_rfc3339();
547 + let times = db::get_response_times(&state.pool, &name, &cutoff)
548 + .await
549 + .unwrap_or_default();
550 + let operational: Vec<i64> = times
551 + .iter()
552 + .filter(|(_, ms)| *ms > 0)
553 + .map(|(_, ms)| *ms)
554 + .collect();
555 + LatencyStats::from_times(&operational).map(|s| s.avg_ms)
556 + };
557 +
558 + let tls = db::get_latest_tls_check(&state.pool, &name)
559 + .await
560 + .ok()
561 + .flatten()
562 + .map(|r| crate::status::TlsView {
563 + valid: r.valid,
564 + days_remaining: r.days_remaining,
565 + checked_at: r.checked_at,
566 + error: r.error,
567 + });
568 +
569 + let incident = db::get_open_incident(&state.pool, &name)
570 + .await
571 + .ok()
572 + .flatten()
573 + .map(|i| crate::status::IncidentView {
574 + from_status: i.from_status,
575 + to_status: i.to_status,
576 + started_at: i.started_at,
577 + });
578 +
579 + let whois = db::get_latest_whois_check(&state.pool, &name)
580 + .await
581 + .ok()
582 + .flatten()
583 + .map(|w| crate::status::WhoisView {
584 + days_remaining: w.days_remaining,
585 + checked_at: w.checked_at,
586 + error: w.error,
587 + });
588 +
589 + let mut backups = Vec::new();
590 + if let Some(backup_config) = &target_config.backups {
591 + for database in &backup_config.databases {
592 + if let Ok(Some(row)) =
593 + db::get_latest_backup_check(&state.pool, &name, database).await
594 + {
595 + backups.push(crate::status::BackupView {
596 + database: row.database_name,
597 + status: row.status,
598 + age_hours: row.age_hours,
599 + checked_at: row.checked_at,
600 + error: row.error,
601 + });
602 + }
603 + }
604 + }
605 +
606 + let scan_pipeline = db::get_latest_scan_pipeline_check(&state.pool, &name)
607 + .await
608 + .ok()
609 + .flatten()
610 + .map(|s| crate::status::ScanView {
611 + issues: s.issue_list(),
612 + status: s.status,
613 + checked_at: s.checked_at,
614 + error: s.error,
615 + });
616 +
617 + targets.push(crate::status::TargetView {
618 + name,
619 + label: target_config.label.clone(),
620 + health,
621 + uptime_24h,
622 + latency_avg_ms,
623 + tls,
624 + incident,
625 + whois,
626 + backups,
627 + scan_pipeline,
628 + });
629 + }
630 +
631 + targets
632 + }
633 +
510 634 // --- Peer endpoints ---
511 635
512 636 /// `GET /api/peer/info` — Returns this instance's identity info.
M pom/src/cli/tasks/scan_pipeline.rs +6
@@ -76,6 +76,12 @@ pub(crate) fn spawn_scan_pipeline_tasks(
76 76 );
77 77 }
78 78
79 + // Persist so /status.json can render the latest result from the
80 + // ledger rather than re-probing on every viewer poll.
81 + if let Err(e) = db::insert_scan_pipeline_check(&pool, &result).await {
82 + tracing::error!("{name}: failed to store scan pipeline check: {e}");
83 + }
84 +
79 85 // Fire alerts on status transitions only.
80 86 if let Some(ref alerter) = alerter {
81 87 let prev_ok = previous_status
M pom/src/db/migrations.rs +19
@@ -226,6 +226,25 @@ const MIGRATIONS: &[(i64, &str, &str)] = &[
226 226 CREATE INDEX IF NOT EXISTS idx_pending_alerts_due ON pending_alerts(next_retry_at, id);
227 227 "#,
228 228 ),
229 + (
230 + 11,
231 + "add scan_pipeline_checks table",
232 + r#"
233 + CREATE TABLE IF NOT EXISTS scan_pipeline_checks (
234 + id INTEGER PRIMARY KEY AUTOINCREMENT,
235 + target TEXT NOT NULL,
236 + status TEXT NOT NULL,
237 + issues TEXT NOT NULL, -- JSON array of fired-threshold lines
238 + queue_pending INTEGER NOT NULL,
239 + queue_running INTEGER NOT NULL,
240 + queue_stuck INTEGER NOT NULL,
241 + held_total INTEGER NOT NULL,
242 + checked_at TEXT NOT NULL,
243 + error TEXT
244 + );
245 + CREATE INDEX IF NOT EXISTS idx_scan_pipeline_checks_target ON scan_pipeline_checks(target, id DESC);
246 + "#,
247 + ),
229 248 ];
230 249
231 250 #[instrument(skip_all)]
M pom/src/db/mod.rs +4 -1
@@ -12,7 +12,8 @@ use std::str::FromStr;
12 12 use crate::error::Result;
13 13 use crate::types::{
14 14 BackupCheckResult, CorsCheckResult, DnsCheckResult, HealthDetails, HealthSnapshot,
15 - HealthStatus, TestDetail, TestRun, TestRunId, TestSummary, TlsStatus, WhoisResult,
15 + HealthStatus, ScanPipelineCheckResult, TestDetail, TestRun, TestRunId, TestSummary, TlsStatus,
16 + WhoisResult,
16 17 };
17 18
18 19 mod alerts;
@@ -25,6 +26,7 @@ mod maintenance;
25 26 mod migrations;
26 27 mod peers;
27 28 mod routes;
29 + mod scan_pipeline;
28 30 mod test_runs;
29 31 mod tls;
30 32 mod whois;
@@ -39,6 +41,7 @@ pub use maintenance::*;
39 41 pub use migrations::*;
40 42 pub use peers::*;
41 43 pub use routes::*;
44 + pub use scan_pipeline::*;
42 45 pub use test_runs::*;
43 46 pub use tls::*;
44 47 pub use whois::*;
A pom/src/db/scan_pipeline.rs +71
@@ -0,0 +1,71 @@
1 + //! Scan-pipeline health-check storage.
2 + //!
3 + //! Unlike the other checks, the scan pipeline was originally only alerted on and
4 + //! never persisted, so `/status.json` had nothing to read. This stores the
5 + //! latest result per target the same way `backup_checks` does, so the release
6 + //! viewer can render it from the ledger rather than re-running a live probe on
7 + //! every poll.
8 +
9 + use super::*;
10 + use tracing::instrument;
11 +
12 + #[instrument(skip_all)]
13 + pub async fn insert_scan_pipeline_check(
14 + pool: &SqlitePool,
15 + result: &ScanPipelineCheckResult,
16 + ) -> Result<i64> {
17 + let issues = serde_json::to_string(&result.issues).unwrap_or_default();
18 + let row = sqlx::query(
19 + "INSERT INTO scan_pipeline_checks (target, status, issues, queue_pending, queue_running, queue_stuck, held_total, checked_at, error)
20 + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
21 + )
22 + .bind(&result.target)
23 + .bind(&result.status)
24 + .bind(&issues)
25 + .bind(result.queue_pending)
26 + .bind(result.queue_running)
27 + .bind(result.queue_stuck)
28 + .bind(result.held_total)
29 + .bind(&result.checked_at)
30 + .bind(&result.error)
31 + .execute(pool)
32 + .await?;
33 + Ok(row.last_insert_rowid())
34 + }
35 +
36 + /// The latest scan-pipeline check for a target, if one has been recorded.
37 + #[instrument(skip_all)]
38 + pub async fn get_latest_scan_pipeline_check(
39 + pool: &SqlitePool,
40 + target: &str,
41 + ) -> Result<Option<ScanPipelineCheckRow>> {
42 + Ok(sqlx::query_as::<_, ScanPipelineCheckRow>(
43 + "SELECT id, target, status, issues, queue_pending, queue_running, queue_stuck, held_total, checked_at, error
44 + FROM scan_pipeline_checks WHERE target = ? ORDER BY id DESC LIMIT 1",
45 + )
46 + .bind(target)
47 + .fetch_optional(pool)
48 + .await?)
49 + }
50 +
51 + #[derive(Debug, Clone, sqlx::FromRow, serde::Serialize)]
52 + pub struct ScanPipelineCheckRow {
53 + pub id: i64,
54 + pub target: String,
55 + pub status: String,
56 + /// JSON array of the fired-threshold issue lines. Use [`Self::issue_list`].
57 + pub issues: String,
58 + pub queue_pending: i64,
59 + pub queue_running: i64,
60 + pub queue_stuck: i64,
61 + pub held_total: i64,
62 + pub checked_at: String,
63 + pub error: Option<String>,
64 + }
65 +
66 + impl ScanPipelineCheckRow {
67 + /// Decode the stored issue lines, tolerating a malformed blob as empty.
68 + pub fn issue_list(&self) -> Vec<String> {
69 + serde_json::from_str(&self.issues).unwrap_or_default()
70 + }
71 + }
M pom/src/lib.rs +1
@@ -17,5 +17,6 @@ pub mod db;
17 17 pub mod display;
18 18 pub mod error;
19 19 pub mod peer;
20 + pub mod status;
20 21 pub mod tools;
21 22 pub mod types;
A pom/src/status.rs +500
@@ -0,0 +1,886 @@
1 + //! PoM's projection onto the shared operator status payload.
2 + //!
3 + //! Spec + rationale: maintainer wiki.
4 + //! <!-- wiki: release-status-payload -->
5 + //!
6 + //! `GET /status.json` serves this. Sando and Bento report on the thing they
7 + //! *are*; PoM reports on the things it *watches*, so every target it monitors
8 + //! becomes one node and the viewer needs no knowledge of health checks, TLS, or
9 + //! incidents to draw PoM.
10 + //!
11 + //! [`payload`] is a pure function of `(&[TargetView], now)`. Every clock is an
12 + //! argument, so a fixture renders identically forever and the mapping is
13 + //! testable without a database.
14 + //!
15 + //! # What maps to what
16 + //!
17 + //! | PoM | payload |
18 + //! |---|---|
19 + //! | monitored target | node, `kind = "target"` |
20 + //! | latest health check | node status + `health` condition |
21 + //! | open incident | `incident` condition carrying the why |
22 + //! | TLS certificate | `tls` condition (expiry) |
23 + //! | domain registration | `whois` condition (expiry) |
24 + //! | uptime / latency / version | fields |
25 + //!
26 + //! # Every signal colors the target
27 + //!
28 + //! The load-bearing difference from Sando. A Sando gate guards *promotion* and
29 + //! must not color a tier — burn-in blocks for 48 hours as routine. PoM has no
30 + //! such notion: a cert 3 days from expiry, an open incident, a domain about to
31 + //! lapse are each a real problem with the target regardless of what the health
32 + //! endpoint says. So a target's status is the worst of every condition on it,
33 + //! and the why is never lost because it is in the conditions either way.
34 + //!
35 + //! PoM declares no actions. It observes; it does not act, and there is no route
36 + //! to trigger a recheck. An empty actions map is the honest shape.
37 +
38 + use chrono::{DateTime, Utc};
39 + use ops_status::{Condition, Field, Node, Payload, Status, Value};
40 +
41 + use crate::types::HealthStatus;
42 +
43 + /// The `source` name PoM answers to in a viewer's config.
44 + pub const SOURCE: &str = "pom";
45 +
46 + /// Days before a certificate expires at which the target goes `degraded`.
47 + const TLS_EXPIRY_WARN_DAYS: i64 = 14;
48 +
49 + /// Days before a domain registration lapses at which the target goes `degraded`.
50 + const WHOIS_EXPIRY_WARN_DAYS: i64 = 30;
51 +
52 + /// Everything the payload needs about one monitored target, already read out of
53 + /// the database.
54 + ///
55 + /// A dedicated view rather than the API's `TargetStatus`: that type is shaped
56 + /// for the existing `/api/status` JSON and carries pre-formatted strings, which
57 + /// is exactly what the shared contract forbids a producer from owning. Building
58 + /// this from typed rows keeps [`payload`] pure and the DB out of the mapping.
59 + pub(crate) struct TargetView {
60 + /// Config key, e.g. "mnw".
61 + pub name: String,
62 + /// Human-readable label, e.g. "MakeNotWork".
63 + pub label: String,
64 + /// The most recent health snapshot. `None` before the first check.
65 + pub health: Option<HealthView>,
66 + /// Uptime percentage over the last 24 hours, if any checks fell in the window.
67 + pub uptime_24h: Option<f64>,
68 + /// Mean response time over the last 24 hours of operational checks, in ms.
69 + pub latency_avg_ms: Option<f64>,
70 + /// Latest TLS check. `None` if TLS is not monitored for this target.
71 + pub tls: Option<TlsView>,
72 + /// Currently open incident, if the target is in one.
73 + pub incident: Option<IncidentView>,
74 + /// Latest WHOIS check. `None` if domain expiry is not monitored.
75 + pub whois: Option<WhoisView>,
76 + /// Latest backup freshness, one per monitored database. Empty if the target
77 + /// has no backup monitoring configured.
78 + pub backups: Vec<BackupView>,
79 + /// Latest scan-pipeline check. `None` if not monitored for this target.
80 + pub scan_pipeline: Option<ScanView>,
81 + }
82 +
83 + pub(crate) struct HealthView {
84 + pub status: HealthStatus,
85 + pub checked_at: String,
86 + pub version: Option<String>,
87 + /// The failure reason when the check did not pass.
88 + pub error: Option<String>,
89 + }
90 +
91 + pub(crate) struct TlsView {
92 + pub valid: bool,
93 + pub days_remaining: i64,
94 + pub checked_at: String,
95 + pub error: Option<String>,
96 + }
97 +
98 + pub(crate) struct IncidentView {
99 + pub from_status: String,
100 + pub to_status: String,
101 + pub started_at: String,
102 + }
103 +
104 + pub(crate) struct WhoisView {
105 + pub days_remaining: Option<i64>,
106 + pub checked_at: String,
107 + pub error: Option<String>,
108 + }
109 +
110 + pub(crate) struct BackupView {
111 + pub database: String,
112 + /// One of "ok", "stale", "missing", "error".
113 + pub status: String,
114 + pub age_hours: Option<i64>,
115 + pub checked_at: String,
116 + pub error: Option<String>,
117 + }
118 +
119 + pub(crate) struct ScanView {
120 + /// One of "operational", "degraded", "unreachable".
121 + pub status: String,
122 + /// Fired-threshold issue lines, the why behind a non-operational status.
123 + pub issues: Vec<String>,
124 + pub checked_at: String,
125 + pub error: Option<String>,
126 + }
127 +
128 + /// Restate every monitored target as the shared payload.
129 + ///
130 + /// `now` is an argument rather than read from the clock so the mapping stays
131 + /// pure and snapshot-testable.
132 + pub(crate) fn payload(targets: &[TargetView], now: DateTime<Utc>) -> Payload {
133 + let mut payload = Payload::new(SOURCE, now);
134 + for target in targets {
135 + payload.nodes.push(target_node(target));
136 + }
137 + payload
138 + }
139 +
140 + fn target_node(target: &TargetView) -> Node {
141 + let mut conditions = vec![health_condition(target.health.as_ref())];
142 + if let Some(incident) = &target.incident {
143 + conditions.push(incident_condition(incident));
144 + }
145 + if let Some(tls) = &target.tls {
146 + conditions.push(tls_condition(tls));
147 + }
148 + if let Some(whois) = &target.whois
149 + && let Some(condition) = whois_condition(whois)
150 + {
151 + conditions.push(condition);
152 + }
153 + for backup in &target.backups {
154 + conditions.push(backup_condition(backup));
155 + }
156 + if let Some(scan) = &target.scan_pipeline {
157 + conditions.push(scan_condition(scan));
158 + }
159 +
160 + // The target's status is the worst thing said about it. There is always at
161 + // least the health condition, so the max is well-defined.
162 + let status = conditions
163 + .iter()
164 + .map(|c| c.status)
165 + .max()
166 + .unwrap_or(Status::Unknown);
167 +
168 + Node {
169 + id: format!("target:{}", target.name),
170 + kind: "target".into(),
171 + label: target.label.clone(),
172 + status,
173 + fields: target_fields(target),
174 + conditions,
175 + children: Vec::new(),
176 + actions: Vec::new(),
177 + }
178 + }
179 +
180 + /// A health snapshot's status, or `pending` when the target has never been
181 + /// checked — PoM has evidence of nothing rather than evidence of health.
182 + fn health_condition(health: Option<&HealthView>) -> Condition {
183 + let Some(health) = health else {
184 + return Condition {
185 + condition_type: "health".into(),
186 + status: Status::Pending,
187 + since: None,
188 + detail: Some("no health check recorded yet".into()),
189 + };
190 + };
191 + Condition {
192 + condition_type: "health".into(),
193 + status: health_status(health.status),
194 + since: parse_instant(&health.checked_at),
195 + // The error is the why on a failing check; a passing one needs no words.
196 + detail: health.error.clone(),
197 + }
198 + }
199 +
200 + /// PoM's four-value health vocabulary onto the shared five.
201 + ///
202 + /// `unreachable` and `error` both map to `failed`: from an operator's chair a
203 + /// target that 5xxs and one that will not answer are the same event — it is
204 + /// down. The distinction is preserved in the health snapshot's own error text.
205 + fn health_status(status: HealthStatus) -> Status {
206 + match status {
207 + HealthStatus::Operational => Status::Ok,
208 + HealthStatus::Degraded => Status::Degraded,
209 + HealthStatus::Error | HealthStatus::Unreachable => Status::Failed,
210 + }
211 + }
212 +
213 + /// An open incident is reported at the severity it escalated *to*, carrying when
214 + /// it started so the viewer can age it.
215 + fn incident_condition(incident: &IncidentView) -> Condition {
216 + Condition {
217 + condition_type: "incident".into(),
218 + status: health_status_from_str(&incident.to_status),
219 + since: parse_instant(&incident.started_at),
220 + detail: Some(format!(
221 + "{} to {}",
222 + incident.from_status, incident.to_status
223 + )),
224 + }
225 + }
226 +
227 + /// Incident rows store the health status as a string. An unrecognized value is
228 + /// `unknown` rather than a parse failure, keeping the node legible against a
229 + /// future status PoM learns before the viewer does.
230 + fn health_status_from_str(raw: &str) -> Status {
231 + raw.parse::<HealthStatus>()
232 + .map(health_status)
233 + .unwrap_or(Status::Unknown)
234 + }
235 +
236 + /// TLS state as a condition. A failed probe or an invalid chain is `failed`; a
237 + /// certificate inside the warning window is `degraded`; anything further out is
238 + /// `ok` and still carries its days-remaining, since the number is the point.
239 + fn tls_condition(tls: &TlsView) -> Condition {
240 + let (status, detail) = if let Some(error) = &tls.error {
241 + (Status::Failed, format!("tls check failed: {error}"))
242 + } else if !tls.valid {
243 + (Status::Failed, "certificate chain is not valid".into())
244 + } else if tls.days_remaining < 0 {
245 + (
246 + Status::Failed,
247 + format!("certificate expired {} days ago", -tls.days_remaining),
248 + )
249 + } else if tls.days_remaining <= TLS_EXPIRY_WARN_DAYS {
250 + (
251 + Status::Degraded,
252 + format!("certificate expires in {} days", tls.days_remaining),
253 + )
254 + } else {
255 + (
256 + Status::Ok,
257 + format!("certificate valid, {} days remaining", tls.days_remaining),
258 + )
259 + };
260 + Condition {
261 + condition_type: "tls".into(),
262 + status,
263 + since: parse_instant(&tls.checked_at),
264 + detail: Some(detail),
265 + }
266 + }
267 +
268 + /// Domain-expiry state as a condition, or `None` when there is nothing to say.
269 + ///
270 + /// A WHOIS lookup that failed is `degraded`, not `failed`: registrar WHOIS is
271 + /// flaky and a lookup error is not evidence the domain lapsed. Absent both an
272 + /// error and a days count there is genuinely no signal, so no condition is
273 + /// emitted rather than a permanently `unknown` one that trains the eye to skip
274 + /// the target.
275 + fn whois_condition(whois: &WhoisView) -> Option<Condition> {
276 + let (status, detail) = if let Some(error) = &whois.error {
277 + (Status::Degraded, format!("whois lookup failed: {error}"))
278 + } else {
279 + match whois.days_remaining {
280 + Some(days) if days < 0 => (
281 + Status::Failed,
282 + format!("domain registration expired {} days ago", -days),
283 + ),
284 + Some(days) if days <= WHOIS_EXPIRY_WARN_DAYS => (
285 + Status::Degraded,
286 + format!("domain registration expires in {days} days"),
287 + ),
288 + Some(days) => (
289 + Status::Ok,
290 + format!("domain registration valid, {days} days remaining"),
291 + ),
292 + None => return None,
293 + }
294 + };
295 + Some(Condition {
296 + condition_type: "whois".into(),
297 + status,
298 + since: parse_instant(&whois.checked_at),
299 + detail: Some(detail),
300 + })
301 + }
302 +
303 + /// Backup freshness as a condition, one per database. A stale backup or a check
304 + /// error is `degraded`; a missing backup is `failed`. The 40-day-stale backup
305 + /// that stayed green by every check that existed is exactly the signal this
306 + /// surfaces — the `type` names the database so several read as distinct rows.
307 + fn backup_condition(backup: &BackupView) -> Condition {
308 + let db = &backup.database;
309 + let (status, detail) = match backup.status.as_str() {
310 + "ok" => (
311 + Status::Ok,
312 + match backup.age_hours {
313 + Some(hours) => format!("{db} backup is {hours}h old"),
314 + None => format!("{db} backup present"),
315 + },
316 + ),
317 + "stale" => (
318 + Status::Degraded,
319 + match backup.age_hours {
320 + Some(hours) => format!("{db} backup is stale, {hours}h old"),
321 + None => format!("{db} backup is stale"),
322 + },
323 + ),
324 + "missing" => (Status::Failed, format!("no backup found for {db}")),
325 + "error" => (
326 + Status::Degraded,
327 + match &backup.error {
328 + Some(error) => format!("{db} backup check failed: {error}"),
329 + None => format!("{db} backup check failed"),
330 + },
331 + ),
332 + other => (Status::Unknown, format!("{db} backup status {other:?}")),
333 + };
334 + Condition {
335 + condition_type: format!("backup:{db}"),
336 + status,
337 + since: parse_instant(&backup.checked_at),
338 + detail: Some(detail),
339 + }
340 + }
341 +
342 + /// Scan-pipeline state as a condition. `operational` is ok, `degraded` is
343 + /// degraded, `unreachable` is failed; the fired-threshold issues, or the probe
344 + /// error, are the why.
345 + fn scan_condition(scan: &ScanView) -> Condition {
346 + let status = match scan.status.as_str() {
347 + "operational" => Status::Ok,
348 + "degraded" => Status::Degraded,
349 + "unreachable" => Status::Failed,
350 + _ => Status::Unknown,
351 + };
352 + let detail = if let Some(error) = &scan.error {
353 + format!("unreachable: {error}")
354 + } else if !scan.issues.is_empty() {
355 + scan.issues.join("; ")
356 + } else {
357 + "pipeline operational".into()
358 + };
359 + Condition {
360 + condition_type: "scan_pipeline".into(),
361 + status,
362 + since: parse_instant(&scan.checked_at),
363 + detail: Some(detail),
364 + }
365 + }
366 +
367 + /// Uptime as a bar, latency as a magnitude, version as a comparable — each a
368 + /// number that *means* something rather than a pre-rendered string.
369 + fn target_fields(target: &TargetView) -> Vec<Field> {
370 + let mut fields = Vec::new();
371 +
372 + if let Some(version) = target.health.as_ref().and_then(|h| h.version.clone()) {
373 + fields.push(Field::new("version", Value::Version { value: version }));
374 + }
375 + if let Some(uptime) = target.uptime_24h {
376 + fields.push(Field::new(
377 + "uptime 24h",
378 + Value::Progress {
379 + value: uptime,
380 + max: 100.0,
381 + unit: Some("%".into()),
382 + },
383 + ));
384 + }
385 + if let Some(latency) = target.latency_avg_ms {
386 + fields.push(Field::new(
387 + "latency 24h",
388 + Value::Quantity {
389 + value: latency,
390 + unit: Some("ms".into()),
391 + },
392 + ));
393 + }
394 + if let Some(checked) = target
395 + .health
396 + .as_ref()
397 + .and_then(|h| parse_instant(&h.checked_at))
398 + {
399 + fields.push(Field::new("checked", Value::Instant { value: checked }));
400 + }
401 +
402 + fields
403 + }
404 +
405 + /// A timestamp that fails to parse costs only itself. A viewer that blanks on
406 + /// one malformed row is worse than one missing a tooltip.
407 + fn parse_instant(raw: &str) -> Option<DateTime<Utc>> {
408 + DateTime::parse_from_rfc3339(raw)
409 + .ok()
410 + .map(|d| d.with_timezone(&Utc))
411 + }
412 +
413 + #[cfg(test)]
414 + mod tests {
415 + use super::*;
416 +
417 + fn now() -> DateTime<Utc> {
418 + "2026-07-21T18:24:39Z".parse().unwrap()
419 + }
420 +
421 + fn checked_at() -> String {
422 + "2026-07-21T18:24:00Z".into()
423 + }
424 +
425 + fn healthy(name: &str) -> TargetView {
426 + TargetView {
427 + name: name.into(),
428 + label: name.to_uppercase(),
429 + health: Some(HealthView {
430 + status: HealthStatus::Operational,
431 + checked_at: checked_at(),
432 + version: Some("1.4.0".into()),
433 + error: None,
434 + }),
435 + uptime_24h: Some(100.0),
436 + latency_avg_ms: Some(42.0),
437 + tls: None,
438 + incident: None,
439 + whois: None,
440 + backups: Vec::new(),
441 + scan_pipeline: None,
442 + }
443 + }
444 +
445 + fn node<'a>(p: &'a Payload, id: &str) -> &'a Node {
446 + p.node(id).unwrap_or_else(|| panic!("no node {id}"))
447 + }
448 +
449 + #[test]
450 + fn a_healthy_target_is_ok_and_structurally_sound() {
451 + let p = payload(&[healthy("mnw")], now());
452 +
453 + assert_eq!(p.source, SOURCE);
454 + assert_eq!(p.schema, ops_status::SCHEMA_VERSION);
455 + assert_eq!(p.validate(), Ok(()));
456 + assert_eq!(node(&p, "target:mnw").status, Status::Ok);
457 + assert_eq!(node(&p, "target:mnw").label, "MNW");
458 + assert_eq!(p.worst_status(), Status::Ok);
459 + }
460 +
461 + #[test]
462 + fn uptime_and_latency_are_typed_values_not_strings() {
463 + let p = payload(&[healthy("mnw")], now());
464 + let n = node(&p, "target:mnw");
465 +
466 + let uptime = n.fields.iter().find(|f| f.label == "uptime 24h").unwrap();
467 + assert_eq!(
468 + uptime.value,
469 + Value::Progress {
470 + value: 100.0,
471 + max: 100.0,
472 + unit: Some("%".into())
473 + }
474 + );
475 + let latency = n.fields.iter().find(|f| f.label == "latency 24h").unwrap();
476 + assert_eq!(
477 + latency.value,
478 + Value::Quantity {
479 + value: 42.0,
480 + unit: Some("ms".into())
481 + }
482 + );
483 + }
484 +
485 + #[test]
486 + fn an_unreachable_target_is_failed_and_says_why() {
487 + let mut t = healthy("mnw");
488 + t.health = Some(HealthView {
489 + status: HealthStatus::Unreachable,
490 + checked_at: checked_at(),
491 + version: None,
492 + error: Some("connection timed out".into()),
493 + });
494 + let p = payload(&[t], now());
495 +
496 + let n = node(&p, "target:mnw");
497 + assert_eq!(n.status, Status::Failed);
498 + assert_eq!(n.conditions[0].condition_type, "health");
499 + assert_eq!(
500 + n.conditions[0].detail.as_deref(),
Lines truncated
M pom/tests/integration.rs +15 -13
@@ -412,7 +412,7 @@ async fn migration_fresh_db_reaches_latest_version() {
412 412 // A fresh in-memory DB should run all migrations and reach the latest version.
413 413 let pool = db::connect_in_memory().await.unwrap();
414 414 let version = db::get_schema_version(&pool).await.unwrap();
415 - assert_eq!(version, 10);
415 + assert_eq!(version, 11);
416 416
417 417 // Verify the schema_version table has entries for each migration
418 418 let rows = sqlx::query_as::<_, (i64, String)>(
@@ -421,7 +421,7 @@ async fn migration_fresh_db_reaches_latest_version() {
421 421 .fetch_all(&pool)
422 422 .await
423 423 .unwrap();
424 - assert_eq!(rows.len(), 10);
424 + assert_eq!(rows.len(), 11);
425 425 assert_eq!(rows[0].0, 1);
426 426 assert_eq!(rows[0].1, "initial schema");
427 427 assert_eq!(rows[1].0, 2);
@@ -442,6 +442,8 @@ async fn migration_fresh_db_reaches_latest_version() {
442 442 assert_eq!(rows[8].1, "add backup_checks table");
443 443 assert_eq!(rows[9].0, 10);
444 444 assert_eq!(rows[9].1, "add pending_alerts retry queue");
445 + assert_eq!(rows[10].0, 11);
446 + assert_eq!(rows[10].1, "add scan_pipeline_checks table");
445 447
446 448 // Verify actual tables were created by inserting data
447 449 let snapshot = HealthSnapshot {
@@ -461,18 +463,18 @@ async fn migration_fresh_db_reaches_latest_version() {
461 463 async fn migration_already_current_is_idempotent() {
462 464 // Running migrations on an already-migrated DB should be a no-op.
463 465 let pool = db::connect_in_memory().await.unwrap();
464 - assert_eq!(db::get_schema_version(&pool).await.unwrap(), 10);
466 + assert_eq!(db::get_schema_version(&pool).await.unwrap(), 11);
465 467
466 468 // Run migrations again
467 469 db::run_migrations(&pool).await.unwrap();
468 - assert_eq!(db::get_schema_version(&pool).await.unwrap(), 10);
470 + assert_eq!(db::get_schema_version(&pool).await.unwrap(), 11);
469 471
470 - // schema_version should still have exactly ten entries (not duplicated)
472 + // schema_version should still have exactly eleven entries (not duplicated)
471 473 let count = sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM schema_version")
472 474 .fetch_one(&pool)
473 475 .await
474 476 .unwrap();
475 - assert_eq!(count.0, 10);
477 + assert_eq!(count.0, 11);
476 478 }
477 479
478 480 #[tokio::test]
@@ -531,8 +533,8 @@ async fn migration_detects_pre_migration_database() {
531 533 // Now run migrations — should detect existing tables, stamp as v1, then run v2+v3+v4+v5+v6
532 534 db::run_migrations(&pool).await.unwrap();
533 535
534 - // Version should be 10 (stamped v1 + ran v2..v10)
535 - assert_eq!(db::get_schema_version(&pool).await.unwrap(), 10);
536 + // Version should be 11 (stamped v1 + ran v2..v11)
537 + assert_eq!(db::get_schema_version(&pool).await.unwrap(), 11);
536 538
537 539 // Description should indicate pre-existing
538 540 let row =
@@ -852,7 +854,7 @@ async fn tool_run_tests_no_test_config() {
852 854 async fn migration_v2_creates_alerts_table() {
853 855 let pool = db::connect_in_memory().await.unwrap();
854 856 let version = db::get_schema_version(&pool).await.unwrap();
855 - assert_eq!(version, 10);
857 + assert_eq!(version, 11);
856 858
857 859 // Verify alerts table exists by inserting
858 860 let id = db::insert_alert(
@@ -938,7 +940,7 @@ async fn prune_removes_old_alerts() {
938 940 async fn migration_v3_creates_tls_checks_table() {
939 941 let pool = db::connect_in_memory().await.unwrap();
940 942 let version = db::get_schema_version(&pool).await.unwrap();
941 - assert_eq!(version, 10);
943 + assert_eq!(version, 11);
942 944
943 945 // Verify tls_checks table exists by inserting
944 946 let status = pom::types::TlsStatus {
@@ -1176,7 +1178,7 @@ cooldown_secs = 120
1176 1178 async fn migration_v4_creates_incidents_table() {
1177 1179 let pool = db::connect_in_memory().await.unwrap();
1178 1180 let version = db::get_schema_version(&pool).await.unwrap();
1179 - assert_eq!(version, 10);
1181 + assert_eq!(version, 11);
1180 1182
1181 1183 // Verify incidents table exists by inserting
1182 1184 let id = db::insert_incident(&pool, "mnw", "operational", "degraded")
@@ -1316,7 +1318,7 @@ async fn api_status_no_incidents_omits_fields() {
1316 1318 async fn migration_v5_creates_route_checks_table() {
1317 1319 let pool = db::connect_in_memory().await.unwrap();
1318 1320 let version = db::get_schema_version(&pool).await.unwrap();
1319 - assert_eq!(version, 10);
1321 + assert_eq!(version, 11);
1320 1322
1321 1323 // Verify route_checks table exists by inserting
1322 1324 let result = pom::checks::routes::RouteCheckResult {
@@ -2458,7 +2460,7 @@ async fn peer_uuid_mismatch_updates_db_identity() {
2458 2460 async fn migration_v6_creates_dns_and_whois_tables() {
2459 2461 let pool = db::connect_in_memory().await.unwrap();
2460 2462 let version = db::get_schema_version(&pool).await.unwrap();
2461 - assert_eq!(version, 10);
2463 + assert_eq!(version, 11);
2462 2464
2463 2465 // Verify dns_checks table exists
2464 2466 let dns_result = DnsCheckResult {
M viewer/deploy/viewer.toml.example +11 -6
@@ -31,9 +31,14 @@ url = "http://fw13:8090"
31 31 poll_secs = 10
32 32
33 33 # PoM polls its own targets on a slower cycle, so hold it to a slower staleness
34 - # limit rather than flagging it degraded between its own checks.
35 - # [[source]]
36 - # name = "pom"
37 - # url = "http://pom:9000"
38 - # poll_secs = 30
39 - # stale_after_secs = 600
34 + # limit rather than flagging it degraded between its own checks. Each target PoM
35 + # watches is a node; the rollup shows the worst of them. PoM gates /status.json
36 + # behind the same bearer token as its /api/* reads, so name the token here — it
37 + # is the value in pom's serve.api_token, supplied to the viewer via the
38 + # environment, never pasted into this file.
39 + [[source]]
40 + name = "pom"
41 + url = "http://pom:9000"
42 + token_env = "POM_API_TOKEN"
43 + poll_secs = 30
44 + stale_after_secs = 600