Skip to main content

max / makenotwork

61.9 KB · 1806 lines History Blame Raw
1 //! PoM's projection onto the shared operator status payload.
2 //!
3 //! Spec + rationale: maintainer wiki.
4 //! <!-- wiki: release-status-payload -->
5 //!
6 //! `GET /status.json` serves this. Sando and Bento report on the thing they
7 //! *are*; PoM reports on the things it *watches*, so every target it monitors
8 //! becomes one node and the viewer needs no knowledge of health checks, TLS, or
9 //! incidents to draw PoM.
10 //!
11 //! [`payload`] is a pure function of `(&[TargetView], now)`. Every clock is an
12 //! argument, so a fixture renders identically forever and the mapping is
13 //! testable without a database.
14 //!
15 //! # What maps to what
16 //!
17 //! | PoM | payload |
18 //! |---|---|
19 //! | monitored target | node, `kind = "target"` |
20 //! | latest health check | node status + `health` condition |
21 //! | open incident | `incident` condition carrying the why |
22 //! | TLS certificate | `tls` condition (expiry) |
23 //! | domain registration | `whois` condition (expiry) |
24 //! | uptime / latency / version | fields |
25 //!
26 //! # Every signal colors the target
27 //!
28 //! The load-bearing difference from Sando. A Sando gate guards *promotion* and
29 //! must not color a tier, burn-in blocks for 48 hours as routine. PoM has no
30 //! such notion: a cert 3 days from expiry, an open incident, a domain about to
31 //! lapse are each a real problem with the target regardless of what the health
32 //! endpoint says. So a target's status is the worst of every condition on it,
33 //! and the why is never lost because it is in the conditions either way.
34 //!
35 //! PoM declares no actions. It observes; it does not act, and there is no route
36 //! to trigger a recheck. An empty actions map is the honest shape.
37
38 use chrono::{DateTime, Utc};
39 use ops_status::{Condition, Field, Node, Payload, Status, Value};
40
41 use crate::types::HealthStatus;
42
43 /// The `source` name PoM answers to in a viewer's config.
44 pub const SOURCE: &str = "pom";
45
46 /// Days before a certificate expires at which the target goes `degraded`.
47 const TLS_EXPIRY_WARN_DAYS: i64 = 14;
48
49 /// Days before a domain registration lapses at which the target goes `degraded`.
50 const WHOIS_EXPIRY_WARN_DAYS: i64 = 30;
51
52 /// Everything the payload needs about one monitored target, already read out of
53 /// the database.
54 ///
55 /// A dedicated view rather than the API's `TargetStatus`: that type is shaped
56 /// for the existing `/api/status` JSON and carries pre-formatted strings, which
57 /// is exactly what the shared contract forbids a producer from owning. Building
58 /// this from typed rows keeps [`payload`] pure and the DB out of the mapping.
59 pub(crate) struct TargetView {
60 /// Config key, e.g. "mnw".
61 pub name: String,
62 /// Human-readable label, e.g. "MakeNotWork".
63 pub label: String,
64 /// The most recent health snapshot. `None` before the first check.
65 pub health: Option<HealthView>,
66 /// Whether the target configures an HTTP health check at all.
67 ///
68 /// Distinct from `health` being `None`, which only says no snapshot has been
69 /// stored. Without this the two collapse and a target that will never have a
70 /// health endpoint reports `health: pending` forever: the desktop apps and the
71 /// SDK are test-only targets, so af, bb, go and sk sat permanently pending and
72 /// held the whole source there with them. A row that cannot ever go green is
73 /// worse than no row, because a surface where most rows are unsatisfiable is
74 /// one nobody reads.
75 pub health_configured: bool,
76 /// Uptime percentage over the last 24 hours, if any checks fell in the window.
77 pub uptime_24h: Option<f64>,
78 /// Mean response time over the last 24 hours of operational checks, in ms.
79 pub latency_avg_ms: Option<f64>,
80 /// Latest TLS check. `None` if TLS is not monitored for this target.
81 pub tls: Option<TlsView>,
82 /// Currently open incident, if the target is in one.
83 pub incident: Option<IncidentView>,
84 /// Latest WHOIS check. `None` if domain expiry is not monitored.
85 pub whois: Option<WhoisView>,
86 /// Latest backup freshness, one per monitored database. Empty if the target
87 /// has no backup monitoring configured.
88 pub backups: Vec<BackupView>,
89 /// Latest scan-pipeline check. `None` if not monitored for this target.
90 pub scan_pipeline: Option<ScanView>,
91 /// Latest local systemd daemon-health check. `None` if not monitored.
92 pub systemd: Option<SystemdView>,
93 /// Latest local CA-bundle freshness check. `None` if not monitored.
94 pub ca_bundle: Option<CaBundleView>,
95 /// Latest SyncKit field-version readout. `None` if not monitored.
96 pub synckit_fleet: Option<SyncKitFleetView>,
97 /// Latest test run and PoM's staleness verdict. `None` if the target has no
98 /// test config.
99 pub tests: Option<TestsView>,
100 /// Latest DNS record checks. `None` if DNS is not monitored for this target.
101 pub dns: Option<DnsView>,
102 /// Latest CORS preflight checks. `None` if CORS is not monitored.
103 pub cors: Option<CorsView>,
104 }
105
106 pub(crate) struct HealthView {
107 pub status: HealthStatus,
108 pub checked_at: String,
109 pub version: Option<String>,
110 /// The failure reason when the check did not pass.
111 pub error: Option<String>,
112 }
113
114 pub(crate) struct TlsView {
115 pub valid: bool,
116 pub days_remaining: i64,
117 pub checked_at: String,
118 pub error: Option<String>,
119 /// Per-trust-store results. `None` means the reading predates migration 14,
120 /// not that the probe failed. See [`tls_condition`].
121 pub webpki_trusted: Option<bool>,
122 pub platform_trusted: Option<bool>,
123 pub platform_error: Option<String>,
124 }
125
126 pub(crate) struct IncidentView {
127 pub from_status: String,
128 pub to_status: String,
129 pub started_at: String,
130 }
131
132 pub(crate) struct WhoisView {
133 pub days_remaining: Option<i64>,
134 pub checked_at: String,
135 pub error: Option<String>,
136 }
137
138 pub(crate) struct BackupView {
139 pub database: String,
140 /// One of "ok", "stale", "missing", "error".
141 pub status: String,
142 pub age_hours: Option<i64>,
143 pub checked_at: String,
144 pub error: Option<String>,
145 }
146
147 pub(crate) struct ScanView {
148 /// One of "operational", "degraded", "unreachable".
149 pub status: String,
150 /// Fired-threshold issue lines, the why behind a non-operational status.
151 pub issues: Vec<String>,
152 pub checked_at: String,
153 pub error: Option<String>,
154 }
155
156 pub(crate) struct SystemdView {
157 /// One of "operational", "degraded", "down".
158 pub status: String,
159 /// Unhealthy-unit / failed-sweep lines, the why behind a non-operational
160 /// status.
161 pub issues: Vec<String>,
162 pub checked_at: String,
163 pub error: Option<String>,
164 }
165
166 pub(crate) struct CaBundleView {
167 /// One of "ok", "stale", "thin", "unknown", "error".
168 pub status: String,
169 /// The why behind a non-ok status, one line per fired signal.
170 pub issues: Vec<String>,
171 pub checked_at: String,
172 pub error: Option<String>,
173 }
174
175 /// A SyncKit field-version readout. No status field: the reading cannot be
176 /// unhealthy, only unavailable. See [`crate::checks::synckit_fleet`].
177 pub(crate) struct SyncKitFleetView {
178 /// Devices that synced within the server's activity window.
179 pub devices: i64,
180 /// The window that count covers, in days.
181 pub window_days: i64,
182 /// Version and device count, most devices first. `None` version means
183 /// "syncing, version unknown".
184 pub versions: Vec<(Option<String>, i64)>,
185 pub checked_at: String,
186 /// Why the readout could not be taken. `None` on a successful reading.
187 pub error: Option<String>,
188 }
189
190 pub(crate) struct TestsView {
191 /// Whether any run has ever been recorded. Separates "no evidence"
192 /// (pending) from a run that failed.
193 pub ran: bool,
194 /// Whether the most recent run passed.
195 pub passed: bool,
196 pub total_passed: Option<i64>,
197 pub total_failed: Option<i64>,
198 /// When the most recent run started, RFC 3339.
199 pub started_at: Option<String>,
200 /// PoM's staleness verdict for the run: true when it is older than the
201 /// configured threshold or a version was deployed after it ran.
202 pub stale: bool,
203 pub stale_reason: Option<String>,
204 }
205
206 pub(crate) struct DnsView {
207 /// One entry per monitored record; non-empty by construction.
208 pub records: Vec<DnsRecordView>,
209 /// Latest checked_at across the records, for the condition's `since`.
210 pub checked_at: Option<String>,
211 }
212
213 pub(crate) struct DnsRecordView {
214 pub name: String,
215 pub record_type: String,
216 pub matches: bool,
217 pub error: Option<String>,
218 }
219
220 pub(crate) struct CorsView {
221 /// One entry per monitored URL; non-empty by construction.
222 pub checks: Vec<CorsCheckView>,
223 pub checked_at: Option<String>,
224 }
225
226 pub(crate) struct CorsCheckView {
227 pub url: String,
228 pub origin: String,
229 pub passes: bool,
230 pub error: Option<String>,
231 }
232
233 /// Restate every monitored target as the shared payload.
234 ///
235 /// `now` is an argument rather than read from the clock so the mapping stays
236 /// pure and snapshot-testable.
237 pub(crate) fn payload(targets: &[TargetView], now: DateTime<Utc>) -> Payload {
238 let mut payload = Payload::new(SOURCE, now);
239 for target in targets {
240 payload.nodes.push(target_node(target));
241 }
242 payload
243 }
244
245 fn target_node(target: &TargetView) -> Node {
246 // Only speak about health where health is watched. A stored snapshot still
247 // reports even if the config was since removed, so history stays visible.
248 let mut conditions = Vec::new();
249 if target.health_configured || target.health.is_some() {
250 conditions.push(health_condition(target.health.as_ref()));
251 }
252 if let Some(incident) = &target.incident {
253 conditions.push(incident_condition(incident));
254 }
255 if let Some(tls) = &target.tls {
256 conditions.push(tls_condition(tls));
257 }
258 if let Some(whois) = &target.whois
259 && let Some(condition) = whois_condition(whois)
260 {
261 conditions.push(condition);
262 }
263 for backup in &target.backups {
264 conditions.push(backup_condition(backup));
265 }
266 if let Some(scan) = &target.scan_pipeline {
267 conditions.push(scan_condition(scan));
268 }
269 if let Some(sd) = &target.systemd {
270 conditions.push(systemd_condition(sd));
271 }
272 if let Some(ca) = &target.ca_bundle {
273 conditions.push(ca_bundle_condition(ca));
274 }
275 if let Some(fleet) = &target.synckit_fleet {
276 conditions.push(synckit_fleet_condition(fleet));
277 }
278 if let Some(tests) = &target.tests {
279 conditions.push(tests_condition(tests));
280 }
281 if let Some(dns) = &target.dns {
282 conditions.push(dns_condition(dns));
283 }
284 if let Some(cors) = &target.cors {
285 conditions.push(cors_condition(cors));
286 }
287
288 // The target's status is the worst thing said about it. There is always at
289 // least the health condition, so the max is well-defined.
290 let status = conditions
291 .iter()
292 .map(|c| c.status)
293 .max()
294 .unwrap_or(Status::Unknown);
295
296 Node {
297 id: format!("target:{}", target.name),
298 kind: "target".into(),
299 label: target.label.clone(),
300 status,
301 fields: target_fields(target),
302 conditions,
303 children: Vec::new(),
304 actions: Vec::new(),
305 }
306 }
307
308 /// A health snapshot's status, or `pending` when the target has never been
309 /// checked, PoM has evidence of nothing rather than evidence of health.
310 fn health_condition(health: Option<&HealthView>) -> Condition {
311 let Some(health) = health else {
312 return Condition {
313 condition_type: "health".into(),
314 status: Status::Pending,
315 since: None,
316 detail: Some("no health check recorded yet".into()),
317 };
318 };
319 Condition {
320 condition_type: "health".into(),
321 status: health_status(health.status),
322 since: parse_instant(&health.checked_at),
323 // The error is the why on a failing check; a passing one needs no words.
324 detail: health.error.clone(),
325 }
326 }
327
328 /// PoM's four-value health vocabulary onto the shared five.
329 ///
330 /// `unreachable` and `error` both map to `failed`: from an operator's chair a
331 /// target that 5xxs and one that will not answer are the same event, it is
332 /// down. The distinction is preserved in the health snapshot's own error text.
333 fn health_status(status: HealthStatus) -> Status {
334 match status {
335 HealthStatus::Operational => Status::Ok,
336 HealthStatus::Degraded => Status::Degraded,
337 HealthStatus::Error | HealthStatus::Unreachable => Status::Failed,
338 }
339 }
340
341 /// An open incident is reported at the severity it escalated *to*, carrying when
342 /// it started so the viewer can age it.
343 fn incident_condition(incident: &IncidentView) -> Condition {
344 Condition {
345 condition_type: "incident".into(),
346 status: health_status_from_str(&incident.to_status),
347 since: parse_instant(&incident.started_at),
348 detail: Some(format!(
349 "{} to {}",
350 incident.from_status, incident.to_status
351 )),
352 }
353 }
354
355 /// Incident rows store the health status as a string. An unrecognized value is
356 /// `unknown` rather than a parse failure, keeping the node legible against a
357 /// future status PoM learns before the viewer does.
358 fn health_status_from_str(raw: &str) -> Status {
359 raw.parse::<HealthStatus>()
360 .map_or(Status::Unknown, health_status)
361 }
362
363 /// TLS state as a condition. A failed probe or an invalid chain is `failed`; a
364 /// certificate inside the warning window is `degraded`; anything further out is
365 /// `ok` and still carries its days-remaining, since the number is the point.
366 fn tls_condition(tls: &TlsView) -> Condition {
367 let (status, detail) = if let Some(error) = &tls.error {
368 (Status::Failed, format!("tls check failed: {error}"))
369 } else if !tls.valid {
370 (Status::Failed, "certificate chain is not valid".into())
371 } else if tls.days_remaining < 0 {
372 (
373 Status::Failed,
374 format!("certificate expired {} days ago", -tls.days_remaining),
375 )
376 } else if tls.webpki_trusted == Some(true) && tls.platform_trusted == Some(false) {
377 // The chain is fine by the public web PKI and rejected by this host's
378 // own trust store, so the fault is the host's trust anchors rather than
379 // the certificate. That is the CA-bundle failure multithreaded would hit
380 // on all three of its outbound paths, and it used to arrive as a login
381 // outage because this reading was computed and thrown away.
382 //
383 // Ordered ahead of the expiry warning deliberately: both are degraded,
384 // and a broken trust store is the rarer and more actionable of the two.
385 (
386 Status::Degraded,
387 match &tls.platform_error {
388 Some(error) => {
389 format!("host trust store rejects a chain the public web PKI accepts: {error}")
390 }
391 None => "host trust store rejects a chain the public web PKI accepts".into(),
392 },
393 )
394 } else if tls.days_remaining <= TLS_EXPIRY_WARN_DAYS {
395 (
396 Status::Degraded,
397 format!("certificate expires in {} days", tls.days_remaining),
398 )
399 } else {
400 (
401 Status::Ok,
402 format!("certificate valid, {} days remaining", tls.days_remaining),
403 )
404 };
405 Condition {
406 condition_type: "tls".into(),
407 status,
408 since: parse_instant(&tls.checked_at),
409 detail: Some(detail),
410 }
411 }
412
413 /// Domain-expiry state as a condition, or `None` when there is nothing to say.
414 ///
415 /// A WHOIS lookup that failed is `degraded`, not `failed`: registrar WHOIS is
416 /// flaky and a lookup error is not evidence the domain lapsed. Absent both an
417 /// error and a days count there is genuinely no signal, so no condition is
418 /// emitted rather than a permanently `unknown` one that trains the eye to skip
419 /// the target.
420 fn whois_condition(whois: &WhoisView) -> Option<Condition> {
421 let (status, detail) = if let Some(error) = &whois.error {
422 (Status::Degraded, format!("whois lookup failed: {error}"))
423 } else {
424 match whois.days_remaining {
425 Some(days) if days < 0 => (
426 Status::Failed,
427 format!("domain registration expired {} days ago", -days),
428 ),
429 Some(days) if days <= WHOIS_EXPIRY_WARN_DAYS => (
430 Status::Degraded,
431 format!("domain registration expires in {days} days"),
432 ),
433 Some(days) => (
434 Status::Ok,
435 format!("domain registration valid, {days} days remaining"),
436 ),
437 None => return None,
438 }
439 };
440 Some(Condition {
441 condition_type: "whois".into(),
442 status,
443 since: parse_instant(&whois.checked_at),
444 detail: Some(detail),
445 })
446 }
447
448 /// Backup freshness as a condition, one per database. A stale backup or a check
449 /// error is `degraded`; a missing backup is `failed`. The 40-day-stale backup
450 /// that stayed green by every check that existed is exactly the signal this
451 /// surfaces, the `type` names the database so several read as distinct rows.
452 fn backup_condition(backup: &BackupView) -> Condition {
453 let db = &backup.database;
454 let (status, detail) = match backup.status.as_str() {
455 "ok" => (
456 Status::Ok,
457 match backup.age_hours {
458 Some(hours) => format!("{db} backup is {hours}h old"),
459 None => format!("{db} backup present"),
460 },
461 ),
462 "stale" => (
463 Status::Degraded,
464 match backup.age_hours {
465 Some(hours) => format!("{db} backup is stale, {hours}h old"),
466 None => format!("{db} backup is stale"),
467 },
468 ),
469 "missing" => (Status::Failed, format!("no backup found for {db}")),
470 "error" => (
471 Status::Degraded,
472 match &backup.error {
473 Some(error) => format!("{db} backup check failed: {error}"),
474 None => format!("{db} backup check failed"),
475 },
476 ),
477 other => (Status::Unknown, format!("{db} backup status {other:?}")),
478 };
479 Condition {
480 condition_type: format!("backup:{db}"),
481 status,
482 since: parse_instant(&backup.checked_at),
483 detail: Some(detail),
484 }
485 }
486
487 /// Scan-pipeline state as a condition. `operational` is ok, `degraded` is
488 /// degraded, `unreachable` is failed; the fired-threshold issues, or the probe
489 /// error, are the why.
490 fn scan_condition(scan: &ScanView) -> Condition {
491 let status = match scan.status.as_str() {
492 "operational" => Status::Ok,
493 "degraded" => Status::Degraded,
494 "unreachable" => Status::Failed,
495 _ => Status::Unknown,
496 };
497 let detail = if let Some(error) = &scan.error {
498 format!("unreachable: {error}")
499 } else if !scan.issues.is_empty() {
500 scan.issues.join("; ")
501 } else {
502 "pipeline operational".into()
503 };
504 Condition {
505 condition_type: "scan_pipeline".into(),
506 status,
507 since: parse_instant(&scan.checked_at),
508 detail: Some(detail),
509 }
510 }
511
512 /// Local systemd daemon health as a condition. A down watched unit is `failed`
513 /// (red, a daemon that watches the platform is itself dead); a crash-loop or a
514 /// host-wide failed unit is `degraded` (yellow, look at this). A probe failure
515 /// (no systemd, no `--user` bus) is `failed` and names the error.
516 fn systemd_condition(sd: &SystemdView) -> Condition {
517 let status = match sd.status.as_str() {
518 "operational" => Status::Ok,
519 "degraded" => Status::Degraded,
520 "down" => Status::Failed,
521 _ => Status::Unknown,
522 };
523 let detail = if let Some(error) = &sd.error {
524 format!("probe error: {error}")
525 } else if !sd.issues.is_empty() {
526 sd.issues.join("; ")
527 } else {
528 "all watched daemons healthy".into()
529 };
530 Condition {
531 condition_type: "systemd".into(),
532 status,
533 since: parse_instant(&sd.checked_at),
534 detail: Some(detail),
535 }
536 }
537
538 /// CA-bundle freshness as a condition.
539 ///
540 /// `thin` is the only red: a bundle below the certificate floor cannot validate
541 /// the public web PKI, so outbound TLS on this host is already broken or one
542 /// handshake from it. `stale` (a newer package is available) and `unknown` (the
543 /// package lists are too old for the comparison to mean anything) are drift, and
544 /// drift is yellow. A probe that could not run is yellow for the same reason the
545 /// SyncKit readout is: PoM has stopped being able to answer the question, which
546 /// is worth saying out loud but is not itself an outage.
547 fn ca_bundle_condition(ca: &CaBundleView) -> Condition {
548 let status = match ca.status.as_str() {
549 "ok" => Status::Ok,
550 "thin" => Status::Failed,
551 "stale" | "unknown" | "error" => Status::Degraded,
552 _ => Status::Unknown,
553 };
554 let detail = if let Some(error) = &ca.error {
555 format!("probe error: {error}")
556 } else if !ca.issues.is_empty() {
557 ca.issues.join("; ")
558 } else {
559 "trust anchors current".into()
560 };
561 Condition {
562 condition_type: "ca_bundle".into(),
563 status,
564 since: parse_instant(&ca.checked_at),
565 detail: Some(detail),
566 }
567 }
568
569 /// The SyncKit field-version readout as a condition. Only two outcomes, and
570 /// neither of them depends on which versions came back: `ok` when the reading was
571 /// taken (the detail is the distribution), `degraded` when it could not be. No
572 /// version mix can fail this condition, because a user still on an old SDK is a
573 /// fact about the world rather than an incident PoM can page anyone about. A
574 /// broken readout *is* worth yellow: PoM has stopped being able to answer "which
575 /// SyncKit is in the field" and would otherwise go on reporting green while
576 /// knowing nothing. It stays out of red because whether the platform is up is the
577 /// `health` condition's job.
578 fn synckit_fleet_condition(fleet: &SyncKitFleetView) -> Condition {
579 let since = parse_instant(&fleet.checked_at);
580 let (status, detail) = match &fleet.error {
581 Some(error) => (
582 Status::Degraded,
583 format!("fleet readout unavailable: {error}"),
584 ),
585 None if fleet.devices == 0 => (
586 Status::Ok,
587 format!("no devices synced in {}d", fleet.window_days),
588 ),
589 None => (
590 Status::Ok,
591 format!(
592 "{} device{} in {}d: {}",
593 fleet.devices,
594 plural(fleet.devices as usize),
595 fleet.window_days,
596 fleet_summary(&fleet.versions)
597 ),
598 ),
599 };
600 Condition {
601 condition_type: "synckit_fleet".into(),
602 status,
603 since,
604 detail: Some(detail),
605 }
606 }
607
608 /// The distribution as one line, e.g. `0.6.0 x12, unknown x3`. An absent version
609 /// is spelled out rather than dropped: a fleet that is half unknown is a real
610 /// reading and hiding the unknowns would overstate what PoM knows.
611 fn fleet_summary(versions: &[(Option<String>, i64)]) -> String {
612 if versions.is_empty() {
613 return "no versions reported".into();
614 }
615 versions
616 .iter()
617 .map(|(version, devices)| format!("{} x{devices}", version.as_deref().unwrap_or("unknown")))
618 .collect::<Vec<_>>()
619 .join(", ")
620 }
621
622 /// Test state as a condition. A never-run target is `pending`: evidence of
623 /// nothing, like one never health-checked. A failing or stale run is `degraded`,
624 /// not `failed`: a red suite is a real regression signal, but the running
625 /// service is covered by `health`, so a test result colors the target yellow
626 /// (look at this) rather than red (it is down). Staleness reuses PoM's own
627 /// verdict, which fires on age past the configured threshold or a version
628 /// deployed since the last run.
629 fn tests_condition(tests: &TestsView) -> Condition {
630 if !tests.ran {
631 return Condition {
632 condition_type: "tests".into(),
633 status: Status::Pending,
634 since: None,
635 detail: Some("no tests have been run yet".into()),
636 };
637 }
638 let since = tests.started_at.as_deref().and_then(parse_instant);
639 let (status, detail) = if !tests.passed {
640 let detail = match (tests.total_passed, tests.total_failed) {
641 (Some(p), Some(f)) => format!("last run failed: {f} failed, {p} passed"),
642 _ => "last test run failed".into(),
643 };
644 (Status::Degraded, detail)
645 } else if tests.stale {
646 (
647 Status::Degraded,
648 tests
649 .stale_reason
650 .clone()
651 .unwrap_or_else(|| "test results are stale".into()),
652 )
653 } else {
654 let detail = match tests.total_passed {
655 Some(p) => format!("tests passing, {p} passed"),
656 None => "tests passing".into(),
657 };
658 (Status::Ok, detail)
659 };
660 Condition {
661 condition_type: "tests".into(),
662 status,
663 since,
664 detail: Some(detail),
665 }
666 }
667
668 /// DNS state as one aggregate condition across a target's monitored records. A
669 /// record whose lookup errored or whose value does not match is `degraded`: a
670 /// wrong or unverifiable record is a real misconfiguration, but a record that
671 /// points the site somewhere dead already shows as a failed `health` check, so
672 /// DNS colors the target yellow and names what is off. All records clean is
673 /// `ok`. One aggregate condition rather than one per record keeps a target that
674 /// watches several records to a single legible row.
675 fn dns_condition(dns: &DnsView) -> Condition {
676 let bad: Vec<String> = dns
677 .records
678 .iter()
679 .filter_map(|r| {
680 if let Some(error) = &r.error {
681 Some(format!(
682 "{} {} lookup failed: {error}",
683 r.name, r.record_type
684 ))
685 } else if !r.matches {
686 Some(format!("{} {} does not match", r.name, r.record_type))
687 } else {
688 None
689 }
690 })
691 .collect();
692 let since = dns.checked_at.as_deref().and_then(parse_instant);
693 let (status, detail) = if bad.is_empty() {
694 (
695 Status::Ok,
696 format!(
697 "{} record{} match",
698 dns.records.len(),
699 plural(dns.records.len())
700 ),
701 )
702 } else {
703 (Status::Degraded, bad.join("; "))
704 };
705 Condition {
706 condition_type: "dns".into(),
707 status,
708 since,
709 detail: Some(detail),
710 }
711 }
712
713 /// CORS state as one aggregate condition across a target's monitored URLs. A
714 /// preflight that failed to run, or that the server answered without allowing
715 /// the origin, is `degraded`: broken CORS breaks a browser client, but it is a
716 /// configuration fault rather than the service being down, so it colors the
717 /// target yellow. All preflights allowed is `ok`.
718 fn cors_condition(cors: &CorsView) -> Condition {
719 let bad: Vec<String> = cors
720 .checks
721 .iter()
722 .filter_map(|c| {
723 if let Some(error) = &c.error {
724 Some(format!("{} preflight failed: {error}", c.url))
725 } else if !c.passes {
726 Some(format!("{} does not allow {}", c.url, c.origin))
727 } else {
728 None
729 }
730 })
731 .collect();
732 let since = cors.checked_at.as_deref().and_then(parse_instant);
733 let (status, detail) = if bad.is_empty() {
734 (
735 Status::Ok,
736 format!(
737 "{} preflight check{} pass",
738 cors.checks.len(),
739 plural(cors.checks.len())
740 ),
741 )
742 } else {
743 (Status::Degraded, bad.join("; "))
744 };
745 Condition {
746 condition_type: "cors".into(),
747 status,
748 since,
749 detail: Some(detail),
750 }
751 }
752
753 /// The plural suffix for a count: "" for one, "s" otherwise.
754 fn plural(n: usize) -> &'static str {
755 if n == 1 { "" } else { "s" }
756 }
757
758 /// Uptime as a bar, latency as a magnitude, version as a comparable, each a
759 /// number that *means* something rather than a pre-rendered string.
760 fn target_fields(target: &TargetView) -> Vec<Field> {
761 let mut fields = Vec::new();
762
763 if let Some(version) = target.health.as_ref().and_then(|h| h.version.clone()) {
764 fields.push(Field::new("version", Value::Version { value: version }));
765 }
766 if let Some(uptime) = target.uptime_24h {
767 fields.push(Field::new(
768 "uptime 24h",
769 Value::Progress {
770 value: uptime,
771 max: 100.0,
772 unit: Some("%".into()),
773 },
774 ));
775 }
776 if let Some(latency) = target.latency_avg_ms {
777 fields.push(Field::new(
778 "latency 24h",
779 Value::Quantity {
780 value: latency,
781 unit: Some("ms".into()),
782 },
783 ));
784 }
785 // The fleet distribution is a field, not just a condition detail: it is the
786 // answer to a question an operator asks on purpose ("which SyncKit is out
787 // there"), not a by-product of something being wrong. A successful readout of
788 // zero devices still reports, so the number is visibly zero rather than
789 // missing.
790 if let Some(fleet) = &target.synckit_fleet
791 && fleet.error.is_none()
792 {
793 fields.push(Field::new(
794 "synckit fleet",
795 Value::Text {
796 value: fleet_summary(&fleet.versions),
797 },
798 ));
799 fields.push(Field::new(
800 "synckit devices",
801 Value::Quantity {
802 value: fleet.devices as f64,
803 unit: None,
804 },
805 ));
806 }
807 if let Some(checked) = target
808 .health
809 .as_ref()
810 .and_then(|h| parse_instant(&h.checked_at))
811 {
812 fields.push(Field::new("checked", Value::Instant { value: checked }));
813 }
814
815 fields
816 }
817
818 /// A timestamp that fails to parse costs only itself. A viewer that blanks on
819 /// one malformed row is worse than one missing a tooltip.
820 fn parse_instant(raw: &str) -> Option<DateTime<Utc>> {
821 DateTime::parse_from_rfc3339(raw)
822 .ok()
823 .map(|d| d.with_timezone(&Utc))
824 }
825
826 #[cfg(test)]
827 mod tests {
828 use super::*;
829
830 fn now() -> DateTime<Utc> {
831 "2026-07-21T18:24:39Z".parse().unwrap()
832 }
833
834 fn checked_at() -> String {
835 "2026-07-21T18:24:00Z".into()
836 }
837
838 fn healthy(name: &str) -> TargetView {
839 TargetView {
840 name: name.into(),
841 label: name.to_uppercase(),
842 health_configured: true,
843 health: Some(HealthView {
844 status: HealthStatus::Operational,
845 checked_at: checked_at(),
846 version: Some("1.4.0".into()),
847 error: None,
848 }),
849 uptime_24h: Some(100.0),
850 latency_avg_ms: Some(42.0),
851 tls: None,
852 incident: None,
853 whois: None,
854 backups: Vec::new(),
855 scan_pipeline: None,
856 systemd: None,
857 ca_bundle: None,
858 synckit_fleet: None,
859 tests: None,
860 dns: None,
861 cors: None,
862 }
863 }
864
865 fn node<'a>(p: &'a Payload, id: &str) -> &'a Node {
866 p.node(id).unwrap_or_else(|| panic!("no node {id}"))
867 }
868
869 #[test]
870 fn a_healthy_target_is_ok_and_structurally_sound() {
871 let p = payload(&[healthy("mnw")], now());
872
873 assert_eq!(p.source, SOURCE);
874 assert_eq!(p.schema, ops_status::SCHEMA_VERSION);
875 assert_eq!(p.validate(), Ok(()));
876 assert_eq!(node(&p, "target:mnw").status, Status::Ok);
877 assert_eq!(node(&p, "target:mnw").label, "MNW");
878 assert_eq!(p.worst_status(), Status::Ok);
879 }
880
881 #[test]
882 fn uptime_and_latency_are_typed_values_not_strings() {
883 let p = payload(&[healthy("mnw")], now());
884 let n = node(&p, "target:mnw");
885
886 let uptime = n.fields.iter().find(|f| f.label == "uptime 24h").unwrap();
887 assert_eq!(
888 uptime.value,
889 Value::Progress {
890 value: 100.0,
891 max: 100.0,
892 unit: Some("%".into())
893 }
894 );
895 let latency = n.fields.iter().find(|f| f.label == "latency 24h").unwrap();
896 assert_eq!(
897 latency.value,
898 Value::Quantity {
899 value: 42.0,
900 unit: Some("ms".into())
901 }
902 );
903 }
904
905 #[test]
906 fn an_unreachable_target_is_failed_and_says_why() {
907 let mut t = healthy("mnw");
908 t.health = Some(HealthView {
909 status: HealthStatus::Unreachable,
910 checked_at: checked_at(),
911 version: None,
912 error: Some("connection timed out".into()),
913 });
914 let p = payload(&[t], now());
915
916 let n = node(&p, "target:mnw");
917 assert_eq!(n.status, Status::Failed);
918 assert_eq!(n.conditions[0].condition_type, "health");
919 assert_eq!(
920 n.conditions[0].detail.as_deref(),
921 Some("connection timed out")
922 );
923 assert_eq!(p.worst_status(), Status::Failed);
924 }
925
926 #[test]
927 fn a_target_never_checked_is_pending_not_healthy() {
928 let mut t = healthy("new");
929 t.health = None;
930 let p = payload(&[t], now());
931
932 let n = node(&p, "target:new");
933 assert_eq!(n.status, Status::Pending);
934 assert_eq!(
935 n.conditions[0].detail.as_deref(),
936 Some("no health check recorded yet")
937 );
938 // No health snapshot means no version and no checked-at field.
939 assert!(n.fields.iter().all(|f| f.label != "version"));
940 assert!(n.fields.iter().all(|f| f.label != "checked"));
941 assert_eq!(p.validate(), Ok(()));
942 }
943
944 #[test]
945 fn an_expiring_certificate_degrades_an_otherwise_healthy_target() {
946 // The Sando contrast: unlike a promotion gate, an expiring cert is a real
947 // problem with the target and must color it even when health is green.
948 let mut t = healthy("mnw");
949 t.tls = Some(TlsView {
950 valid: true,
951 days_remaining: 9,
952 checked_at: checked_at(),
953 error: None,
954 webpki_trusted: Some(true),
955 platform_trusted: Some(true),
956 platform_error: None,
957 });
958 let p = payload(&[t], now());
959
960 let n = node(&p, "target:mnw");
961 assert_eq!(n.status, Status::Degraded);
962 let tls = n
963 .conditions
964 .iter()
965 .find(|c| c.condition_type == "tls")
966 .unwrap();
967 assert_eq!(tls.status, Status::Degraded);
968 assert!(tls.detail.as_deref().unwrap().contains("9 days"));
969 }
970
971 #[test]
972 fn an_expired_certificate_fails_the_target() {
973 let mut t = healthy("mnw");
974 t.tls = Some(TlsView {
975 valid: true,
976 days_remaining: -3,
977 checked_at: checked_at(),
978 error: None,
979 webpki_trusted: Some(true),
980 platform_trusted: Some(true),
981 platform_error: None,
982 });
983 let p = payload(&[t], now());
984
985 let n = node(&p, "target:mnw");
986 assert_eq!(n.status, Status::Failed);
987 let tls = n
988 .conditions
989 .iter()
990 .find(|c| c.condition_type == "tls")
991 .unwrap();
992 assert!(
993 tls.detail
994 .as_deref()
995 .unwrap()
996 .contains("expired 3 days ago")
997 );
998 }
999
1000 #[test]
1001 fn a_healthy_certificate_stays_ok_but_still_reports_its_runway() {
1002 let mut t = healthy("mnw");
1003 t.tls = Some(TlsView {
1004 valid: true,
1005 days_remaining: 60,
1006 checked_at: checked_at(),
1007 error: None,
1008 webpki_trusted: Some(true),
1009 platform_trusted: Some(true),
1010 platform_error: None,
1011 });
1012 let p = payload(&[t], now());
1013
1014 let n = node(&p, "target:mnw");
1015 assert_eq!(n.status, Status::Ok);
1016 let tls = n
1017 .conditions
1018 .iter()
1019 .find(|c| c.condition_type == "tls")
1020 .unwrap();
1021 assert_eq!(tls.status, Status::Ok);
1022 assert!(tls.detail.as_deref().unwrap().contains("60 days remaining"));
1023 }
1024
1025 #[test]
1026 fn a_host_trust_store_that_rejects_a_publicly_valid_chain_degrades_the_target() {
1027 // multithreaded takes its outbound trust anchors from the host CA
1028 // bundle on every path it has, with no in-binary fallback. A bundle
1029 // that goes stale or thin therefore breaks OAuth, link previews and S3
1030 // at once while the certificates themselves are perfectly good, which
1031 // is why the two stores are reported separately rather than folded.
1032 let mut t = healthy("mnw");
1033 t.tls = Some(TlsView {
1034 valid: true,
1035 days_remaining: 60,
1036 checked_at: checked_at(),
1037 error: None,
1038 webpki_trusted: Some(true),
1039 platform_trusted: Some(false),
1040 platform_error: Some("invalid peer certificate: UnknownIssuer".into()),
1041 });
1042 let p = payload(&[t], now());
1043
1044 let n = node(&p, "target:mnw");
1045 assert_eq!(n.status, Status::Degraded);
1046 let tls = n
1047 .conditions
1048 .iter()
1049 .find(|c| c.condition_type == "tls")
1050 .unwrap();
1051 assert_eq!(tls.status, Status::Degraded);
1052 let detail = tls.detail.as_deref().unwrap();
1053 assert!(detail.contains("host trust store"));
1054 assert!(detail.contains("UnknownIssuer"));
1055 }
1056
1057 #[test]
1058 fn a_thin_ca_bundle_fails_the_target_and_a_stale_one_only_degrades_it() {
1059 // The severity split is the point: a bundle below the certificate floor
1060 // means outbound TLS is broken now, while a package one release behind
1061 // is drift. Collapsing them would either page on drift or bury an outage.
1062 let mut thin = healthy("mnw");
1063 thin.ca_bundle = Some(CaBundleView {
1064 status: "thin".into(),
1065 issues: vec!["bundle holds 3 certificates, below the floor of 80".into()],
1066 checked_at: checked_at(),
1067 error: None,
1068 });
1069 assert_eq!(
1070 node(&payload(&[thin], now()), "target:mnw").status,
1071 Status::Failed
1072 );
1073
1074 let mut stale = healthy("mnw");
1075 stale.ca_bundle = Some(CaBundleView {
1076 status: "stale".into(),
1077 issues: vec!["ca-certificates 20260601 installed, 20261101 available".into()],
1078 checked_at: checked_at(),
1079 error: None,
1080 });
1081 let p = payload(&[stale], now());
1082 let n = node(&p, "target:mnw");
1083 assert_eq!(n.status, Status::Degraded);
1084 let ca = n
1085 .conditions
1086 .iter()
1087 .find(|c| c.condition_type == "ca_bundle")
1088 .unwrap();
1089 assert!(ca.detail.as_deref().unwrap().contains("20261101 available"));
1090 }
1091
1092 #[test]
1093 fn a_ca_bundle_probe_that_could_not_run_says_so_rather_than_reading_green() {
1094 let mut t = healthy("mnw");
1095 t.ca_bundle = Some(CaBundleView {
1096 status: "error".into(),
1097 issues: Vec::new(),
1098 checked_at: checked_at(),
1099 error: Some("apt-cache policy failed to run: No such file or directory".into()),
1100 });
1101 let p = payload(&[t], now());
1102 let n = node(&p, "target:mnw");
1103 assert_eq!(n.status, Status::Degraded);
1104 let ca = n
1105 .conditions
1106 .iter()
1107 .find(|c| c.condition_type == "ca_bundle")
1108 .unwrap();
1109 assert!(ca.detail.as_deref().unwrap().contains("probe error"));
1110 }
1111
1112 #[test]
1113 fn a_pre_migration_tls_row_reports_expiry_and_claims_nothing_about_trust() {
1114 // Rows written before the trust columns existed carry NULL, not false.
1115 // Reading those as "untrusted" would light up every target on the first
1116 // run after an upgrade, which trains the eye to ignore the condition.
1117 let mut t = healthy("mnw");
1118 t.tls = Some(TlsView {
1119 valid: true,
1120 days_remaining: 60,
1121 checked_at: checked_at(),
1122 error: None,
1123 webpki_trusted: None,
1124 platform_trusted: None,
1125 platform_error: None,
1126 });
1127 let p = payload(&[t], now());
1128
1129 let n = node(&p, "target:mnw");
1130 assert_eq!(n.status, Status::Ok);
1131 let tls = n
1132 .conditions
1133 .iter()
1134 .find(|c| c.condition_type == "tls")
1135 .unwrap();
1136 assert_eq!(tls.status, Status::Ok);
1137 assert!(tls.detail.as_deref().unwrap().contains("60 days remaining"));
1138 }
1139
1140 #[test]
1141 fn an_open_incident_surfaces_with_its_transition_and_start() {
1142 let mut t = healthy("mnw");
1143 t.incident = Some(IncidentView {
1144 from_status: "operational".into(),
1145 to_status: "unreachable".into(),
1146 started_at: "2026-07-21T17:00:00Z".into(),
1147 });
1148 // Health has recovered on paper but the incident is still open: the node
1149 // must not read green while an incident stands.
1150 let p = payload(&[t], now());
1151
1152 let n = node(&p, "target:mnw");
1153 assert_eq!(n.status, Status::Failed);
1154 let incident = n
1155 .conditions
1156 .iter()
1157 .find(|c| c.condition_type == "incident")
1158 .unwrap();
1159 assert_eq!(incident.status, Status::Failed);
1160 assert_eq!(
1161 incident.detail.as_deref(),
1162 Some("operational to unreachable")
1163 );
1164 assert_eq!(
1165 incident.since,
1166 Some("2026-07-21T17:00:00Z".parse::<DateTime<Utc>>().unwrap())
1167 );
1168 }
1169
1170 #[test]
1171 fn a_failed_whois_lookup_is_degraded_not_failed() {
1172 // Registrar WHOIS is flaky; a lookup error is not proof the domain lapsed.
1173 let mut t = healthy("mnw");
1174 t.whois = Some(WhoisView {
1175 days_remaining: None,
1176 checked_at: checked_at(),
1177 error: Some("connection reset".into()),
1178 });
1179 let p = payload(&[t], now());
1180
1181 let n = node(&p, "target:mnw");
1182 assert_eq!(n.status, Status::Degraded);
1183 }
1184
1185 #[test]
1186 fn an_expiring_domain_degrades_the_target() {
1187 let mut t = healthy("mnw");
1188 t.whois = Some(WhoisView {
1189 days_remaining: Some(12),
1190 checked_at: checked_at(),
1191 error: None,
1192 });
1193 let p = payload(&[t], now());
1194
1195 let n = node(&p, "target:mnw");
1196 assert_eq!(n.status, Status::Degraded);
1197 let whois = n
1198 .conditions
1199 .iter()
1200 .find(|c| c.condition_type == "whois")
1201 .unwrap();
1202 assert!(whois.detail.as_deref().unwrap().contains("12 days"));
1203 }
1204
1205 #[test]
1206 fn a_whois_check_with_no_signal_emits_no_condition() {
1207 let mut t = healthy("mnw");
1208 t.whois = Some(WhoisView {
1209 days_remaining: None,
1210 checked_at: checked_at(),
1211 error: None,
1212 });
1213 let p = payload(&[t], now());
1214
1215 let n = node(&p, "target:mnw");
1216 assert!(n.conditions.iter().all(|c| c.condition_type != "whois"));
1217 assert_eq!(n.status, Status::Ok);
1218 }
1219
1220 #[test]
1221 fn the_loudest_of_several_problems_wins_the_target() {
1222 let mut t = healthy("mnw");
1223 t.health = Some(HealthView {
1224 status: HealthStatus::Degraded,
1225 checked_at: checked_at(),
1226 version: Some("1.4.0".into()),
1227 error: Some("unexpected status 503".into()),
1228 });
1229 t.tls = Some(TlsView {
1230 valid: true,
1231 days_remaining: -1,
1232 checked_at: checked_at(),
1233 error: None,
1234 webpki_trusted: Some(true),
1235 platform_trusted: Some(true),
1236 platform_error: None,
1237 });
1238 let p = payload(&[t], now());
1239
1240 // health is degraded, tls is failed: the target is failed.
1241 assert_eq!(node(&p, "target:mnw").status, Status::Failed);
1242 }
1243
1244 #[test]
1245 fn one_targets_failure_does_not_touch_another() {
1246 let mut down = healthy("mt");
1247 down.health = Some(HealthView {
1248 status: HealthStatus::Error,
1249 checked_at: checked_at(),
1250 version: None,
1251 error: Some("500 Internal Server Error".into()),
1252 });
1253 let p = payload(&[healthy("mnw"), down], now());
1254
1255 assert_eq!(node(&p, "target:mnw").status, Status::Ok);
1256 assert_eq!(node(&p, "target:mt").status, Status::Failed);
1257 assert_eq!(p.worst_status(), Status::Failed);
1258 assert_eq!(p.validate(), Ok(()));
1259 }
1260
1261 #[test]
1262 fn an_unknown_incident_status_stays_legible() {
1263 let mut t = healthy("mnw");
1264 t.incident = Some(IncidentView {
1265 from_status: "operational".into(),
1266 to_status: "sideways".into(),
1267 started_at: checked_at(),
1268 });
1269 let p = payload(&[t], now());
1270
1271 let incident = node(&p, "target:mnw")
1272 .conditions
1273 .iter()
1274 .find(|c| c.condition_type == "incident")
1275 .unwrap();
1276 assert_eq!(incident.status, Status::Unknown);
1277 }
1278
1279 #[test]
1280 fn a_malformed_timestamp_costs_only_that_timestamp() {
1281 let mut t = healthy("mnw");
1282 t.health = Some(HealthView {
1283 status: HealthStatus::Operational,
1284 checked_at: "not a timestamp".into(),
1285 version: Some("1.4.0".into()),
1286 error: None,
1287 });
1288 let p = payload(&[t], now());
1289
1290 let n = node(&p, "target:mnw");
1291 assert_eq!(n.status, Status::Ok);
1292 assert_eq!(n.conditions[0].since, None);
1293 assert!(n.fields.iter().all(|f| f.label != "checked"));
1294 }
1295
1296 #[test]
1297 fn a_stale_backup_degrades_the_target_and_names_the_database() {
1298 // The 40-day-stale backup that stayed green by every check that existed.
1299 let mut t = healthy("mnw");
1300 t.backups = vec![BackupView {
1301 database: "makenotwork".into(),
1302 status: "stale".into(),
1303 age_hours: Some(960),
1304 checked_at: checked_at(),
1305 error: None,
1306 }];
1307 let p = payload(&[t], now());
1308
1309 let n = node(&p, "target:mnw");
1310 assert_eq!(n.status, Status::Degraded);
1311 let backup = n
1312 .conditions
1313 .iter()
1314 .find(|c| c.condition_type == "backup:makenotwork")
1315 .unwrap();
1316 assert_eq!(backup.status, Status::Degraded);
1317 assert!(backup.detail.as_deref().unwrap().contains("960h"));
1318 }
1319
1320 #[test]
1321 fn a_missing_backup_fails_the_target() {
1322 let mut t = healthy("mnw");
1323 t.backups = vec![BackupView {
1324 database: "makenotwork".into(),
1325 status: "missing".into(),
1326 age_hours: None,
1327 checked_at: checked_at(),
1328 error: None,
1329 }];
1330 let p = payload(&[t], now());
1331 assert_eq!(node(&p, "target:mnw").status, Status::Failed);
1332 }
1333
1334 #[test]
1335 fn several_databases_read_as_distinct_conditions() {
1336 let mut t = healthy("mnw");
1337 t.backups = vec![
1338 BackupView {
1339 database: "makenotwork".into(),
1340 status: "ok".into(),
1341 age_hours: Some(6),
1342 checked_at: checked_at(),
1343 error: None,
1344 },
1345 BackupView {
1346 database: "multithreaded".into(),
1347 status: "ok".into(),
1348 age_hours: Some(7),
1349 checked_at: checked_at(),
1350 error: None,
1351 },
1352 ];
1353 let p = payload(&[t], now());
1354
1355 let n = node(&p, "target:mnw");
1356 assert_eq!(n.status, Status::Ok);
1357 assert!(
1358 n.conditions
1359 .iter()
1360 .any(|c| c.condition_type == "backup:makenotwork")
1361 );
1362 assert!(
1363 n.conditions
1364 .iter()
1365 .any(|c| c.condition_type == "backup:multithreaded")
1366 );
1367 }
1368
1369 #[test]
1370 fn a_degraded_scan_pipeline_carries_its_issues() {
1371 let mut t = healthy("mnw");
1372 t.scan_pipeline = Some(ScanView {
1373 status: "degraded".into(),
1374 issues: vec!["thumbnail error rate 22%".into(), "queue stuck: 4".into()],
1375 checked_at: checked_at(),
1376 error: None,
1377 });
1378 let p = payload(&[t], now());
1379
1380 let n = node(&p, "target:mnw");
1381 assert_eq!(n.status, Status::Degraded);
1382 let scan = n
1383 .conditions
1384 .iter()
1385 .find(|c| c.condition_type == "scan_pipeline")
1386 .unwrap();
1387 assert!(
1388 scan.detail
1389 .as_deref()
1390 .unwrap()
1391 .contains("thumbnail error rate 22%")
1392 );
1393 assert!(scan.detail.as_deref().unwrap().contains("queue stuck: 4"));
1394 }
1395
1396 #[test]
1397 fn an_unreachable_scan_pipeline_fails_the_target() {
1398 let mut t = healthy("mnw");
1399 t.scan_pipeline = Some(ScanView {
1400 status: "unreachable".into(),
1401 issues: Vec::new(),
1402 checked_at: checked_at(),
1403 error: Some("502 Bad Gateway".into()),
1404 });
1405 let p = payload(&[t], now());
1406
1407 let n = node(&p, "target:mnw");
1408 assert_eq!(n.status, Status::Failed);
1409 let scan = n
1410 .conditions
1411 .iter()
1412 .find(|c| c.condition_type == "scan_pipeline")
1413 .unwrap();
1414 assert!(scan.detail.as_deref().unwrap().contains("502 Bad Gateway"));
1415 }
1416
1417 fn fleet_view(versions: Vec<(Option<&str>, i64)>) -> SyncKitFleetView {
1418 SyncKitFleetView {
1419 devices: versions.iter().map(|(_, d)| d).sum(),
1420 window_days: 30,
1421 versions: versions
1422 .into_iter()
1423 .map(|(v, d)| (v.map(str::to_string), d))
1424 .collect(),
1425 checked_at: checked_at(),
1426 error: None,
1427 }
1428 }
1429
1430 #[test]
1431 fn a_fleet_readout_reports_the_distribution_without_degrading() {
1432 let mut t = healthy("mnw");
1433 t.synckit_fleet = Some(fleet_view(vec![(Some("0.6.0"), 12), (None, 3)]));
1434 let p = payload(&[t], now());
1435
1436 let n = node(&p, "target:mnw");
1437 assert_eq!(n.status, Status::Ok);
1438 let fleet = n
1439 .conditions
1440 .iter()
1441 .find(|c| c.condition_type == "synckit_fleet")
1442 .unwrap();
1443 assert_eq!(fleet.status, Status::Ok);
1444 let detail = fleet.detail.as_deref().unwrap();
1445 assert!(detail.contains("15 devices in 30d"), "got {detail}");
1446 assert!(detail.contains("0.6.0 x12"), "got {detail}");
1447 assert!(detail.contains("unknown x3"), "got {detail}");
1448 }
1449
1450 #[test]
1451 fn an_ancient_version_in_the_field_does_not_degrade_the_target() {
1452 // The whole design call: version age is a fact, not an incident. A fleet
1453 // entirely on a year-old SDK must still read green.
1454 let mut t = healthy("mnw");
1455 t.synckit_fleet = Some(fleet_view(vec![(Some("0.1.0"), 200)]));
1456 let p = payload(&[t], now());
1457
1458 let n = node(&p, "target:mnw");
1459 assert_eq!(n.status, Status::Ok);
1460 }
1461
1462 #[test]
1463 fn an_empty_fleet_is_ok_and_says_so() {
1464 let mut t = healthy("mnw");
1465 t.synckit_fleet = Some(fleet_view(vec![]));
1466 let p = payload(&[t], now());
1467
1468 let n = node(&p, "target:mnw");
1469 assert_eq!(n.status, Status::Ok);
1470 let fleet = n
1471 .conditions
1472 .iter()
1473 .find(|c| c.condition_type == "synckit_fleet")
1474 .unwrap();
1475 assert!(
1476 fleet
1477 .detail
1478 .as_deref()
1479 .unwrap()
1480 .contains("no devices synced in 30d")
1481 );
1482 }
1483
1484 #[test]
1485 fn an_unavailable_fleet_readout_degrades_but_never_fails() {
1486 let mut t = healthy("mnw");
1487 let mut fleet = fleet_view(vec![]);
1488 fleet.error = Some("HTTP 401 (alerts ingest token rejected)".into());
1489 t.synckit_fleet = Some(fleet);
1490 let p = payload(&[t], now());
1491
1492 let n = node(&p, "target:mnw");
1493 assert_eq!(
1494 n.status,
1495 Status::Degraded,
1496 "a readout PoM cannot take is yellow, not red: health owns whether MNW is up"
1497 );
1498 let fleet = n
1499 .conditions
1500 .iter()
1501 .find(|c| c.condition_type == "synckit_fleet")
1502 .unwrap();
1503 assert!(fleet.detail.as_deref().unwrap().contains("401"));
1504 }
1505
1506 #[test]
1507 fn a_successful_readout_becomes_target_fields() {
1508 let mut t = healthy("mnw");
1509 t.synckit_fleet = Some(fleet_view(vec![(Some("0.6.0"), 12)]));
1510 let p = payload(&[t], now());
1511
1512 let n = node(&p, "target:mnw");
1513 assert!(n.fields.iter().any(|f| f.label == "synckit fleet"));
1514 assert!(n.fields.iter().any(|f| f.label == "synckit devices"));
1515 }
1516
1517 #[test]
1518 fn an_unavailable_readout_contributes_no_fields() {
1519 // A blank column beats a field reading "0 devices" that is really "PoM
1520 // could not ask".
1521 let mut t = healthy("mnw");
1522 let mut fleet = fleet_view(vec![]);
1523 fleet.error = Some("request: timed out".into());
1524 t.synckit_fleet = Some(fleet);
1525 let p = payload(&[t], now());
1526
1527 let n = node(&p, "target:mnw");
1528 assert!(!n.fields.iter().any(|f| f.label.starts_with("synckit")));
1529 }
1530
1531 fn tests_view(passed: bool, stale: bool) -> TestsView {
1532 TestsView {
1533 ran: true,
1534 passed,
1535 total_passed: Some(226),
1536 total_failed: if passed { Some(0) } else { Some(3) },
1537 started_at: Some(checked_at()),
1538 stale,
1539 stale_reason: if stale {
1540 Some("tests are 12 days old (threshold: 7d)".into())
1541 } else {
1542 None
1543 },
1544 }
1545 }
1546
1547 fn condition<'a>(n: &'a Node, ty: &str) -> &'a Condition {
1548 n.conditions
1549 .iter()
1550 .find(|c| c.condition_type == ty)
1551 .unwrap_or_else(|| panic!("no {ty} condition"))
1552 }
1553
1554 #[test]
1555 fn a_never_run_test_target_is_pending_not_healthy() {
1556 let mut t = healthy("mnw");
1557 t.tests = Some(TestsView {
1558 ran: false,
1559 passed: false,
1560 total_passed: None,
1561 total_failed: None,
1562 started_at: None,
1563 stale: true,
1564 stale_reason: Some("no tests have been run".into()),
1565 });
1566 let p = payload(&[t], now());
1567 let n = node(&p, "target:mnw");
1568 // Pending is quieter than degraded: no evidence is not a failure.
1569 assert_eq!(n.status, Status::Pending);
1570 assert_eq!(
1571 condition(n, "tests").detail.as_deref(),
1572 Some("no tests have been run yet")
1573 );
1574 }
1575
1576 #[test]
1577 fn a_failing_test_run_degrades_but_does_not_fail_the_target() {
1578 // A red suite is a regression signal; the running service is health's job.
1579 let mut t = healthy("mnw");
1580 t.tests = Some(tests_view(false, false));
1581 let p = payload(&[t], now());
1582 let n = node(&p, "target:mnw");
1583 assert_eq!(n.status, Status::Degraded);
1584 let c = condition(n, "tests");
1585 assert_eq!(c.status, Status::Degraded);
1586 assert!(c.detail.as_deref().unwrap().contains("3 failed"));
1587 }
1588
1589 #[test]
1590 fn a_stale_but_passing_test_run_degrades_the_target() {
1591 let mut t = healthy("mnw");
1592 t.tests = Some(tests_view(true, true));
1593 let p = payload(&[t], now());
1594 let n = node(&p, "target:mnw");
1595 assert_eq!(n.status, Status::Degraded);
1596 assert!(
1597 condition(n, "tests")
1598 .detail
1599 .as_deref()
1600 .unwrap()
1601 .contains("12 days old")
1602 );
1603 }
1604
1605 #[test]
1606 fn a_fresh_passing_test_run_leaves_the_target_ok() {
1607 let mut t = healthy("mnw");
1608 t.tests = Some(tests_view(true, false));
1609 let p = payload(&[t], now());
1610 let n = node(&p, "target:mnw");
1611 assert_eq!(n.status, Status::Ok);
1612 assert_eq!(condition(n, "tests").status, Status::Ok);
1613 }
1614
1615 #[test]
1616 fn a_dns_mismatch_degrades_the_target_and_names_the_record() {
1617 let mut t = healthy("mnw");
1618 t.dns = Some(DnsView {
1619 records: vec![
1620 DnsRecordView {
1621 name: "makenot.work".into(),
1622 record_type: "A".into(),
1623 matches: true,
1624 error: None,
1625 },
1626 DnsRecordView {
1627 name: "makenot.work".into(),
1628 record_type: "MX".into(),
1629 matches: false,
1630 error: None,
1631 },
1632 ],
1633 checked_at: Some(checked_at()),
1634 });
1635 let p = payload(&[t], now());
1636 let n = node(&p, "target:mnw");
1637 assert_eq!(n.status, Status::Degraded);
1638 let c = condition(n, "dns");
1639 assert_eq!(c.status, Status::Degraded);
1640 assert!(c.detail.as_deref().unwrap().contains("MX does not match"));
1641 }
1642
1643 #[test]
1644 fn a_dns_lookup_error_reads_as_a_flaky_degrade_not_a_mismatch() {
1645 let mut t = healthy("mnw");
1646 t.dns = Some(DnsView {
1647 records: vec![DnsRecordView {
1648 name: "makenot.work".into(),
1649 record_type: "TXT".into(),
1650 matches: false,
1651 error: Some("SERVFAIL".into()),
1652 }],
1653 checked_at: Some(checked_at()),
1654 });
1655 let p = payload(&[t], now());
1656 let c = condition(node(&p, "target:mnw"), "dns");
1657 assert_eq!(c.status, Status::Degraded);
1658 assert!(
1659 c.detail
1660 .as_deref()
1661 .unwrap()
1662 .contains("lookup failed: SERVFAIL")
1663 );
1664 }
1665
1666 #[test]
1667 fn all_dns_records_matching_stays_ok_and_counts_them() {
1668 let mut t = healthy("mnw");
1669 t.dns = Some(DnsView {
1670 records: vec![DnsRecordView {
1671 name: "makenot.work".into(),
1672 record_type: "A".into(),
1673 matches: true,
1674 error: None,
1675 }],
1676 checked_at: Some(checked_at()),
1677 });
1678 let p = payload(&[t], now());
1679 let n = node(&p, "target:mnw");
1680 assert_eq!(n.status, Status::Ok);
1681 // Singular, not "1 records match".
1682 assert_eq!(
1683 condition(n, "dns").detail.as_deref(),
1684 Some("1 record match")
1685 );
1686 }
1687
1688 #[test]
1689 fn a_cors_misconfiguration_degrades_the_target() {
1690 let mut t = healthy("mnw");
1691 t.cors = Some(CorsView {
1692 checks: vec![CorsCheckView {
1693 url: "https://makenot.work/api".into(),
1694 origin: "https://app.makenot.work".into(),
1695 passes: false,
1696 error: None,
1697 }],
1698 checked_at: Some(checked_at()),
1699 });
1700 let p = payload(&[t], now());
1701 let n = node(&p, "target:mnw");
1702 assert_eq!(n.status, Status::Degraded);
1703 assert!(
1704 condition(n, "cors")
1705 .detail
1706 .as_deref()
1707 .unwrap()
1708 .contains("does not allow https://app.makenot.work")
1709 );
1710 }
1711
1712 #[test]
1713 fn passing_cors_preflights_stay_ok() {
1714 let mut t = healthy("mnw");
1715 t.cors = Some(CorsView {
1716 checks: vec![CorsCheckView {
1717 url: "https://makenot.work/api".into(),
1718 origin: "https://app.makenot.work".into(),
1719 passes: true,
1720 error: None,
1721 }],
1722 checked_at: Some(checked_at()),
1723 });
1724 let p = payload(&[t], now());
1725 assert_eq!(node(&p, "target:mnw").status, Status::Ok);
1726 }
1727
1728 #[test]
1729 fn the_new_conditions_do_not_disturb_a_target_that_has_none_of_them() {
1730 // A target without test/dns/cors config emits none of the three, exactly
1731 // as before, the additive property the shared contract exists to give.
1732 let p = payload(&[healthy("mnw")], now());
1733 let n = node(&p, "target:mnw");
1734 assert!(n.conditions.iter().all(|c| c.condition_type != "tests"));
1735 assert!(n.conditions.iter().all(|c| c.condition_type != "dns"));
1736 assert!(n.conditions.iter().all(|c| c.condition_type != "cors"));
1737 assert_eq!(p.validate(), Ok(()));
1738 }
1739
1740 #[test]
1741 fn a_target_that_does_not_watch_health_says_nothing_about_health() {
1742 // The test-only targets (the desktop apps, the SDK) have no HTTP endpoint
1743 // and never will. Emitting `health: pending` for them is a row that can
1744 // never go green, which held af/bb/go/sk pending forever and the whole
1745 // source with them.
1746 let mut t = healthy("af");
1747 t.health_configured = false;
1748 t.health = None;
1749 let p = payload(&[t], now());
1750 let n = node(&p, "target:af");
1751 assert!(
1752 n.conditions.iter().all(|c| c.condition_type != "health"),
1753 "unconfigured health must emit no condition, got {:?}",
1754 n.conditions
1755 );
1756 assert_eq!(p.validate(), Ok(()));
1757 }
1758
1759 #[test]
1760 fn a_stored_snapshot_still_reports_after_the_config_is_removed() {
1761 // Config is truth for whether to watch, but evidence already collected
1762 // should not vanish: dropping the block should not silently erase the
1763 // last thing PoM knew about that target's health.
1764 let mut t = healthy("mnw");
1765 t.health_configured = false;
1766 let p = payload(&[t], now());
1767 let n = node(&p, "target:mnw");
1768 assert!(n.conditions.iter().any(|c| c.condition_type == "health"));
1769 assert_eq!(p.validate(), Ok(()));
1770 }
1771
1772 #[test]
1773 fn a_watched_target_with_no_snapshot_yet_is_still_pending() {
1774 // The case the flag must not break: health IS configured, no check has
1775 // run, so "evidence of nothing" is the honest answer.
1776 let mut t = healthy("mnw");
1777 t.health = None;
1778 let p = payload(&[t], now());
1779 let n = node(&p, "target:mnw");
1780 let h = n
1781 .conditions
1782 .iter()
1783 .find(|c| c.condition_type == "health")
1784 .expect("configured health must still emit a condition");
1785 assert_eq!(h.status, Status::Pending);
1786 assert_eq!(p.validate(), Ok(()));
1787 }
1788
1789 #[test]
1790 fn render_is_a_pure_function_of_state_and_clock() {
1791 let a = payload(&[healthy("mnw")], now());
1792 let b = payload(&[healthy("mnw")], now());
1793 assert_eq!(
1794 serde_json::to_value(&a).unwrap(),
1795 serde_json::to_value(&b).unwrap()
1796 );
1797 }
1798
1799 #[test]
1800 fn pom_declares_no_actions() {
1801 let p = payload(&[healthy("mnw")], now());
1802 assert!(p.actions.is_empty());
1803 assert!(node(&p, "target:mnw").actions.is_empty());
1804 }
1805 }
1806