Skip to main content

max / makenotwork

42.4 KB · 1293 lines History Blame Raw
1 //! PoM's projection onto the shared operator status payload.
2 //!
3 //! Spec + rationale: maintainer wiki.
4 //! <!-- wiki: release-status-payload -->
5 //!
6 //! `GET /status.json` serves this. Sando and Bento report on the thing they
7 //! *are*; PoM reports on the things it *watches*, so every target it monitors
8 //! becomes one node and the viewer needs no knowledge of health checks, TLS, or
9 //! incidents to draw PoM.
10 //!
11 //! [`payload`] is a pure function of `(&[TargetView], now)`. Every clock is an
12 //! argument, so a fixture renders identically forever and the mapping is
13 //! testable without a database.
14 //!
15 //! # What maps to what
16 //!
17 //! | PoM | payload |
18 //! |---|---|
19 //! | monitored target | node, `kind = "target"` |
20 //! | latest health check | node status + `health` condition |
21 //! | open incident | `incident` condition carrying the why |
22 //! | TLS certificate | `tls` condition (expiry) |
23 //! | domain registration | `whois` condition (expiry) |
24 //! | uptime / latency / version | fields |
25 //!
26 //! # Every signal colors the target
27 //!
28 //! The load-bearing difference from Sando. A Sando gate guards *promotion* and
29 //! must not color a tier, burn-in blocks for 48 hours as routine. PoM has no
30 //! such notion: a cert 3 days from expiry, an open incident, a domain about to
31 //! lapse are each a real problem with the target regardless of what the health
32 //! endpoint says. So a target's status is the worst of every condition on it,
33 //! and the why is never lost because it is in the conditions either way.
34 //!
35 //! PoM declares no actions. It observes; it does not act, and there is no route
36 //! to trigger a recheck. An empty actions map is the honest shape.
37
38 use chrono::{DateTime, Utc};
39 use ops_status::{Condition, Field, Node, Payload, Status, Value};
40
41 use crate::types::HealthStatus;
42
43 /// The `source` name PoM answers to in a viewer's config.
44 pub const SOURCE: &str = "pom";
45
46 /// Days before a certificate expires at which the target goes `degraded`.
47 const TLS_EXPIRY_WARN_DAYS: i64 = 14;
48
49 /// Days before a domain registration lapses at which the target goes `degraded`.
50 const WHOIS_EXPIRY_WARN_DAYS: i64 = 30;
51
52 /// Everything the payload needs about one monitored target, already read out of
53 /// the database.
54 ///
55 /// A dedicated view rather than the API's `TargetStatus`: that type is shaped
56 /// for the existing `/api/status` JSON and carries pre-formatted strings, which
57 /// is exactly what the shared contract forbids a producer from owning. Building
58 /// this from typed rows keeps [`payload`] pure and the DB out of the mapping.
59 pub(crate) struct TargetView {
60 /// Config key, e.g. "mnw".
61 pub name: String,
62 /// Human-readable label, e.g. "MakeNotWork".
63 pub label: String,
64 /// The most recent health snapshot. `None` before the first check.
65 pub health: Option<HealthView>,
66 /// Uptime percentage over the last 24 hours, if any checks fell in the window.
67 pub uptime_24h: Option<f64>,
68 /// Mean response time over the last 24 hours of operational checks, in ms.
69 pub latency_avg_ms: Option<f64>,
70 /// Latest TLS check. `None` if TLS is not monitored for this target.
71 pub tls: Option<TlsView>,
72 /// Currently open incident, if the target is in one.
73 pub incident: Option<IncidentView>,
74 /// Latest WHOIS check. `None` if domain expiry is not monitored.
75 pub whois: Option<WhoisView>,
76 /// Latest backup freshness, one per monitored database. Empty if the target
77 /// has no backup monitoring configured.
78 pub backups: Vec<BackupView>,
79 /// Latest scan-pipeline check. `None` if not monitored for this target.
80 pub scan_pipeline: Option<ScanView>,
81 /// Latest test run and PoM's staleness verdict. `None` if the target has no
82 /// test config.
83 pub tests: Option<TestsView>,
84 /// Latest DNS record checks. `None` if DNS is not monitored for this target.
85 pub dns: Option<DnsView>,
86 /// Latest CORS preflight checks. `None` if CORS is not monitored.
87 pub cors: Option<CorsView>,
88 }
89
90 pub(crate) struct HealthView {
91 pub status: HealthStatus,
92 pub checked_at: String,
93 pub version: Option<String>,
94 /// The failure reason when the check did not pass.
95 pub error: Option<String>,
96 }
97
98 pub(crate) struct TlsView {
99 pub valid: bool,
100 pub days_remaining: i64,
101 pub checked_at: String,
102 pub error: Option<String>,
103 }
104
105 pub(crate) struct IncidentView {
106 pub from_status: String,
107 pub to_status: String,
108 pub started_at: String,
109 }
110
111 pub(crate) struct WhoisView {
112 pub days_remaining: Option<i64>,
113 pub checked_at: String,
114 pub error: Option<String>,
115 }
116
117 pub(crate) struct BackupView {
118 pub database: String,
119 /// One of "ok", "stale", "missing", "error".
120 pub status: String,
121 pub age_hours: Option<i64>,
122 pub checked_at: String,
123 pub error: Option<String>,
124 }
125
126 pub(crate) struct ScanView {
127 /// One of "operational", "degraded", "unreachable".
128 pub status: String,
129 /// Fired-threshold issue lines, the why behind a non-operational status.
130 pub issues: Vec<String>,
131 pub checked_at: String,
132 pub error: Option<String>,
133 }
134
135 pub(crate) struct TestsView {
136 /// Whether any run has ever been recorded. Separates "no evidence"
137 /// (pending) from a run that failed.
138 pub ran: bool,
139 /// Whether the most recent run passed.
140 pub passed: bool,
141 pub total_passed: Option<i64>,
142 pub total_failed: Option<i64>,
143 /// When the most recent run started, RFC 3339.
144 pub started_at: Option<String>,
145 /// PoM's staleness verdict for the run: true when it is older than the
146 /// configured threshold or a version was deployed after it ran.
147 pub stale: bool,
148 pub stale_reason: Option<String>,
149 }
150
151 pub(crate) struct DnsView {
152 /// One entry per monitored record; non-empty by construction.
153 pub records: Vec<DnsRecordView>,
154 /// Latest checked_at across the records, for the condition's `since`.
155 pub checked_at: Option<String>,
156 }
157
158 pub(crate) struct DnsRecordView {
159 pub name: String,
160 pub record_type: String,
161 pub matches: bool,
162 pub error: Option<String>,
163 }
164
165 pub(crate) struct CorsView {
166 /// One entry per monitored URL; non-empty by construction.
167 pub checks: Vec<CorsCheckView>,
168 pub checked_at: Option<String>,
169 }
170
171 pub(crate) struct CorsCheckView {
172 pub url: String,
173 pub origin: String,
174 pub passes: bool,
175 pub error: Option<String>,
176 }
177
178 /// Restate every monitored target as the shared payload.
179 ///
180 /// `now` is an argument rather than read from the clock so the mapping stays
181 /// pure and snapshot-testable.
182 pub(crate) fn payload(targets: &[TargetView], now: DateTime<Utc>) -> Payload {
183 let mut payload = Payload::new(SOURCE, now);
184 for target in targets {
185 payload.nodes.push(target_node(target));
186 }
187 payload
188 }
189
190 fn target_node(target: &TargetView) -> Node {
191 let mut conditions = vec![health_condition(target.health.as_ref())];
192 if let Some(incident) = &target.incident {
193 conditions.push(incident_condition(incident));
194 }
195 if let Some(tls) = &target.tls {
196 conditions.push(tls_condition(tls));
197 }
198 if let Some(whois) = &target.whois
199 && let Some(condition) = whois_condition(whois)
200 {
201 conditions.push(condition);
202 }
203 for backup in &target.backups {
204 conditions.push(backup_condition(backup));
205 }
206 if let Some(scan) = &target.scan_pipeline {
207 conditions.push(scan_condition(scan));
208 }
209 if let Some(tests) = &target.tests {
210 conditions.push(tests_condition(tests));
211 }
212 if let Some(dns) = &target.dns {
213 conditions.push(dns_condition(dns));
214 }
215 if let Some(cors) = &target.cors {
216 conditions.push(cors_condition(cors));
217 }
218
219 // The target's status is the worst thing said about it. There is always at
220 // least the health condition, so the max is well-defined.
221 let status = conditions
222 .iter()
223 .map(|c| c.status)
224 .max()
225 .unwrap_or(Status::Unknown);
226
227 Node {
228 id: format!("target:{}", target.name),
229 kind: "target".into(),
230 label: target.label.clone(),
231 status,
232 fields: target_fields(target),
233 conditions,
234 children: Vec::new(),
235 actions: Vec::new(),
236 }
237 }
238
239 /// A health snapshot's status, or `pending` when the target has never been
240 /// checked, PoM has evidence of nothing rather than evidence of health.
241 fn health_condition(health: Option<&HealthView>) -> Condition {
242 let Some(health) = health else {
243 return Condition {
244 condition_type: "health".into(),
245 status: Status::Pending,
246 since: None,
247 detail: Some("no health check recorded yet".into()),
248 };
249 };
250 Condition {
251 condition_type: "health".into(),
252 status: health_status(health.status),
253 since: parse_instant(&health.checked_at),
254 // The error is the why on a failing check; a passing one needs no words.
255 detail: health.error.clone(),
256 }
257 }
258
259 /// PoM's four-value health vocabulary onto the shared five.
260 ///
261 /// `unreachable` and `error` both map to `failed`: from an operator's chair a
262 /// target that 5xxs and one that will not answer are the same event, it is
263 /// down. The distinction is preserved in the health snapshot's own error text.
264 fn health_status(status: HealthStatus) -> Status {
265 match status {
266 HealthStatus::Operational => Status::Ok,
267 HealthStatus::Degraded => Status::Degraded,
268 HealthStatus::Error | HealthStatus::Unreachable => Status::Failed,
269 }
270 }
271
272 /// An open incident is reported at the severity it escalated *to*, carrying when
273 /// it started so the viewer can age it.
274 fn incident_condition(incident: &IncidentView) -> Condition {
275 Condition {
276 condition_type: "incident".into(),
277 status: health_status_from_str(&incident.to_status),
278 since: parse_instant(&incident.started_at),
279 detail: Some(format!(
280 "{} to {}",
281 incident.from_status, incident.to_status
282 )),
283 }
284 }
285
286 /// Incident rows store the health status as a string. An unrecognized value is
287 /// `unknown` rather than a parse failure, keeping the node legible against a
288 /// future status PoM learns before the viewer does.
289 fn health_status_from_str(raw: &str) -> Status {
290 raw.parse::<HealthStatus>()
291 .map_or(Status::Unknown, health_status)
292 }
293
294 /// TLS state as a condition. A failed probe or an invalid chain is `failed`; a
295 /// certificate inside the warning window is `degraded`; anything further out is
296 /// `ok` and still carries its days-remaining, since the number is the point.
297 fn tls_condition(tls: &TlsView) -> Condition {
298 let (status, detail) = if let Some(error) = &tls.error {
299 (Status::Failed, format!("tls check failed: {error}"))
300 } else if !tls.valid {
301 (Status::Failed, "certificate chain is not valid".into())
302 } else if tls.days_remaining < 0 {
303 (
304 Status::Failed,
305 format!("certificate expired {} days ago", -tls.days_remaining),
306 )
307 } else if tls.days_remaining <= TLS_EXPIRY_WARN_DAYS {
308 (
309 Status::Degraded,
310 format!("certificate expires in {} days", tls.days_remaining),
311 )
312 } else {
313 (
314 Status::Ok,
315 format!("certificate valid, {} days remaining", tls.days_remaining),
316 )
317 };
318 Condition {
319 condition_type: "tls".into(),
320 status,
321 since: parse_instant(&tls.checked_at),
322 detail: Some(detail),
323 }
324 }
325
326 /// Domain-expiry state as a condition, or `None` when there is nothing to say.
327 ///
328 /// A WHOIS lookup that failed is `degraded`, not `failed`: registrar WHOIS is
329 /// flaky and a lookup error is not evidence the domain lapsed. Absent both an
330 /// error and a days count there is genuinely no signal, so no condition is
331 /// emitted rather than a permanently `unknown` one that trains the eye to skip
332 /// the target.
333 fn whois_condition(whois: &WhoisView) -> Option<Condition> {
334 let (status, detail) = if let Some(error) = &whois.error {
335 (Status::Degraded, format!("whois lookup failed: {error}"))
336 } else {
337 match whois.days_remaining {
338 Some(days) if days < 0 => (
339 Status::Failed,
340 format!("domain registration expired {} days ago", -days),
341 ),
342 Some(days) if days <= WHOIS_EXPIRY_WARN_DAYS => (
343 Status::Degraded,
344 format!("domain registration expires in {days} days"),
345 ),
346 Some(days) => (
347 Status::Ok,
348 format!("domain registration valid, {days} days remaining"),
349 ),
350 None => return None,
351 }
352 };
353 Some(Condition {
354 condition_type: "whois".into(),
355 status,
356 since: parse_instant(&whois.checked_at),
357 detail: Some(detail),
358 })
359 }
360
361 /// Backup freshness as a condition, one per database. A stale backup or a check
362 /// error is `degraded`; a missing backup is `failed`. The 40-day-stale backup
363 /// that stayed green by every check that existed is exactly the signal this
364 /// surfaces, the `type` names the database so several read as distinct rows.
365 fn backup_condition(backup: &BackupView) -> Condition {
366 let db = &backup.database;
367 let (status, detail) = match backup.status.as_str() {
368 "ok" => (
369 Status::Ok,
370 match backup.age_hours {
371 Some(hours) => format!("{db} backup is {hours}h old"),
372 None => format!("{db} backup present"),
373 },
374 ),
375 "stale" => (
376 Status::Degraded,
377 match backup.age_hours {
378 Some(hours) => format!("{db} backup is stale, {hours}h old"),
379 None => format!("{db} backup is stale"),
380 },
381 ),
382 "missing" => (Status::Failed, format!("no backup found for {db}")),
383 "error" => (
384 Status::Degraded,
385 match &backup.error {
386 Some(error) => format!("{db} backup check failed: {error}"),
387 None => format!("{db} backup check failed"),
388 },
389 ),
390 other => (Status::Unknown, format!("{db} backup status {other:?}")),
391 };
392 Condition {
393 condition_type: format!("backup:{db}"),
394 status,
395 since: parse_instant(&backup.checked_at),
396 detail: Some(detail),
397 }
398 }
399
400 /// Scan-pipeline state as a condition. `operational` is ok, `degraded` is
401 /// degraded, `unreachable` is failed; the fired-threshold issues, or the probe
402 /// error, are the why.
403 fn scan_condition(scan: &ScanView) -> Condition {
404 let status = match scan.status.as_str() {
405 "operational" => Status::Ok,
406 "degraded" => Status::Degraded,
407 "unreachable" => Status::Failed,
408 _ => Status::Unknown,
409 };
410 let detail = if let Some(error) = &scan.error {
411 format!("unreachable: {error}")
412 } else if !scan.issues.is_empty() {
413 scan.issues.join("; ")
414 } else {
415 "pipeline operational".into()
416 };
417 Condition {
418 condition_type: "scan_pipeline".into(),
419 status,
420 since: parse_instant(&scan.checked_at),
421 detail: Some(detail),
422 }
423 }
424
425 /// Test state as a condition. A never-run target is `pending`: evidence of
426 /// nothing, like one never health-checked. A failing or stale run is `degraded`,
427 /// not `failed`: a red suite is a real regression signal, but the running
428 /// service is covered by `health`, so a test result colors the target yellow
429 /// (look at this) rather than red (it is down). Staleness reuses PoM's own
430 /// verdict, which fires on age past the configured threshold or a version
431 /// deployed since the last run.
432 fn tests_condition(tests: &TestsView) -> Condition {
433 if !tests.ran {
434 return Condition {
435 condition_type: "tests".into(),
436 status: Status::Pending,
437 since: None,
438 detail: Some("no tests have been run yet".into()),
439 };
440 }
441 let since = tests.started_at.as_deref().and_then(parse_instant);
442 let (status, detail) = if !tests.passed {
443 let detail = match (tests.total_passed, tests.total_failed) {
444 (Some(p), Some(f)) => format!("last run failed: {f} failed, {p} passed"),
445 _ => "last test run failed".into(),
446 };
447 (Status::Degraded, detail)
448 } else if tests.stale {
449 (
450 Status::Degraded,
451 tests
452 .stale_reason
453 .clone()
454 .unwrap_or_else(|| "test results are stale".into()),
455 )
456 } else {
457 let detail = match tests.total_passed {
458 Some(p) => format!("tests passing, {p} passed"),
459 None => "tests passing".into(),
460 };
461 (Status::Ok, detail)
462 };
463 Condition {
464 condition_type: "tests".into(),
465 status,
466 since,
467 detail: Some(detail),
468 }
469 }
470
471 /// DNS state as one aggregate condition across a target's monitored records. A
472 /// record whose lookup errored or whose value does not match is `degraded`: a
473 /// wrong or unverifiable record is a real misconfiguration, but a record that
474 /// points the site somewhere dead already shows as a failed `health` check, so
475 /// DNS colors the target yellow and names what is off. All records clean is
476 /// `ok`. One aggregate condition rather than one per record keeps a target that
477 /// watches several records to a single legible row.
478 fn dns_condition(dns: &DnsView) -> Condition {
479 let bad: Vec<String> = dns
480 .records
481 .iter()
482 .filter_map(|r| {
483 if let Some(error) = &r.error {
484 Some(format!(
485 "{} {} lookup failed: {error}",
486 r.name, r.record_type
487 ))
488 } else if !r.matches {
489 Some(format!("{} {} does not match", r.name, r.record_type))
490 } else {
491 None
492 }
493 })
494 .collect();
495 let since = dns.checked_at.as_deref().and_then(parse_instant);
496 let (status, detail) = if bad.is_empty() {
497 (
498 Status::Ok,
499 format!(
500 "{} record{} match",
501 dns.records.len(),
502 plural(dns.records.len())
503 ),
504 )
505 } else {
506 (Status::Degraded, bad.join("; "))
507 };
508 Condition {
509 condition_type: "dns".into(),
510 status,
511 since,
512 detail: Some(detail),
513 }
514 }
515
516 /// CORS state as one aggregate condition across a target's monitored URLs. A
517 /// preflight that failed to run, or that the server answered without allowing
518 /// the origin, is `degraded`: broken CORS breaks a browser client, but it is a
519 /// configuration fault rather than the service being down, so it colors the
520 /// target yellow. All preflights allowed is `ok`.
521 fn cors_condition(cors: &CorsView) -> Condition {
522 let bad: Vec<String> = cors
523 .checks
524 .iter()
525 .filter_map(|c| {
526 if let Some(error) = &c.error {
527 Some(format!("{} preflight failed: {error}", c.url))
528 } else if !c.passes {
529 Some(format!("{} does not allow {}", c.url, c.origin))
530 } else {
531 None
532 }
533 })
534 .collect();
535 let since = cors.checked_at.as_deref().and_then(parse_instant);
536 let (status, detail) = if bad.is_empty() {
537 (
538 Status::Ok,
539 format!(
540 "{} preflight check{} pass",
541 cors.checks.len(),
542 plural(cors.checks.len())
543 ),
544 )
545 } else {
546 (Status::Degraded, bad.join("; "))
547 };
548 Condition {
549 condition_type: "cors".into(),
550 status,
551 since,
552 detail: Some(detail),
553 }
554 }
555
556 /// The plural suffix for a count: "" for one, "s" otherwise.
557 fn plural(n: usize) -> &'static str {
558 if n == 1 { "" } else { "s" }
559 }
560
561 /// Uptime as a bar, latency as a magnitude, version as a comparable, each a
562 /// number that *means* something rather than a pre-rendered string.
563 fn target_fields(target: &TargetView) -> Vec<Field> {
564 let mut fields = Vec::new();
565
566 if let Some(version) = target.health.as_ref().and_then(|h| h.version.clone()) {
567 fields.push(Field::new("version", Value::Version { value: version }));
568 }
569 if let Some(uptime) = target.uptime_24h {
570 fields.push(Field::new(
571 "uptime 24h",
572 Value::Progress {
573 value: uptime,
574 max: 100.0,
575 unit: Some("%".into()),
576 },
577 ));
578 }
579 if let Some(latency) = target.latency_avg_ms {
580 fields.push(Field::new(
581 "latency 24h",
582 Value::Quantity {
583 value: latency,
584 unit: Some("ms".into()),
585 },
586 ));
587 }
588 if let Some(checked) = target
589 .health
590 .as_ref()
591 .and_then(|h| parse_instant(&h.checked_at))
592 {
593 fields.push(Field::new("checked", Value::Instant { value: checked }));
594 }
595
596 fields
597 }
598
599 /// A timestamp that fails to parse costs only itself. A viewer that blanks on
600 /// one malformed row is worse than one missing a tooltip.
601 fn parse_instant(raw: &str) -> Option<DateTime<Utc>> {
602 DateTime::parse_from_rfc3339(raw)
603 .ok()
604 .map(|d| d.with_timezone(&Utc))
605 }
606
607 #[cfg(test)]
608 mod tests {
609 use super::*;
610
611 fn now() -> DateTime<Utc> {
612 "2026-07-21T18:24:39Z".parse().unwrap()
613 }
614
615 fn checked_at() -> String {
616 "2026-07-21T18:24:00Z".into()
617 }
618
619 fn healthy(name: &str) -> TargetView {
620 TargetView {
621 name: name.into(),
622 label: name.to_uppercase(),
623 health: Some(HealthView {
624 status: HealthStatus::Operational,
625 checked_at: checked_at(),
626 version: Some("1.4.0".into()),
627 error: None,
628 }),
629 uptime_24h: Some(100.0),
630 latency_avg_ms: Some(42.0),
631 tls: None,
632 incident: None,
633 whois: None,
634 backups: Vec::new(),
635 scan_pipeline: None,
636 tests: None,
637 dns: None,
638 cors: None,
639 }
640 }
641
642 fn node<'a>(p: &'a Payload, id: &str) -> &'a Node {
643 p.node(id).unwrap_or_else(|| panic!("no node {id}"))
644 }
645
646 #[test]
647 fn a_healthy_target_is_ok_and_structurally_sound() {
648 let p = payload(&[healthy("mnw")], now());
649
650 assert_eq!(p.source, SOURCE);
651 assert_eq!(p.schema, ops_status::SCHEMA_VERSION);
652 assert_eq!(p.validate(), Ok(()));
653 assert_eq!(node(&p, "target:mnw").status, Status::Ok);
654 assert_eq!(node(&p, "target:mnw").label, "MNW");
655 assert_eq!(p.worst_status(), Status::Ok);
656 }
657
658 #[test]
659 fn uptime_and_latency_are_typed_values_not_strings() {
660 let p = payload(&[healthy("mnw")], now());
661 let n = node(&p, "target:mnw");
662
663 let uptime = n.fields.iter().find(|f| f.label == "uptime 24h").unwrap();
664 assert_eq!(
665 uptime.value,
666 Value::Progress {
667 value: 100.0,
668 max: 100.0,
669 unit: Some("%".into())
670 }
671 );
672 let latency = n.fields.iter().find(|f| f.label == "latency 24h").unwrap();
673 assert_eq!(
674 latency.value,
675 Value::Quantity {
676 value: 42.0,
677 unit: Some("ms".into())
678 }
679 );
680 }
681
682 #[test]
683 fn an_unreachable_target_is_failed_and_says_why() {
684 let mut t = healthy("mnw");
685 t.health = Some(HealthView {
686 status: HealthStatus::Unreachable,
687 checked_at: checked_at(),
688 version: None,
689 error: Some("connection timed out".into()),
690 });
691 let p = payload(&[t], now());
692
693 let n = node(&p, "target:mnw");
694 assert_eq!(n.status, Status::Failed);
695 assert_eq!(n.conditions[0].condition_type, "health");
696 assert_eq!(
697 n.conditions[0].detail.as_deref(),
698 Some("connection timed out")
699 );
700 assert_eq!(p.worst_status(), Status::Failed);
701 }
702
703 #[test]
704 fn a_target_never_checked_is_pending_not_healthy() {
705 let mut t = healthy("new");
706 t.health = None;
707 let p = payload(&[t], now());
708
709 let n = node(&p, "target:new");
710 assert_eq!(n.status, Status::Pending);
711 assert_eq!(
712 n.conditions[0].detail.as_deref(),
713 Some("no health check recorded yet")
714 );
715 // No health snapshot means no version and no checked-at field.
716 assert!(n.fields.iter().all(|f| f.label != "version"));
717 assert!(n.fields.iter().all(|f| f.label != "checked"));
718 assert_eq!(p.validate(), Ok(()));
719 }
720
721 #[test]
722 fn an_expiring_certificate_degrades_an_otherwise_healthy_target() {
723 // The Sando contrast: unlike a promotion gate, an expiring cert is a real
724 // problem with the target and must color it even when health is green.
725 let mut t = healthy("mnw");
726 t.tls = Some(TlsView {
727 valid: true,
728 days_remaining: 9,
729 checked_at: checked_at(),
730 error: None,
731 });
732 let p = payload(&[t], now());
733
734 let n = node(&p, "target:mnw");
735 assert_eq!(n.status, Status::Degraded);
736 let tls = n
737 .conditions
738 .iter()
739 .find(|c| c.condition_type == "tls")
740 .unwrap();
741 assert_eq!(tls.status, Status::Degraded);
742 assert!(tls.detail.as_deref().unwrap().contains("9 days"));
743 }
744
745 #[test]
746 fn an_expired_certificate_fails_the_target() {
747 let mut t = healthy("mnw");
748 t.tls = Some(TlsView {
749 valid: true,
750 days_remaining: -3,
751 checked_at: checked_at(),
752 error: None,
753 });
754 let p = payload(&[t], now());
755
756 let n = node(&p, "target:mnw");
757 assert_eq!(n.status, Status::Failed);
758 let tls = n
759 .conditions
760 .iter()
761 .find(|c| c.condition_type == "tls")
762 .unwrap();
763 assert!(
764 tls.detail
765 .as_deref()
766 .unwrap()
767 .contains("expired 3 days ago")
768 );
769 }
770
771 #[test]
772 fn a_healthy_certificate_stays_ok_but_still_reports_its_runway() {
773 let mut t = healthy("mnw");
774 t.tls = Some(TlsView {
775 valid: true,
776 days_remaining: 60,
777 checked_at: checked_at(),
778 error: None,
779 });
780 let p = payload(&[t], now());
781
782 let n = node(&p, "target:mnw");
783 assert_eq!(n.status, Status::Ok);
784 let tls = n
785 .conditions
786 .iter()
787 .find(|c| c.condition_type == "tls")
788 .unwrap();
789 assert_eq!(tls.status, Status::Ok);
790 assert!(tls.detail.as_deref().unwrap().contains("60 days remaining"));
791 }
792
793 #[test]
794 fn an_open_incident_surfaces_with_its_transition_and_start() {
795 let mut t = healthy("mnw");
796 t.incident = Some(IncidentView {
797 from_status: "operational".into(),
798 to_status: "unreachable".into(),
799 started_at: "2026-07-21T17:00:00Z".into(),
800 });
801 // Health has recovered on paper but the incident is still open: the node
802 // must not read green while an incident stands.
803 let p = payload(&[t], now());
804
805 let n = node(&p, "target:mnw");
806 assert_eq!(n.status, Status::Failed);
807 let incident = n
808 .conditions
809 .iter()
810 .find(|c| c.condition_type == "incident")
811 .unwrap();
812 assert_eq!(incident.status, Status::Failed);
813 assert_eq!(
814 incident.detail.as_deref(),
815 Some("operational to unreachable")
816 );
817 assert_eq!(
818 incident.since,
819 Some("2026-07-21T17:00:00Z".parse::<DateTime<Utc>>().unwrap())
820 );
821 }
822
823 #[test]
824 fn a_failed_whois_lookup_is_degraded_not_failed() {
825 // Registrar WHOIS is flaky; a lookup error is not proof the domain lapsed.
826 let mut t = healthy("mnw");
827 t.whois = Some(WhoisView {
828 days_remaining: None,
829 checked_at: checked_at(),
830 error: Some("connection reset".into()),
831 });
832 let p = payload(&[t], now());
833
834 let n = node(&p, "target:mnw");
835 assert_eq!(n.status, Status::Degraded);
836 }
837
838 #[test]
839 fn an_expiring_domain_degrades_the_target() {
840 let mut t = healthy("mnw");
841 t.whois = Some(WhoisView {
842 days_remaining: Some(12),
843 checked_at: checked_at(),
844 error: None,
845 });
846 let p = payload(&[t], now());
847
848 let n = node(&p, "target:mnw");
849 assert_eq!(n.status, Status::Degraded);
850 let whois = n
851 .conditions
852 .iter()
853 .find(|c| c.condition_type == "whois")
854 .unwrap();
855 assert!(whois.detail.as_deref().unwrap().contains("12 days"));
856 }
857
858 #[test]
859 fn a_whois_check_with_no_signal_emits_no_condition() {
860 let mut t = healthy("mnw");
861 t.whois = Some(WhoisView {
862 days_remaining: None,
863 checked_at: checked_at(),
864 error: None,
865 });
866 let p = payload(&[t], now());
867
868 let n = node(&p, "target:mnw");
869 assert!(n.conditions.iter().all(|c| c.condition_type != "whois"));
870 assert_eq!(n.status, Status::Ok);
871 }
872
873 #[test]
874 fn the_loudest_of_several_problems_wins_the_target() {
875 let mut t = healthy("mnw");
876 t.health = Some(HealthView {
877 status: HealthStatus::Degraded,
878 checked_at: checked_at(),
879 version: Some("1.4.0".into()),
880 error: Some("unexpected status 503".into()),
881 });
882 t.tls = Some(TlsView {
883 valid: true,
884 days_remaining: -1,
885 checked_at: checked_at(),
886 error: None,
887 });
888 let p = payload(&[t], now());
889
890 // health is degraded, tls is failed: the target is failed.
891 assert_eq!(node(&p, "target:mnw").status, Status::Failed);
892 }
893
894 #[test]
895 fn one_targets_failure_does_not_touch_another() {
896 let mut down = healthy("mt");
897 down.health = Some(HealthView {
898 status: HealthStatus::Error,
899 checked_at: checked_at(),
900 version: None,
901 error: Some("500 Internal Server Error".into()),
902 });
903 let p = payload(&[healthy("mnw"), down], now());
904
905 assert_eq!(node(&p, "target:mnw").status, Status::Ok);
906 assert_eq!(node(&p, "target:mt").status, Status::Failed);
907 assert_eq!(p.worst_status(), Status::Failed);
908 assert_eq!(p.validate(), Ok(()));
909 }
910
911 #[test]
912 fn an_unknown_incident_status_stays_legible() {
913 let mut t = healthy("mnw");
914 t.incident = Some(IncidentView {
915 from_status: "operational".into(),
916 to_status: "sideways".into(),
917 started_at: checked_at(),
918 });
919 let p = payload(&[t], now());
920
921 let incident = node(&p, "target:mnw")
922 .conditions
923 .iter()
924 .find(|c| c.condition_type == "incident")
925 .unwrap();
926 assert_eq!(incident.status, Status::Unknown);
927 }
928
929 #[test]
930 fn a_malformed_timestamp_costs_only_that_timestamp() {
931 let mut t = healthy("mnw");
932 t.health = Some(HealthView {
933 status: HealthStatus::Operational,
934 checked_at: "not a timestamp".into(),
935 version: Some("1.4.0".into()),
936 error: None,
937 });
938 let p = payload(&[t], now());
939
940 let n = node(&p, "target:mnw");
941 assert_eq!(n.status, Status::Ok);
942 assert_eq!(n.conditions[0].since, None);
943 assert!(n.fields.iter().all(|f| f.label != "checked"));
944 }
945
946 #[test]
947 fn a_stale_backup_degrades_the_target_and_names_the_database() {
948 // The 40-day-stale backup that stayed green by every check that existed.
949 let mut t = healthy("mnw");
950 t.backups = vec![BackupView {
951 database: "makenotwork".into(),
952 status: "stale".into(),
953 age_hours: Some(960),
954 checked_at: checked_at(),
955 error: None,
956 }];
957 let p = payload(&[t], now());
958
959 let n = node(&p, "target:mnw");
960 assert_eq!(n.status, Status::Degraded);
961 let backup = n
962 .conditions
963 .iter()
964 .find(|c| c.condition_type == "backup:makenotwork")
965 .unwrap();
966 assert_eq!(backup.status, Status::Degraded);
967 assert!(backup.detail.as_deref().unwrap().contains("960h"));
968 }
969
970 #[test]
971 fn a_missing_backup_fails_the_target() {
972 let mut t = healthy("mnw");
973 t.backups = vec![BackupView {
974 database: "makenotwork".into(),
975 status: "missing".into(),
976 age_hours: None,
977 checked_at: checked_at(),
978 error: None,
979 }];
980 let p = payload(&[t], now());
981 assert_eq!(node(&p, "target:mnw").status, Status::Failed);
982 }
983
984 #[test]
985 fn several_databases_read_as_distinct_conditions() {
986 let mut t = healthy("mnw");
987 t.backups = vec![
988 BackupView {
989 database: "makenotwork".into(),
990 status: "ok".into(),
991 age_hours: Some(6),
992 checked_at: checked_at(),
993 error: None,
994 },
995 BackupView {
996 database: "multithreaded".into(),
997 status: "ok".into(),
998 age_hours: Some(7),
999 checked_at: checked_at(),
1000 error: None,
1001 },
1002 ];
1003 let p = payload(&[t], now());
1004
1005 let n = node(&p, "target:mnw");
1006 assert_eq!(n.status, Status::Ok);
1007 assert!(
1008 n.conditions
1009 .iter()
1010 .any(|c| c.condition_type == "backup:makenotwork")
1011 );
1012 assert!(
1013 n.conditions
1014 .iter()
1015 .any(|c| c.condition_type == "backup:multithreaded")
1016 );
1017 }
1018
1019 #[test]
1020 fn a_degraded_scan_pipeline_carries_its_issues() {
1021 let mut t = healthy("mnw");
1022 t.scan_pipeline = Some(ScanView {
1023 status: "degraded".into(),
1024 issues: vec!["thumbnail error rate 22%".into(), "queue stuck: 4".into()],
1025 checked_at: checked_at(),
1026 error: None,
1027 });
1028 let p = payload(&[t], now());
1029
1030 let n = node(&p, "target:mnw");
1031 assert_eq!(n.status, Status::Degraded);
1032 let scan = n
1033 .conditions
1034 .iter()
1035 .find(|c| c.condition_type == "scan_pipeline")
1036 .unwrap();
1037 assert!(
1038 scan.detail
1039 .as_deref()
1040 .unwrap()
1041 .contains("thumbnail error rate 22%")
1042 );
1043 assert!(scan.detail.as_deref().unwrap().contains("queue stuck: 4"));
1044 }
1045
1046 #[test]
1047 fn an_unreachable_scan_pipeline_fails_the_target() {
1048 let mut t = healthy("mnw");
1049 t.scan_pipeline = Some(ScanView {
1050 status: "unreachable".into(),
1051 issues: Vec::new(),
1052 checked_at: checked_at(),
1053 error: Some("502 Bad Gateway".into()),
1054 });
1055 let p = payload(&[t], now());
1056
1057 let n = node(&p, "target:mnw");
1058 assert_eq!(n.status, Status::Failed);
1059 let scan = n
1060 .conditions
1061 .iter()
1062 .find(|c| c.condition_type == "scan_pipeline")
1063 .unwrap();
1064 assert!(scan.detail.as_deref().unwrap().contains("502 Bad Gateway"));
1065 }
1066
1067 fn tests_view(passed: bool, stale: bool) -> TestsView {
1068 TestsView {
1069 ran: true,
1070 passed,
1071 total_passed: Some(226),
1072 total_failed: if passed { Some(0) } else { Some(3) },
1073 started_at: Some(checked_at()),
1074 stale,
1075 stale_reason: if stale {
1076 Some("tests are 12 days old (threshold: 7d)".into())
1077 } else {
1078 None
1079 },
1080 }
1081 }
1082
1083 fn condition<'a>(n: &'a Node, ty: &str) -> &'a Condition {
1084 n.conditions
1085 .iter()
1086 .find(|c| c.condition_type == ty)
1087 .unwrap_or_else(|| panic!("no {ty} condition"))
1088 }
1089
1090 #[test]
1091 fn a_never_run_test_target_is_pending_not_healthy() {
1092 let mut t = healthy("mnw");
1093 t.tests = Some(TestsView {
1094 ran: false,
1095 passed: false,
1096 total_passed: None,
1097 total_failed: None,
1098 started_at: None,
1099 stale: true,
1100 stale_reason: Some("no tests have been run".into()),
1101 });
1102 let p = payload(&[t], now());
1103 let n = node(&p, "target:mnw");
1104 // Pending is quieter than degraded: no evidence is not a failure.
1105 assert_eq!(n.status, Status::Pending);
1106 assert_eq!(
1107 condition(n, "tests").detail.as_deref(),
1108 Some("no tests have been run yet")
1109 );
1110 }
1111
1112 #[test]
1113 fn a_failing_test_run_degrades_but_does_not_fail_the_target() {
1114 // A red suite is a regression signal; the running service is health's job.
1115 let mut t = healthy("mnw");
1116 t.tests = Some(tests_view(false, false));
1117 let p = payload(&[t], now());
1118 let n = node(&p, "target:mnw");
1119 assert_eq!(n.status, Status::Degraded);
1120 let c = condition(n, "tests");
1121 assert_eq!(c.status, Status::Degraded);
1122 assert!(c.detail.as_deref().unwrap().contains("3 failed"));
1123 }
1124
1125 #[test]
1126 fn a_stale_but_passing_test_run_degrades_the_target() {
1127 let mut t = healthy("mnw");
1128 t.tests = Some(tests_view(true, true));
1129 let p = payload(&[t], now());
1130 let n = node(&p, "target:mnw");
1131 assert_eq!(n.status, Status::Degraded);
1132 assert!(
1133 condition(n, "tests")
1134 .detail
1135 .as_deref()
1136 .unwrap()
1137 .contains("12 days old")
1138 );
1139 }
1140
1141 #[test]
1142 fn a_fresh_passing_test_run_leaves_the_target_ok() {
1143 let mut t = healthy("mnw");
1144 t.tests = Some(tests_view(true, false));
1145 let p = payload(&[t], now());
1146 let n = node(&p, "target:mnw");
1147 assert_eq!(n.status, Status::Ok);
1148 assert_eq!(condition(n, "tests").status, Status::Ok);
1149 }
1150
1151 #[test]
1152 fn a_dns_mismatch_degrades_the_target_and_names_the_record() {
1153 let mut t = healthy("mnw");
1154 t.dns = Some(DnsView {
1155 records: vec![
1156 DnsRecordView {
1157 name: "makenot.work".into(),
1158 record_type: "A".into(),
1159 matches: true,
1160 error: None,
1161 },
1162 DnsRecordView {
1163 name: "makenot.work".into(),
1164 record_type: "MX".into(),
1165 matches: false,
1166 error: None,
1167 },
1168 ],
1169 checked_at: Some(checked_at()),
1170 });
1171 let p = payload(&[t], now());
1172 let n = node(&p, "target:mnw");
1173 assert_eq!(n.status, Status::Degraded);
1174 let c = condition(n, "dns");
1175 assert_eq!(c.status, Status::Degraded);
1176 assert!(c.detail.as_deref().unwrap().contains("MX does not match"));
1177 }
1178
1179 #[test]
1180 fn a_dns_lookup_error_reads_as_a_flaky_degrade_not_a_mismatch() {
1181 let mut t = healthy("mnw");
1182 t.dns = Some(DnsView {
1183 records: vec![DnsRecordView {
1184 name: "makenot.work".into(),
1185 record_type: "TXT".into(),
1186 matches: false,
1187 error: Some("SERVFAIL".into()),
1188 }],
1189 checked_at: Some(checked_at()),
1190 });
1191 let p = payload(&[t], now());
1192 let c = condition(node(&p, "target:mnw"), "dns");
1193 assert_eq!(c.status, Status::Degraded);
1194 assert!(
1195 c.detail
1196 .as_deref()
1197 .unwrap()
1198 .contains("lookup failed: SERVFAIL")
1199 );
1200 }
1201
1202 #[test]
1203 fn all_dns_records_matching_stays_ok_and_counts_them() {
1204 let mut t = healthy("mnw");
1205 t.dns = Some(DnsView {
1206 records: vec![DnsRecordView {
1207 name: "makenot.work".into(),
1208 record_type: "A".into(),
1209 matches: true,
1210 error: None,
1211 }],
1212 checked_at: Some(checked_at()),
1213 });
1214 let p = payload(&[t], now());
1215 let n = node(&p, "target:mnw");
1216 assert_eq!(n.status, Status::Ok);
1217 // Singular, not "1 records match".
1218 assert_eq!(
1219 condition(n, "dns").detail.as_deref(),
1220 Some("1 record match")
1221 );
1222 }
1223
1224 #[test]
1225 fn a_cors_misconfiguration_degrades_the_target() {
1226 let mut t = healthy("mnw");
1227 t.cors = Some(CorsView {
1228 checks: vec![CorsCheckView {
1229 url: "https://makenot.work/api".into(),
1230 origin: "https://app.makenot.work".into(),
1231 passes: false,
1232 error: None,
1233 }],
1234 checked_at: Some(checked_at()),
1235 });
1236 let p = payload(&[t], now());
1237 let n = node(&p, "target:mnw");
1238 assert_eq!(n.status, Status::Degraded);
1239 assert!(
1240 condition(n, "cors")
1241 .detail
1242 .as_deref()
1243 .unwrap()
1244 .contains("does not allow https://app.makenot.work")
1245 );
1246 }
1247
1248 #[test]
1249 fn passing_cors_preflights_stay_ok() {
1250 let mut t = healthy("mnw");
1251 t.cors = Some(CorsView {
1252 checks: vec![CorsCheckView {
1253 url: "https://makenot.work/api".into(),
1254 origin: "https://app.makenot.work".into(),
1255 passes: true,
1256 error: None,
1257 }],
1258 checked_at: Some(checked_at()),
1259 });
1260 let p = payload(&[t], now());
1261 assert_eq!(node(&p, "target:mnw").status, Status::Ok);
1262 }
1263
1264 #[test]
1265 fn the_new_conditions_do_not_disturb_a_target_that_has_none_of_them() {
1266 // A target without test/dns/cors config emits none of the three, exactly
1267 // as before, the additive property the shared contract exists to give.
1268 let p = payload(&[healthy("mnw")], now());
1269 let n = node(&p, "target:mnw");
1270 assert!(n.conditions.iter().all(|c| c.condition_type != "tests"));
1271 assert!(n.conditions.iter().all(|c| c.condition_type != "dns"));
1272 assert!(n.conditions.iter().all(|c| c.condition_type != "cors"));
1273 assert_eq!(p.validate(), Ok(()));
1274 }
1275
1276 #[test]
1277 fn render_is_a_pure_function_of_state_and_clock() {
1278 let a = payload(&[healthy("mnw")], now());
1279 let b = payload(&[healthy("mnw")], now());
1280 assert_eq!(
1281 serde_json::to_value(&a).unwrap(),
1282 serde_json::to_value(&b).unwrap()
1283 );
1284 }
1285
1286 #[test]
1287 fn pom_declares_no_actions() {
1288 let p = payload(&[healthy("mnw")], now());
1289 assert!(p.actions.is_empty());
1290 assert!(node(&p, "target:mnw").actions.is_empty());
1291 }
1292 }
1293