Skip to main content

max / makenotwork

30.0 KB · 910 lines History Blame Raw
1 //! Sando's projection onto the shared operator status payload.
2 //!
3 //! Spec + rationale: maintainer wiki.
4 //! <!-- wiki: release-status-payload -->
5 //!
6 //! `GET /status.json` serves this. It is the same read as `/state`, restated in
7 //! the vocabulary every operator surface renders, so a viewer needs no
8 //! knowledge of tiers, gates, or promotion to draw Sando.
9 //!
10 //! [`payload`] is a pure function of `(StateView, now)`. Every clock is an
11 //! argument, so a fixture state renders identically forever and the mapping is
12 //! testable without a database.
13 //!
14 //! # What maps to what
15 //!
16 //! | Sando | payload |
17 //! |---|---|
18 //! | tier | node, `kind = "tier"` |
19 //! | deploy target | node, `kind = "node"`, child of its tier |
20 //! | latest build | node, `kind = "build"` |
21 //! | gate | condition on its tier |
22 //! | promote / rollback / confirm | declared action |
23 //!
24 //! # Gates are promotion guards, not tier health
25 //!
26 //! The load-bearing judgement here. A gate blocks promotion *out of* a tier; it
27 //! does not describe whether that tier is serving. `burn_in` sits blocked for
28 //! 48 hours as a matter of routine, and a tier whose status went yellow for two
29 //! days every release is a tier nobody looks at.
30 //!
31 //! So a blocked or in-flight gate is reported as a `pending` condition and
32 //! leaves the tier `ok`. Only evidence of something actually wrong — a failed
33 //! gate, or a partial (mixed-version) tier — degrades the tier itself. The why
34 //! is never lost: it is in the conditions either way.
35
36 use chrono::{DateTime, Utc};
37 use ops_status::{Action, Condition, Field, Method, Node, Payload, Status, Value};
38
39 use crate::outcome::GateOutcome;
40 use crate::routes::{GateView, StateView, TierView};
41
42 /// The `source` name Sando answers to in a viewer's config.
43 pub const SOURCE: &str = "sando";
44
45 /// Restate `/state` as the shared payload.
46 ///
47 /// `now` is an argument rather than read from the clock so the mapping stays
48 /// pure and snapshot-testable.
49 pub(crate) fn payload(view: &StateView, now: DateTime<Utc>) -> Payload {
50 let mut payload = Payload::new(SOURCE, now);
51
52 payload.actions.insert(
53 "rebuild".into(),
54 Action {
55 label: "Rebuild".into(),
56 method: Method::Post,
57 url: "/rebuild".into(),
58 confirm: false,
59 danger: false,
60 body: None,
61 },
62 );
63
64 // Always emitted, even before the first `/rebuild`. Actions reach a viewer
65 // only by hanging off a node, so a daemon with no build history and no
66 // build node would offer no way to start one.
67 payload.nodes.push(build_node(view.build.as_ref()));
68
69 for tier in &view.tiers {
70 for action in tier_actions(tier) {
71 payload.actions.insert(action.0, action.1);
72 }
73 payload.nodes.push(tier_node(tier));
74 payload.nodes.extend(node_nodes(tier));
75 }
76
77 payload
78 }
79
80 // ---------------------------------------------------------------------------
81 // Build
82 // ---------------------------------------------------------------------------
83
84 fn build_node(build: Option<&crate::runs::BuildSummary>) -> Node {
85 let Some(build) = build else {
86 return Node {
87 id: "build".into(),
88 kind: "build".into(),
89 label: "latest build".into(),
90 status: Status::Pending,
91 fields: Vec::new(),
92 conditions: vec![Condition {
93 condition_type: "build".into(),
94 status: Status::Pending,
95 since: None,
96 detail: Some("no build run yet".into()),
97 }],
98 children: Vec::new(),
99 actions: vec!["rebuild".into()],
100 };
101 };
102
103 // `build_runs.result` is one of building / passed / failed / aborted. An
104 // aborted run is degraded rather than failed: the daemon settling an
105 // orphaned run on restart is not the same event as a build that broke.
106 let status = match build.result.as_str() {
107 "passed" => Status::Ok,
108 "failed" => Status::Failed,
109 "building" => Status::Pending,
110 "aborted" => Status::Degraded,
111 _ => Status::Unknown,
112 };
113
114 let mut fields = vec![
115 Field::new(
116 "sha",
117 Value::Ident {
118 value: build.sha.clone(),
119 abbrev_to: Some(8),
120 },
121 ),
122 Field::new(
123 "phase",
124 Value::Text {
125 value: build.phase.clone(),
126 },
127 ),
128 Field::new(
129 "elapsed",
130 Value::Duration {
131 seconds: build.elapsed_s,
132 },
133 ),
134 ];
135 if let Some(version) = &build.version {
136 fields.insert(
137 0,
138 Field::new(
139 "version",
140 Value::Version {
141 value: version.clone(),
142 },
143 ),
144 );
145 }
146
147 let mut conditions = vec![Condition {
148 condition_type: "build".into(),
149 status,
150 since: None,
151 detail: Some(format!("run {} {}", build.run_id, build.result)),
152 }];
153 if let Some(failure) = &build.failure_summary {
154 conditions.push(Condition {
155 condition_type: "build_failure".into(),
156 status: Status::Failed,
157 since: None,
158 detail: Some(failure.clone()),
159 });
160 }
161
162 Node {
163 id: "build".into(),
164 kind: "build".into(),
165 label: "latest build".into(),
166 status,
167 fields,
168 conditions,
169 children: Vec::new(),
170 actions: vec!["rebuild".into()],
171 }
172 }
173
174 // ---------------------------------------------------------------------------
175 // Tiers
176 // ---------------------------------------------------------------------------
177
178 fn tier_id(tier: &TierView) -> String {
179 format!("tier:{}", tier.name)
180 }
181
182 fn tier_node(tier: &TierView) -> Node {
183 let mut conditions: Vec<Condition> = tier.gates.iter().map(gate_condition).collect();
184
185 // A partial tier is mid-version and could not be compensated back to
186 // consistency. It is the loudest thing Sando can report.
187 if let Some(reason) = &tier.partial_reason {
188 conditions.insert(
189 0,
190 Condition {
191 condition_type: "consistent".into(),
192 status: Status::Failed,
193 since: None,
194 detail: Some(reason.clone()),
195 },
196 );
197 }
198 // An artifact the tier still names is gone from the store. The tier is
199 // serving, so this is not a `consistent` failure — what is lost is the
200 // ability to redeploy or roll back, which nothing else on this node would
201 // say. Reported after `consistent` and before the routine conditions.
202 if let Some(detail) = &tier.missing_artifact {
203 conditions.push(Condition {
204 condition_type: "artifact".into(),
205 status: Status::Degraded,
206 since: None,
207 detail: Some(format!("referenced artifact missing: {detail}")),
208 });
209 }
210 if !tier.provisioned {
211 conditions.push(Condition {
212 condition_type: "provisioned".into(),
213 status: Status::Pending,
214 since: None,
215 detail: Some("declared in topology, not provisioned".into()),
216 });
217 }
218
219 let mut fields = vec![Field::new(
220 "canary",
221 Value::Text {
222 value: tier.canary.clone(),
223 },
224 )];
225 if let Some(version) = &tier.current_version {
226 fields.insert(
227 0,
228 Field::new(
229 "version",
230 Value::Version {
231 value: version.clone(),
232 },
233 ),
234 );
235 }
236 if let Some(previous) = &tier.previous_version {
237 fields.push(Field::new(
238 "previous",
239 Value::Version {
240 value: previous.clone(),
241 },
242 ));
243 }
244 if let Some(started) = tier.burn_in_started_at.as_deref().and_then(parse_instant) {
245 fields.push(Field::new(
246 "burn-in since",
247 Value::Instant { value: started },
248 ));
249 }
250 if let Some(progress) = burn_in_progress(tier) {
251 fields.push(progress);
252 }
253
254 Node {
255 id: tier_id(tier),
256 kind: "tier".into(),
257 label: tier.name.clone(),
258 status: tier_status(tier),
259 fields,
260 conditions,
261 children: tier.nodes.iter().map(|n| format!("node:{n}")).collect(),
262 actions: tier_actions(tier).into_iter().map(|(key, _)| key).collect(),
263 }
264 }
265
266 /// Whether the tier itself is healthy — deliberately not "are its gates green".
267 ///
268 /// See the module docs: gates guard promotion out, so a blocked one is routine
269 /// and must not color the tier.
270 fn tier_status(tier: &TierView) -> Status {
271 if tier.partial_reason.is_some() {
272 return Status::Failed;
273 }
274 if tier
275 .gates
276 .iter()
277 .any(|g| g.status.as_deref() == Some("failed"))
278 {
279 return Status::Failed;
280 }
281 // Degraded rather than failed: the tier is serving the version it says it
282 // is. What is broken is the next promote and every rollback, which is worse
283 // than routine and less than an outage. Reporting it green is what let two
284 // occurrences go unnoticed until an operator reached for a rollback.
285 if tier.missing_artifact.is_some() {
286 return Status::Degraded;
287 }
288 // An unrecognized gate status is contract drift between daemon and viewer,
289 // which is worth surfacing rather than smoothing over. The TUI renders it
290 // magenta for the same reason.
291 if tier.gates.iter().any(|g| gate_status(g) == Status::Unknown) {
292 return Status::Degraded;
293 }
294 if !tier.provisioned || tier.current_version.is_none() {
295 return Status::Pending;
296 }
297 Status::Ok
298 }
299
300 /// Burn-in as a bar rather than a sentence, so it sorts and colors like every
301 /// other progress value in the UI.
302 fn burn_in_progress(tier: &TierView) -> Option<Field> {
303 let gate = tier.gates.iter().find(|g| g.kind == "burn_in")?;
304 let outcome = gate.outcome.clone()?;
305 let outcome: GateOutcome = serde_json::from_value(outcome).ok()?;
306
307 let (value, max) = match outcome.status {
308 crate::outcome::GateStatus::Blocked {
309 blocker:
310 crate::outcome::GateBlocker::BurnInRemaining {
311 hours_remaining,
312 hours_total,
313 },
314 } => (
315 f64::from(hours_total.saturating_sub(hours_remaining)),
316 f64::from(hours_total),
317 ),
318 crate::outcome::GateStatus::Passed {
319 note: crate::outcome::PassNote::BurnInElapsed { hours },
320 } => (f64::from(hours), f64::from(hours)),
321 _ => return None,
322 };
323
324 Some(Field::new(
325 "burn-in",
326 Value::Progress {
327 value,
328 max,
329 unit: Some("hour".into()),
330 },
331 ))
332 }
333
334 /// Promote, roll back, confirm — declared as data so the viewer issues them
335 /// without knowing what any of them mean.
336 ///
337 /// All three carry `confirm` and `danger`. Sando has no flag marking which
338 /// tiers are production, and inventing one from the tier name would be a
339 /// heuristic that fails silently the day the topology changes. Over-confirming
340 /// a deploy is the cheap direction to be wrong in.
341 fn tier_actions(tier: &TierView) -> Vec<(String, Action)> {
342 if !tier.provisioned {
343 return Vec::new();
344 }
345 let name = &tier.name;
346 vec![
347 (
348 format!("promote-{name}"),
349 Action {
350 label: format!("Promote to {name}"),
351 method: Method::Post,
352 url: format!("/promote/{name}"),
353 confirm: true,
354 danger: true,
355 body: None,
356 },
357 ),
358 (
359 format!("rollback-{name}"),
360 Action {
361 label: format!("Roll back {name}"),
362 method: Method::Post,
363 url: format!("/rollback/{name}"),
364 confirm: true,
365 danger: true,
366 body: None,
367 },
368 ),
369 (
370 format!("confirm-{name}"),
371 Action {
372 label: format!("Confirm {name}"),
373 method: Method::Post,
374 url: format!("/confirm/{name}"),
375 confirm: true,
376 danger: true,
377 body: None,
378 },
379 ),
380 ]
381 }
382
383 // ---------------------------------------------------------------------------
384 // Gates
385 // ---------------------------------------------------------------------------
386
387 /// `passed | failed | blocked`, or NULL while the gate is in flight.
388 ///
389 /// `blocked` becomes `pending`: a blocked gate is waiting on something, which
390 /// is what pending means here. An unrecognized string is `unknown`, not
391 /// in-flight — that distinction is why the TUI renders it magenta.
392 fn gate_status(gate: &GateView) -> Status {
393 match gate.status.as_deref() {
394 Some("passed") => Status::Ok,
395 Some("failed") => Status::Failed,
396 Some("blocked") | None => Status::Pending,
397 Some(_) => Status::Unknown,
398 }
399 }
400
401 fn gate_condition(gate: &GateView) -> Condition {
402 Condition {
403 condition_type: gate.kind.clone(),
404 status: gate_status(gate),
405 since: gate.finished_at.as_deref().and_then(parse_instant),
406 detail: gate_detail(gate),
407 }
408 }
409
410 /// The why, from the typed outcome when there is one.
411 ///
412 /// "blocked" is useless; "blocked because burn_in has 17 hours remaining of 48"
413 /// is what saves an SSH.
414 fn gate_detail(gate: &GateView) -> Option<String> {
415 if gate.status.is_none() {
416 return Some("running".into());
417 }
418 let outcome = gate.outcome.clone()?;
419 let outcome: GateOutcome = serde_json::from_value(outcome).ok()?;
420 Some(match outcome.status {
421 crate::outcome::GateStatus::Passed { note } => note.summary(),
422 crate::outcome::GateStatus::Failed { failure } => failure.summary(),
423 crate::outcome::GateStatus::Blocked { blocker } => blocker.summary(),
424 })
425 }
426
427 // ---------------------------------------------------------------------------
428 // Deploy targets
429 // ---------------------------------------------------------------------------
430
431 /// One node per deploy target, carrying whatever independent health evidence
432 /// exists for it.
433 ///
434 /// `node_health` is the only per-node signal Sando has. When it failed on a
435 /// named node, that node is failed and its siblings are not. With no evidence
436 /// either way a node inherits its tier: claiming `unknown` for every node on a
437 /// tier without the gate would make Sando permanently loud in the rollup for a
438 /// topology choice, and a viewer that cries wolf is the thing this replaces.
439 fn node_nodes(tier: &TierView) -> Vec<Node> {
440 let unhealthy = unhealthy_node(tier);
441 let health_gate = tier.gates.iter().find(|g| g.kind == "node_health");
442
443 tier.nodes
444 .iter()
445 .map(|name| {
446 let status = if unhealthy.as_deref() == Some(name.as_str()) {
447 Status::Failed
448 } else {
449 match health_gate.map(gate_status) {
450 Some(Status::Ok) => Status::Ok,
451 // The gate failed on a *different* node, so this one is
452 // healthy by the same evidence.
453 Some(Status::Failed) if unhealthy.is_some() => Status::Ok,
454 Some(_) | None => tier_status(tier),
455 }
456 };
457
458 let mut conditions = Vec::new();
459 if let Some(gate) = health_gate {
460 conditions.push(Condition {
461 condition_type: "node_health".into(),
462 status,
463 since: gate.finished_at.as_deref().and_then(parse_instant),
464 detail: gate_detail(gate),
465 });
466 }
467
468 Node {
469 id: format!("node:{name}"),
470 kind: "node".into(),
471 label: name.clone(),
472 status,
473 fields: tier
474 .current_version
475 .as_ref()
476 .map(|v| vec![Field::new("version", Value::Version { value: v.clone() })])
477 .unwrap_or_default(),
478 conditions,
479 children: Vec::new(),
480 actions: Vec::new(),
481 }
482 })
483 .collect()
484 }
485
486 /// The node named by a `node_health` failure, if that is why the gate is red.
487 fn unhealthy_node(tier: &TierView) -> Option<String> {
488 let gate = tier.gates.iter().find(|g| g.kind == "node_health")?;
489 let outcome: GateOutcome = serde_json::from_value(gate.outcome.clone()?).ok()?;
490 match outcome.status {
491 crate::outcome::GateStatus::Failed {
492 failure: crate::outcome::GateFailure::NodeUnhealthy { node, .. },
493 } => Some(node),
494 _ => None,
495 }
496 }
497
498 // ---------------------------------------------------------------------------
499
500 /// Timestamps cross the DB boundary as RFC 3339 strings. A row that fails to
501 /// parse loses its `since` rather than failing the whole payload: a viewer that
502 /// blanks on one malformed timestamp is worse than one missing a tooltip.
503 fn parse_instant(raw: &str) -> Option<DateTime<Utc>> {
504 DateTime::parse_from_rfc3339(raw)
505 .ok()
506 .map(|d| d.with_timezone(&Utc))
507 }
508
509 #[cfg(test)]
510 mod tests {
511 use super::*;
512 use crate::outcome::{GateBlocker, GateFailure, GateStatus, PassNote};
513
514 fn now() -> DateTime<Utc> {
515 "2026-07-21T18:24:39Z".parse().unwrap()
516 }
517
518 fn gate(kind: &str, status: Option<&str>, outcome: Option<GateOutcome>) -> GateView {
519 GateView {
520 kind: kind.into(),
521 finished_at: Some("2026-07-21T14:02:00Z".into()),
522 status: status.map(Into::into),
523 outcome: outcome.map(|o| serde_json::to_value(o).unwrap()),
524 log_ref: None,
525 }
526 }
527
528 fn passed(note: PassNote) -> GateOutcome {
529 GateOutcome {
530 status: GateStatus::Passed { note },
531 log_ref: None,
532 }
533 }
534
535 fn blocked(blocker: GateBlocker) -> GateOutcome {
536 GateOutcome {
537 status: GateStatus::Blocked { blocker },
538 log_ref: None,
539 }
540 }
541
542 fn failed(failure: GateFailure) -> GateOutcome {
543 GateOutcome {
544 status: GateStatus::Failed { failure },
545 log_ref: None,
546 }
547 }
548
549 fn tier(name: &str) -> TierView {
550 TierView {
551 name: name.into(),
552 ord: 1,
553 provisioned: true,
554 canary: "sequential".into(),
555 current_version: Some("0.10.14".into()),
556 previous_version: Some("0.10.13".into()),
557 burn_in_started_at: Some("2026-07-21T14:02:00Z".into()),
558 partial_reason: None,
559 missing_artifact: None,
560 nodes: vec![format!("{name}-1")],
561 gates: Vec::new(),
562 }
563 }
564
565 fn view(tiers: Vec<TierView>) -> StateView {
566 StateView {
567 sandod_version: "0.2.2",
568 tiers,
569 build: None,
570 }
571 }
572
573 fn node<'a>(p: &'a Payload, id: &str) -> &'a Node {
574 p.node(id).unwrap_or_else(|| panic!("no node {id}"))
575 }
576
577 #[test]
578 fn a_healthy_tier_is_ok_and_structurally_sound() {
579 let mut t = tier("b");
580 t.gates = vec![gate(
581 "node_health",
582 Some("passed"),
583 Some(passed(PassNote::NodesHealthy { nodes: 1 })),
584 )];
585 let mut v = view(vec![t]);
586 v.build = Some(crate::runs::BuildSummary {
587 run_id: 40,
588 sha: "68f44d7ac21b7e4d".into(),
589 version: Some("0.10.14".into()),
590 phase: "done".into(),
591 result: "passed".into(),
592 failure_summary: None,
593 elapsed_s: 512,
594 });
595 let p = payload(&v, now());
596
597 assert_eq!(p.source, SOURCE);
598 assert_eq!(p.schema, ops_status::SCHEMA_VERSION);
599 assert_eq!(p.validate(), Ok(()));
600 assert_eq!(node(&p, "tier:b").status, Status::Ok);
601 assert_eq!(node(&p, "node:b-1").status, Status::Ok);
602 assert_eq!(p.worst_status(), Status::Ok);
603 }
604
605 #[test]
606 fn a_blocked_burn_in_leaves_the_tier_ok() {
607 // The judgement this module turns on: burn-in blocks for 48 hours as a
608 // matter of routine and must not make prod look sick for two days.
609 let mut t = tier("b");
610 t.gates = vec![gate(
611 "burn_in",
612 Some("blocked"),
613 Some(blocked(GateBlocker::BurnInRemaining {
614 hours_remaining: 17,
615 hours_total: 48,
616 })),
617 )];
618 let p = payload(&view(vec![t]), now());
619
620 assert_eq!(node(&p, "tier:b").status, Status::Ok);
621 let c = &node(&p, "tier:b").conditions[0];
622 assert_eq!(c.status, Status::Pending);
623 assert_eq!(c.detail.as_deref(), Some("17 hours remaining of 48"));
624 }
625
626 #[test]
627 fn burn_in_becomes_a_bar_not_a_sentence() {
628 let mut t = tier("b");
629 t.gates = vec![gate(
630 "burn_in",
631 Some("blocked"),
632 Some(blocked(GateBlocker::BurnInRemaining {
633 hours_remaining: 17,
634 hours_total: 48,
635 })),
636 )];
637 let p = payload(&view(vec![t]), now());
638
639 let field = node(&p, "tier:b")
640 .fields
641 .iter()
642 .find(|f| f.label == "burn-in")
643 .unwrap();
644 assert_eq!(
645 field.value,
646 Value::Progress {
647 value: 31.0,
648 max: 48.0,
649 unit: Some("hour".into())
650 }
651 );
652 }
653
654 #[test]
655 fn a_failed_gate_fails_the_tier() {
656 let mut t = tier("b");
657 t.gates = vec![gate(
658 "cargo_test",
659 Some("failed"),
660 Some(failed(GateFailure::CargoTest {
661 failed_count: 3,
662 first_failed: Some("auth::rate_limit".into()),
663 first_panic: None,
664 })),
665 )];
666 let p = payload(&view(vec![t]), now());
667
668 assert_eq!(node(&p, "tier:b").status, Status::Failed);
669 assert_eq!(p.worst_status(), Status::Failed);
670 assert!(
671 node(&p, "tier:b").conditions[0]
672 .detail
673 .as_deref()
674 .unwrap()
675 .contains("3 test(s)")
676 );
677 }
678
679 #[test]
680 fn a_partial_tier_is_failed_and_says_why_first() {
681 let mut t = tier("b");
682 t.partial_reason = Some("rsync failed on prod-1 after symlink swap".into());
683 let p = payload(&view(vec![t]), now());
684
685 let n = node(&p, "tier:b");
686 assert_eq!(n.status, Status::Failed);
687 assert_eq!(n.conditions[0].condition_type, "consistent");
688 assert_eq!(n.conditions[0].status, Status::Failed);
689 }
690
691 #[test]
692 fn an_unprovisioned_tier_is_pending_with_no_actions() {
693 let mut t = tier("c");
694 t.provisioned = false;
695 t.current_version = None;
696 t.nodes = Vec::new();
697 let p = payload(&view(vec![t]), now());
698
699 let n = node(&p, "tier:c");
700 assert_eq!(n.status, Status::Pending);
701 assert!(
702 n.actions.is_empty(),
703 "an unprovisioned tier must not offer a promote"
704 );
705 assert_eq!(n.conditions.last().unwrap().condition_type, "provisioned");
706 }
707
708 #[test]
709 fn node_health_blames_only_the_named_node() {
710 let mut t = tier("b");
711 t.nodes = vec!["prod-1".into(), "prod-2".into()];
712 t.gates = vec![gate(
713 "node_health",
714 Some("failed"),
715 Some(failed(GateFailure::NodeUnhealthy {
716 node: "prod-2".into(),
717 detail: "inactive".into(),
718 })),
719 )];
720 let p = payload(&view(vec![t]), now());
721
722 assert_eq!(node(&p, "node:prod-2").status, Status::Failed);
723 assert_eq!(node(&p, "node:prod-1").status, Status::Ok);
724 assert_eq!(node(&p, "tier:b").status, Status::Failed);
725 }
726
727 #[test]
728 fn a_node_with_no_health_evidence_inherits_its_tier() {
729 // Not `unknown`: a tier that simply has no node_health gate must not
730 // make Sando permanently loud in the rollup.
731 let t = tier("host");
732 let p = payload(&view(vec![t]), now());
733 assert_eq!(node(&p, "node:host-1").status, Status::Ok);
734 }
735
736 #[test]
737 fn an_unrecognized_gate_status_degrades_rather_than_hiding() {
738 let mut t = tier("b");
739 t.gates = vec![gate("mystery", Some("sideways"), None)];
740 let p = payload(&view(vec![t]), now());
741
742 assert_eq!(node(&p, "tier:b").conditions[0].status, Status::Unknown);
743 assert_eq!(node(&p, "tier:b").status, Status::Degraded);
744 }
745
746 #[test]
747 fn an_in_flight_gate_is_pending_and_says_so() {
748 let mut t = tier("b");
749 t.gates = vec![gate("cargo_test", None, None)];
750 let p = payload(&view(vec![t]), now());
751
752 let c = &node(&p, "tier:b").conditions[0];
753 assert_eq!(c.status, Status::Pending);
754 assert_eq!(c.detail.as_deref(), Some("running"));
755 assert_eq!(node(&p, "tier:b").status, Status::Ok);
756 }
757
758 #[test]
759 fn every_declared_action_is_referenced_and_every_reference_declared() {
760 let mut t = tier("b");
761 t.gates = vec![gate(
762 "node_health",
763 Some("passed"),
764 Some(passed(PassNote::NodesHealthy { nodes: 1 })),
765 )];
766 let p = payload(&view(vec![t, tier("a")]), now());
767
768 assert_eq!(p.validate(), Ok(()));
769 for action in &node(&p, "tier:b").actions {
770 let declared = &p.actions[action];
771 assert!(declared.confirm, "{action} must confirm");
772 assert!(declared.danger, "{action} must read as dangerous");
773 }
774 }
775
776 #[test]
777 fn a_failed_build_is_a_node_and_carries_its_summary() {
778 let mut v = view(vec![tier("b")]);
779 v.build = Some(crate::runs::BuildSummary {
780 run_id: 41,
781 sha: "68f44d7ac21b7e4d".into(),
782 version: Some("0.10.15".into()),
783 phase: "done".into(),
784 result: "failed".into(),
785 failure_summary: Some("error[E0063]: missing field user_pages_host".into()),
786 elapsed_s: 214,
787 });
788 let p = payload(&v, now());
789
790 let n = node(&p, "build");
791 assert_eq!(n.status, Status::Failed);
792 assert_eq!(n.actions, vec!["rebuild".to_string()]);
793 assert!(
794 n.conditions
795 .iter()
796 .any(|c| c.condition_type == "build_failure")
797 );
798 assert_eq!(p.worst_status(), Status::Failed);
799 assert_eq!(p.validate(), Ok(()));
800 }
801
802 #[test]
803 fn a_daemon_that_has_never_built_can_still_be_told_to_build() {
804 // An action reaches a viewer only through a node, so the build node is
805 // unconditional even before the first `/rebuild`.
806 let p = payload(&view(vec![tier("b")]), now());
807 let n = node(&p, "build");
808 assert_eq!(n.status, Status::Pending);
809 assert_eq!(n.actions, vec!["rebuild".to_string()]);
810 assert_eq!(n.conditions[0].detail.as_deref(), Some("no build run yet"));
811 assert_eq!(p.validate(), Ok(()));
812 }
813
814 #[test]
815 fn an_in_flight_build_is_pending_not_failed() {
816 let mut v = view(vec![tier("b")]);
817 v.build = Some(crate::runs::BuildSummary {
818 run_id: 42,
819 sha: "68f44d7a".into(),
820 version: None,
821 phase: "cargo build".into(),
822 result: "building".into(),
823 failure_summary: None,
824 elapsed_s: 61,
825 });
826 let p = payload(&v, now());
827 assert_eq!(node(&p, "build").status, Status::Pending);
828 }
829
830 #[test]
831 fn render_is_a_pure_function_of_state_and_clock() {
832 let v = view(vec![tier("b")]);
833 let a = payload(&v, now());
834 let b = payload(&v, now());
835 assert_eq!(
836 serde_json::to_value(&a).unwrap(),
837 serde_json::to_value(&b).unwrap()
838 );
839 }
840
841 #[test]
842 fn tiers_appear_in_topology_order() {
843 let p = payload(&view(vec![tier("host"), tier("a"), tier("b")]), now());
844 let tiers: Vec<&str> = p
845 .nodes
846 .iter()
847 .filter(|n| n.kind == "tier")
848 .map(|n| n.label.as_str())
849 .collect();
850 assert_eq!(tiers, vec!["host", "a", "b"]);
851 }
852
853 #[test]
854 fn a_malformed_timestamp_costs_only_that_timestamp() {
855 let mut t = tier("b");
856 t.gates = vec![GateView {
857 kind: "node_health".into(),
858 finished_at: Some("not a timestamp".into()),
859 status: Some("passed".into()),
860 outcome: None,
861 log_ref: None,
862 }];
863 let p = payload(&view(vec![t]), now());
864 assert!(node(&p, "tier:b").conditions[0].since.is_none());
865 assert_eq!(node(&p, "tier:b").status, Status::Ok);
866 }
867
868 #[test]
869 fn a_tier_missing_a_referenced_artifact_is_degraded_and_says_which() {
870 // The state that reported green through two occurrences: the tier is
871 // serving, and the artifact behind its rollback is gone.
872 let mut t = tier("b");
873 t.missing_artifact = Some("b previous 0.11.20 (/srv/sando/releases/ed29/mnw)".into());
874 let p = payload(&view(vec![t]), now());
875
876 let n = node(&p, "tier:b");
877 assert_eq!(n.status, Status::Degraded);
878 let c = n
879 .conditions
880 .iter()
881 .find(|c| c.condition_type == "artifact")
882 .expect("an artifact condition");
883 assert_eq!(c.status, Status::Degraded);
884 assert!(
885 c.detail.as_deref().unwrap_or_default().contains("0.11.20"),
886 "the condition has to name what is gone: {:?}",
887 c.detail
888 );
889 assert_eq!(p.validate(), Ok(()));
890 }
891
892 #[test]
893 fn a_partial_tier_stays_failed_even_with_a_missing_artifact() {
894 // Degraded must not mask the loudest thing Sando can report.
895 let mut t = tier("b");
896 t.partial_reason = Some("mixed versions after a failed promote".into());
897 t.missing_artifact = Some("b previous 0.11.20 (/srv/sando/releases/ed29/mnw)".into());
898 let p = payload(&view(vec![t]), now());
899
900 let n = node(&p, "tier:b");
901 assert_eq!(n.status, Status::Failed);
902 assert!(n.conditions.iter().any(|c| c.condition_type == "artifact"));
903 assert!(
904 n.conditions
905 .iter()
906 .any(|c| c.condition_type == "consistent")
907 );
908 }
909 }
910