//! Sando's projection onto the shared operator status payload. //! //! Spec + rationale: maintainer wiki. //! //! //! `GET /status.json` serves this. It is the same read as `/state`, restated in //! the vocabulary every operator surface renders, so a viewer needs no //! knowledge of tiers, gates, or promotion to draw Sando. //! //! [`payload`] is a pure function of `(StateView, now)`. Every clock is an //! argument, so a fixture state renders identically forever and the mapping is //! testable without a database. //! //! # What maps to what //! //! | Sando | payload | //! |---|---| //! | tier | node, `kind = "tier"` | //! | deploy target | node, `kind = "node"`, child of its tier | //! | latest build | node, `kind = "build"` | //! | gate | condition on its tier | //! | promote / rollback / confirm | declared action | //! //! # Gates are promotion guards, not tier health //! //! The load-bearing judgement here. A gate blocks promotion *out of* a tier; it //! does not describe whether that tier is serving. `burn_in` sits blocked for //! 48 hours as a matter of routine, and a tier whose status went yellow for two //! days every release is a tier nobody looks at. //! //! So a blocked or in-flight gate is reported as a `pending` condition and //! leaves the tier `ok`. Only evidence of something actually wrong — a failed //! gate, or a partial (mixed-version) tier — degrades the tier itself. The why //! is never lost: it is in the conditions either way. use chrono::{DateTime, Utc}; use ops_status::{Action, Condition, Field, Method, Node, Payload, Status, Value}; use crate::outcome::GateOutcome; use crate::routes::{GateView, StateView, TierView}; /// The `source` name Sando answers to in a viewer's config. pub const SOURCE: &str = "sando"; /// Restate `/state` as the shared payload. /// /// `now` is an argument rather than read from the clock so the mapping stays /// pure and snapshot-testable. pub(crate) fn payload(view: &StateView, now: DateTime) -> Payload { let mut payload = Payload::new(SOURCE, now); payload.actions.insert( "rebuild".into(), Action { label: "Rebuild".into(), method: Method::Post, url: "/rebuild".into(), confirm: false, danger: false, body: None, }, ); // Always emitted, even before the first `/rebuild`. Actions reach a viewer // only by hanging off a node, so a daemon with no build history and no // build node would offer no way to start one. payload.nodes.push(build_node(view.build.as_ref())); for tier in &view.tiers { for action in tier_actions(tier) { payload.actions.insert(action.0, action.1); } payload.nodes.push(tier_node(tier)); payload.nodes.extend(node_nodes(tier)); } payload } // --------------------------------------------------------------------------- // Build // --------------------------------------------------------------------------- fn build_node(build: Option<&crate::runs::BuildSummary>) -> Node { let Some(build) = build else { return Node { id: "build".into(), kind: "build".into(), label: "latest build".into(), status: Status::Pending, fields: Vec::new(), conditions: vec![Condition { condition_type: "build".into(), status: Status::Pending, since: None, detail: Some("no build run yet".into()), }], children: Vec::new(), actions: vec!["rebuild".into()], }; }; // `build_runs.result` is one of building / passed / failed / aborted. An // aborted run is degraded rather than failed: the daemon settling an // orphaned run on restart is not the same event as a build that broke. let status = match build.result.as_str() { "passed" => Status::Ok, "failed" => Status::Failed, "building" => Status::Pending, "aborted" => Status::Degraded, _ => Status::Unknown, }; let mut fields = vec![ Field::new( "sha", Value::Ident { value: build.sha.clone(), abbrev_to: Some(8), }, ), Field::new( "phase", Value::Text { value: build.phase.clone(), }, ), Field::new( "elapsed", Value::Duration { seconds: build.elapsed_s, }, ), ]; if let Some(version) = &build.version { fields.insert( 0, Field::new( "version", Value::Version { value: version.clone(), }, ), ); } let mut conditions = vec![Condition { condition_type: "build".into(), status, since: None, detail: Some(format!("run {} {}", build.run_id, build.result)), }]; if let Some(failure) = &build.failure_summary { conditions.push(Condition { condition_type: "build_failure".into(), status: Status::Failed, since: None, detail: Some(failure.clone()), }); } Node { id: "build".into(), kind: "build".into(), label: "latest build".into(), status, fields, conditions, children: Vec::new(), actions: vec!["rebuild".into()], } } // --------------------------------------------------------------------------- // Tiers // --------------------------------------------------------------------------- fn tier_id(tier: &TierView) -> String { format!("tier:{}", tier.name) } fn tier_node(tier: &TierView) -> Node { let mut conditions: Vec = tier.gates.iter().map(gate_condition).collect(); // A partial tier is mid-version and could not be compensated back to // consistency. It is the loudest thing Sando can report. if let Some(reason) = &tier.partial_reason { conditions.insert( 0, Condition { condition_type: "consistent".into(), status: Status::Failed, since: None, detail: Some(reason.clone()), }, ); } if !tier.provisioned { conditions.push(Condition { condition_type: "provisioned".into(), status: Status::Pending, since: None, detail: Some("declared in topology, not provisioned".into()), }); } let mut fields = vec![Field::new( "canary", Value::Text { value: tier.canary.clone(), }, )]; if let Some(version) = &tier.current_version { fields.insert( 0, Field::new( "version", Value::Version { value: version.clone(), }, ), ); } if let Some(previous) = &tier.previous_version { fields.push(Field::new( "previous", Value::Version { value: previous.clone(), }, )); } if let Some(started) = tier.burn_in_started_at.as_deref().and_then(parse_instant) { fields.push(Field::new( "burn-in since", Value::Instant { value: started }, )); } if let Some(progress) = burn_in_progress(tier) { fields.push(progress); } Node { id: tier_id(tier), kind: "tier".into(), label: tier.name.clone(), status: tier_status(tier), fields, conditions, children: tier.nodes.iter().map(|n| format!("node:{n}")).collect(), actions: tier_actions(tier).into_iter().map(|(key, _)| key).collect(), } } /// Whether the tier itself is healthy — deliberately not "are its gates green". /// /// See the module docs: gates guard promotion out, so a blocked one is routine /// and must not color the tier. fn tier_status(tier: &TierView) -> Status { if tier.partial_reason.is_some() { return Status::Failed; } if tier .gates .iter() .any(|g| g.status.as_deref() == Some("failed")) { return Status::Failed; } // An unrecognized gate status is contract drift between daemon and viewer, // which is worth surfacing rather than smoothing over. The TUI renders it // magenta for the same reason. if tier.gates.iter().any(|g| gate_status(g) == Status::Unknown) { return Status::Degraded; } if !tier.provisioned || tier.current_version.is_none() { return Status::Pending; } Status::Ok } /// Burn-in as a bar rather than a sentence, so it sorts and colors like every /// other progress value in the UI. fn burn_in_progress(tier: &TierView) -> Option { let gate = tier.gates.iter().find(|g| g.kind == "burn_in")?; let outcome = gate.outcome.clone()?; let outcome: GateOutcome = serde_json::from_value(outcome).ok()?; let (value, max) = match outcome.status { crate::outcome::GateStatus::Blocked { blocker: crate::outcome::GateBlocker::BurnInRemaining { hours_remaining, hours_total, }, } => ( f64::from(hours_total.saturating_sub(hours_remaining)), f64::from(hours_total), ), crate::outcome::GateStatus::Passed { note: crate::outcome::PassNote::BurnInElapsed { hours }, } => (f64::from(hours), f64::from(hours)), _ => return None, }; Some(Field::new( "burn-in", Value::Progress { value, max, unit: Some("hour".into()), }, )) } /// Promote, roll back, confirm — declared as data so the viewer issues them /// without knowing what any of them mean. /// /// All three carry `confirm` and `danger`. Sando has no flag marking which /// tiers are production, and inventing one from the tier name would be a /// heuristic that fails silently the day the topology changes. Over-confirming /// a deploy is the cheap direction to be wrong in. fn tier_actions(tier: &TierView) -> Vec<(String, Action)> { if !tier.provisioned { return Vec::new(); } let name = &tier.name; vec![ ( format!("promote-{name}"), Action { label: format!("Promote to {name}"), method: Method::Post, url: format!("/promote/{name}"), confirm: true, danger: true, body: None, }, ), ( format!("rollback-{name}"), Action { label: format!("Roll back {name}"), method: Method::Post, url: format!("/rollback/{name}"), confirm: true, danger: true, body: None, }, ), ( format!("confirm-{name}"), Action { label: format!("Confirm {name}"), method: Method::Post, url: format!("/confirm/{name}"), confirm: true, danger: true, body: None, }, ), ] } // --------------------------------------------------------------------------- // Gates // --------------------------------------------------------------------------- /// `passed | failed | blocked`, or NULL while the gate is in flight. /// /// `blocked` becomes `pending`: a blocked gate is waiting on something, which /// is what pending means here. An unrecognized string is `unknown`, not /// in-flight — that distinction is why the TUI renders it magenta. fn gate_status(gate: &GateView) -> Status { match gate.status.as_deref() { Some("passed") => Status::Ok, Some("failed") => Status::Failed, Some("blocked") | None => Status::Pending, Some(_) => Status::Unknown, } } fn gate_condition(gate: &GateView) -> Condition { Condition { condition_type: gate.kind.clone(), status: gate_status(gate), since: gate.finished_at.as_deref().and_then(parse_instant), detail: gate_detail(gate), } } /// The why, from the typed outcome when there is one. /// /// "blocked" is useless; "blocked because burn_in has 17 hours remaining of 48" /// is what saves an SSH. fn gate_detail(gate: &GateView) -> Option { if gate.status.is_none() { return Some("running".into()); } let outcome = gate.outcome.clone()?; let outcome: GateOutcome = serde_json::from_value(outcome).ok()?; Some(match outcome.status { crate::outcome::GateStatus::Passed { note } => note.summary(), crate::outcome::GateStatus::Failed { failure } => failure.summary(), crate::outcome::GateStatus::Blocked { blocker } => blocker.summary(), }) } // --------------------------------------------------------------------------- // Deploy targets // --------------------------------------------------------------------------- /// One node per deploy target, carrying whatever independent health evidence /// exists for it. /// /// `node_health` is the only per-node signal Sando has. When it failed on a /// named node, that node is failed and its siblings are not. With no evidence /// either way a node inherits its tier: claiming `unknown` for every node on a /// tier without the gate would make Sando permanently loud in the rollup for a /// topology choice, and a viewer that cries wolf is the thing this replaces. fn node_nodes(tier: &TierView) -> Vec { let unhealthy = unhealthy_node(tier); let health_gate = tier.gates.iter().find(|g| g.kind == "node_health"); tier.nodes .iter() .map(|name| { let status = if unhealthy.as_deref() == Some(name.as_str()) { Status::Failed } else { match health_gate.map(gate_status) { Some(Status::Ok) => Status::Ok, // The gate failed on a *different* node, so this one is // healthy by the same evidence. Some(Status::Failed) if unhealthy.is_some() => Status::Ok, Some(_) | None => tier_status(tier), } }; let mut conditions = Vec::new(); if let Some(gate) = health_gate { conditions.push(Condition { condition_type: "node_health".into(), status, since: gate.finished_at.as_deref().and_then(parse_instant), detail: gate_detail(gate), }); } Node { id: format!("node:{name}"), kind: "node".into(), label: name.clone(), status, fields: tier .current_version .as_ref() .map(|v| vec![Field::new("version", Value::Version { value: v.clone() })]) .unwrap_or_default(), conditions, children: Vec::new(), actions: Vec::new(), } }) .collect() } /// The node named by a `node_health` failure, if that is why the gate is red. fn unhealthy_node(tier: &TierView) -> Option { let gate = tier.gates.iter().find(|g| g.kind == "node_health")?; let outcome: GateOutcome = serde_json::from_value(gate.outcome.clone()?).ok()?; match outcome.status { crate::outcome::GateStatus::Failed { failure: crate::outcome::GateFailure::NodeUnhealthy { node, .. }, } => Some(node), _ => None, } } // --------------------------------------------------------------------------- /// Timestamps cross the DB boundary as RFC 3339 strings. A row that fails to /// parse loses its `since` rather than failing the whole payload: a viewer that /// blanks on one malformed timestamp is worse than one missing a tooltip. fn parse_instant(raw: &str) -> Option> { DateTime::parse_from_rfc3339(raw) .ok() .map(|d| d.with_timezone(&Utc)) } #[cfg(test)] mod tests { use super::*; use crate::outcome::{GateBlocker, GateFailure, GateStatus, PassNote}; fn now() -> DateTime { "2026-07-21T18:24:39Z".parse().unwrap() } fn gate(kind: &str, status: Option<&str>, outcome: Option) -> GateView { GateView { kind: kind.into(), finished_at: Some("2026-07-21T14:02:00Z".into()), status: status.map(Into::into), outcome: outcome.map(|o| serde_json::to_value(o).unwrap()), log_ref: None, } } fn passed(note: PassNote) -> GateOutcome { GateOutcome { status: GateStatus::Passed { note }, log_ref: None, } } fn blocked(blocker: GateBlocker) -> GateOutcome { GateOutcome { status: GateStatus::Blocked { blocker }, log_ref: None, } } fn failed(failure: GateFailure) -> GateOutcome { GateOutcome { status: GateStatus::Failed { failure }, log_ref: None, } } fn tier(name: &str) -> TierView { TierView { name: name.into(), ord: 1, provisioned: true, canary: "sequential".into(), current_version: Some("0.10.14".into()), previous_version: Some("0.10.13".into()), burn_in_started_at: Some("2026-07-21T14:02:00Z".into()), partial_reason: None, nodes: vec![format!("{name}-1")], gates: Vec::new(), } } fn view(tiers: Vec) -> StateView { StateView { sandod_version: "0.2.2", tiers, build: None, } } fn node<'a>(p: &'a Payload, id: &str) -> &'a Node { p.node(id).unwrap_or_else(|| panic!("no node {id}")) } #[test] fn a_healthy_tier_is_ok_and_structurally_sound() { let mut t = tier("b"); t.gates = vec![gate( "node_health", Some("passed"), Some(passed(PassNote::NodesHealthy { nodes: 1 })), )]; let mut v = view(vec![t]); v.build = Some(crate::runs::BuildSummary { run_id: 40, sha: "68f44d7ac21b7e4d".into(), version: Some("0.10.14".into()), phase: "done".into(), result: "passed".into(), failure_summary: None, elapsed_s: 512, }); let p = payload(&v, now()); assert_eq!(p.source, SOURCE); assert_eq!(p.schema, ops_status::SCHEMA_VERSION); assert_eq!(p.validate(), Ok(())); assert_eq!(node(&p, "tier:b").status, Status::Ok); assert_eq!(node(&p, "node:b-1").status, Status::Ok); assert_eq!(p.worst_status(), Status::Ok); } #[test] fn a_blocked_burn_in_leaves_the_tier_ok() { // The judgement this module turns on: burn-in blocks for 48 hours as a // matter of routine and must not make prod look sick for two days. let mut t = tier("b"); t.gates = vec![gate( "burn_in", Some("blocked"), Some(blocked(GateBlocker::BurnInRemaining { hours_remaining: 17, hours_total: 48, })), )]; let p = payload(&view(vec![t]), now()); assert_eq!(node(&p, "tier:b").status, Status::Ok); let c = &node(&p, "tier:b").conditions[0]; assert_eq!(c.status, Status::Pending); assert_eq!(c.detail.as_deref(), Some("17 hours remaining of 48")); } #[test] fn burn_in_becomes_a_bar_not_a_sentence() { let mut t = tier("b"); t.gates = vec![gate( "burn_in", Some("blocked"), Some(blocked(GateBlocker::BurnInRemaining { hours_remaining: 17, hours_total: 48, })), )]; let p = payload(&view(vec![t]), now()); let field = node(&p, "tier:b") .fields .iter() .find(|f| f.label == "burn-in") .unwrap(); assert_eq!( field.value, Value::Progress { value: 31.0, max: 48.0, unit: Some("hour".into()) } ); } #[test] fn a_failed_gate_fails_the_tier() { let mut t = tier("b"); t.gates = vec![gate( "cargo_test", Some("failed"), Some(failed(GateFailure::CargoTest { failed_count: 3, first_failed: Some("auth::rate_limit".into()), first_panic: None, })), )]; let p = payload(&view(vec![t]), now()); assert_eq!(node(&p, "tier:b").status, Status::Failed); assert_eq!(p.worst_status(), Status::Failed); assert!( node(&p, "tier:b").conditions[0] .detail .as_deref() .unwrap() .contains("3 test(s)") ); } #[test] fn a_partial_tier_is_failed_and_says_why_first() { let mut t = tier("b"); t.partial_reason = Some("rsync failed on prod-1 after symlink swap".into()); let p = payload(&view(vec![t]), now()); let n = node(&p, "tier:b"); assert_eq!(n.status, Status::Failed); assert_eq!(n.conditions[0].condition_type, "consistent"); assert_eq!(n.conditions[0].status, Status::Failed); } #[test] fn an_unprovisioned_tier_is_pending_with_no_actions() { let mut t = tier("c"); t.provisioned = false; t.current_version = None; t.nodes = Vec::new(); let p = payload(&view(vec![t]), now()); let n = node(&p, "tier:c"); assert_eq!(n.status, Status::Pending); assert!( n.actions.is_empty(), "an unprovisioned tier must not offer a promote" ); assert_eq!(n.conditions.last().unwrap().condition_type, "provisioned"); } #[test] fn node_health_blames_only_the_named_node() { let mut t = tier("b"); t.nodes = vec!["prod-1".into(), "prod-2".into()]; t.gates = vec![gate( "node_health", Some("failed"), Some(failed(GateFailure::NodeUnhealthy { node: "prod-2".into(), detail: "inactive".into(), })), )]; let p = payload(&view(vec![t]), now()); assert_eq!(node(&p, "node:prod-2").status, Status::Failed); assert_eq!(node(&p, "node:prod-1").status, Status::Ok); assert_eq!(node(&p, "tier:b").status, Status::Failed); } #[test] fn a_node_with_no_health_evidence_inherits_its_tier() { // Not `unknown`: a tier that simply has no node_health gate must not // make Sando permanently loud in the rollup. let t = tier("host"); let p = payload(&view(vec![t]), now()); assert_eq!(node(&p, "node:host-1").status, Status::Ok); } #[test] fn an_unrecognized_gate_status_degrades_rather_than_hiding() { let mut t = tier("b"); t.gates = vec![gate("mystery", Some("sideways"), None)]; let p = payload(&view(vec![t]), now()); assert_eq!(node(&p, "tier:b").conditions[0].status, Status::Unknown); assert_eq!(node(&p, "tier:b").status, Status::Degraded); } #[test] fn an_in_flight_gate_is_pending_and_says_so() { let mut t = tier("b"); t.gates = vec![gate("cargo_test", None, None)]; let p = payload(&view(vec![t]), now()); let c = &node(&p, "tier:b").conditions[0]; assert_eq!(c.status, Status::Pending); assert_eq!(c.detail.as_deref(), Some("running")); assert_eq!(node(&p, "tier:b").status, Status::Ok); } #[test] fn every_declared_action_is_referenced_and_every_reference_declared() { let mut t = tier("b"); t.gates = vec![gate( "node_health", Some("passed"), Some(passed(PassNote::NodesHealthy { nodes: 1 })), )]; let p = payload(&view(vec![t, tier("a")]), now()); assert_eq!(p.validate(), Ok(())); for action in &node(&p, "tier:b").actions { let declared = &p.actions[action]; assert!(declared.confirm, "{action} must confirm"); assert!(declared.danger, "{action} must read as dangerous"); } } #[test] fn a_failed_build_is_a_node_and_carries_its_summary() { let mut v = view(vec![tier("b")]); v.build = Some(crate::runs::BuildSummary { run_id: 41, sha: "68f44d7ac21b7e4d".into(), version: Some("0.10.15".into()), phase: "done".into(), result: "failed".into(), failure_summary: Some("error[E0063]: missing field user_pages_host".into()), elapsed_s: 214, }); let p = payload(&v, now()); let n = node(&p, "build"); assert_eq!(n.status, Status::Failed); assert_eq!(n.actions, vec!["rebuild".to_string()]); assert!( n.conditions .iter() .any(|c| c.condition_type == "build_failure") ); assert_eq!(p.worst_status(), Status::Failed); assert_eq!(p.validate(), Ok(())); } #[test] fn a_daemon_that_has_never_built_can_still_be_told_to_build() { // An action reaches a viewer only through a node, so the build node is // unconditional even before the first `/rebuild`. let p = payload(&view(vec![tier("b")]), now()); let n = node(&p, "build"); assert_eq!(n.status, Status::Pending); assert_eq!(n.actions, vec!["rebuild".to_string()]); assert_eq!(n.conditions[0].detail.as_deref(), Some("no build run yet")); assert_eq!(p.validate(), Ok(())); } #[test] fn an_in_flight_build_is_pending_not_failed() { let mut v = view(vec![tier("b")]); v.build = Some(crate::runs::BuildSummary { run_id: 42, sha: "68f44d7a".into(), version: None, phase: "cargo build".into(), result: "building".into(), failure_summary: None, elapsed_s: 61, }); let p = payload(&v, now()); assert_eq!(node(&p, "build").status, Status::Pending); } #[test] fn render_is_a_pure_function_of_state_and_clock() { let v = view(vec![tier("b")]); let a = payload(&v, now()); let b = payload(&v, now()); assert_eq!( serde_json::to_value(&a).unwrap(), serde_json::to_value(&b).unwrap() ); } #[test] fn tiers_appear_in_topology_order() { let p = payload(&view(vec![tier("host"), tier("a"), tier("b")]), now()); let tiers: Vec<&str> = p .nodes .iter() .filter(|n| n.kind == "tier") .map(|n| n.label.as_str()) .collect(); assert_eq!(tiers, vec!["host", "a", "b"]); } #[test] fn a_malformed_timestamp_costs_only_that_timestamp() { let mut t = tier("b"); t.gates = vec![GateView { kind: "node_health".into(), finished_at: Some("not a timestamp".into()), status: Some("passed".into()), outcome: None, log_ref: None, }]; let p = payload(&view(vec![t]), now()); assert!(node(&p, "tier:b").conditions[0].since.is_none()); assert_eq!(node(&p, "tier:b").status, Status::Ok); } }