Skip to main content

max / makenotwork

sando: serve GET /status.json Sando's projection onto the shared ops-status contract: tiers become nodes, deploy targets their children, the latest build a node of its own, gates conditions, and promote/rollback/confirm/rebuild declared actions. /state and /status.json are now two renderings of one read. The query moved into state_view() and both handlers call it, so the two cannot drift into disagreeing about what a tier is. The mapping is a pure function of (StateView, now) with the clock passed in, so all 18 of its tests run without a database. Gates guard promotion OUT of a tier; they are not that tier's health. burn_in sits blocked for 48 hours as a matter of routine, and a tier that went yellow for two days every release is a tier nobody looks at. So a blocked or in-flight gate reports as a pending condition and leaves the tier ok; only a failed gate or a partial (mixed-version) tier degrades the tier itself. The why is never lost either way -- it is in the conditions. Every promote/rollback/confirm is declared confirm + danger. Sando has no flag marking which tiers are production, and deriving one from the tier name would be a heuristic that fails silently the day the topology changes. Over-confirming a deploy is the cheap direction to be wrong in. A real `production = true` in sando.toml would be better than the guess. A node with no node_health evidence inherits its tier rather than reporting unknown. Claiming unknown for every node on a tier that simply has no such gate would make Sando permanently loud in the rollup because of a topology choice, and a viewer that cries wolf is what this replaces. The build node is emitted even before the first /rebuild. Actions reach a viewer only by hanging off a node, so a daemon with no build history and no build node would offer no way to start one. Found by reading the rendered payload rather than by a test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-21 23:38 UTC
Commit: f83156b0529754e8a0da56aee0b5293cf5a999b4
Parent: d57f442
5 files changed, +586 insertions, -24 deletions
M sando/Cargo.lock +10
@@ -1354,6 +1354,15 @@ dependencies = [
1354 1354 ]
1355 1355
1356 1356 [[package]]
1357 + name = "ops-status"
1358 + version = "0.1.0"
1359 + dependencies = [
1360 + "chrono",
1361 + "serde",
1362 + "serde_json",
1363 + ]
1364 +
1365 + [[package]]
1357 1366 name = "parking"
1358 1367 version = "2.2.1"
1359 1368 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1832,6 +1841,7 @@ dependencies = [
1832 1841 "http-body-util",
1833 1842 "ops-core",
1834 1843 "ops-exec",
1844 + "ops-status",
1835 1845 "reqwest",
1836 1846 "semver",
1837 1847 "serde",
M sando/daemon/Cargo.toml +1
@@ -12,6 +12,7 @@ path = "src/main.rs"
12 12 axum = { version = "0.8.8", features = ["macros", "ws"] }
13 13 tokio = { version = "1.50.0", features = ["macros", "rt-multi-thread", "net", "signal", "fs", "process"] }
14 14 ops-core = { path = "../../shared/ops-core" }
15 + ops-status = { path = "../../shared/ops-status" }
15 16 ops-exec = { path = "../../shared/ops-exec" }
16 17 async-trait = "0.1.83"
17 18 serde = { version = "1.0.228", features = ["derive"] }
M sando/daemon/src/lib.rs +1
@@ -29,5 +29,6 @@ pub mod outcome;
29 29 pub mod routes;
30 30 pub mod runs;
31 31 pub mod state;
32 + pub mod status;
32 33 pub mod sync;
33 34 pub mod topology;
M sando/daemon/src/routes/mod.rs +74 -24
@@ -31,6 +31,7 @@ pub fn router(state: AppState) -> Router {
31 31 .route("/backup/fetch", post(backup_fetch))
32 32 // reads — prod state, now also gated
33 33 .route("/state", get(get_state))
34 + .route("/status.json", get(get_status_json))
34 35 .route("/runs/{id}", get(get_run))
35 36 .route("/runs/{id}/wait", get(get_run_wait))
36 37 .route("/logs/{version}/{gate}", get(get_gate_log))
@@ -102,52 +103,64 @@ fn ct_eq(a: &str, b: &str) -> bool {
102 103 }
103 104
104 105 #[derive(Serialize)]
105 - struct StateView {
106 + pub(crate) struct StateView {
106 107 /// The running sandod's own package version. Lets a self-update caller
107 108 /// confirm the new binary is live after the restart (the tier versions are
108 109 /// the *deployed product*, not the controller).
109 - sandod_version: &'static str,
110 - tiers: Vec<TierView>,
110 + pub(crate) sandod_version: &'static str,
111 + pub(crate) tiers: Vec<TierView>,
111 112 /// The most recent build run (the resource `GET /runs/{id}` exposes in
112 113 /// full). Surfaced here so a `/state` poller sees an in-flight or failed
113 114 /// build — the tier versions only ever reflect the last *success*, so
114 115 /// without this `/state` looks frozen for the whole build. `null` until
115 116 /// the first `/rebuild`.
116 - build: Option<crate::runs::BuildSummary>,
117 + pub(crate) build: Option<crate::runs::BuildSummary>,
117 118 }
118 119
119 120 #[derive(Serialize)]
120 - struct TierView {
121 - name: String,
122 - ord: i64,
123 - provisioned: bool,
124 - canary: String,
125 - current_version: Option<String>,
126 - previous_version: Option<String>,
127 - burn_in_started_at: Option<String>,
121 + pub(crate) struct TierView {
122 + pub(crate) name: String,
123 + pub(crate) ord: i64,
124 + pub(crate) provisioned: bool,
125 + pub(crate) canary: String,
126 + pub(crate) current_version: Option<String>,
127 + pub(crate) previous_version: Option<String>,
128 + pub(crate) burn_in_started_at: Option<String>,
128 129 /// Non-null when the tier was left in a partial / mixed-version state by a
129 130 /// failed promote or rollback whose compensation could not fully restore
130 131 /// consistency. NULL when the tier is consistent. The TUI flags it red.
131 - partial_reason: Option<String>,
132 - nodes: Vec<String>,
133 - gates: Vec<GateView>,
132 + pub(crate) partial_reason: Option<String>,
133 + pub(crate) nodes: Vec<String>,
134 + pub(crate) gates: Vec<GateView>,
134 135 }
135 136
136 137 #[derive(Serialize)]
137 - struct GateView {
138 - kind: String,
139 - finished_at: Option<String>,
138 + pub(crate) struct GateView {
139 + pub(crate) kind: String,
140 + pub(crate) finished_at: Option<String>,
140 141 /// `'passed' | 'failed' | 'blocked'` or NULL while in-flight. The TUI
141 142 /// uses this to choose green/red/yellow rendering.
142 - status: Option<String>,
143 + pub(crate) status: Option<String>,
143 144 /// Full typed `GateOutcome` as a JSON object, when present.
144 145 /// Deserialized lazily by the consumer; sandod doesn't re-parse it.
145 - outcome: Option<serde_json::Value>,
146 + pub(crate) outcome: Option<serde_json::Value>,
146 147 /// Relative path under `cfg.logs_root` to the persisted stdout/stderr.
147 - log_ref: Option<String>,
148 + pub(crate) log_ref: Option<String>,
148 149 }
149 150
150 151 async fn get_state(State(s): State<AppState>) -> Result<Json<StateView>> {
152 + Ok(Json(state_view(&s).await?))
153 + }
154 +
155 + /// `GET /status.json` — the same state, in the shared cross-service payload
156 + /// every operator surface renders. See `crate::status`.
157 + async fn get_status_json(State(s): State<AppState>) -> Result<Json<ops_status::Payload>> {
158 + let view = state_view(&s).await?;
159 + Ok(Json(crate::status::payload(&view, chrono::Utc::now())))
160 + }
161 +
162 + /// The shared read behind `/state` and `/status.json`.
163 + pub(crate) async fn state_view(s: &AppState) -> Result<StateView> {
151 164 let rows = sqlx::query(
152 165 "SELECT t.name, t.ord, t.provisioned, t.canary,
153 166 ts.current_version, ts.previous_version, ts.burn_in_started_at,
@@ -159,7 +172,7 @@ async fn get_state(State(s): State<AppState>) -> Result<Json<StateView>> {
159 172 .fetch_all(&s.pool)
160 173 .await?;
161 174
162 - let mut tiers = Vec::with_capacity(rows.len());
175 + let mut tiers: Vec<TierView> = Vec::with_capacity(rows.len());
163 176 for r in rows {
164 177 let name: String = r.get("name");
165 178 let current_version: Option<String> = r.get("current_version");
@@ -231,11 +244,11 @@ async fn get_state(State(s): State<AppState>) -> Result<Json<StateView>> {
231 244 }
232 245
233 246 let build = crate::runs::latest_summary(&s.pool).await?;
234 - Ok(Json(StateView {
247 + Ok(StateView {
235 248 sandod_version: env!("CARGO_PKG_VERSION"),
236 249 tiers,
237 250 build,
238 - }))
251 + })
239 252 }
240 253
241 254 #[derive(Deserialize, Default)]
@@ -1913,6 +1926,43 @@ mod tests {
1913 1926 }
1914 1927
1915 1928 #[tokio::test]
1929 + async fn status_json_serves_the_shared_payload_over_the_real_router() {
1930 + // The mapping itself is tested in `crate::status`. This asserts the
1931 + // route is wired, serves valid JSON, and stays internally consistent
1932 + // (no dangling child or action references) against a real topology
1933 + // rather than a hand-built fixture.
1934 + let state = test_state().await;
1935 + set_partial(&state, &tid("a"), "canary rollback incomplete: 1/2").await;
1936 +
1937 + let resp = router(state)
1938 + .oneshot(
1939 + Request::builder()
1940 + .uri("/status.json")
1941 + .body(Body::empty())
1942 + .unwrap(),
1943 + )
1944 + .await
1945 + .unwrap();
1946 + assert_eq!(resp.status(), StatusCode::OK);
1947 +
1948 + let body = http_body_util::BodyExt::collect(resp.into_body())
1949 + .await
1950 + .unwrap()
1951 + .to_bytes();
1952 + let payload: ops_status::Payload = serde_json::from_slice(&body).unwrap();
1953 +
1954 + assert_eq!(payload.source, crate::status::SOURCE);
1955 + assert_eq!(payload.schema, ops_status::SCHEMA_VERSION);
1956 + assert_eq!(payload.validate(), Ok(()));
1957 + assert_eq!(
1958 + payload.node("tier:a").unwrap().status,
1959 + ops_status::Status::Failed,
1960 + "a partial tier must surface as failed over the wire"
1961 + );
1962 + assert_eq!(payload.worst_status(), ops_status::Status::Failed);
1963 + }
1964 +
1965 + #[tokio::test]
1916 1966 async fn promote_with_explicit_version_but_missing_artifact_404s() {
1917 1967 // Explicit version supplied, gates trivially pass (mm has none in
1918 1968 // test_topo), but `versions` table has no row → 404.
A sando/daemon/src/status.rs +500
@@ -0,0 +1,847 @@
1 + //! Sando's projection onto the shared operator status payload.
2 + //!
3 + //! Spec + rationale: maintainer wiki.
4 + //! <!-- wiki: release-status-payload -->
5 + //!
6 + //! `GET /status.json` serves this. It is the same read as `/state`, restated in
7 + //! the vocabulary every operator surface renders, so a viewer needs no
8 + //! knowledge of tiers, gates, or promotion to draw Sando.
9 + //!
10 + //! [`payload`] is a pure function of `(StateView, now)`. Every clock is an
11 + //! argument, so a fixture state renders identically forever and the mapping is
12 + //! testable without a database.
13 + //!
14 + //! # What maps to what
15 + //!
16 + //! | Sando | payload |
17 + //! |---|---|
18 + //! | tier | node, `kind = "tier"` |
19 + //! | deploy target | node, `kind = "node"`, child of its tier |
20 + //! | latest build | node, `kind = "build"` |
21 + //! | gate | condition on its tier |
22 + //! | promote / rollback / confirm | declared action |
23 + //!
24 + //! # Gates are promotion guards, not tier health
25 + //!
26 + //! The load-bearing judgement here. A gate blocks promotion *out of* a tier; it
27 + //! does not describe whether that tier is serving. `burn_in` sits blocked for
28 + //! 48 hours as a matter of routine, and a tier whose status went yellow for two
29 + //! days every release is a tier nobody looks at.
30 + //!
31 + //! So a blocked or in-flight gate is reported as a `pending` condition and
32 + //! leaves the tier `ok`. Only evidence of something actually wrong — a failed
33 + //! gate, or a partial (mixed-version) tier — degrades the tier itself. The why
34 + //! is never lost: it is in the conditions either way.
35 +
36 + use chrono::{DateTime, Utc};
37 + use ops_status::{Action, Condition, Field, Method, Node, Payload, Status, Value};
38 +
39 + use crate::outcome::GateOutcome;
40 + use crate::routes::{GateView, StateView, TierView};
41 +
42 + /// The `source` name Sando answers to in a viewer's config.
43 + pub const SOURCE: &str = "sando";
44 +
45 + /// Restate `/state` as the shared payload.
46 + ///
47 + /// `now` is an argument rather than read from the clock so the mapping stays
48 + /// pure and snapshot-testable.
49 + pub(crate) fn payload(view: &StateView, now: DateTime<Utc>) -> Payload {
50 + let mut payload = Payload::new(SOURCE, now);
51 +
52 + payload.actions.insert(
53 + "rebuild".into(),
54 + Action {
55 + label: "Rebuild".into(),
56 + method: Method::Post,
57 + url: "/rebuild".into(),
58 + confirm: false,
59 + danger: false,
60 + body: None,
61 + },
62 + );
63 +
64 + // Always emitted, even before the first `/rebuild`. Actions reach a viewer
65 + // only by hanging off a node, so a daemon with no build history and no
66 + // build node would offer no way to start one.
67 + payload.nodes.push(build_node(view.build.as_ref()));
68 +
69 + for tier in &view.tiers {
70 + for action in tier_actions(tier) {
71 + payload.actions.insert(action.0, action.1);
72 + }
73 + payload.nodes.push(tier_node(tier));
74 + payload.nodes.extend(node_nodes(tier));
75 + }
76 +
77 + payload
78 + }
79 +
80 + // ---------------------------------------------------------------------------
81 + // Build
82 + // ---------------------------------------------------------------------------
83 +
84 + fn build_node(build: Option<&crate::runs::BuildSummary>) -> Node {
85 + let Some(build) = build else {
86 + return Node {
87 + id: "build".into(),
88 + kind: "build".into(),
89 + label: "latest build".into(),
90 + status: Status::Pending,
91 + fields: Vec::new(),
92 + conditions: vec![Condition {
93 + condition_type: "build".into(),
94 + status: Status::Pending,
95 + since: None,
96 + detail: Some("no build run yet".into()),
97 + }],
98 + children: Vec::new(),
99 + actions: vec!["rebuild".into()],
100 + };
101 + };
102 +
103 + // `build_runs.result` is one of building / passed / failed / aborted. An
104 + // aborted run is degraded rather than failed: the daemon settling an
105 + // orphaned run on restart is not the same event as a build that broke.
106 + let status = match build.result.as_str() {
107 + "passed" => Status::Ok,
108 + "failed" => Status::Failed,
109 + "building" => Status::Pending,
110 + "aborted" => Status::Degraded,
111 + _ => Status::Unknown,
112 + };
113 +
114 + let mut fields = vec![
115 + Field::new(
116 + "sha",
117 + Value::Ident {
118 + value: build.sha.clone(),
119 + abbrev_to: Some(8),
120 + },
121 + ),
122 + Field::new(
123 + "phase",
124 + Value::Text {
125 + value: build.phase.clone(),
126 + },
127 + ),
128 + Field::new(
129 + "elapsed",
130 + Value::Duration {
131 + seconds: build.elapsed_s,
132 + },
133 + ),
134 + ];
135 + if let Some(version) = &build.version {
136 + fields.insert(
137 + 0,
138 + Field::new(
139 + "version",
140 + Value::Version {
141 + value: version.clone(),
142 + },
143 + ),
144 + );
145 + }
146 +
147 + let mut conditions = vec![Condition {
148 + condition_type: "build".into(),
149 + status,
150 + since: None,
151 + detail: Some(format!("run {} {}", build.run_id, build.result)),
152 + }];
153 + if let Some(failure) = &build.failure_summary {
154 + conditions.push(Condition {
155 + condition_type: "build_failure".into(),
156 + status: Status::Failed,
157 + since: None,
158 + detail: Some(failure.clone()),
159 + });
160 + }
161 +
162 + Node {
163 + id: "build".into(),
164 + kind: "build".into(),
165 + label: "latest build".into(),
166 + status,
167 + fields,
168 + conditions,
169 + children: Vec::new(),
170 + actions: vec!["rebuild".into()],
171 + }
172 + }
173 +
174 + // ---------------------------------------------------------------------------
175 + // Tiers
176 + // ---------------------------------------------------------------------------
177 +
178 + fn tier_id(tier: &TierView) -> String {
179 + format!("tier:{}", tier.name)
180 + }
181 +
182 + fn tier_node(tier: &TierView) -> Node {
183 + let mut conditions: Vec<Condition> = tier.gates.iter().map(gate_condition).collect();
184 +
185 + // A partial tier is mid-version and could not be compensated back to
186 + // consistency. It is the loudest thing Sando can report.
187 + if let Some(reason) = &tier.partial_reason {
188 + conditions.insert(
189 + 0,
190 + Condition {
191 + condition_type: "consistent".into(),
192 + status: Status::Failed,
193 + since: None,
194 + detail: Some(reason.clone()),
195 + },
196 + );
197 + }
198 + if !tier.provisioned {
199 + conditions.push(Condition {
200 + condition_type: "provisioned".into(),
201 + status: Status::Pending,
202 + since: None,
203 + detail: Some("declared in topology, not provisioned".into()),
204 + });
205 + }
206 +
207 + let mut fields = vec![Field::new(
208 + "canary",
209 + Value::Text {
210 + value: tier.canary.clone(),
211 + },
212 + )];
213 + if let Some(version) = &tier.current_version {
214 + fields.insert(
215 + 0,
216 + Field::new(
217 + "version",
218 + Value::Version {
219 + value: version.clone(),
220 + },
221 + ),
222 + );
223 + }
224 + if let Some(previous) = &tier.previous_version {
225 + fields.push(Field::new(
226 + "previous",
227 + Value::Version {
228 + value: previous.clone(),
229 + },
230 + ));
231 + }
232 + if let Some(started) = tier.burn_in_started_at.as_deref().and_then(parse_instant) {
233 + fields.push(Field::new(
234 + "burn-in since",
235 + Value::Instant { value: started },
236 + ));
237 + }
238 + if let Some(progress) = burn_in_progress(tier) {
239 + fields.push(progress);
240 + }
241 +
242 + Node {
243 + id: tier_id(tier),
244 + kind: "tier".into(),
245 + label: tier.name.clone(),
246 + status: tier_status(tier),
247 + fields,
248 + conditions,
249 + children: tier.nodes.iter().map(|n| format!("node:{n}")).collect(),
250 + actions: tier_actions(tier).into_iter().map(|(key, _)| key).collect(),
251 + }
252 + }
253 +
254 + /// Whether the tier itself is healthy — deliberately not "are its gates green".
255 + ///
256 + /// See the module docs: gates guard promotion out, so a blocked one is routine
257 + /// and must not color the tier.
258 + fn tier_status(tier: &TierView) -> Status {
259 + if tier.partial_reason.is_some() {
260 + return Status::Failed;
261 + }
262 + if tier
263 + .gates
264 + .iter()
265 + .any(|g| g.status.as_deref() == Some("failed"))
266 + {
267 + return Status::Failed;
268 + }
269 + // An unrecognized gate status is contract drift between daemon and viewer,
270 + // which is worth surfacing rather than smoothing over. The TUI renders it
271 + // magenta for the same reason.
272 + if tier.gates.iter().any(|g| gate_status(g) == Status::Unknown) {
273 + return Status::Degraded;
274 + }
275 + if !tier.provisioned || tier.current_version.is_none() {
276 + return Status::Pending;
277 + }
278 + Status::Ok
279 + }
280 +
281 + /// Burn-in as a bar rather than a sentence, so it sorts and colors like every
282 + /// other progress value in the UI.
283 + fn burn_in_progress(tier: &TierView) -> Option<Field> {
284 + let gate = tier.gates.iter().find(|g| g.kind == "burn_in")?;
285 + let outcome = gate.outcome.clone()?;
286 + let outcome: GateOutcome = serde_json::from_value(outcome).ok()?;
287 +
288 + let (value, max) = match outcome.status {
289 + crate::outcome::GateStatus::Blocked {
290 + blocker:
291 + crate::outcome::GateBlocker::BurnInRemaining {
292 + hours_remaining,
293 + hours_total,
294 + },
295 + } => (
296 + f64::from(hours_total.saturating_sub(hours_remaining)),
297 + f64::from(hours_total),
298 + ),
299 + crate::outcome::GateStatus::Passed {
300 + note: crate::outcome::PassNote::BurnInElapsed { hours },
301 + } => (f64::from(hours), f64::from(hours)),
302 + _ => return None,
303 + };
304 +
305 + Some(Field::new(
306 + "burn-in",
307 + Value::Progress {
308 + value,
309 + max,
310 + unit: Some("hour".into()),
311 + },
312 + ))
313 + }
314 +
315 + /// Promote, roll back, confirm — declared as data so the viewer issues them
316 + /// without knowing what any of them mean.
317 + ///
318 + /// All three carry `confirm` and `danger`. Sando has no flag marking which
319 + /// tiers are production, and inventing one from the tier name would be a
320 + /// heuristic that fails silently the day the topology changes. Over-confirming
321 + /// a deploy is the cheap direction to be wrong in.
322 + fn tier_actions(tier: &TierView) -> Vec<(String, Action)> {
323 + if !tier.provisioned {
324 + return Vec::new();
325 + }
326 + let name = &tier.name;
327 + vec![
328 + (
329 + format!("promote-{name}"),
330 + Action {
331 + label: format!("Promote to {name}"),
332 + method: Method::Post,
333 + url: format!("/promote/{name}"),
334 + confirm: true,
335 + danger: true,
336 + body: None,
337 + },
338 + ),
339 + (
340 + format!("rollback-{name}"),
341 + Action {
342 + label: format!("Roll back {name}"),
343 + method: Method::Post,
344 + url: format!("/rollback/{name}"),
345 + confirm: true,
346 + danger: true,
347 + body: None,
348 + },
349 + ),
350 + (
351 + format!("confirm-{name}"),
352 + Action {
353 + label: format!("Confirm {name}"),
354 + method: Method::Post,
355 + url: format!("/confirm/{name}"),
356 + confirm: true,
357 + danger: true,
358 + body: None,
359 + },
360 + ),
361 + ]
362 + }
363 +
364 + // ---------------------------------------------------------------------------
365 + // Gates
366 + // ---------------------------------------------------------------------------
367 +
368 + /// `passed | failed | blocked`, or NULL while the gate is in flight.
369 + ///
370 + /// `blocked` becomes `pending`: a blocked gate is waiting on something, which
371 + /// is what pending means here. An unrecognized string is `unknown`, not
372 + /// in-flight — that distinction is why the TUI renders it magenta.
373 + fn gate_status(gate: &GateView) -> Status {
374 + match gate.status.as_deref() {
375 + Some("passed") => Status::Ok,
376 + Some("failed") => Status::Failed,
377 + Some("blocked") | None => Status::Pending,
378 + Some(_) => Status::Unknown,
379 + }
380 + }
381 +
382 + fn gate_condition(gate: &GateView) -> Condition {
383 + Condition {
384 + condition_type: gate.kind.clone(),
385 + status: gate_status(gate),
386 + since: gate.finished_at.as_deref().and_then(parse_instant),
387 + detail: gate_detail(gate),
388 + }
389 + }
390 +
391 + /// The why, from the typed outcome when there is one.
392 + ///
393 + /// "blocked" is useless; "blocked because burn_in has 17 hours remaining of 48"
394 + /// is what saves an SSH.
395 + fn gate_detail(gate: &GateView) -> Option<String> {
396 + if gate.status.is_none() {
397 + return Some("running".into());
398 + }
399 + let outcome = gate.outcome.clone()?;
400 + let outcome: GateOutcome = serde_json::from_value(outcome).ok()?;
401 + Some(match outcome.status {
402 + crate::outcome::GateStatus::Passed { note } => note.summary(),
403 + crate::outcome::GateStatus::Failed { failure } => failure.summary(),
404 + crate::outcome::GateStatus::Blocked { blocker } => blocker.summary(),
405 + })
406 + }
407 +
408 + // ---------------------------------------------------------------------------
409 + // Deploy targets
410 + // ---------------------------------------------------------------------------
411 +
412 + /// One node per deploy target, carrying whatever independent health evidence
413 + /// exists for it.
414 + ///
415 + /// `node_health` is the only per-node signal Sando has. When it failed on a
416 + /// named node, that node is failed and its siblings are not. With no evidence
417 + /// either way a node inherits its tier: claiming `unknown` for every node on a
418 + /// tier without the gate would make Sando permanently loud in the rollup for a
419 + /// topology choice, and a viewer that cries wolf is the thing this replaces.
420 + fn node_nodes(tier: &TierView) -> Vec<Node> {
421 + let unhealthy = unhealthy_node(tier);
422 + let health_gate = tier.gates.iter().find(|g| g.kind == "node_health");
423 +
424 + tier.nodes
425 + .iter()
426 + .map(|name| {
427 + let status = if unhealthy.as_deref() == Some(name.as_str()) {
428 + Status::Failed
429 + } else {
430 + match health_gate.map(gate_status) {
431 + Some(Status::Ok) => Status::Ok,
432 + // The gate failed on a *different* node, so this one is
433 + // healthy by the same evidence.
434 + Some(Status::Failed) if unhealthy.is_some() => Status::Ok,
435 + Some(_) | None => tier_status(tier),
436 + }
437 + };
438 +
439 + let mut conditions = Vec::new();
440 + if let Some(gate) = health_gate {
441 + conditions.push(Condition {
442 + condition_type: "node_health".into(),
443 + status,
444 + since: gate.finished_at.as_deref().and_then(parse_instant),
445 + detail: gate_detail(gate),
446 + });
447 + }
448 +
449 + Node {
450 + id: format!("node:{name}"),
451 + kind: "node".into(),
452 + label: name.clone(),
453 + status,
454 + fields: tier
455 + .current_version
456 + .as_ref()
457 + .map(|v| vec![Field::new("version", Value::Version { value: v.clone() })])
458 + .unwrap_or_default(),
459 + conditions,
460 + children: Vec::new(),
461 + actions: Vec::new(),
462 + }
463 + })
464 + .collect()
465 + }
466 +
467 + /// The node named by a `node_health` failure, if that is why the gate is red.
468 + fn unhealthy_node(tier: &TierView) -> Option<String> {
469 + let gate = tier.gates.iter().find(|g| g.kind == "node_health")?;
470 + let outcome: GateOutcome = serde_json::from_value(gate.outcome.clone()?).ok()?;
471 + match outcome.status {
472 + crate::outcome::GateStatus::Failed {
473 + failure: crate::outcome::GateFailure::NodeUnhealthy { node, .. },
474 + } => Some(node),
475 + _ => None,
476 + }
477 + }
478 +
479 + // ---------------------------------------------------------------------------
480 +
481 + /// Timestamps cross the DB boundary as RFC 3339 strings. A row that fails to
482 + /// parse loses its `since` rather than failing the whole payload: a viewer that
483 + /// blanks on one malformed timestamp is worse than one missing a tooltip.
484 + fn parse_instant(raw: &str) -> Option<DateTime<Utc>> {
485 + DateTime::parse_from_rfc3339(raw)
486 + .ok()
487 + .map(|d| d.with_timezone(&Utc))
488 + }
489 +
490 + #[cfg(test)]
491 + mod tests {
492 + use super::*;
493 + use crate::outcome::{GateBlocker, GateFailure, GateStatus, PassNote};
494 +
495 + fn now() -> DateTime<Utc> {
496 + "2026-07-21T18:24:39Z".parse().unwrap()
497 + }
498 +
499 + fn gate(kind: &str, status: Option<&str>, outcome: Option<GateOutcome>) -> GateView {
500 + GateView {
Lines truncated