Skip to main content

max / makenotwork

sando: add node_health post-deploy gate; boot_smoke is host-only (Gate SM A+, ultra-fuzz Run 2 deep Phase 2) Drive the Gate State Machine axis from A- to A+: - node_health gate (SERIOUS-3): a new post-deploy gate that probes each deployed node's service over its executor (systemctl is-active + an optional per-node health_url curl, retried ~10s) under the read-only Observe(Health) capability. It records the gate_runs row the next promote checks, so a node that took a corrupt artifact / wrong-arch binary / failed restart is caught instead of waved through. - boot_smoke split: boot_smoke no longer runs post-deploy and no longer counts as a promotion gate — it is a build-host readiness gate only. A serving tier declaring only boot_smoke now fails Topology::validate, structurally forcing a real node-level / operator gate (the Run-2 blind spot was boot_smoke re-running the staged binary locally and the next promote trusting a binary that never touched the node). - exhaustive gate dispatch (D4): unsatisfied_gates' `_` catch-all is replaced with explicit Gate arms, so adding a variant is a compile error until its promotion semantics are decided. - sando.toml: tiers a/b/c now gate on node_health; host keeps boot_smoke. - optional Node.health_url for the HTTP readiness probe (unset = is-active only; backwards-compatible). Tests: node_health fail-closed on empty nodes; boot_smoke-only serving tier rejected. clippy -D warnings clean; native fw13. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-23 03:31 UTC
Commit: 675dec133088f19a8e5f899df13e8662596e70a3
Parent: 907b159
10 files changed, +255 insertions, -31 deletions
M sando/daemon/src/build.rs +4
@@ -230,6 +230,10 @@ pub async fn build_and_run_host(
230 230 version: art.version.clone(),
231 231 worktree: art.worktree.clone(),
232 232 events: events.clone(),
233 + // Host runs build-time gates (cargo_test / migration_dry_run /
234 + // boot_smoke) only — `node_health` never appears here, so there are no
235 + // nodes to probe.
236 + nodes: Vec::new(),
233 237 };
234 238 let ok = gates::run_all(&ctx, &host.gates).await?;
235 239
M sando/daemon/src/deploy.rs +3
@@ -428,6 +428,7 @@ mod tests {
428 428 ssh_target: "deploy@192.0.2.1".into(),
429 429 release_root: "/opt/never".into(),
430 430 service_name: "makenotwork.service".into(),
431 + health_url: None,
431 432 actuate: crate::topology::default_actuate(),
432 433 observe: crate::topology::default_observe(),
433 434 };
@@ -465,6 +466,7 @@ mod tests {
465 466 ssh_target: "local".into(),
466 467 release_root: release_root.to_string_lossy().into_owned(),
467 468 service_name: "makenotwork.service".into(),
469 + health_url: None,
468 470 actuate: crate::topology::default_actuate(),
469 471 observe: crate::topology::default_observe(),
470 472 };
@@ -590,6 +592,7 @@ mod tests {
590 592 ssh_target: "local".into(),
591 593 release_root: release_root.to_string_lossy().into_owned(),
592 594 service_name: "makenotwork.service".into(),
595 + health_url: None,
593 596 actuate: vec!["restart".into()], // no deploy
594 597 observe: vec![],
595 598 };
M sando/daemon/src/domain.rs +7
@@ -239,6 +239,11 @@ pub enum GateKind {
239 239 CargoTest,
240 240 MigrationDryRun,
241 241 BootSmoke,
242 + /// Post-deploy readiness of the *deployed nodes* (distinct from `BootSmoke`,
243 + /// which boots the staged artifact on the build host). Probes each node's
244 + /// service over its executor — the gate that actually proves a tier's nodes
245 + /// are serving before the next promote.
246 + NodeHealth,
242 247 BurnIn,
243 248 ManualConfirm,
244 249 }
@@ -249,6 +254,7 @@ impl GateKind {
249 254 GateKind::CargoTest => "cargo_test",
250 255 GateKind::MigrationDryRun => "migration_dry_run",
251 256 GateKind::BootSmoke => "boot_smoke",
257 + GateKind::NodeHealth => "node_health",
252 258 GateKind::BurnIn => "burn_in",
253 259 GateKind::ManualConfirm => "manual_confirm",
254 260 }
@@ -266,6 +272,7 @@ impl FromStr for GateKind {
266 272 "cargo_test" => Ok(GateKind::CargoTest),
267 273 "migration_dry_run" => Ok(GateKind::MigrationDryRun),
268 274 "boot_smoke" => Ok(GateKind::BootSmoke),
275 + "node_health" => Ok(GateKind::NodeHealth),
269 276 "burn_in" => Ok(GateKind::BurnIn),
270 277 "manual_confirm" => Ok(GateKind::ManualConfirm),
271 278 other => Err(GateKindParseError(other.to_owned())),
M sando/daemon/src/gates.rs +130
@@ -25,6 +25,20 @@ pub struct GateCtx {
25 25 pub version: Version,
26 26 pub worktree: PathBuf,
27 27 pub events: EventTx,
28 + /// Nodes the `node_health` post-deploy gate probes. Empty for build-time
29 + /// gate runs on the host (where `node_health` never appears); filled at
30 + /// promote time with each freshly-deployed node and its executor.
31 + pub nodes: Vec<NodeProbe>,
32 + }
33 +
34 + /// One node the `node_health` gate verifies: its id, the systemd unit to
35 + /// confirm active after the restart, an optional HTTP readiness URL, and the
36 + /// executor that reaches it (the same transport the deploy used).
37 + pub struct NodeProbe {
38 + pub node: crate::domain::NodeId,
39 + pub service: String,
40 + pub health_url: Option<String>,
41 + pub executor: Arc<dyn ops_exec::Executor>,
28 42 }
29 43
30 44 /// Run a single gate end-to-end: insert the in-flight row, execute the gate,
@@ -74,6 +88,7 @@ pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> {
74 88 }
75 89 }
76 90 Gate::BootSmoke => boot_smoke(ctx, run_id).await,
91 + Gate::NodeHealth => node_health(ctx).await,
77 92 Gate::BurnIn { hours } => burn_in(ctx, *hours).await,
78 93 Gate::ManualConfirm => manual_confirm(ctx).await,
79 94 };
@@ -669,6 +684,82 @@ async fn probe_health(port: u16) -> std::result::Result<(), String> {
669 684 }
670 685 }
671 686
687 + /// Sink that drops streamed output. `node_health` keeps the probe's stderr from
688 + /// its `RunOutput` for the failure note, so the live byte stream isn't needed.
689 + struct DiscardSink;
690 + #[async_trait::async_trait]
691 + impl ops_exec::LogSink for DiscardSink {
692 + async fn write_chunk(&mut self, _bytes: &[u8]) {}
693 + }
694 +
695 + /// `node_health` — the post-deploy gate that proves the *deployed nodes* are
696 + /// serving, recording one outcome per (tier, version) that the next promote
697 + /// checks. Distinct from `boot_smoke`, which boots the staged artifact on the
698 + /// build host: this probes each node over the same executor the deploy used, so
699 + /// a node that took a corrupt artifact, wrong-arch binary, or failed restart is
700 + /// caught here rather than waved through (Run-2 SERIOUS-3). Fails closed: any
701 + /// unhealthy node fails the gate, and an empty node set is `Blocked`.
702 + async fn node_health(ctx: &GateCtx) -> Result<GateOutcome> {
703 + if ctx.nodes.is_empty() {
704 + return Ok(GateOutcome::blocked(GateBlocker::NoNodesToProbe));
705 + }
706 + for probe in &ctx.nodes {
707 + if let Err(detail) = probe_node(probe).await {
708 + return Ok(GateOutcome::failed(GateFailure::NodeUnhealthy {
709 + node: probe.node.to_string(),
710 + detail,
711 + }));
712 + }
713 + }
714 + Ok(GateOutcome::passed(PassNote::NodesHealthy { nodes: ctx.nodes.len() as u32 }))
715 + }
716 +
717 + /// Probe one node over its executor: confirm the unit is active post-restart
718 + /// and, when a `health_url` is configured, that it serves a 2xx. Retries across
719 + /// ~10s because the service may still be restarting / warming. Runs under the
720 + /// read-only `Observe(Health)` capability (every Sando node grants it), so the
721 + /// probe needs no deploy authority. `Ok(())` = healthy; `Err(detail)` carries a
722 + /// short reason for the gate's failure note.
723 + async fn probe_node(probe: &NodeProbe) -> std::result::Result<(), String> {
724 + use ops_exec::{Action, ObserveKind, Step, sh_quote};
725 + let svc = sh_quote(&probe.service);
726 + let url = probe
727 + .health_url
728 + .as_deref()
729 + .map(sh_quote)
730 + .unwrap_or_else(|| "''".to_string());
731 + // One executor round-trip with the retry loop on the node: is-active, then
732 + // (if a url is set) curl it for a 2xx. Exit 0 only when both hold.
733 + let script = format!(
734 + "svc={svc}; url={url}; \
735 + for _ in $(seq 1 10); do \
736 + if systemctl is-active --quiet \"$svc\"; then \
737 + if [ -z \"$url\" ] || curl -fsS --max-time 5 \"$url\" >/dev/null 2>&1; then exit 0; fi; \
738 + fi; \
739 + sleep 1; \
740 + done; \
741 + echo 'service not active or health url not 2xx after retries' >&2; exit 1"
742 + );
743 + let step = Step::shell(Action::Observe(ObserveKind::Health), script);
744 + let mut sink = DiscardSink;
745 + let out = probe
746 + .executor
747 + .run_streaming(&step, &mut sink)
748 + .await
749 + .map_err(|e| format!("probe spawn: {e}"))?;
750 + if out.status.success() {
751 + Ok(())
752 + } else {
753 + let code = out
754 + .status
755 + .code()
756 + .map(|c| c.to_string())
757 + .unwrap_or_else(|| "signal".to_string());
758 + let stderr: String = String::from_utf8_lossy(&out.stderr).chars().take(200).collect();
759 + Err(format!("exit {code}: {stderr}"))
760 + }
761 + }
762 +
672 763 /// Drain `stream` into the shared `LiveLog` (which forwards each chunk to
673 764 /// the on-disk log file AND broadcasts a `GateLogChunk` event), and return
674 765 /// the concatenated bytes so the classifier can still operate on the full
@@ -901,6 +992,7 @@ mod tests {
901 992 version: "0.1.0".parse().unwrap(),
902 993 worktree: std::path::PathBuf::from("/tmp/unused"),
903 994 events: events::channel(),
995 + nodes: Vec::new(),
904 996 };
905 997 let out = run(&ctx, &Gate::BurnIn { hours: 24 }).await.unwrap();
906 998 assert_eq!(out.status_str(), "blocked");
@@ -916,6 +1008,44 @@ mod tests {
916 1008 assert_eq!(json["status"]["blocker"]["kind"], "burn_in_clock_not_started");
917 1009 }
918 1010
1011 + /// node_health fails closed when there are no nodes to probe: a serving tier
1012 + /// should always carry nodes, so an empty set is a misconfiguration that must
1013 + /// block promotion, not pass it.
1014 + #[tokio::test]
1015 + async fn node_health_blocks_with_no_nodes() {
1016 + let pool = SqlitePoolOptions::new()
1017 + .max_connections(1)
1018 + .connect("sqlite::memory:")
1019 + .await
1020 + .unwrap();
1021 + crate::db::migrate(&pool).await.unwrap();
1022 + sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('b', 2, 1, 'sequential')")
1023 + .execute(&pool).await.unwrap();
1024 + sqlx::query("INSERT INTO tier_state (tier) VALUES ('b')")
1025 + .execute(&pool).await.unwrap();
1026 + sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
1027 + .execute(&pool).await.unwrap();
1028 +
1029 + let cfg = std::sync::Arc::new(crate::config::Config::for_tests());
1030 + let ctx = GateCtx {
1031 + pool: pool.clone(),
1032 + cfg,
1033 + tier: TierId::new("b"),
1034 + version: "0.1.0".parse().unwrap(),
1035 + worktree: std::path::PathBuf::new(),
1036 + events: events::channel(),
1037 + nodes: Vec::new(), // no nodes -> fail closed
1038 + };
1039 + let out = run(&ctx, &Gate::NodeHealth).await.unwrap();
1040 + assert_eq!(out.status_str(), "blocked");
1041 + assert!(!out.is_passed());
1042 + let row: (Option<String>, Option<String>) = sqlx::query_as(
1043 + "SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1",
1044 + ).fetch_one(&pool).await.unwrap();
1045 + let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
1046 + assert_eq!(json["status"]["blocker"]["kind"], "no_nodes_to_probe");
1047 + }
1048 +
919 1049 /// reset_scratch must drop every non-system schema, not just `public` —
920 1050 /// otherwise migrations that create custom schemas (e.g. tower_sessions)
921 1051 /// collide on the next run. This regressed once (Phase 0) and the fix is
M sando/daemon/src/outcome.rs +15
@@ -91,6 +91,10 @@ pub enum PassNote {
91 91 TestsPassed { duration_s: u32 },
92 92 /// `manual_confirm` — an operator inserted a passing row out-of-band.
93 93 OperatorConfirmed { at: DateTime<Utc> },
94 + /// `node_health` — every deployed node's service was active and (where a
95 + /// `health_url` is configured) served its readiness probe. `nodes` is how
96 + /// many nodes were verified.
97 + NodesHealthy { nodes: u32 },
94 98 /// Legacy rows backfilled from the pre-typed schema. Carries the
95 99 /// original `detail` string so nothing is lost.
96 100 Legacy { text: String },
@@ -104,6 +108,7 @@ impl PassNote {
104 108 PassNote::Migrated { backup_path } => format!("restored {backup_path} + migrated"),
105 109 PassNote::TestsPassed { duration_s } => format!("tests passed in {duration_s}s"),
106 110 PassNote::OperatorConfirmed { at } => format!("operator confirmed at {at}"),
111 + PassNote::NodesHealthy { nodes } => format!("{nodes} node(s) healthy"),
107 112 PassNote::Legacy { text } => text.clone(),
108 113 }
109 114 }
@@ -126,6 +131,9 @@ pub enum GateBlocker {
126 131 ScratchDbUrlUnset,
127 132 /// `boot_smoke`: no `artifact_path` in `versions` for this version.
128 133 ArtifactMissing { version: Version },
134 + /// `node_health`: the gate ran with no nodes to probe (a serving tier should
135 + /// always have nodes; this fails closed so a misconfigured tier can't pass).
136 + NoNodesToProbe,
129 137 }
130 138
131 139 impl GateBlocker {
@@ -138,6 +146,7 @@ impl GateBlocker {
138 146 GateBlocker::NoBackupAvailable => "no backup fetched; call /backup/fetch first".into(),
139 147 GateBlocker::ScratchDbUrlUnset => "scratch_db_url unset in daemon config".into(),
140 148 GateBlocker::ArtifactMissing { version } => format!("no artifact for version {version}"),
149 + GateBlocker::NoNodesToProbe => "node_health has no nodes to probe".into(),
141 150 }
142 151 }
143 152 }
@@ -184,6 +193,10 @@ pub enum GateFailure {
184 193 /// the smoke window — started but not ready. `last_error` is the final probe
185 194 /// error (connection refused, non-200, timeout).
186 195 BootHealthProbeFailed { last_error: String },
196 + /// `node_health`: a deployed node's service was not active, or did not serve
197 + /// its `health_url`, within the probe window. `node` is the failing node;
198 + /// `detail` carries the probe's stderr/reason.
199 + NodeUnhealthy { node: String, detail: String },
187 200 /// `cargo_test` / `boot_smoke`: tokio could not spawn the child.
188 201 SpawnFailed { message: String },
189 202 /// Gate took longer than the configured ceiling.
@@ -222,6 +235,8 @@ impl GateFailure {
222 235 GateFailure::BootExitedEarly { exit_code: None } => "binary exited early".into(),
223 236 GateFailure::BootHealthProbeFailed { last_error } =>
224 237 format!("started but never served /health: {last_error}"),
238 + GateFailure::NodeUnhealthy { node, detail } =>
239 + format!("node {node} unhealthy: {detail}"),
225 240 GateFailure::SpawnFailed { message } => format!("spawn: {message}"),
226 241 GateFailure::Timeout { gate, after_s } => format!("{gate} timed out after {after_s}s"),
227 242 GateFailure::Unclassified { legacy_detail: Some(d) } => d.clone(),
M sando/daemon/src/routes.rs +40 -11
@@ -419,27 +419,47 @@ async fn promote(
419 419 });
420 420 }
421 421
422 - // 3b. Run this tier's post-deploy gates (boot_smoke) against the freshly
423 - // deployed artifact and record their outcomes. These rows are the
424 - // evidence the NEXT promote (this tier -> the following one) checks via
422 + // 3b. Run this tier's post-deploy gates (node_health) against the freshly
423 + // deployed nodes and record their outcomes. These rows are the evidence
424 + // the NEXT promote (this tier -> the following one) checks via
425 425 // `unsatisfied_gates`. Before CF1, only the host tier ran gates, so
426 - // A/B/C had no evidence and promotion waved through. burn_in /
427 - // manual_confirm are not run here — they are evaluated live / by the
428 - // operator at the next promote. A failed gate does not unwind this
429 - // deploy (the artifact is already live on the tier); it blocks the next
430 - // promote, which is the fail-closed behavior we want.
426 + // A/B/C had no evidence and promotion waved through; node_health now
427 + // proves the deployed nodes are serving (Run-2 SERIOUS-3: boot_smoke
428 + // used to re-run the staged binary locally and proved nothing about the
429 + // node). burn_in / manual_confirm are not run here — they are evaluated
430 + // live / by the operator at the next promote. A failed gate does not
431 + // unwind this deploy (the artifact is already live on the tier); it
432 + // blocks the next promote, which is the fail-closed behavior we want.
431 433 let post_deploy: Vec<crate::topology::Gate> =
432 434 target.gates.iter().filter(|g| g.runs_post_deploy()).cloned().collect();
433 435 if !post_deploy.is_empty() {
436 + // node_health probes each node the deploy just shipped to, over the same
437 + // executor the deploy used. Build the probe set from the tier's nodes and
438 + // the startup executor map; a node missing an executor (shouldn't happen
439 + // — both come from the same topology) is skipped, and an empty set makes
440 + // node_health Blocked (fail closed).
441 + let nodes: Vec<crate::gates::NodeProbe> = target
442 + .nodes
443 + .iter()
444 + .filter_map(|n| {
445 + s.executors.get(&n.name).map(|exec| crate::gates::NodeProbe {
446 + node: n.name.clone(),
447 + service: n.service_name.clone(),
448 + health_url: n.health_url.clone(),
449 + executor: exec.clone(),
450 + })
451 + })
452 + .collect();
434 453 let ctx = crate::gates::GateCtx {
435 454 pool: s.pool.clone(),
436 455 cfg: s.cfg.clone(),
437 456 tier: target.name.clone(),
438 457 version: version.clone(),
439 - // boot_smoke checks the staged artifact (versions.artifact_path),
440 - // not a worktree; none is available at promote time.
458 + // No worktree at promote time; node_health works over executors, not
459 + // a checkout.
441 460 worktree: std::path::PathBuf::new(),
442 461 events: s.events.clone(),
462 + nodes,
443 463 };
444 464 match crate::gates::run_all(&ctx, &post_deploy).await {
445 465 Ok(true) => {}
@@ -655,7 +675,13 @@ async fn unsatisfied_gates(
655 675 bad.push(kind.as_str().to_string());
656 676 }
657 677 }
658 - _ => {
678 + // Build/post-deploy gates that leave a `gate_runs` row: the latest
679 + // row for this (tier, version, kind) must be `passed`. Listed
680 + // explicitly (no `_` catch-all) so adding a new `Gate` variant is a
681 + // compile error here until its promotion semantics are decided —
682 + // a transient-`blocked` kind silently falling into "needs a passed
683 + // row" would be permanently unsatisfiable.
684 + Gate::CargoTest | Gate::MigrationDryRun | Gate::BootSmoke | Gate::NodeHealth => {
659 685 // Latest row for this configured gate kind; NULL/missing/any
660 686 // non-'passed' status all count as unsatisfied (fail closed).
661 687 let status: Option<String> = sqlx::query_scalar(
@@ -1169,6 +1195,7 @@ mod tests {
1169 1195 ssh_target: "local".into(),
1170 1196 release_root: "/tmp/a-node".into(),
1171 1197 service_name: "makenotwork.service".into(),
1198 + health_url: None,
1172 1199 actuate: crate::topology::default_actuate(),
1173 1200 observe: crate::topology::default_observe(),
1174 1201 }],
@@ -1768,6 +1795,7 @@ mod tests {
1768 1795 ssh_target: "local".into(),
1769 1796 release_root: rr.to_string_lossy().into_owned(),
1770 1797 service_name: "x.service".into(),
1798 + health_url: None,
1771 1799 actuate: default_actuate(),
1772 1800 observe: default_observe(),
1773 1801 });
@@ -1806,6 +1834,7 @@ mod tests {
1806 1834 ssh_target: "local".into(),
1807 1835 release_root: rr.to_string_lossy().into_owned(),
1808 1836 service_name: "x.service".into(),
1837 + health_url: None,
1809 1838 actuate: default_actuate(),
1810 1839 observe: default_observe(),
1811 1840 };
M sando/daemon/src/sync.rs +1
@@ -152,6 +152,7 @@ mod tests {
152 152 ssh_target: format!("deploy@{name}"),
153 153 release_root: "/opt/mnw".into(),
154 154 service_name: "makenotwork.service".into(),
155 + health_url: None,
155 156 actuate: crate::topology::default_actuate(),
156 157 observe: crate::topology::default_observe(),
157 158 }
M sando/daemon/src/topology.rs +41 -17
@@ -58,6 +58,14 @@ pub struct Node {
58 58 pub actuate: Vec<String>,
59 59 #[serde(default = "default_observe")]
60 60 pub observe: Vec<String>,
61 + /// Optional HTTP readiness URL the `node_health` gate curls on the node (over
62 + /// its executor) after `systemctl is-active`. Typically a loopback address
63 + /// the service binds, e.g. `http://127.0.0.1:8080/health`. When unset, the
64 + /// gate proves the unit is active post-restart but does not HTTP-probe; set
65 + /// it for a full readiness check. Kept optional so an existing `sando.toml`
66 + /// needs no edit.
67 + #[serde(default)]
68 + pub health_url: Option<String>,
61 69 }
62 70
63 71 fn default_service_name() -> String { "makenotwork.service".into() }
@@ -91,6 +99,7 @@ pub enum Gate {
91 99 CargoTest,
92 100 MigrationDryRun,
93 101 BootSmoke,
102 + NodeHealth,
94 103 BurnIn { hours: u32 },
95 104 ManualConfirm,
96 105 }
@@ -104,29 +113,34 @@ impl Gate {
104 113 Gate::CargoTest => GateKind::CargoTest,
105 114 Gate::MigrationDryRun => GateKind::MigrationDryRun,
106 115 Gate::BootSmoke => GateKind::BootSmoke,
116 + Gate::NodeHealth => GateKind::NodeHealth,
107 117 Gate::BurnIn { .. } => GateKind::BurnIn,
108 118 Gate::ManualConfirm => GateKind::ManualConfirm,
109 119 }
110 120 }
111 121
112 - /// Gates that execute against a tier's freshly-deployed artifact, recording
113 - /// a `gate_runs` row, at the end of a successful promote to that tier. This
114 - /// is distinct from build-time gates (`cargo_test`, `migration_dry_run`, run
115 - /// once on the host) and from live/operator gates (`burn_in`,
116 - /// `manual_confirm`, evaluated at promote time rather than run). Currently
117 - /// only `boot_smoke` — it boots the staged artifact and needs no worktree.
122 + /// Gates that execute against a tier's freshly-deployed nodes, recording a
123 + /// `gate_runs` row, at the end of a successful promote to that tier — the
124 + /// evidence the *next* promote checks. Only `node_health`: it probes the
125 + /// nodes the deploy just shipped to. `boot_smoke` is NOT post-deploy — it is
126 + /// a build-time gate that boots the staged artifact on the build host and
127 + /// proves nothing about a deployed node (the Run-2 SERIOUS-3 blind spot:
128 + /// boot_smoke used to re-run locally at promote time and the next promote
129 + /// trusted a binary that never touched the node).
118 130 pub fn runs_post_deploy(&self) -> bool {
119 - matches!(self, Gate::BootSmoke)
131 + matches!(self, Gate::NodeHealth)
120 132 }
121 133
122 - /// Gates evaluated at promote time against the deployed artifact or the
134 + /// Gates evaluated at promote time against the deployed nodes or the
123 135 /// operator, as opposed to build-time gates (`cargo_test`,
124 - /// `migration_dry_run`) that run once on the build host and prove nothing
125 - /// about a promote. A serving tier whose gate list contains none of these
126 - /// would wave every promote straight through; `Topology::validate` refuses
127 - /// to load such a config (the structural form of CF1's fail-closed default).
136 + /// `migration_dry_run`, `boot_smoke`) that run once on the build host and
137 + /// prove nothing about a promote. A serving tier whose gate list contains
138 + /// none of these would wave every promote straight through; `Topology
139 + /// ::validate` refuses to load such a config (the structural form of CF1's
140 + /// fail-closed default). `boot_smoke` no longer counts here — a serving tier
141 + /// must declare a real node-level / operator gate, not a host smoke test.
128 142 pub fn guards_promotion(&self) -> bool {
129 - matches!(self, Gate::BootSmoke | Gate::BurnIn { .. } | Gate::ManualConfirm)
143 + matches!(self, Gate::NodeHealth | Gate::BurnIn { .. } | Gate::ManualConfirm)
130 144 }
131 145 }
132 146
@@ -188,7 +202,7 @@ impl Topology {
188 202 if t.provisioned && !is_build_tier && !t.gates.iter().any(Gate::guards_promotion) {
189 203 anyhow::bail!(
190 204 "tier {} is provisioned to serve but declares no promotion gate \
191 - (need at least one of boot_smoke / burn_in / manual_confirm)",
205 + (need at least one of node_health / burn_in / manual_confirm)",
192 206 t.name
193 207 );
194 208 }
@@ -244,11 +258,21 @@ release_root = "/srv/mnw"
244 258
245 259 #[test]
246 260 fn provisioned_serving_tier_with_a_promotion_gate_is_accepted() {
247 - let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
261 + let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
248 262 assert!(topo.validate_for_test().is_ok());
249 263 }
250 264
251 265 #[test]
266 + fn provisioned_serving_tier_with_only_boot_smoke_is_rejected() {
267 + // boot_smoke is a build-host gate now; it proves nothing about a node, so
268 + // a serving tier carrying only boot_smoke must fail closed exactly like an
269 + // empty gate list (Run-2 SERIOUS-3 structural close).
270 + let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
271 + let err = topo.validate_for_test().unwrap_err().to_string();
272 + assert!(err.contains("no promotion gate"), "{err}");
273 + }
274 +
275 + #[test]
252 276 fn unprovisioned_tier_with_empty_gates_is_skipped() {
253 277 // A declared-but-not-yet-provisioned tier (e.g. tier c) carries no
254 278 // promote authority, so the gate requirement does not apply yet.
@@ -260,14 +284,14 @@ release_root = "/srv/mnw"
260 284 fn build_host_matching_a_serving_node_is_rejected() {
261 285 // prod-1 is a node in the provisioned serving tier built above; naming it
262 286 // as the builder must fail closed.
263 - let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
287 + let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
264 288 let err = topo.ensure_build_host_not_serving("prod-1").unwrap_err().to_string();
265 289 assert!(err.contains("must not be a prod/serving node"), "{err}");
266 290 }
267 291
268 292 #[test]
269 293 fn build_host_distinct_from_serving_nodes_is_accepted() {
270 - let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
294 + let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
271 295 assert!(topo.ensure_build_host_not_serving("fw13").is_ok());
272 296 }
273 297
M sando/sando.toml +9 -3
@@ -45,8 +45,11 @@ gates = [
45 45 name = "a"
46 46 provisioned = true
47 47 canary = "sequential"
48 + # node_health (post-deploy) probes the deployed node over its executor — the
49 + # gate boot_smoke used to stand in for, but boot_smoke runs on the build host and
50 + # proves nothing about testnot-1. boot_smoke stays a host build-time gate only.
48 51 gates = [
49 - { kind = "boot_smoke" },
52 + { kind = "node_health" },
50 53 { kind = "burn_in", hours = 48 },
51 54 ]
52 55 [[tier.node]]
@@ -54,6 +57,9 @@ name = "testnot-1"
54 57 ssh_target = "deploy@testnot" # tailnet name; never the public IP / testnot.work hostname
55 58 release_root = "/opt/mnw"
56 59 service_name = "makenotwork.service"
60 + # Set health_url to add an HTTP readiness probe on top of `systemctl is-active`,
61 + # e.g. health_url = "http://127.0.0.1:8080/health" (the local port the service
62 + # binds). Unset = unit-active check only.
57 63
58 64 # ---- B: prod-1 ----
59 65 [[tier]]
@@ -61,7 +67,7 @@ name = "b"
61 67 provisioned = true
62 68 canary = "sequential"
63 69 gates = [
64 - { kind = "boot_smoke" },
70 + { kind = "node_health" },
65 71 { kind = "manual_confirm" },
66 72 ]
67 73 [[tier.node]]
@@ -79,6 +85,6 @@ name = "c"
79 85 provisioned = false
80 86 canary = "sequential"
81 87 gates = [
82 - { kind = "boot_smoke" },
88 + { kind = "node_health" },
83 89 ]
84 90 # [[tier.node]] entries to be added when the second prod node ships.
M sando/tui/src/main.rs +5
@@ -518,6 +518,7 @@ fn pass_note_short(n: &PassNote) -> String {
518 518 }
519 519 PassNote::TestsPassed { duration_s } => format!("{duration_s}s"),
520 520 PassNote::OperatorConfirmed { .. } => "operator".into(),
521 + PassNote::NodesHealthy { nodes } => format!("{nodes} node(s) healthy"),
521 522 PassNote::Legacy { text } => text.clone(),
522 523 }
523 524 }
@@ -545,6 +546,9 @@ fn failure_short(f: &GateFailure) -> String {
545 546 GateFailure::BootHealthProbeFailed { last_error } => {
546 547 format!("no /health: {}", last_error.chars().take(40).collect::<String>())
547 548 }
549 + GateFailure::NodeUnhealthy { node, detail } => {
550 + format!("{node} unhealthy: {}", detail.chars().take(40).collect::<String>())
551 + }
548 552 GateFailure::SpawnFailed { message } => format!("spawn: {message}"),
549 553 GateFailure::Timeout { gate, after_s } => format!("{gate} timeout {after_s}s"),
550 554 GateFailure::Unclassified { legacy_detail: Some(d) } => {
@@ -593,6 +597,7 @@ fn blocker_short(b: &GateBlocker) -> String {
593 597 GateBlocker::NoBackupAvailable => "no backup".into(),
594 598 GateBlocker::ScratchDbUrlUnset => "scratch_db_url unset".into(),
595 599 GateBlocker::ArtifactMissing { version } => format!("no artifact for {version}"),
600 + GateBlocker::NoNodesToProbe => "no nodes to probe".into(),
596 601 }
597 602 }
598 603