Skip to main content

max / makenotwork

sando: wire gate wall-clock ceiling (Timeout now live); drop dead StayedUp (Run 2) A hung cargo_test/migration_dry_run gate would block the pipeline until a new /rebuild aborted it — GateFailure::Timeout was defined but never constructed. Add config gate_timeout_secs (default 2400s/40min); cargo_test wraps its full run in tokio::time::timeout (kill_on_drop already set), migration_dry_run is bounded at the dispatcher (its bash restore now sets kill_on_drop so a timeout can't orphan psql). Past the ceiling the gate fails Timeout. Remove PassNote::StayedUp: it was constructed nowhere (HealthyProbe replaced it). The new workspace gate caught the TUI's now-orphaned match arm at build time. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-23 01:03 UTC
Commit: 83486aa9684b5d53deb35e5fc69d17a34f871522
Parent: 84c516a
6 files changed, +74 insertions, -9 deletions
M sando/daemon/src/classify.rs +2 -2
@@ -214,8 +214,8 @@ fn extract_sqlstate(err: &str) -> Option<String> {
214 214 /// `boot_smoke`: process exit info is the dominant signal. If the
215 215 /// binary exited with a status during the smoke window, we map exit
216 216 /// code 101 (Rust default for panic) to `BootPanic`, everything else
217 - /// to `BootExitedEarly`. If it never exited (stayed up), the caller
218 - /// constructs `PassNote::StayedUp` directly without consulting this.
217 + /// to `BootExitedEarly`. If it stayed up and served `/health`, the caller
218 + /// constructs `PassNote::HealthyProbe` directly without consulting this.
219 219 pub fn classify_boot_smoke(exit_code: Option<i32>) -> GateFailure {
220 220 match exit_code {
221 221 Some(101) => GateFailure::BootPanic { exit_code: Some(101) },
M sando/daemon/src/config.rs +22
@@ -56,6 +56,13 @@ pub struct Config {
56 56 /// daemon config so the sando code stays project-agnostic.
57 57 #[serde(default)]
58 58 pub release_contents: Vec<ReleaseEntry>,
59 + /// Wall-clock ceiling (seconds) for the `cargo_test` and `migration_dry_run`
60 + /// gates. A hung suite (deadlocked test, wedged child) otherwise blocks the
61 + /// pipeline until a new `/rebuild` aborts it; past the ceiling the gate is
62 + /// killed and fails with `GateFailure::Timeout`. Default 2400s (40 min) —
63 + /// generous for a full release test suite, fatal only to a genuine hang.
64 + #[serde(default = "default_gate_timeout_secs")]
65 + pub gate_timeout_secs: u64,
59 66 }
60 67
61 68 /// A directory or file copied from the worktree into the staged release dir.
@@ -76,6 +83,7 @@ pub struct ReleaseEntry {
76 83 fn default_bin_names() -> Vec<String> { vec!["server".into()] }
77 84 fn default_logs_root() -> PathBuf { PathBuf::from("/srv/sando/logs") }
78 85 fn default_boot_smoke_port() -> u16 { 18181 }
86 + fn default_gate_timeout_secs() -> u64 { 2400 }
79 87
80 88 impl Config {
81 89 /// Primary binary — the one the systemd unit's ExecStart points at.
@@ -105,6 +113,7 @@ impl Config {
105 113 logs_root: PathBuf::from("/tmp/sando-test-logs"),
106 114 release_contents: Vec::new(),
107 115 cargo_target_dir: None,
116 + gate_timeout_secs: default_gate_timeout_secs(),
108 117 }
109 118 }
110 119 }
@@ -136,6 +145,19 @@ mod tests {
136 145 }
137 146
138 147 #[test]
148 + fn gate_timeout_defaults_when_omitted() {
149 + let cfg: Config = toml::from_str(MINIMAL).unwrap();
150 + assert_eq!(cfg.gate_timeout_secs, 2400, "omitting it keeps the 40-min ceiling");
151 + }
152 +
153 + #[test]
154 + fn gate_timeout_parses_when_present() {
155 + let raw = format!("{MINIMAL}\ngate_timeout_secs = 600\n");
156 + let cfg: Config = toml::from_str(&raw).unwrap();
157 + assert_eq!(cfg.gate_timeout_secs, 600);
158 + }
159 +
160 + #[test]
139 161 fn build_host_is_required() {
140 162 // No safe default: a config without build_host must not parse, so the
141 163 // no-build-on-prod guard can never be silently skipped.
M sando/daemon/src/gates.rs +35 -3
@@ -57,8 +57,22 @@ pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> {
57 57 });
58 58
59 59 let outcome = match gate {
60 + // cargo_test bounds its own run internally (it kills the specific child).
60 61 Gate::CargoTest => cargo_test(ctx, run_id).await,
61 - Gate::MigrationDryRun => migration_dry_run(ctx).await,
62 + // migration_dry_run's psql restore + sqlx migrate could wedge; bound the
63 + // whole gate here. Its bash restore sets kill_on_drop, so a timeout-drop
64 + // doesn't orphan it.
65 + Gate::MigrationDryRun => {
66 + let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
67 + match tokio::time::timeout(ceiling, migration_dry_run(ctx)).await {
68 + Ok(res) => res,
69 + Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout {
70 + gate: GateKind::MigrationDryRun,
71 + after_s: ctx.cfg.gate_timeout_secs as u32,
72 + })
73 + .with_log_ref(LogRef::new(&ctx.version, GateKind::MigrationDryRun))),
74 + }
75 + }
62 76 Gate::BootSmoke => boot_smoke(ctx, run_id).await,
63 77 Gate::BurnIn { hours } => burn_in(ctx, *hours).await,
64 78 Gate::ManualConfirm => manual_confirm(ctx).await,
@@ -165,8 +179,23 @@ async fn cargo_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
165 179 }).with_log_ref(log_ref));
166 180 }
167 181 };
168 - let (stdout_buf, stderr_buf, status) =
169 - stream_child_to_live_log(&mut child, ctx.events.clone(), run_id, log_path).await?;
182 + // Wall-clock ceiling: a hung suite would otherwise block the pipeline until a
183 + // new /rebuild aborts it. Past the ceiling, kill the child and fail Timeout.
184 + let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
185 + let stream = stream_child_to_live_log(&mut child, ctx.events.clone(), run_id, log_path);
186 + let (stdout_buf, stderr_buf, status) = match tokio::time::timeout(ceiling, stream).await {
187 + Ok(res) => res?,
188 + Err(_elapsed) => {
189 + child.start_kill().ok();
190 + let _ = child.wait().await;
191 + let after_s = started.elapsed().as_secs() as u32;
192 + return Ok(GateOutcome::failed(GateFailure::Timeout {
193 + gate: GateKind::CargoTest,
194 + after_s,
195 + })
196 + .with_log_ref(log_ref));
197 + }
198 + };
170 199 let duration_s = started.elapsed().as_secs() as u32;
171 200 if status.success() {
172 201 Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref))
@@ -403,6 +432,9 @@ async fn restore_dump(db_url: &str, dump: &str, log_buf: &mut Vec<u8>) -> Result
403 432 // locally on the Sando host (fw13), which has bash.
404 433 let mut cmd = Command::new("bash");
405 434 cmd.arg("-c").arg(&shell);
435 + // kill_on_drop so the gate's wall-clock ceiling (dispatcher-level timeout on
436 + // migration_dry_run) can't orphan a wedged psql restore.
437 + cmd.kill_on_drop(true);
406 438 if let Some(pw) = password {
407 439 cmd.env("PGPASSWORD", pw);
408 440 }
M sando/daemon/src/outcome.rs +14 -3
@@ -78,8 +78,6 @@ pub enum GateStatus {
78 78 #[derive(Debug, Clone, Serialize, Deserialize)]
79 79 #[serde(tag = "kind", rename_all = "snake_case")]
80 80 pub enum PassNote {
81 - /// `boot_smoke` — the binary stayed up for the smoke window.
82 - StayedUp { duration_s: u32 },
83 81 /// `boot_smoke` — the binary came up and served `GET /health` (readiness,
84 82 /// not just liveness). `after_ms` is how long until the first good probe.
85 83 HealthyProbe { after_ms: u32 },
@@ -101,7 +99,6 @@ pub enum PassNote {
101 99 impl PassNote {
102 100 pub fn summary(&self) -> String {
103 101 match self {
104 - PassNote::StayedUp { duration_s } => format!("stayed up for {duration_s}s"),
105 102 PassNote::HealthyProbe { after_ms } => format!("served /health in {after_ms}ms"),
106 103 PassNote::BurnInElapsed { hours } => format!("{hours} hours elapsed"),
107 104 PassNote::Migrated { backup_path } => format!("restored {backup_path} + migrated"),
@@ -350,6 +347,20 @@ mod tests {
350 347 }
351 348
352 349 #[test]
350 + fn timeout_failure_renders_and_is_not_passed() {
351 + // Timeout is now a live failure (cargo_test / migration_dry_run ceilings),
352 + // not an aspirational variant — keep it mapped in summary() and serde.
353 + let o = GateOutcome::failed(GateFailure::Timeout {
354 + gate: GateKind::CargoTest,
355 + after_s: 2400,
356 + });
357 + assert!(!o.is_passed());
358 + let v: serde_json::Value = serde_json::to_value(&o).unwrap();
359 + assert_eq!(v["status"]["failure"]["kind"], "timeout");
360 + assert!(matches!(&o.status, GateStatus::Failed { failure } if failure.summary().contains("timed out")));
361 + }
362 +
363 + #[test]
353 364 fn blocked_is_not_passed() {
354 365 let o = GateOutcome::blocked(GateBlocker::BurnInClockNotStarted);
355 366 assert!(!o.is_passed());
M sando/daemon/src/routes.rs +1
@@ -1166,6 +1166,7 @@ mod tests {
1166 1166 logs_root: PathBuf::from("/tmp/sando-logs"),
1167 1167 release_contents: vec![],
1168 1168 cargo_target_dir: None,
1169 + gate_timeout_secs: 2400,
1169 1170 }
1170 1171 }
1171 1172
M sando/tui/src/main.rs -1
@@ -474,7 +474,6 @@ fn format_event_body(e: &Event) -> String {
474 474
475 475 fn pass_note_short(n: &PassNote) -> String {
476 476 match n {
477 - PassNote::StayedUp { duration_s } => format!("up {duration_s}s"),
478 477 PassNote::HealthyProbe { after_ms } => format!("/health {after_ms}ms"),
479 478 PassNote::BurnInElapsed { hours } => format!("{hours}h elapsed"),
480 479 PassNote::Migrated { backup_path } => {