Skip to main content

max / makenotwork

sando+server: boot_smoke probes /health on a known port instead of liveness-only boot_smoke was liveness-only (process stayed up 3s = pass). Now it tells the staged artifact which loopback port to bind (SANDO_BOOT_SMOKE_PORT, configurable boot_smoke_port, default 18181) and polls GET /health within the window, passing only on a 200 (PassNote::HealthyProbe) and failing distinctly when the process stays up but never serves (GateFailure::BootHealthProbeFailed). The MNW smoke server reads the port (falls back to an ephemeral bind when unset, preserving prior behavior). The probe is a hand-rolled HTTP/1.0 GET over tokio TcpStream so the daemon gains no runtime HTTP-client dep. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-19 01:27 UTC
Commit: a0a22c5662f1606edaa10c251784ee16daab9305
Parent: c58cbb7
5 files changed, +154 insertions, -19 deletions
M sando/daemon/src/config.rs +8
@@ -22,6 +22,12 @@ pub struct Config {
22 22 /// you care about.
23 23 #[serde(default)]
24 24 pub scratch_db_url: Option<String>,
25 + /// Loopback port the `boot_smoke` gate tells the staged artifact to bind
26 + /// (`SANDO_BOOT_SMOKE_PORT`), then probes `GET /health` on. Lets the gate
27 + /// prove readiness, not just liveness. Fixed rather than ephemeral so the
28 + /// gate knows where to probe; builds are serialized so there's no contention.
29 + #[serde(default = "default_boot_smoke_port")]
30 + pub boot_smoke_port: u16,
25 31 /// Names of cargo bin targets the server crate produces (files under
26 32 /// `target/release/`). First entry is the primary unit (referenced from
27 33 /// the systemd unit's ExecStart). Defaults to `["server"]`; MNW ships
@@ -69,6 +75,7 @@ pub struct ReleaseEntry {
69 75
70 76 fn default_bin_names() -> Vec<String> { vec!["server".into()] }
71 77 fn default_logs_root() -> PathBuf { PathBuf::from("/srv/sando/logs") }
78 + fn default_boot_smoke_port() -> u16 { 18181 }
72 79
73 80 impl Config {
74 81 /// Primary binary — the one the systemd unit's ExecStart points at.
@@ -93,6 +100,7 @@ impl Config {
93 100 workdir: PathBuf::from("/tmp/sando-test-workdir"),
94 101 release_root: PathBuf::from("/tmp/sando-test-release-root"),
95 102 scratch_db_url: None,
103 + boot_smoke_port: default_boot_smoke_port(),
96 104 bin_names: vec!["server".into()],
97 105 logs_root: PathBuf::from("/tmp/sando-test-logs"),
98 106 release_contents: Vec::new(),
M sando/daemon/src/gates.rs +124 -16
@@ -499,19 +499,21 @@ async fn boot_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
499 499 }));
500 500 };
501 501
502 - // Lowest-bar smoke: start the binary and verify it stays up for a few
503 - // seconds without exiting. Panics in main, missing config, port-bind
504 - // failures show up here. Anything more ambitious (probing /healthz on a
505 - // real port) needs server config we don't generically know.
502 + // Readiness smoke: start the binary and confirm it serves `GET /health`
503 + // within the window, not merely that the process stays up. Panics in main,
504 + // missing config, and port-bind failures still surface as an early exit; a
505 + // process that comes up but never serves /health is now its own failure.
506 506 //
507 507 // The server requires DATABASE_URL or it panics on config load before
508 508 // we can observe anything. We point it at the scratch DB (already
509 509 // migrated by the build step and refreshed by migration_dry_run if
510 510 // that gate ran first). SCAN_ENABLED=false skips loading YARA rules
511 511 // from /opt/makenotwork/yara-rules which doesn't exist on the build
512 - // host. Other config has sane optional defaults.
512 + // host. SANDO_BOOT_SMOKE_PORT tells the smoke server which loopback port
513 + // to bind so we know where to probe. Other config has sane optional defaults.
513 514 let mut cmd = tokio::process::Command::new(&bin);
514 515 cmd.env("SANDO_BOOT_SMOKE", "1")
516 + .env("SANDO_BOOT_SMOKE_PORT", ctx.cfg.boot_smoke_port.to_string())
515 517 .env("SCAN_ENABLED", "false")
516 518 .stdout(std::process::Stdio::piped())
517 519 .stderr(std::process::Stdio::piped())
@@ -546,15 +548,45 @@ async fn boot_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
546 548 let stdout_task = tokio::spawn(stream_into_log(child.stdout.take(), log.clone()));
547 549 let stderr_task = tokio::spawn(stream_into_log(child.stderr.take(), log.clone()));
548 550
549 - tokio::time::sleep(std::time::Duration::from_secs(3)).await;
550 -
551 - let exit = child.try_wait()?;
552 - if exit.is_none() {
553 - let _ = child.kill().await;
551 + // Poll readiness across the 3s window instead of a flat sleep: GET /health
552 + // must return 2xx. A crash mid-window short-circuits to the exit-code
553 + // failure path (try_wait below); a process that stays up but never serves
554 + // /health is a distinct readiness failure.
555 + let probe_timeout = std::time::Duration::from_millis(500);
556 + let started = std::time::Instant::now();
557 + let window = std::time::Duration::from_secs(3);
558 + let mut probe_ok_after: Option<u32> = None;
559 + let mut last_probe_err = "never responded".to_string();
560 + let mut early_exit = None;
561 + while started.elapsed() < window {
562 + if let Some(status) = child.try_wait()? {
563 + early_exit = Some(status);
564 + break;
565 + }
566 + match tokio::time::timeout(probe_timeout, probe_health(ctx.cfg.boot_smoke_port)).await {
567 + Ok(Ok(())) => {
568 + probe_ok_after = Some(started.elapsed().as_millis() as u32);
569 + break;
570 + }
571 + Ok(Err(e)) => last_probe_err = e,
572 + Err(_) => last_probe_err = "probe timed out".to_string(),
573 + }
574 + tokio::time::sleep(std::time::Duration::from_millis(150)).await;
554 575 }
555 - // The boot_smoke classifier looks at exit code only — the streamed
556 - // bytes already landed in the live log and the on-disk file for the
557 - // post-mortem reader. Drain the join handles to avoid hangs.
576 +
577 + // Stop the child unless it already exited, then drain the log tasks.
578 + let exit = match early_exit {
579 + Some(status) => Some(status),
580 + None => {
581 + let e = child.try_wait()?;
582 + if e.is_none() {
583 + let _ = child.kill().await;
584 + }
585 + e
586 + }
587 + };
588 + // The streamed bytes already landed in the live log and the on-disk file for
589 + // the post-mortem reader. Drain the join handles to avoid hangs.
558 590 let _ = stdout_task.await;
559 591 let _ = stderr_task.await;
560 592 // Unique owner of the Arc at this point (both tasks dropped their clones).
@@ -562,13 +594,46 @@ async fn boot_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
562 594 mutex.into_inner().close().await;
563 595 }
564 596
565 - match exit {
566 - Some(status) => {
597 + match (exit, probe_ok_after) {
598 + // Exited on its own within the window — a crash/panic/bind failure.
599 + (Some(status), _) => {
567 600 let failure = classify::classify_boot_smoke(status.code());
568 601 Ok(GateOutcome::failed(failure).with_log_ref(log_ref))
569 602 }
570 - None => Ok(GateOutcome::passed(PassNote::StayedUp { duration_s: 3 })
603 + // Stayed up and served /health — readiness proven.
604 + (None, Some(after_ms)) => Ok(GateOutcome::passed(PassNote::HealthyProbe { after_ms })
571 605 .with_log_ref(log_ref)),
606 + // Stayed up but never served /health — started, not ready.
607 + (None, None) => Ok(GateOutcome::failed(GateFailure::BootHealthProbeFailed {
608 + last_error: last_probe_err,
609 + })
610 + .with_log_ref(log_ref)),
611 + }
612 + }
613 +
614 + /// One readiness probe of the boot-smoke server: connect to `127.0.0.1:port`
615 + /// and `GET /health`, returning `Ok(())` only on a `200`. A hand-rolled HTTP/1.0
616 + /// request over a raw `TcpStream` keeps the outbound probe dependency-free
617 + /// (reqwest is dev-only); the smoke server serves the one route over axum, which
618 + /// speaks 1.0. `Err` carries a short reason for the operator's failure note. The
619 + /// caller wraps each call in a timeout.
620 + async fn probe_health(port: u16) -> std::result::Result<(), String> {
621 + use tokio::io::{AsyncReadExt, AsyncWriteExt};
622 + let mut stream = tokio::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, port))
623 + .await
624 + .map_err(|e| format!("connect: {e}"))?;
625 + stream
626 + .write_all(b"GET /health HTTP/1.0\r\nHost: localhost\r\nConnection: close\r\n\r\n")
627 + .await
628 + .map_err(|e| format!("write: {e}"))?;
629 + let mut buf = Vec::new();
630 + stream.read_to_end(&mut buf).await.map_err(|e| format!("read: {e}"))?;
631 + let text = String::from_utf8_lossy(&buf);
632 + let status_line = text.lines().next().unwrap_or("");
633 + if status_line.contains(" 200 ") {
634 + Ok(())
635 + } else {
636 + Err(format!("unexpected status line: {status_line:?}"))
572 637 }
573 638 }
574 639
@@ -733,6 +798,49 @@ mod tests {
733 798 use crate::events;
734 799 use sqlx::sqlite::SqlitePoolOptions;
735 800
801 + /// Spawn a one-shot loopback server that answers the first connection with
802 + /// `status_line` + a tiny body, then closes. Returns the bound port.
803 + async fn oneshot_http(status_line: &'static str) -> u16 {
804 + use tokio::io::{AsyncReadExt, AsyncWriteExt};
805 + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
806 + .await
807 + .unwrap();
808 + let port = listener.local_addr().unwrap().port();
809 + tokio::spawn(async move {
810 + if let Ok((mut sock, _)) = listener.accept().await {
811 + let mut scratch = [0u8; 1024];
812 + let _ = sock.read(&mut scratch).await; // drain the request line
813 + let resp = format!("{status_line}\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok");
814 + let _ = sock.write_all(resp.as_bytes()).await;
815 + }
816 + });
817 + port
818 + }
819 +
820 + #[tokio::test]
821 + async fn probe_health_ok_on_200() {
822 + let port = oneshot_http("HTTP/1.1 200 OK").await;
823 + assert!(probe_health(port).await.is_ok());
824 + }
825 +
826 + #[tokio::test]
827 + async fn probe_health_err_on_non_200() {
828 + let port = oneshot_http("HTTP/1.1 503 Service Unavailable").await;
829 + let err = probe_health(port).await.unwrap_err();
830 + assert!(err.contains("status line"), "{err}");
831 + }
832 +
833 + #[tokio::test]
834 + async fn probe_health_err_on_connection_refused() {
835 + // Bind then drop to get an almost-certainly-free port nothing listens on.
836 + let port = {
837 + let l = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).await.unwrap();
838 + l.local_addr().unwrap().port()
839 + };
840 + let err = probe_health(port).await.unwrap_err();
841 + assert!(err.contains("connect"), "{err}");
842 + }
843 +
736 844 /// burn_in returns a typed Blocked when the clock isn't started; the
737 845 /// runner persists status='blocked' + outcome_json (the json carries
738 846 /// blocker.kind = 'burn_in_clock_not_started').
M sando/daemon/src/outcome.rs +10
@@ -80,6 +80,9 @@ pub enum GateStatus {
80 80 pub enum PassNote {
81 81 /// `boot_smoke` — the binary stayed up for the smoke window.
82 82 StayedUp { duration_s: u32 },
83 + /// `boot_smoke` — the binary came up and served `GET /health` (readiness,
84 + /// not just liveness). `after_ms` is how long until the first good probe.
85 + HealthyProbe { after_ms: u32 },
83 86 /// `burn_in` — the configured number of hours have elapsed since
84 87 /// the gate's clock started.
85 88 BurnInElapsed { hours: u32 },
@@ -99,6 +102,7 @@ impl PassNote {
99 102 pub fn summary(&self) -> String {
100 103 match self {
101 104 PassNote::StayedUp { duration_s } => format!("stayed up for {duration_s}s"),
105 + PassNote::HealthyProbe { after_ms } => format!("served /health in {after_ms}ms"),
102 106 PassNote::BurnInElapsed { hours } => format!("{hours} hours elapsed"),
103 107 PassNote::Migrated { backup_path } => format!("restored {backup_path} + migrated"),
104 108 PassNote::TestsPassed { duration_s } => format!("tests passed in {duration_s}s"),
@@ -179,6 +183,10 @@ pub enum GateFailure {
179 183 BootPanic { exit_code: Option<i32> },
180 184 /// `boot_smoke`: binary exited 0 before the smoke window elapsed.
181 185 BootExitedEarly { exit_code: Option<i32> },
186 + /// `boot_smoke`: binary stayed up but never served `GET /health` 200 within
187 + /// the smoke window — started but not ready. `last_error` is the final probe
188 + /// error (connection refused, non-200, timeout).
189 + BootHealthProbeFailed { last_error: String },
182 190 /// `cargo_test` / `boot_smoke`: tokio could not spawn the child.
183 191 SpawnFailed { message: String },
184 192 /// Gate took longer than the configured ceiling.
@@ -215,6 +223,8 @@ impl GateFailure {
215 223 GateFailure::BootPanic { exit_code: None } => "binary panicked".into(),
216 224 GateFailure::BootExitedEarly { exit_code: Some(c) } => format!("binary exited early: exit {c}"),
217 225 GateFailure::BootExitedEarly { exit_code: None } => "binary exited early".into(),
226 + GateFailure::BootHealthProbeFailed { last_error } =>
227 + format!("started but never served /health: {last_error}"),
218 228 GateFailure::SpawnFailed { message } => format!("spawn: {message}"),
219 229 GateFailure::Timeout { gate, after_s } => format!("{gate} timed out after {after_s}s"),
220 230 GateFailure::Unclassified { legacy_detail: Some(d) } => d.clone(),
M sando/daemon/src/routes.rs +1
@@ -1159,6 +1159,7 @@ mod tests {
1159 1159 workdir: PathBuf::from("/tmp/sando-work"),
1160 1160 release_root: PathBuf::from("/tmp/sando-releases"),
1161 1161 scratch_db_url: None,
1162 + boot_smoke_port: 18181,
1162 1163 bin_names: vec!["makenotwork".into()],
1163 1164 logs_root: PathBuf::from("/tmp/sando-logs"),
1164 1165 release_contents: vec![],
M server/src/main.rs +11 -3
@@ -46,11 +46,19 @@ async fn main() {
46 46 // proves the binary, tokio runtime, axum, and TCP bind all work,
47 47 // then idles until the gate kills it.
48 48 if std::env::var("SANDO_BOOT_SMOKE").is_ok() {
49 - tracing::info!("SANDO_BOOT_SMOKE=1; running minimal smoke server");
49 + // Sando passes SANDO_BOOT_SMOKE_PORT so the gate can actually probe
50 + // GET /health (readiness), not just check the process stays up. A bad
51 + // value or unset falls back to an ephemeral port (liveness-only, the
52 + // historical behavior) rather than failing the smoke for a config typo.
53 + let port = std::env::var("SANDO_BOOT_SMOKE_PORT")
54 + .ok()
55 + .and_then(|p| p.parse::<u16>().ok())
56 + .unwrap_or(0);
57 + tracing::info!(port, "SANDO_BOOT_SMOKE=1; running minimal smoke server");
50 58 let app = axum::Router::new()
51 59 .route("/health", axum::routing::get(|| async { "ok" }));
52 - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await
53 - .expect("smoke: bind 127.0.0.1:0");
60 + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, port)).await
61 + .expect("smoke: bind 127.0.0.1");
54 62 axum::serve(listener, app).await.expect("smoke: serve");
55 63 return;
56 64 }