Skip to main content

max / makenotwork

sando: detached-task promote + opt-in pre-swap config-drift guard Two resilience fixes from the 0.10.12 deploy. /promote now runs its deploy in a detached tokio task and awaits the join handle, instead of inline in the request future. A promote's rsync+restart can outlast a client's socket timeout; running inline meant a disconnect dropped the handler and abandoned the rollout mid-flight. Detaching lets the task own the deploy lock and finish regardless; a connected client still gets the result. (Pairs with the ops-exec rsync kill_on_drop fix.) Node gains an opt-in `config_check_env_file`. When set, deploy_remote sources that env file and runs the freshly-rsynced binary in MNW_CHECK_CONFIG mode BEFORE the symlink swap, so a required var missing on the node (how testnot crash-looped on a missing CDN_BASE_URL) fails the promote with the service still intact, instead of after the swap+restart. Default-unset: never runs against a binary too old to support the mode; enable per node once a check-capable version is deployed. Bounded by a 20s timeout (fail closed). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-08 23:57 UTC
Commit: 942b5bbf1727c5998ac97d61f66c80211e0302fd
Parent: 1800c40
4 files changed, +84 insertions, -1 deletion
M sando/daemon/src/deploy.rs +49
@@ -180,6 +180,19 @@ async fn deploy_remote(
180 180 .await
181 181 .context("deployed binary architecture does not match the target node (current symlink left intact)")?;
182 182
183 + // Config-drift guard (opt-in per node). Runs the freshly-rsynced binary in
184 + // config-only mode with the node's env sourced, BEFORE the swap, so a
185 + // required var missing on this node fails here — service still intact —
186 + // rather than after the restart, which would crash-loop it (how testnot
187 + // went down on a missing CDN_BASE_URL). Skipped unless the node sets
188 + // `config_check_env_file`.
189 + if let Some(env_file) = node.config_check_env_file.as_deref() {
190 + tracing::info!(node = %node.name, version, "deploy: pre-swap config check");
191 + check_target_config(executor, &deployed_bin, env_file)
192 + .await
193 + .context("pre-swap config check failed (current symlink left intact)")?;
194 + }
195 +
183 196 tracing::info!(node = %node.name, version, "deploy: symlink swap + service reload");
184 197 let restart_cmd = format!("sudo /bin/systemctl reload-or-restart {}", sh_quote(service));
185 198 let swap_and_restart = swap_and_restart_script(release_root, version, &restart_cmd);
@@ -192,6 +205,39 @@ async fn deploy_remote(
192 205 Ok(PathBuf::from(release_root).join("releases").join(version))
193 206 }
194 207
208 + /// Pre-swap config-drift check: source the node's env file, then run the
209 + /// freshly-deployed binary in `MNW_CHECK_CONFIG=1` mode (loads config, exits
210 + /// 0/1, no DB/migrations/bind). A non-zero exit — a required var missing — is
211 + /// surfaced by `run_checked` as an error, failing the promote before the swap.
212 + ///
213 + /// Bounded by a timeout as a backstop: a binary predating `MNW_CHECK_CONFIG`
214 + /// would ignore the var and try to start normally, which must not hang the
215 + /// deploy. A timeout is reported as a failure (fail closed) — the operator only
216 + /// opts a node in once a check-capable version is deployed, so a timeout means
217 + /// something is wrong, not a routine older binary.
218 + async fn check_target_config(
219 + executor: &dyn Executor,
220 + deployed_bin: &str,
221 + env_file: &str,
222 + ) -> Result<()> {
223 + // `set -a` exports every var the env file assigns so Config::from_env sees
224 + // them; the binary then validates and exits. Quoted paths; the env file is
225 + // operator-controlled config, not user input.
226 + let script = format!(
227 + "set -eu; set -a; . {env}; set +a; MNW_CHECK_CONFIG=1 {bin}",
228 + env = sh_quote(env_file),
229 + bin = sh_quote(deployed_bin),
230 + );
231 + let fut = run_checked(executor, &script, "pre-swap config check");
232 + match tokio::time::timeout(std::time::Duration::from_secs(20), fut).await {
233 + Ok(result) => result.map(|_| ()),
234 + Err(_) => anyhow::bail!(
235 + "pre-swap config check timed out after 20s — the binary may predate \
236 + MNW_CHECK_CONFIG or the check hung; refusing to swap"
237 + ),
238 + }
239 + }
240 +
195 241 /// Build the swap-and-restart shell script for a remote node.
196 242 ///
197 243 /// The symlink swap is atomic via `mv -T` of a freshly-created symlink over the
@@ -429,6 +475,7 @@ mod tests {
429 475 release_root: "/opt/never".into(),
430 476 service_name: "makenotwork.service".into(),
431 477 health_url: None,
478 + config_check_env_file: None,
432 479 actuate: crate::topology::default_actuate(),
433 480 observe: crate::topology::default_observe(),
434 481 };
@@ -467,6 +514,7 @@ mod tests {
467 514 release_root: release_root.to_string_lossy().into_owned(),
468 515 service_name: "makenotwork.service".into(),
469 516 health_url: None,
517 + config_check_env_file: None,
470 518 actuate: crate::topology::default_actuate(),
471 519 observe: crate::topology::default_observe(),
472 520 };
@@ -593,6 +641,7 @@ mod tests {
593 641 release_root: release_root.to_string_lossy().into_owned(),
594 642 service_name: "makenotwork.service".into(),
595 643 health_url: None,
644 + config_check_env_file: None,
596 645 actuate: vec!["restart".into()], // no deploy
597 646 observe: vec![],
598 647 };
M sando/daemon/src/routes.rs +23 -1
@@ -246,8 +246,27 @@ async fn promote(
246 246 body: Option<Json<PromoteBody>>,
247 247 ) -> Result<Json<serde_json::Value>> {
248 248 let body = body.map(|Json(b)| b).unwrap_or_default();
249 + // Run the deploy in a detached task, not inline in the request future.
250 + // A promote's rsync + restart can outlast a client's socket timeout; if the
251 + // client disconnects, axum drops the handler future — and if the deploy ran
252 + // inline that would abandon the rollout mid-flight (symlink half-swapped,
253 + // tier_state not advanced, and — before the kill_on_drop fix — an orphaned
254 + // rsync). Spawning detaches the work from the connection: the task owns the
255 + // deploy lock and runs to completion regardless, while a still-connected
256 + // client still receives the result. Mirrors how /rebuild already runs its
257 + // build off-request.
258 + tokio::spawn(promote_inner(s, tier, body))
259 + .await
260 + .map_err(|e| crate::error::Error::Other(anyhow::anyhow!("promote task failed to join: {e}")))?
261 + }
262 +
263 + async fn promote_inner(
264 + s: AppState,
265 + tier: String,
266 + body: PromoteBody,
267 + ) -> Result<Json<serde_json::Value>> {
249 268 // Serialize the whole check -> deploy -> advance against any concurrent
250 - // promote/rollback (CF3). Held for the entire handler.
269 + // promote/rollback (CF3). Held for the entire task.
251 270 let _deploy_guard = s.deploy_lock.lock().await;
252 271 let tier = crate::domain::TierId::new(tier);
253 272 let idx = s.topo.tiers.iter().position(|t| t.name == tier)
@@ -1218,6 +1237,7 @@ mod tests {
1218 1237 release_root: "/tmp/a-node".into(),
1219 1238 service_name: "makenotwork.service".into(),
1220 1239 health_url: None,
1240 + config_check_env_file: None,
1221 1241 actuate: crate::topology::default_actuate(),
1222 1242 observe: crate::topology::default_observe(),
1223 1243 }],
@@ -1818,6 +1838,7 @@ mod tests {
1818 1838 release_root: rr.to_string_lossy().into_owned(),
1819 1839 service_name: "x.service".into(),
1820 1840 health_url: None,
1841 + config_check_env_file: None,
1821 1842 actuate: default_actuate(),
1822 1843 observe: default_observe(),
1823 1844 });
@@ -1857,6 +1878,7 @@ mod tests {
1857 1878 release_root: rr.to_string_lossy().into_owned(),
1858 1879 service_name: "x.service".into(),
1859 1880 health_url: None,
1881 + config_check_env_file: None,
1860 1882 actuate: default_actuate(),
1861 1883 observe: default_observe(),
1862 1884 };
M sando/daemon/src/sync.rs +1
@@ -153,6 +153,7 @@ mod tests {
153 153 release_root: "/opt/mnw".into(),
154 154 service_name: "makenotwork.service".into(),
155 155 health_url: None,
156 + config_check_env_file: None,
156 157 actuate: crate::topology::default_actuate(),
157 158 observe: crate::topology::default_observe(),
158 159 }
M sando/daemon/src/topology.rs +11
@@ -51,6 +51,17 @@ pub struct Node {
51 51 /// Defaults to "makenotwork.service" because that's MNW's prod unit.
52 52 #[serde(default = "default_service_name")]
53 53 pub service_name: String,
54 + /// Opt-in config-drift guard. When set to the node's env-file path (e.g.
55 + /// `/etc/mnw/makenotwork.env`), the deploy sources that file and runs the
56 + /// freshly-rsynced binary in `MNW_CHECK_CONFIG=1` mode BEFORE the symlink
57 + /// swap. A required var missing on this node (a var added upstream but never
58 + /// added to the node's env — how testnot crash-looped on CDN_BASE_URL) then
59 + /// fails the promote with the running service still intact, instead of after
60 + /// the swap+restart. Unset (default) skips the check, so an existing
61 + /// `sando.toml` keeps working and it never runs against a binary too old to
62 + /// support the mode; enable it once a check-capable version is deployed.
63 + #[serde(default)]
64 + pub config_check_env_file: Option<String>,
54 65 /// Capability grant for this node's executor (see `ops_exec`). Defaults to
55 66 /// the current behavior of every Sando node — actuate deploy+restart,
56 67 /// observe health — so an existing `sando.toml` keeps working unedited.