Skip to main content

max / makenotwork

Guard companion binaries before the swap, not by their unit failing to start arch_guard_script and ldd_guard_script both took the primary binary alone. A node's [[companion]] binaries were checked by nothing: a companion of the wrong architecture, or one linking a symbol version the node lacks, was discovered by its unit failing during the install loop. That loop runs AFTER the symlink swap and restart, so the promote failed on the expensive side of the line these guards exist to stay on. mnw-cli ships to production exactly that way. Both guards now run against every companion before the swap, at FailureStage::BeforeSwap, and the refusal names which companion. Nothing has to wait for them: the companion bytes arrive in the same rsync and manifest_verify_script already covers the whole release directory, companions/ included. companion_src is one function because the guards and the installer must read the same path. A guard checking a path the installer does not use is a check of nothing, and looks exactly like a passing check; a test pins the two together. The glibc half of this was already closed in 0.2.12, earlier still, because check_bundle_fits_node walks the whole staged bundle. This is the architecture half and the full loader check, which that number comparison cannot reach.
Author: Max Johnson <me@maxj.phd> · 2026-08-25 14:55 UTC
Signed with PGP, not checked
Commit: 2e8c6ee23d75394ca1951a381c57057ef6654967
Parent: dbcdfa0
3 files changed, +164 insertions, -18 deletions
M sando/Cargo.lock +13 -13
@@ -1228,7 +1228,7 @@
1228 1228
1229 1229 [[package]]
1230 1230 name = "sando-daemon"
1231 - version = "0.2.12"
1231 + version = "0.2.13"
1232 1232 dependencies = [
1233 1233 "anyhow",
1234 1234 "async-trait",
@@ -2270,18 +2270,6 @@
2270 2270 name = "docengine"
2271 2271 version = "0.7.0"
2272 2272
2273 - [[patch.unused]]
2274 - name = "quasi-type"
2275 - version = "0.1.0"
2276 -
2277 - [[patch.unused]]
2278 - name = "synckit-client"
2279 - version = "0.9.1"
2280 -
2281 - [[patch.unused]]
2282 - name = "synckit-config"
2283 - version = "0.2.0"
2284 -
2285 2273 [[patch.unused]]
2286 2274 name = "quasi-axum"
2287 2275 version = "0.56.0"
@@ -2318,6 +2306,18 @@
2318 2306 name = "quasi-webview"
2319 2307 version = "0.56.0"
2320 2308
2309 + [[patch.unused]]
2310 + name = "quasi-type"
2311 + version = "0.1.0"
2312 +
2313 + [[patch.unused]]
2314 + name = "synckit-client"
2315 + version = "0.9.1"
2316 +
2317 + [[patch.unused]]
2318 + name = "synckit-config"
2319 + version = "0.2.0"
2320 +
2321 2321 [[patch.unused]]
2322 2322 name = "kberg"
2323 2323 version = "0.1.0"
@@ -1,6 +1,6 @@
1 1 [package]
2 2 name = "sando-daemon"
3 - version = "0.2.12"
3 + version = "0.2.13"
4 4 edition = "2024"
5 5 license = "MIT"
6 6
@@ -567,6 +567,47 @@
567 567 .context("the target node cannot satisfy the deployed binary's dynamic dependencies")
568 568 .context(FailureStage::BeforeSwap)?;
569 569
570 + // The same two guards for every companion, and for the same reason. Both
571 + // checks above take the primary binary alone, so a companion of the wrong
572 + // architecture, or one linking a symbol version this node lacks, used to be
573 + // discovered by its unit failing to start — during the install loop below,
574 + // which runs AFTER the swap. That is the expensive side of the line these
575 + // guards exist to stay on: the promote fails either way, but with the server
576 + // already restarted onto the new release.
577 + //
578 + // The companion bytes are present and verified by now: they arrived in the
579 + // same rsync and `manifest_verify_script` above covers the whole release
580 + // directory, `companions/` included. So there is nothing to wait for.
581 + for c in &node.companions {
582 + let src = companion_src(&release_dir, &c.name);
583 + run_checked(
584 + executor,
585 + &arch_guard_script(&src),
586 + "verifying companion arch matches node",
587 + )
588 + .await
589 + .with_context(|| {
590 + format!(
591 + "companion {} architecture does not match the target node",
592 + c.name
593 + )
594 + })
595 + .context(FailureStage::BeforeSwap)?;
596 + run_checked(
597 + executor,
598 + &ldd_guard_script(&src),
599 + "verifying the node can resolve the companion's dynamic dependencies",
600 + )
601 + .await
602 + .with_context(|| {
603 + format!(
604 + "the target node cannot satisfy companion {}'s dynamic dependencies",
605 + c.name
606 + )
607 + })
608 + .context(FailureStage::BeforeSwap)?;
609 + }
610 +
570 611 // Config-drift guard (opt-in per node). Runs the freshly-rsynced binary in
571 612 // config-only mode with the node's env sourced, BEFORE the swap, so a
572 613 // required var missing on this node fails here — service still intact —
@@ -601,10 +642,7 @@
601 642 // in this SAME bundle — the lockstep guarantee. A failure here fails the
602 643 // promote: a companion is part of the deploy, not a best-effort side effect.
603 644 for c in &node.companions {
604 - let src = format!(
605 - "{release_root}/releases/{release_id}/companions/{name}",
606 - name = c.name,
607 - );
645 + let src = companion_src(&release_dir, &c.name);
608 646 tracing::info!(node = %node.name, companion = %c.name, "deploy: install companion + restart");
609 647 let cmd = install_companion_cmd(&src, &c.install_path, &c.service_name);
610 648 run_checked(executor, &cmd, "install companion + restart")
@@ -627,6 +665,16 @@
627 665 .join(release_id))
628 666 }
629 667
668 + /// Where a companion's binary sits inside the staged release directory.
669 + ///
670 + /// One function because two places need it and they must not drift: the guards
671 + /// that run before the swap check this path, and the installer after the swap
672 + /// reads it. A guard that checked a path the installer did not use would be a
673 + /// check of nothing, and would look exactly like a passing check.
674 + fn companion_src(release_dir: &str, name: &str) -> String {
675 + format!("{release_dir}/companions/{name}")
676 + }
677 +
630 678 /// Absolute path of the node-side companion installer (shipped once per node;
631 679 /// granted to the deploy user by a single scoped sudoers line). It installs the
632 680 /// staged binary to its `ExecStart` path and restarts the unit — keeping the
@@ -2262,6 +2310,104 @@
2262 2310 );
2263 2311 }
2264 2312
2313 + /// A companion is guarded on the same terms as the primary, and BEFORE the
2314 + /// swap. It used to be checked by nothing at all, so the first thing that
2315 + /// noticed a bad companion was its unit failing to start during the install
2316 + /// loop, which runs after the server has already been restarted.
2317 + #[tokio::test]
2318 + async fn companions_are_guarded_before_the_swap() {
2319 + let tmp = tempfile::tempdir().unwrap();
2320 + let staged = tmp.path().join("releases").join("0.9.0");
2321 + tokio::fs::create_dir_all(&staged).await.unwrap();
2322 +
2323 + let node = remote_node(false, vec![companion()]);
2324 + let exec = FakeExec::new();
2325 + deploy_node(
2326 + &exec,
2327 + Placement::check(&node, &staged, None).unwrap(),
2328 + "0.9.0",
2329 + "makenotwork",
2330 + )
2331 + .await
2332 + .unwrap();
2333 +
2334 + let log = exec.log();
2335 + // The companion's own arch and loader checks, named by its path so they
2336 + // cannot be confused with the primary's.
2337 + let guard = pos(&log, "companions/mnw-cli");
2338 + let swap = pos(&log, "reload-or-restart");
2339 + let install = pos(&log, "install-companion.sh");
2340 + assert!(
2341 + guard < swap && swap < install,
2342 + "a companion must be guarded before the swap and installed after it: {log:#?}"
2343 + );
2344 + let companion_guards = log
2345 + .iter()
2346 + .filter(|c| c.contains("companions/mnw-cli") && !c.contains("install-companion.sh"))
2347 + .count();
2348 + assert_eq!(
2349 + companion_guards, 2,
2350 + "both guards must run against the companion, not just one: {log:#?}"
2351 + );
2352 + }
2353 +
2354 + /// And failing one of them fails the promote with the service intact, which
2355 + /// is the whole point of moving the check ahead of the swap.
2356 + #[tokio::test]
2357 + async fn a_companion_failing_its_guard_aborts_before_the_swap() {
2358 + let tmp = tempfile::tempdir().unwrap();
2359 + let staged = tmp.path().join("releases").join("0.9.0");
2360 + tokio::fs::create_dir_all(&staged).await.unwrap();
2361 +
2362 + let node = remote_node(false, vec![companion()]);
2363 + let mut exec = FakeExec::new();
2364 + // Fails the first script naming the companion, which is its arch guard.
2365 + // The primary's guards name the primary and are unaffected.
2366 + exec.fail_run_matching = Some("companions/mnw-cli".into());
2367 + let err = deploy_node(
2368 + &exec,
2369 + Placement::check(&node, &staged, None).unwrap(),
2370 + "0.9.0",
2371 + "makenotwork",
2372 + )
2373 + .await
2374 + .expect_err("a bad companion must fail the deploy");
2375 +
2376 + let msg = format!("{err:#}");
2377 + assert!(
2378 + msg.contains("mnw-cli"),
2379 + "the refusal must name which companion: {msg}"
2380 + );
2381 + assert_eq!(
2382 + stage_of(&err),
2383 + Some(FailureStage::BeforeSwap),
2384 + "a companion guard failing must leave the service intact: {msg}"
2385 + );
2386 + let log = exec.log();
2387 + assert!(
2388 + !log.iter().any(|c| c.contains("reload-or-restart")),
2389 + "swap must not run after a failed companion guard: {log:#?}"
2390 + );
2391 + assert!(
2392 + !log.iter().any(|c| c.contains("install-companion.sh")),
2393 + "nothing should be installed after a failed companion guard: {log:#?}"
2394 + );
2395 + }
2396 +
2397 + /// The guards and the installer must read the same path. A guard checking a
2398 + /// path the installer does not use is a check of nothing, and passes.
2399 + #[test]
2400 + fn the_guarded_companion_path_is_the_one_installed() {
2401 + let release_dir = "/opt/mnw/releases/0.9.0";
2402 + let src = companion_src(release_dir, "mnw-cli");
2403 + assert_eq!(src, "/opt/mnw/releases/0.9.0/companions/mnw-cli");
2404 + let cmd = install_companion_cmd(&src, "/opt/mnw-cli/mnw-cli", "mnw-cli.service");
2405 + assert!(
2406 + cmd.contains(&src),
2407 + "the installer must read the path the guards checked: {cmd}"
2408 + );
2409 + }
2410 +
2265 2411 #[tokio::test]
2266 2412 async fn deploy_remote_installs_companion_after_the_swap() {
2267 2413 // Companions are After= the server: their install must land after the