Skip to main content

max / makenotwork

Refuse a binary the node cannot resolve, before the swap Bento's recipe compared the built binary's highest GLIBC symbol against ldd --version on the service host. It could only do that while it held a [[deploy]] entry naming that host, and a handed-off service has none, so the check came out of pom's recipe when pom moved to the boundary. Which machine runs the bytes is environment knowledge; this is where it lives. It asks the stronger question, because here it can. Bento compared two version numbers from two machines. This runs the node's own loader against the bytes just rsynced onto it, which covers every library and every symbol version rather than glibc alone, and answers whether the thing will exec here rather than whether one number is smaller than another. Sibling of the arch guard and placed beside it: right architecture, right ELF, and still unrunnable for a symbol version the node does not carry. A static binary passes (nothing to resolve) and a node with no ldd passes with a log, which is the call the arch guard already makes for an unmapped arch -- cannot verify is not known bad. Closes sando acd295ad.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-08-09 18:06 UTC
Signed with PGP, not checked
Commit: af10cde612535e19602502ddc52fee08391033e6
Parent: 2818996
1 file changed, +181 insertions, -0 deletions
M sando/daemon/src/deploy.rs +181
@@ -436,6 +436,20 @@
436 436 .context("deployed binary architecture does not match the target node")
437 437 .context(FailureStage::BeforeSwap)?;
438 438
439 + // And that the node can actually resolve what the binary links, which the
440 + // arch check above cannot see: right architecture, right ELF, and still
441 + // unrunnable because it wants a glibc symbol version this box does not have.
442 + // Bento used to catch that at build time; under the Sando/Bento boundary the
443 + // builder no longer knows which machine runs the bytes, so it lands here.
444 + run_checked(
445 + executor,
446 + &ldd_guard_script(&deployed_bin),
447 + "verifying the node can resolve the binary's dynamic dependencies",
448 + )
449 + .await
450 + .context("the target node cannot satisfy the deployed binary's dynamic dependencies")
451 + .context(FailureStage::BeforeSwap)?;
452 +
439 453 // Config-drift guard (opt-in per node). Runs the freshly-rsynced binary in
440 454 // config-only mode with the node's env sourced, BEFORE the swap, so a
441 455 // required var missing on this node fails here — service still intact —
@@ -702,6 +716,57 @@
702 716 )
703 717 }
704 718
719 + /// Refuse a binary whose dynamic dependencies the node cannot satisfy, before
720 + /// the symlink swap.
721 + ///
722 + /// The sibling of [`arch_guard_script`], and it exists because the boundary took
723 + /// the check away from the builder. Bento's recipe used to compare the binary's
724 + /// highest `GLIBC_` symbol against `ldd --version` on the service host, which it
725 + /// could only do while it held a `[[deploy]]` entry naming that host. A
726 + /// handed-off service has none: which machine runs the bytes is environment
727 + /// knowledge, which is Sando's half. So the check moves here, where the node and
728 + /// the artifact are already in the same value.
729 + ///
730 + /// It asks the stronger question, because here it can. Bento compared two
731 + /// version numbers from two machines; this runs the node's own loader against
732 + /// the bytes that were just rsynced onto it. That covers every shared library
733 + /// and every symbol version, not glibc alone, and it answers "will this exec
734 + /// here" rather than "is this number smaller than that one".
735 + ///
736 + /// Three outcomes, and only one of them fails:
737 + ///
738 + /// - `not found` in `ldd` output — a missing library or an unsatisfiable symbol
739 + /// version. This is the failure, and it is exactly what would otherwise be
740 + /// discovered by the unit failing to start after the swap.
741 + /// - not a dynamic executable — `ldd` exits non-zero and says so. A static
742 + /// binary has nothing to resolve, so it passes.
743 + /// - no `ldd` on the node — nothing to check with. Logged and passed: "cannot
744 + /// verify" is not "known bad", the same call `arch_guard_script` makes for an
745 + /// unmapped arch.
746 + ///
747 + /// `ldd` runs the loader, which for an arbitrary binary is code execution. These
748 + /// bytes are ours, already verified against their MANIFEST on this node, and
749 + /// about to be exec'd by the service unit a second later.
750 + fn ldd_guard_script(bin: &str) -> String {
751 + format!(
752 + "set -e\n\
753 + bin={bin}\n\
754 + command -v ldd >/dev/null 2>&1 || {{ echo \"deploy: ldd check skipped (no ldd on node)\" >&2; exit 0; }}\n\
755 + out=$(ldd \"$bin\" 2>&1) || {{ \n\
756 + case \"$out\" in\n\
757 + *\"not a dynamic executable\"*) echo \"deploy: ldd check passed (static binary)\" >&2; exit 0 ;;\n\
758 + *) echo \"deploy: ldd failed on $bin: $out\" >&2; exit 1 ;;\n\
759 + esac\n\
760 + }}\n\
761 + if printf '%s' \"$out\" | grep -q 'not found'; then\n\
762 + echo \"deploy: this node cannot satisfy the binary's dynamic dependencies:\" >&2\n\
763 + printf '%s\\n' \"$out\" | grep 'not found' >&2\n\
764 + exit 1\n\
765 + fi\n",
766 + bin = sh_quote(bin),
767 + )
768 + }
769 +
705 770 async fn gc_local_releases(release_root: &Path) -> Result<()> {
706 771 let releases = release_root.join("releases");
707 772 if !releases.exists() {
@@ -1402,6 +1467,122 @@
1402 1467 );
1403 1468 }
1404 1469
1470 + // ---- ldd_guard_script: a binary this node cannot resolve fails closed ----
1471 +
1472 + /// A fake `ldd` on PATH that prints `body` and exits `code`, so the guard's
1473 + /// three outcomes can be exercised without a binary that genuinely fails to
1474 + /// link. The real `ldd` cannot be made to produce a `not found` on demand.
1475 + async fn run_ldd_guard_with_fake(body: &str, code: i32) -> std::process::Output {
1476 + let dir = tempfile::tempdir().unwrap();
1477 + let fake = dir.path().join("ldd");
1478 + std::fs::write(
1479 + &fake,
1480 + format!("#!/bin/sh\ncat <<'EOF'\n{body}\nEOF\nexit {code}\n"),
1481 + )
1482 + .unwrap();
1483 + let mut perms = std::fs::metadata(&fake).unwrap().permissions();
1484 + std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
1485 + std::fs::set_permissions(&fake, perms).unwrap();
1486 + let bin = dir.path().join("subject");
1487 + std::fs::write(&bin, b"x").unwrap();
1488 + Command::new("sh")
1489 + .arg("-c")
1490 + .arg(ldd_guard_script(&bin.to_string_lossy()))
1491 + .env("PATH", format!("{}:/usr/bin:/bin", dir.path().display()))
1492 + .output()
1493 + .await
1494 + .unwrap()
1495 + }
1496 +
1497 + #[tokio::test]
1498 + async fn ldd_guard_fails_closed_on_an_unsatisfiable_symbol_version() {
1499 + // The exact failure Bento's glibc_check used to catch at build time, and
1500 + // the reason this guard exists: right arch, resolves every library, and
1501 + // still cannot exec because the node's glibc is older than the build
1502 + // host's.
1503 + let out = run_ldd_guard_with_fake(
1504 + "\tlinux-vdso.so.1 (0x00007fff)\n\
1505 + \t/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.40' not found (required by ./pom)\n\
1506 + \tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
1507 + 0,
1508 + )
1509 + .await;
1510 + assert!(
1511 + !out.status.success(),
1512 + "an unsatisfiable symbol version must fail before the symlink swap"
1513 + );
1514 + let stderr = String::from_utf8_lossy(&out.stderr);
1515 + assert!(
1516 + stderr.contains("GLIBC_2.40"),
1517 + "the offending line must reach the operator, not just a verdict: {stderr}"
1518 + );
1519 + }
1520 +
1521 + #[tokio::test]
1522 + async fn ldd_guard_fails_closed_on_a_missing_library() {
1523 + let out = run_ldd_guard_with_fake("\tlibfoo.so.1 => not found", 0).await;
1524 + assert!(!out.status.success(), "a missing library must fail closed");
1525 + }
1526 +
1527 + #[tokio::test]
1528 + async fn ldd_guard_passes_a_resolvable_binary() {
1529 + let out = run_ldd_guard_with_fake(
1530 + "\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
1531 + 0,
1532 + )
1533 + .await;
1534 + assert!(
1535 + out.status.success(),
1536 + "a fully resolved binary must pass: {}",
1537 + String::from_utf8_lossy(&out.stderr),
1538 + );
1539 + }
1540 +
1541 + #[tokio::test]
1542 + async fn ldd_guard_passes_a_static_binary() {
1543 + // ldd exits non-zero for these. Nothing to resolve is not a failure.
1544 + let out = run_ldd_guard_with_fake("\tnot a dynamic executable", 1).await;
1545 + assert!(
1546 + out.status.success(),
1547 + "a static binary has no dependencies to satisfy: {}",
1548 + String::from_utf8_lossy(&out.stderr),
1549 + );
1550 + }
1551 +
1552 + #[tokio::test]
1553 + async fn ldd_guard_fails_when_ldd_errors_for_another_reason() {
1554 + // Not the static case: ldd said something else and exited non-zero. We
1555 + // do not know the binary is fine, so we do not say it is.
1556 + let out = run_ldd_guard_with_fake("ldd: cannot read file", 1).await;
1557 + assert!(
1558 + !out.status.success(),
1559 + "an unexplained ldd failure must not read as a pass"
1560 + );
1561 + }
1562 +
1563 + #[tokio::test]
1564 + async fn ldd_guard_skips_when_the_node_has_no_ldd() {
1565 + // Cannot verify is not known bad, matching arch_guard's unmapped-arch
1566 + // call. PATH holds nothing, so `command -v ldd` finds none.
1567 + let dir = tempfile::tempdir().unwrap();
1568 + let bin = dir.path().join("subject");
1569 + std::fs::write(&bin, b"x").unwrap();
1570 + // Absolute path to the shell: PATH is what this test empties, so
1571 + // resolving `sh` through it would fail before the script ever ran.
1572 + let out = Command::new("/bin/sh")
1573 + .arg("-c")
1574 + .arg(ldd_guard_script(&bin.to_string_lossy()))
1575 + .env("PATH", dir.path().display().to_string())
1576 + .output()
1577 + .await
1578 + .unwrap();
1579 + assert!(
1580 + out.status.success(),
1581 + "a node with no ldd must not fail the deploy: {}",
1582 + String::from_utf8_lossy(&out.stderr),
1583 + );
1584 + }
1585 +
1405 1586 #[tokio::test]
1406 1587 async fn swap_and_restart_first_deploy_failure_has_no_prev_to_restore() {
1407 1588 // No prior `current`. A restart failure leaves `current` at new (the only