Skip to main content

max / makenotwork

sando: verify binary arch at promote; gzip -t the testnot refresh (Run 2) Promote-time arch guard: after rsync and before the symlink swap, compare the deployed binary's ELF e_machine to the node's uname -m and fail closed on a real mismatch. 'Never cross-compile' was enforced only at build time; nothing checked the artifact matched the *target* node, so an aarch64 node added to an x86_64-built tier would silently symlink an unrunnable binary live. Unmapped node arches log and proceed. Arch-agnostic tests cover match + fail-closed. testnot refresh now gzip -t's the prod dump before dropping the staging schema, so a backup truncated after fetch can't feed a corrupt stream into the restore (matches backup.rs verify_backup). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-23 00:57 UTC
Commit: 84c516a707e5ec327e593c3196a8657cc9646360
Parent: 007494e
2 files changed, +86 insertions, -1 deletion
@@ -169,6 +169,17 @@ async fn deploy_remote(
169 169 .await
170 170 .context("rsync failed (current symlink left intact)")?;
171 171
172 + // Fail closed on a wrong-architecture binary before the symlink swap. The
173 + // "never cross-compile" rule is enforced at build time (build_host check),
174 + // but nothing verified the artifact's arch matched the *target* node — so
175 + // adding an aarch64 node to a tier built on x86_64 would silently symlink an
176 + // unrunnable binary live. Compare the deployed binary's ELF e_machine to the
177 + // node's `uname -m`; unknown arches log and proceed (can't verify != known-bad).
178 + let deployed_bin = format!("{release_dir}/{primary_bin}");
179 + run_checked(executor, &arch_guard_script(&deployed_bin), "verifying binary arch matches node")
180 + .await
181 + .context("deployed binary architecture does not match the target node (current symlink left intact)")?;
182 +
172 183 tracing::info!(node = %node.name, version, "deploy: symlink swap + service reload");
173 184 let restart_cmd = format!("sudo /bin/systemctl reload-or-restart {}", sh_quote(service));
174 185 let swap_and_restart = swap_and_restart_script(release_root, version, &restart_cmd);
@@ -217,6 +228,30 @@ fn swap_and_restart_script(release_root: &str, version: &str, restart_cmd: &str)
217 228 )
218 229 }
219 230
231 + /// Shell that aborts (exit 1) if `bin`'s ELF architecture doesn't match the
232 + /// node it's running on. Reads the ELF `e_machine` field (2 bytes LE at offset
233 + /// 18) and compares it to the value implied by `uname -m`. An arch we don't have
234 + /// a mapping for logs and proceeds — the guard exists to catch the concrete
235 + /// x86_64-vs-aarch64 confusion, not to gate genuinely-new targets.
236 + fn arch_guard_script(bin: &str) -> String {
237 + format!(
238 + "set -e\n\
239 + bin={bin}\n\
240 + arch=$(uname -m)\n\
241 + machine=$(od -An -tx1 -j18 -N2 \"$bin\" 2>/dev/null | tr -d ' \\n')\n\
242 + case \"$arch\" in\n\
243 + x86_64|amd64) want=3e00 ;;\n\
244 + aarch64|arm64) want=b700 ;;\n\
245 + *) echo \"deploy: arch check skipped (unmapped node arch $arch)\" >&2; want= ;;\n\
246 + esac\n\
247 + if [ -n \"$want\" ] && [ \"$machine\" != \"$want\" ]; then\n\
248 + echo \"deploy: arch mismatch — node $arch expects e_machine $want but binary has ${{machine:-<unreadable>}}\" >&2\n\
249 + exit 1\n\
250 + fi\n",
251 + bin = sh_quote(bin),
252 + )
253 + }
254 +
220 255 async fn gc_local_releases(release_root: &Path) -> Result<()> {
221 256 let releases = release_root.join("releases");
222 257 if !releases.exists() {
@@ -483,6 +518,52 @@ mod tests {
483 518 );
484 519 }
485 520
521 + // ---- arch_guard_script: wrong-arch artifacts fail closed ----
522 +
523 + /// A 20-byte stub whose ELF e_machine field (offset 18, 2 bytes LE) is set.
524 + fn elf_stub_with_machine(b18: u8, b19: u8) -> tempfile::NamedTempFile {
525 + let mut data = vec![0u8; 20];
526 + data[18] = b18;
527 + data[19] = b19;
528 + let f = tempfile::NamedTempFile::new().unwrap();
529 + std::fs::write(f.path(), &data).unwrap();
530 + f
531 + }
532 +
533 + /// e_machine low byte for the host running the test, if mapped.
534 + fn host_machine_lo() -> Option<u8> {
535 + match std::env::consts::ARCH {
536 + "x86_64" => Some(0x3e),
537 + "aarch64" => Some(0xb7),
538 + _ => None,
539 + }
540 + }
541 +
542 + #[tokio::test]
543 + async fn arch_guard_passes_for_matching_binary() {
544 + let Some(lo) = host_machine_lo() else { return };
545 + let f = elf_stub_with_machine(lo, 0x00);
546 + let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
547 + assert!(
548 + out.status.success(),
549 + "matching arch must pass: {}",
550 + String::from_utf8_lossy(&out.stderr),
551 + );
552 + }
553 +
554 + #[tokio::test]
555 + async fn arch_guard_fails_closed_for_wrong_binary() {
556 + // Use the other arch's e_machine so it can't match the host.
557 + let wrong = match std::env::consts::ARCH {
558 + "x86_64" => 0xb7, // aarch64 binary on an x86_64 node
559 + "aarch64" => 0x3e, // x86_64 binary on an aarch64 node
560 + _ => return,
561 + };
562 + let f = elf_stub_with_machine(wrong, 0x00);
563 + let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
564 + assert!(!out.status.success(), "wrong-arch binary must fail closed before the symlink swap");
565 + }
566 +
486 567 #[tokio::test]
487 568 async fn swap_and_restart_first_deploy_failure_has_no_prev_to_restore() {
488 569 // No prior `current`. A restart failure leaves `current` at new (the only
@@ -36,7 +36,11 @@ if [ -e "$PAUSE_FLAG" ]; then
36 36 fi
37 37
38 38 [ -r "$BACKUP" ] || { echo "backup not readable: $BACKUP" >&2; exit 1; }
39 - log "backup: $BACKUP ($(stat -c %s "$BACKUP") bytes)"
39 + # Verify gzip integrity before we drop the staging schema — a backup truncated
40 + # after sandod fetched it (disk-fill, partial copy) would otherwise feed a corrupt
41 + # stream into the restore. Matches the daemon's backup.rs verify_backup gzip -t.
42 + gzip -t "$BACKUP" || { echo "backup failed gzip integrity check: $BACKUP" >&2; exit 1; }
43 + log "backup: $BACKUP ($(stat -c %s "$BACKUP") bytes, gzip ok)"
40 44
41 45 log "stopping $SERVICE on testnot"
42 46 ts_ssh "systemctl stop $SERVICE"