Skip to main content

max / makenotwork

83.2 KB · 2090 lines History Blame Raw
1 //! Atomic symlink-swap deploys.
2 //!
3 //! Layout on every target (local host, A nodes, B nodes, ...). Release dirs are
4 //! named for their content digest, not the version (wiki note
5 //! `release-artifact-identity`), so a rebuild can never overwrite an earlier
6 //! build's dir in place:
7 //!
8 //! <release_root>/
9 //! releases/
10 //! a1b2c3d4e5f60718/ <- <digest16>
11 //! <bin_name>
12 //! MANIFEST <- <sha256> <relpath> per file, node-verified
13 //! f0e1d2c3b4a59687/
14 //! <bin_name>
15 //! current -> releases/f0e1d2c3b4a59687
16 //!
17 //! `ln -sfn` swaps the symlink. systemd units point at
18 //! `<release_root>/current/<bin_name>` so reload-or-restart picks up the new
19 //! binary without ever pointing at a missing path.
20 //!
21 //! The host-side transport — `ssh` for shell steps, `rsync` for the release
22 //! dir — comes from the shared [`ops_exec::Executor`] (a `LocalExec` for
23 //! `ssh_target = "local"`, an `SshExec` otherwise), built once per node in
24 //! [`crate::state`]. This module owns the *deploy choreography* (mkdir, push,
25 //! atomic swap, restart, gc); the transport is the crate's. SSH push behavior
26 //! is identical to the pre-extraction code — this is a transport extraction,
27 //! not a model change.
28
29 use crate::domain::Platform;
30 use crate::topology::Node;
31 use anyhow::{Context, Result};
32 use async_trait::async_trait;
33 use ops_exec::{Action, Executor, LogSink, RunOutput, Step, SyncOpts, sh_quote};
34 use std::path::{Path, PathBuf};
35 use tokio::process::Command;
36
37 /// A staged bundle proven to be for the node it is about to be pushed to.
38 ///
39 /// This exists because "ship aarch64 bytes to an x86_64 box" was, until pom, a
40 /// mistake nobody could make: one product, one build host, one architecture, so
41 /// the pairing of a bundle and a node was correct by having no alternative. pom
42 /// has two architectures under one version, so the pairing becomes a real
43 /// choice, and a wrong choice deploys a binary the node cannot exec.
44 ///
45 /// The answer is not a check before the call. A check is something a later
46 /// caller forgets, and the failure it guards is discovered by a production node
47 /// failing to start. [`Placement::check`] is the *only* way to obtain one of
48 /// these, and [`deploy_node`] takes one instead of a loose `(node, dir)` pair —
49 /// so a mismatched deploy is not a bug the code has to avoid, it is a value the
50 /// code cannot construct.
51 #[derive(Debug, Clone)]
52 pub struct Placement<'a> {
53 node: &'a Node,
54 bundle: &'a Path,
55 }
56
57 /// Why a bundle may not be placed on a node.
58 ///
59 /// All four cases are refusals, including both "one side said nothing" cases.
60 /// Silence is not agreement: a node that does not state its platform cannot
61 /// vouch that it runs a bundle built for a stated one, and a bundle that does
62 /// not state its platform cannot satisfy a node that requires one. The only
63 /// admissible pairing besides a match is both sides silent, which is the
64 /// single-platform world Sando lived in and MNW still lives in.
65 #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
66 pub enum PlacementError {
67 #[error(
68 "node {node} runs {node_platform} and this bundle was built for {artifact_platform}; \
69 refusing to deploy a binary the node cannot execute"
70 )]
71 Mismatch {
72 node: String,
73 node_platform: Platform,
74 artifact_platform: Platform,
75 },
76 #[error(
77 "node {node} does not declare a platform, and this bundle was built for \
78 {artifact_platform}. Declare `platform` on the node so the two can be compared"
79 )]
80 NodeSilent {
81 node: String,
82 artifact_platform: Platform,
83 },
84 #[error(
85 "node {node} requires {node_platform} and this bundle records no platform. \
86 An artifact whose platform is unknown cannot be shown to satisfy one that is"
87 )]
88 ArtifactSilent {
89 node: String,
90 node_platform: Platform,
91 },
92 }
93
94 impl<'a> Placement<'a> {
95 /// The one constructor. `artifact` is the platform the bundle records, which
96 /// for an accepted artifact comes from its `ArtifactRecord` provenance and
97 /// for a Sando-built one comes from the app config.
98 pub fn check(
99 node: &'a Node,
100 bundle: &'a Path,
101 artifact: Option<&Platform>,
102 ) -> Result<Self, PlacementError> {
103 match (node.platform.as_ref(), artifact) {
104 (Some(n), Some(a)) if n == a => Ok(Self { node, bundle }),
105 (Some(n), Some(a)) => Err(PlacementError::Mismatch {
106 node: node.name.to_string(),
107 node_platform: n.clone(),
108 artifact_platform: a.clone(),
109 }),
110 (None, Some(a)) => Err(PlacementError::NodeSilent {
111 node: node.name.to_string(),
112 artifact_platform: a.clone(),
113 }),
114 (Some(n), None) => Err(PlacementError::ArtifactSilent {
115 node: node.name.to_string(),
116 node_platform: n.clone(),
117 }),
118 // Both silent: the single-platform world. MNW is here, and stays
119 // here until its nodes declare a platform — at which point its
120 // builds have to as well, which is the forcing function rather than
121 // a silently mixed state.
122 (None, None) => Ok(Self { node, bundle }),
123 }
124 }
125
126 pub fn node(&self) -> &'a Node {
127 self.node
128 }
129
130 pub fn bundle(&self) -> &'a Path {
131 self.bundle
132 }
133 }
134
135 /// Keep this many release dirs per node; older ones get gc'd after a
136 /// successful deploy. Fixed for now; promote to config if the constant ever
137 /// needs to vary by tier.
138 const RELEASES_TO_KEEP: usize = 5;
139
140 /// A sink that drops streamed bytes. Deploy steps don't have a live-log handle
141 /// (gates do), so output is discarded as it streams; [`RunOutput`] still
142 /// captures the full stdout/stderr for error reporting, preserving the
143 /// pre-extraction behavior of surfacing `stderr` in failure messages.
144 struct DiscardSink;
145
146 #[async_trait]
147 impl LogSink for DiscardSink {
148 async fn write_chunk(&mut self, _bytes: &[u8]) {}
149 }
150
151 /// Run a shell step through `executor`, treating a non-zero exit as an error
152 /// whose message carries the captured stderr — exactly as the old bespoke
153 /// `ssh()` helper did (`ssh <target> failed: <stderr>`).
154 async fn run_checked(executor: &dyn Executor, script: &str, what: &str) -> Result<RunOutput> {
155 let step = Step::shell(Action::Deploy, script);
156 let mut sink = DiscardSink;
157 let out = executor
158 .run_streaming(&step, &mut sink)
159 .await
160 .with_context(|| format!("{what}: spawning command"))?;
161 anyhow::ensure!(
162 out.status.success(),
163 "{what} failed (exit {}): {}",
164 out.status
165 .code()
166 .map_or_else(|| "signal".into(), |c| c.to_string()),
167 String::from_utf8_lossy(&out.stderr),
168 );
169 Ok(out)
170 }
171
172 /// Stage built binaries into `staging/<build_id>/` on the Sando host — a
173 /// private, mutable scratch dir that is not yet a release (no symlink, no gc).
174 /// The caller adds `release_contents` + companions, hashes the result, writes
175 /// the `MANIFEST`, then publishes it content-addressed via
176 /// [`finalize_local_release`]. Splitting staging from publish is what lets the
177 /// bundle be hashed before it is named (wiki [[release-artifact-identity]]).
178 ///
179 /// A stale `staging/<build_id>` from a killed prior run at the same id is
180 /// removed first, so a retry stages clean.
181 pub async fn stage_local_bundle(
182 release_root: &Path,
183 build_id: i64,
184 binaries: &[PathBuf],
185 ) -> Result<PathBuf> {
186 let staging = release_root.join("staging").join(build_id.to_string());
187 if tokio::fs::try_exists(&staging).await.unwrap_or(false) {
188 tokio::fs::remove_dir_all(&staging)
189 .await
190 .with_context(|| format!("clearing stale staging dir {}", staging.display()))?;
191 }
192 tokio::fs::create_dir_all(&staging).await?;
193 for binary in binaries {
194 let name = binary.file_name().context("binary path has no file name")?;
195 let dest = staging.join(name);
196 tokio::fs::copy(binary, &dest)
197 .await
198 .with_context(|| format!("copy {} -> {}", binary.display(), dest.display()))?;
199 }
200 Ok(staging)
201 }
202
203 /// Publish a fully-staged bundle content-addressed: rename
204 /// `staging/<build_id>` to `releases/<digest16>` (atomic same-filesystem
205 /// rename), flip `current` to it, gc old releases. Returns the released dir.
206 ///
207 /// The rename is the load-bearing step: **a directory whose name derives from
208 /// its contents cannot be rewritten, because rewriting it changes its name.**
209 /// The overwrite class that let a dev rebuild inherit an earlier build's gate
210 /// rows and burn-in clock stops being something to guard against and becomes
211 /// something that cannot be expressed. If a release with this digest already
212 /// exists (identical bytes rebuilt), the staging copy is redundant and dropped.
213 pub async fn finalize_local_release(
214 release_root: &Path,
215 staging: &Path,
216 digest16: &str,
217 ) -> Result<PathBuf> {
218 let releases = release_root.join("releases");
219 tokio::fs::create_dir_all(&releases).await?;
220 let released = releases.join(digest16);
221
222 if tokio::fs::try_exists(&released).await.unwrap_or(false) {
223 // Same digest already published — reuse it, discard the redundant stage.
224 tokio::fs::remove_dir_all(staging).await.ok();
225 } else {
226 tokio::fs::rename(staging, &released)
227 .await
228 .with_context(|| format!("publish {} -> {}", staging.display(), released.display()))?;
229 }
230
231 let current = release_root.join("current");
232 let target = format!("releases/{digest16}");
233 let out = Command::new("ln")
234 .args(["-sfn", &target])
235 .arg(&current)
236 .output()
237 .await?;
238 anyhow::ensure!(
239 out.status.success(),
240 "symlink swap failed: {}",
241 String::from_utf8_lossy(&out.stderr),
242 );
243
244 if let Err(e) = gc_local_releases(release_root).await {
245 tracing::warn!(error = %e, "local release GC failed (non-fatal)");
246 }
247 Ok(released)
248 }
249
250 /// Deploy a [`Placement`]'s bundle to its node using `executor` (the node's
251 /// transport from the topology executor map). For `ssh_target=local`, this is
252 /// just a symlink swap; for remote nodes, we rsync the whole dir over the
253 /// executor.
254 ///
255 /// The bundle and the node arrive together inside the placement, so there is no
256 /// signature here that accepts a bundle and a node that were never compared.
257 ///
258 /// `primary_bin` is only used for logging — every file present in the staged
259 /// dir gets shipped.
260 pub async fn deploy_node(
261 executor: &dyn Executor,
262 placement: Placement<'_>,
263 version: &str,
264 primary_bin: &str,
265 ) -> Result<PathBuf> {
266 let node = placement.node();
267 let staged_release_dir = placement.bundle();
268 // The release dir is named for its content digest (`releases/<digest16>`),
269 // not the version. The node mirrors that name so host and node agree on the
270 // artifact's identity; the version is only a log label here. Legacy staged
271 // dirs (pre-identity, still `releases/<version>`) work unchanged — the name
272 // is whatever the host staged under.
273 let release_id = staged_release_dir
274 .file_name()
275 .and_then(|n| n.to_str())
276 .with_context(|| {
277 format!(
278 "staged release dir {} has no usable name",
279 staged_release_dir.display()
280 )
281 })?;
282 if node.ssh_target == "local" || node.ssh_target.is_empty() {
283 // Local deploy already happened when we staged on the Sando host.
284 // Just re-point `current` at the staged dir.
285 return reset_local_current(executor, Path::new(&node.release_root), release_id).await;
286 }
287 deploy_remote(
288 executor,
289 node,
290 version,
291 release_id,
292 staged_release_dir,
293 primary_bin,
294 )
295 .await
296 }
297
298 /// Where a node deploy failed, relative to the symlink swap.
299 ///
300 /// The distinction is the whole difference between "nothing happened" and "go
301 /// look at production now", and it used to be carried only in the wording of a
302 /// `.context()` string, which meant the reporting layer could not act on it. It
303 /// reported every rollback failure as though the node were stranded on the new
304 /// version — including the case where the node had never left the old one,
305 /// which is the safe case and the common one.
306 ///
307 /// Attached as `anyhow` context, so it both reads correctly in the error chain
308 /// and can be recovered with `stage_of`.
309 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
310 pub enum FailureStage {
311 /// Failed before the swap ran. `current` still points at the old release
312 /// and the service was never restarted, so the node is on the OLD version.
313 /// Nothing is stranded and nothing needs doing.
314 BeforeSwap,
315 /// Failed at or after the swap. The node's version is not knowable from
316 /// here: the swap script rolls `current` back if the restart fails, but a
317 /// failure between the two, or in a companion after the server is already
318 /// live, can leave the node on either version.
319 AtOrAfterSwap,
320 }
321
322 impl std::fmt::Display for FailureStage {
323 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
324 match self {
325 Self::BeforeSwap => {
326 f.write_str("current symlink left intact; node is on the previous version")
327 }
328 Self::AtOrAfterSwap => {
329 f.write_str("the symlink swap had already run; node version is indeterminate")
330 }
331 }
332 }
333 }
334
335 /// Recover the [`FailureStage`] from a deploy error's context chain.
336 ///
337 /// `None` means the error predates the stage annotation or came from somewhere
338 /// that does not set one. Callers must treat that as indeterminate rather than
339 /// as safe: guessing "before the swap" would reintroduce the bug in the
340 /// opposite, worse direction.
341 pub fn stage_of(err: &anyhow::Error) -> Option<FailureStage> {
342 // anyhow's own downcast_ref searches attached context values, not just the
343 // source chain, which is where a `.context(FailureStage::…)` lands.
344 err.downcast_ref::<FailureStage>().copied()
345 }
346
347 async fn reset_local_current(
348 executor: &dyn Executor,
349 release_root: &Path,
350 release_id: &str,
351 ) -> Result<PathBuf> {
352 let current = release_root.join("current");
353 let target = format!("releases/{release_id}");
354 run_checked(
355 executor,
356 &format!(
357 "ln -sfn {} {}",
358 sh_quote(&target),
359 sh_quote(&current.to_string_lossy())
360 ),
361 "local symlink swap",
362 )
363 .await?;
364 Ok(release_root.join("releases").join(release_id))
365 }
366
367 async fn deploy_remote(
368 executor: &dyn Executor,
369 node: &Node,
370 version: &str,
371 release_id: &str,
372 staged_release_dir: &Path,
373 primary_bin: &str,
374 ) -> Result<PathBuf> {
375 let release_root = &node.release_root;
376 let service = &node.service_name;
377 let release_dir = format!("{release_root}/releases/{release_id}");
378
379 tracing::info!(node = %node.name, version, release_id, "deploy: mkdir release dir");
380 run_checked(
381 executor,
382 &format!("set -e; mkdir -p {q}", q = sh_quote(&release_dir)),
383 "creating remote release dir",
384 )
385 .await
386 .context(FailureStage::BeforeSwap)?;
387
388 tracing::info!(node = %node.name, version, primary = %primary_bin, "deploy: rsync release dir");
389 // Rsync the whole staged dir (binaries + every release_contents entry).
390 // `SyncOpts::release_mirror()` is the exact pre-extraction rsync flag set:
391 // -az --partial --delete --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r,F+X.
392 // --delete: removed assets across versions don't accumulate on the
393 // target; the bundle stays self-contained per version.
394 // --chmod: F+X preserves the execute bit per-file (binaries land 0755,
395 // data files 0644) instead of a blanket 0755.
396 executor
397 .push_dir(
398 staged_release_dir,
399 Path::new(&release_dir),
400 &SyncOpts::release_mirror(),
401 )
402 .await
403 .context("rsync failed")
404 .context(FailureStage::BeforeSwap)?;
405
406 // Verify the bundle on the node against its own MANIFEST before the swap
407 // (invariant 3, wiki [[release-artifact-identity]]). The MANIFEST shipped in
408 // the bundle is exactly `sha256sum` check format (`<hash> <relpath>`), so
409 // this re-hashes every file on the node and names any that drifted in
410 // transit — the hash is load-bearing, not merely recorded. Bundles staged
411 // by a pre-identity build carry no MANIFEST; those skip verification (logged)
412 // rather than fail, so a mid-migration deploy of a legacy artifact still
413 // ships. A mismatch fails the promote with the running service intact.
414 run_checked(
415 executor,
416 &manifest_verify_script(&release_dir),
417 "verifying bundle digest on node",
418 )
419 .await
420 .context("node-side bundle verification failed")
421 .context(FailureStage::BeforeSwap)?;
422
423 // Fail closed on a wrong-architecture binary before the symlink swap. The
424 // "never cross-compile" rule is enforced at build time (build_host check),
425 // but nothing verified the artifact's arch matched the *target* node — so
426 // adding an aarch64 node to a tier built on x86_64 would silently symlink an
427 // unrunnable binary live. Compare the deployed binary's ELF e_machine to the
428 // node's `uname -m`; unknown arches log and proceed (can't verify != known-bad).
429 let deployed_bin = format!("{release_dir}/{primary_bin}");
430 run_checked(
431 executor,
432 &arch_guard_script(&deployed_bin),
433 "verifying binary arch matches node",
434 )
435 .await
436 .context("deployed binary architecture does not match the target node")
437 .context(FailureStage::BeforeSwap)?;
438
439 // And that the node can actually resolve what the binary links, which the
440 // arch check above cannot see: right architecture, right ELF, and still
441 // unrunnable because it wants a glibc symbol version this box does not have.
442 // Bento used to catch that at build time; under the Sando/Bento boundary the
443 // builder no longer knows which machine runs the bytes, so it lands here.
444 run_checked(
445 executor,
446 &ldd_guard_script(&deployed_bin),
447 "verifying the node can resolve the binary's dynamic dependencies",
448 )
449 .await
450 .context("the target node cannot satisfy the deployed binary's dynamic dependencies")
451 .context(FailureStage::BeforeSwap)?;
452
453 // Config-drift guard (opt-in per node). Runs the freshly-rsynced binary in
454 // config-only mode with the node's env sourced, BEFORE the swap, so a
455 // required var missing on this node fails here — service still intact —
456 // rather than after the restart, which would crash-loop it (how testnot
457 // went down on a missing CDN_BASE_URL). Skipped unless the node sets
458 // `config_check_env_file`.
459 if let Some(env_file) = node.config_check_env_file.as_deref() {
460 tracing::info!(node = %node.name, version, "deploy: pre-swap config check");
461 check_target_config(executor, &deployed_bin, env_file)
462 .await
463 .context("pre-swap config check failed")
464 .context(FailureStage::BeforeSwap)?;
465 }
466
467 tracing::info!(node = %node.name, version, "deploy: symlink swap + service reload");
468 let restart_cmd = format!(
469 "sudo /bin/systemctl reload-or-restart {}",
470 sh_quote(service)
471 );
472 let swap_and_restart = swap_and_restart_script(release_root, release_id, &restart_cmd);
473 run_checked(
474 executor,
475 &swap_and_restart,
476 "symlink swap + systemctl reload-or-restart",
477 )
478 .await
479 .context(FailureStage::AtOrAfterSwap)?;
480
481 // Companion services (opt-in per node): install each from the just-rsynced
482 // bundle and restart its unit via the node-side wrapper, AFTER the server is
483 // up (mnw-cli is `After=makenotwork.service`). They shipped from the SAME sha
484 // in this SAME bundle — the lockstep guarantee. A failure here fails the
485 // promote: a companion is part of the deploy, not a best-effort side effect.
486 for c in &node.companions {
487 let src = format!(
488 "{release_root}/releases/{release_id}/companions/{name}",
489 name = c.name,
490 );
491 tracing::info!(node = %node.name, companion = %c.name, "deploy: install companion + restart");
492 let cmd = install_companion_cmd(&src, &c.install_path, &c.service_name);
493 run_checked(executor, &cmd, "install companion + restart")
494 .await
495 .with_context(|| {
496 format!(
497 "companion {} deploy failed (server already swapped)",
498 c.name
499 )
500 })
501 .context(FailureStage::AtOrAfterSwap)?;
502 }
503
504 if let Err(e) = gc_remote_releases(executor, release_root).await {
505 tracing::warn!(error = %e, "remote release GC failed (non-fatal)");
506 }
507
508 Ok(PathBuf::from(release_root)
509 .join("releases")
510 .join(release_id))
511 }
512
513 /// Absolute path of the node-side companion installer (shipped once per node;
514 /// granted to the deploy user by a single scoped sudoers line). It installs the
515 /// staged binary to its `ExecStart` path and restarts the unit — keeping the
516 /// sudo grant to one script rather than a broad `install`/`systemctl` grant.
517 const COMPANION_INSTALLER: &str = "/usr/local/lib/mnw/install-companion.sh";
518
519 /// Command run on the node to install a staged companion binary and restart its
520 /// unit, via the wrapper. Pure builder so it can be unit-tested; all three args
521 /// are shell-quoted (paths/unit names, operator config — but quoted regardless).
522 fn install_companion_cmd(src: &str, install_path: &str, service: &str) -> String {
523 format!(
524 "sudo {installer} {src} {dst} {svc}",
525 installer = sh_quote(COMPANION_INSTALLER),
526 src = sh_quote(src),
527 dst = sh_quote(install_path),
528 svc = sh_quote(service),
529 )
530 }
531
532 /// Pre-swap config-drift check: load the node's env file the way systemd loads
533 /// it, then run the freshly-deployed binary in `MNW_CHECK_CONFIG=1` mode (loads
534 /// config, exits 0/1, no DB/migrations/bind). A non-zero exit — a required var
535 /// missing — is surfaced by `run_checked` as an error, failing the promote
536 /// before the swap.
537 ///
538 /// Bounded by a timeout as a backstop: a binary predating `MNW_CHECK_CONFIG`
539 /// would ignore the var and try to start normally, which must not hang the
540 /// deploy. A timeout is reported as a failure (fail closed) — the operator only
541 /// opts a node in once a check-capable version is deployed, so a timeout means
542 /// something is wrong, not a routine older binary.
543 async fn check_target_config(
544 executor: &dyn Executor,
545 deployed_bin: &str,
546 env_file: &str,
547 ) -> Result<()> {
548 // Readability first, as its own step with its own message.
549 //
550 // The env file is read by this check AS THE DEPLOY USER, and it is the only
551 // thing that does. systemd loads `EnvironmentFile=` as root before dropping
552 // to `User=`, so the running service does not care about the mode — which
553 // means a file rewritten 0600 breaks the next deploy while the current one
554 // keeps serving, and the breakage is invisible until someone ships. That is
555 // exactly how prod deploy 0.11.3 failed on 2026-08-01.
556 //
557 // Without this step the operator gets `bash: line 9: <file>: Permission
558 // denied` out of a generated script and has to reverse-engineer which user
559 // and which file. Naming the user, the mode and the owner turns that into a
560 // one-line read.
561 let probe = readability_probe_script(env_file);
562 if let Ok(Err(e)) = tokio::time::timeout(
563 std::time::Duration::from_secs(20),
564 run_checked(executor, &probe, "env file readability"),
565 )
566 .await
567 {
568 return Err(e).context(format!(
569 "the deploy user cannot read {env_file}. systemd reads EnvironmentFile= as root, so \
570 the running service is unaffected and this breaks only deploys. Expected mode 0640 \
571 owned root:<service user> (see sando/deploy/bootstrap-node.sh); something that \
572 rewrote the file likely did so with a 077 umask"
573 ));
574 }
575
576 let script = config_check_script(env_file, deployed_bin);
577 let fut = run_checked(executor, &script, "pre-swap config check");
578 match tokio::time::timeout(std::time::Duration::from_secs(20), fut).await {
579 Ok(result) => result.map(|_| ()),
580 Err(_) => anyhow::bail!(
581 "pre-swap config check timed out after 20s — the binary may predate \
582 MNW_CHECK_CONFIG or the check hung; refusing to swap"
583 ),
584 }
585 }
586
587 /// Assert the deploy user can read `env_file`, reporting who it is and what the
588 /// file actually looks like when it cannot.
589 ///
590 /// `stat` output is best-effort: a node without it (or a file that does not
591 /// exist) still gets the identity line, which is the half an operator cannot
592 /// derive from the failure on their own.
593 fn readability_probe_script(env_file: &str) -> String {
594 format!(
595 "if [ ! -e {env} ]; then\n\
596 \techo \"{env_disp}: does not exist on this node\" >&2; exit 1\n\
597 fi\n\
598 if [ ! -r {env} ]; then\n\
599 \techo \"cannot read {env_disp} as $(id -un) (groups: $(id -Gn))\" >&2\n\
600 \tstat -c 'actual: mode %a owner %U:%G' {env} >&2 2>/dev/null || true\n\
601 \texit 1\n\
602 fi\n",
603 env = sh_quote(env_file),
604 env_disp = env_file,
605 )
606 }
607
608 /// Shell that loads `env_file` with systemd `EnvironmentFile=` semantics, then
609 /// runs `bin` under `MNW_CHECK_CONFIG=1`.
610 ///
611 /// Load the file line by line and `export` each `KEY=VALUE` verbatim rather than
612 /// `. env_file`. Dot-sourcing runs the file as a script, so any shell
613 /// metacharacter in a value (`$`, backticks, `;`, `&`, a glob, whitespace) is
614 /// expanded or word-split — a DB URL carrying a password silently dropped
615 /// `DATABASE_URL` to empty on our nodes, which would fail the check (and thus
616 /// every deploy) even though systemd starts the service fine. `export "$line"`
617 /// assigns the already-expanded word literally, matching systemd's "no variable
618 /// expansion" rule. Comments and blank lines are skipped; the `|| [ -n "$line" ]`
619 /// guard processes a final line with no trailing newline. (Quoted values —
620 /// `KEY="v"` — aren't unquoted here the way systemd would, but our env files use
621 /// bare `KEY=VALUE`, and a stray quote can only make the check stricter, never
622 /// wave a bad config through.)
623 fn config_check_script(env_file: &str, bin: &str) -> String {
624 format!(
625 "set -eu\n\
626 while IFS= read -r __sando_l || [ -n \"$__sando_l\" ]; do\n\
627 \tcase \"$__sando_l\" in ''|'#'*) continue ;; esac\n\
628 \texport \"$__sando_l\"\n\
629 done < {env}\n\
630 MNW_CHECK_CONFIG=1 {bin}\n",
631 env = sh_quote(env_file),
632 bin = sh_quote(bin),
633 )
634 }
635
636 /// Build the swap-and-restart shell script for a remote node.
637 ///
638 /// The symlink swap is atomic via `mv -T` of a freshly-created symlink over the
639 /// old one (the rename(2) is the atomic step; `ln -sfn` alone does
640 /// unlink+symlink, which has a window). The load-bearing part: if `restart_cmd`
641 /// fails *after* the flip, `current` is rolled back to its prior target before
642 /// the script exits non-zero. Otherwise a failed restart would leave `current`
643 /// pointing at the new, un-activated release while the service still runs the
644 /// old one — and a later reboot/cron restart would then silently bring up the
645 /// release the deploy reported as failed. Best-effort re-restart of the prior
646 /// version keeps the running service consistent with the restored symlink.
647 ///
648 /// `restart_cmd` is injected (rather than hardcoded) so tests can drive the
649 /// failure and success paths with a `false`/`true` stand-in.
650 fn swap_and_restart_script(release_root: &str, release_id: &str, restart_cmd: &str) -> String {
651 format!(
652 "set -e\n\
653 cd {root}\n\
654 prev=$(readlink current 2>/dev/null || true)\n\
655 ln -sfn releases/{rel} current.new\n\
656 mv -Tf current.new current\n\
657 if ! {restart}; then\n\
658 if [ -n \"$prev\" ]; then\n\
659 ln -sfn \"$prev\" current.rollback\n\
660 mv -Tf current.rollback current\n\
661 {restart} || true\n\
662 fi\n\
663 echo \"deploy: restart failed; rolled symlink back to ${{prev:-<none>}}\" >&2\n\
664 exit 1\n\
665 fi\n",
666 root = sh_quote(release_root),
667 rel = sh_quote(release_id),
668 restart = restart_cmd,
669 )
670 }
671
672 /// Shell that re-hashes the rsynced bundle on the node against its shipped
673 /// `MANIFEST` and aborts (exit 1) if any file drifted (invariant 3, wiki note
674 /// `release-artifact-identity`). The `MANIFEST` is `sha256sum` check format,
675 /// so `sha256sum -c` verifies every listed file with node-native tooling and
676 /// names the one that failed. `--strict` fails on a malformed manifest line;
677 /// `--quiet` drops the per-file OK spam and keeps only failures.
678 ///
679 /// A bundle staged by a pre-identity build carries no `MANIFEST`; that is not an
680 /// error — it logs a skip and exits 0, so a mid-migration deploy of a legacy
681 /// artifact still ships. Once every tier has cycled once, every bundle has one.
682 fn manifest_verify_script(release_dir: &str) -> String {
683 format!(
684 "set -e\n\
685 cd {dir}\n\
686 if [ ! -f MANIFEST ]; then\n\
687 echo \"deploy: no MANIFEST in bundle; skipping digest verification (legacy artifact)\" >&2\n\
688 exit 0\n\
689 fi\n\
690 sha256sum --quiet --strict -c MANIFEST\n",
691 dir = sh_quote(release_dir),
692 )
693 }
694
695 /// Shell that aborts (exit 1) if `bin`'s ELF architecture doesn't match the
696 /// node it's running on. Reads the ELF `e_machine` field (2 bytes LE at offset
697 /// 18) and compares it to the value implied by `uname -m`. An arch we don't have
698 /// a mapping for logs and proceeds — the guard exists to catch the concrete
699 /// x86_64-vs-aarch64 confusion, not to gate genuinely-new targets.
700 fn arch_guard_script(bin: &str) -> String {
701 format!(
702 "set -e\n\
703 bin={bin}\n\
704 arch=$(uname -m)\n\
705 machine=$(od -An -tx1 -j18 -N2 \"$bin\" 2>/dev/null | tr -d ' \\n')\n\
706 case \"$arch\" in\n\
707 x86_64|amd64) want=3e00 ;;\n\
708 aarch64|arm64) want=b700 ;;\n\
709 *) echo \"deploy: arch check skipped (unmapped node arch $arch)\" >&2; want= ;;\n\
710 esac\n\
711 if [ -n \"$want\" ] && [ \"$machine\" != \"$want\" ]; then\n\
712 echo \"deploy: arch mismatch — node $arch expects e_machine $want but binary has ${{machine:-<unreadable>}}\" >&2\n\
713 exit 1\n\
714 fi\n",
715 bin = sh_quote(bin),
716 )
717 }
718
719 /// Refuse a binary whose dynamic dependencies the node cannot satisfy, before
720 /// the symlink swap.
721 ///
722 /// The sibling of [`arch_guard_script`], and it exists because the boundary took
723 /// the check away from the builder. Bento's recipe used to compare the binary's
724 /// highest `GLIBC_` symbol against `ldd --version` on the service host, which it
725 /// could only do while it held a `[[deploy]]` entry naming that host. A
726 /// handed-off service has none: which machine runs the bytes is environment
727 /// knowledge, which is Sando's half. So the check moves here, where the node and
728 /// the artifact are already in the same value.
729 ///
730 /// It asks the stronger question, because here it can. Bento compared two
731 /// version numbers from two machines; this runs the node's own loader against
732 /// the bytes that were just rsynced onto it. That covers every shared library
733 /// and every symbol version, not glibc alone, and it answers "will this exec
734 /// here" rather than "is this number smaller than that one".
735 ///
736 /// Three outcomes, and only one of them fails:
737 ///
738 /// - `not found` in `ldd` output — a missing library or an unsatisfiable symbol
739 /// version. This is the failure, and it is exactly what would otherwise be
740 /// discovered by the unit failing to start after the swap.
741 /// - not a dynamic executable — `ldd` exits non-zero and says so. A static
742 /// binary has nothing to resolve, so it passes.
743 /// - no `ldd` on the node — nothing to check with. Logged and passed: "cannot
744 /// verify" is not "known bad", the same call `arch_guard_script` makes for an
745 /// unmapped arch.
746 ///
747 /// `ldd` runs the loader, which for an arbitrary binary is code execution. These
748 /// bytes are ours, already verified against their MANIFEST on this node, and
749 /// about to be exec'd by the service unit a second later.
750 fn ldd_guard_script(bin: &str) -> String {
751 format!(
752 "set -e\n\
753 bin={bin}\n\
754 command -v ldd >/dev/null 2>&1 || {{ echo \"deploy: ldd check skipped (no ldd on node)\" >&2; exit 0; }}\n\
755 out=$(ldd \"$bin\" 2>&1) || {{ \n\
756 case \"$out\" in\n\
757 *\"not a dynamic executable\"*) echo \"deploy: ldd check passed (static binary)\" >&2; exit 0 ;;\n\
758 *) echo \"deploy: ldd failed on $bin: $out\" >&2; exit 1 ;;\n\
759 esac\n\
760 }}\n\
761 if printf '%s' \"$out\" | grep -q 'not found'; then\n\
762 echo \"deploy: this node cannot satisfy the binary's dynamic dependencies:\" >&2\n\
763 printf '%s\\n' \"$out\" | grep 'not found' >&2\n\
764 exit 1\n\
765 fi\n",
766 bin = sh_quote(bin),
767 )
768 }
769
770 async fn gc_local_releases(release_root: &Path) -> Result<()> {
771 let releases = release_root.join("releases");
772 if !releases.exists() {
773 return Ok(());
774 }
775 let mut entries = Vec::new();
776 let mut rd = tokio::fs::read_dir(&releases).await?;
777 while let Some(entry) = rd.next_entry().await? {
778 if !entry.file_type().await?.is_dir() {
779 continue;
780 }
781 let meta = entry.metadata().await?;
782 entries.push((entry.path(), meta.modified()?));
783 }
784 entries.sort_by_key(|e| std::cmp::Reverse(e.1));
785 for (path, _) in entries.into_iter().skip(RELEASES_TO_KEEP) {
786 if let Err(e) = tokio::fs::remove_dir_all(&path).await {
787 tracing::warn!(path = %path.display(), error = %e, "gc: rm failed");
788 } else {
789 tracing::debug!(path = %path.display(), "gc: removed old release");
790 }
791 }
792 Ok(())
793 }
794
795 async fn gc_remote_releases(executor: &dyn Executor, release_root: &str) -> Result<()> {
796 // `ls -t` orders by mtime desc. Skip the first N, rm the rest. `xargs -r`
797 // is a no-op when stdin is empty (avoids `rm` complaining).
798 let script = format!(
799 "set -e; cd {root}/releases 2>/dev/null || exit 0; \
800 ls -1t | tail -n +{keep_plus_one} | xargs -r -I{{}} rm -rf -- {{}}",
801 root = sh_quote(release_root),
802 keep_plus_one = RELEASES_TO_KEEP + 1,
803 );
804 run_checked(executor, &script, "remote release gc")
805 .await
806 .map(|_| ())
807 }
808
809 #[cfg(test)]
810 mod tests {
811 use super::*;
812 use crate::topology::NodeCompanion;
813 use ops_exec::{CapabilitySet, LocalExec, SshExec};
814 use std::os::unix::process::ExitStatusExt;
815 use std::sync::{Arc, Mutex as StdMutex};
816 use std::time::SystemTime;
817
818 // ---- placement ----
819 //
820 // The whole table, because the interesting cases are the two where one side
821 // said nothing. Treating silence as agreement is how a wrong-architecture
822 // deploy would get through, and it is the shape a "check it before you call"
823 // guard tends to end up with.
824
825 fn node_on(platform: Option<&str>) -> Node {
826 Node {
827 name: crate::domain::NodeId::new("n1"),
828 ssh_target: "deploy@n1".into(),
829 release_root: "/opt/x".into(),
830 platform: platform.map(|p| Platform::parse(p).unwrap()),
831 service_name: "x.service".into(),
832 config_check_env_file: None,
833 actuate: crate::topology::default_actuate(),
834 observe: crate::topology::default_observe(),
835 health_url: None,
836 companions: Vec::new(),
837 }
838 }
839
840 #[test]
841 fn matching_platforms_are_placeable() {
842 let node = node_on(Some("linux/aarch64"));
843 let art = Platform::parse("linux/aarch64").unwrap();
844 let p = Placement::check(&node, Path::new("/r/abc"), Some(&art)).expect("a match places");
845 assert_eq!(p.bundle(), Path::new("/r/abc"));
846 assert_eq!(p.node().name.as_str(), "n1");
847 }
848
849 #[test]
850 fn a_different_architecture_is_refused() {
851 // The failure this type exists for: pom's aarch64 bundle reaching the
852 // x86_64 box, which execs nothing and takes the watcher down.
853 let node = node_on(Some("linux/x86_64"));
854 let art = Platform::parse("linux/aarch64").unwrap();
855 let err = Placement::check(&node, Path::new("/r/abc"), Some(&art)).unwrap_err();
856 assert!(
857 matches!(err, PlacementError::Mismatch { .. }),
858 "expected a mismatch, got {err}"
859 );
860 // The message has to name both, or an operator cannot tell which half
861 // is wrong.
862 let msg = err.to_string();
863 assert!(
864 msg.contains("linux/x86_64") && msg.contains("linux/aarch64"),
865 "{msg}"
866 );
867 }
868
869 #[test]
870 fn a_silent_node_refuses_a_stated_artifact() {
871 // Not "the node probably runs it". A node that never said what it is
872 // cannot vouch for a bundle that did, and the pairing that looks
873 // harmless here is exactly the one that ships the wrong half of a
874 // two-architecture release.
875 let node = node_on(None);
876 let art = Platform::parse("linux/aarch64").unwrap();
877 assert!(matches!(
878 Placement::check(&node, Path::new("/r/abc"), Some(&art)),
879 Err(PlacementError::NodeSilent { .. })
880 ));
881 }
882
883 #[test]
884 fn a_stated_node_refuses_a_silent_artifact() {
885 let node = node_on(Some("linux/aarch64"));
886 assert!(matches!(
887 Placement::check(&node, Path::new("/r/abc"), None),
888 Err(PlacementError::ArtifactSilent { .. })
889 ));
890 }
891
892 #[test]
893 fn both_silent_is_the_single_platform_world_and_still_places() {
894 // MNW is here and stays here. Its nodes declare nothing and its builds
895 // record nothing, which is the truth about a product with one build host
896 // and one architecture. The moment either side starts stating, the other
897 // has to as well — that is the forcing function, and it is why this cell
898 // is the only admissible non-match.
899 let node = node_on(None);
900 Placement::check(&node, Path::new("/r/abc"), None).expect("the pre-pom world still ships");
901 }
902
903 #[test]
904 fn platform_parsing_is_a_shape_not_a_spelling() {
905 assert_eq!(
906 Platform::parse("Linux/AArch64").unwrap(),
907 Platform::parse("linux/aarch64").unwrap(),
908 "case is not a distinction between two machines"
909 );
910 for bad in ["linux", "linux/", "/aarch64", "linux/aarch64/gnu", ""] {
911 assert!(Platform::parse(bad).is_err(), "{bad:?} should not parse");
912 }
913 }
914
915 // ---- failure stage ----
916 //
917 // The 2026-08-01 prod deploy failed its pre-swap config check, and the
918 // rollback then failed the same way — which left the node safely on the old
919 // version, and was reported as "it remains on the new version, manual
920 // intervention needed". These pin the distinction the reporting layer now
921 // depends on.
922
923 #[test]
924 fn a_pre_swap_failure_is_recoverable_as_such() {
925 let e = anyhow::anyhow!("Permission denied")
926 .context("pre-swap config check failed")
927 .context(FailureStage::BeforeSwap);
928 assert_eq!(stage_of(&e), Some(FailureStage::BeforeSwap));
929 // The reason survives alongside the stage; the stage does not replace it.
930 let rendered = format!("{e:#}");
931 assert!(
932 rendered.contains("pre-swap config check failed"),
933 "{rendered}"
934 );
935 assert!(rendered.contains("Permission denied"), "{rendered}");
936 }
937
938 #[test]
939 fn a_post_swap_failure_is_recoverable_as_such() {
940 let e = anyhow::anyhow!("unit failed to start")
941 .context("companion x deploy failed (server already swapped)")
942 .context(FailureStage::AtOrAfterSwap);
943 assert_eq!(stage_of(&e), Some(FailureStage::AtOrAfterSwap));
944 }
945
946 #[test]
947 fn an_unannotated_failure_has_no_stage() {
948 // Must be None, not a default. A caller seeing None has to treat the
949 // node as indeterminate; inferring "before the swap" would reintroduce
950 // the original bug pointing the other way, which is the dangerous way.
951 let e = anyhow::anyhow!("something older, from before stages existed");
952 assert_eq!(stage_of(&e), None);
953 }
954
955 // ---- env file readability probe ----
956
957 #[tokio::test]
958 async fn readability_probe_passes_on_a_readable_file() {
959 let tmp = tempfile::tempdir().unwrap();
960 let f = tmp.path().join("ok.env");
961 tokio::fs::write(&f, "A=1\n").await.unwrap();
962 let script = readability_probe_script(&f.to_string_lossy());
963 let out = run_checked(&local_executor(), &script, "probe").await;
964 assert!(out.is_ok(), "{:?}", out.err().map(|e| format!("{e:#}")));
965 }
966
967 #[tokio::test]
968 async fn readability_probe_names_the_user_and_mode_when_unreadable() {
969 // Root can read anything, so a mode-based test would pass spuriously
970 // there. Skip rather than assert something false. No libc dependency
971 // for one probe: a 0-mode temp file is readable iff we are root.
972 let probe_dir = tempfile::tempdir().unwrap();
973 let probe_file = probe_dir.path().join("root-check");
974 tokio::fs::write(&probe_file, "x").await.unwrap();
975 tokio::fs::set_permissions(
976 &probe_file,
977 std::os::unix::fs::PermissionsExt::from_mode(0o000),
978 )
979 .await
980 .unwrap();
981 if tokio::fs::read(&probe_file).await.is_ok() {
982 return; // running as root
983 }
984 let tmp = tempfile::tempdir().unwrap();
985 let f = tmp.path().join("locked.env");
986 tokio::fs::write(&f, "A=1\n").await.unwrap();
987 tokio::fs::set_permissions(&f, std::os::unix::fs::PermissionsExt::from_mode(0o000))
988 .await
989 .unwrap();
990
991 let script = readability_probe_script(&f.to_string_lossy());
992 let err = run_checked(&local_executor(), &script, "probe")
993 .await
994 .expect_err("an unreadable file must fail the probe");
995 let msg = format!("{err:#}");
996 // The two things the raw bash error does not tell you.
997 assert!(msg.contains("cannot read"), "{msg}");
998 assert!(msg.contains("mode 0") || msg.contains("mode "), "{msg}");
999 }
1000
1001 #[tokio::test]
1002 async fn readability_probe_distinguishes_missing_from_unreadable() {
1003 let tmp = tempfile::tempdir().unwrap();
1004 let missing = tmp.path().join("nope.env");
1005 let script = readability_probe_script(&missing.to_string_lossy());
1006 let err = run_checked(&local_executor(), &script, "probe")
1007 .await
1008 .expect_err("a missing file must fail the probe");
1009 let msg = format!("{err:#}");
1010 assert!(msg.contains("does not exist"), "{msg}");
1011 }
1012
1013 #[test]
1014 fn the_two_stages_read_differently() {
1015 // These strings end up in an operator's terminal during an incident.
1016 let before = FailureStage::BeforeSwap.to_string();
1017 let after = FailureStage::AtOrAfterSwap.to_string();
1018 assert!(before.contains("previous version"), "{before}");
1019 assert!(after.contains("indeterminate"), "{after}");
1020 assert_ne!(before, after);
1021 }
1022
1023 /// A LocalExec granted the default node capabilities (deploy + restart).
1024 fn local_executor() -> LocalExec {
1025 LocalExec::new(CapabilitySet::from_tokens(
1026 ["deploy", "restart"],
1027 ["health"],
1028 ))
1029 }
1030
1031 #[tokio::test]
1032 async fn deploy_local_copies_multiple_binaries_and_swaps_symlink() {
1033 let tmp = tempfile::tempdir().unwrap();
1034 let root = tmp.path();
1035
1036 let src_dir = root.join("src");
1037 tokio::fs::create_dir_all(&src_dir).await.unwrap();
1038 let primary = src_dir.join("makenotwork");
1039 let admin = src_dir.join("mnw-admin");
1040 tokio::fs::write(&primary, b"PRIMARY").await.unwrap();
1041 tokio::fs::write(&admin, b"ADMIN").await.unwrap();
1042
1043 let release_root = root.join("releases-root");
1044 tokio::fs::create_dir_all(&release_root).await.unwrap();
1045
1046 // Stage into staging/<build_id> (no publish yet).
1047 let staging = stage_local_bundle(&release_root, 42, &[primary.clone(), admin.clone()])
1048 .await
1049 .expect("stage_local_bundle should succeed");
1050 assert_eq!(staging, release_root.join("staging").join("42"));
1051 assert!(
1052 !release_root.join("current").exists(),
1053 "staging must not publish or flip current"
1054 );
1055
1056 // Publish content-addressed at releases/<digest16>.
1057 let released = finalize_local_release(&release_root, &staging, "deadbeefcafe0000")
1058 .await
1059 .expect("finalize_local_release should succeed");
1060 assert_eq!(
1061 released,
1062 release_root.join("releases").join("deadbeefcafe0000")
1063 );
1064 assert!(
1065 !staging.exists(),
1066 "staging dir is consumed by the publish rename"
1067 );
1068 assert_eq!(
1069 tokio::fs::read(released.join("makenotwork")).await.unwrap(),
1070 b"PRIMARY"
1071 );
1072 assert_eq!(
1073 tokio::fs::read(released.join("mnw-admin")).await.unwrap(),
1074 b"ADMIN"
1075 );
1076
1077 let current = release_root.join("current");
1078 let target = tokio::fs::read_link(&current).await.unwrap();
1079 assert_eq!(target.to_string_lossy(), "releases/deadbeefcafe0000");
1080 let via_current = tokio::fs::read(current.join("makenotwork")).await.unwrap();
1081 assert_eq!(via_current, b"PRIMARY");
1082 }
1083
1084 #[tokio::test]
1085 async fn finalize_second_release_swaps_symlink_and_keeps_old_dir() {
1086 let tmp = tempfile::tempdir().unwrap();
1087 let root = tmp.path();
1088 let src_dir = root.join("src");
1089 tokio::fs::create_dir_all(&src_dir).await.unwrap();
1090 let bin = src_dir.join("server");
1091 tokio::fs::write(&bin, b"V1").await.unwrap();
1092
1093 let release_root = root.join("rr");
1094 tokio::fs::create_dir_all(&release_root).await.unwrap();
1095
1096 // Two builds, distinct digests (distinct content) -> two release dirs.
1097 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1098 .await
1099 .unwrap();
1100 finalize_local_release(&release_root, &s1, "1111111111111111")
1101 .await
1102 .unwrap();
1103 tokio::fs::write(&bin, b"V2").await.unwrap();
1104 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1105 .await
1106 .unwrap();
1107 finalize_local_release(&release_root, &s2, "2222222222222222")
1108 .await
1109 .unwrap();
1110
1111 assert!(
1112 release_root
1113 .join("releases/1111111111111111/server")
1114 .exists()
1115 );
1116 assert!(
1117 release_root
1118 .join("releases/2222222222222222/server")
1119 .exists()
1120 );
1121 let target = tokio::fs::read_link(release_root.join("current"))
1122 .await
1123 .unwrap();
1124 assert_eq!(target.to_string_lossy(), "releases/2222222222222222");
1125 let via_current = tokio::fs::read(release_root.join("current/server"))
1126 .await
1127 .unwrap();
1128 assert_eq!(via_current, b"V2");
1129 }
1130
1131 #[tokio::test]
1132 async fn finalize_reuses_an_existing_release_of_the_same_digest() {
1133 let tmp = tempfile::tempdir().unwrap();
1134 let root = tmp.path();
1135 let bin = root.join("server");
1136 tokio::fs::write(&bin, b"BYTES").await.unwrap();
1137 let release_root = root.join("rr");
1138 tokio::fs::create_dir_all(&release_root).await.unwrap();
1139
1140 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1141 .await
1142 .unwrap();
1143 finalize_local_release(&release_root, &s1, "abc123abc123abc1")
1144 .await
1145 .unwrap();
1146 // Same digest rebuilt (e.g. a re-run at the same content): finalize must
1147 // reuse the existing release and drop the redundant staging dir, not error.
1148 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1149 .await
1150 .unwrap();
1151 let released = finalize_local_release(&release_root, &s2, "abc123abc123abc1")
1152 .await
1153 .expect("finalize is idempotent on a repeated digest");
1154 assert_eq!(released, release_root.join("releases/abc123abc123abc1"));
1155 assert!(!s2.exists(), "redundant staging dropped");
1156 }
1157
1158 #[tokio::test]
1159 async fn manifest_verify_script_passes_on_match_fails_on_drift_and_skips_when_absent() {
1160 // The node-side verification is a shell running `sha256sum -c MANIFEST`;
1161 // drive the real script through bash to prove it accepts a good bundle,
1162 // rejects a tampered one, and no-ops on a legacy (MANIFEST-less) bundle.
1163 let dir = tempfile::tempdir().unwrap();
1164 tokio::fs::write(dir.path().join("server"), b"BINARY")
1165 .await
1166 .unwrap();
1167 tokio::fs::create_dir(dir.path().join("static"))
1168 .await
1169 .unwrap();
1170 tokio::fs::write(dir.path().join("static/app.css"), b"body{}")
1171 .await
1172 .unwrap();
1173 let digest = crate::bundle::digest_dir(dir.path()).await.unwrap();
1174 tokio::fs::write(dir.path().join("MANIFEST"), digest.manifest.as_bytes())
1175 .await
1176 .unwrap();
1177
1178 let run = |d: &std::path::Path| {
1179 let script = manifest_verify_script(d.to_str().unwrap());
1180 async move {
1181 Command::new("bash")
1182 .arg("-c")
1183 .arg(&script)
1184 .output()
1185 .await
1186 .unwrap()
1187 }
1188 };
1189
1190 let ok = run(dir.path()).await;
1191 assert!(
1192 ok.status.success(),
1193 "matching bundle verifies: {}",
1194 String::from_utf8_lossy(&ok.stderr)
1195 );
1196
1197 // Drift one file: sha256sum -c must fail (current symlink left intact).
1198 tokio::fs::write(dir.path().join("static/app.css"), b"TAMPERED")
1199 .await
1200 .unwrap();
1201 let bad = run(dir.path()).await;
1202 assert!(!bad.status.success(), "a drifted file fails verification");
1203
1204 // Legacy bundle with no MANIFEST: skip, not fail.
1205 let legacy = tempfile::tempdir().unwrap();
1206 tokio::fs::write(legacy.path().join("server"), b"x")
1207 .await
1208 .unwrap();
1209 let skip = run(legacy.path()).await;
1210 assert!(
1211 skip.status.success(),
1212 "a bundle without a MANIFEST skips verification rather than failing"
1213 );
1214 }
1215
1216 #[tokio::test]
1217 async fn gc_local_releases_keeps_last_n_by_mtime() {
1218 let tmp = tempfile::tempdir().unwrap();
1219 let root = tmp.path();
1220 let releases = root.join("releases");
1221 tokio::fs::create_dir_all(&releases).await.unwrap();
1222
1223 let total = RELEASES_TO_KEEP + 3;
1224 let mut names = Vec::new();
1225 for i in 0..total {
1226 let name = format!("v{i:02}");
1227 let dir = releases.join(&name);
1228 tokio::fs::create_dir(&dir).await.unwrap();
1229 let f = std::fs::File::open(&dir).unwrap();
1230 let when =
1231 SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
1232 let times = std::fs::FileTimes::new().set_modified(when);
1233 f.set_times(times).unwrap();
1234 names.push(name);
1235 }
1236
1237 gc_local_releases(root).await.unwrap();
1238
1239 let surviving_expected: Vec<_> = names
1240 .iter()
1241 .skip(total - RELEASES_TO_KEEP)
1242 .cloned()
1243 .collect();
1244 for name in &surviving_expected {
1245 assert!(releases.join(name).exists(), "expected to survive: {name}");
1246 }
1247 for name in names.iter().take(total - RELEASES_TO_KEEP) {
1248 assert!(
1249 !releases.join(name).exists(),
1250 "expected to be pruned: {name}"
1251 );
1252 }
1253 }
1254
1255 #[tokio::test]
1256 async fn gc_local_releases_noop_when_below_threshold() {
1257 let tmp = tempfile::tempdir().unwrap();
1258 let root = tmp.path();
1259 let releases = root.join("releases");
1260 tokio::fs::create_dir_all(&releases).await.unwrap();
1261 for i in 0..3 {
1262 tokio::fs::create_dir(releases.join(format!("v{i}")))
1263 .await
1264 .unwrap();
1265 }
1266 gc_local_releases(root).await.unwrap();
1267 for i in 0..3 {
1268 assert!(releases.join(format!("v{i}")).exists());
1269 }
1270 }
1271
1272 #[tokio::test]
1273 async fn gc_local_releases_noop_when_releases_dir_missing() {
1274 let tmp = tempfile::tempdir().unwrap();
1275 gc_local_releases(tmp.path()).await.unwrap();
1276 }
1277
1278 #[tokio::test]
1279 async fn deploy_remote_fails_cleanly_when_host_unreachable() {
1280 // 192.0.2.0/24 is reserved for documentation and routes nowhere.
1281 // ConnectTimeout=10 limits the test wallclock to ~10s worst case.
1282 let tmp = tempfile::tempdir().unwrap();
1283 let staged = tmp.path().join("releases").join("0.0.1");
1284 tokio::fs::create_dir_all(&staged).await.unwrap();
1285 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
1286
1287 let node = crate::topology::Node {
1288 platform: None,
1289 name: "unreachable".into(),
1290 ssh_target: "deploy@192.0.2.1".into(),
1291 release_root: "/opt/never".into(),
1292 service_name: "makenotwork.service".into(),
1293 health_url: None,
1294 config_check_env_file: None,
1295 actuate: crate::topology::default_actuate(),
1296 observe: crate::topology::default_observe(),
1297 companions: Vec::new(),
1298 };
1299 let executor = SshExec::new(
1300 node.ssh_target.clone(),
1301 CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1302 );
1303
1304 let placement = Placement::check(&node, &staged, None).expect("both sides silent");
1305 let result = deploy_node(&executor, placement, "0.0.1", "server").await;
1306 let err = result.expect_err("deploy to unreachable host should fail");
1307 let msg = format!("{err:#}");
1308 // Don't pin exact wording, just that the failure is attributed (ssh /
1309 // rsync / connection) and that no panic / hang happened.
1310 assert!(
1311 msg.contains("ssh")
1312 || msg.contains("rsync")
1313 || msg.contains("connection")
1314 || msg.contains("Connection"),
1315 "unexpected error: {msg}"
1316 );
1317 }
1318
1319 #[tokio::test]
1320 async fn deploy_node_with_local_ssh_target_swaps_symlink() {
1321 // ssh_target="local" routes to the local fast-path: just a symlink
1322 // swap, no remote calls.
1323 let tmp = tempfile::tempdir().unwrap();
1324 let release_root = tmp.path().to_path_buf();
1325 let staged = release_root.join("releases").join("0.0.1");
1326 tokio::fs::create_dir_all(&staged).await.unwrap();
1327 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
1328
1329 let node = crate::topology::Node {
1330 platform: None,
1331 name: "local-dev".into(),
1332 ssh_target: "local".into(),
1333 release_root: release_root.to_string_lossy().into_owned(),
1334 service_name: "makenotwork.service".into(),
1335 health_url: None,
1336 config_check_env_file: None,
1337 actuate: crate::topology::default_actuate(),
1338 observe: crate::topology::default_observe(),
1339 companions: Vec::new(),
1340 };
1341 let executor = local_executor();
1342
1343 let out = deploy_node(
1344 &executor,
1345 Placement::check(&node, &staged, None).unwrap(),
1346 "0.0.1",
1347 "server",
1348 )
1349 .await
1350 .unwrap();
1351 assert_eq!(out, staged);
1352 let target = tokio::fs::read_link(release_root.join("current"))
1353 .await
1354 .unwrap();
1355 assert_eq!(target.to_string_lossy(), "releases/0.0.1");
1356 }
1357
1358 // ---- swap_and_restart_script: symlink/restart consistency ----
1359
1360 async fn run_script(script: &str) -> std::process::Output {
1361 Command::new("sh")
1362 .arg("-c")
1363 .arg(script)
1364 .output()
1365 .await
1366 .unwrap()
1367 }
1368
1369 async fn setup_release_root(with_current: bool) -> tempfile::TempDir {
1370 let tmp = tempfile::tempdir().unwrap();
1371 let root = tmp.path();
1372 tokio::fs::create_dir_all(root.join("releases/old"))
1373 .await
1374 .unwrap();
1375 tokio::fs::create_dir_all(root.join("releases/new"))
1376 .await
1377 .unwrap();
1378 if with_current {
1379 std::os::unix::fs::symlink("releases/old", root.join("current")).unwrap();
1380 }
1381 tmp
1382 }
1383
1384 #[tokio::test]
1385 async fn swap_and_restart_keeps_new_symlink_when_restart_succeeds() {
1386 let tmp = setup_release_root(true).await;
1387 let root = tmp.path().to_string_lossy().into_owned();
1388 let out = run_script(&swap_and_restart_script(&root, "new", "true")).await;
1389 assert!(
1390 out.status.success(),
1391 "script should succeed when restart succeeds"
1392 );
1393 let target = tokio::fs::read_link(tmp.path().join("current"))
1394 .await
1395 .unwrap();
1396 assert_eq!(
1397 target.to_string_lossy(),
1398 "releases/new",
1399 "symlink advanced to new"
1400 );
1401 }
1402
1403 #[tokio::test]
1404 async fn swap_and_restart_rolls_symlink_back_when_restart_fails() {
1405 // The bug: a restart failure after the flip must NOT leave `current`
1406 // pointing at the new (un-activated) release.
1407 let tmp = setup_release_root(true).await;
1408 let root = tmp.path().to_string_lossy().into_owned();
1409 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
1410 assert!(!out.status.success(), "script must fail when restart fails");
1411 let target = tokio::fs::read_link(tmp.path().join("current"))
1412 .await
1413 .unwrap();
1414 assert_eq!(
1415 target.to_string_lossy(),
1416 "releases/old",
1417 "symlink rolled back to prev so a later restart can't silently activate new",
1418 );
1419 }
1420
1421 // ---- arch_guard_script: wrong-arch artifacts fail closed ----
1422
1423 /// A 20-byte stub whose ELF e_machine field (offset 18, 2 bytes LE) is set.
1424 fn elf_stub_with_machine(b18: u8, b19: u8) -> tempfile::NamedTempFile {
1425 let mut data = vec![0u8; 20];
1426 data[18] = b18;
1427 data[19] = b19;
1428 let f = tempfile::NamedTempFile::new().unwrap();
1429 std::fs::write(f.path(), &data).unwrap();
1430 f
1431 }
1432
1433 /// e_machine low byte for the host running the test, if mapped.
1434 fn host_machine_lo() -> Option<u8> {
1435 match std::env::consts::ARCH {
1436 "x86_64" => Some(0x3e),
1437 "aarch64" => Some(0xb7),
1438 _ => None,
1439 }
1440 }
1441
1442 #[tokio::test]
1443 async fn arch_guard_passes_for_matching_binary() {
1444 let Some(lo) = host_machine_lo() else { return };
1445 let f = elf_stub_with_machine(lo, 0x00);
1446 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
1447 assert!(
1448 out.status.success(),
1449 "matching arch must pass: {}",
1450 String::from_utf8_lossy(&out.stderr),
1451 );
1452 }
1453
1454 #[tokio::test]
1455 async fn arch_guard_fails_closed_for_wrong_binary() {
1456 // Use the other arch's e_machine so it can't match the host.
1457 let wrong = match std::env::consts::ARCH {
1458 "x86_64" => 0xb7, // aarch64 binary on an x86_64 node
1459 "aarch64" => 0x3e, // x86_64 binary on an aarch64 node
1460 _ => return,
1461 };
1462 let f = elf_stub_with_machine(wrong, 0x00);
1463 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
1464 assert!(
1465 !out.status.success(),
1466 "wrong-arch binary must fail closed before the symlink swap"
1467 );
1468 }
1469
1470 // ---- ldd_guard_script: a binary this node cannot resolve fails closed ----
1471
1472 /// A fake `ldd` on PATH that prints `body` and exits `code`, so the guard's
1473 /// three outcomes can be exercised without a binary that genuinely fails to
1474 /// link. The real `ldd` cannot be made to produce a `not found` on demand.
1475 async fn run_ldd_guard_with_fake(body: &str, code: i32) -> std::process::Output {
1476 let dir = tempfile::tempdir().unwrap();
1477 let fake = dir.path().join("ldd");
1478 std::fs::write(
1479 &fake,
1480 format!("#!/bin/sh\ncat <<'EOF'\n{body}\nEOF\nexit {code}\n"),
1481 )
1482 .unwrap();
1483 let mut perms = std::fs::metadata(&fake).unwrap().permissions();
1484 std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
1485 std::fs::set_permissions(&fake, perms).unwrap();
1486 let bin = dir.path().join("subject");
1487 std::fs::write(&bin, b"x").unwrap();
1488 Command::new("sh")
1489 .arg("-c")
1490 .arg(ldd_guard_script(&bin.to_string_lossy()))
1491 .env("PATH", format!("{}:/usr/bin:/bin", dir.path().display()))
1492 .output()
1493 .await
1494 .unwrap()
1495 }
1496
1497 #[tokio::test]
1498 async fn ldd_guard_fails_closed_on_an_unsatisfiable_symbol_version() {
1499 // The exact failure Bento's glibc_check used to catch at build time, and
1500 // the reason this guard exists: right arch, resolves every library, and
1501 // still cannot exec because the node's glibc is older than the build
1502 // host's.
1503 let out = run_ldd_guard_with_fake(
1504 "\tlinux-vdso.so.1 (0x00007fff)\n\
1505 \t/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.40' not found (required by ./pom)\n\
1506 \tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
1507 0,
1508 )
1509 .await;
1510 assert!(
1511 !out.status.success(),
1512 "an unsatisfiable symbol version must fail before the symlink swap"
1513 );
1514 let stderr = String::from_utf8_lossy(&out.stderr);
1515 assert!(
1516 stderr.contains("GLIBC_2.40"),
1517 "the offending line must reach the operator, not just a verdict: {stderr}"
1518 );
1519 }
1520
1521 #[tokio::test]
1522 async fn ldd_guard_fails_closed_on_a_missing_library() {
1523 let out = run_ldd_guard_with_fake("\tlibfoo.so.1 => not found", 0).await;
1524 assert!(!out.status.success(), "a missing library must fail closed");
1525 }
1526
1527 #[tokio::test]
1528 async fn ldd_guard_passes_a_resolvable_binary() {
1529 let out = run_ldd_guard_with_fake(
1530 "\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
1531 0,
1532 )
1533 .await;
1534 assert!(
1535 out.status.success(),
1536 "a fully resolved binary must pass: {}",
1537 String::from_utf8_lossy(&out.stderr),
1538 );
1539 }
1540
1541 #[tokio::test]
1542 async fn ldd_guard_passes_a_static_binary() {
1543 // ldd exits non-zero for these. Nothing to resolve is not a failure.
1544 let out = run_ldd_guard_with_fake("\tnot a dynamic executable", 1).await;
1545 assert!(
1546 out.status.success(),
1547 "a static binary has no dependencies to satisfy: {}",
1548 String::from_utf8_lossy(&out.stderr),
1549 );
1550 }
1551
1552 #[tokio::test]
1553 async fn ldd_guard_fails_when_ldd_errors_for_another_reason() {
1554 // Not the static case: ldd said something else and exited non-zero. We
1555 // do not know the binary is fine, so we do not say it is.
1556 let out = run_ldd_guard_with_fake("ldd: cannot read file", 1).await;
1557 assert!(
1558 !out.status.success(),
1559 "an unexplained ldd failure must not read as a pass"
1560 );
1561 }
1562
1563 #[tokio::test]
1564 async fn ldd_guard_skips_when_the_node_has_no_ldd() {
1565 // Cannot verify is not known bad, matching arch_guard's unmapped-arch
1566 // call. PATH holds nothing, so `command -v ldd` finds none.
1567 let dir = tempfile::tempdir().unwrap();
1568 let bin = dir.path().join("subject");
1569 std::fs::write(&bin, b"x").unwrap();
1570 // Absolute path to the shell: PATH is what this test empties, so
1571 // resolving `sh` through it would fail before the script ever ran.
1572 let out = Command::new("/bin/sh")
1573 .arg("-c")
1574 .arg(ldd_guard_script(&bin.to_string_lossy()))
1575 .env("PATH", dir.path().display().to_string())
1576 .output()
1577 .await
1578 .unwrap();
1579 assert!(
1580 out.status.success(),
1581 "a node with no ldd must not fail the deploy: {}",
1582 String::from_utf8_lossy(&out.stderr),
1583 );
1584 }
1585
1586 #[tokio::test]
1587 async fn swap_and_restart_first_deploy_failure_has_no_prev_to_restore() {
1588 // No prior `current`. A restart failure leaves `current` at new (the only
1589 // version) and still reports failure — documented degenerate case.
1590 let tmp = setup_release_root(false).await;
1591 let root = tmp.path().to_string_lossy().into_owned();
1592 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
1593 assert!(!out.status.success(), "script must fail when restart fails");
1594 let target = tokio::fs::read_link(tmp.path().join("current"))
1595 .await
1596 .unwrap();
1597 assert_eq!(
1598 target.to_string_lossy(),
1599 "releases/new",
1600 "no prev existed to roll back to"
1601 );
1602 }
1603
1604 // ---- config_check_script: systemd-faithful env loading ----
1605
1606 #[tokio::test]
1607 async fn config_check_script_loads_values_with_shell_metachars() {
1608 // The bug: `. env_file` expands/word-splits values, so a URL or a
1609 // password containing a shell metacharacter is mangled — it dropped
1610 // DATABASE_URL to empty on a real node, which would fail every deploy.
1611 // The export-loop must load such a value intact. The "binary" is a
1612 // checker script (a real path, like a deployed binary) that exits 0 only
1613 // if the var arrived byte-for-byte — it compares against the expected
1614 // value read from a file, so nothing re-interprets the metacharacters.
1615 let tricky = "postgres://u:p$ss;w&rd@h/db `x` $(y)";
1616 // Plain files in a tempdir: no lingering write fd, so the checker can be
1617 // exec'd (a NamedTempFile stays open and would ETXTBSY).
1618 let dir = tempfile::tempdir().unwrap();
1619 let expected_path = dir.path().join("expected");
1620 std::fs::write(&expected_path, tricky).unwrap(); // no trailing newline
1621
1622 let env_path = dir.path().join("node.env");
1623 std::fs::write(
1624 &env_path,
1625 format!(
1626 "# a comment\n\nDATABASE_URL={tricky}\nOTHER=plain\nEXPECTED_FILE={ef}\n",
1627 ef = expected_path.display(),
1628 ),
1629 )
1630 .unwrap();
1631
1632 let checker_path = dir.path().join("checker.sh");
1633 std::fs::write(
1634 &checker_path,
1635 "#!/bin/sh\nwant=$(cat \"$EXPECTED_FILE\")\n\
1636 [ \"$DATABASE_URL\" = \"$want\" ] || { echo \"DB [$DATABASE_URL] != [$want]\" >&2; exit 1; }\n\
1637 [ \"$OTHER\" = plain ] || { echo \"OTHER [$OTHER]\" >&2; exit 1; }\n",
1638 )
1639 .unwrap();
1640 std::fs::set_permissions(
1641 &checker_path,
1642 std::os::unix::fs::PermissionsExt::from_mode(0o755),
1643 )
1644 .unwrap();
1645
1646 let script =
1647 config_check_script(&env_path.to_string_lossy(), &checker_path.to_string_lossy());
1648 let out = run_script(&script).await;
1649 assert!(
1650 out.status.success(),
1651 "value with shell metachars must load intact; stderr: {}",
1652 String::from_utf8_lossy(&out.stderr),
1653 );
1654 }
1655
1656 // ---- install-companion.sh: the node-side guard rails ----
1657
1658 /// Run the shipped installer script with three args; returns its exit code.
1659 /// Exercises the real file rather than a copy of its logic, because the
1660 /// script is the ONLY control on a NOPASSWD sudo grant.
1661 fn run_installer(src: &str, dst: &str, service: &str) -> i32 {
1662 let script =
1663 std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../deploy/install-companion.sh");
1664 std::process::Command::new("bash")
1665 .arg(&script)
1666 .args([src, dst, service])
1667 .output()
1668 .expect("running install-companion.sh")
1669 .status
1670 .code()
1671 .expect("script exited via signal")
1672 }
1673
1674 // Guards run before any filesystem write, so these never install anything.
1675 // Exit 3 = refused by a guard; exit 4 = guards passed, src simply absent.
1676 const REFUSED: i32 = 3;
1677 const PASSED_GUARDS: i32 = 4;
1678
1679 #[test]
1680 fn installer_refuses_a_dst_that_escapes_opt_via_dotdot() {
1681 // `/opt/../etc/...` matches a bare `/opt/*` glob. With the sudoers
1682 // wildcard that meant `install -m 0755` as root to anywhere, plus a
1683 // restart of any unit — so the path must be normalised before the test.
1684 assert_eq!(
1685 run_installer(
1686 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1687 "/opt/../etc/systemd/system/evil.service",
1688 "mnw-cli.service",
1689 ),
1690 REFUSED,
1691 );
1692 }
1693
1694 #[test]
1695 fn installer_refuses_a_src_that_escapes_the_bundle_via_dotdot() {
1696 assert_eq!(
1697 run_installer(
1698 "/opt/mnw/releases/1.0.0/companions/../../../../../etc/shadow",
1699 "/opt/mnw-cli/mnw-cli",
1700 "mnw-cli.service",
1701 ),
1702 REFUSED,
1703 );
1704 }
1705
1706 #[test]
1707 fn installer_accepts_the_real_companion_paths() {
1708 // The guards must not have been tightened into uselessness: the shape
1709 // Sando actually sends has to get past them. It stops at the missing
1710 // src (exit 4), which is proof the guards accepted it.
1711 assert_eq!(
1712 run_installer(
1713 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1714 "/opt/mnw-cli/mnw-cli",
1715 "mnw-cli.service",
1716 ),
1717 PASSED_GUARDS,
1718 );
1719 }
1720
1721 #[test]
1722 fn installer_refuses_a_service_name_with_a_path_separator() {
1723 assert_eq!(
1724 run_installer(
1725 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1726 "/opt/mnw-cli/mnw-cli",
1727 "../../etc/evil.service",
1728 ),
1729 REFUSED,
1730 );
1731 }
1732
1733 // ---- install_companion_cmd: shape + quoting ----
1734
1735 #[test]
1736 fn install_companion_cmd_shape_and_quoting() {
1737 let cmd = install_companion_cmd(
1738 "/opt/mnw/releases/0.10.14/companions/mnw-cli",
1739 "/opt/mnw-cli/mnw-cli",
1740 "mnw-cli.service",
1741 );
1742 // Routes through the wrapper (single sudoers grant), sudo-invoked, with
1743 // src, dst, service in that order.
1744 assert!(cmd.starts_with("sudo "), "must be sudo-invoked: {cmd}");
1745 assert!(
1746 cmd.contains("/usr/local/lib/mnw/install-companion.sh"),
1747 "{cmd}"
1748 );
1749 let installer_pos = cmd.find("install-companion.sh").unwrap();
1750 let src_pos = cmd.find("companions/mnw-cli").unwrap();
1751 let dst_pos = cmd.find("/opt/mnw-cli/mnw-cli").unwrap();
1752 let svc_pos = cmd.find("mnw-cli.service").unwrap();
1753 assert!(
1754 installer_pos < src_pos && src_pos < dst_pos && dst_pos < svc_pos,
1755 "arg order: {cmd}"
1756 );
1757 }
1758
1759 #[test]
1760 fn install_companion_cmd_quotes_metachars() {
1761 // A path with a space/quote must be shell-safe (defense in depth even
1762 // though these come from operator config).
1763 let cmd = install_companion_cmd("/a b/src", "/dst'x", "u.service");
1764 let out = std::process::Command::new("sh")
1765 .arg("-c")
1766 .arg(format!(
1767 "set -- {}; echo \"$#\"",
1768 cmd.strip_prefix("sudo ").unwrap()
1769 ))
1770 .output()
1771 .unwrap();
1772 // installer + 3 args = 4 positional words after quoting.
1773 assert_eq!(
1774 String::from_utf8_lossy(&out.stdout).trim(),
1775 "4",
1776 "quoting split wrong: {cmd}"
1777 );
1778 }
1779
1780 #[tokio::test]
1781 async fn config_check_script_propagates_binary_failure() {
1782 // A required var missing (the binary exits non-zero) must fail the check.
1783 let env = tempfile::NamedTempFile::new().unwrap();
1784 std::fs::write(env.path(), "FOO=bar\n").unwrap();
1785 let script = config_check_script(&env.path().to_string_lossy(), "false");
1786 let out = run_script(&script).await;
1787 assert!(
1788 !out.status.success(),
1789 "a non-zero MNW_CHECK_CONFIG exit must fail the check"
1790 );
1791 }
1792
1793 #[tokio::test]
1794 async fn deploy_node_denied_when_executor_lacks_deploy_grant() {
1795 // Defense in depth: an executor without the deploy grant refuses the
1796 // step before any filesystem / ssh action.
1797 let tmp = tempfile::tempdir().unwrap();
1798 let release_root = tmp.path().to_path_buf();
1799 let staged = release_root.join("releases").join("0.0.1");
1800 tokio::fs::create_dir_all(&staged).await.unwrap();
1801
1802 let node = crate::topology::Node {
1803 platform: None,
1804 name: "local-dev".into(),
1805 ssh_target: "local".into(),
1806 release_root: release_root.to_string_lossy().into_owned(),
1807 service_name: "makenotwork.service".into(),
1808 health_url: None,
1809 config_check_env_file: None,
1810 actuate: vec!["restart".into()], // no deploy
1811 observe: vec![],
1812 companions: Vec::new(),
1813 };
1814 let executor = LocalExec::new(CapabilitySet::from_tokens(["restart"], Vec::<&str>::new()));
1815 let err = deploy_node(
1816 &executor,
1817 Placement::check(&node, &staged, None).unwrap(),
1818 "0.0.1",
1819 "server",
1820 )
1821 .await
1822 .unwrap_err();
1823 assert!(
1824 format!("{err:#}").contains("capability denied"),
1825 "expected capability denial"
1826 );
1827 }
1828
1829 // ---- FakeExec: the deploy_remote choreography without a real host ----
1830 //
1831 // deploy_node's local fast-path is covered above with a real LocalExec, but
1832 // the remote path (rsync + arch guard + config-drift + swap + companions +
1833 // gc) short-circuits on `ssh_target != "local"` and so never ran under test
1834 // without a reachable node. FakeExec records every executor call in order
1835 // and can be told to fail one shell step (matched by substring) or the rsync
1836 // push, so the ordering and the fail-closed-before-swap contract are
1837 // assertable in-process.
1838
1839 struct FakeExec {
1840 caps: CapabilitySet,
1841 calls: Arc<StdMutex<Vec<String>>>,
1842 /// The first `run_streaming` whose script contains this substring exits
1843 /// non-zero (a failed shell step), e.g. the arch guard.
1844 fail_run_matching: Option<String>,
1845 /// `push_dir` (the rsync) returns an error.
1846 fail_push_dir: bool,
1847 }
1848
1849 impl FakeExec {
1850 fn new() -> Self {
1851 Self {
1852 caps: CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1853 calls: Arc::new(StdMutex::new(Vec::new())),
1854 fail_run_matching: None,
1855 fail_push_dir: false,
1856 }
1857 }
1858 fn log(&self) -> Vec<String> {
1859 self.calls.lock().unwrap().clone()
1860 }
1861 }
1862
1863 #[async_trait]
1864 impl Executor for FakeExec {
1865 async fn run_streaming(&self, step: &Step, _sink: &mut dyn LogSink) -> Result<RunOutput> {
1866 // Every deploy step is a `Step::shell`, so the script is argv's tail.
1867 let script = step.argv.last().cloned().unwrap_or_default();
1868 self.calls.lock().unwrap().push(format!("run:{script}"));
1869 let fail = self
1870 .fail_run_matching
1871 .as_deref()
1872 .is_some_and(|m| script.contains(m));
1873 Ok(RunOutput {
1874 status: std::process::ExitStatus::from_raw(if fail { 1 << 8 } else { 0 }),
1875 stdout: Vec::new(),
1876 stderr: if fail {
1877 b"fake step failure".to_vec()
1878 } else {
1879 Vec::new()
1880 },
1881 })
1882 }
1883 async fn pull_file(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1884 self.calls.lock().unwrap().push("pull_file".into());
1885 Ok(())
1886 }
1887 async fn pull_dir(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1888 self.calls.lock().unwrap().push("pull_dir".into());
1889 Ok(())
1890 }
1891 async fn pull_glob(&self, _glob: &str, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1892 self.calls.lock().unwrap().push("pull_glob".into());
1893 Ok(())
1894 }
1895 async fn push_dir(&self, _local: &Path, remote: &Path, _opts: &SyncOpts) -> Result<()> {
1896 self.calls
1897 .lock()
1898 .unwrap()
1899 .push(format!("push_dir:{}", remote.display()));
1900 if self.fail_push_dir {
1901 anyhow::bail!("fake rsync failure");
1902 }
1903 Ok(())
1904 }
1905 fn capabilities(&self) -> &CapabilitySet {
1906 &self.caps
1907 }
1908 }
1909
1910 fn remote_node(config_check: bool, companions: Vec<NodeCompanion>) -> Node {
1911 Node {
1912 platform: None,
1913 name: "web-a".into(),
1914 ssh_target: "deploy@web-a".into(),
1915 release_root: "/opt/mnw".into(),
1916 service_name: "makenotwork.service".into(),
1917 health_url: None,
1918 config_check_env_file: config_check.then(|| "/etc/mnw/node.env".to_string()),
1919 actuate: crate::topology::default_actuate(),
1920 observe: crate::topology::default_observe(),
1921 companions,
1922 }
1923 }
1924
1925 fn companion() -> NodeCompanion {
1926 NodeCompanion {
1927 name: "mnw-cli".into(),
1928 install_path: "/opt/mnw-cli/mnw-cli".into(),
1929 service_name: "mnw-cli.service".into(),
1930 }
1931 }
1932
1933 /// Index of the first recorded call whose text contains `needle` (panics if
1934 /// absent — the assertion message names what was missing).
1935 fn pos(log: &[String], needle: &str) -> usize {
1936 log.iter()
1937 .position(|c| c.contains(needle))
1938 .unwrap_or_else(|| panic!("no call matched {needle:?} in {log:#?}"))
1939 }
1940
1941 #[tokio::test]
1942 async fn deploy_remote_runs_the_full_choreography_in_order() {
1943 // A node opted into the config-drift check and carrying one companion:
1944 // mkdir -> rsync -> arch guard -> config check -> swap+restart ->
1945 // companion install -> gc, in that order.
1946 let tmp = tempfile::tempdir().unwrap();
1947 let staged = tmp.path().join("releases").join("0.9.0");
1948 tokio::fs::create_dir_all(&staged).await.unwrap();
1949
1950 let node = remote_node(true, vec![companion()]);
1951 let exec = FakeExec::new();
1952 let out = deploy_node(
1953 &exec,
1954 Placement::check(&node, &staged, None).unwrap(),
1955 "0.9.0",
1956 "makenotwork",
1957 )
1958 .await
1959 .expect("deploy_remote should succeed against the fake");
1960 assert_eq!(out, PathBuf::from("/opt/mnw/releases/0.9.0"));
1961
1962 let log = exec.log();
1963 let mkdir = pos(&log, "mkdir -p");
1964 let rsync = pos(&log, "push_dir:/opt/mnw/releases/0.9.0");
1965 let arch = pos(&log, "e_machine");
1966 let cfg = pos(&log, "MNW_CHECK_CONFIG=1");
1967 let swap = pos(&log, "reload-or-restart");
1968 let comp = pos(&log, "install-companion.sh");
1969 let gc = pos(&log, "ls -1t");
1970 assert!(
1971 mkdir < rsync && rsync < arch && arch < cfg && cfg < swap && swap < comp && comp < gc,
1972 "deploy steps out of order: {log:#?}"
1973 );
1974 }
1975
1976 #[tokio::test]
1977 async fn deploy_remote_aborts_before_swap_when_rsync_fails() {
1978 // The rsync failing must fail the deploy BEFORE the symlink swap — the
1979 // "current symlink left intact" contract. Assert the swap never ran.
1980 let tmp = tempfile::tempdir().unwrap();
1981 let staged = tmp.path().join("releases").join("0.9.0");
1982 tokio::fs::create_dir_all(&staged).await.unwrap();
1983
1984 let node = remote_node(false, Vec::new());
1985 let mut exec = FakeExec::new();
1986 exec.fail_push_dir = true;
1987 let err = deploy_node(
1988 &exec,
1989 Placement::check(&node, &staged, None).unwrap(),
1990 "0.9.0",
1991 "makenotwork",
1992 )
1993 .await
1994 .expect_err("rsync failure must fail the deploy");
1995 assert!(
1996 format!("{err:#}").contains("rsync"),
1997 "error should attribute the rsync: {err:#}"
1998 );
1999 let log = exec.log();
2000 assert!(
2001 !log.iter().any(|c| c.contains("reload-or-restart")),
2002 "swap must not run after a failed rsync: {log:#?}"
2003 );
2004 }
2005
2006 #[tokio::test]
2007 async fn deploy_remote_aborts_before_swap_when_arch_guard_fails() {
2008 // A wrong-arch binary must fail closed before the swap. The fake fails
2009 // the arch-guard shell step; the swap must not follow.
2010 let tmp = tempfile::tempdir().unwrap();
2011 let staged = tmp.path().join("releases").join("0.9.0");
2012 tokio::fs::create_dir_all(&staged).await.unwrap();
2013
2014 let node = remote_node(false, Vec::new());
2015 let mut exec = FakeExec::new();
2016 exec.fail_run_matching = Some("e_machine".into());
2017 let err = deploy_node(
2018 &exec,
2019 Placement::check(&node, &staged, None).unwrap(),
2020 "0.9.0",
2021 "makenotwork",
2022 )
2023 .await
2024 .expect_err("arch mismatch must fail the deploy");
2025 assert!(
2026 format!("{err:#}").contains("architecture"),
2027 "error should mention the arch check: {err:#}"
2028 );
2029 let log = exec.log();
2030 assert!(
2031 !log.iter().any(|c| c.contains("reload-or-restart")),
2032 "swap must not run after a failed arch guard: {log:#?}"
2033 );
2034 }
2035
2036 #[tokio::test]
2037 async fn deploy_remote_skips_config_check_when_node_opts_out() {
2038 // No config_check_env_file => the pre-swap config check is skipped, but
2039 // the rest of the choreography (including the swap) still runs.
2040 let tmp = tempfile::tempdir().unwrap();
2041 let staged = tmp.path().join("releases").join("0.9.0");
2042 tokio::fs::create_dir_all(&staged).await.unwrap();
2043
2044 let node = remote_node(false, Vec::new());
2045 let exec = FakeExec::new();
2046 deploy_node(
2047 &exec,
2048 Placement::check(&node, &staged, None).unwrap(),
2049 "0.9.0",
2050 "makenotwork",
2051 )
2052 .await
2053 .unwrap();
2054 let log = exec.log();
2055 assert!(
2056 !log.iter().any(|c| c.contains("MNW_CHECK_CONFIG=1")),
2057 "config check must be skipped when the node opts out: {log:#?}"
2058 );
2059 assert!(
2060 log.iter().any(|c| c.contains("reload-or-restart")),
2061 "the swap must still run: {log:#?}"
2062 );
2063 }
2064
2065 #[tokio::test]
2066 async fn deploy_remote_installs_companion_after_the_swap() {
2067 // Companions are After= the server: their install must land after the
2068 // symlink swap + service restart, never before.
2069 let tmp = tempfile::tempdir().unwrap();
2070 let staged = tmp.path().join("releases").join("0.9.0");
2071 tokio::fs::create_dir_all(&staged).await.unwrap();
2072
2073 let node = remote_node(false, vec![companion()]);
2074 let exec = FakeExec::new();
2075 deploy_node(
2076 &exec,
2077 Placement::check(&node, &staged, None).unwrap(),
2078 "0.9.0",
2079 "makenotwork",
2080 )
2081 .await
2082 .unwrap();
2083 let log = exec.log();
2084 assert!(
2085 pos(&log, "reload-or-restart") < pos(&log, "install-companion.sh"),
2086 "companion install must follow the swap: {log:#?}"
2087 );
2088 }
2089 }
2090