Skip to main content

max / makenotwork

75.4 KB · 1909 lines History Blame Raw
1 //! Atomic symlink-swap deploys.
2 //!
3 //! Layout on every target (local host, A nodes, B nodes, ...). Release dirs are
4 //! named for their content digest, not the version (wiki note
5 //! `release-artifact-identity`), so a rebuild can never overwrite an earlier
6 //! build's dir in place:
7 //!
8 //! <release_root>/
9 //! releases/
10 //! a1b2c3d4e5f60718/ <- <digest16>
11 //! <bin_name>
12 //! MANIFEST <- <sha256> <relpath> per file, node-verified
13 //! f0e1d2c3b4a59687/
14 //! <bin_name>
15 //! current -> releases/f0e1d2c3b4a59687
16 //!
17 //! `ln -sfn` swaps the symlink. systemd units point at
18 //! `<release_root>/current/<bin_name>` so reload-or-restart picks up the new
19 //! binary without ever pointing at a missing path.
20 //!
21 //! The host-side transport — `ssh` for shell steps, `rsync` for the release
22 //! dir — comes from the shared [`ops_exec::Executor`] (a `LocalExec` for
23 //! `ssh_target = "local"`, an `SshExec` otherwise), built once per node in
24 //! [`crate::state`]. This module owns the *deploy choreography* (mkdir, push,
25 //! atomic swap, restart, gc); the transport is the crate's. SSH push behavior
26 //! is identical to the pre-extraction code — this is a transport extraction,
27 //! not a model change.
28
29 use crate::domain::Platform;
30 use crate::topology::Node;
31 use anyhow::{Context, Result};
32 use async_trait::async_trait;
33 use ops_exec::{Action, Executor, LogSink, RunOutput, Step, SyncOpts, sh_quote};
34 use std::path::{Path, PathBuf};
35 use tokio::process::Command;
36
37 /// A staged bundle proven to be for the node it is about to be pushed to.
38 ///
39 /// This exists because "ship aarch64 bytes to an x86_64 box" was, until pom, a
40 /// mistake nobody could make: one product, one build host, one architecture, so
41 /// the pairing of a bundle and a node was correct by having no alternative. pom
42 /// has two architectures under one version, so the pairing becomes a real
43 /// choice, and a wrong choice deploys a binary the node cannot exec.
44 ///
45 /// The answer is not a check before the call. A check is something a later
46 /// caller forgets, and the failure it guards is discovered by a production node
47 /// failing to start. [`Placement::check`] is the *only* way to obtain one of
48 /// these, and [`deploy_node`] takes one instead of a loose `(node, dir)` pair —
49 /// so a mismatched deploy is not a bug the code has to avoid, it is a value the
50 /// code cannot construct.
51 #[derive(Debug, Clone)]
52 pub struct Placement<'a> {
53 node: &'a Node,
54 bundle: &'a Path,
55 }
56
57 /// Why a bundle may not be placed on a node.
58 ///
59 /// All four cases are refusals, including both "one side said nothing" cases.
60 /// Silence is not agreement: a node that does not state its platform cannot
61 /// vouch that it runs a bundle built for a stated one, and a bundle that does
62 /// not state its platform cannot satisfy a node that requires one. The only
63 /// admissible pairing besides a match is both sides silent, which is the
64 /// single-platform world Sando lived in and MNW still lives in.
65 #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
66 pub enum PlacementError {
67 #[error(
68 "node {node} runs {node_platform} and this bundle was built for {artifact_platform}; \
69 refusing to deploy a binary the node cannot execute"
70 )]
71 Mismatch {
72 node: String,
73 node_platform: Platform,
74 artifact_platform: Platform,
75 },
76 #[error(
77 "node {node} does not declare a platform, and this bundle was built for \
78 {artifact_platform}. Declare `platform` on the node so the two can be compared"
79 )]
80 NodeSilent {
81 node: String,
82 artifact_platform: Platform,
83 },
84 #[error(
85 "node {node} requires {node_platform} and this bundle records no platform. \
86 An artifact whose platform is unknown cannot be shown to satisfy one that is"
87 )]
88 ArtifactSilent {
89 node: String,
90 node_platform: Platform,
91 },
92 }
93
94 impl<'a> Placement<'a> {
95 /// The one constructor. `artifact` is the platform the bundle records, which
96 /// for an accepted artifact comes from its `ArtifactRecord` provenance and
97 /// for a Sando-built one comes from the app config.
98 pub fn check(
99 node: &'a Node,
100 bundle: &'a Path,
101 artifact: Option<&Platform>,
102 ) -> Result<Self, PlacementError> {
103 match (node.platform.as_ref(), artifact) {
104 (Some(n), Some(a)) if n == a => Ok(Self { node, bundle }),
105 (Some(n), Some(a)) => Err(PlacementError::Mismatch {
106 node: node.name.to_string(),
107 node_platform: n.clone(),
108 artifact_platform: a.clone(),
109 }),
110 (None, Some(a)) => Err(PlacementError::NodeSilent {
111 node: node.name.to_string(),
112 artifact_platform: a.clone(),
113 }),
114 (Some(n), None) => Err(PlacementError::ArtifactSilent {
115 node: node.name.to_string(),
116 node_platform: n.clone(),
117 }),
118 // Both silent: the single-platform world. MNW is here, and stays
119 // here until its nodes declare a platform — at which point its
120 // builds have to as well, which is the forcing function rather than
121 // a silently mixed state.
122 (None, None) => Ok(Self { node, bundle }),
123 }
124 }
125
126 pub fn node(&self) -> &'a Node {
127 self.node
128 }
129
130 pub fn bundle(&self) -> &'a Path {
131 self.bundle
132 }
133 }
134
135 /// Keep this many release dirs per node; older ones get gc'd after a
136 /// successful deploy. Fixed for now; promote to config if the constant ever
137 /// needs to vary by tier.
138 const RELEASES_TO_KEEP: usize = 5;
139
140 /// A sink that drops streamed bytes. Deploy steps don't have a live-log handle
141 /// (gates do), so output is discarded as it streams; [`RunOutput`] still
142 /// captures the full stdout/stderr for error reporting, preserving the
143 /// pre-extraction behavior of surfacing `stderr` in failure messages.
144 struct DiscardSink;
145
146 #[async_trait]
147 impl LogSink for DiscardSink {
148 async fn write_chunk(&mut self, _bytes: &[u8]) {}
149 }
150
151 /// Run a shell step through `executor`, treating a non-zero exit as an error
152 /// whose message carries the captured stderr — exactly as the old bespoke
153 /// `ssh()` helper did (`ssh <target> failed: <stderr>`).
154 async fn run_checked(executor: &dyn Executor, script: &str, what: &str) -> Result<RunOutput> {
155 let step = Step::shell(Action::Deploy, script);
156 let mut sink = DiscardSink;
157 let out = executor
158 .run_streaming(&step, &mut sink)
159 .await
160 .with_context(|| format!("{what}: spawning command"))?;
161 anyhow::ensure!(
162 out.status.success(),
163 "{what} failed (exit {}): {}",
164 out.status
165 .code()
166 .map_or_else(|| "signal".into(), |c| c.to_string()),
167 String::from_utf8_lossy(&out.stderr),
168 );
169 Ok(out)
170 }
171
172 /// Stage built binaries into `staging/<build_id>/` on the Sando host — a
173 /// private, mutable scratch dir that is not yet a release (no symlink, no gc).
174 /// The caller adds `release_contents` + companions, hashes the result, writes
175 /// the `MANIFEST`, then publishes it content-addressed via
176 /// [`finalize_local_release`]. Splitting staging from publish is what lets the
177 /// bundle be hashed before it is named (wiki [[release-artifact-identity]]).
178 ///
179 /// A stale `staging/<build_id>` from a killed prior run at the same id is
180 /// removed first, so a retry stages clean.
181 pub async fn stage_local_bundle(
182 release_root: &Path,
183 build_id: i64,
184 binaries: &[PathBuf],
185 ) -> Result<PathBuf> {
186 let staging = release_root.join("staging").join(build_id.to_string());
187 if tokio::fs::try_exists(&staging).await.unwrap_or(false) {
188 tokio::fs::remove_dir_all(&staging)
189 .await
190 .with_context(|| format!("clearing stale staging dir {}", staging.display()))?;
191 }
192 tokio::fs::create_dir_all(&staging).await?;
193 for binary in binaries {
194 let name = binary.file_name().context("binary path has no file name")?;
195 let dest = staging.join(name);
196 tokio::fs::copy(binary, &dest)
197 .await
198 .with_context(|| format!("copy {} -> {}", binary.display(), dest.display()))?;
199 }
200 Ok(staging)
201 }
202
203 /// Publish a fully-staged bundle content-addressed: rename
204 /// `staging/<build_id>` to `releases/<digest16>` (atomic same-filesystem
205 /// rename), flip `current` to it, gc old releases. Returns the released dir.
206 ///
207 /// The rename is the load-bearing step: **a directory whose name derives from
208 /// its contents cannot be rewritten, because rewriting it changes its name.**
209 /// The overwrite class that let a dev rebuild inherit an earlier build's gate
210 /// rows and burn-in clock stops being something to guard against and becomes
211 /// something that cannot be expressed. If a release with this digest already
212 /// exists (identical bytes rebuilt), the staging copy is redundant and dropped.
213 pub async fn finalize_local_release(
214 release_root: &Path,
215 staging: &Path,
216 digest16: &str,
217 ) -> Result<PathBuf> {
218 let releases = release_root.join("releases");
219 tokio::fs::create_dir_all(&releases).await?;
220 let released = releases.join(digest16);
221
222 if tokio::fs::try_exists(&released).await.unwrap_or(false) {
223 // Same digest already published — reuse it, discard the redundant stage.
224 tokio::fs::remove_dir_all(staging).await.ok();
225 } else {
226 tokio::fs::rename(staging, &released)
227 .await
228 .with_context(|| format!("publish {} -> {}", staging.display(), released.display()))?;
229 }
230
231 let current = release_root.join("current");
232 let target = format!("releases/{digest16}");
233 let out = Command::new("ln")
234 .args(["-sfn", &target])
235 .arg(&current)
236 .output()
237 .await?;
238 anyhow::ensure!(
239 out.status.success(),
240 "symlink swap failed: {}",
241 String::from_utf8_lossy(&out.stderr),
242 );
243
244 if let Err(e) = gc_local_releases(release_root).await {
245 tracing::warn!(error = %e, "local release GC failed (non-fatal)");
246 }
247 Ok(released)
248 }
249
250 /// Deploy a [`Placement`]'s bundle to its node using `executor` (the node's
251 /// transport from the topology executor map). For `ssh_target=local`, this is
252 /// just a symlink swap; for remote nodes, we rsync the whole dir over the
253 /// executor.
254 ///
255 /// The bundle and the node arrive together inside the placement, so there is no
256 /// signature here that accepts a bundle and a node that were never compared.
257 ///
258 /// `primary_bin` is only used for logging — every file present in the staged
259 /// dir gets shipped.
260 pub async fn deploy_node(
261 executor: &dyn Executor,
262 placement: Placement<'_>,
263 version: &str,
264 primary_bin: &str,
265 ) -> Result<PathBuf> {
266 let node = placement.node();
267 let staged_release_dir = placement.bundle();
268 // The release dir is named for its content digest (`releases/<digest16>`),
269 // not the version. The node mirrors that name so host and node agree on the
270 // artifact's identity; the version is only a log label here. Legacy staged
271 // dirs (pre-identity, still `releases/<version>`) work unchanged — the name
272 // is whatever the host staged under.
273 let release_id = staged_release_dir
274 .file_name()
275 .and_then(|n| n.to_str())
276 .with_context(|| {
277 format!(
278 "staged release dir {} has no usable name",
279 staged_release_dir.display()
280 )
281 })?;
282 if node.ssh_target == "local" || node.ssh_target.is_empty() {
283 // Local deploy already happened when we staged on the Sando host.
284 // Just re-point `current` at the staged dir.
285 return reset_local_current(executor, Path::new(&node.release_root), release_id).await;
286 }
287 deploy_remote(
288 executor,
289 node,
290 version,
291 release_id,
292 staged_release_dir,
293 primary_bin,
294 )
295 .await
296 }
297
298 /// Where a node deploy failed, relative to the symlink swap.
299 ///
300 /// The distinction is the whole difference between "nothing happened" and "go
301 /// look at production now", and it used to be carried only in the wording of a
302 /// `.context()` string, which meant the reporting layer could not act on it. It
303 /// reported every rollback failure as though the node were stranded on the new
304 /// version — including the case where the node had never left the old one,
305 /// which is the safe case and the common one.
306 ///
307 /// Attached as `anyhow` context, so it both reads correctly in the error chain
308 /// and can be recovered with `stage_of`.
309 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
310 pub enum FailureStage {
311 /// Failed before the swap ran. `current` still points at the old release
312 /// and the service was never restarted, so the node is on the OLD version.
313 /// Nothing is stranded and nothing needs doing.
314 BeforeSwap,
315 /// Failed at or after the swap. The node's version is not knowable from
316 /// here: the swap script rolls `current` back if the restart fails, but a
317 /// failure between the two, or in a companion after the server is already
318 /// live, can leave the node on either version.
319 AtOrAfterSwap,
320 }
321
322 impl std::fmt::Display for FailureStage {
323 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
324 match self {
325 Self::BeforeSwap => {
326 f.write_str("current symlink left intact; node is on the previous version")
327 }
328 Self::AtOrAfterSwap => {
329 f.write_str("the symlink swap had already run; node version is indeterminate")
330 }
331 }
332 }
333 }
334
335 /// Recover the [`FailureStage`] from a deploy error's context chain.
336 ///
337 /// `None` means the error predates the stage annotation or came from somewhere
338 /// that does not set one. Callers must treat that as indeterminate rather than
339 /// as safe: guessing "before the swap" would reintroduce the bug in the
340 /// opposite, worse direction.
341 pub fn stage_of(err: &anyhow::Error) -> Option<FailureStage> {
342 // anyhow's own downcast_ref searches attached context values, not just the
343 // source chain, which is where a `.context(FailureStage::…)` lands.
344 err.downcast_ref::<FailureStage>().copied()
345 }
346
347 async fn reset_local_current(
348 executor: &dyn Executor,
349 release_root: &Path,
350 release_id: &str,
351 ) -> Result<PathBuf> {
352 let current = release_root.join("current");
353 let target = format!("releases/{release_id}");
354 run_checked(
355 executor,
356 &format!(
357 "ln -sfn {} {}",
358 sh_quote(&target),
359 sh_quote(&current.to_string_lossy())
360 ),
361 "local symlink swap",
362 )
363 .await?;
364 Ok(release_root.join("releases").join(release_id))
365 }
366
367 async fn deploy_remote(
368 executor: &dyn Executor,
369 node: &Node,
370 version: &str,
371 release_id: &str,
372 staged_release_dir: &Path,
373 primary_bin: &str,
374 ) -> Result<PathBuf> {
375 let release_root = &node.release_root;
376 let service = &node.service_name;
377 let release_dir = format!("{release_root}/releases/{release_id}");
378
379 tracing::info!(node = %node.name, version, release_id, "deploy: mkdir release dir");
380 run_checked(
381 executor,
382 &format!("set -e; mkdir -p {q}", q = sh_quote(&release_dir)),
383 "creating remote release dir",
384 )
385 .await
386 .context(FailureStage::BeforeSwap)?;
387
388 tracing::info!(node = %node.name, version, primary = %primary_bin, "deploy: rsync release dir");
389 // Rsync the whole staged dir (binaries + every release_contents entry).
390 // `SyncOpts::release_mirror()` is the exact pre-extraction rsync flag set:
391 // -az --partial --delete --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r,F+X.
392 // --delete: removed assets across versions don't accumulate on the
393 // target; the bundle stays self-contained per version.
394 // --chmod: F+X preserves the execute bit per-file (binaries land 0755,
395 // data files 0644) instead of a blanket 0755.
396 executor
397 .push_dir(
398 staged_release_dir,
399 Path::new(&release_dir),
400 &SyncOpts::release_mirror(),
401 )
402 .await
403 .context("rsync failed")
404 .context(FailureStage::BeforeSwap)?;
405
406 // Verify the bundle on the node against its own MANIFEST before the swap
407 // (invariant 3, wiki [[release-artifact-identity]]). The MANIFEST shipped in
408 // the bundle is exactly `sha256sum` check format (`<hash> <relpath>`), so
409 // this re-hashes every file on the node and names any that drifted in
410 // transit — the hash is load-bearing, not merely recorded. Bundles staged
411 // by a pre-identity build carry no MANIFEST; those skip verification (logged)
412 // rather than fail, so a mid-migration deploy of a legacy artifact still
413 // ships. A mismatch fails the promote with the running service intact.
414 run_checked(
415 executor,
416 &manifest_verify_script(&release_dir),
417 "verifying bundle digest on node",
418 )
419 .await
420 .context("node-side bundle verification failed")
421 .context(FailureStage::BeforeSwap)?;
422
423 // Fail closed on a wrong-architecture binary before the symlink swap. The
424 // "never cross-compile" rule is enforced at build time (build_host check),
425 // but nothing verified the artifact's arch matched the *target* node — so
426 // adding an aarch64 node to a tier built on x86_64 would silently symlink an
427 // unrunnable binary live. Compare the deployed binary's ELF e_machine to the
428 // node's `uname -m`; unknown arches log and proceed (can't verify != known-bad).
429 let deployed_bin = format!("{release_dir}/{primary_bin}");
430 run_checked(
431 executor,
432 &arch_guard_script(&deployed_bin),
433 "verifying binary arch matches node",
434 )
435 .await
436 .context("deployed binary architecture does not match the target node")
437 .context(FailureStage::BeforeSwap)?;
438
439 // Config-drift guard (opt-in per node). Runs the freshly-rsynced binary in
440 // config-only mode with the node's env sourced, BEFORE the swap, so a
441 // required var missing on this node fails here — service still intact —
442 // rather than after the restart, which would crash-loop it (how testnot
443 // went down on a missing CDN_BASE_URL). Skipped unless the node sets
444 // `config_check_env_file`.
445 if let Some(env_file) = node.config_check_env_file.as_deref() {
446 tracing::info!(node = %node.name, version, "deploy: pre-swap config check");
447 check_target_config(executor, &deployed_bin, env_file)
448 .await
449 .context("pre-swap config check failed")
450 .context(FailureStage::BeforeSwap)?;
451 }
452
453 tracing::info!(node = %node.name, version, "deploy: symlink swap + service reload");
454 let restart_cmd = format!(
455 "sudo /bin/systemctl reload-or-restart {}",
456 sh_quote(service)
457 );
458 let swap_and_restart = swap_and_restart_script(release_root, release_id, &restart_cmd);
459 run_checked(
460 executor,
461 &swap_and_restart,
462 "symlink swap + systemctl reload-or-restart",
463 )
464 .await
465 .context(FailureStage::AtOrAfterSwap)?;
466
467 // Companion services (opt-in per node): install each from the just-rsynced
468 // bundle and restart its unit via the node-side wrapper, AFTER the server is
469 // up (mnw-cli is `After=makenotwork.service`). They shipped from the SAME sha
470 // in this SAME bundle — the lockstep guarantee. A failure here fails the
471 // promote: a companion is part of the deploy, not a best-effort side effect.
472 for c in &node.companions {
473 let src = format!(
474 "{release_root}/releases/{release_id}/companions/{name}",
475 name = c.name,
476 );
477 tracing::info!(node = %node.name, companion = %c.name, "deploy: install companion + restart");
478 let cmd = install_companion_cmd(&src, &c.install_path, &c.service_name);
479 run_checked(executor, &cmd, "install companion + restart")
480 .await
481 .with_context(|| {
482 format!(
483 "companion {} deploy failed (server already swapped)",
484 c.name
485 )
486 })
487 .context(FailureStage::AtOrAfterSwap)?;
488 }
489
490 if let Err(e) = gc_remote_releases(executor, release_root).await {
491 tracing::warn!(error = %e, "remote release GC failed (non-fatal)");
492 }
493
494 Ok(PathBuf::from(release_root)
495 .join("releases")
496 .join(release_id))
497 }
498
499 /// Absolute path of the node-side companion installer (shipped once per node;
500 /// granted to the deploy user by a single scoped sudoers line). It installs the
501 /// staged binary to its `ExecStart` path and restarts the unit — keeping the
502 /// sudo grant to one script rather than a broad `install`/`systemctl` grant.
503 const COMPANION_INSTALLER: &str = "/usr/local/lib/mnw/install-companion.sh";
504
505 /// Command run on the node to install a staged companion binary and restart its
506 /// unit, via the wrapper. Pure builder so it can be unit-tested; all three args
507 /// are shell-quoted (paths/unit names, operator config — but quoted regardless).
508 fn install_companion_cmd(src: &str, install_path: &str, service: &str) -> String {
509 format!(
510 "sudo {installer} {src} {dst} {svc}",
511 installer = sh_quote(COMPANION_INSTALLER),
512 src = sh_quote(src),
513 dst = sh_quote(install_path),
514 svc = sh_quote(service),
515 )
516 }
517
518 /// Pre-swap config-drift check: load the node's env file the way systemd loads
519 /// it, then run the freshly-deployed binary in `MNW_CHECK_CONFIG=1` mode (loads
520 /// config, exits 0/1, no DB/migrations/bind). A non-zero exit — a required var
521 /// missing — is surfaced by `run_checked` as an error, failing the promote
522 /// before the swap.
523 ///
524 /// Bounded by a timeout as a backstop: a binary predating `MNW_CHECK_CONFIG`
525 /// would ignore the var and try to start normally, which must not hang the
526 /// deploy. A timeout is reported as a failure (fail closed) — the operator only
527 /// opts a node in once a check-capable version is deployed, so a timeout means
528 /// something is wrong, not a routine older binary.
529 async fn check_target_config(
530 executor: &dyn Executor,
531 deployed_bin: &str,
532 env_file: &str,
533 ) -> Result<()> {
534 // Readability first, as its own step with its own message.
535 //
536 // The env file is read by this check AS THE DEPLOY USER, and it is the only
537 // thing that does. systemd loads `EnvironmentFile=` as root before dropping
538 // to `User=`, so the running service does not care about the mode — which
539 // means a file rewritten 0600 breaks the next deploy while the current one
540 // keeps serving, and the breakage is invisible until someone ships. That is
541 // exactly how prod deploy 0.11.3 failed on 2026-08-01.
542 //
543 // Without this step the operator gets `bash: line 9: <file>: Permission
544 // denied` out of a generated script and has to reverse-engineer which user
545 // and which file. Naming the user, the mode and the owner turns that into a
546 // one-line read.
547 let probe = readability_probe_script(env_file);
548 if let Ok(Err(e)) = tokio::time::timeout(
549 std::time::Duration::from_secs(20),
550 run_checked(executor, &probe, "env file readability"),
551 )
552 .await
553 {
554 return Err(e).context(format!(
555 "the deploy user cannot read {env_file}. systemd reads EnvironmentFile= as root, so \
556 the running service is unaffected and this breaks only deploys. Expected mode 0640 \
557 owned root:<service user> (see sando/deploy/bootstrap-node.sh); something that \
558 rewrote the file likely did so with a 077 umask"
559 ));
560 }
561
562 let script = config_check_script(env_file, deployed_bin);
563 let fut = run_checked(executor, &script, "pre-swap config check");
564 match tokio::time::timeout(std::time::Duration::from_secs(20), fut).await {
565 Ok(result) => result.map(|_| ()),
566 Err(_) => anyhow::bail!(
567 "pre-swap config check timed out after 20s — the binary may predate \
568 MNW_CHECK_CONFIG or the check hung; refusing to swap"
569 ),
570 }
571 }
572
573 /// Assert the deploy user can read `env_file`, reporting who it is and what the
574 /// file actually looks like when it cannot.
575 ///
576 /// `stat` output is best-effort: a node without it (or a file that does not
577 /// exist) still gets the identity line, which is the half an operator cannot
578 /// derive from the failure on their own.
579 fn readability_probe_script(env_file: &str) -> String {
580 format!(
581 "if [ ! -e {env} ]; then\n\
582 \techo \"{env_disp}: does not exist on this node\" >&2; exit 1\n\
583 fi\n\
584 if [ ! -r {env} ]; then\n\
585 \techo \"cannot read {env_disp} as $(id -un) (groups: $(id -Gn))\" >&2\n\
586 \tstat -c 'actual: mode %a owner %U:%G' {env} >&2 2>/dev/null || true\n\
587 \texit 1\n\
588 fi\n",
589 env = sh_quote(env_file),
590 env_disp = env_file,
591 )
592 }
593
594 /// Shell that loads `env_file` with systemd `EnvironmentFile=` semantics, then
595 /// runs `bin` under `MNW_CHECK_CONFIG=1`.
596 ///
597 /// Load the file line by line and `export` each `KEY=VALUE` verbatim rather than
598 /// `. env_file`. Dot-sourcing runs the file as a script, so any shell
599 /// metacharacter in a value (`$`, backticks, `;`, `&`, a glob, whitespace) is
600 /// expanded or word-split — a DB URL carrying a password silently dropped
601 /// `DATABASE_URL` to empty on our nodes, which would fail the check (and thus
602 /// every deploy) even though systemd starts the service fine. `export "$line"`
603 /// assigns the already-expanded word literally, matching systemd's "no variable
604 /// expansion" rule. Comments and blank lines are skipped; the `|| [ -n "$line" ]`
605 /// guard processes a final line with no trailing newline. (Quoted values —
606 /// `KEY="v"` — aren't unquoted here the way systemd would, but our env files use
607 /// bare `KEY=VALUE`, and a stray quote can only make the check stricter, never
608 /// wave a bad config through.)
609 fn config_check_script(env_file: &str, bin: &str) -> String {
610 format!(
611 "set -eu\n\
612 while IFS= read -r __sando_l || [ -n \"$__sando_l\" ]; do\n\
613 \tcase \"$__sando_l\" in ''|'#'*) continue ;; esac\n\
614 \texport \"$__sando_l\"\n\
615 done < {env}\n\
616 MNW_CHECK_CONFIG=1 {bin}\n",
617 env = sh_quote(env_file),
618 bin = sh_quote(bin),
619 )
620 }
621
622 /// Build the swap-and-restart shell script for a remote node.
623 ///
624 /// The symlink swap is atomic via `mv -T` of a freshly-created symlink over the
625 /// old one (the rename(2) is the atomic step; `ln -sfn` alone does
626 /// unlink+symlink, which has a window). The load-bearing part: if `restart_cmd`
627 /// fails *after* the flip, `current` is rolled back to its prior target before
628 /// the script exits non-zero. Otherwise a failed restart would leave `current`
629 /// pointing at the new, un-activated release while the service still runs the
630 /// old one — and a later reboot/cron restart would then silently bring up the
631 /// release the deploy reported as failed. Best-effort re-restart of the prior
632 /// version keeps the running service consistent with the restored symlink.
633 ///
634 /// `restart_cmd` is injected (rather than hardcoded) so tests can drive the
635 /// failure and success paths with a `false`/`true` stand-in.
636 fn swap_and_restart_script(release_root: &str, release_id: &str, restart_cmd: &str) -> String {
637 format!(
638 "set -e\n\
639 cd {root}\n\
640 prev=$(readlink current 2>/dev/null || true)\n\
641 ln -sfn releases/{rel} current.new\n\
642 mv -Tf current.new current\n\
643 if ! {restart}; then\n\
644 if [ -n \"$prev\" ]; then\n\
645 ln -sfn \"$prev\" current.rollback\n\
646 mv -Tf current.rollback current\n\
647 {restart} || true\n\
648 fi\n\
649 echo \"deploy: restart failed; rolled symlink back to ${{prev:-<none>}}\" >&2\n\
650 exit 1\n\
651 fi\n",
652 root = sh_quote(release_root),
653 rel = sh_quote(release_id),
654 restart = restart_cmd,
655 )
656 }
657
658 /// Shell that re-hashes the rsynced bundle on the node against its shipped
659 /// `MANIFEST` and aborts (exit 1) if any file drifted (invariant 3, wiki note
660 /// `release-artifact-identity`). The `MANIFEST` is `sha256sum` check format,
661 /// so `sha256sum -c` verifies every listed file with node-native tooling and
662 /// names the one that failed. `--strict` fails on a malformed manifest line;
663 /// `--quiet` drops the per-file OK spam and keeps only failures.
664 ///
665 /// A bundle staged by a pre-identity build carries no `MANIFEST`; that is not an
666 /// error — it logs a skip and exits 0, so a mid-migration deploy of a legacy
667 /// artifact still ships. Once every tier has cycled once, every bundle has one.
668 fn manifest_verify_script(release_dir: &str) -> String {
669 format!(
670 "set -e\n\
671 cd {dir}\n\
672 if [ ! -f MANIFEST ]; then\n\
673 echo \"deploy: no MANIFEST in bundle; skipping digest verification (legacy artifact)\" >&2\n\
674 exit 0\n\
675 fi\n\
676 sha256sum --quiet --strict -c MANIFEST\n",
677 dir = sh_quote(release_dir),
678 )
679 }
680
681 /// Shell that aborts (exit 1) if `bin`'s ELF architecture doesn't match the
682 /// node it's running on. Reads the ELF `e_machine` field (2 bytes LE at offset
683 /// 18) and compares it to the value implied by `uname -m`. An arch we don't have
684 /// a mapping for logs and proceeds — the guard exists to catch the concrete
685 /// x86_64-vs-aarch64 confusion, not to gate genuinely-new targets.
686 fn arch_guard_script(bin: &str) -> String {
687 format!(
688 "set -e\n\
689 bin={bin}\n\
690 arch=$(uname -m)\n\
691 machine=$(od -An -tx1 -j18 -N2 \"$bin\" 2>/dev/null | tr -d ' \\n')\n\
692 case \"$arch\" in\n\
693 x86_64|amd64) want=3e00 ;;\n\
694 aarch64|arm64) want=b700 ;;\n\
695 *) echo \"deploy: arch check skipped (unmapped node arch $arch)\" >&2; want= ;;\n\
696 esac\n\
697 if [ -n \"$want\" ] && [ \"$machine\" != \"$want\" ]; then\n\
698 echo \"deploy: arch mismatch — node $arch expects e_machine $want but binary has ${{machine:-<unreadable>}}\" >&2\n\
699 exit 1\n\
700 fi\n",
701 bin = sh_quote(bin),
702 )
703 }
704
705 async fn gc_local_releases(release_root: &Path) -> Result<()> {
706 let releases = release_root.join("releases");
707 if !releases.exists() {
708 return Ok(());
709 }
710 let mut entries = Vec::new();
711 let mut rd = tokio::fs::read_dir(&releases).await?;
712 while let Some(entry) = rd.next_entry().await? {
713 if !entry.file_type().await?.is_dir() {
714 continue;
715 }
716 let meta = entry.metadata().await?;
717 entries.push((entry.path(), meta.modified()?));
718 }
719 entries.sort_by_key(|e| std::cmp::Reverse(e.1));
720 for (path, _) in entries.into_iter().skip(RELEASES_TO_KEEP) {
721 if let Err(e) = tokio::fs::remove_dir_all(&path).await {
722 tracing::warn!(path = %path.display(), error = %e, "gc: rm failed");
723 } else {
724 tracing::debug!(path = %path.display(), "gc: removed old release");
725 }
726 }
727 Ok(())
728 }
729
730 async fn gc_remote_releases(executor: &dyn Executor, release_root: &str) -> Result<()> {
731 // `ls -t` orders by mtime desc. Skip the first N, rm the rest. `xargs -r`
732 // is a no-op when stdin is empty (avoids `rm` complaining).
733 let script = format!(
734 "set -e; cd {root}/releases 2>/dev/null || exit 0; \
735 ls -1t | tail -n +{keep_plus_one} | xargs -r -I{{}} rm -rf -- {{}}",
736 root = sh_quote(release_root),
737 keep_plus_one = RELEASES_TO_KEEP + 1,
738 );
739 run_checked(executor, &script, "remote release gc")
740 .await
741 .map(|_| ())
742 }
743
744 #[cfg(test)]
745 mod tests {
746 use super::*;
747 use crate::topology::NodeCompanion;
748 use ops_exec::{CapabilitySet, LocalExec, SshExec};
749 use std::os::unix::process::ExitStatusExt;
750 use std::sync::{Arc, Mutex as StdMutex};
751 use std::time::SystemTime;
752
753 // ---- placement ----
754 //
755 // The whole table, because the interesting cases are the two where one side
756 // said nothing. Treating silence as agreement is how a wrong-architecture
757 // deploy would get through, and it is the shape a "check it before you call"
758 // guard tends to end up with.
759
760 fn node_on(platform: Option<&str>) -> Node {
761 Node {
762 name: crate::domain::NodeId::new("n1"),
763 ssh_target: "deploy@n1".into(),
764 release_root: "/opt/x".into(),
765 platform: platform.map(|p| Platform::parse(p).unwrap()),
766 service_name: "x.service".into(),
767 config_check_env_file: None,
768 actuate: crate::topology::default_actuate(),
769 observe: crate::topology::default_observe(),
770 health_url: None,
771 companions: Vec::new(),
772 }
773 }
774
775 #[test]
776 fn matching_platforms_are_placeable() {
777 let node = node_on(Some("linux/aarch64"));
778 let art = Platform::parse("linux/aarch64").unwrap();
779 let p = Placement::check(&node, Path::new("/r/abc"), Some(&art)).expect("a match places");
780 assert_eq!(p.bundle(), Path::new("/r/abc"));
781 assert_eq!(p.node().name.as_str(), "n1");
782 }
783
784 #[test]
785 fn a_different_architecture_is_refused() {
786 // The failure this type exists for: pom's aarch64 bundle reaching the
787 // x86_64 box, which execs nothing and takes the watcher down.
788 let node = node_on(Some("linux/x86_64"));
789 let art = Platform::parse("linux/aarch64").unwrap();
790 let err = Placement::check(&node, Path::new("/r/abc"), Some(&art)).unwrap_err();
791 assert!(
792 matches!(err, PlacementError::Mismatch { .. }),
793 "expected a mismatch, got {err}"
794 );
795 // The message has to name both, or an operator cannot tell which half
796 // is wrong.
797 let msg = err.to_string();
798 assert!(
799 msg.contains("linux/x86_64") && msg.contains("linux/aarch64"),
800 "{msg}"
801 );
802 }
803
804 #[test]
805 fn a_silent_node_refuses_a_stated_artifact() {
806 // Not "the node probably runs it". A node that never said what it is
807 // cannot vouch for a bundle that did, and the pairing that looks
808 // harmless here is exactly the one that ships the wrong half of a
809 // two-architecture release.
810 let node = node_on(None);
811 let art = Platform::parse("linux/aarch64").unwrap();
812 assert!(matches!(
813 Placement::check(&node, Path::new("/r/abc"), Some(&art)),
814 Err(PlacementError::NodeSilent { .. })
815 ));
816 }
817
818 #[test]
819 fn a_stated_node_refuses_a_silent_artifact() {
820 let node = node_on(Some("linux/aarch64"));
821 assert!(matches!(
822 Placement::check(&node, Path::new("/r/abc"), None),
823 Err(PlacementError::ArtifactSilent { .. })
824 ));
825 }
826
827 #[test]
828 fn both_silent_is_the_single_platform_world_and_still_places() {
829 // MNW is here and stays here. Its nodes declare nothing and its builds
830 // record nothing, which is the truth about a product with one build host
831 // and one architecture. The moment either side starts stating, the other
832 // has to as well — that is the forcing function, and it is why this cell
833 // is the only admissible non-match.
834 let node = node_on(None);
835 Placement::check(&node, Path::new("/r/abc"), None).expect("the pre-pom world still ships");
836 }
837
838 #[test]
839 fn platform_parsing_is_a_shape_not_a_spelling() {
840 assert_eq!(
841 Platform::parse("Linux/AArch64").unwrap(),
842 Platform::parse("linux/aarch64").unwrap(),
843 "case is not a distinction between two machines"
844 );
845 for bad in ["linux", "linux/", "/aarch64", "linux/aarch64/gnu", ""] {
846 assert!(Platform::parse(bad).is_err(), "{bad:?} should not parse");
847 }
848 }
849
850 // ---- failure stage ----
851 //
852 // The 2026-08-01 prod deploy failed its pre-swap config check, and the
853 // rollback then failed the same way — which left the node safely on the old
854 // version, and was reported as "it remains on the new version, manual
855 // intervention needed". These pin the distinction the reporting layer now
856 // depends on.
857
858 #[test]
859 fn a_pre_swap_failure_is_recoverable_as_such() {
860 let e = anyhow::anyhow!("Permission denied")
861 .context("pre-swap config check failed")
862 .context(FailureStage::BeforeSwap);
863 assert_eq!(stage_of(&e), Some(FailureStage::BeforeSwap));
864 // The reason survives alongside the stage; the stage does not replace it.
865 let rendered = format!("{e:#}");
866 assert!(
867 rendered.contains("pre-swap config check failed"),
868 "{rendered}"
869 );
870 assert!(rendered.contains("Permission denied"), "{rendered}");
871 }
872
873 #[test]
874 fn a_post_swap_failure_is_recoverable_as_such() {
875 let e = anyhow::anyhow!("unit failed to start")
876 .context("companion x deploy failed (server already swapped)")
877 .context(FailureStage::AtOrAfterSwap);
878 assert_eq!(stage_of(&e), Some(FailureStage::AtOrAfterSwap));
879 }
880
881 #[test]
882 fn an_unannotated_failure_has_no_stage() {
883 // Must be None, not a default. A caller seeing None has to treat the
884 // node as indeterminate; inferring "before the swap" would reintroduce
885 // the original bug pointing the other way, which is the dangerous way.
886 let e = anyhow::anyhow!("something older, from before stages existed");
887 assert_eq!(stage_of(&e), None);
888 }
889
890 // ---- env file readability probe ----
891
892 #[tokio::test]
893 async fn readability_probe_passes_on_a_readable_file() {
894 let tmp = tempfile::tempdir().unwrap();
895 let f = tmp.path().join("ok.env");
896 tokio::fs::write(&f, "A=1\n").await.unwrap();
897 let script = readability_probe_script(&f.to_string_lossy());
898 let out = run_checked(&local_executor(), &script, "probe").await;
899 assert!(out.is_ok(), "{:?}", out.err().map(|e| format!("{e:#}")));
900 }
901
902 #[tokio::test]
903 async fn readability_probe_names_the_user_and_mode_when_unreadable() {
904 // Root can read anything, so a mode-based test would pass spuriously
905 // there. Skip rather than assert something false. No libc dependency
906 // for one probe: a 0-mode temp file is readable iff we are root.
907 let probe_dir = tempfile::tempdir().unwrap();
908 let probe_file = probe_dir.path().join("root-check");
909 tokio::fs::write(&probe_file, "x").await.unwrap();
910 tokio::fs::set_permissions(
911 &probe_file,
912 std::os::unix::fs::PermissionsExt::from_mode(0o000),
913 )
914 .await
915 .unwrap();
916 if tokio::fs::read(&probe_file).await.is_ok() {
917 return; // running as root
918 }
919 let tmp = tempfile::tempdir().unwrap();
920 let f = tmp.path().join("locked.env");
921 tokio::fs::write(&f, "A=1\n").await.unwrap();
922 tokio::fs::set_permissions(&f, std::os::unix::fs::PermissionsExt::from_mode(0o000))
923 .await
924 .unwrap();
925
926 let script = readability_probe_script(&f.to_string_lossy());
927 let err = run_checked(&local_executor(), &script, "probe")
928 .await
929 .expect_err("an unreadable file must fail the probe");
930 let msg = format!("{err:#}");
931 // The two things the raw bash error does not tell you.
932 assert!(msg.contains("cannot read"), "{msg}");
933 assert!(msg.contains("mode 0") || msg.contains("mode "), "{msg}");
934 }
935
936 #[tokio::test]
937 async fn readability_probe_distinguishes_missing_from_unreadable() {
938 let tmp = tempfile::tempdir().unwrap();
939 let missing = tmp.path().join("nope.env");
940 let script = readability_probe_script(&missing.to_string_lossy());
941 let err = run_checked(&local_executor(), &script, "probe")
942 .await
943 .expect_err("a missing file must fail the probe");
944 let msg = format!("{err:#}");
945 assert!(msg.contains("does not exist"), "{msg}");
946 }
947
948 #[test]
949 fn the_two_stages_read_differently() {
950 // These strings end up in an operator's terminal during an incident.
951 let before = FailureStage::BeforeSwap.to_string();
952 let after = FailureStage::AtOrAfterSwap.to_string();
953 assert!(before.contains("previous version"), "{before}");
954 assert!(after.contains("indeterminate"), "{after}");
955 assert_ne!(before, after);
956 }
957
958 /// A LocalExec granted the default node capabilities (deploy + restart).
959 fn local_executor() -> LocalExec {
960 LocalExec::new(CapabilitySet::from_tokens(
961 ["deploy", "restart"],
962 ["health"],
963 ))
964 }
965
966 #[tokio::test]
967 async fn deploy_local_copies_multiple_binaries_and_swaps_symlink() {
968 let tmp = tempfile::tempdir().unwrap();
969 let root = tmp.path();
970
971 let src_dir = root.join("src");
972 tokio::fs::create_dir_all(&src_dir).await.unwrap();
973 let primary = src_dir.join("makenotwork");
974 let admin = src_dir.join("mnw-admin");
975 tokio::fs::write(&primary, b"PRIMARY").await.unwrap();
976 tokio::fs::write(&admin, b"ADMIN").await.unwrap();
977
978 let release_root = root.join("releases-root");
979 tokio::fs::create_dir_all(&release_root).await.unwrap();
980
981 // Stage into staging/<build_id> (no publish yet).
982 let staging = stage_local_bundle(&release_root, 42, &[primary.clone(), admin.clone()])
983 .await
984 .expect("stage_local_bundle should succeed");
985 assert_eq!(staging, release_root.join("staging").join("42"));
986 assert!(
987 !release_root.join("current").exists(),
988 "staging must not publish or flip current"
989 );
990
991 // Publish content-addressed at releases/<digest16>.
992 let released = finalize_local_release(&release_root, &staging, "deadbeefcafe0000")
993 .await
994 .expect("finalize_local_release should succeed");
995 assert_eq!(
996 released,
997 release_root.join("releases").join("deadbeefcafe0000")
998 );
999 assert!(
1000 !staging.exists(),
1001 "staging dir is consumed by the publish rename"
1002 );
1003 assert_eq!(
1004 tokio::fs::read(released.join("makenotwork")).await.unwrap(),
1005 b"PRIMARY"
1006 );
1007 assert_eq!(
1008 tokio::fs::read(released.join("mnw-admin")).await.unwrap(),
1009 b"ADMIN"
1010 );
1011
1012 let current = release_root.join("current");
1013 let target = tokio::fs::read_link(&current).await.unwrap();
1014 assert_eq!(target.to_string_lossy(), "releases/deadbeefcafe0000");
1015 let via_current = tokio::fs::read(current.join("makenotwork")).await.unwrap();
1016 assert_eq!(via_current, b"PRIMARY");
1017 }
1018
1019 #[tokio::test]
1020 async fn finalize_second_release_swaps_symlink_and_keeps_old_dir() {
1021 let tmp = tempfile::tempdir().unwrap();
1022 let root = tmp.path();
1023 let src_dir = root.join("src");
1024 tokio::fs::create_dir_all(&src_dir).await.unwrap();
1025 let bin = src_dir.join("server");
1026 tokio::fs::write(&bin, b"V1").await.unwrap();
1027
1028 let release_root = root.join("rr");
1029 tokio::fs::create_dir_all(&release_root).await.unwrap();
1030
1031 // Two builds, distinct digests (distinct content) -> two release dirs.
1032 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1033 .await
1034 .unwrap();
1035 finalize_local_release(&release_root, &s1, "1111111111111111")
1036 .await
1037 .unwrap();
1038 tokio::fs::write(&bin, b"V2").await.unwrap();
1039 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1040 .await
1041 .unwrap();
1042 finalize_local_release(&release_root, &s2, "2222222222222222")
1043 .await
1044 .unwrap();
1045
1046 assert!(
1047 release_root
1048 .join("releases/1111111111111111/server")
1049 .exists()
1050 );
1051 assert!(
1052 release_root
1053 .join("releases/2222222222222222/server")
1054 .exists()
1055 );
1056 let target = tokio::fs::read_link(release_root.join("current"))
1057 .await
1058 .unwrap();
1059 assert_eq!(target.to_string_lossy(), "releases/2222222222222222");
1060 let via_current = tokio::fs::read(release_root.join("current/server"))
1061 .await
1062 .unwrap();
1063 assert_eq!(via_current, b"V2");
1064 }
1065
1066 #[tokio::test]
1067 async fn finalize_reuses_an_existing_release_of_the_same_digest() {
1068 let tmp = tempfile::tempdir().unwrap();
1069 let root = tmp.path();
1070 let bin = root.join("server");
1071 tokio::fs::write(&bin, b"BYTES").await.unwrap();
1072 let release_root = root.join("rr");
1073 tokio::fs::create_dir_all(&release_root).await.unwrap();
1074
1075 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1076 .await
1077 .unwrap();
1078 finalize_local_release(&release_root, &s1, "abc123abc123abc1")
1079 .await
1080 .unwrap();
1081 // Same digest rebuilt (e.g. a re-run at the same content): finalize must
1082 // reuse the existing release and drop the redundant staging dir, not error.
1083 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1084 .await
1085 .unwrap();
1086 let released = finalize_local_release(&release_root, &s2, "abc123abc123abc1")
1087 .await
1088 .expect("finalize is idempotent on a repeated digest");
1089 assert_eq!(released, release_root.join("releases/abc123abc123abc1"));
1090 assert!(!s2.exists(), "redundant staging dropped");
1091 }
1092
1093 #[tokio::test]
1094 async fn manifest_verify_script_passes_on_match_fails_on_drift_and_skips_when_absent() {
1095 // The node-side verification is a shell running `sha256sum -c MANIFEST`;
1096 // drive the real script through bash to prove it accepts a good bundle,
1097 // rejects a tampered one, and no-ops on a legacy (MANIFEST-less) bundle.
1098 let dir = tempfile::tempdir().unwrap();
1099 tokio::fs::write(dir.path().join("server"), b"BINARY")
1100 .await
1101 .unwrap();
1102 tokio::fs::create_dir(dir.path().join("static"))
1103 .await
1104 .unwrap();
1105 tokio::fs::write(dir.path().join("static/app.css"), b"body{}")
1106 .await
1107 .unwrap();
1108 let digest = crate::bundle::digest_dir(dir.path()).await.unwrap();
1109 tokio::fs::write(dir.path().join("MANIFEST"), digest.manifest.as_bytes())
1110 .await
1111 .unwrap();
1112
1113 let run = |d: &std::path::Path| {
1114 let script = manifest_verify_script(d.to_str().unwrap());
1115 async move {
1116 Command::new("bash")
1117 .arg("-c")
1118 .arg(&script)
1119 .output()
1120 .await
1121 .unwrap()
1122 }
1123 };
1124
1125 let ok = run(dir.path()).await;
1126 assert!(
1127 ok.status.success(),
1128 "matching bundle verifies: {}",
1129 String::from_utf8_lossy(&ok.stderr)
1130 );
1131
1132 // Drift one file: sha256sum -c must fail (current symlink left intact).
1133 tokio::fs::write(dir.path().join("static/app.css"), b"TAMPERED")
1134 .await
1135 .unwrap();
1136 let bad = run(dir.path()).await;
1137 assert!(!bad.status.success(), "a drifted file fails verification");
1138
1139 // Legacy bundle with no MANIFEST: skip, not fail.
1140 let legacy = tempfile::tempdir().unwrap();
1141 tokio::fs::write(legacy.path().join("server"), b"x")
1142 .await
1143 .unwrap();
1144 let skip = run(legacy.path()).await;
1145 assert!(
1146 skip.status.success(),
1147 "a bundle without a MANIFEST skips verification rather than failing"
1148 );
1149 }
1150
1151 #[tokio::test]
1152 async fn gc_local_releases_keeps_last_n_by_mtime() {
1153 let tmp = tempfile::tempdir().unwrap();
1154 let root = tmp.path();
1155 let releases = root.join("releases");
1156 tokio::fs::create_dir_all(&releases).await.unwrap();
1157
1158 let total = RELEASES_TO_KEEP + 3;
1159 let mut names = Vec::new();
1160 for i in 0..total {
1161 let name = format!("v{i:02}");
1162 let dir = releases.join(&name);
1163 tokio::fs::create_dir(&dir).await.unwrap();
1164 let f = std::fs::File::open(&dir).unwrap();
1165 let when =
1166 SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
1167 let times = std::fs::FileTimes::new().set_modified(when);
1168 f.set_times(times).unwrap();
1169 names.push(name);
1170 }
1171
1172 gc_local_releases(root).await.unwrap();
1173
1174 let surviving_expected: Vec<_> = names
1175 .iter()
1176 .skip(total - RELEASES_TO_KEEP)
1177 .cloned()
1178 .collect();
1179 for name in &surviving_expected {
1180 assert!(releases.join(name).exists(), "expected to survive: {name}");
1181 }
1182 for name in names.iter().take(total - RELEASES_TO_KEEP) {
1183 assert!(
1184 !releases.join(name).exists(),
1185 "expected to be pruned: {name}"
1186 );
1187 }
1188 }
1189
1190 #[tokio::test]
1191 async fn gc_local_releases_noop_when_below_threshold() {
1192 let tmp = tempfile::tempdir().unwrap();
1193 let root = tmp.path();
1194 let releases = root.join("releases");
1195 tokio::fs::create_dir_all(&releases).await.unwrap();
1196 for i in 0..3 {
1197 tokio::fs::create_dir(releases.join(format!("v{i}")))
1198 .await
1199 .unwrap();
1200 }
1201 gc_local_releases(root).await.unwrap();
1202 for i in 0..3 {
1203 assert!(releases.join(format!("v{i}")).exists());
1204 }
1205 }
1206
1207 #[tokio::test]
1208 async fn gc_local_releases_noop_when_releases_dir_missing() {
1209 let tmp = tempfile::tempdir().unwrap();
1210 gc_local_releases(tmp.path()).await.unwrap();
1211 }
1212
1213 #[tokio::test]
1214 async fn deploy_remote_fails_cleanly_when_host_unreachable() {
1215 // 192.0.2.0/24 is reserved for documentation and routes nowhere.
1216 // ConnectTimeout=10 limits the test wallclock to ~10s worst case.
1217 let tmp = tempfile::tempdir().unwrap();
1218 let staged = tmp.path().join("releases").join("0.0.1");
1219 tokio::fs::create_dir_all(&staged).await.unwrap();
1220 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
1221
1222 let node = crate::topology::Node {
1223 platform: None,
1224 name: "unreachable".into(),
1225 ssh_target: "deploy@192.0.2.1".into(),
1226 release_root: "/opt/never".into(),
1227 service_name: "makenotwork.service".into(),
1228 health_url: None,
1229 config_check_env_file: None,
1230 actuate: crate::topology::default_actuate(),
1231 observe: crate::topology::default_observe(),
1232 companions: Vec::new(),
1233 };
1234 let executor = SshExec::new(
1235 node.ssh_target.clone(),
1236 CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1237 );
1238
1239 let placement = Placement::check(&node, &staged, None).expect("both sides silent");
1240 let result = deploy_node(&executor, placement, "0.0.1", "server").await;
1241 let err = result.expect_err("deploy to unreachable host should fail");
1242 let msg = format!("{err:#}");
1243 // Don't pin exact wording, just that the failure is attributed (ssh /
1244 // rsync / connection) and that no panic / hang happened.
1245 assert!(
1246 msg.contains("ssh")
1247 || msg.contains("rsync")
1248 || msg.contains("connection")
1249 || msg.contains("Connection"),
1250 "unexpected error: {msg}"
1251 );
1252 }
1253
1254 #[tokio::test]
1255 async fn deploy_node_with_local_ssh_target_swaps_symlink() {
1256 // ssh_target="local" routes to the local fast-path: just a symlink
1257 // swap, no remote calls.
1258 let tmp = tempfile::tempdir().unwrap();
1259 let release_root = tmp.path().to_path_buf();
1260 let staged = release_root.join("releases").join("0.0.1");
1261 tokio::fs::create_dir_all(&staged).await.unwrap();
1262 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
1263
1264 let node = crate::topology::Node {
1265 platform: None,
1266 name: "local-dev".into(),
1267 ssh_target: "local".into(),
1268 release_root: release_root.to_string_lossy().into_owned(),
1269 service_name: "makenotwork.service".into(),
1270 health_url: None,
1271 config_check_env_file: None,
1272 actuate: crate::topology::default_actuate(),
1273 observe: crate::topology::default_observe(),
1274 companions: Vec::new(),
1275 };
1276 let executor = local_executor();
1277
1278 let out = deploy_node(
1279 &executor,
1280 Placement::check(&node, &staged, None).unwrap(),
1281 "0.0.1",
1282 "server",
1283 )
1284 .await
1285 .unwrap();
1286 assert_eq!(out, staged);
1287 let target = tokio::fs::read_link(release_root.join("current"))
1288 .await
1289 .unwrap();
1290 assert_eq!(target.to_string_lossy(), "releases/0.0.1");
1291 }
1292
1293 // ---- swap_and_restart_script: symlink/restart consistency ----
1294
1295 async fn run_script(script: &str) -> std::process::Output {
1296 Command::new("sh")
1297 .arg("-c")
1298 .arg(script)
1299 .output()
1300 .await
1301 .unwrap()
1302 }
1303
1304 async fn setup_release_root(with_current: bool) -> tempfile::TempDir {
1305 let tmp = tempfile::tempdir().unwrap();
1306 let root = tmp.path();
1307 tokio::fs::create_dir_all(root.join("releases/old"))
1308 .await
1309 .unwrap();
1310 tokio::fs::create_dir_all(root.join("releases/new"))
1311 .await
1312 .unwrap();
1313 if with_current {
1314 std::os::unix::fs::symlink("releases/old", root.join("current")).unwrap();
1315 }
1316 tmp
1317 }
1318
1319 #[tokio::test]
1320 async fn swap_and_restart_keeps_new_symlink_when_restart_succeeds() {
1321 let tmp = setup_release_root(true).await;
1322 let root = tmp.path().to_string_lossy().into_owned();
1323 let out = run_script(&swap_and_restart_script(&root, "new", "true")).await;
1324 assert!(
1325 out.status.success(),
1326 "script should succeed when restart succeeds"
1327 );
1328 let target = tokio::fs::read_link(tmp.path().join("current"))
1329 .await
1330 .unwrap();
1331 assert_eq!(
1332 target.to_string_lossy(),
1333 "releases/new",
1334 "symlink advanced to new"
1335 );
1336 }
1337
1338 #[tokio::test]
1339 async fn swap_and_restart_rolls_symlink_back_when_restart_fails() {
1340 // The bug: a restart failure after the flip must NOT leave `current`
1341 // pointing at the new (un-activated) release.
1342 let tmp = setup_release_root(true).await;
1343 let root = tmp.path().to_string_lossy().into_owned();
1344 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
1345 assert!(!out.status.success(), "script must fail when restart fails");
1346 let target = tokio::fs::read_link(tmp.path().join("current"))
1347 .await
1348 .unwrap();
1349 assert_eq!(
1350 target.to_string_lossy(),
1351 "releases/old",
1352 "symlink rolled back to prev so a later restart can't silently activate new",
1353 );
1354 }
1355
1356 // ---- arch_guard_script: wrong-arch artifacts fail closed ----
1357
1358 /// A 20-byte stub whose ELF e_machine field (offset 18, 2 bytes LE) is set.
1359 fn elf_stub_with_machine(b18: u8, b19: u8) -> tempfile::NamedTempFile {
1360 let mut data = vec![0u8; 20];
1361 data[18] = b18;
1362 data[19] = b19;
1363 let f = tempfile::NamedTempFile::new().unwrap();
1364 std::fs::write(f.path(), &data).unwrap();
1365 f
1366 }
1367
1368 /// e_machine low byte for the host running the test, if mapped.
1369 fn host_machine_lo() -> Option<u8> {
1370 match std::env::consts::ARCH {
1371 "x86_64" => Some(0x3e),
1372 "aarch64" => Some(0xb7),
1373 _ => None,
1374 }
1375 }
1376
1377 #[tokio::test]
1378 async fn arch_guard_passes_for_matching_binary() {
1379 let Some(lo) = host_machine_lo() else { return };
1380 let f = elf_stub_with_machine(lo, 0x00);
1381 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
1382 assert!(
1383 out.status.success(),
1384 "matching arch must pass: {}",
1385 String::from_utf8_lossy(&out.stderr),
1386 );
1387 }
1388
1389 #[tokio::test]
1390 async fn arch_guard_fails_closed_for_wrong_binary() {
1391 // Use the other arch's e_machine so it can't match the host.
1392 let wrong = match std::env::consts::ARCH {
1393 "x86_64" => 0xb7, // aarch64 binary on an x86_64 node
1394 "aarch64" => 0x3e, // x86_64 binary on an aarch64 node
1395 _ => return,
1396 };
1397 let f = elf_stub_with_machine(wrong, 0x00);
1398 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
1399 assert!(
1400 !out.status.success(),
1401 "wrong-arch binary must fail closed before the symlink swap"
1402 );
1403 }
1404
1405 #[tokio::test]
1406 async fn swap_and_restart_first_deploy_failure_has_no_prev_to_restore() {
1407 // No prior `current`. A restart failure leaves `current` at new (the only
1408 // version) and still reports failure — documented degenerate case.
1409 let tmp = setup_release_root(false).await;
1410 let root = tmp.path().to_string_lossy().into_owned();
1411 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
1412 assert!(!out.status.success(), "script must fail when restart fails");
1413 let target = tokio::fs::read_link(tmp.path().join("current"))
1414 .await
1415 .unwrap();
1416 assert_eq!(
1417 target.to_string_lossy(),
1418 "releases/new",
1419 "no prev existed to roll back to"
1420 );
1421 }
1422
1423 // ---- config_check_script: systemd-faithful env loading ----
1424
1425 #[tokio::test]
1426 async fn config_check_script_loads_values_with_shell_metachars() {
1427 // The bug: `. env_file` expands/word-splits values, so a URL or a
1428 // password containing a shell metacharacter is mangled — it dropped
1429 // DATABASE_URL to empty on a real node, which would fail every deploy.
1430 // The export-loop must load such a value intact. The "binary" is a
1431 // checker script (a real path, like a deployed binary) that exits 0 only
1432 // if the var arrived byte-for-byte — it compares against the expected
1433 // value read from a file, so nothing re-interprets the metacharacters.
1434 let tricky = "postgres://u:p$ss;w&rd@h/db `x` $(y)";
1435 // Plain files in a tempdir: no lingering write fd, so the checker can be
1436 // exec'd (a NamedTempFile stays open and would ETXTBSY).
1437 let dir = tempfile::tempdir().unwrap();
1438 let expected_path = dir.path().join("expected");
1439 std::fs::write(&expected_path, tricky).unwrap(); // no trailing newline
1440
1441 let env_path = dir.path().join("node.env");
1442 std::fs::write(
1443 &env_path,
1444 format!(
1445 "# a comment\n\nDATABASE_URL={tricky}\nOTHER=plain\nEXPECTED_FILE={ef}\n",
1446 ef = expected_path.display(),
1447 ),
1448 )
1449 .unwrap();
1450
1451 let checker_path = dir.path().join("checker.sh");
1452 std::fs::write(
1453 &checker_path,
1454 "#!/bin/sh\nwant=$(cat \"$EXPECTED_FILE\")\n\
1455 [ \"$DATABASE_URL\" = \"$want\" ] || { echo \"DB [$DATABASE_URL] != [$want]\" >&2; exit 1; }\n\
1456 [ \"$OTHER\" = plain ] || { echo \"OTHER [$OTHER]\" >&2; exit 1; }\n",
1457 )
1458 .unwrap();
1459 std::fs::set_permissions(
1460 &checker_path,
1461 std::os::unix::fs::PermissionsExt::from_mode(0o755),
1462 )
1463 .unwrap();
1464
1465 let script =
1466 config_check_script(&env_path.to_string_lossy(), &checker_path.to_string_lossy());
1467 let out = run_script(&script).await;
1468 assert!(
1469 out.status.success(),
1470 "value with shell metachars must load intact; stderr: {}",
1471 String::from_utf8_lossy(&out.stderr),
1472 );
1473 }
1474
1475 // ---- install-companion.sh: the node-side guard rails ----
1476
1477 /// Run the shipped installer script with three args; returns its exit code.
1478 /// Exercises the real file rather than a copy of its logic, because the
1479 /// script is the ONLY control on a NOPASSWD sudo grant.
1480 fn run_installer(src: &str, dst: &str, service: &str) -> i32 {
1481 let script =
1482 std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../deploy/install-companion.sh");
1483 std::process::Command::new("bash")
1484 .arg(&script)
1485 .args([src, dst, service])
1486 .output()
1487 .expect("running install-companion.sh")
1488 .status
1489 .code()
1490 .expect("script exited via signal")
1491 }
1492
1493 // Guards run before any filesystem write, so these never install anything.
1494 // Exit 3 = refused by a guard; exit 4 = guards passed, src simply absent.
1495 const REFUSED: i32 = 3;
1496 const PASSED_GUARDS: i32 = 4;
1497
1498 #[test]
1499 fn installer_refuses_a_dst_that_escapes_opt_via_dotdot() {
1500 // `/opt/../etc/...` matches a bare `/opt/*` glob. With the sudoers
1501 // wildcard that meant `install -m 0755` as root to anywhere, plus a
1502 // restart of any unit — so the path must be normalised before the test.
1503 assert_eq!(
1504 run_installer(
1505 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1506 "/opt/../etc/systemd/system/evil.service",
1507 "mnw-cli.service",
1508 ),
1509 REFUSED,
1510 );
1511 }
1512
1513 #[test]
1514 fn installer_refuses_a_src_that_escapes_the_bundle_via_dotdot() {
1515 assert_eq!(
1516 run_installer(
1517 "/opt/mnw/releases/1.0.0/companions/../../../../../etc/shadow",
1518 "/opt/mnw-cli/mnw-cli",
1519 "mnw-cli.service",
1520 ),
1521 REFUSED,
1522 );
1523 }
1524
1525 #[test]
1526 fn installer_accepts_the_real_companion_paths() {
1527 // The guards must not have been tightened into uselessness: the shape
1528 // Sando actually sends has to get past them. It stops at the missing
1529 // src (exit 4), which is proof the guards accepted it.
1530 assert_eq!(
1531 run_installer(
1532 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1533 "/opt/mnw-cli/mnw-cli",
1534 "mnw-cli.service",
1535 ),
1536 PASSED_GUARDS,
1537 );
1538 }
1539
1540 #[test]
1541 fn installer_refuses_a_service_name_with_a_path_separator() {
1542 assert_eq!(
1543 run_installer(
1544 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1545 "/opt/mnw-cli/mnw-cli",
1546 "../../etc/evil.service",
1547 ),
1548 REFUSED,
1549 );
1550 }
1551
1552 // ---- install_companion_cmd: shape + quoting ----
1553
1554 #[test]
1555 fn install_companion_cmd_shape_and_quoting() {
1556 let cmd = install_companion_cmd(
1557 "/opt/mnw/releases/0.10.14/companions/mnw-cli",
1558 "/opt/mnw-cli/mnw-cli",
1559 "mnw-cli.service",
1560 );
1561 // Routes through the wrapper (single sudoers grant), sudo-invoked, with
1562 // src, dst, service in that order.
1563 assert!(cmd.starts_with("sudo "), "must be sudo-invoked: {cmd}");
1564 assert!(
1565 cmd.contains("/usr/local/lib/mnw/install-companion.sh"),
1566 "{cmd}"
1567 );
1568 let installer_pos = cmd.find("install-companion.sh").unwrap();
1569 let src_pos = cmd.find("companions/mnw-cli").unwrap();
1570 let dst_pos = cmd.find("/opt/mnw-cli/mnw-cli").unwrap();
1571 let svc_pos = cmd.find("mnw-cli.service").unwrap();
1572 assert!(
1573 installer_pos < src_pos && src_pos < dst_pos && dst_pos < svc_pos,
1574 "arg order: {cmd}"
1575 );
1576 }
1577
1578 #[test]
1579 fn install_companion_cmd_quotes_metachars() {
1580 // A path with a space/quote must be shell-safe (defense in depth even
1581 // though these come from operator config).
1582 let cmd = install_companion_cmd("/a b/src", "/dst'x", "u.service");
1583 let out = std::process::Command::new("sh")
1584 .arg("-c")
1585 .arg(format!(
1586 "set -- {}; echo \"$#\"",
1587 cmd.strip_prefix("sudo ").unwrap()
1588 ))
1589 .output()
1590 .unwrap();
1591 // installer + 3 args = 4 positional words after quoting.
1592 assert_eq!(
1593 String::from_utf8_lossy(&out.stdout).trim(),
1594 "4",
1595 "quoting split wrong: {cmd}"
1596 );
1597 }
1598
1599 #[tokio::test]
1600 async fn config_check_script_propagates_binary_failure() {
1601 // A required var missing (the binary exits non-zero) must fail the check.
1602 let env = tempfile::NamedTempFile::new().unwrap();
1603 std::fs::write(env.path(), "FOO=bar\n").unwrap();
1604 let script = config_check_script(&env.path().to_string_lossy(), "false");
1605 let out = run_script(&script).await;
1606 assert!(
1607 !out.status.success(),
1608 "a non-zero MNW_CHECK_CONFIG exit must fail the check"
1609 );
1610 }
1611
1612 #[tokio::test]
1613 async fn deploy_node_denied_when_executor_lacks_deploy_grant() {
1614 // Defense in depth: an executor without the deploy grant refuses the
1615 // step before any filesystem / ssh action.
1616 let tmp = tempfile::tempdir().unwrap();
1617 let release_root = tmp.path().to_path_buf();
1618 let staged = release_root.join("releases").join("0.0.1");
1619 tokio::fs::create_dir_all(&staged).await.unwrap();
1620
1621 let node = crate::topology::Node {
1622 platform: None,
1623 name: "local-dev".into(),
1624 ssh_target: "local".into(),
1625 release_root: release_root.to_string_lossy().into_owned(),
1626 service_name: "makenotwork.service".into(),
1627 health_url: None,
1628 config_check_env_file: None,
1629 actuate: vec!["restart".into()], // no deploy
1630 observe: vec![],
1631 companions: Vec::new(),
1632 };
1633 let executor = LocalExec::new(CapabilitySet::from_tokens(["restart"], Vec::<&str>::new()));
1634 let err = deploy_node(
1635 &executor,
1636 Placement::check(&node, &staged, None).unwrap(),
1637 "0.0.1",
1638 "server",
1639 )
1640 .await
1641 .unwrap_err();
1642 assert!(
1643 format!("{err:#}").contains("capability denied"),
1644 "expected capability denial"
1645 );
1646 }
1647
1648 // ---- FakeExec: the deploy_remote choreography without a real host ----
1649 //
1650 // deploy_node's local fast-path is covered above with a real LocalExec, but
1651 // the remote path (rsync + arch guard + config-drift + swap + companions +
1652 // gc) short-circuits on `ssh_target != "local"` and so never ran under test
1653 // without a reachable node. FakeExec records every executor call in order
1654 // and can be told to fail one shell step (matched by substring) or the rsync
1655 // push, so the ordering and the fail-closed-before-swap contract are
1656 // assertable in-process.
1657
1658 struct FakeExec {
1659 caps: CapabilitySet,
1660 calls: Arc<StdMutex<Vec<String>>>,
1661 /// The first `run_streaming` whose script contains this substring exits
1662 /// non-zero (a failed shell step), e.g. the arch guard.
1663 fail_run_matching: Option<String>,
1664 /// `push_dir` (the rsync) returns an error.
1665 fail_push_dir: bool,
1666 }
1667
1668 impl FakeExec {
1669 fn new() -> Self {
1670 Self {
1671 caps: CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1672 calls: Arc::new(StdMutex::new(Vec::new())),
1673 fail_run_matching: None,
1674 fail_push_dir: false,
1675 }
1676 }
1677 fn log(&self) -> Vec<String> {
1678 self.calls.lock().unwrap().clone()
1679 }
1680 }
1681
1682 #[async_trait]
1683 impl Executor for FakeExec {
1684 async fn run_streaming(&self, step: &Step, _sink: &mut dyn LogSink) -> Result<RunOutput> {
1685 // Every deploy step is a `Step::shell`, so the script is argv's tail.
1686 let script = step.argv.last().cloned().unwrap_or_default();
1687 self.calls.lock().unwrap().push(format!("run:{script}"));
1688 let fail = self
1689 .fail_run_matching
1690 .as_deref()
1691 .is_some_and(|m| script.contains(m));
1692 Ok(RunOutput {
1693 status: std::process::ExitStatus::from_raw(if fail { 1 << 8 } else { 0 }),
1694 stdout: Vec::new(),
1695 stderr: if fail {
1696 b"fake step failure".to_vec()
1697 } else {
1698 Vec::new()
1699 },
1700 })
1701 }
1702 async fn pull_file(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1703 self.calls.lock().unwrap().push("pull_file".into());
1704 Ok(())
1705 }
1706 async fn pull_dir(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1707 self.calls.lock().unwrap().push("pull_dir".into());
1708 Ok(())
1709 }
1710 async fn pull_glob(&self, _glob: &str, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1711 self.calls.lock().unwrap().push("pull_glob".into());
1712 Ok(())
1713 }
1714 async fn push_dir(&self, _local: &Path, remote: &Path, _opts: &SyncOpts) -> Result<()> {
1715 self.calls
1716 .lock()
1717 .unwrap()
1718 .push(format!("push_dir:{}", remote.display()));
1719 if self.fail_push_dir {
1720 anyhow::bail!("fake rsync failure");
1721 }
1722 Ok(())
1723 }
1724 fn capabilities(&self) -> &CapabilitySet {
1725 &self.caps
1726 }
1727 }
1728
1729 fn remote_node(config_check: bool, companions: Vec<NodeCompanion>) -> Node {
1730 Node {
1731 platform: None,
1732 name: "web-a".into(),
1733 ssh_target: "deploy@web-a".into(),
1734 release_root: "/opt/mnw".into(),
1735 service_name: "makenotwork.service".into(),
1736 health_url: None,
1737 config_check_env_file: config_check.then(|| "/etc/mnw/node.env".to_string()),
1738 actuate: crate::topology::default_actuate(),
1739 observe: crate::topology::default_observe(),
1740 companions,
1741 }
1742 }
1743
1744 fn companion() -> NodeCompanion {
1745 NodeCompanion {
1746 name: "mnw-cli".into(),
1747 install_path: "/opt/mnw-cli/mnw-cli".into(),
1748 service_name: "mnw-cli.service".into(),
1749 }
1750 }
1751
1752 /// Index of the first recorded call whose text contains `needle` (panics if
1753 /// absent — the assertion message names what was missing).
1754 fn pos(log: &[String], needle: &str) -> usize {
1755 log.iter()
1756 .position(|c| c.contains(needle))
1757 .unwrap_or_else(|| panic!("no call matched {needle:?} in {log:#?}"))
1758 }
1759
1760 #[tokio::test]
1761 async fn deploy_remote_runs_the_full_choreography_in_order() {
1762 // A node opted into the config-drift check and carrying one companion:
1763 // mkdir -> rsync -> arch guard -> config check -> swap+restart ->
1764 // companion install -> gc, in that order.
1765 let tmp = tempfile::tempdir().unwrap();
1766 let staged = tmp.path().join("releases").join("0.9.0");
1767 tokio::fs::create_dir_all(&staged).await.unwrap();
1768
1769 let node = remote_node(true, vec![companion()]);
1770 let exec = FakeExec::new();
1771 let out = deploy_node(
1772 &exec,
1773 Placement::check(&node, &staged, None).unwrap(),
1774 "0.9.0",
1775 "makenotwork",
1776 )
1777 .await
1778 .expect("deploy_remote should succeed against the fake");
1779 assert_eq!(out, PathBuf::from("/opt/mnw/releases/0.9.0"));
1780
1781 let log = exec.log();
1782 let mkdir = pos(&log, "mkdir -p");
1783 let rsync = pos(&log, "push_dir:/opt/mnw/releases/0.9.0");
1784 let arch = pos(&log, "e_machine");
1785 let cfg = pos(&log, "MNW_CHECK_CONFIG=1");
1786 let swap = pos(&log, "reload-or-restart");
1787 let comp = pos(&log, "install-companion.sh");
1788 let gc = pos(&log, "ls -1t");
1789 assert!(
1790 mkdir < rsync && rsync < arch && arch < cfg && cfg < swap && swap < comp && comp < gc,
1791 "deploy steps out of order: {log:#?}"
1792 );
1793 }
1794
1795 #[tokio::test]
1796 async fn deploy_remote_aborts_before_swap_when_rsync_fails() {
1797 // The rsync failing must fail the deploy BEFORE the symlink swap — the
1798 // "current symlink left intact" contract. Assert the swap never ran.
1799 let tmp = tempfile::tempdir().unwrap();
1800 let staged = tmp.path().join("releases").join("0.9.0");
1801 tokio::fs::create_dir_all(&staged).await.unwrap();
1802
1803 let node = remote_node(false, Vec::new());
1804 let mut exec = FakeExec::new();
1805 exec.fail_push_dir = true;
1806 let err = deploy_node(
1807 &exec,
1808 Placement::check(&node, &staged, None).unwrap(),
1809 "0.9.0",
1810 "makenotwork",
1811 )
1812 .await
1813 .expect_err("rsync failure must fail the deploy");
1814 assert!(
1815 format!("{err:#}").contains("rsync"),
1816 "error should attribute the rsync: {err:#}"
1817 );
1818 let log = exec.log();
1819 assert!(
1820 !log.iter().any(|c| c.contains("reload-or-restart")),
1821 "swap must not run after a failed rsync: {log:#?}"
1822 );
1823 }
1824
1825 #[tokio::test]
1826 async fn deploy_remote_aborts_before_swap_when_arch_guard_fails() {
1827 // A wrong-arch binary must fail closed before the swap. The fake fails
1828 // the arch-guard shell step; the swap must not follow.
1829 let tmp = tempfile::tempdir().unwrap();
1830 let staged = tmp.path().join("releases").join("0.9.0");
1831 tokio::fs::create_dir_all(&staged).await.unwrap();
1832
1833 let node = remote_node(false, Vec::new());
1834 let mut exec = FakeExec::new();
1835 exec.fail_run_matching = Some("e_machine".into());
1836 let err = deploy_node(
1837 &exec,
1838 Placement::check(&node, &staged, None).unwrap(),
1839 "0.9.0",
1840 "makenotwork",
1841 )
1842 .await
1843 .expect_err("arch mismatch must fail the deploy");
1844 assert!(
1845 format!("{err:#}").contains("architecture"),
1846 "error should mention the arch check: {err:#}"
1847 );
1848 let log = exec.log();
1849 assert!(
1850 !log.iter().any(|c| c.contains("reload-or-restart")),
1851 "swap must not run after a failed arch guard: {log:#?}"
1852 );
1853 }
1854
1855 #[tokio::test]
1856 async fn deploy_remote_skips_config_check_when_node_opts_out() {
1857 // No config_check_env_file => the pre-swap config check is skipped, but
1858 // the rest of the choreography (including the swap) still runs.
1859 let tmp = tempfile::tempdir().unwrap();
1860 let staged = tmp.path().join("releases").join("0.9.0");
1861 tokio::fs::create_dir_all(&staged).await.unwrap();
1862
1863 let node = remote_node(false, Vec::new());
1864 let exec = FakeExec::new();
1865 deploy_node(
1866 &exec,
1867 Placement::check(&node, &staged, None).unwrap(),
1868 "0.9.0",
1869 "makenotwork",
1870 )
1871 .await
1872 .unwrap();
1873 let log = exec.log();
1874 assert!(
1875 !log.iter().any(|c| c.contains("MNW_CHECK_CONFIG=1")),
1876 "config check must be skipped when the node opts out: {log:#?}"
1877 );
1878 assert!(
1879 log.iter().any(|c| c.contains("reload-or-restart")),
1880 "the swap must still run: {log:#?}"
1881 );
1882 }
1883
1884 #[tokio::test]
1885 async fn deploy_remote_installs_companion_after_the_swap() {
1886 // Companions are After= the server: their install must land after the
1887 // symlink swap + service restart, never before.
1888 let tmp = tempfile::tempdir().unwrap();
1889 let staged = tmp.path().join("releases").join("0.9.0");
1890 tokio::fs::create_dir_all(&staged).await.unwrap();
1891
1892 let node = remote_node(false, vec![companion()]);
1893 let exec = FakeExec::new();
1894 deploy_node(
1895 &exec,
1896 Placement::check(&node, &staged, None).unwrap(),
1897 "0.9.0",
1898 "makenotwork",
1899 )
1900 .await
1901 .unwrap();
1902 let log = exec.log();
1903 assert!(
1904 pos(&log, "reload-or-restart") < pos(&log, "install-companion.sh"),
1905 "companion install must follow the swap: {log:#?}"
1906 );
1907 }
1908 }
1909