Skip to main content

max / makenotwork

57.5 KB · 1458 lines History Blame Raw
1 //! Atomic symlink-swap deploys.
2 //!
3 //! Layout on every target (local host, A nodes, B nodes, ...). Release dirs are
4 //! named for their content digest, not the version (wiki note
5 //! `release-artifact-identity`), so a rebuild can never overwrite an earlier
6 //! build's dir in place:
7 //!
8 //! <release_root>/
9 //! releases/
10 //! a1b2c3d4e5f60718/ <- <digest16>
11 //! <bin_name>
12 //! MANIFEST <- <sha256> <relpath> per file, node-verified
13 //! f0e1d2c3b4a59687/
14 //! <bin_name>
15 //! current -> releases/f0e1d2c3b4a59687
16 //!
17 //! `ln -sfn` swaps the symlink. systemd units point at
18 //! `<release_root>/current/<bin_name>` so reload-or-restart picks up the new
19 //! binary without ever pointing at a missing path.
20 //!
21 //! The host-side transport — `ssh` for shell steps, `rsync` for the release
22 //! dir — comes from the shared [`ops_exec::Executor`] (a `LocalExec` for
23 //! `ssh_target = "local"`, an `SshExec` otherwise), built once per node in
24 //! [`crate::state`]. This module owns the *deploy choreography* (mkdir, push,
25 //! atomic swap, restart, gc); the transport is the crate's. SSH push behavior
26 //! is identical to the pre-extraction code — this is a transport extraction,
27 //! not a model change.
28
29 use crate::topology::Node;
30 use anyhow::{Context, Result};
31 use async_trait::async_trait;
32 use ops_exec::{Action, Executor, LogSink, RunOutput, Step, SyncOpts, sh_quote};
33 use std::path::{Path, PathBuf};
34 use tokio::process::Command;
35
36 /// Keep this many release dirs per node; older ones get gc'd after a
37 /// successful deploy. Fixed for now; promote to config if the constant ever
38 /// needs to vary by tier.
39 const RELEASES_TO_KEEP: usize = 5;
40
41 /// A sink that drops streamed bytes. Deploy steps don't have a live-log handle
42 /// (gates do), so output is discarded as it streams; [`RunOutput`] still
43 /// captures the full stdout/stderr for error reporting, preserving the
44 /// pre-extraction behavior of surfacing `stderr` in failure messages.
45 struct DiscardSink;
46
47 #[async_trait]
48 impl LogSink for DiscardSink {
49 async fn write_chunk(&mut self, _bytes: &[u8]) {}
50 }
51
52 /// Run a shell step through `executor`, treating a non-zero exit as an error
53 /// whose message carries the captured stderr — exactly as the old bespoke
54 /// `ssh()` helper did (`ssh <target> failed: <stderr>`).
55 async fn run_checked(executor: &dyn Executor, script: &str, what: &str) -> Result<RunOutput> {
56 let step = Step::shell(Action::Deploy, script);
57 let mut sink = DiscardSink;
58 let out = executor
59 .run_streaming(&step, &mut sink)
60 .await
61 .with_context(|| format!("{what}: spawning command"))?;
62 anyhow::ensure!(
63 out.status.success(),
64 "{what} failed (exit {}): {}",
65 out.status
66 .code()
67 .map_or_else(|| "signal".into(), |c| c.to_string()),
68 String::from_utf8_lossy(&out.stderr),
69 );
70 Ok(out)
71 }
72
73 /// Stage built binaries into `staging/<build_id>/` on the Sando host — a
74 /// private, mutable scratch dir that is not yet a release (no symlink, no gc).
75 /// The caller adds `release_contents` + companions, hashes the result, writes
76 /// the `MANIFEST`, then publishes it content-addressed via
77 /// [`finalize_local_release`]. Splitting staging from publish is what lets the
78 /// bundle be hashed before it is named (wiki [[release-artifact-identity]]).
79 ///
80 /// A stale `staging/<build_id>` from a killed prior run at the same id is
81 /// removed first, so a retry stages clean.
82 pub async fn stage_local_bundle(
83 release_root: &Path,
84 build_id: i64,
85 binaries: &[PathBuf],
86 ) -> Result<PathBuf> {
87 let staging = release_root.join("staging").join(build_id.to_string());
88 if tokio::fs::try_exists(&staging).await.unwrap_or(false) {
89 tokio::fs::remove_dir_all(&staging)
90 .await
91 .with_context(|| format!("clearing stale staging dir {}", staging.display()))?;
92 }
93 tokio::fs::create_dir_all(&staging).await?;
94 for binary in binaries {
95 let name = binary.file_name().context("binary path has no file name")?;
96 let dest = staging.join(name);
97 tokio::fs::copy(binary, &dest)
98 .await
99 .with_context(|| format!("copy {} -> {}", binary.display(), dest.display()))?;
100 }
101 Ok(staging)
102 }
103
104 /// Publish a fully-staged bundle content-addressed: rename
105 /// `staging/<build_id>` to `releases/<digest16>` (atomic same-filesystem
106 /// rename), flip `current` to it, gc old releases. Returns the released dir.
107 ///
108 /// The rename is the load-bearing step: **a directory whose name derives from
109 /// its contents cannot be rewritten, because rewriting it changes its name.**
110 /// The overwrite class that let a dev rebuild inherit an earlier build's gate
111 /// rows and burn-in clock stops being something to guard against and becomes
112 /// something that cannot be expressed. If a release with this digest already
113 /// exists (identical bytes rebuilt), the staging copy is redundant and dropped.
114 pub async fn finalize_local_release(
115 release_root: &Path,
116 staging: &Path,
117 digest16: &str,
118 ) -> Result<PathBuf> {
119 let releases = release_root.join("releases");
120 tokio::fs::create_dir_all(&releases).await?;
121 let released = releases.join(digest16);
122
123 if tokio::fs::try_exists(&released).await.unwrap_or(false) {
124 // Same digest already published — reuse it, discard the redundant stage.
125 tokio::fs::remove_dir_all(staging).await.ok();
126 } else {
127 tokio::fs::rename(staging, &released)
128 .await
129 .with_context(|| format!("publish {} -> {}", staging.display(), released.display()))?;
130 }
131
132 let current = release_root.join("current");
133 let target = format!("releases/{digest16}");
134 let out = Command::new("ln")
135 .args(["-sfn", &target])
136 .arg(&current)
137 .output()
138 .await?;
139 anyhow::ensure!(
140 out.status.success(),
141 "symlink swap failed: {}",
142 String::from_utf8_lossy(&out.stderr),
143 );
144
145 if let Err(e) = gc_local_releases(release_root).await {
146 tracing::warn!(error = %e, "local release GC failed (non-fatal)");
147 }
148 Ok(released)
149 }
150
151 /// Deploy `staged_release_dir` (a directory built on the Sando host by
152 /// `deploy_local`) to `node` using `executor` (its transport from the topology
153 /// executor map). For `ssh_target=local`, this is just a symlink swap; for
154 /// remote nodes, we rsync the whole dir over the executor.
155 ///
156 /// `primary_bin` is only used for logging — every file present in the staged
157 /// dir gets shipped.
158 pub async fn deploy_node(
159 executor: &dyn Executor,
160 node: &Node,
161 version: &str,
162 staged_release_dir: &Path,
163 primary_bin: &str,
164 ) -> Result<PathBuf> {
165 // The release dir is named for its content digest (`releases/<digest16>`),
166 // not the version. The node mirrors that name so host and node agree on the
167 // artifact's identity; the version is only a log label here. Legacy staged
168 // dirs (pre-identity, still `releases/<version>`) work unchanged — the name
169 // is whatever the host staged under.
170 let release_id = staged_release_dir
171 .file_name()
172 .and_then(|n| n.to_str())
173 .with_context(|| {
174 format!(
175 "staged release dir {} has no usable name",
176 staged_release_dir.display()
177 )
178 })?;
179 if node.ssh_target == "local" || node.ssh_target.is_empty() {
180 // Local deploy already happened when we staged on the Sando host.
181 // Just re-point `current` at the staged dir.
182 return reset_local_current(executor, Path::new(&node.release_root), release_id).await;
183 }
184 deploy_remote(
185 executor,
186 node,
187 version,
188 release_id,
189 staged_release_dir,
190 primary_bin,
191 )
192 .await
193 }
194
195 async fn reset_local_current(
196 executor: &dyn Executor,
197 release_root: &Path,
198 release_id: &str,
199 ) -> Result<PathBuf> {
200 let current = release_root.join("current");
201 let target = format!("releases/{release_id}");
202 run_checked(
203 executor,
204 &format!(
205 "ln -sfn {} {}",
206 sh_quote(&target),
207 sh_quote(&current.to_string_lossy())
208 ),
209 "local symlink swap",
210 )
211 .await?;
212 Ok(release_root.join("releases").join(release_id))
213 }
214
215 async fn deploy_remote(
216 executor: &dyn Executor,
217 node: &Node,
218 version: &str,
219 release_id: &str,
220 staged_release_dir: &Path,
221 primary_bin: &str,
222 ) -> Result<PathBuf> {
223 let release_root = &node.release_root;
224 let service = &node.service_name;
225 let release_dir = format!("{release_root}/releases/{release_id}");
226
227 tracing::info!(node = %node.name, version, release_id, "deploy: mkdir release dir");
228 run_checked(
229 executor,
230 &format!("set -e; mkdir -p {q}", q = sh_quote(&release_dir)),
231 "creating remote release dir",
232 )
233 .await?;
234
235 tracing::info!(node = %node.name, version, primary = %primary_bin, "deploy: rsync release dir");
236 // Rsync the whole staged dir (binaries + every release_contents entry).
237 // `SyncOpts::release_mirror()` is the exact pre-extraction rsync flag set:
238 // -az --partial --delete --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r,F+X.
239 // --delete: removed assets across versions don't accumulate on the
240 // target; the bundle stays self-contained per version.
241 // --chmod: F+X preserves the execute bit per-file (binaries land 0755,
242 // data files 0644) instead of a blanket 0755.
243 executor
244 .push_dir(
245 staged_release_dir,
246 Path::new(&release_dir),
247 &SyncOpts::release_mirror(),
248 )
249 .await
250 .context("rsync failed (current symlink left intact)")?;
251
252 // Verify the bundle on the node against its own MANIFEST before the swap
253 // (invariant 3, wiki [[release-artifact-identity]]). The MANIFEST shipped in
254 // the bundle is exactly `sha256sum` check format (`<hash> <relpath>`), so
255 // this re-hashes every file on the node and names any that drifted in
256 // transit — the hash is load-bearing, not merely recorded. Bundles staged
257 // by a pre-identity build carry no MANIFEST; those skip verification (logged)
258 // rather than fail, so a mid-migration deploy of a legacy artifact still
259 // ships. A mismatch fails the promote with the running service intact.
260 run_checked(
261 executor,
262 &manifest_verify_script(&release_dir),
263 "verifying bundle digest on node",
264 )
265 .await
266 .context("node-side bundle verification failed (current symlink left intact)")?;
267
268 // Fail closed on a wrong-architecture binary before the symlink swap. The
269 // "never cross-compile" rule is enforced at build time (build_host check),
270 // but nothing verified the artifact's arch matched the *target* node — so
271 // adding an aarch64 node to a tier built on x86_64 would silently symlink an
272 // unrunnable binary live. Compare the deployed binary's ELF e_machine to the
273 // node's `uname -m`; unknown arches log and proceed (can't verify != known-bad).
274 let deployed_bin = format!("{release_dir}/{primary_bin}");
275 run_checked(
276 executor,
277 &arch_guard_script(&deployed_bin),
278 "verifying binary arch matches node",
279 )
280 .await
281 .context(
282 "deployed binary architecture does not match the target node (current symlink left intact)",
283 )?;
284
285 // Config-drift guard (opt-in per node). Runs the freshly-rsynced binary in
286 // config-only mode with the node's env sourced, BEFORE the swap, so a
287 // required var missing on this node fails here — service still intact —
288 // rather than after the restart, which would crash-loop it (how testnot
289 // went down on a missing CDN_BASE_URL). Skipped unless the node sets
290 // `config_check_env_file`.
291 if let Some(env_file) = node.config_check_env_file.as_deref() {
292 tracing::info!(node = %node.name, version, "deploy: pre-swap config check");
293 check_target_config(executor, &deployed_bin, env_file)
294 .await
295 .context("pre-swap config check failed (current symlink left intact)")?;
296 }
297
298 tracing::info!(node = %node.name, version, "deploy: symlink swap + service reload");
299 let restart_cmd = format!(
300 "sudo /bin/systemctl reload-or-restart {}",
301 sh_quote(service)
302 );
303 let swap_and_restart = swap_and_restart_script(release_root, release_id, &restart_cmd);
304 run_checked(
305 executor,
306 &swap_and_restart,
307 "symlink swap + systemctl reload-or-restart",
308 )
309 .await?;
310
311 // Companion services (opt-in per node): install each from the just-rsynced
312 // bundle and restart its unit via the node-side wrapper, AFTER the server is
313 // up (mnw-cli is `After=makenotwork.service`). They shipped from the SAME sha
314 // in this SAME bundle — the lockstep guarantee. A failure here fails the
315 // promote: a companion is part of the deploy, not a best-effort side effect.
316 for c in &node.companions {
317 let src = format!(
318 "{release_root}/releases/{release_id}/companions/{name}",
319 name = c.name,
320 );
321 tracing::info!(node = %node.name, companion = %c.name, "deploy: install companion + restart");
322 let cmd = install_companion_cmd(&src, &c.install_path, &c.service_name);
323 run_checked(executor, &cmd, "install companion + restart")
324 .await
325 .with_context(|| {
326 format!(
327 "companion {} deploy failed (server already swapped)",
328 c.name
329 )
330 })?;
331 }
332
333 if let Err(e) = gc_remote_releases(executor, release_root).await {
334 tracing::warn!(error = %e, "remote release GC failed (non-fatal)");
335 }
336
337 Ok(PathBuf::from(release_root)
338 .join("releases")
339 .join(release_id))
340 }
341
342 /// Absolute path of the node-side companion installer (shipped once per node;
343 /// granted to the deploy user by a single scoped sudoers line). It installs the
344 /// staged binary to its `ExecStart` path and restarts the unit — keeping the
345 /// sudo grant to one script rather than a broad `install`/`systemctl` grant.
346 const COMPANION_INSTALLER: &str = "/usr/local/lib/mnw/install-companion.sh";
347
348 /// Command run on the node to install a staged companion binary and restart its
349 /// unit, via the wrapper. Pure builder so it can be unit-tested; all three args
350 /// are shell-quoted (paths/unit names, operator config — but quoted regardless).
351 fn install_companion_cmd(src: &str, install_path: &str, service: &str) -> String {
352 format!(
353 "sudo {installer} {src} {dst} {svc}",
354 installer = sh_quote(COMPANION_INSTALLER),
355 src = sh_quote(src),
356 dst = sh_quote(install_path),
357 svc = sh_quote(service),
358 )
359 }
360
361 /// Pre-swap config-drift check: load the node's env file the way systemd loads
362 /// it, then run the freshly-deployed binary in `MNW_CHECK_CONFIG=1` mode (loads
363 /// config, exits 0/1, no DB/migrations/bind). A non-zero exit — a required var
364 /// missing — is surfaced by `run_checked` as an error, failing the promote
365 /// before the swap.
366 ///
367 /// Bounded by a timeout as a backstop: a binary predating `MNW_CHECK_CONFIG`
368 /// would ignore the var and try to start normally, which must not hang the
369 /// deploy. A timeout is reported as a failure (fail closed) — the operator only
370 /// opts a node in once a check-capable version is deployed, so a timeout means
371 /// something is wrong, not a routine older binary.
372 async fn check_target_config(
373 executor: &dyn Executor,
374 deployed_bin: &str,
375 env_file: &str,
376 ) -> Result<()> {
377 let script = config_check_script(env_file, deployed_bin);
378 let fut = run_checked(executor, &script, "pre-swap config check");
379 match tokio::time::timeout(std::time::Duration::from_secs(20), fut).await {
380 Ok(result) => result.map(|_| ()),
381 Err(_) => anyhow::bail!(
382 "pre-swap config check timed out after 20s — the binary may predate \
383 MNW_CHECK_CONFIG or the check hung; refusing to swap"
384 ),
385 }
386 }
387
388 /// Shell that loads `env_file` with systemd `EnvironmentFile=` semantics, then
389 /// runs `bin` under `MNW_CHECK_CONFIG=1`.
390 ///
391 /// Load the file line by line and `export` each `KEY=VALUE` verbatim rather than
392 /// `. env_file`. Dot-sourcing runs the file as a script, so any shell
393 /// metacharacter in a value (`$`, backticks, `;`, `&`, a glob, whitespace) is
394 /// expanded or word-split — a DB URL carrying a password silently dropped
395 /// `DATABASE_URL` to empty on our nodes, which would fail the check (and thus
396 /// every deploy) even though systemd starts the service fine. `export "$line"`
397 /// assigns the already-expanded word literally, matching systemd's "no variable
398 /// expansion" rule. Comments and blank lines are skipped; the `|| [ -n "$line" ]`
399 /// guard processes a final line with no trailing newline. (Quoted values —
400 /// `KEY="v"` — aren't unquoted here the way systemd would, but our env files use
401 /// bare `KEY=VALUE`, and a stray quote can only make the check stricter, never
402 /// wave a bad config through.)
403 fn config_check_script(env_file: &str, bin: &str) -> String {
404 format!(
405 "set -eu\n\
406 while IFS= read -r __sando_l || [ -n \"$__sando_l\" ]; do\n\
407 \tcase \"$__sando_l\" in ''|'#'*) continue ;; esac\n\
408 \texport \"$__sando_l\"\n\
409 done < {env}\n\
410 MNW_CHECK_CONFIG=1 {bin}\n",
411 env = sh_quote(env_file),
412 bin = sh_quote(bin),
413 )
414 }
415
416 /// Build the swap-and-restart shell script for a remote node.
417 ///
418 /// The symlink swap is atomic via `mv -T` of a freshly-created symlink over the
419 /// old one (the rename(2) is the atomic step; `ln -sfn` alone does
420 /// unlink+symlink, which has a window). The load-bearing part: if `restart_cmd`
421 /// fails *after* the flip, `current` is rolled back to its prior target before
422 /// the script exits non-zero. Otherwise a failed restart would leave `current`
423 /// pointing at the new, un-activated release while the service still runs the
424 /// old one — and a later reboot/cron restart would then silently bring up the
425 /// release the deploy reported as failed. Best-effort re-restart of the prior
426 /// version keeps the running service consistent with the restored symlink.
427 ///
428 /// `restart_cmd` is injected (rather than hardcoded) so tests can drive the
429 /// failure and success paths with a `false`/`true` stand-in.
430 fn swap_and_restart_script(release_root: &str, release_id: &str, restart_cmd: &str) -> String {
431 format!(
432 "set -e\n\
433 cd {root}\n\
434 prev=$(readlink current 2>/dev/null || true)\n\
435 ln -sfn releases/{rel} current.new\n\
436 mv -Tf current.new current\n\
437 if ! {restart}; then\n\
438 if [ -n \"$prev\" ]; then\n\
439 ln -sfn \"$prev\" current.rollback\n\
440 mv -Tf current.rollback current\n\
441 {restart} || true\n\
442 fi\n\
443 echo \"deploy: restart failed; rolled symlink back to ${{prev:-<none>}}\" >&2\n\
444 exit 1\n\
445 fi\n",
446 root = sh_quote(release_root),
447 rel = sh_quote(release_id),
448 restart = restart_cmd,
449 )
450 }
451
452 /// Shell that re-hashes the rsynced bundle on the node against its shipped
453 /// `MANIFEST` and aborts (exit 1) if any file drifted (invariant 3, wiki note
454 /// `release-artifact-identity`). The `MANIFEST` is `sha256sum` check format,
455 /// so `sha256sum -c` verifies every listed file with node-native tooling and
456 /// names the one that failed. `--strict` fails on a malformed manifest line;
457 /// `--quiet` drops the per-file OK spam and keeps only failures.
458 ///
459 /// A bundle staged by a pre-identity build carries no `MANIFEST`; that is not an
460 /// error — it logs a skip and exits 0, so a mid-migration deploy of a legacy
461 /// artifact still ships. Once every tier has cycled once, every bundle has one.
462 fn manifest_verify_script(release_dir: &str) -> String {
463 format!(
464 "set -e\n\
465 cd {dir}\n\
466 if [ ! -f MANIFEST ]; then\n\
467 echo \"deploy: no MANIFEST in bundle; skipping digest verification (legacy artifact)\" >&2\n\
468 exit 0\n\
469 fi\n\
470 sha256sum --quiet --strict -c MANIFEST\n",
471 dir = sh_quote(release_dir),
472 )
473 }
474
475 /// Shell that aborts (exit 1) if `bin`'s ELF architecture doesn't match the
476 /// node it's running on. Reads the ELF `e_machine` field (2 bytes LE at offset
477 /// 18) and compares it to the value implied by `uname -m`. An arch we don't have
478 /// a mapping for logs and proceeds — the guard exists to catch the concrete
479 /// x86_64-vs-aarch64 confusion, not to gate genuinely-new targets.
480 fn arch_guard_script(bin: &str) -> String {
481 format!(
482 "set -e\n\
483 bin={bin}\n\
484 arch=$(uname -m)\n\
485 machine=$(od -An -tx1 -j18 -N2 \"$bin\" 2>/dev/null | tr -d ' \\n')\n\
486 case \"$arch\" in\n\
487 x86_64|amd64) want=3e00 ;;\n\
488 aarch64|arm64) want=b700 ;;\n\
489 *) echo \"deploy: arch check skipped (unmapped node arch $arch)\" >&2; want= ;;\n\
490 esac\n\
491 if [ -n \"$want\" ] && [ \"$machine\" != \"$want\" ]; then\n\
492 echo \"deploy: arch mismatch — node $arch expects e_machine $want but binary has ${{machine:-<unreadable>}}\" >&2\n\
493 exit 1\n\
494 fi\n",
495 bin = sh_quote(bin),
496 )
497 }
498
499 async fn gc_local_releases(release_root: &Path) -> Result<()> {
500 let releases = release_root.join("releases");
501 if !releases.exists() {
502 return Ok(());
503 }
504 let mut entries = Vec::new();
505 let mut rd = tokio::fs::read_dir(&releases).await?;
506 while let Some(entry) = rd.next_entry().await? {
507 if !entry.file_type().await?.is_dir() {
508 continue;
509 }
510 let meta = entry.metadata().await?;
511 entries.push((entry.path(), meta.modified()?));
512 }
513 entries.sort_by_key(|e| std::cmp::Reverse(e.1));
514 for (path, _) in entries.into_iter().skip(RELEASES_TO_KEEP) {
515 if let Err(e) = tokio::fs::remove_dir_all(&path).await {
516 tracing::warn!(path = %path.display(), error = %e, "gc: rm failed");
517 } else {
518 tracing::debug!(path = %path.display(), "gc: removed old release");
519 }
520 }
521 Ok(())
522 }
523
524 async fn gc_remote_releases(executor: &dyn Executor, release_root: &str) -> Result<()> {
525 // `ls -t` orders by mtime desc. Skip the first N, rm the rest. `xargs -r`
526 // is a no-op when stdin is empty (avoids `rm` complaining).
527 let script = format!(
528 "set -e; cd {root}/releases 2>/dev/null || exit 0; \
529 ls -1t | tail -n +{keep_plus_one} | xargs -r -I{{}} rm -rf -- {{}}",
530 root = sh_quote(release_root),
531 keep_plus_one = RELEASES_TO_KEEP + 1,
532 );
533 run_checked(executor, &script, "remote release gc")
534 .await
535 .map(|_| ())
536 }
537
538 #[cfg(test)]
539 mod tests {
540 use super::*;
541 use crate::topology::NodeCompanion;
542 use ops_exec::{CapabilitySet, LocalExec, SshExec};
543 use std::os::unix::process::ExitStatusExt;
544 use std::sync::{Arc, Mutex as StdMutex};
545 use std::time::SystemTime;
546
547 /// A LocalExec granted the default node capabilities (deploy + restart).
548 fn local_executor() -> LocalExec {
549 LocalExec::new(CapabilitySet::from_tokens(
550 ["deploy", "restart"],
551 ["health"],
552 ))
553 }
554
555 #[tokio::test]
556 async fn deploy_local_copies_multiple_binaries_and_swaps_symlink() {
557 let tmp = tempfile::tempdir().unwrap();
558 let root = tmp.path();
559
560 let src_dir = root.join("src");
561 tokio::fs::create_dir_all(&src_dir).await.unwrap();
562 let primary = src_dir.join("makenotwork");
563 let admin = src_dir.join("mnw-admin");
564 tokio::fs::write(&primary, b"PRIMARY").await.unwrap();
565 tokio::fs::write(&admin, b"ADMIN").await.unwrap();
566
567 let release_root = root.join("releases-root");
568 tokio::fs::create_dir_all(&release_root).await.unwrap();
569
570 // Stage into staging/<build_id> (no publish yet).
571 let staging = stage_local_bundle(&release_root, 42, &[primary.clone(), admin.clone()])
572 .await
573 .expect("stage_local_bundle should succeed");
574 assert_eq!(staging, release_root.join("staging").join("42"));
575 assert!(
576 !release_root.join("current").exists(),
577 "staging must not publish or flip current"
578 );
579
580 // Publish content-addressed at releases/<digest16>.
581 let released = finalize_local_release(&release_root, &staging, "deadbeefcafe0000")
582 .await
583 .expect("finalize_local_release should succeed");
584 assert_eq!(
585 released,
586 release_root.join("releases").join("deadbeefcafe0000")
587 );
588 assert!(
589 !staging.exists(),
590 "staging dir is consumed by the publish rename"
591 );
592 assert_eq!(
593 tokio::fs::read(released.join("makenotwork")).await.unwrap(),
594 b"PRIMARY"
595 );
596 assert_eq!(
597 tokio::fs::read(released.join("mnw-admin")).await.unwrap(),
598 b"ADMIN"
599 );
600
601 let current = release_root.join("current");
602 let target = tokio::fs::read_link(&current).await.unwrap();
603 assert_eq!(target.to_string_lossy(), "releases/deadbeefcafe0000");
604 let via_current = tokio::fs::read(current.join("makenotwork")).await.unwrap();
605 assert_eq!(via_current, b"PRIMARY");
606 }
607
608 #[tokio::test]
609 async fn finalize_second_release_swaps_symlink_and_keeps_old_dir() {
610 let tmp = tempfile::tempdir().unwrap();
611 let root = tmp.path();
612 let src_dir = root.join("src");
613 tokio::fs::create_dir_all(&src_dir).await.unwrap();
614 let bin = src_dir.join("server");
615 tokio::fs::write(&bin, b"V1").await.unwrap();
616
617 let release_root = root.join("rr");
618 tokio::fs::create_dir_all(&release_root).await.unwrap();
619
620 // Two builds, distinct digests (distinct content) -> two release dirs.
621 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
622 .await
623 .unwrap();
624 finalize_local_release(&release_root, &s1, "1111111111111111")
625 .await
626 .unwrap();
627 tokio::fs::write(&bin, b"V2").await.unwrap();
628 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
629 .await
630 .unwrap();
631 finalize_local_release(&release_root, &s2, "2222222222222222")
632 .await
633 .unwrap();
634
635 assert!(
636 release_root
637 .join("releases/1111111111111111/server")
638 .exists()
639 );
640 assert!(
641 release_root
642 .join("releases/2222222222222222/server")
643 .exists()
644 );
645 let target = tokio::fs::read_link(release_root.join("current"))
646 .await
647 .unwrap();
648 assert_eq!(target.to_string_lossy(), "releases/2222222222222222");
649 let via_current = tokio::fs::read(release_root.join("current/server"))
650 .await
651 .unwrap();
652 assert_eq!(via_current, b"V2");
653 }
654
655 #[tokio::test]
656 async fn finalize_reuses_an_existing_release_of_the_same_digest() {
657 let tmp = tempfile::tempdir().unwrap();
658 let root = tmp.path();
659 let bin = root.join("server");
660 tokio::fs::write(&bin, b"BYTES").await.unwrap();
661 let release_root = root.join("rr");
662 tokio::fs::create_dir_all(&release_root).await.unwrap();
663
664 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
665 .await
666 .unwrap();
667 finalize_local_release(&release_root, &s1, "abc123abc123abc1")
668 .await
669 .unwrap();
670 // Same digest rebuilt (e.g. a re-run at the same content): finalize must
671 // reuse the existing release and drop the redundant staging dir, not error.
672 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
673 .await
674 .unwrap();
675 let released = finalize_local_release(&release_root, &s2, "abc123abc123abc1")
676 .await
677 .expect("finalize is idempotent on a repeated digest");
678 assert_eq!(released, release_root.join("releases/abc123abc123abc1"));
679 assert!(!s2.exists(), "redundant staging dropped");
680 }
681
682 #[tokio::test]
683 async fn manifest_verify_script_passes_on_match_fails_on_drift_and_skips_when_absent() {
684 // The node-side verification is a shell running `sha256sum -c MANIFEST`;
685 // drive the real script through bash to prove it accepts a good bundle,
686 // rejects a tampered one, and no-ops on a legacy (MANIFEST-less) bundle.
687 let dir = tempfile::tempdir().unwrap();
688 tokio::fs::write(dir.path().join("server"), b"BINARY")
689 .await
690 .unwrap();
691 tokio::fs::create_dir(dir.path().join("static"))
692 .await
693 .unwrap();
694 tokio::fs::write(dir.path().join("static/app.css"), b"body{}")
695 .await
696 .unwrap();
697 let digest = crate::bundle::digest_dir(dir.path()).await.unwrap();
698 tokio::fs::write(dir.path().join("MANIFEST"), digest.manifest.as_bytes())
699 .await
700 .unwrap();
701
702 let run = |d: &std::path::Path| {
703 let script = manifest_verify_script(d.to_str().unwrap());
704 async move {
705 Command::new("bash")
706 .arg("-c")
707 .arg(&script)
708 .output()
709 .await
710 .unwrap()
711 }
712 };
713
714 let ok = run(dir.path()).await;
715 assert!(
716 ok.status.success(),
717 "matching bundle verifies: {}",
718 String::from_utf8_lossy(&ok.stderr)
719 );
720
721 // Drift one file: sha256sum -c must fail (current symlink left intact).
722 tokio::fs::write(dir.path().join("static/app.css"), b"TAMPERED")
723 .await
724 .unwrap();
725 let bad = run(dir.path()).await;
726 assert!(!bad.status.success(), "a drifted file fails verification");
727
728 // Legacy bundle with no MANIFEST: skip, not fail.
729 let legacy = tempfile::tempdir().unwrap();
730 tokio::fs::write(legacy.path().join("server"), b"x")
731 .await
732 .unwrap();
733 let skip = run(legacy.path()).await;
734 assert!(
735 skip.status.success(),
736 "a bundle without a MANIFEST skips verification rather than failing"
737 );
738 }
739
740 #[tokio::test]
741 async fn gc_local_releases_keeps_last_n_by_mtime() {
742 let tmp = tempfile::tempdir().unwrap();
743 let root = tmp.path();
744 let releases = root.join("releases");
745 tokio::fs::create_dir_all(&releases).await.unwrap();
746
747 let total = RELEASES_TO_KEEP + 3;
748 let mut names = Vec::new();
749 for i in 0..total {
750 let name = format!("v{i:02}");
751 let dir = releases.join(&name);
752 tokio::fs::create_dir(&dir).await.unwrap();
753 let f = std::fs::File::open(&dir).unwrap();
754 let when =
755 SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
756 let times = std::fs::FileTimes::new().set_modified(when);
757 f.set_times(times).unwrap();
758 names.push(name);
759 }
760
761 gc_local_releases(root).await.unwrap();
762
763 let surviving_expected: Vec<_> = names
764 .iter()
765 .skip(total - RELEASES_TO_KEEP)
766 .cloned()
767 .collect();
768 for name in &surviving_expected {
769 assert!(releases.join(name).exists(), "expected to survive: {name}");
770 }
771 for name in names.iter().take(total - RELEASES_TO_KEEP) {
772 assert!(
773 !releases.join(name).exists(),
774 "expected to be pruned: {name}"
775 );
776 }
777 }
778
779 #[tokio::test]
780 async fn gc_local_releases_noop_when_below_threshold() {
781 let tmp = tempfile::tempdir().unwrap();
782 let root = tmp.path();
783 let releases = root.join("releases");
784 tokio::fs::create_dir_all(&releases).await.unwrap();
785 for i in 0..3 {
786 tokio::fs::create_dir(releases.join(format!("v{i}")))
787 .await
788 .unwrap();
789 }
790 gc_local_releases(root).await.unwrap();
791 for i in 0..3 {
792 assert!(releases.join(format!("v{i}")).exists());
793 }
794 }
795
796 #[tokio::test]
797 async fn gc_local_releases_noop_when_releases_dir_missing() {
798 let tmp = tempfile::tempdir().unwrap();
799 gc_local_releases(tmp.path()).await.unwrap();
800 }
801
802 #[tokio::test]
803 async fn deploy_remote_fails_cleanly_when_host_unreachable() {
804 // 192.0.2.0/24 is reserved for documentation and routes nowhere.
805 // ConnectTimeout=10 limits the test wallclock to ~10s worst case.
806 let tmp = tempfile::tempdir().unwrap();
807 let staged = tmp.path().join("releases").join("0.0.1");
808 tokio::fs::create_dir_all(&staged).await.unwrap();
809 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
810
811 let node = crate::topology::Node {
812 name: "unreachable".into(),
813 ssh_target: "deploy@192.0.2.1".into(),
814 release_root: "/opt/never".into(),
815 service_name: "makenotwork.service".into(),
816 health_url: None,
817 config_check_env_file: None,
818 actuate: crate::topology::default_actuate(),
819 observe: crate::topology::default_observe(),
820 companions: Vec::new(),
821 };
822 let executor = SshExec::new(
823 node.ssh_target.clone(),
824 CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
825 );
826
827 let result = deploy_node(&executor, &node, "0.0.1", &staged, "server").await;
828 let err = result.expect_err("deploy to unreachable host should fail");
829 let msg = format!("{err:#}");
830 // Don't pin exact wording, just that the failure is attributed (ssh /
831 // rsync / connection) and that no panic / hang happened.
832 assert!(
833 msg.contains("ssh")
834 || msg.contains("rsync")
835 || msg.contains("connection")
836 || msg.contains("Connection"),
837 "unexpected error: {msg}"
838 );
839 }
840
841 #[tokio::test]
842 async fn deploy_node_with_local_ssh_target_swaps_symlink() {
843 // ssh_target="local" routes to the local fast-path: just a symlink
844 // swap, no remote calls.
845 let tmp = tempfile::tempdir().unwrap();
846 let release_root = tmp.path().to_path_buf();
847 let staged = release_root.join("releases").join("0.0.1");
848 tokio::fs::create_dir_all(&staged).await.unwrap();
849 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
850
851 let node = crate::topology::Node {
852 name: "local-dev".into(),
853 ssh_target: "local".into(),
854 release_root: release_root.to_string_lossy().into_owned(),
855 service_name: "makenotwork.service".into(),
856 health_url: None,
857 config_check_env_file: None,
858 actuate: crate::topology::default_actuate(),
859 observe: crate::topology::default_observe(),
860 companions: Vec::new(),
861 };
862 let executor = local_executor();
863
864 let out = deploy_node(&executor, &node, "0.0.1", &staged, "server")
865 .await
866 .unwrap();
867 assert_eq!(out, staged);
868 let target = tokio::fs::read_link(release_root.join("current"))
869 .await
870 .unwrap();
871 assert_eq!(target.to_string_lossy(), "releases/0.0.1");
872 }
873
874 // ---- swap_and_restart_script: symlink/restart consistency ----
875
876 async fn run_script(script: &str) -> std::process::Output {
877 Command::new("sh")
878 .arg("-c")
879 .arg(script)
880 .output()
881 .await
882 .unwrap()
883 }
884
885 async fn setup_release_root(with_current: bool) -> tempfile::TempDir {
886 let tmp = tempfile::tempdir().unwrap();
887 let root = tmp.path();
888 tokio::fs::create_dir_all(root.join("releases/old"))
889 .await
890 .unwrap();
891 tokio::fs::create_dir_all(root.join("releases/new"))
892 .await
893 .unwrap();
894 if with_current {
895 std::os::unix::fs::symlink("releases/old", root.join("current")).unwrap();
896 }
897 tmp
898 }
899
900 #[tokio::test]
901 async fn swap_and_restart_keeps_new_symlink_when_restart_succeeds() {
902 let tmp = setup_release_root(true).await;
903 let root = tmp.path().to_string_lossy().into_owned();
904 let out = run_script(&swap_and_restart_script(&root, "new", "true")).await;
905 assert!(
906 out.status.success(),
907 "script should succeed when restart succeeds"
908 );
909 let target = tokio::fs::read_link(tmp.path().join("current"))
910 .await
911 .unwrap();
912 assert_eq!(
913 target.to_string_lossy(),
914 "releases/new",
915 "symlink advanced to new"
916 );
917 }
918
919 #[tokio::test]
920 async fn swap_and_restart_rolls_symlink_back_when_restart_fails() {
921 // The bug: a restart failure after the flip must NOT leave `current`
922 // pointing at the new (un-activated) release.
923 let tmp = setup_release_root(true).await;
924 let root = tmp.path().to_string_lossy().into_owned();
925 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
926 assert!(!out.status.success(), "script must fail when restart fails");
927 let target = tokio::fs::read_link(tmp.path().join("current"))
928 .await
929 .unwrap();
930 assert_eq!(
931 target.to_string_lossy(),
932 "releases/old",
933 "symlink rolled back to prev so a later restart can't silently activate new",
934 );
935 }
936
937 // ---- arch_guard_script: wrong-arch artifacts fail closed ----
938
939 /// A 20-byte stub whose ELF e_machine field (offset 18, 2 bytes LE) is set.
940 fn elf_stub_with_machine(b18: u8, b19: u8) -> tempfile::NamedTempFile {
941 let mut data = vec![0u8; 20];
942 data[18] = b18;
943 data[19] = b19;
944 let f = tempfile::NamedTempFile::new().unwrap();
945 std::fs::write(f.path(), &data).unwrap();
946 f
947 }
948
949 /// e_machine low byte for the host running the test, if mapped.
950 fn host_machine_lo() -> Option<u8> {
951 match std::env::consts::ARCH {
952 "x86_64" => Some(0x3e),
953 "aarch64" => Some(0xb7),
954 _ => None,
955 }
956 }
957
958 #[tokio::test]
959 async fn arch_guard_passes_for_matching_binary() {
960 let Some(lo) = host_machine_lo() else { return };
961 let f = elf_stub_with_machine(lo, 0x00);
962 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
963 assert!(
964 out.status.success(),
965 "matching arch must pass: {}",
966 String::from_utf8_lossy(&out.stderr),
967 );
968 }
969
970 #[tokio::test]
971 async fn arch_guard_fails_closed_for_wrong_binary() {
972 // Use the other arch's e_machine so it can't match the host.
973 let wrong = match std::env::consts::ARCH {
974 "x86_64" => 0xb7, // aarch64 binary on an x86_64 node
975 "aarch64" => 0x3e, // x86_64 binary on an aarch64 node
976 _ => return,
977 };
978 let f = elf_stub_with_machine(wrong, 0x00);
979 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
980 assert!(
981 !out.status.success(),
982 "wrong-arch binary must fail closed before the symlink swap"
983 );
984 }
985
986 #[tokio::test]
987 async fn swap_and_restart_first_deploy_failure_has_no_prev_to_restore() {
988 // No prior `current`. A restart failure leaves `current` at new (the only
989 // version) and still reports failure — documented degenerate case.
990 let tmp = setup_release_root(false).await;
991 let root = tmp.path().to_string_lossy().into_owned();
992 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
993 assert!(!out.status.success(), "script must fail when restart fails");
994 let target = tokio::fs::read_link(tmp.path().join("current"))
995 .await
996 .unwrap();
997 assert_eq!(
998 target.to_string_lossy(),
999 "releases/new",
1000 "no prev existed to roll back to"
1001 );
1002 }
1003
1004 // ---- config_check_script: systemd-faithful env loading ----
1005
1006 #[tokio::test]
1007 async fn config_check_script_loads_values_with_shell_metachars() {
1008 // The bug: `. env_file` expands/word-splits values, so a URL or a
1009 // password containing a shell metacharacter is mangled — it dropped
1010 // DATABASE_URL to empty on a real node, which would fail every deploy.
1011 // The export-loop must load such a value intact. The "binary" is a
1012 // checker script (a real path, like a deployed binary) that exits 0 only
1013 // if the var arrived byte-for-byte — it compares against the expected
1014 // value read from a file, so nothing re-interprets the metacharacters.
1015 let tricky = "postgres://u:p$ss;w&rd@h/db `x` $(y)";
1016 // Plain files in a tempdir: no lingering write fd, so the checker can be
1017 // exec'd (a NamedTempFile stays open and would ETXTBSY).
1018 let dir = tempfile::tempdir().unwrap();
1019 let expected_path = dir.path().join("expected");
1020 std::fs::write(&expected_path, tricky).unwrap(); // no trailing newline
1021
1022 let env_path = dir.path().join("node.env");
1023 std::fs::write(
1024 &env_path,
1025 format!(
1026 "# a comment\n\nDATABASE_URL={tricky}\nOTHER=plain\nEXPECTED_FILE={ef}\n",
1027 ef = expected_path.display(),
1028 ),
1029 )
1030 .unwrap();
1031
1032 let checker_path = dir.path().join("checker.sh");
1033 std::fs::write(
1034 &checker_path,
1035 "#!/bin/sh\nwant=$(cat \"$EXPECTED_FILE\")\n\
1036 [ \"$DATABASE_URL\" = \"$want\" ] || { echo \"DB [$DATABASE_URL] != [$want]\" >&2; exit 1; }\n\
1037 [ \"$OTHER\" = plain ] || { echo \"OTHER [$OTHER]\" >&2; exit 1; }\n",
1038 )
1039 .unwrap();
1040 std::fs::set_permissions(
1041 &checker_path,
1042 std::os::unix::fs::PermissionsExt::from_mode(0o755),
1043 )
1044 .unwrap();
1045
1046 let script =
1047 config_check_script(&env_path.to_string_lossy(), &checker_path.to_string_lossy());
1048 let out = run_script(&script).await;
1049 assert!(
1050 out.status.success(),
1051 "value with shell metachars must load intact; stderr: {}",
1052 String::from_utf8_lossy(&out.stderr),
1053 );
1054 }
1055
1056 // ---- install-companion.sh: the node-side guard rails ----
1057
1058 /// Run the shipped installer script with three args; returns its exit code.
1059 /// Exercises the real file rather than a copy of its logic, because the
1060 /// script is the ONLY control on a NOPASSWD sudo grant.
1061 fn run_installer(src: &str, dst: &str, service: &str) -> i32 {
1062 let script =
1063 std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../deploy/install-companion.sh");
1064 std::process::Command::new("bash")
1065 .arg(&script)
1066 .args([src, dst, service])
1067 .output()
1068 .expect("running install-companion.sh")
1069 .status
1070 .code()
1071 .expect("script exited via signal")
1072 }
1073
1074 // Guards run before any filesystem write, so these never install anything.
1075 // Exit 3 = refused by a guard; exit 4 = guards passed, src simply absent.
1076 const REFUSED: i32 = 3;
1077 const PASSED_GUARDS: i32 = 4;
1078
1079 #[test]
1080 fn installer_refuses_a_dst_that_escapes_opt_via_dotdot() {
1081 // `/opt/../etc/...` matches a bare `/opt/*` glob. With the sudoers
1082 // wildcard that meant `install -m 0755` as root to anywhere, plus a
1083 // restart of any unit — so the path must be normalised before the test.
1084 assert_eq!(
1085 run_installer(
1086 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1087 "/opt/../etc/systemd/system/evil.service",
1088 "mnw-cli.service",
1089 ),
1090 REFUSED,
1091 );
1092 }
1093
1094 #[test]
1095 fn installer_refuses_a_src_that_escapes_the_bundle_via_dotdot() {
1096 assert_eq!(
1097 run_installer(
1098 "/opt/mnw/releases/1.0.0/companions/../../../../../etc/shadow",
1099 "/opt/mnw-cli/mnw-cli",
1100 "mnw-cli.service",
1101 ),
1102 REFUSED,
1103 );
1104 }
1105
1106 #[test]
1107 fn installer_accepts_the_real_companion_paths() {
1108 // The guards must not have been tightened into uselessness: the shape
1109 // Sando actually sends has to get past them. It stops at the missing
1110 // src (exit 4), which is proof the guards accepted it.
1111 assert_eq!(
1112 run_installer(
1113 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1114 "/opt/mnw-cli/mnw-cli",
1115 "mnw-cli.service",
1116 ),
1117 PASSED_GUARDS,
1118 );
1119 }
1120
1121 #[test]
1122 fn installer_refuses_a_service_name_with_a_path_separator() {
1123 assert_eq!(
1124 run_installer(
1125 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1126 "/opt/mnw-cli/mnw-cli",
1127 "../../etc/evil.service",
1128 ),
1129 REFUSED,
1130 );
1131 }
1132
1133 // ---- install_companion_cmd: shape + quoting ----
1134
1135 #[test]
1136 fn install_companion_cmd_shape_and_quoting() {
1137 let cmd = install_companion_cmd(
1138 "/opt/mnw/releases/0.10.14/companions/mnw-cli",
1139 "/opt/mnw-cli/mnw-cli",
1140 "mnw-cli.service",
1141 );
1142 // Routes through the wrapper (single sudoers grant), sudo-invoked, with
1143 // src, dst, service in that order.
1144 assert!(cmd.starts_with("sudo "), "must be sudo-invoked: {cmd}");
1145 assert!(
1146 cmd.contains("/usr/local/lib/mnw/install-companion.sh"),
1147 "{cmd}"
1148 );
1149 let installer_pos = cmd.find("install-companion.sh").unwrap();
1150 let src_pos = cmd.find("companions/mnw-cli").unwrap();
1151 let dst_pos = cmd.find("/opt/mnw-cli/mnw-cli").unwrap();
1152 let svc_pos = cmd.find("mnw-cli.service").unwrap();
1153 assert!(
1154 installer_pos < src_pos && src_pos < dst_pos && dst_pos < svc_pos,
1155 "arg order: {cmd}"
1156 );
1157 }
1158
1159 #[test]
1160 fn install_companion_cmd_quotes_metachars() {
1161 // A path with a space/quote must be shell-safe (defense in depth even
1162 // though these come from operator config).
1163 let cmd = install_companion_cmd("/a b/src", "/dst'x", "u.service");
1164 let out = std::process::Command::new("sh")
1165 .arg("-c")
1166 .arg(format!(
1167 "set -- {}; echo \"$#\"",
1168 cmd.strip_prefix("sudo ").unwrap()
1169 ))
1170 .output()
1171 .unwrap();
1172 // installer + 3 args = 4 positional words after quoting.
1173 assert_eq!(
1174 String::from_utf8_lossy(&out.stdout).trim(),
1175 "4",
1176 "quoting split wrong: {cmd}"
1177 );
1178 }
1179
1180 #[tokio::test]
1181 async fn config_check_script_propagates_binary_failure() {
1182 // A required var missing (the binary exits non-zero) must fail the check.
1183 let env = tempfile::NamedTempFile::new().unwrap();
1184 std::fs::write(env.path(), "FOO=bar\n").unwrap();
1185 let script = config_check_script(&env.path().to_string_lossy(), "false");
1186 let out = run_script(&script).await;
1187 assert!(
1188 !out.status.success(),
1189 "a non-zero MNW_CHECK_CONFIG exit must fail the check"
1190 );
1191 }
1192
1193 #[tokio::test]
1194 async fn deploy_node_denied_when_executor_lacks_deploy_grant() {
1195 // Defense in depth: an executor without the deploy grant refuses the
1196 // step before any filesystem / ssh action.
1197 let tmp = tempfile::tempdir().unwrap();
1198 let release_root = tmp.path().to_path_buf();
1199 let staged = release_root.join("releases").join("0.0.1");
1200 tokio::fs::create_dir_all(&staged).await.unwrap();
1201
1202 let node = crate::topology::Node {
1203 name: "local-dev".into(),
1204 ssh_target: "local".into(),
1205 release_root: release_root.to_string_lossy().into_owned(),
1206 service_name: "makenotwork.service".into(),
1207 health_url: None,
1208 config_check_env_file: None,
1209 actuate: vec!["restart".into()], // no deploy
1210 observe: vec![],
1211 companions: Vec::new(),
1212 };
1213 let executor = LocalExec::new(CapabilitySet::from_tokens(["restart"], Vec::<&str>::new()));
1214 let err = deploy_node(&executor, &node, "0.0.1", &staged, "server")
1215 .await
1216 .unwrap_err();
1217 assert!(
1218 format!("{err:#}").contains("capability denied"),
1219 "expected capability denial"
1220 );
1221 }
1222
1223 // ---- FakeExec: the deploy_remote choreography without a real host ----
1224 //
1225 // deploy_node's local fast-path is covered above with a real LocalExec, but
1226 // the remote path (rsync + arch guard + config-drift + swap + companions +
1227 // gc) short-circuits on `ssh_target != "local"` and so never ran under test
1228 // without a reachable node. FakeExec records every executor call in order
1229 // and can be told to fail one shell step (matched by substring) or the rsync
1230 // push, so the ordering and the fail-closed-before-swap contract are
1231 // assertable in-process.
1232
1233 struct FakeExec {
1234 caps: CapabilitySet,
1235 calls: Arc<StdMutex<Vec<String>>>,
1236 /// The first `run_streaming` whose script contains this substring exits
1237 /// non-zero (a failed shell step), e.g. the arch guard.
1238 fail_run_matching: Option<String>,
1239 /// `push_dir` (the rsync) returns an error.
1240 fail_push_dir: bool,
1241 }
1242
1243 impl FakeExec {
1244 fn new() -> Self {
1245 Self {
1246 caps: CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1247 calls: Arc::new(StdMutex::new(Vec::new())),
1248 fail_run_matching: None,
1249 fail_push_dir: false,
1250 }
1251 }
1252 fn log(&self) -> Vec<String> {
1253 self.calls.lock().unwrap().clone()
1254 }
1255 }
1256
1257 #[async_trait]
1258 impl Executor for FakeExec {
1259 async fn run_streaming(&self, step: &Step, _sink: &mut dyn LogSink) -> Result<RunOutput> {
1260 // Every deploy step is a `Step::shell`, so the script is argv's tail.
1261 let script = step.argv.last().cloned().unwrap_or_default();
1262 self.calls.lock().unwrap().push(format!("run:{script}"));
1263 let fail = self
1264 .fail_run_matching
1265 .as_deref()
1266 .is_some_and(|m| script.contains(m));
1267 Ok(RunOutput {
1268 status: std::process::ExitStatus::from_raw(if fail { 1 << 8 } else { 0 }),
1269 stdout: Vec::new(),
1270 stderr: if fail {
1271 b"fake step failure".to_vec()
1272 } else {
1273 Vec::new()
1274 },
1275 })
1276 }
1277 async fn pull_file(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1278 self.calls.lock().unwrap().push("pull_file".into());
1279 Ok(())
1280 }
1281 async fn pull_dir(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1282 self.calls.lock().unwrap().push("pull_dir".into());
1283 Ok(())
1284 }
1285 async fn pull_glob(&self, _glob: &str, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1286 self.calls.lock().unwrap().push("pull_glob".into());
1287 Ok(())
1288 }
1289 async fn push_dir(&self, _local: &Path, remote: &Path, _opts: &SyncOpts) -> Result<()> {
1290 self.calls
1291 .lock()
1292 .unwrap()
1293 .push(format!("push_dir:{}", remote.display()));
1294 if self.fail_push_dir {
1295 anyhow::bail!("fake rsync failure");
1296 }
1297 Ok(())
1298 }
1299 fn capabilities(&self) -> &CapabilitySet {
1300 &self.caps
1301 }
1302 }
1303
1304 fn remote_node(config_check: bool, companions: Vec<NodeCompanion>) -> Node {
1305 Node {
1306 name: "web-a".into(),
1307 ssh_target: "deploy@web-a".into(),
1308 release_root: "/opt/mnw".into(),
1309 service_name: "makenotwork.service".into(),
1310 health_url: None,
1311 config_check_env_file: config_check.then(|| "/etc/mnw/node.env".to_string()),
1312 actuate: crate::topology::default_actuate(),
1313 observe: crate::topology::default_observe(),
1314 companions,
1315 }
1316 }
1317
1318 fn companion() -> NodeCompanion {
1319 NodeCompanion {
1320 name: "mnw-cli".into(),
1321 install_path: "/opt/mnw-cli/mnw-cli".into(),
1322 service_name: "mnw-cli.service".into(),
1323 }
1324 }
1325
1326 /// Index of the first recorded call whose text contains `needle` (panics if
1327 /// absent — the assertion message names what was missing).
1328 fn pos(log: &[String], needle: &str) -> usize {
1329 log.iter()
1330 .position(|c| c.contains(needle))
1331 .unwrap_or_else(|| panic!("no call matched {needle:?} in {log:#?}"))
1332 }
1333
1334 #[tokio::test]
1335 async fn deploy_remote_runs_the_full_choreography_in_order() {
1336 // A node opted into the config-drift check and carrying one companion:
1337 // mkdir -> rsync -> arch guard -> config check -> swap+restart ->
1338 // companion install -> gc, in that order.
1339 let tmp = tempfile::tempdir().unwrap();
1340 let staged = tmp.path().join("releases").join("0.9.0");
1341 tokio::fs::create_dir_all(&staged).await.unwrap();
1342
1343 let node = remote_node(true, vec![companion()]);
1344 let exec = FakeExec::new();
1345 let out = deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1346 .await
1347 .expect("deploy_remote should succeed against the fake");
1348 assert_eq!(out, PathBuf::from("/opt/mnw/releases/0.9.0"));
1349
1350 let log = exec.log();
1351 let mkdir = pos(&log, "mkdir -p");
1352 let rsync = pos(&log, "push_dir:/opt/mnw/releases/0.9.0");
1353 let arch = pos(&log, "e_machine");
1354 let cfg = pos(&log, "MNW_CHECK_CONFIG=1");
1355 let swap = pos(&log, "reload-or-restart");
1356 let comp = pos(&log, "install-companion.sh");
1357 let gc = pos(&log, "ls -1t");
1358 assert!(
1359 mkdir < rsync && rsync < arch && arch < cfg && cfg < swap && swap < comp && comp < gc,
1360 "deploy steps out of order: {log:#?}"
1361 );
1362 }
1363
1364 #[tokio::test]
1365 async fn deploy_remote_aborts_before_swap_when_rsync_fails() {
1366 // The rsync failing must fail the deploy BEFORE the symlink swap — the
1367 // "current symlink left intact" contract. Assert the swap never ran.
1368 let tmp = tempfile::tempdir().unwrap();
1369 let staged = tmp.path().join("releases").join("0.9.0");
1370 tokio::fs::create_dir_all(&staged).await.unwrap();
1371
1372 let node = remote_node(false, Vec::new());
1373 let mut exec = FakeExec::new();
1374 exec.fail_push_dir = true;
1375 let err = deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1376 .await
1377 .expect_err("rsync failure must fail the deploy");
1378 assert!(
1379 format!("{err:#}").contains("rsync"),
1380 "error should attribute the rsync: {err:#}"
1381 );
1382 let log = exec.log();
1383 assert!(
1384 !log.iter().any(|c| c.contains("reload-or-restart")),
1385 "swap must not run after a failed rsync: {log:#?}"
1386 );
1387 }
1388
1389 #[tokio::test]
1390 async fn deploy_remote_aborts_before_swap_when_arch_guard_fails() {
1391 // A wrong-arch binary must fail closed before the swap. The fake fails
1392 // the arch-guard shell step; the swap must not follow.
1393 let tmp = tempfile::tempdir().unwrap();
1394 let staged = tmp.path().join("releases").join("0.9.0");
1395 tokio::fs::create_dir_all(&staged).await.unwrap();
1396
1397 let node = remote_node(false, Vec::new());
1398 let mut exec = FakeExec::new();
1399 exec.fail_run_matching = Some("e_machine".into());
1400 let err = deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1401 .await
1402 .expect_err("arch mismatch must fail the deploy");
1403 assert!(
1404 format!("{err:#}").contains("architecture"),
1405 "error should mention the arch check: {err:#}"
1406 );
1407 let log = exec.log();
1408 assert!(
1409 !log.iter().any(|c| c.contains("reload-or-restart")),
1410 "swap must not run after a failed arch guard: {log:#?}"
1411 );
1412 }
1413
1414 #[tokio::test]
1415 async fn deploy_remote_skips_config_check_when_node_opts_out() {
1416 // No config_check_env_file => the pre-swap config check is skipped, but
1417 // the rest of the choreography (including the swap) still runs.
1418 let tmp = tempfile::tempdir().unwrap();
1419 let staged = tmp.path().join("releases").join("0.9.0");
1420 tokio::fs::create_dir_all(&staged).await.unwrap();
1421
1422 let node = remote_node(false, Vec::new());
1423 let exec = FakeExec::new();
1424 deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1425 .await
1426 .unwrap();
1427 let log = exec.log();
1428 assert!(
1429 !log.iter().any(|c| c.contains("MNW_CHECK_CONFIG=1")),
1430 "config check must be skipped when the node opts out: {log:#?}"
1431 );
1432 assert!(
1433 log.iter().any(|c| c.contains("reload-or-restart")),
1434 "the swap must still run: {log:#?}"
1435 );
1436 }
1437
1438 #[tokio::test]
1439 async fn deploy_remote_installs_companion_after_the_swap() {
1440 // Companions are After= the server: their install must land after the
1441 // symlink swap + service restart, never before.
1442 let tmp = tempfile::tempdir().unwrap();
1443 let staged = tmp.path().join("releases").join("0.9.0");
1444 tokio::fs::create_dir_all(&staged).await.unwrap();
1445
1446 let node = remote_node(false, vec![companion()]);
1447 let exec = FakeExec::new();
1448 deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1449 .await
1450 .unwrap();
1451 let log = exec.log();
1452 assert!(
1453 pos(&log, "reload-or-restart") < pos(&log, "install-companion.sh"),
1454 "companion install must follow the swap: {log:#?}"
1455 );
1456 }
1457 }
1458