Skip to main content

max / makenotwork

112.6 KB · 2798 lines History Blame Raw
1 //! Atomic symlink-swap deploys.
2 //!
3 //! Layout on every target (local host, A nodes, B nodes, ...). Release dirs are
4 //! named for their content digest, not the version (wiki note
5 //! `release-artifact-identity`), so a rebuild can never overwrite an earlier
6 //! build's dir in place:
7 //!
8 //! <release_root>/
9 //! releases/
10 //! a1b2c3d4e5f60718/ <- <digest16>
11 //! <bin_name>
12 //! MANIFEST <- <sha256> <relpath> per file, node-verified
13 //! f0e1d2c3b4a59687/
14 //! <bin_name>
15 //! current -> releases/f0e1d2c3b4a59687
16 //!
17 //! `ln -sfn` swaps the symlink. systemd units point at
18 //! `<release_root>/current/<bin_name>` so reload-or-restart picks up the new
19 //! binary without ever pointing at a missing path.
20 //!
21 //! The host-side transport — `ssh` for shell steps, `rsync` for the release
22 //! dir — comes from the shared [`ops_exec::Executor`] (a `LocalExec` for
23 //! `ssh_target = "local"`, an `SshExec` otherwise), built once per node in
24 //! [`crate::state`]. This module owns the *deploy choreography* (mkdir, push,
25 //! atomic swap, restart, gc); the transport is the crate's. SSH push behavior
26 //! is identical to the pre-extraction code — this is a transport extraction,
27 //! not a model change.
28
29 use crate::domain::Platform;
30 use crate::retention::PinnedReleases;
31 use crate::topology::Node;
32 use anyhow::{Context, Result};
33 use async_trait::async_trait;
34 use ops_core::base_image;
35 use ops_exec::{Action, Executor, LogSink, RunOutput, Step, SyncOpts, sh_quote};
36 use std::path::{Path, PathBuf};
37 use tokio::process::Command;
38
39 /// A staged bundle proven to be for the node it is about to be pushed to.
40 ///
41 /// This exists because "ship aarch64 bytes to an x86_64 box" was, until pom, a
42 /// mistake nobody could make: one product, one build host, one architecture, so
43 /// the pairing of a bundle and a node was correct by having no alternative. pom
44 /// has two architectures under one version, so the pairing becomes a real
45 /// choice, and a wrong choice deploys a binary the node cannot exec.
46 ///
47 /// The answer is not a check before the call. A check is something a later
48 /// caller forgets, and the failure it guards is discovered by a production node
49 /// failing to start. [`Placement::check`] is the *only* way to obtain one of
50 /// these, and [`deploy_node`] takes one instead of a loose `(node, dir)` pair —
51 /// so a mismatched deploy is not a bug the code has to avoid, it is a value the
52 /// code cannot construct.
53 #[derive(Debug, Clone)]
54 pub struct Placement<'a> {
55 node: &'a Node,
56 bundle: &'a Path,
57 }
58
59 /// Why a bundle may not be placed on a node.
60 ///
61 /// All four cases are refusals, including both "one side said nothing" cases.
62 /// Silence is not agreement: a node that does not state its platform cannot
63 /// vouch that it runs a bundle built for a stated one, and a bundle that does
64 /// not state its platform cannot satisfy a node that requires one. The only
65 /// admissible pairing besides a match is both sides silent, which is the
66 /// single-platform world Sando lived in and MNW still lives in.
67 #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
68 pub enum PlacementError {
69 #[error(
70 "node {node} runs {node_platform} and this bundle was built for {artifact_platform}; \
71 refusing to deploy a binary the node cannot execute"
72 )]
73 Mismatch {
74 node: String,
75 node_platform: Platform,
76 artifact_platform: Platform,
77 },
78 #[error(
79 "node {node} does not declare a platform, and this bundle was built for \
80 {artifact_platform}. Declare `platform` on the node so the two can be compared"
81 )]
82 NodeSilent {
83 node: String,
84 artifact_platform: Platform,
85 },
86 #[error(
87 "node {node} requires {node_platform} and this bundle records no platform. \
88 An artifact whose platform is unknown cannot be shown to satisfy one that is"
89 )]
90 ArtifactSilent {
91 node: String,
92 node_platform: Platform,
93 },
94 }
95
96 impl<'a> Placement<'a> {
97 /// The one constructor. `artifact` is the platform the bundle records, which
98 /// for an accepted artifact comes from its `ArtifactRecord` provenance and
99 /// for a Sando-built one comes from the app config.
100 pub fn check(
101 node: &'a Node,
102 bundle: &'a Path,
103 artifact: Option<&Platform>,
104 ) -> Result<Self, PlacementError> {
105 match (node.platform.as_ref(), artifact) {
106 (Some(n), Some(a)) if n == a => Ok(Self { node, bundle }),
107 (Some(n), Some(a)) => Err(PlacementError::Mismatch {
108 node: node.name.to_string(),
109 node_platform: n.clone(),
110 artifact_platform: a.clone(),
111 }),
112 (None, Some(a)) => Err(PlacementError::NodeSilent {
113 node: node.name.to_string(),
114 artifact_platform: a.clone(),
115 }),
116 (Some(n), None) => Err(PlacementError::ArtifactSilent {
117 node: node.name.to_string(),
118 node_platform: n.clone(),
119 }),
120 // Both silent: the single-platform world. MNW is here, and stays
121 // here until its nodes declare a platform — at which point its
122 // builds have to as well, which is the forcing function rather than
123 // a silently mixed state.
124 (None, None) => Ok(Self { node, bundle }),
125 }
126 }
127
128 pub fn node(&self) -> &'a Node {
129 self.node
130 }
131
132 pub fn bundle(&self) -> &'a Path {
133 self.bundle
134 }
135 }
136
137 /// Keep at least this many release dirs per node; older ones get gc'd after a
138 /// successful deploy. Fixed for now; promote to config if the constant ever
139 /// needs to vary by tier.
140 ///
141 /// A floor, not a ceiling. Whatever the deployed state still references is set
142 /// aside first and this count applies to the remainder — see
143 /// [`crate::retention`] for why a count alone could not express that.
144 const RELEASES_TO_KEEP: usize = 5;
145
146 /// A sink that drops streamed bytes. Deploy steps don't have a live-log handle
147 /// (gates do), so output is discarded as it streams; [`RunOutput`] still
148 /// captures the full stdout/stderr for error reporting, preserving the
149 /// pre-extraction behavior of surfacing `stderr` in failure messages.
150 struct DiscardSink;
151
152 #[async_trait]
153 impl LogSink for DiscardSink {
154 async fn write_chunk(&mut self, _bytes: &[u8]) {}
155 }
156
157 /// Run a shell step through `executor`, treating a non-zero exit as an error
158 /// whose message carries the captured stderr (`ssh <target> failed: <stderr>`).
159 async fn run_checked(executor: &dyn Executor, script: &str, what: &str) -> Result<RunOutput> {
160 let step = Step::shell(Action::Deploy, script);
161 let mut sink = DiscardSink;
162 let out = executor
163 .run_streaming(&step, &mut sink)
164 .await
165 .with_context(|| format!("{what}: spawning command"))?;
166 anyhow::ensure!(
167 out.status.success(),
168 "{what} failed (exit {}): {}",
169 out.status
170 .code()
171 .map_or_else(|| "signal".into(), |c| c.to_string()),
172 String::from_utf8_lossy(&out.stderr),
173 );
174 Ok(out)
175 }
176
177 /// Stage built binaries into `staging/<build_id>/` on the Sando host: a private,
178 /// mutable scratch dir that is not yet a release (no symlink, no gc).
179 /// The caller adds `release_contents` + companions, hashes the result, writes
180 /// the `MANIFEST`, then publishes it content-addressed via
181 /// [`finalize_local_release`]. Splitting staging from publish is what lets the
182 /// bundle be hashed before it is named (wiki [[release-artifact-identity]]).
183 ///
184 /// A stale `staging/<build_id>` from a killed prior run at the same id is
185 /// removed first, so a retry stages clean.
186 pub async fn stage_local_bundle(
187 release_root: &Path,
188 build_id: i64,
189 binaries: &[PathBuf],
190 ) -> Result<PathBuf> {
191 let staging = release_root.join("staging").join(build_id.to_string());
192 if tokio::fs::try_exists(&staging).await.unwrap_or(false) {
193 tokio::fs::remove_dir_all(&staging)
194 .await
195 .with_context(|| format!("clearing stale staging dir {}", staging.display()))?;
196 }
197 tokio::fs::create_dir_all(&staging).await?;
198 for binary in binaries {
199 let name = binary.file_name().context("binary path has no file name")?;
200 let dest = staging.join(name);
201 tokio::fs::copy(binary, &dest)
202 .await
203 .with_context(|| format!("copy {} -> {}", binary.display(), dest.display()))?;
204 }
205 Ok(staging)
206 }
207
208 /// Publish a fully-staged bundle content-addressed: rename
209 /// `staging/<build_id>` to `releases/<digest16>` (atomic same-filesystem
210 /// rename), flip `current` to it, gc old releases. Returns the released dir.
211 ///
212 /// The rename is the load-bearing step: **a directory whose name derives from
213 /// its contents cannot be rewritten, because rewriting it changes its name.**
214 /// The overwrite class that let a dev rebuild inherit an earlier build's gate
215 /// rows and burn-in clock stops being something to guard against and becomes
216 /// something that cannot be expressed. If a release with this digest already
217 /// exists (identical bytes rebuilt), the staging copy is redundant and dropped.
218 ///
219 /// `pinned` names the release dirs the deployed state still points at
220 /// ([`crate::retention::pinned_dirs`]); they are never gc'd here, however old
221 /// they are. It arrives as data rather than as a pool handle so this stays a
222 /// filesystem operation and the intake seam keeps working without a database.
223 pub async fn finalize_local_release(
224 release_root: &Path,
225 staging: &Path,
226 digest16: &str,
227 pinned: &PinnedReleases,
228 ) -> Result<PathBuf> {
229 let releases = release_root.join("releases");
230 tokio::fs::create_dir_all(&releases).await?;
231 let released = releases.join(digest16);
232
233 if tokio::fs::try_exists(&released).await.unwrap_or(false) {
234 // Same digest already published — reuse it, discard the redundant stage.
235 tokio::fs::remove_dir_all(staging).await.ok();
236 } else {
237 tokio::fs::rename(staging, &released)
238 .await
239 .with_context(|| format!("publish {} -> {}", staging.display(), released.display()))?;
240 }
241
242 let current = release_root.join("current");
243 let target = format!("releases/{digest16}");
244 let out = Command::new("ln")
245 .args(["-sfn", &target])
246 .arg(&current)
247 .output()
248 .await?;
249 anyhow::ensure!(
250 out.status.success(),
251 "symlink swap failed: {}",
252 String::from_utf8_lossy(&out.stderr),
253 );
254
255 if let Err(e) = gc_local_releases(release_root, pinned).await {
256 tracing::warn!(error = %e, "local release GC failed (non-fatal)");
257 }
258 Ok(released)
259 }
260
261 /// Deploy a [`Placement`]'s bundle to its node using `executor` (the node's
262 /// transport from the topology executor map). For `ssh_target=local`, this is
263 /// just a symlink swap; for remote nodes, we rsync the whole dir over the
264 /// executor.
265 ///
266 /// The bundle and the node arrive together inside the placement, so there is no
267 /// signature here that accepts a bundle and a node that were never compared.
268 ///
269 /// `primary_bin` is only used for logging — every file present in the staged
270 /// dir gets shipped.
271 ///
272 /// `pinned` protects the node's own `releases/` from the gc that runs after a
273 /// successful remote deploy. The node mirrors the host's directory name, so the
274 /// set is the host's ([`crate::retention::pinned_dirs`]) with nothing recomputed
275 /// per node. `None` says the caller could not determine it and the gc is skipped
276 /// rather than run blind; `Some(PinnedReleases::none())` says there is genuinely
277 /// nothing deployed to protect. The distinction matters — the first is ignorance
278 /// and the second is knowledge — which is why this is an `Option` and not an
279 /// empty set standing in for both.
280 ///
281 /// Unused on the `ssh_target=local` path: that deploy is a symlink swap over a
282 /// store the host gc already owns.
283 pub async fn deploy_node(
284 executor: &dyn Executor,
285 placement: Placement<'_>,
286 version: &str,
287 primary_bin: &str,
288 pinned: Option<&PinnedReleases>,
289 ) -> Result<PathBuf> {
290 let node = placement.node();
291 let staged_release_dir = placement.bundle();
292 // The release dir is named for its content digest (`releases/<digest16>`),
293 // not the version. The node mirrors that name so host and node agree on the
294 // artifact's identity; the version is only a log label here. Legacy staged
295 // dirs (pre-identity, still `releases/<version>`) work unchanged — the name
296 // is whatever the host staged under.
297 let release_id = staged_release_dir
298 .file_name()
299 .and_then(|n| n.to_str())
300 .with_context(|| {
301 format!(
302 "staged release dir {} has no usable name",
303 staged_release_dir.display()
304 )
305 })?;
306 if node.ssh_target == "local" || node.ssh_target.is_empty() {
307 // Local deploy already happened when we staged on the Sando host.
308 // Just re-point `current` at the staged dir.
309 return reset_local_current(executor, Path::new(&node.release_root), release_id).await;
310 }
311 deploy_remote(
312 executor,
313 node,
314 version,
315 release_id,
316 staged_release_dir,
317 primary_bin,
318 pinned,
319 )
320 .await
321 }
322
323 /// Where a node deploy failed, relative to the symlink swap.
324 ///
325 /// The distinction is the whole difference between "nothing happened" and "go
326 /// look at production now". Carried as a typed value rather than in the wording
327 /// of a `.context()` string, so the reporting layer can act on it instead of
328 /// reporting every rollback failure as though the node were stranded on the new
329 /// version.
330 ///
331 /// Attached as `anyhow` context, so it both reads correctly in the error chain
332 /// and can be recovered with `stage_of`.
333 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
334 pub enum FailureStage {
335 /// Failed before the swap ran. `current` still points at the old release
336 /// and the service was never restarted, so the node is on the OLD version.
337 /// Nothing is stranded and nothing needs doing.
338 BeforeSwap,
339 /// Failed at or after the swap. The node's version is not knowable from
340 /// here: the swap script rolls `current` back if the restart fails, but a
341 /// failure between the two, or in a companion after the server is already
342 /// live, can leave the node on either version.
343 AtOrAfterSwap,
344 }
345
346 impl std::fmt::Display for FailureStage {
347 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
348 match self {
349 Self::BeforeSwap => {
350 f.write_str("current symlink left intact; node is on the previous version")
351 }
352 Self::AtOrAfterSwap => {
353 f.write_str("the symlink swap had already run; node version is indeterminate")
354 }
355 }
356 }
357 }
358
359 /// Recover the [`FailureStage`] from a deploy error's context chain.
360 ///
361 /// `None` means the error predates the stage annotation or came from somewhere
362 /// that does not set one. Callers must treat that as indeterminate rather than
363 /// as safe: guessing "before the swap" would reintroduce the bug in the
364 /// opposite, worse direction.
365 pub fn stage_of(err: &anyhow::Error) -> Option<FailureStage> {
366 // anyhow's own downcast_ref searches attached context values, not just the
367 // source chain, which is where a `.context(FailureStage::…)` lands.
368 err.downcast_ref::<FailureStage>().copied()
369 }
370
371 async fn reset_local_current(
372 executor: &dyn Executor,
373 release_root: &Path,
374 release_id: &str,
375 ) -> Result<PathBuf> {
376 let current = release_root.join("current");
377 let target = format!("releases/{release_id}");
378 run_checked(
379 executor,
380 &format!(
381 "ln -sfn {} {}",
382 sh_quote(&target),
383 sh_quote(&current.to_string_lossy())
384 ),
385 "local symlink swap",
386 )
387 .await?;
388 Ok(release_root.join("releases").join(release_id))
389 }
390
391 /// Confirm the node is what the topology says it is, before anything is pushed.
392 ///
393 /// The declared half of what the pre-swap `ldd` guard does at runtime. That
394 /// guard compares a specific binary against a specific box and is the last line;
395 /// this compares the box against its own declaration and is the first. Both
396 /// exist on purpose: the guard catches a binary nobody declared anything about,
397 /// and this catches a machine that stopped being what the config claims, which
398 /// the guard can only report as a symbol it cannot resolve.
399 ///
400 /// A node that declares nothing is not checked. That is a skip and it is logged
401 /// as one, so an unchecked node is visible rather than looking checked. See
402 /// [`ops_core::base_image`] for why this differs from [`Placement::check`].
403 async fn check_node_identity(executor: &dyn Executor, node: &Node) -> Result<()> {
404 if node.base_image.is_none() && node.libc.is_none() {
405 tracing::info!(
406 node = %node.name,
407 "deploy: node declares no base image; identity not checked"
408 );
409 return Ok(());
410 }
411 let out = run_checked(
412 executor,
413 &base_image::probe_cmd(),
414 "asking the node what it is",
415 )
416 .await
417 .context(FailureStage::BeforeSwap)?;
418 let reported = base_image::parse_probe(&String::from_utf8_lossy(&out.stdout));
419 match base_image::check(
420 node.name.as_str(),
421 node.base_image.as_ref(),
422 node.libc.as_deref(),
423 &reported,
424 ) {
425 Ok(checked) => {
426 if let Some(what) = checked {
427 tracing::info!(node = %node.name, "deploy: identity checked, {what}");
428 }
429 Ok(())
430 }
431 Err(drift) => Err(anyhow::Error::new(drift))
432 .context("the node is not what the topology declares it to be")
433 .context(FailureStage::BeforeSwap),
434 }
435 }
436
437 /// Refuse a bundle whose glibc floor is above what the node declares, before
438 /// the rsync.
439 ///
440 /// The weak, early half of a pair. `ldd_guard_script` runs the node's own loader
441 /// against the actual bytes one step before the symlink swap, which covers every
442 /// shared library and every symbol version rather than glibc alone. Nothing here
443 /// replaces it, and a bundle that passes this can still fail that.
444 ///
445 /// What this adds is *when*. The loader check happens after the bundle is built
446 /// and rsynced; this happens before either, so the subset of failures that two
447 /// declared numbers already prove is refused at the start of the promote instead
448 /// of most of the way through it. The zero-margin state on production makes that
449 /// subset a live one: three of its five binaries sit exactly on the box's glibc,
450 /// so a build host drifting one point release ahead puts every promote here.
451 ///
452 /// Skipped, and logged as skipped, in all three cases where there is nothing to
453 /// compare: the node declares no `libc`, the bundle states no floor (static, or
454 /// no ELF this parser reads), or the declared `libc` is not a version string.
455 /// The last is a config typo rather than a bad bundle, and refusing a deploy
456 /// over it would be answering the wrong question loudly.
457 async fn check_bundle_fits_node(node: &Node, staged_release_dir: &Path) -> Result<()> {
458 let Some(declared) = node.libc.as_deref() else {
459 return Ok(());
460 };
461 let Some(node_libc) = crate::elf::GlibcVersion::parse(declared) else {
462 tracing::warn!(
463 node = %node.name,
464 declared,
465 "deploy: node's declared libc is not a version; glibc floor not compared"
466 );
467 return Ok(());
468 };
469 let digest = crate::bundle::digest_dir(staged_release_dir)
470 .await
471 .context("reading the staged bundle's glibc floor")
472 .context(FailureStage::BeforeSwap)?;
473 let Some(floor) = digest.glibc_floor else {
474 tracing::info!(
475 node = %node.name,
476 "deploy: bundle states no glibc floor; nothing to compare"
477 );
478 return Ok(());
479 };
480 if floor > node_libc {
481 return Err(anyhow::anyhow!(
482 "this bundle needs glibc {floor} and `{node}` declares {node_libc}; \
483 refusing to ship a binary the node cannot load. Either the build host \
484 drifted ahead of the node, or the node's declared libc is stale",
485 node = node.name,
486 ))
487 .context(FailureStage::BeforeSwap);
488 }
489 tracing::info!(
490 node = %node.name,
491 "deploy: glibc floor {floor} fits the node's {node_libc}"
492 );
493 Ok(())
494 }
495
496 async fn deploy_remote(
497 executor: &dyn Executor,
498 node: &Node,
499 version: &str,
500 release_id: &str,
501 staged_release_dir: &Path,
502 primary_bin: &str,
503 pinned: Option<&PinnedReleases>,
504 ) -> Result<PathBuf> {
505 let release_root = &node.release_root;
506 let service = &node.service_name;
507 let release_dir = format!("{release_root}/releases/{release_id}");
508
509 // Identity check first, before a single byte moves. A node that was rebuilt
510 // into something else is refused here rather than after an rsync, and long
511 // before the pre-swap `ldd` guard would have caught the consequence without
512 // naming the cause. Cheap: one shell round-trip that reads /etc/os-release.
513 check_node_identity(executor, node).await?;
514
515 // And that the bundle could load there at all, from two numbers, before the
516 // bytes move. The `ldd` guard below asks the stronger question on the node
517 // itself; this one is only earlier.
518 check_bundle_fits_node(node, staged_release_dir).await?;
519
520 tracing::info!(node = %node.name, version, release_id, "deploy: mkdir release dir");
521 run_checked(
522 executor,
523 &format!("set -e; mkdir -p {q}", q = sh_quote(&release_dir)),
524 "creating remote release dir",
525 )
526 .await
527 .context(FailureStage::BeforeSwap)?;
528
529 tracing::info!(node = %node.name, version, primary = %primary_bin, "deploy: rsync release dir");
530 // Rsync the whole staged dir (binaries + every release_contents entry).
531 // `SyncOpts::release_mirror()` is the exact pre-extraction rsync flag set:
532 // -az --partial --delete --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r,F+X.
533 // --delete: removed assets across versions don't accumulate on the
534 // target; the bundle stays self-contained per version.
535 // --chmod: F+X preserves the execute bit per-file (binaries land 0755,
536 // data files 0644) instead of a blanket 0755.
537 executor
538 .push_dir(
539 staged_release_dir,
540 Path::new(&release_dir),
541 &SyncOpts::release_mirror(),
542 )
543 .await
544 .context("rsync failed")
545 .context(FailureStage::BeforeSwap)?;
546
547 // Verify the bundle on the node against its own MANIFEST before the swap
548 // (invariant 3, wiki [[release-artifact-identity]]). The MANIFEST shipped in
549 // the bundle is exactly `sha256sum` check format (`<hash> <relpath>`), so
550 // this re-hashes every file on the node and names any that drifted in
551 // transit — the hash is load-bearing, not merely recorded. Bundles staged
552 // by a pre-identity build carry no MANIFEST; those skip verification (logged)
553 // rather than fail, so a mid-migration deploy of a legacy artifact still
554 // ships. A mismatch fails the promote with the running service intact.
555 run_checked(
556 executor,
557 &manifest_verify_script(&release_dir),
558 "verifying bundle digest on node",
559 )
560 .await
561 .context("node-side bundle verification failed")
562 .context(FailureStage::BeforeSwap)?;
563
564 // Fail closed on a wrong-architecture binary before the symlink swap. The
565 // "never cross-compile" rule is enforced at build time (build_host check),
566 // but nothing verified the artifact's arch matched the *target* node — so
567 // adding an aarch64 node to a tier built on x86_64 would silently symlink an
568 // unrunnable binary live. Compare the deployed binary's ELF e_machine to the
569 // node's `uname -m`; unknown arches log and proceed (can't verify != known-bad).
570 let deployed_bin = format!("{release_dir}/{primary_bin}");
571 run_checked(
572 executor,
573 &arch_guard_script(&deployed_bin),
574 "verifying binary arch matches node",
575 )
576 .await
577 .context("deployed binary architecture does not match the target node")
578 .context(FailureStage::BeforeSwap)?;
579
580 // And that the node can actually resolve what the binary links, which the
581 // arch check above cannot see: right architecture, right ELF, and still
582 // unrunnable because it wants a glibc symbol version this box does not have.
583 // Bento used to catch that at build time; under the Sando/Bento boundary the
584 // builder no longer knows which machine runs the bytes, so it lands here.
585 run_checked(
586 executor,
587 &ldd_guard_script(&deployed_bin),
588 "verifying the node can resolve the binary's dynamic dependencies",
589 )
590 .await
591 .context("the target node cannot satisfy the deployed binary's dynamic dependencies")
592 .context(FailureStage::BeforeSwap)?;
593
594 // The same two guards for every companion, and for the same reason. Both
595 // checks above take the primary binary alone, so a companion of the wrong
596 // architecture, or one linking a symbol version this node lacks, used to be
597 // discovered by its unit failing to start — during the install loop below,
598 // which runs AFTER the swap. That is the expensive side of the line these
599 // guards exist to stay on: the promote fails either way, but with the server
600 // already restarted onto the new release.
601 //
602 // The companion bytes are present and verified by now: they arrived in the
603 // same rsync and `manifest_verify_script` above covers the whole release
604 // directory, `companions/` included. So there is nothing to wait for.
605 for c in &node.companions {
606 let src = companion_src(&release_dir, &c.name);
607 run_checked(
608 executor,
609 &arch_guard_script(&src),
610 "verifying companion arch matches node",
611 )
612 .await
613 .with_context(|| {
614 format!(
615 "companion {} architecture does not match the target node",
616 c.name
617 )
618 })
619 .context(FailureStage::BeforeSwap)?;
620 run_checked(
621 executor,
622 &ldd_guard_script(&src),
623 "verifying the node can resolve the companion's dynamic dependencies",
624 )
625 .await
626 .with_context(|| {
627 format!(
628 "the target node cannot satisfy companion {}'s dynamic dependencies",
629 c.name
630 )
631 })
632 .context(FailureStage::BeforeSwap)?;
633 }
634
635 // Config-drift guard (opt-in per node). Runs the freshly-rsynced binary in
636 // config-only mode with the node's env sourced, BEFORE the swap, so a
637 // required var missing on this node fails here — service still intact —
638 // rather than after the restart, which would crash-loop it (how testnot
639 // went down on a missing CDN_BASE_URL). Skipped unless the node sets
640 // `config_check_env_file`.
641 if let Some(env_file) = node.config_check_env_file.as_deref() {
642 tracing::info!(node = %node.name, version, "deploy: pre-swap config check");
643 check_target_config(executor, &deployed_bin, env_file)
644 .await
645 .context("pre-swap config check failed")
646 .context(FailureStage::BeforeSwap)?;
647 }
648
649 tracing::info!(node = %node.name, version, "deploy: symlink swap + service reload");
650 let restart_cmd = format!(
651 "sudo /bin/systemctl reload-or-restart {}",
652 sh_quote(service)
653 );
654 let swap_and_restart = swap_and_restart_script(release_root, release_id, &restart_cmd);
655 run_checked(
656 executor,
657 &swap_and_restart,
658 "symlink swap + systemctl reload-or-restart",
659 )
660 .await
661 .context(FailureStage::AtOrAfterSwap)?;
662
663 // Companion services (opt-in per node): install each from the just-rsynced
664 // bundle and restart its unit via the node-side wrapper, AFTER the server is
665 // up (mnw-cli is `After=makenotwork.service`). They shipped from the SAME sha
666 // in this SAME bundle — the lockstep guarantee. A failure here fails the
667 // promote: a companion is part of the deploy, not a best-effort side effect.
668 for c in &node.companions {
669 let src = companion_src(&release_dir, &c.name);
670 tracing::info!(node = %node.name, companion = %c.name, "deploy: install companion + restart");
671 let cmd = install_companion_cmd(&src, &c.install_path, &c.service_name);
672 run_checked(executor, &cmd, "install companion + restart")
673 .await
674 .with_context(|| {
675 format!(
676 "companion {} deploy failed (server already swapped)",
677 c.name
678 )
679 })
680 .context(FailureStage::AtOrAfterSwap)?;
681 }
682
683 // No pinned set means the caller could not determine what is referenced,
684 // and a gc that cannot tell is the exact failure this parameter exists to
685 // stop. Skipping costs disk on the node; running blind costs the artifact a
686 // rollback resolves to. `finalize_local_release` takes the same position by
687 // refusing to publish at all when the pin query fails.
688 match pinned {
689 Some(pinned) => {
690 if let Err(e) = gc_remote_releases(executor, release_root, pinned).await {
691 tracing::warn!(error = %e, "remote release GC failed (non-fatal)");
692 }
693 }
694 None => tracing::warn!(
695 node = %node.name,
696 "remote release GC skipped: the pinned set is unknown, and a gc that \
697 cannot see what is referenced is what stranded the host store twice"
698 ),
699 }
700
701 Ok(PathBuf::from(release_root)
702 .join("releases")
703 .join(release_id))
704 }
705
706 /// Where a companion's binary sits inside the staged release directory.
707 ///
708 /// One function because two places need it and they must not drift: the guards
709 /// that run before the swap check this path, and the installer after the swap
710 /// reads it. A guard that checked a path the installer did not use would be a
711 /// check of nothing, and would look exactly like a passing check.
712 fn companion_src(release_dir: &str, name: &str) -> String {
713 format!("{release_dir}/companions/{name}")
714 }
715
716 /// Absolute path of the node-side companion installer (shipped once per node;
717 /// granted to the deploy user by a single scoped sudoers line). It installs the
718 /// staged binary to its `ExecStart` path and restarts the unit — keeping the
719 /// sudo grant to one script rather than a broad `install`/`systemctl` grant.
720 const COMPANION_INSTALLER: &str = "/usr/local/lib/mnw/install-companion.sh";
721
722 /// Command run on the node to install a staged companion binary and restart its
723 /// unit, via the wrapper. Pure builder so it can be unit-tested; all three args
724 /// are shell-quoted (paths/unit names, operator config — but quoted regardless).
725 fn install_companion_cmd(src: &str, install_path: &str, service: &str) -> String {
726 format!(
727 "sudo {installer} {src} {dst} {svc}",
728 installer = sh_quote(COMPANION_INSTALLER),
729 src = sh_quote(src),
730 dst = sh_quote(install_path),
731 svc = sh_quote(service),
732 )
733 }
734
735 /// Pre-swap config-drift check: load the node's env file the way systemd loads
736 /// it, then run the freshly-deployed binary in `MNW_CHECK_CONFIG=1` mode (loads
737 /// config, exits 0/1, no DB/migrations/bind). A non-zero exit — a required var
738 /// missing — is surfaced by `run_checked` as an error, failing the promote
739 /// before the swap.
740 ///
741 /// Bounded by a timeout as a backstop: a binary predating `MNW_CHECK_CONFIG`
742 /// would ignore the var and try to start normally, which must not hang the
743 /// deploy. A timeout is reported as a failure (fail closed) — the operator only
744 /// opts a node in once a check-capable version is deployed, so a timeout means
745 /// something is wrong, not a routine older binary.
746 async fn check_target_config(
747 executor: &dyn Executor,
748 deployed_bin: &str,
749 env_file: &str,
750 ) -> Result<()> {
751 // Readability first, as its own step with its own message.
752 //
753 // The env file is read by this check AS THE DEPLOY USER, and it is the only
754 // thing that does. systemd loads `EnvironmentFile=` as root before dropping
755 // to `User=`, so the running service does not care about the mode — which
756 // means a file rewritten 0600 breaks the next deploy while the current one
757 // keeps serving, and the breakage is invisible until someone ships. That is
758 // exactly how prod deploy 0.11.3 failed on 2026-08-01.
759 //
760 // Without this step the operator gets `bash: line 9: <file>: Permission
761 // denied` out of a generated script and has to reverse-engineer which user
762 // and which file. Naming the user, the mode and the owner turns that into a
763 // one-line read.
764 let probe = readability_probe_script(env_file);
765 if let Ok(Err(e)) = tokio::time::timeout(
766 std::time::Duration::from_secs(20),
767 run_checked(executor, &probe, "env file readability"),
768 )
769 .await
770 {
771 return Err(e).context(format!(
772 "the deploy user cannot read {env_file}. systemd reads EnvironmentFile= as root, so \
773 the running service is unaffected and this breaks only deploys. Expected mode 0640 \
774 owned root:<service user> (see sando/deploy/bootstrap-node.sh); something that \
775 rewrote the file likely did so with a 077 umask"
776 ));
777 }
778
779 let script = config_check_script(env_file, deployed_bin);
780 let fut = run_checked(executor, &script, "pre-swap config check");
781 match tokio::time::timeout(std::time::Duration::from_secs(20), fut).await {
782 Ok(result) => result.map(|_| ()),
783 Err(_) => anyhow::bail!(
784 "pre-swap config check timed out after 20s — the binary may predate \
785 MNW_CHECK_CONFIG or the check hung; refusing to swap"
786 ),
787 }
788 }
789
790 /// Assert the deploy user can read `env_file`, reporting who it is and what the
791 /// file actually looks like when it cannot.
792 ///
793 /// `stat` output is best-effort: a node without it (or a file that does not
794 /// exist) still gets the identity line, which is the half an operator cannot
795 /// derive from the failure on their own.
796 fn readability_probe_script(env_file: &str) -> String {
797 format!(
798 "if [ ! -e {env} ]; then\n\
799 \techo \"{env_disp}: does not exist on this node\" >&2; exit 1\n\
800 fi\n\
801 if [ ! -r {env} ]; then\n\
802 \techo \"cannot read {env_disp} as $(id -un) (groups: $(id -Gn))\" >&2\n\
803 \tstat -c 'actual: mode %a owner %U:%G' {env} >&2 2>/dev/null || true\n\
804 \texit 1\n\
805 fi\n",
806 env = sh_quote(env_file),
807 env_disp = env_file,
808 )
809 }
810
811 /// Shell that loads `env_file` with systemd `EnvironmentFile=` semantics, then
812 /// runs `bin` under `MNW_CHECK_CONFIG=1`.
813 ///
814 /// Load the file line by line and `export` each `KEY=VALUE` verbatim rather than
815 /// `. env_file`. Dot-sourcing runs the file as a script, so any shell
816 /// metacharacter in a value (`$`, backticks, `;`, `&`, a glob, whitespace) is
817 /// expanded or word-split — a DB URL carrying a password silently dropped
818 /// `DATABASE_URL` to empty on our nodes, which would fail the check (and thus
819 /// every deploy) even though systemd starts the service fine. `export "$line"`
820 /// assigns the already-expanded word literally, matching systemd's "no variable
821 /// expansion" rule. Comments and blank lines are skipped; the `|| [ -n "$line" ]`
822 /// guard processes a final line with no trailing newline. (Quoted values —
823 /// `KEY="v"` — aren't unquoted here the way systemd would, but our env files use
824 /// bare `KEY=VALUE`, and a stray quote can only make the check stricter, never
825 /// wave a bad config through.)
826 fn config_check_script(env_file: &str, bin: &str) -> String {
827 format!(
828 "set -eu\n\
829 while IFS= read -r __sando_l || [ -n \"$__sando_l\" ]; do\n\
830 \tcase \"$__sando_l\" in ''|'#'*) continue ;; esac\n\
831 \texport \"$__sando_l\"\n\
832 done < {env}\n\
833 MNW_CHECK_CONFIG=1 {bin}\n",
834 env = sh_quote(env_file),
835 bin = sh_quote(bin),
836 )
837 }
838
839 /// Build the swap-and-restart shell script for a remote node.
840 ///
841 /// The symlink swap is atomic via `mv -T` of a freshly-created symlink over the
842 /// old one (the rename(2) is the atomic step; `ln -sfn` alone does
843 /// unlink+symlink, which has a window). The load-bearing part: if `restart_cmd`
844 /// fails *after* the flip, `current` is rolled back to its prior target before
845 /// the script exits non-zero. Otherwise a failed restart would leave `current`
846 /// pointing at the new, un-activated release while the service still runs the
847 /// old one — and a later reboot/cron restart would then silently bring up the
848 /// release the deploy reported as failed. Best-effort re-restart of the prior
849 /// version keeps the running service consistent with the restored symlink.
850 ///
851 /// `restart_cmd` is injected (rather than hardcoded) so tests can drive the
852 /// failure and success paths with a `false`/`true` stand-in.
853 fn swap_and_restart_script(release_root: &str, release_id: &str, restart_cmd: &str) -> String {
854 format!(
855 "set -e\n\
856 cd {root}\n\
857 prev=$(readlink current 2>/dev/null || true)\n\
858 ln -sfn releases/{rel} current.new\n\
859 mv -Tf current.new current\n\
860 if ! {restart}; then\n\
861 if [ -n \"$prev\" ]; then\n\
862 ln -sfn \"$prev\" current.rollback\n\
863 mv -Tf current.rollback current\n\
864 {restart} || true\n\
865 fi\n\
866 echo \"deploy: restart failed; rolled symlink back to ${{prev:-<none>}}\" >&2\n\
867 exit 1\n\
868 fi\n",
869 root = sh_quote(release_root),
870 rel = sh_quote(release_id),
871 restart = restart_cmd,
872 )
873 }
874
875 /// Shell that re-hashes the rsynced bundle on the node against its shipped
876 /// `MANIFEST` and aborts (exit 1) if any file drifted (invariant 3, wiki note
877 /// `release-artifact-identity`). The `MANIFEST` is `sha256sum` check format,
878 /// so `sha256sum -c` verifies every listed file with node-native tooling and
879 /// names the one that failed. `--strict` fails on a malformed manifest line;
880 /// `--quiet` drops the per-file OK spam and keeps only failures.
881 ///
882 /// A bundle with no `MANIFEST` is not an error: the script logs a skip and exits
883 /// 0, so a deploy of such an artifact still ships.
884 fn manifest_verify_script(release_dir: &str) -> String {
885 format!(
886 "set -e\n\
887 cd {dir}\n\
888 if [ ! -f MANIFEST ]; then\n\
889 echo \"deploy: no MANIFEST in bundle; skipping digest verification (legacy artifact)\" >&2\n\
890 exit 0\n\
891 fi\n\
892 sha256sum --quiet --strict -c MANIFEST\n",
893 dir = sh_quote(release_dir),
894 )
895 }
896
897 /// Shell that aborts (exit 1) if `bin`'s ELF architecture doesn't match the
898 /// node it's running on. Reads the ELF `e_machine` field (2 bytes LE at offset
899 /// 18) and compares it to the value implied by `uname -m`. An arch we don't have
900 /// a mapping for logs and proceeds — the guard exists to catch the concrete
901 /// x86_64-vs-aarch64 confusion, not to gate genuinely-new targets.
902 fn arch_guard_script(bin: &str) -> String {
903 format!(
904 "set -e\n\
905 bin={bin}\n\
906 arch=$(uname -m)\n\
907 machine=$(od -An -tx1 -j18 -N2 \"$bin\" 2>/dev/null | tr -d ' \\n')\n\
908 case \"$arch\" in\n\
909 x86_64|amd64) want=3e00 ;;\n\
910 aarch64|arm64) want=b700 ;;\n\
911 *) echo \"deploy: arch check skipped (unmapped node arch $arch)\" >&2; want= ;;\n\
912 esac\n\
913 if [ -n \"$want\" ] && [ \"$machine\" != \"$want\" ]; then\n\
914 echo \"deploy: arch mismatch — node $arch expects e_machine $want but binary has ${{machine:-<unreadable>}}\" >&2\n\
915 exit 1\n\
916 fi\n",
917 bin = sh_quote(bin),
918 )
919 }
920
921 /// Refuse a binary whose dynamic dependencies the node cannot satisfy, before
922 /// the symlink swap.
923 ///
924 /// The sibling of [`arch_guard_script`]. Which machine runs the bytes is
925 /// environment knowledge, which is Sando's half, so the check belongs here,
926 /// where the node and the artifact are already in the same value.
927 ///
928 /// It runs the node's own loader against the bytes just rsynced onto it, which
929 /// covers every shared library and every symbol version rather than glibc alone,
930 /// and answers "will this exec here" rather than "is this version number smaller
931 /// than that one".
932 ///
933 /// Three outcomes, and only one of them fails:
934 ///
935 /// - `not found` in `ldd` output — a missing library or an unsatisfiable symbol
936 /// version. This is the failure, and it is exactly what would otherwise be
937 /// discovered by the unit failing to start after the swap.
938 /// - not a dynamic executable — `ldd` exits non-zero and says so. A static
939 /// binary has nothing to resolve, so it passes.
940 /// - no `ldd` on the node — nothing to check with. Logged and passed: "cannot
941 /// verify" is not "known bad", the same call `arch_guard_script` makes for an
942 /// unmapped arch.
943 ///
944 /// `ldd` runs the loader, which for an arbitrary binary is code execution. These
945 /// bytes are ours, already verified against their MANIFEST on this node, and
946 /// about to be exec'd by the service unit a second later.
947 fn ldd_guard_script(bin: &str) -> String {
948 format!(
949 "set -e\n\
950 bin={bin}\n\
951 command -v ldd >/dev/null 2>&1 || {{ echo \"deploy: ldd check skipped (no ldd on node)\" >&2; exit 0; }}\n\
952 out=$(ldd \"$bin\" 2>&1) || {{ \n\
953 case \"$out\" in\n\
954 *\"not a dynamic executable\"*) echo \"deploy: ldd check passed (static binary)\" >&2; exit 0 ;;\n\
955 *) echo \"deploy: ldd failed on $bin: $out\" >&2; exit 1 ;;\n\
956 esac\n\
957 }}\n\
958 if printf '%s' \"$out\" | grep -q 'not found'; then\n\
959 echo \"deploy: this node cannot satisfy the binary's dynamic dependencies:\" >&2\n\
960 printf '%s\\n' \"$out\" | grep 'not found' >&2\n\
961 exit 1\n\
962 fi\n",
963 bin = sh_quote(bin),
964 )
965 }
966
967 /// Trim `releases/` to the pinned set plus the [`RELEASES_TO_KEEP`] newest of
968 /// what is left.
969 ///
970 /// Pinning is applied before the count, so a referenced artifact cannot be aged
971 /// out by rebuilds of a newer version — the failure that stranded production
972 /// twice. See [`crate::retention`].
973 async fn gc_local_releases(release_root: &Path, pinned: &PinnedReleases) -> Result<()> {
974 let releases = release_root.join("releases");
975 if !releases.exists() {
976 return Ok(());
977 }
978 let mut entries = Vec::new();
979 let mut rd = tokio::fs::read_dir(&releases).await?;
980 while let Some(entry) = rd.next_entry().await? {
981 if !entry.file_type().await?.is_dir() {
982 continue;
983 }
984 // Set aside before anything is ordered or counted: a pinned dir is not
985 // a candidate, so it can never occupy one of the count's slots either.
986 if entry
987 .file_name()
988 .to_str()
989 .is_some_and(|n| pinned.contains(n))
990 {
991 continue;
992 }
993 let meta = entry.metadata().await?;
994 entries.push((entry.path(), meta.modified()?));
995 }
996 entries.sort_by_key(|e| std::cmp::Reverse(e.1));
997 for (path, _) in entries.into_iter().skip(RELEASES_TO_KEEP) {
998 if let Err(e) = tokio::fs::remove_dir_all(&path).await {
999 tracing::warn!(path = %path.display(), error = %e, "gc: rm failed");
1000 } else {
1001 tracing::debug!(path = %path.display(), "gc: removed old release");
1002 }
1003 }
1004 Ok(())
1005 }
1006
1007 /// Trim a node's `releases/` to the pinned set plus the [`RELEASES_TO_KEEP`]
1008 /// newest of what is left.
1009 ///
1010 /// The node mirrors the host's directory name (see [`deploy_node`]), so the same
1011 /// [`PinnedReleases`] the host gc subtracts is the right set here — nothing has
1012 /// to be recomputed per node.
1013 ///
1014 /// Weaker instance of the host defect: every promote and rollback rsyncs from
1015 /// the host store, so a node directory evicted here is re-pushed rather than
1016 /// lost. What it costs is a full rsync of a bundle the node already had, at the
1017 /// worst moment — mid-rollback, with a tier already failing.
1018 ///
1019 /// Pinning is applied before the count, matching [`gc_local_releases`]: a pinned
1020 /// directory is not a candidate, so it cannot occupy one of the count's slots.
1021 async fn gc_remote_releases(
1022 executor: &dyn Executor,
1023 release_root: &str,
1024 pinned: &PinnedReleases,
1025 ) -> Result<()> {
1026 run_checked(
1027 executor,
1028 &remote_gc_script(release_root, pinned),
1029 "remote release gc",
1030 )
1031 .await
1032 .map(|_| ())
1033 }
1034
1035 /// The remote gc as shell.
1036 ///
1037 /// Split out so the script is testable without an executor: it is the half of
1038 /// this that can be wrong in a way `rm -rf` makes expensive.
1039 ///
1040 /// `ls -1t` orders by mtime desc. Pinned names are carried in the positional
1041 /// parameters rather than a here-doc piped through `grep -v`, because `grep`
1042 /// exits 1 when it selects no lines — which happens on both edges that matter
1043 /// (nothing pinned, or everything pinned) and would abort the script under
1044 /// `set -e` for the two cases that are perfectly normal. Comparing with `case`
1045 /// has no exit status to trip over, and matches whole names rather than
1046 /// substrings, which a `grep -F` without `-x` would not.
1047 fn remote_gc_script(release_root: &str, pinned: &PinnedReleases) -> String {
1048 let pins = pinned
1049 .sorted_names()
1050 .into_iter()
1051 .map(sh_quote)
1052 .collect::<Vec<_>>()
1053 .join(" ");
1054 // `set --` with no operands unsets the positional parameters, which is
1055 // exactly what a nothing-pinned gc wants: the `for` below then iterates zero
1056 // times and every directory is a candidate. Written as one branch because
1057 // `set -- ` with an empty expansion is the same statement.
1058 let set_pins = format!("set -- {pins}");
1059 format!(
1060 "set -e; cd {root}/releases 2>/dev/null || exit 0; \
1061 {set_pins}; \
1062 n=0; \
1063 ls -1t | while IFS= read -r d; do \
1064 for p in \"$@\"; do \
1065 if [ \"$d\" = \"$p\" ]; then continue 2; fi; \
1066 done; \
1067 n=$((n+1)); \
1068 if [ \"$n\" -le {keep} ]; then continue; fi; \
1069 rm -rf -- \"$d\"; \
1070 done",
1071 root = sh_quote(release_root),
1072 keep = RELEASES_TO_KEEP,
1073 )
1074 }
1075
1076 #[cfg(test)]
1077 mod tests {
1078 use super::*;
1079
1080 /// Nothing deployed, so nothing pinned: the tests that exercise the count
1081 /// alone pass this, and the ones that exercise pinning build their own set.
1082 fn no_pins() -> PinnedReleases {
1083 PinnedReleases::none()
1084 }
1085
1086 use crate::topology::NodeCompanion;
1087 use ops_exec::{CapabilitySet, LocalExec, SshExec};
1088 use std::os::unix::process::ExitStatusExt;
1089 use std::sync::{Arc, Mutex as StdMutex};
1090 use std::time::SystemTime;
1091
1092 // ---- placement ----
1093 //
1094 // The whole table, because the interesting cases are the two where one side
1095 // said nothing. Treating silence as agreement is how a wrong-architecture
1096 // deploy would get through, and it is the shape a "check it before you call"
1097 // guard tends to end up with.
1098
1099 fn node_on(platform: Option<&str>) -> Node {
1100 Node {
1101 name: crate::domain::NodeId::new("n1"),
1102 ssh_target: "deploy@n1".into(),
1103 release_root: "/opt/x".into(),
1104 platform: platform.map(|p| Platform::parse(p).unwrap()),
1105 base_image: None,
1106 libc: None,
1107 service_name: "x.service".into(),
1108 config_check_env_file: None,
1109 actuate: crate::topology::default_actuate(),
1110 observe: crate::topology::default_observe(),
1111 health_url: None,
1112 companions: Vec::new(),
1113 }
1114 }
1115
1116 /// A node that declares a glibc older than the bundle needs is refused
1117 /// before the rsync, and the message names both numbers so the operator
1118 /// knows which side to fix.
1119 #[tokio::test]
1120 async fn a_bundle_above_the_node_s_declared_glibc_is_refused_before_the_rsync() {
1121 let dir = tempfile::tempdir().unwrap();
1122 let exe = std::fs::read(std::env::current_exe().unwrap()).unwrap();
1123 std::fs::write(dir.path().join("bin"), &exe).unwrap();
1124 let Some(floor) = crate::elf::glibc_floor(&exe) else {
1125 return; // a static test binary states no floor; nothing to compare
1126 };
1127
1128 let mut node = node_on(None);
1129 node.libc = Some("2.0".into()); // older than anything real
1130 let err = check_bundle_fits_node(&node, dir.path())
1131 .await
1132 .expect_err("a bundle above the node's glibc must be refused");
1133 // `{:#}` walks the context chain: the outermost context is the
1134 // `FailureStage`, whose Display is the operator-facing "nothing moved"
1135 // line, and the cause below it is the reason.
1136 let msg = format!("{err:#}");
1137 assert!(
1138 msg.contains(&floor.to_string()) && msg.contains("2.0"),
1139 "the refusal must name both numbers: {msg}"
1140 );
1141 assert_eq!(
1142 stage_of(&err),
1143 Some(FailureStage::BeforeSwap),
1144 "refusing here must be recoverable: nothing has moved yet"
1145 );
1146 }
1147
1148 #[tokio::test]
1149 async fn a_bundle_within_the_node_s_declared_glibc_passes() {
1150 let dir = tempfile::tempdir().unwrap();
1151 let exe = std::fs::read(std::env::current_exe().unwrap()).unwrap();
1152 std::fs::write(dir.path().join("bin"), &exe).unwrap();
1153
1154 let mut node = node_on(None);
1155 node.libc = Some("99.0".into()); // newer than anything real
1156 check_bundle_fits_node(&node, dir.path())
1157 .await
1158 .expect("a bundle the node can load must pass");
1159 }
1160
1161 /// The three ways there is nothing to compare. All three pass, because
1162 /// "cannot verify" is not "known bad" — the same call `arch_guard_script`
1163 /// makes for an unmapped architecture.
1164 #[tokio::test]
1165 async fn nothing_to_compare_is_a_pass_not_a_refusal() {
1166 let dir = tempfile::tempdir().unwrap();
1167 let exe = std::fs::read(std::env::current_exe().unwrap()).unwrap();
1168 std::fs::write(dir.path().join("bin"), &exe).unwrap();
1169
1170 // 1. The node declares no libc.
1171 let node = node_on(None);
1172 check_bundle_fits_node(&node, dir.path()).await.unwrap();
1173
1174 // 2. The node's declared libc is not a version (a config typo).
1175 let mut typo = node_on(None);
1176 typo.libc = Some("noble".into());
1177 check_bundle_fits_node(&typo, dir.path()).await.unwrap();
1178
1179 // 3. The bundle holds no ELF, so it states no floor.
1180 let empty = tempfile::tempdir().unwrap();
1181 std::fs::write(empty.path().join("style.css"), b"body{}").unwrap();
1182 let mut strict = node_on(None);
1183 strict.libc = Some("2.0".into());
1184 check_bundle_fits_node(&strict, empty.path())
1185 .await
1186 .expect("a bundle with no binaries has no floor to exceed");
1187 }
1188
1189 #[test]
1190 fn matching_platforms_are_placeable() {
1191 let node = node_on(Some("linux/aarch64"));
1192 let art = Platform::parse("linux/aarch64").unwrap();
1193 let p = Placement::check(&node, Path::new("/r/abc"), Some(&art)).expect("a match places");
1194 assert_eq!(p.bundle(), Path::new("/r/abc"));
1195 assert_eq!(p.node().name.as_str(), "n1");
1196 }
1197
1198 #[test]
1199 fn a_different_architecture_is_refused() {
1200 // The failure this type exists for: pom's aarch64 bundle reaching the
1201 // x86_64 box, which execs nothing and takes the watcher down.
1202 let node = node_on(Some("linux/x86_64"));
1203 let art = Platform::parse("linux/aarch64").unwrap();
1204 let err = Placement::check(&node, Path::new("/r/abc"), Some(&art)).unwrap_err();
1205 assert!(
1206 matches!(err, PlacementError::Mismatch { .. }),
1207 "expected a mismatch, got {err}"
1208 );
1209 // The message has to name both, or an operator cannot tell which half
1210 // is wrong.
1211 let msg = err.to_string();
1212 assert!(
1213 msg.contains("linux/x86_64") && msg.contains("linux/aarch64"),
1214 "{msg}"
1215 );
1216 }
1217
1218 #[test]
1219 fn a_silent_node_refuses_a_stated_artifact() {
1220 // Not "the node probably runs it". A node that never said what it is
1221 // cannot vouch for a bundle that did, and the pairing that looks
1222 // harmless here is exactly the one that ships the wrong half of a
1223 // two-architecture release.
1224 let node = node_on(None);
1225 let art = Platform::parse("linux/aarch64").unwrap();
1226 assert!(matches!(
1227 Placement::check(&node, Path::new("/r/abc"), Some(&art)),
1228 Err(PlacementError::NodeSilent { .. })
1229 ));
1230 }
1231
1232 #[test]
1233 fn a_stated_node_refuses_a_silent_artifact() {
1234 let node = node_on(Some("linux/aarch64"));
1235 assert!(matches!(
1236 Placement::check(&node, Path::new("/r/abc"), None),
1237 Err(PlacementError::ArtifactSilent { .. })
1238 ));
1239 }
1240
1241 #[test]
1242 fn both_silent_is_the_single_platform_world_and_still_places() {
1243 // MNW is here and stays here. Its nodes declare nothing and its builds
1244 // record nothing, which is the truth about a product with one build host
1245 // and one architecture. The moment either side starts stating, the other
1246 // has to as well — that is the forcing function, and it is why this cell
1247 // is the only admissible non-match.
1248 let node = node_on(None);
1249 Placement::check(&node, Path::new("/r/abc"), None).expect("the pre-pom world still ships");
1250 }
1251
1252 #[test]
1253 fn platform_parsing_is_a_shape_not_a_spelling() {
1254 assert_eq!(
1255 Platform::parse("Linux/AArch64").unwrap(),
1256 Platform::parse("linux/aarch64").unwrap(),
1257 "case is not a distinction between two machines"
1258 );
1259 for bad in ["linux", "linux/", "/aarch64", "linux/aarch64/gnu", ""] {
1260 assert!(Platform::parse(bad).is_err(), "{bad:?} should not parse");
1261 }
1262 }
1263
1264 // ---- failure stage ----
1265 //
1266 // The 2026-08-01 prod deploy failed its pre-swap config check, and the
1267 // rollback then failed the same way — which left the node safely on the old
1268 // version, and was reported as "it remains on the new version, manual
1269 // intervention needed". These pin the distinction the reporting layer now
1270 // depends on.
1271
1272 #[test]
1273 fn a_pre_swap_failure_is_recoverable_as_such() {
1274 let e = anyhow::anyhow!("Permission denied")
1275 .context("pre-swap config check failed")
1276 .context(FailureStage::BeforeSwap);
1277 assert_eq!(stage_of(&e), Some(FailureStage::BeforeSwap));
1278 // The reason survives alongside the stage; the stage does not replace it.
1279 let rendered = format!("{e:#}");
1280 assert!(
1281 rendered.contains("pre-swap config check failed"),
1282 "{rendered}"
1283 );
1284 assert!(rendered.contains("Permission denied"), "{rendered}");
1285 }
1286
1287 #[test]
1288 fn a_post_swap_failure_is_recoverable_as_such() {
1289 let e = anyhow::anyhow!("unit failed to start")
1290 .context("companion x deploy failed (server already swapped)")
1291 .context(FailureStage::AtOrAfterSwap);
1292 assert_eq!(stage_of(&e), Some(FailureStage::AtOrAfterSwap));
1293 }
1294
1295 #[test]
1296 fn an_unannotated_failure_has_no_stage() {
1297 // Must be None, not a default. A caller seeing None has to treat the
1298 // node as indeterminate; inferring "before the swap" would reintroduce
1299 // the original bug pointing the other way, which is the dangerous way.
1300 let e = anyhow::anyhow!("something older, from before stages existed");
1301 assert_eq!(stage_of(&e), None);
1302 }
1303
1304 // ---- env file readability probe ----
1305
1306 #[tokio::test]
1307 async fn readability_probe_passes_on_a_readable_file() {
1308 let tmp = tempfile::tempdir().unwrap();
1309 let f = tmp.path().join("ok.env");
1310 tokio::fs::write(&f, "A=1\n").await.unwrap();
1311 let script = readability_probe_script(&f.to_string_lossy());
1312 let out = run_checked(&local_executor(), &script, "probe").await;
1313 assert!(out.is_ok(), "{:?}", out.err().map(|e| format!("{e:#}")));
1314 }
1315
1316 #[tokio::test]
1317 async fn readability_probe_names_the_user_and_mode_when_unreadable() {
1318 // Root can read anything, so a mode-based test would pass spuriously
1319 // there. Skip rather than assert something false. No libc dependency
1320 // for one probe: a 0-mode temp file is readable iff we are root.
1321 let probe_dir = tempfile::tempdir().unwrap();
1322 let probe_file = probe_dir.path().join("root-check");
1323 tokio::fs::write(&probe_file, "x").await.unwrap();
1324 tokio::fs::set_permissions(
1325 &probe_file,
1326 std::os::unix::fs::PermissionsExt::from_mode(0o000),
1327 )
1328 .await
1329 .unwrap();
1330 if tokio::fs::read(&probe_file).await.is_ok() {
1331 return; // running as root
1332 }
1333 let tmp = tempfile::tempdir().unwrap();
1334 let f = tmp.path().join("locked.env");
1335 tokio::fs::write(&f, "A=1\n").await.unwrap();
1336 tokio::fs::set_permissions(&f, std::os::unix::fs::PermissionsExt::from_mode(0o000))
1337 .await
1338 .unwrap();
1339
1340 let script = readability_probe_script(&f.to_string_lossy());
1341 let err = run_checked(&local_executor(), &script, "probe")
1342 .await
1343 .expect_err("an unreadable file must fail the probe");
1344 let msg = format!("{err:#}");
1345 // The two things the raw bash error does not tell you.
1346 assert!(msg.contains("cannot read"), "{msg}");
1347 assert!(msg.contains("mode 0") || msg.contains("mode "), "{msg}");
1348 }
1349
1350 #[tokio::test]
1351 async fn readability_probe_distinguishes_missing_from_unreadable() {
1352 let tmp = tempfile::tempdir().unwrap();
1353 let missing = tmp.path().join("nope.env");
1354 let script = readability_probe_script(&missing.to_string_lossy());
1355 let err = run_checked(&local_executor(), &script, "probe")
1356 .await
1357 .expect_err("a missing file must fail the probe");
1358 let msg = format!("{err:#}");
1359 assert!(msg.contains("does not exist"), "{msg}");
1360 }
1361
1362 #[test]
1363 fn the_two_stages_read_differently() {
1364 // These strings end up in an operator's terminal during an incident.
1365 let before = FailureStage::BeforeSwap.to_string();
1366 let after = FailureStage::AtOrAfterSwap.to_string();
1367 assert!(before.contains("previous version"), "{before}");
1368 assert!(after.contains("indeterminate"), "{after}");
1369 assert_ne!(before, after);
1370 }
1371
1372 /// A LocalExec granted the default node capabilities (deploy + restart).
1373 fn local_executor() -> LocalExec {
1374 LocalExec::new(CapabilitySet::from_tokens(
1375 ["deploy", "restart"],
1376 ["health"],
1377 ))
1378 }
1379
1380 #[tokio::test]
1381 async fn deploy_local_copies_multiple_binaries_and_swaps_symlink() {
1382 let tmp = tempfile::tempdir().unwrap();
1383 let root = tmp.path();
1384
1385 let src_dir = root.join("src");
1386 tokio::fs::create_dir_all(&src_dir).await.unwrap();
1387 let primary = src_dir.join("makenotwork");
1388 let admin = src_dir.join("mnw-admin");
1389 tokio::fs::write(&primary, b"PRIMARY").await.unwrap();
1390 tokio::fs::write(&admin, b"ADMIN").await.unwrap();
1391
1392 let release_root = root.join("releases-root");
1393 tokio::fs::create_dir_all(&release_root).await.unwrap();
1394
1395 // Stage into staging/<build_id> (no publish yet).
1396 let staging = stage_local_bundle(&release_root, 42, &[primary.clone(), admin.clone()])
1397 .await
1398 .expect("stage_local_bundle should succeed");
1399 assert_eq!(staging, release_root.join("staging").join("42"));
1400 assert!(
1401 !release_root.join("current").exists(),
1402 "staging must not publish or flip current"
1403 );
1404
1405 // Publish content-addressed at releases/<digest16>.
1406 let released =
1407 finalize_local_release(&release_root, &staging, "deadbeefcafe0000", &no_pins())
1408 .await
1409 .expect("finalize_local_release should succeed");
1410 assert_eq!(
1411 released,
1412 release_root.join("releases").join("deadbeefcafe0000")
1413 );
1414 assert!(
1415 !staging.exists(),
1416 "staging dir is consumed by the publish rename"
1417 );
1418 assert_eq!(
1419 tokio::fs::read(released.join("makenotwork")).await.unwrap(),
1420 b"PRIMARY"
1421 );
1422 assert_eq!(
1423 tokio::fs::read(released.join("mnw-admin")).await.unwrap(),
1424 b"ADMIN"
1425 );
1426
1427 let current = release_root.join("current");
1428 let target = tokio::fs::read_link(&current).await.unwrap();
1429 assert_eq!(target.to_string_lossy(), "releases/deadbeefcafe0000");
1430 let via_current = tokio::fs::read(current.join("makenotwork")).await.unwrap();
1431 assert_eq!(via_current, b"PRIMARY");
1432 }
1433
1434 #[tokio::test]
1435 async fn finalize_second_release_swaps_symlink_and_keeps_old_dir() {
1436 let tmp = tempfile::tempdir().unwrap();
1437 let root = tmp.path();
1438 let src_dir = root.join("src");
1439 tokio::fs::create_dir_all(&src_dir).await.unwrap();
1440 let bin = src_dir.join("server");
1441 tokio::fs::write(&bin, b"V1").await.unwrap();
1442
1443 let release_root = root.join("rr");
1444 tokio::fs::create_dir_all(&release_root).await.unwrap();
1445
1446 // Two builds, distinct digests (distinct content) -> two release dirs.
1447 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1448 .await
1449 .unwrap();
1450 finalize_local_release(&release_root, &s1, "1111111111111111", &no_pins())
1451 .await
1452 .unwrap();
1453 tokio::fs::write(&bin, b"V2").await.unwrap();
1454 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1455 .await
1456 .unwrap();
1457 finalize_local_release(&release_root, &s2, "2222222222222222", &no_pins())
1458 .await
1459 .unwrap();
1460
1461 assert!(
1462 release_root
1463 .join("releases/1111111111111111/server")
1464 .exists()
1465 );
1466 assert!(
1467 release_root
1468 .join("releases/2222222222222222/server")
1469 .exists()
1470 );
1471 let target = tokio::fs::read_link(release_root.join("current"))
1472 .await
1473 .unwrap();
1474 assert_eq!(target.to_string_lossy(), "releases/2222222222222222");
1475 let via_current = tokio::fs::read(release_root.join("current/server"))
1476 .await
1477 .unwrap();
1478 assert_eq!(via_current, b"V2");
1479 }
1480
1481 #[tokio::test]
1482 async fn finalize_reuses_an_existing_release_of_the_same_digest() {
1483 let tmp = tempfile::tempdir().unwrap();
1484 let root = tmp.path();
1485 let bin = root.join("server");
1486 tokio::fs::write(&bin, b"BYTES").await.unwrap();
1487 let release_root = root.join("rr");
1488 tokio::fs::create_dir_all(&release_root).await.unwrap();
1489
1490 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1491 .await
1492 .unwrap();
1493 finalize_local_release(&release_root, &s1, "abc123abc123abc1", &no_pins())
1494 .await
1495 .unwrap();
1496 // Same digest rebuilt (e.g. a re-run at the same content): finalize must
1497 // reuse the existing release and drop the redundant staging dir, not error.
1498 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1499 .await
1500 .unwrap();
1501 let released = finalize_local_release(&release_root, &s2, "abc123abc123abc1", &no_pins())
1502 .await
1503 .expect("finalize is idempotent on a repeated digest");
1504 assert_eq!(released, release_root.join("releases/abc123abc123abc1"));
1505 assert!(!s2.exists(), "redundant staging dropped");
1506 }
1507
1508 #[tokio::test]
1509 async fn manifest_verify_script_passes_on_match_fails_on_drift_and_skips_when_absent() {
1510 // The node-side verification is a shell running `sha256sum -c MANIFEST`;
1511 // drive the real script through bash to prove it accepts a good bundle,
1512 // rejects a tampered one, and no-ops on a legacy (MANIFEST-less) bundle.
1513 let dir = tempfile::tempdir().unwrap();
1514 tokio::fs::write(dir.path().join("server"), b"BINARY")
1515 .await
1516 .unwrap();
1517 tokio::fs::create_dir(dir.path().join("static"))
1518 .await
1519 .unwrap();
1520 tokio::fs::write(dir.path().join("static/app.css"), b"body{}")
1521 .await
1522 .unwrap();
1523 let digest = crate::bundle::digest_dir(dir.path()).await.unwrap();
1524 tokio::fs::write(dir.path().join("MANIFEST"), digest.manifest.as_bytes())
1525 .await
1526 .unwrap();
1527
1528 let run = |d: &std::path::Path| {
1529 let script = manifest_verify_script(d.to_str().unwrap());
1530 async move {
1531 Command::new("bash")
1532 .arg("-c")
1533 .arg(&script)
1534 .output()
1535 .await
1536 .unwrap()
1537 }
1538 };
1539
1540 let ok = run(dir.path()).await;
1541 assert!(
1542 ok.status.success(),
1543 "matching bundle verifies: {}",
1544 String::from_utf8_lossy(&ok.stderr)
1545 );
1546
1547 // Drift one file: sha256sum -c must fail (current symlink left intact).
1548 tokio::fs::write(dir.path().join("static/app.css"), b"TAMPERED")
1549 .await
1550 .unwrap();
1551 let bad = run(dir.path()).await;
1552 assert!(!bad.status.success(), "a drifted file fails verification");
1553
1554 // Legacy bundle with no MANIFEST: skip, not fail.
1555 let legacy = tempfile::tempdir().unwrap();
1556 tokio::fs::write(legacy.path().join("server"), b"x")
1557 .await
1558 .unwrap();
1559 let skip = run(legacy.path()).await;
1560 assert!(
1561 skip.status.success(),
1562 "a bundle without a MANIFEST skips verification rather than failing"
1563 );
1564 }
1565
1566 #[tokio::test]
1567 async fn gc_local_releases_keeps_last_n_by_mtime() {
1568 let tmp = tempfile::tempdir().unwrap();
1569 let root = tmp.path();
1570 let releases = root.join("releases");
1571 tokio::fs::create_dir_all(&releases).await.unwrap();
1572
1573 let total = RELEASES_TO_KEEP + 3;
1574 let mut names = Vec::new();
1575 for i in 0..total {
1576 let name = format!("v{i:02}");
1577 let dir = releases.join(&name);
1578 tokio::fs::create_dir(&dir).await.unwrap();
1579 let f = std::fs::File::open(&dir).unwrap();
1580 let when =
1581 SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
1582 let times = std::fs::FileTimes::new().set_modified(when);
1583 f.set_times(times).unwrap();
1584 names.push(name);
1585 }
1586
1587 gc_local_releases(root, &no_pins()).await.unwrap();
1588
1589 let surviving_expected: Vec<_> = names
1590 .iter()
1591 .skip(total - RELEASES_TO_KEEP)
1592 .cloned()
1593 .collect();
1594 for name in &surviving_expected {
1595 assert!(releases.join(name).exists(), "expected to survive: {name}");
1596 }
1597 for name in names.iter().take(total - RELEASES_TO_KEEP) {
1598 assert!(
1599 !releases.join(name).exists(),
1600 "expected to be pruned: {name}"
1601 );
1602 }
1603 }
1604
1605 #[tokio::test]
1606 async fn gc_local_releases_never_evicts_a_pinned_dir() {
1607 // The 2026-08-25 shape exactly: the oldest dir is the one production is
1608 // running, and enough newer rebuilds exist to push it past the count.
1609 // Under the count alone it was the first thing deleted.
1610 let tmp = tempfile::tempdir().unwrap();
1611 let root = tmp.path();
1612 let releases = root.join("releases");
1613 tokio::fs::create_dir_all(&releases).await.unwrap();
1614
1615 let total = RELEASES_TO_KEEP + 3;
1616 let mut names = Vec::new();
1617 for i in 0..total {
1618 let name = format!("v{i:02}");
1619 let dir = releases.join(&name);
1620 tokio::fs::create_dir(&dir).await.unwrap();
1621 let f = std::fs::File::open(&dir).unwrap();
1622 let when =
1623 SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
1624 f.set_times(std::fs::FileTimes::new().set_modified(when))
1625 .unwrap();
1626 names.push(name);
1627 }
1628
1629 // The two oldest: what a tier is running and what it would roll back to.
1630 let pinned: PinnedReleases = [names[0].clone(), names[1].clone()].into_iter().collect();
1631 gc_local_releases(root, &pinned).await.unwrap();
1632
1633 for name in [&names[0], &names[1]] {
1634 assert!(
1635 releases.join(name).exists(),
1636 "a referenced artifact was evicted: {name}"
1637 );
1638 }
1639 // And the count still applies to everything else, from a floor that the
1640 // pinned pair did not eat into: the newest RELEASES_TO_KEEP unpinned
1641 // dirs survive, so pinning two costs two extra slots rather than two of
1642 // the five.
1643 let unpinned: Vec<_> = names.iter().filter(|n| !pinned.contains(n)).collect();
1644 let cut = unpinned.len() - RELEASES_TO_KEEP;
1645 for name in unpinned.iter().take(cut) {
1646 assert!(
1647 !releases.join(name).exists(),
1648 "expected to be pruned: {name}"
1649 );
1650 }
1651 for name in unpinned.iter().skip(cut) {
1652 assert!(releases.join(name).exists(), "expected to survive: {name}");
1653 }
1654 }
1655
1656 #[tokio::test]
1657 async fn gc_local_releases_keeps_a_pinned_dir_that_is_not_even_present() {
1658 // A pinned name with nothing on disk must not disturb the count. This is
1659 // the state the bug leaves behind, and gc runs again while it holds.
1660 let tmp = tempfile::tempdir().unwrap();
1661 let root = tmp.path();
1662 let releases = root.join("releases");
1663 tokio::fs::create_dir_all(&releases).await.unwrap();
1664 for i in 0..=RELEASES_TO_KEEP {
1665 tokio::fs::create_dir(releases.join(format!("v{i}")))
1666 .await
1667 .unwrap();
1668 }
1669 let pinned: PinnedReleases = ["gone-already".to_string()].into_iter().collect();
1670 gc_local_releases(root, &pinned).await.unwrap();
1671
1672 let left = std::fs::read_dir(&releases).unwrap().count();
1673 assert_eq!(left, RELEASES_TO_KEEP);
1674 }
1675
1676 #[tokio::test]
1677 async fn gc_local_releases_noop_when_below_threshold() {
1678 let tmp = tempfile::tempdir().unwrap();
1679 let root = tmp.path();
1680 let releases = root.join("releases");
1681 tokio::fs::create_dir_all(&releases).await.unwrap();
1682 for i in 0..3 {
1683 tokio::fs::create_dir(releases.join(format!("v{i}")))
1684 .await
1685 .unwrap();
1686 }
1687 gc_local_releases(root, &no_pins()).await.unwrap();
1688 for i in 0..3 {
1689 assert!(releases.join(format!("v{i}")).exists());
1690 }
1691 }
1692
1693 // ---- remote gc ----
1694 //
1695 // Driven through `LocalExec`, so these run the real shell the node runs
1696 // rather than asserting on the script's text. The script is the half of the
1697 // remote gc that can be wrong, and it is wrong with `rm -rf`.
1698
1699 /// `releases/` with `total` dirs named `v00..`, oldest first by mtime.
1700 async fn releases_by_age(root: &Path, total: usize) -> Vec<String> {
1701 let releases = root.join("releases");
1702 tokio::fs::create_dir_all(&releases).await.unwrap();
1703 let mut names = Vec::new();
1704 for i in 0..total {
1705 let name = format!("v{i:02}");
1706 let dir = releases.join(&name);
1707 tokio::fs::create_dir(&dir).await.unwrap();
1708 // A file inside, so a deletion is visible as more than an empty dir.
1709 tokio::fs::write(dir.join("makenotwork"), b"x")
1710 .await
1711 .unwrap();
1712 let f = std::fs::File::open(&dir).unwrap();
1713 let when =
1714 SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
1715 f.set_times(std::fs::FileTimes::new().set_modified(when))
1716 .unwrap();
1717 names.push(name);
1718 }
1719 names
1720 }
1721
1722 /// Nothing pinned: the newest `RELEASES_TO_KEEP` survive.
1723 #[tokio::test]
1724 async fn gc_remote_releases_keeps_last_n_by_mtime_when_nothing_is_pinned() {
1725 let tmp = tempfile::tempdir().unwrap();
1726 let root = tmp.path();
1727 let total = RELEASES_TO_KEEP + 3;
1728 let names = releases_by_age(root, total).await;
1729
1730 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &no_pins())
1731 .await
1732 .unwrap();
1733
1734 let releases = root.join("releases");
1735 for name in names.iter().take(total - RELEASES_TO_KEEP) {
1736 assert!(!releases.join(name).exists(), "expected pruned: {name}");
1737 }
1738 for name in names.iter().skip(total - RELEASES_TO_KEEP) {
1739 assert!(releases.join(name).exists(), "expected to survive: {name}");
1740 }
1741 }
1742
1743 /// The done condition, on the node: the dirs a tier's current and previous
1744 /// artifacts name survive even when they are the oldest on disk and well
1745 /// past the count. Same shape as the host-store test above, which is the
1746 /// point — the two stores now answer the same question the same way.
1747 #[tokio::test]
1748 async fn gc_remote_releases_never_evicts_a_pinned_dir() {
1749 let tmp = tempfile::tempdir().unwrap();
1750 let root = tmp.path();
1751 let total = RELEASES_TO_KEEP + 3;
1752 let names = releases_by_age(root, total).await;
1753
1754 let pinned: PinnedReleases = [names[0].clone(), names[1].clone()].into_iter().collect();
1755 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
1756 .await
1757 .unwrap();
1758
1759 let releases = root.join("releases");
1760 for name in [&names[0], &names[1]] {
1761 assert!(
1762 releases.join(name).exists(),
1763 "a referenced artifact was evicted from the node: {name}"
1764 );
1765 }
1766 // And pinning does not spend the count's slots, again matching the host.
1767 let unpinned: Vec<_> = names.iter().filter(|n| !pinned.contains(n)).collect();
1768 let cut = unpinned.len() - RELEASES_TO_KEEP;
1769 for name in unpinned.iter().take(cut) {
1770 assert!(!releases.join(name).exists(), "expected pruned: {name}");
1771 }
1772 for name in unpinned.iter().skip(cut) {
1773 assert!(releases.join(name).exists(), "expected to survive: {name}");
1774 }
1775 }
1776
1777 /// Every dir pinned means the loop deletes nothing and the script still
1778 /// exits 0. Worth its own test because the obvious implementation of this
1779 /// filter is `grep -v`, which exits 1 when it selects no lines and would
1780 /// have failed the deploy here under `set -e`.
1781 #[tokio::test]
1782 async fn gc_remote_releases_succeeds_when_everything_is_pinned() {
1783 let tmp = tempfile::tempdir().unwrap();
1784 let root = tmp.path();
1785 let names = releases_by_age(root, RELEASES_TO_KEEP + 3).await;
1786 let pinned: PinnedReleases = names.iter().cloned().collect();
1787
1788 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
1789 .await
1790 .unwrap();
1791
1792 let releases = root.join("releases");
1793 for name in &names {
1794 assert!(releases.join(name).exists(), "expected to survive: {name}");
1795 }
1796 }
1797
1798 /// A `releases/` that does not exist is not an error: a node's first deploy
1799 /// creates the dir, and gc runs on the same path.
1800 #[tokio::test]
1801 async fn gc_remote_releases_is_a_noop_when_the_store_is_missing() {
1802 let tmp = tempfile::tempdir().unwrap();
1803 gc_remote_releases(&local_executor(), tmp.path().to_str().unwrap(), &no_pins())
1804 .await
1805 .unwrap();
1806 }
1807
1808 /// Names reach the script as positional parameters, so a name that looks
1809 /// like shell must be compared whole rather than expanded or split. None of
1810 /// these can be a digest16, but the pre-identity names are version strings
1811 /// and the pinned set is data read out of a database.
1812 #[tokio::test]
1813 async fn gc_remote_releases_quotes_pinned_names() {
1814 let tmp = tempfile::tempdir().unwrap();
1815 let root = tmp.path();
1816 let releases = root.join("releases");
1817 tokio::fs::create_dir_all(&releases).await.unwrap();
1818 let awkward = ["a b", "x'y", "*"];
1819 for name in awkward {
1820 tokio::fs::create_dir(releases.join(name)).await.unwrap();
1821 }
1822 // Enough newer dirs that the count alone would evict all three.
1823 let filler: Vec<String> = (0..=RELEASES_TO_KEEP).map(|i| format!("f{i}")).collect();
1824 for name in &filler {
1825 tokio::fs::create_dir(releases.join(name)).await.unwrap();
1826 }
1827
1828 let pinned: PinnedReleases = awkward.iter().map(|s| (*s).to_string()).collect();
1829 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
1830 .await
1831 .unwrap();
1832
1833 for name in awkward {
1834 assert!(releases.join(name).exists(), "expected to survive: {name}");
1835 }
1836 }
1837
1838 /// A pinned name matches a whole directory name, never a prefix of one.
1839 /// `case`-with-globbing or a `grep -F` without `-x` would keep `v0` and
1840 /// `v01` both because one contains the other, quietly widening the pinned
1841 /// set past what the database said.
1842 #[tokio::test]
1843 async fn gc_remote_releases_matches_whole_names_not_prefixes() {
1844 let tmp = tempfile::tempdir().unwrap();
1845 let root = tmp.path();
1846 let names = releases_by_age(root, RELEASES_TO_KEEP + 3).await;
1847
1848 // Pin the oldest by an exact name; its neighbours share the prefix.
1849 let pinned: PinnedReleases = [names[0].clone()].into_iter().collect();
1850 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
1851 .await
1852 .unwrap();
1853
1854 let releases = root.join("releases");
1855 assert!(
1856 releases.join(&names[0]).exists(),
1857 "the pinned dir was evicted"
1858 );
1859 assert!(
1860 !releases.join(&names[1]).exists(),
1861 "a dir sharing the pinned name's prefix was treated as pinned"
1862 );
1863 }
1864
1865 #[tokio::test]
1866 async fn gc_local_releases_noop_when_releases_dir_missing() {
1867 let tmp = tempfile::tempdir().unwrap();
1868 gc_local_releases(tmp.path(), &no_pins()).await.unwrap();
1869 }
1870
1871 #[tokio::test]
1872 async fn deploy_remote_fails_cleanly_when_host_unreachable() {
1873 // 192.0.2.0/24 is reserved for documentation and routes nowhere.
1874 // ConnectTimeout=10 limits the test wallclock to ~10s worst case.
1875 let tmp = tempfile::tempdir().unwrap();
1876 let staged = tmp.path().join("releases").join("0.0.1");
1877 tokio::fs::create_dir_all(&staged).await.unwrap();
1878 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
1879
1880 let node = crate::topology::Node {
1881 platform: None,
1882 base_image: None,
1883 libc: None,
1884 name: "unreachable".into(),
1885 ssh_target: "deploy@192.0.2.1".into(),
1886 release_root: "/opt/never".into(),
1887 service_name: "makenotwork.service".into(),
1888 health_url: None,
1889 config_check_env_file: None,
1890 actuate: crate::topology::default_actuate(),
1891 observe: crate::topology::default_observe(),
1892 companions: Vec::new(),
1893 };
1894 let executor = SshExec::new(
1895 node.ssh_target.clone(),
1896 CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1897 );
1898
1899 let placement = Placement::check(&node, &staged, None).expect("both sides silent");
1900 let result = deploy_node(&executor, placement, "0.0.1", "server", Some(&no_pins())).await;
1901 let err = result.expect_err("deploy to unreachable host should fail");
1902 let msg = format!("{err:#}");
1903 // Don't pin exact wording, just that the failure is attributed (ssh /
1904 // rsync / connection) and that no panic / hang happened.
1905 assert!(
1906 msg.contains("ssh")
1907 || msg.contains("rsync")
1908 || msg.contains("connection")
1909 || msg.contains("Connection"),
1910 "unexpected error: {msg}"
1911 );
1912 }
1913
1914 #[tokio::test]
1915 async fn deploy_node_with_local_ssh_target_swaps_symlink() {
1916 // ssh_target="local" routes to the local fast-path: just a symlink
1917 // swap, no remote calls.
1918 let tmp = tempfile::tempdir().unwrap();
1919 let release_root = tmp.path().to_path_buf();
1920 let staged = release_root.join("releases").join("0.0.1");
1921 tokio::fs::create_dir_all(&staged).await.unwrap();
1922 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
1923
1924 let node = crate::topology::Node {
1925 platform: None,
1926 base_image: None,
1927 libc: None,
1928 name: "local-dev".into(),
1929 ssh_target: "local".into(),
1930 release_root: release_root.to_string_lossy().into_owned(),
1931 service_name: "makenotwork.service".into(),
1932 health_url: None,
1933 config_check_env_file: None,
1934 actuate: crate::topology::default_actuate(),
1935 observe: crate::topology::default_observe(),
1936 companions: Vec::new(),
1937 };
1938 let executor = local_executor();
1939
1940 let out = deploy_node(
1941 &executor,
1942 Placement::check(&node, &staged, None).unwrap(),
1943 "0.0.1",
1944 "server",
1945 Some(&no_pins()),
1946 )
1947 .await
1948 .unwrap();
1949 assert_eq!(out, staged);
1950 let target = tokio::fs::read_link(release_root.join("current"))
1951 .await
1952 .unwrap();
1953 assert_eq!(target.to_string_lossy(), "releases/0.0.1");
1954 }
1955
1956 // ---- swap_and_restart_script: symlink/restart consistency ----
1957
1958 async fn run_script(script: &str) -> std::process::Output {
1959 Command::new("sh")
1960 .arg("-c")
1961 .arg(script)
1962 .output()
1963 .await
1964 .unwrap()
1965 }
1966
1967 async fn setup_release_root(with_current: bool) -> tempfile::TempDir {
1968 let tmp = tempfile::tempdir().unwrap();
1969 let root = tmp.path();
1970 tokio::fs::create_dir_all(root.join("releases/old"))
1971 .await
1972 .unwrap();
1973 tokio::fs::create_dir_all(root.join("releases/new"))
1974 .await
1975 .unwrap();
1976 if with_current {
1977 std::os::unix::fs::symlink("releases/old", root.join("current")).unwrap();
1978 }
1979 tmp
1980 }
1981
1982 #[tokio::test]
1983 async fn swap_and_restart_keeps_new_symlink_when_restart_succeeds() {
1984 let tmp = setup_release_root(true).await;
1985 let root = tmp.path().to_string_lossy().into_owned();
1986 let out = run_script(&swap_and_restart_script(&root, "new", "true")).await;
1987 assert!(
1988 out.status.success(),
1989 "script should succeed when restart succeeds"
1990 );
1991 let target = tokio::fs::read_link(tmp.path().join("current"))
1992 .await
1993 .unwrap();
1994 assert_eq!(
1995 target.to_string_lossy(),
1996 "releases/new",
1997 "symlink advanced to new"
1998 );
1999 }
2000
2001 #[tokio::test]
2002 async fn swap_and_restart_rolls_symlink_back_when_restart_fails() {
2003 // The bug: a restart failure after the flip must NOT leave `current`
2004 // pointing at the new (un-activated) release.
2005 let tmp = setup_release_root(true).await;
2006 let root = tmp.path().to_string_lossy().into_owned();
2007 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
2008 assert!(!out.status.success(), "script must fail when restart fails");
2009 let target = tokio::fs::read_link(tmp.path().join("current"))
2010 .await
2011 .unwrap();
2012 assert_eq!(
2013 target.to_string_lossy(),
2014 "releases/old",
2015 "symlink rolled back to prev so a later restart can't silently activate new",
2016 );
2017 }
2018
2019 // ---- arch_guard_script: wrong-arch artifacts fail closed ----
2020
2021 /// A 20-byte stub whose ELF e_machine field (offset 18, 2 bytes LE) is set.
2022 fn elf_stub_with_machine(b18: u8, b19: u8) -> tempfile::NamedTempFile {
2023 let mut data = vec![0u8; 20];
2024 data[18] = b18;
2025 data[19] = b19;
2026 let f = tempfile::NamedTempFile::new().unwrap();
2027 std::fs::write(f.path(), &data).unwrap();
2028 f
2029 }
2030
2031 /// e_machine low byte for the host running the test, if mapped.
2032 fn host_machine_lo() -> Option<u8> {
2033 match std::env::consts::ARCH {
2034 "x86_64" => Some(0x3e),
2035 "aarch64" => Some(0xb7),
2036 _ => None,
2037 }
2038 }
2039
2040 #[tokio::test]
2041 async fn arch_guard_passes_for_matching_binary() {
2042 let Some(lo) = host_machine_lo() else { return };
2043 let f = elf_stub_with_machine(lo, 0x00);
2044 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
2045 assert!(
2046 out.status.success(),
2047 "matching arch must pass: {}",
2048 String::from_utf8_lossy(&out.stderr),
2049 );
2050 }
2051
2052 #[tokio::test]
2053 async fn arch_guard_fails_closed_for_wrong_binary() {
2054 // Use the other arch's e_machine so it can't match the host.
2055 let wrong = match std::env::consts::ARCH {
2056 "x86_64" => 0xb7, // aarch64 binary on an x86_64 node
2057 "aarch64" => 0x3e, // x86_64 binary on an aarch64 node
2058 _ => return,
2059 };
2060 let f = elf_stub_with_machine(wrong, 0x00);
2061 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
2062 assert!(
2063 !out.status.success(),
2064 "wrong-arch binary must fail closed before the symlink swap"
2065 );
2066 }
2067
2068 // ---- ldd_guard_script: a binary this node cannot resolve fails closed ----
2069
2070 /// A fake `ldd` on PATH that prints `body` and exits `code`, so the guard's
2071 /// three outcomes can be exercised without a binary that genuinely fails to
2072 /// link. The real `ldd` cannot be made to produce a `not found` on demand.
2073 async fn run_ldd_guard_with_fake(body: &str, code: i32) -> std::process::Output {
2074 let dir = tempfile::tempdir().unwrap();
2075 let fake = dir.path().join("ldd");
2076 std::fs::write(
2077 &fake,
2078 format!("#!/bin/sh\ncat <<'EOF'\n{body}\nEOF\nexit {code}\n"),
2079 )
2080 .unwrap();
2081 let mut perms = std::fs::metadata(&fake).unwrap().permissions();
2082 std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
2083 std::fs::set_permissions(&fake, perms).unwrap();
2084 let bin = dir.path().join("subject");
2085 std::fs::write(&bin, b"x").unwrap();
2086 Command::new("sh")
2087 .arg("-c")
2088 .arg(ldd_guard_script(&bin.to_string_lossy()))
2089 .env("PATH", format!("{}:/usr/bin:/bin", dir.path().display()))
2090 .output()
2091 .await
2092 .unwrap()
2093 }
2094
2095 #[tokio::test]
2096 async fn ldd_guard_fails_closed_on_an_unsatisfiable_symbol_version() {
2097 // The exact failure Bento's glibc_check used to catch at build time, and
2098 // the reason this guard exists: right arch, resolves every library, and
2099 // still cannot exec because the node's glibc is older than the build
2100 // host's.
2101 let out = run_ldd_guard_with_fake(
2102 "\tlinux-vdso.so.1 (0x00007fff)\n\
2103 \t/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.40' not found (required by ./pom)\n\
2104 \tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
2105 0,
2106 )
2107 .await;
2108 assert!(
2109 !out.status.success(),
2110 "an unsatisfiable symbol version must fail before the symlink swap"
2111 );
2112 let stderr = String::from_utf8_lossy(&out.stderr);
2113 assert!(
2114 stderr.contains("GLIBC_2.40"),
2115 "the offending line must reach the operator, not just a verdict: {stderr}"
2116 );
2117 }
2118
2119 #[tokio::test]
2120 async fn ldd_guard_fails_closed_on_a_missing_library() {
2121 let out = run_ldd_guard_with_fake("\tlibfoo.so.1 => not found", 0).await;
2122 assert!(!out.status.success(), "a missing library must fail closed");
2123 }
2124
2125 #[tokio::test]
2126 async fn ldd_guard_passes_a_resolvable_binary() {
2127 let out = run_ldd_guard_with_fake(
2128 "\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
2129 0,
2130 )
2131 .await;
2132 assert!(
2133 out.status.success(),
2134 "a fully resolved binary must pass: {}",
2135 String::from_utf8_lossy(&out.stderr),
2136 );
2137 }
2138
2139 #[tokio::test]
2140 async fn ldd_guard_passes_a_static_binary() {
2141 // ldd exits non-zero for these. Nothing to resolve is not a failure.
2142 let out = run_ldd_guard_with_fake("\tnot a dynamic executable", 1).await;
2143 assert!(
2144 out.status.success(),
2145 "a static binary has no dependencies to satisfy: {}",
2146 String::from_utf8_lossy(&out.stderr),
2147 );
2148 }
2149
2150 #[tokio::test]
2151 async fn ldd_guard_fails_when_ldd_errors_for_another_reason() {
2152 // Not the static case: ldd said something else and exited non-zero. We
2153 // do not know the binary is fine, so we do not say it is.
2154 let out = run_ldd_guard_with_fake("ldd: cannot read file", 1).await;
2155 assert!(
2156 !out.status.success(),
2157 "an unexplained ldd failure must not read as a pass"
2158 );
2159 }
2160
2161 #[tokio::test]
2162 async fn ldd_guard_skips_when_the_node_has_no_ldd() {
2163 // Cannot verify is not known bad, matching arch_guard's unmapped-arch
2164 // call. PATH holds nothing, so `command -v ldd` finds none.
2165 let dir = tempfile::tempdir().unwrap();
2166 let bin = dir.path().join("subject");
2167 std::fs::write(&bin, b"x").unwrap();
2168 // Absolute path to the shell: PATH is what this test empties, so
2169 // resolving `sh` through it would fail before the script ever ran.
2170 let out = Command::new("/bin/sh")
2171 .arg("-c")
2172 .arg(ldd_guard_script(&bin.to_string_lossy()))
2173 .env("PATH", dir.path().display().to_string())
2174 .output()
2175 .await
2176 .unwrap();
2177 assert!(
2178 out.status.success(),
2179 "a node with no ldd must not fail the deploy: {}",
2180 String::from_utf8_lossy(&out.stderr),
2181 );
2182 }
2183
2184 #[tokio::test]
2185 async fn swap_and_restart_first_deploy_failure_has_no_prev_to_restore() {
2186 // No prior `current`. A restart failure leaves `current` at new (the only
2187 // version) and still reports failure — documented degenerate case.
2188 let tmp = setup_release_root(false).await;
2189 let root = tmp.path().to_string_lossy().into_owned();
2190 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
2191 assert!(!out.status.success(), "script must fail when restart fails");
2192 let target = tokio::fs::read_link(tmp.path().join("current"))
2193 .await
2194 .unwrap();
2195 assert_eq!(
2196 target.to_string_lossy(),
2197 "releases/new",
2198 "no prev existed to roll back to"
2199 );
2200 }
2201
2202 // ---- config_check_script: systemd-faithful env loading ----
2203
2204 #[tokio::test]
2205 async fn config_check_script_loads_values_with_shell_metachars() {
2206 // The bug: `. env_file` expands/word-splits values, so a URL or a
2207 // password containing a shell metacharacter is mangled — it dropped
2208 // DATABASE_URL to empty on a real node, which would fail every deploy.
2209 // The export-loop must load such a value intact. The "binary" is a
2210 // checker script (a real path, like a deployed binary) that exits 0 only
2211 // if the var arrived byte-for-byte — it compares against the expected
2212 // value read from a file, so nothing re-interprets the metacharacters.
2213 let tricky = "postgres://u:p$ss;w&rd@h/db `x` $(y)";
2214 // Plain files in a tempdir: no lingering write fd, so the checker can be
2215 // exec'd (a NamedTempFile stays open and would ETXTBSY).
2216 let dir = tempfile::tempdir().unwrap();
2217 let expected_path = dir.path().join("expected");
2218 std::fs::write(&expected_path, tricky).unwrap(); // no trailing newline
2219
2220 let env_path = dir.path().join("node.env");
2221 std::fs::write(
2222 &env_path,
2223 format!(
2224 "# a comment\n\nDATABASE_URL={tricky}\nOTHER=plain\nEXPECTED_FILE={ef}\n",
2225 ef = expected_path.display(),
2226 ),
2227 )
2228 .unwrap();
2229
2230 let checker_path = dir.path().join("checker.sh");
2231 std::fs::write(
2232 &checker_path,
2233 "#!/bin/sh\nwant=$(cat \"$EXPECTED_FILE\")\n\
2234 [ \"$DATABASE_URL\" = \"$want\" ] || { echo \"DB [$DATABASE_URL] != [$want]\" >&2; exit 1; }\n\
2235 [ \"$OTHER\" = plain ] || { echo \"OTHER [$OTHER]\" >&2; exit 1; }\n",
2236 )
2237 .unwrap();
2238 std::fs::set_permissions(
2239 &checker_path,
2240 std::os::unix::fs::PermissionsExt::from_mode(0o755),
2241 )
2242 .unwrap();
2243
2244 let script =
2245 config_check_script(&env_path.to_string_lossy(), &checker_path.to_string_lossy());
2246 let out = run_script(&script).await;
2247 assert!(
2248 out.status.success(),
2249 "value with shell metachars must load intact; stderr: {}",
2250 String::from_utf8_lossy(&out.stderr),
2251 );
2252 }
2253
2254 // ---- install-companion.sh: the node-side guard rails ----
2255
2256 /// Run the shipped installer script with three args; returns its exit code.
2257 /// Exercises the real file rather than a copy of its logic, because the
2258 /// script is the ONLY control on a NOPASSWD sudo grant.
2259 fn run_installer(src: &str, dst: &str, service: &str) -> i32 {
2260 let script =
2261 std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../deploy/install-companion.sh");
2262 std::process::Command::new("bash")
2263 .arg(&script)
2264 .args([src, dst, service])
2265 .output()
2266 .expect("running install-companion.sh")
2267 .status
2268 .code()
2269 .expect("script exited via signal")
2270 }
2271
2272 // Guards run before any filesystem write, so these never install anything.
2273 // Exit 3 = refused by a guard; exit 4 = guards passed, src simply absent.
2274 const REFUSED: i32 = 3;
2275 const PASSED_GUARDS: i32 = 4;
2276
2277 #[test]
2278 fn installer_refuses_a_dst_that_escapes_opt_via_dotdot() {
2279 // `/opt/../etc/...` matches a bare `/opt/*` glob. With the sudoers
2280 // wildcard that meant `install -m 0755` as root to anywhere, plus a
2281 // restart of any unit — so the path must be normalised before the test.
2282 assert_eq!(
2283 run_installer(
2284 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
2285 "/opt/../etc/systemd/system/evil.service",
2286 "mnw-cli.service",
2287 ),
2288 REFUSED,
2289 );
2290 }
2291
2292 #[test]
2293 fn installer_refuses_a_src_that_escapes_the_bundle_via_dotdot() {
2294 assert_eq!(
2295 run_installer(
2296 "/opt/mnw/releases/1.0.0/companions/../../../../../etc/shadow",
2297 "/opt/mnw-cli/mnw-cli",
2298 "mnw-cli.service",
2299 ),
2300 REFUSED,
2301 );
2302 }
2303
2304 #[test]
2305 fn installer_accepts_the_real_companion_paths() {
2306 // The guards must not have been tightened into uselessness: the shape
2307 // Sando actually sends has to get past them. It stops at the missing
2308 // src (exit 4), which is proof the guards accepted it.
2309 assert_eq!(
2310 run_installer(
2311 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
2312 "/opt/mnw-cli/mnw-cli",
2313 "mnw-cli.service",
2314 ),
2315 PASSED_GUARDS,
2316 );
2317 }
2318
2319 #[test]
2320 fn installer_refuses_a_service_name_with_a_path_separator() {
2321 assert_eq!(
2322 run_installer(
2323 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
2324 "/opt/mnw-cli/mnw-cli",
2325 "../../etc/evil.service",
2326 ),
2327 REFUSED,
2328 );
2329 }
2330
2331 // ---- install_companion_cmd: shape + quoting ----
2332
2333 #[test]
2334 fn install_companion_cmd_shape_and_quoting() {
2335 let cmd = install_companion_cmd(
2336 "/opt/mnw/releases/0.10.14/companions/mnw-cli",
2337 "/opt/mnw-cli/mnw-cli",
2338 "mnw-cli.service",
2339 );
2340 // Routes through the wrapper (single sudoers grant), sudo-invoked, with
2341 // src, dst, service in that order.
2342 assert!(cmd.starts_with("sudo "), "must be sudo-invoked: {cmd}");
2343 assert!(
2344 cmd.contains("/usr/local/lib/mnw/install-companion.sh"),
2345 "{cmd}"
2346 );
2347 let installer_pos = cmd.find("install-companion.sh").unwrap();
2348 let src_pos = cmd.find("companions/mnw-cli").unwrap();
2349 let dst_pos = cmd.find("/opt/mnw-cli/mnw-cli").unwrap();
2350 let svc_pos = cmd.find("mnw-cli.service").unwrap();
2351 assert!(
2352 installer_pos < src_pos && src_pos < dst_pos && dst_pos < svc_pos,
2353 "arg order: {cmd}"
2354 );
2355 }
2356
2357 #[test]
2358 fn install_companion_cmd_quotes_metachars() {
2359 // A path with a space/quote must be shell-safe (defense in depth even
2360 // though these come from operator config).
2361 let cmd = install_companion_cmd("/a b/src", "/dst'x", "u.service");
2362 let out = std::process::Command::new("sh")
2363 .arg("-c")
2364 .arg(format!(
2365 "set -- {}; echo \"$#\"",
2366 cmd.strip_prefix("sudo ").unwrap()
2367 ))
2368 .output()
2369 .unwrap();
2370 // installer + 3 args = 4 positional words after quoting.
2371 assert_eq!(
2372 String::from_utf8_lossy(&out.stdout).trim(),
2373 "4",
2374 "quoting split wrong: {cmd}"
2375 );
2376 }
2377
2378 #[tokio::test]
2379 async fn config_check_script_propagates_binary_failure() {
2380 // A required var missing (the binary exits non-zero) must fail the check.
2381 let env = tempfile::NamedTempFile::new().unwrap();
2382 std::fs::write(env.path(), "FOO=bar\n").unwrap();
2383 let script = config_check_script(&env.path().to_string_lossy(), "false");
2384 let out = run_script(&script).await;
2385 assert!(
2386 !out.status.success(),
2387 "a non-zero MNW_CHECK_CONFIG exit must fail the check"
2388 );
2389 }
2390
2391 #[tokio::test]
2392 async fn deploy_node_denied_when_executor_lacks_deploy_grant() {
2393 // Defense in depth: an executor without the deploy grant refuses the
2394 // step before any filesystem / ssh action.
2395 let tmp = tempfile::tempdir().unwrap();
2396 let release_root = tmp.path().to_path_buf();
2397 let staged = release_root.join("releases").join("0.0.1");
2398 tokio::fs::create_dir_all(&staged).await.unwrap();
2399
2400 let node = crate::topology::Node {
2401 platform: None,
2402 base_image: None,
2403 libc: None,
2404 name: "local-dev".into(),
2405 ssh_target: "local".into(),
2406 release_root: release_root.to_string_lossy().into_owned(),
2407 service_name: "makenotwork.service".into(),
2408 health_url: None,
2409 config_check_env_file: None,
2410 actuate: vec!["restart".into()], // no deploy
2411 observe: vec![],
2412 companions: Vec::new(),
2413 };
2414 let executor = LocalExec::new(CapabilitySet::from_tokens(["restart"], Vec::<&str>::new()));
2415 let err = deploy_node(
2416 &executor,
2417 Placement::check(&node, &staged, None).unwrap(),
2418 "0.0.1",
2419 "server",
2420 Some(&no_pins()),
2421 )
2422 .await
2423 .unwrap_err();
2424 assert!(
2425 format!("{err:#}").contains("capability denied"),
2426 "expected capability denial"
2427 );
2428 }
2429
2430 // ---- FakeExec: the deploy_remote choreography without a real host ----
2431 //
2432 // deploy_node's local fast-path is covered above with a real LocalExec, but
2433 // the remote path (rsync + arch guard + config-drift + swap + companions +
2434 // gc) short-circuits on `ssh_target != "local"` and so never ran under test
2435 // without a reachable node. FakeExec records every executor call in order
2436 // and can be told to fail one shell step (matched by substring) or the rsync
2437 // push, so the ordering and the fail-closed-before-swap contract are
2438 // assertable in-process.
2439
2440 struct FakeExec {
2441 caps: CapabilitySet,
2442 calls: Arc<StdMutex<Vec<String>>>,
2443 /// The first `run_streaming` whose script contains this substring exits
2444 /// non-zero (a failed shell step), e.g. the arch guard.
2445 fail_run_matching: Option<String>,
2446 /// `push_dir` (the rsync) returns an error.
2447 fail_push_dir: bool,
2448 }
2449
2450 impl FakeExec {
2451 fn new() -> Self {
2452 Self {
2453 caps: CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
2454 calls: Arc::new(StdMutex::new(Vec::new())),
2455 fail_run_matching: None,
2456 fail_push_dir: false,
2457 }
2458 }
2459 fn log(&self) -> Vec<String> {
2460 self.calls.lock().unwrap().clone()
2461 }
2462 }
2463
2464 #[async_trait]
2465 impl Executor for FakeExec {
2466 async fn run_streaming(&self, step: &Step, _sink: &mut dyn LogSink) -> Result<RunOutput> {
2467 // Every deploy step is a `Step::shell`, so the script is argv's tail.
2468 let script = step.argv.last().cloned().unwrap_or_default();
2469 self.calls.lock().unwrap().push(format!("run:{script}"));
2470 let fail = self
2471 .fail_run_matching
2472 .as_deref()
2473 .is_some_and(|m| script.contains(m));
2474 Ok(RunOutput {
2475 status: std::process::ExitStatus::from_raw(if fail { 1 << 8 } else { 0 }),
2476 stdout: Vec::new(),
2477 stderr: if fail {
2478 b"fake step failure".to_vec()
2479 } else {
2480 Vec::new()
2481 },
2482 })
2483 }
2484 async fn pull_file(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
2485 self.calls.lock().unwrap().push("pull_file".into());
2486 Ok(())
2487 }
2488 async fn pull_dir(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
2489 self.calls.lock().unwrap().push("pull_dir".into());
2490 Ok(())
2491 }
2492 async fn pull_glob(&self, _glob: &str, _local: &Path, _opts: &SyncOpts) -> Result<()> {
2493 self.calls.lock().unwrap().push("pull_glob".into());
2494 Ok(())
2495 }
2496 async fn push_dir(&self, _local: &Path, remote: &Path, _opts: &SyncOpts) -> Result<()> {
2497 self.calls
2498 .lock()
2499 .unwrap()
2500 .push(format!("push_dir:{}", remote.display()));
2501 if self.fail_push_dir {
2502 anyhow::bail!("fake rsync failure");
2503 }
2504 Ok(())
2505 }
2506 fn capabilities(&self) -> &CapabilitySet {
2507 &self.caps
2508 }
2509 }
2510
2511 fn remote_node(config_check: bool, companions: Vec<NodeCompanion>) -> Node {
2512 Node {
2513 platform: None,
2514 base_image: None,
2515 libc: None,
2516 name: "web-a".into(),
2517 ssh_target: "deploy@web-a".into(),
2518 release_root: "/opt/mnw".into(),
2519 service_name: "makenotwork.service".into(),
2520 health_url: None,
2521 config_check_env_file: config_check.then(|| "/etc/mnw/node.env".to_string()),
2522 actuate: crate::topology::default_actuate(),
2523 observe: crate::topology::default_observe(),
2524 companions,
2525 }
2526 }
2527
2528 fn companion() -> NodeCompanion {
2529 NodeCompanion {
2530 name: "mnw-cli".into(),
2531 install_path: "/opt/mnw-cli/mnw-cli".into(),
2532 service_name: "mnw-cli.service".into(),
2533 }
2534 }
2535
2536 /// Index of the first recorded call whose text contains `needle` (panics if
2537 /// absent — the assertion message names what was missing).
2538 fn pos(log: &[String], needle: &str) -> usize {
2539 log.iter()
2540 .position(|c| c.contains(needle))
2541 .unwrap_or_else(|| panic!("no call matched {needle:?} in {log:#?}"))
2542 }
2543
2544 #[tokio::test]
2545 async fn deploy_remote_runs_the_full_choreography_in_order() {
2546 // A node opted into the config-drift check and carrying one companion:
2547 // mkdir -> rsync -> arch guard -> config check -> swap+restart ->
2548 // companion install -> gc, in that order.
2549 let tmp = tempfile::tempdir().unwrap();
2550 let staged = tmp.path().join("releases").join("0.9.0");
2551 tokio::fs::create_dir_all(&staged).await.unwrap();
2552
2553 let node = remote_node(true, vec![companion()]);
2554 let exec = FakeExec::new();
2555 let out = deploy_node(
2556 &exec,
2557 Placement::check(&node, &staged, None).unwrap(),
2558 "0.9.0",
2559 "makenotwork",
2560 Some(&no_pins()),
2561 )
2562 .await
2563 .expect("deploy_remote should succeed against the fake");
2564 assert_eq!(out, PathBuf::from("/opt/mnw/releases/0.9.0"));
2565
2566 let log = exec.log();
2567 let mkdir = pos(&log, "mkdir -p");
2568 let rsync = pos(&log, "push_dir:/opt/mnw/releases/0.9.0");
2569 let arch = pos(&log, "e_machine");
2570 let cfg = pos(&log, "MNW_CHECK_CONFIG=1");
2571 let swap = pos(&log, "reload-or-restart");
2572 let comp = pos(&log, "install-companion.sh");
2573 let gc = pos(&log, "ls -1t");
2574 assert!(
2575 mkdir < rsync && rsync < arch && arch < cfg && cfg < swap && swap < comp && comp < gc,
2576 "deploy steps out of order: {log:#?}"
2577 );
2578 }
2579
2580 #[tokio::test]
2581 async fn deploy_remote_aborts_before_swap_when_rsync_fails() {
2582 // The rsync failing must fail the deploy BEFORE the symlink swap — the
2583 // "current symlink left intact" contract. Assert the swap never ran.
2584 let tmp = tempfile::tempdir().unwrap();
2585 let staged = tmp.path().join("releases").join("0.9.0");
2586 tokio::fs::create_dir_all(&staged).await.unwrap();
2587
2588 let node = remote_node(false, Vec::new());
2589 let mut exec = FakeExec::new();
2590 exec.fail_push_dir = true;
2591 let err = deploy_node(
2592 &exec,
2593 Placement::check(&node, &staged, None).unwrap(),
2594 "0.9.0",
2595 "makenotwork",
2596 Some(&no_pins()),
2597 )
2598 .await
2599 .expect_err("rsync failure must fail the deploy");
2600 assert!(
2601 format!("{err:#}").contains("rsync"),
2602 "error should attribute the rsync: {err:#}"
2603 );
2604 let log = exec.log();
2605 assert!(
2606 !log.iter().any(|c| c.contains("reload-or-restart")),
2607 "swap must not run after a failed rsync: {log:#?}"
2608 );
2609 }
2610
2611 #[tokio::test]
2612 async fn deploy_remote_aborts_before_swap_when_arch_guard_fails() {
2613 // A wrong-arch binary must fail closed before the swap. The fake fails
2614 // the arch-guard shell step; the swap must not follow.
2615 let tmp = tempfile::tempdir().unwrap();
2616 let staged = tmp.path().join("releases").join("0.9.0");
2617 tokio::fs::create_dir_all(&staged).await.unwrap();
2618
2619 let node = remote_node(false, Vec::new());
2620 let mut exec = FakeExec::new();
2621 exec.fail_run_matching = Some("e_machine".into());
2622 let err = deploy_node(
2623 &exec,
2624 Placement::check(&node, &staged, None).unwrap(),
2625 "0.9.0",
2626 "makenotwork",
2627 Some(&no_pins()),
2628 )
2629 .await
2630 .expect_err("arch mismatch must fail the deploy");
2631 assert!(
2632 format!("{err:#}").contains("architecture"),
2633 "error should mention the arch check: {err:#}"
2634 );
2635 let log = exec.log();
2636 assert!(
2637 !log.iter().any(|c| c.contains("reload-or-restart")),
2638 "swap must not run after a failed arch guard: {log:#?}"
2639 );
2640 }
2641
2642 #[tokio::test]
2643 async fn deploy_remote_skips_config_check_when_node_opts_out() {
2644 // No config_check_env_file => the pre-swap config check is skipped, but
2645 // the rest of the choreography (including the swap) still runs.
2646 let tmp = tempfile::tempdir().unwrap();
2647 let staged = tmp.path().join("releases").join("0.9.0");
2648 tokio::fs::create_dir_all(&staged).await.unwrap();
2649
2650 let node = remote_node(false, Vec::new());
2651 let exec = FakeExec::new();
2652 deploy_node(
2653 &exec,
2654 Placement::check(&node, &staged, None).unwrap(),
2655 "0.9.0",
2656 "makenotwork",
2657 Some(&no_pins()),
2658 )
2659 .await
2660 .unwrap();
2661 let log = exec.log();
2662 assert!(
2663 !log.iter().any(|c| c.contains("MNW_CHECK_CONFIG=1")),
2664 "config check must be skipped when the node opts out: {log:#?}"
2665 );
2666 assert!(
2667 log.iter().any(|c| c.contains("reload-or-restart")),
2668 "the swap must still run: {log:#?}"
2669 );
2670 }
2671
2672 /// A companion is guarded on the same terms as the primary, and BEFORE the
2673 /// swap. Unguarded, the first thing to notice a bad companion is its unit
2674 /// failing to start during the install loop, which runs after the server has
2675 /// already been restarted.
2676 #[tokio::test]
2677 async fn companions_are_guarded_before_the_swap() {
2678 let tmp = tempfile::tempdir().unwrap();
2679 let staged = tmp.path().join("releases").join("0.9.0");
2680 tokio::fs::create_dir_all(&staged).await.unwrap();
2681
2682 let node = remote_node(false, vec![companion()]);
2683 let exec = FakeExec::new();
2684 deploy_node(
2685 &exec,
2686 Placement::check(&node, &staged, None).unwrap(),
2687 "0.9.0",
2688 "makenotwork",
2689 Some(&no_pins()),
2690 )
2691 .await
2692 .unwrap();
2693
2694 let log = exec.log();
2695 // The companion's own arch and loader checks, named by its path so they
2696 // cannot be confused with the primary's.
2697 let guard = pos(&log, "companions/mnw-cli");
2698 let swap = pos(&log, "reload-or-restart");
2699 let install = pos(&log, "install-companion.sh");
2700 assert!(
2701 guard < swap && swap < install,
2702 "a companion must be guarded before the swap and installed after it: {log:#?}"
2703 );
2704 let companion_guards = log
2705 .iter()
2706 .filter(|c| c.contains("companions/mnw-cli") && !c.contains("install-companion.sh"))
2707 .count();
2708 assert_eq!(
2709 companion_guards, 2,
2710 "both guards must run against the companion, not just one: {log:#?}"
2711 );
2712 }
2713
2714 /// And failing one of them fails the promote with the service intact, which
2715 /// is the whole point of moving the check ahead of the swap.
2716 #[tokio::test]
2717 async fn a_companion_failing_its_guard_aborts_before_the_swap() {
2718 let tmp = tempfile::tempdir().unwrap();
2719 let staged = tmp.path().join("releases").join("0.9.0");
2720 tokio::fs::create_dir_all(&staged).await.unwrap();
2721
2722 let node = remote_node(false, vec![companion()]);
2723 let mut exec = FakeExec::new();
2724 // Fails the first script naming the companion, which is its arch guard.
2725 // The primary's guards name the primary and are unaffected.
2726 exec.fail_run_matching = Some("companions/mnw-cli".into());
2727 let err = deploy_node(
2728 &exec,
2729 Placement::check(&node, &staged, None).unwrap(),
2730 "0.9.0",
2731 "makenotwork",
2732 Some(&no_pins()),
2733 )
2734 .await
2735 .expect_err("a bad companion must fail the deploy");
2736
2737 let msg = format!("{err:#}");
2738 assert!(
2739 msg.contains("mnw-cli"),
2740 "the refusal must name which companion: {msg}"
2741 );
2742 assert_eq!(
2743 stage_of(&err),
2744 Some(FailureStage::BeforeSwap),
2745 "a companion guard failing must leave the service intact: {msg}"
2746 );
2747 let log = exec.log();
2748 assert!(
2749 !log.iter().any(|c| c.contains("reload-or-restart")),
2750 "swap must not run after a failed companion guard: {log:#?}"
2751 );
2752 assert!(
2753 !log.iter().any(|c| c.contains("install-companion.sh")),
2754 "nothing should be installed after a failed companion guard: {log:#?}"
2755 );
2756 }
2757
2758 /// The guards and the installer must read the same path. A guard checking a
2759 /// path the installer does not use is a check of nothing, and passes.
2760 #[test]
2761 fn the_guarded_companion_path_is_the_one_installed() {
2762 let release_dir = "/opt/mnw/releases/0.9.0";
2763 let src = companion_src(release_dir, "mnw-cli");
2764 assert_eq!(src, "/opt/mnw/releases/0.9.0/companions/mnw-cli");
2765 let cmd = install_companion_cmd(&src, "/opt/mnw-cli/mnw-cli", "mnw-cli.service");
2766 assert!(
2767 cmd.contains(&src),
2768 "the installer must read the path the guards checked: {cmd}"
2769 );
2770 }
2771
2772 #[tokio::test]
2773 async fn deploy_remote_installs_companion_after_the_swap() {
2774 // Companions are After= the server: their install must land after the
2775 // symlink swap + service restart, never before.
2776 let tmp = tempfile::tempdir().unwrap();
2777 let staged = tmp.path().join("releases").join("0.9.0");
2778 tokio::fs::create_dir_all(&staged).await.unwrap();
2779
2780 let node = remote_node(false, vec![companion()]);
2781 let exec = FakeExec::new();
2782 deploy_node(
2783 &exec,
2784 Placement::check(&node, &staged, None).unwrap(),
2785 "0.9.0",
2786 "makenotwork",
2787 Some(&no_pins()),
2788 )
2789 .await
2790 .unwrap();
2791 let log = exec.log();
2792 assert!(
2793 pos(&log, "reload-or-restart") < pos(&log, "install-companion.sh"),
2794 "companion install must follow the swap: {log:#?}"
2795 );
2796 }
2797 }
2798