max / makenotwork

113.5 KB · 2808 lines History Blame Raw

1	//! Atomic symlink-swap deploys.
2	//!
3	//! Layout on every target (local host, A nodes, B nodes, ...). Release dirs are
4	//! named for their content digest, not the version (wiki note
5	//! `release-artifact-identity`), so a rebuild can never overwrite an earlier
6	//! build's dir in place:
7	//!
8	//! <release_root>/
9	//! releases/
10	//! a1b2c3d4e5f60718/ <- <digest16>
11	//! <bin_name>
12	//! MANIFEST <- <sha256> <relpath> per file, node-verified
13	//! f0e1d2c3b4a59687/
14	//! <bin_name>
15	//! current -> releases/f0e1d2c3b4a59687
16	//!
17	//! `ln -sfn` swaps the symlink. systemd units point at
18	//! `<release_root>/current/<bin_name>` so reload-or-restart picks up the new
19	//! binary without ever pointing at a missing path.
20	//!
21	//! The host-side transport — `ssh` for shell steps, `rsync` for the release
22	//! dir — comes from the shared [`ops_exec::Executor`] (a `LocalExec` for
23	//! `ssh_target = "local"`, an `SshExec` otherwise), built once per node in
24	//! [`crate::state`]. This module owns the deploy choreography (mkdir, push,
25	//! atomic swap, restart, gc); the transport is the crate's. SSH push behavior
26	//! is identical to the pre-extraction code — this is a transport extraction,
27	//! not a model change.
28
29	use crate::domain::Platform;
30	use crate::retention::PinnedReleases;
31	use crate::topology::Node;
32	use anyhow::{Context, Result};
33	use async_trait::async_trait;
34	use ops_core::base_image;
35	use ops_exec::{Action, Executor, LogSink, RunOutput, Step, SyncOpts, sh_quote};
36	use std::path::{Path, PathBuf};
37	use tokio::process::Command;
38
39	/// A staged bundle proven to be for the node it is about to be pushed to.
40	///
41	/// This exists because "ship aarch64 bytes to an x86_64 box" was, until pom, a
42	/// mistake nobody could make: one product, one build host, one architecture, so
43	/// the pairing of a bundle and a node was correct by having no alternative. pom
44	/// has two architectures under one version, so the pairing becomes a real
45	/// choice, and a wrong choice deploys a binary the node cannot exec.
46	///
47	/// The answer is not a check before the call. A check is something a later
48	/// caller forgets, and the failure it guards is discovered by a production node
49	/// failing to start. [`Placement::check`] is the only way to obtain one of
50	/// these, and [`deploy_node`] takes one instead of a loose `(node, dir)` pair —
51	/// so a mismatched deploy is not a bug the code has to avoid, it is a value the
52	/// code cannot construct.
53	#[derive(Debug, Clone)]
54	pub struct Placement<'a> {
55	node: &'a Node,
56	bundle: &'a Path,
57	}
58
59	/// Why a bundle may not be placed on a node.
60	///
61	/// All four cases are refusals, including both "one side said nothing" cases.
62	/// Silence is not agreement: a node that does not state its platform cannot
63	/// vouch that it runs a bundle built for a stated one, and a bundle that does
64	/// not state its platform cannot satisfy a node that requires one. The only
65	/// admissible pairing besides a match is both sides silent, which is the
66	/// single-platform world Sando lived in and MNW still lives in.
67	#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
68	pub enum PlacementError {
69	#[error(
70	"node {node} runs {node_platform} and this bundle was built for {artifact_platform}; \
71	refusing to deploy a binary the node cannot execute"
72	)]
73	Mismatch {
74	node: String,
75	node_platform: Platform,
76	artifact_platform: Platform,
77	},
78	#[error(
79	"node {node} does not declare a platform, and this bundle was built for \
80	{artifact_platform}. Declare `platform` on the node so the two can be compared"
81	)]
82	NodeSilent {
83	node: String,
84	artifact_platform: Platform,
85	},
86	#[error(
87	"node {node} requires {node_platform} and this bundle records no platform. \
88	An artifact whose platform is unknown cannot be shown to satisfy one that is"
89	)]
90	ArtifactSilent {
91	node: String,
92	node_platform: Platform,
93	},
94	}
95
96	impl<'a> Placement<'a> {
97	/// The one constructor. `artifact` is the platform the bundle records, which
98	/// for an accepted artifact comes from its `ArtifactRecord` provenance and
99	/// for a Sando-built one comes from the app config.
100	pub fn check(
101	node: &'a Node,
102	bundle: &'a Path,
103	artifact: Option<&Platform>,
104	) -> Result<Self, PlacementError> {
105	match (node.platform.as_ref(), artifact) {
106	(Some(n), Some(a)) if n == a => Ok(Self { node, bundle }),
107	(Some(n), Some(a)) => Err(PlacementError::Mismatch {
108	node: node.name.to_string(),
109	node_platform: n.clone(),
110	artifact_platform: a.clone(),
111	}),
112	(None, Some(a)) => Err(PlacementError::NodeSilent {
113	node: node.name.to_string(),
114	artifact_platform: a.clone(),
115	}),
116	(Some(n), None) => Err(PlacementError::ArtifactSilent {
117	node: node.name.to_string(),
118	node_platform: n.clone(),
119	}),
120	// Both silent: the single-platform world. MNW is here, and stays
121	// here until its nodes declare a platform — at which point its
122	// builds have to as well, which is the forcing function rather than
123	// a silently mixed state.
124	(None, None) => Ok(Self { node, bundle }),
125	}
126	}
127
128	pub fn node(&self) -> &'a Node {
129	self.node
130	}
131
132	pub fn bundle(&self) -> &'a Path {
133	self.bundle
134	}
135	}
136
137	/// Keep at least this many release dirs per node; older ones get gc'd after a
138	/// successful deploy. Fixed for now; promote to config if the constant ever
139	/// needs to vary by tier.
140	///
141	/// A floor, not a ceiling. Whatever the deployed state still references is set
142	/// aside first and this count applies to the remainder — see
143	/// [`crate::retention`] for why a count alone could not express that.
144	const RELEASES_TO_KEEP: usize = 5;
145
146	/// A sink that drops streamed bytes. Deploy steps don't have a live-log handle
147	/// (gates do), so output is discarded as it streams; [`RunOutput`] still
148	/// captures the full stdout/stderr for error reporting, preserving the
149	/// pre-extraction behavior of surfacing `stderr` in failure messages.
150	struct DiscardSink;
151
152	#[async_trait]
153	impl LogSink for DiscardSink {
154	async fn write_chunk(&mut self, _bytes: &[u8]) {}
155	}
156
157	/// Run a shell step through `executor`, treating a non-zero exit as an error
158	/// whose message carries the captured stderr — exactly as the old bespoke
159	/// `ssh()` helper did (`ssh <target> failed: <stderr>`).
160	async fn run_checked(executor: &dyn Executor, script: &str, what: &str) -> Result<RunOutput> {
161	let step = Step::shell(Action::Deploy, script);
162	let mut sink = DiscardSink;
163	let out = executor
164	.run_streaming(&step, &mut sink)
165	.await
166	.with_context(\|\| format!("{what}: spawning command"))?;
167	anyhow::ensure!(
168	out.status.success(),
169	"{what} failed (exit {}): {}",
170	out.status
171	.code()
172	.map_or_else(\|\| "signal".into(), \|c\| c.to_string()),
173	String::from_utf8_lossy(&out.stderr),
174	);
175	Ok(out)
176	}
177
178	/// Stage built binaries into `staging/<build_id>/` on the Sando host — a
179	/// private, mutable scratch dir that is not yet a release (no symlink, no gc).
180	/// The caller adds `release_contents` + companions, hashes the result, writes
181	/// the `MANIFEST`, then publishes it content-addressed via
182	/// [`finalize_local_release`]. Splitting staging from publish is what lets the
183	/// bundle be hashed before it is named (wiki [[release-artifact-identity]]).
184	///
185	/// A stale `staging/<build_id>` from a killed prior run at the same id is
186	/// removed first, so a retry stages clean.
187	pub async fn stage_local_bundle(
188	release_root: &Path,
189	build_id: i64,
190	binaries: &[PathBuf],
191	) -> Result<PathBuf> {
192	let staging = release_root.join("staging").join(build_id.to_string());
193	if tokio::fs::try_exists(&staging).await.unwrap_or(false) {
194	tokio::fs::remove_dir_all(&staging)
195	.await
196	.with_context(\|\| format!("clearing stale staging dir {}", staging.display()))?;
197	}
198	tokio::fs::create_dir_all(&staging).await?;
199	for binary in binaries {
200	let name = binary.file_name().context("binary path has no file name")?;
201	let dest = staging.join(name);
202	tokio::fs::copy(binary, &dest)
203	.await
204	.with_context(\|\| format!("copy {} -> {}", binary.display(), dest.display()))?;
205	}
206	Ok(staging)
207	}
208
209	/// Publish a fully-staged bundle content-addressed: rename
210	/// `staging/<build_id>` to `releases/<digest16>` (atomic same-filesystem
211	/// rename), flip `current` to it, gc old releases. Returns the released dir.
212	///
213	/// The rename is the load-bearing step: **a directory whose name derives from
214	/// its contents cannot be rewritten, because rewriting it changes its name.**
215	/// The overwrite class that let a dev rebuild inherit an earlier build's gate
216	/// rows and burn-in clock stops being something to guard against and becomes
217	/// something that cannot be expressed. If a release with this digest already
218	/// exists (identical bytes rebuilt), the staging copy is redundant and dropped.
219	///
220	/// `pinned` names the release dirs the deployed state still points at
221	/// ([`crate::retention::pinned_dirs`]); they are never gc'd here, however old
222	/// they are. It arrives as data rather than as a pool handle so this stays a
223	/// filesystem operation and the intake seam keeps working without a database.
224	pub async fn finalize_local_release(
225	release_root: &Path,
226	staging: &Path,
227	digest16: &str,
228	pinned: &PinnedReleases,
229	) -> Result<PathBuf> {
230	let releases = release_root.join("releases");
231	tokio::fs::create_dir_all(&releases).await?;
232	let released = releases.join(digest16);
233
234	if tokio::fs::try_exists(&released).await.unwrap_or(false) {
235	// Same digest already published — reuse it, discard the redundant stage.
236	tokio::fs::remove_dir_all(staging).await.ok();
237	} else {
238	tokio::fs::rename(staging, &released)
239	.await
240	.with_context(\|\| format!("publish {} -> {}", staging.display(), released.display()))?;
241	}
242
243	let current = release_root.join("current");
244	let target = format!("releases/{digest16}");
245	let out = Command::new("ln")
246	.args(["-sfn", &target])
247	.arg(&current)
248	.output()
249	.await?;
250	anyhow::ensure!(
251	out.status.success(),
252	"symlink swap failed: {}",
253	String::from_utf8_lossy(&out.stderr),
254	);
255
256	if let Err(e) = gc_local_releases(release_root, pinned).await {
257	tracing::warn!(error = %e, "local release GC failed (non-fatal)");
258	}
259	Ok(released)
260	}
261
262	/// Deploy a [`Placement`]'s bundle to its node using `executor` (the node's
263	/// transport from the topology executor map). For `ssh_target=local`, this is
264	/// just a symlink swap; for remote nodes, we rsync the whole dir over the
265	/// executor.
266	///
267	/// The bundle and the node arrive together inside the placement, so there is no
268	/// signature here that accepts a bundle and a node that were never compared.
269	///
270	/// `primary_bin` is only used for logging — every file present in the staged
271	/// dir gets shipped.
272	///
273	/// `pinned` protects the node's own `releases/` from the gc that runs after a
274	/// successful remote deploy. The node mirrors the host's directory name, so the
275	/// set is the host's ([`crate::retention::pinned_dirs`]) with nothing recomputed
276	/// per node. `None` says the caller could not determine it and the gc is skipped
277	/// rather than run blind; `Some(PinnedReleases::none())` says there is genuinely
278	/// nothing deployed to protect. The distinction matters — the first is ignorance
279	/// and the second is knowledge — which is why this is an `Option` and not an
280	/// empty set standing in for both.
281	///
282	/// Unused on the `ssh_target=local` path: that deploy is a symlink swap over a
283	/// store the host gc already owns.
284	pub async fn deploy_node(
285	executor: &dyn Executor,
286	placement: Placement<'_>,
287	version: &str,
288	primary_bin: &str,
289	pinned: Option<&PinnedReleases>,
290	) -> Result<PathBuf> {
291	let node = placement.node();
292	let staged_release_dir = placement.bundle();
293	// The release dir is named for its content digest (`releases/<digest16>`),
294	// not the version. The node mirrors that name so host and node agree on the
295	// artifact's identity; the version is only a log label here. Legacy staged
296	// dirs (pre-identity, still `releases/<version>`) work unchanged — the name
297	// is whatever the host staged under.
298	let release_id = staged_release_dir
299	.file_name()
300	.and_then(\|n\| n.to_str())
301	.with_context(\|\| {
302	format!(
303	"staged release dir {} has no usable name",
304	staged_release_dir.display()
305	)
306	})?;
307	if node.ssh_target == "local" \|\| node.ssh_target.is_empty() {
308	// Local deploy already happened when we staged on the Sando host.
309	// Just re-point `current` at the staged dir.
310	return reset_local_current(executor, Path::new(&node.release_root), release_id).await;
311	}
312	deploy_remote(
313	executor,
314	node,
315	version,
316	release_id,
317	staged_release_dir,
318	primary_bin,
319	pinned,
320	)
321	.await
322	}
323
324	/// Where a node deploy failed, relative to the symlink swap.
325	///
326	/// The distinction is the whole difference between "nothing happened" and "go
327	/// look at production now", and it used to be carried only in the wording of a
328	/// `.context()` string, which meant the reporting layer could not act on it. It
329	/// reported every rollback failure as though the node were stranded on the new
330	/// version — including the case where the node had never left the old one,
331	/// which is the safe case and the common one.
332	///
333	/// Attached as `anyhow` context, so it both reads correctly in the error chain
334	/// and can be recovered with `stage_of`.
335	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
336	pub enum FailureStage {
337	/// Failed before the swap ran. `current` still points at the old release
338	/// and the service was never restarted, so the node is on the OLD version.
339	/// Nothing is stranded and nothing needs doing.
340	BeforeSwap,
341	/// Failed at or after the swap. The node's version is not knowable from
342	/// here: the swap script rolls `current` back if the restart fails, but a
343	/// failure between the two, or in a companion after the server is already
344	/// live, can leave the node on either version.
345	AtOrAfterSwap,
346	}
347
348	impl std::fmt::Display for FailureStage {
349	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
350	match self {
351	Self::BeforeSwap => {
352	f.write_str("current symlink left intact; node is on the previous version")
353	}
354	Self::AtOrAfterSwap => {
355	f.write_str("the symlink swap had already run; node version is indeterminate")
356	}
357	}
358	}
359	}
360
361	/// Recover the [`FailureStage`] from a deploy error's context chain.
362	///
363	/// `None` means the error predates the stage annotation or came from somewhere
364	/// that does not set one. Callers must treat that as indeterminate rather than
365	/// as safe: guessing "before the swap" would reintroduce the bug in the
366	/// opposite, worse direction.
367	pub fn stage_of(err: &anyhow::Error) -> Option<FailureStage> {
368	// anyhow's own downcast_ref searches attached context values, not just the
369	// source chain, which is where a `.context(FailureStage::…)` lands.
370	err.downcast_ref::<FailureStage>().copied()
371	}
372
373	async fn reset_local_current(
374	executor: &dyn Executor,
375	release_root: &Path,
376	release_id: &str,
377	) -> Result<PathBuf> {
378	let current = release_root.join("current");
379	let target = format!("releases/{release_id}");
380	run_checked(
381	executor,
382	&format!(
383	"ln -sfn {} {}",
384	sh_quote(&target),
385	sh_quote(&current.to_string_lossy())
386	),
387	"local symlink swap",
388	)
389	.await?;
390	Ok(release_root.join("releases").join(release_id))
391	}
392
393	/// Confirm the node is what the topology says it is, before anything is pushed.
394	///
395	/// The declared half of what the pre-swap `ldd` guard does at runtime. That
396	/// guard compares a specific binary against a specific box and is the last line;
397	/// this compares the box against its own declaration and is the first. Both
398	/// exist on purpose: the guard catches a binary nobody declared anything about,
399	/// and this catches a machine that stopped being what the config claims, which
400	/// the guard can only report as a symbol it cannot resolve.
401	///
402	/// A node that declares nothing is not checked. That is a skip and it is logged
403	/// as one, so an unchecked node is visible rather than looking checked. See
404	/// [`ops_core::base_image`] for why this differs from [`Placement::check`].
405	async fn check_node_identity(executor: &dyn Executor, node: &Node) -> Result<()> {
406	if node.base_image.is_none() && node.libc.is_none() {
407	tracing::info!(
408	node = %node.name,
409	"deploy: node declares no base image; identity not checked"
410	);
411	return Ok(());
412	}
413	let out = run_checked(
414	executor,
415	&base_image::probe_cmd(),
416	"asking the node what it is",
417	)
418	.await
419	.context(FailureStage::BeforeSwap)?;
420	let reported = base_image::parse_probe(&String::from_utf8_lossy(&out.stdout));
421	match base_image::check(
422	node.name.as_str(),
423	node.base_image.as_ref(),
424	node.libc.as_deref(),
425	&reported,
426	) {
427	Ok(checked) => {
428	if let Some(what) = checked {
429	tracing::info!(node = %node.name, "deploy: identity checked, {what}");
430	}
431	Ok(())
432	}
433	Err(drift) => Err(anyhow::Error::new(drift))
434	.context("the node is not what the topology declares it to be")
435	.context(FailureStage::BeforeSwap),
436	}
437	}
438
439	/// Refuse a bundle whose glibc floor is above what the node declares, before
440	/// the rsync.
441	///
442	/// The weak, early half of a pair. `ldd_guard_script` runs the node's own loader
443	/// against the actual bytes one step before the symlink swap, which covers every
444	/// shared library and every symbol version rather than glibc alone. Nothing here
445	/// replaces it, and a bundle that passes this can still fail that.
446	///
447	/// What this adds is when. The loader check happens after the bundle is built
448	/// and rsynced; this happens before either, so the subset of failures that two
449	/// declared numbers already prove is refused at the start of the promote instead
450	/// of most of the way through it. The zero-margin state on production makes that
451	/// subset a live one: three of its five binaries sit exactly on the box's glibc,
452	/// so a build host drifting one point release ahead puts every promote here.
453	///
454	/// Skipped, and logged as skipped, in all three cases where there is nothing to
455	/// compare: the node declares no `libc`, the bundle states no floor (static, or
456	/// no ELF this parser reads), or the declared `libc` is not a version string.
457	/// The last is a config typo rather than a bad bundle, and refusing a deploy
458	/// over it would be answering the wrong question loudly.
459	async fn check_bundle_fits_node(node: &Node, staged_release_dir: &Path) -> Result<()> {
460	let Some(declared) = node.libc.as_deref() else {
461	return Ok(());
462	};
463	let Some(node_libc) = crate::elf::GlibcVersion::parse(declared) else {
464	tracing::warn!(
465	node = %node.name,
466	declared,
467	"deploy: node's declared libc is not a version; glibc floor not compared"
468	);
469	return Ok(());
470	};
471	let digest = crate::bundle::digest_dir(staged_release_dir)
472	.await
473	.context("reading the staged bundle's glibc floor")
474	.context(FailureStage::BeforeSwap)?;
475	let Some(floor) = digest.glibc_floor else {
476	tracing::info!(
477	node = %node.name,
478	"deploy: bundle states no glibc floor; nothing to compare"
479	);
480	return Ok(());
481	};
482	if floor > node_libc {
483	return Err(anyhow::anyhow!(
484	"this bundle needs glibc {floor} and `{node}` declares {node_libc}; \
485	refusing to ship a binary the node cannot load. Either the build host \
486	drifted ahead of the node, or the node's declared libc is stale",
487	node = node.name,
488	))
489	.context(FailureStage::BeforeSwap);
490	}
491	tracing::info!(
492	node = %node.name,
493	"deploy: glibc floor {floor} fits the node's {node_libc}"
494	);
495	Ok(())
496	}
497
498	async fn deploy_remote(
499	executor: &dyn Executor,
500	node: &Node,
501	version: &str,
502	release_id: &str,
503	staged_release_dir: &Path,
504	primary_bin: &str,
505	pinned: Option<&PinnedReleases>,
506	) -> Result<PathBuf> {
507	let release_root = &node.release_root;
508	let service = &node.service_name;
509	let release_dir = format!("{release_root}/releases/{release_id}");
510
511	// Identity check first, before a single byte moves. A node that was rebuilt
512	// into something else is refused here rather than after an rsync, and long
513	// before the pre-swap `ldd` guard would have caught the consequence without
514	// naming the cause. Cheap: one shell round-trip that reads /etc/os-release.
515	check_node_identity(executor, node).await?;
516
517	// And that the bundle could load there at all, from two numbers, before the
518	// bytes move. The `ldd` guard below asks the stronger question on the node
519	// itself; this one is only earlier.
520	check_bundle_fits_node(node, staged_release_dir).await?;
521
522	tracing::info!(node = %node.name, version, release_id, "deploy: mkdir release dir");
523	run_checked(
524	executor,
525	&format!("set -e; mkdir -p {q}", q = sh_quote(&release_dir)),
526	"creating remote release dir",
527	)
528	.await
529	.context(FailureStage::BeforeSwap)?;
530
531	tracing::info!(node = %node.name, version, primary = %primary_bin, "deploy: rsync release dir");
532	// Rsync the whole staged dir (binaries + every release_contents entry).
533	// `SyncOpts::release_mirror()` is the exact pre-extraction rsync flag set:
534	// -az --partial --delete --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r,F+X.
535	// --delete: removed assets across versions don't accumulate on the
536	// target; the bundle stays self-contained per version.
537	// --chmod: F+X preserves the execute bit per-file (binaries land 0755,
538	// data files 0644) instead of a blanket 0755.
539	executor
540	.push_dir(
541	staged_release_dir,
542	Path::new(&release_dir),
543	&SyncOpts::release_mirror(),
544	)
545	.await
546	.context("rsync failed")
547	.context(FailureStage::BeforeSwap)?;
548
549	// Verify the bundle on the node against its own MANIFEST before the swap
550	// (invariant 3, wiki [[release-artifact-identity]]). The MANIFEST shipped in
551	// the bundle is exactly `sha256sum` check format (`<hash> <relpath>`), so
552	// this re-hashes every file on the node and names any that drifted in
553	// transit — the hash is load-bearing, not merely recorded. Bundles staged
554	// by a pre-identity build carry no MANIFEST; those skip verification (logged)
555	// rather than fail, so a mid-migration deploy of a legacy artifact still
556	// ships. A mismatch fails the promote with the running service intact.
557	run_checked(
558	executor,
559	&manifest_verify_script(&release_dir),
560	"verifying bundle digest on node",
561	)
562	.await
563	.context("node-side bundle verification failed")
564	.context(FailureStage::BeforeSwap)?;
565
566	// Fail closed on a wrong-architecture binary before the symlink swap. The
567	// "never cross-compile" rule is enforced at build time (build_host check),
568	// but nothing verified the artifact's arch matched the target node — so
569	// adding an aarch64 node to a tier built on x86_64 would silently symlink an
570	// unrunnable binary live. Compare the deployed binary's ELF e_machine to the
571	// node's `uname -m`; unknown arches log and proceed (can't verify != known-bad).
572	let deployed_bin = format!("{release_dir}/{primary_bin}");
573	run_checked(
574	executor,
575	&arch_guard_script(&deployed_bin),
576	"verifying binary arch matches node",
577	)
578	.await
579	.context("deployed binary architecture does not match the target node")
580	.context(FailureStage::BeforeSwap)?;
581
582	// And that the node can actually resolve what the binary links, which the
583	// arch check above cannot see: right architecture, right ELF, and still
584	// unrunnable because it wants a glibc symbol version this box does not have.
585	// Bento used to catch that at build time; under the Sando/Bento boundary the
586	// builder no longer knows which machine runs the bytes, so it lands here.
587	run_checked(
588	executor,
589	&ldd_guard_script(&deployed_bin),
590	"verifying the node can resolve the binary's dynamic dependencies",
591	)
592	.await
593	.context("the target node cannot satisfy the deployed binary's dynamic dependencies")
594	.context(FailureStage::BeforeSwap)?;
595
596	// The same two guards for every companion, and for the same reason. Both
597	// checks above take the primary binary alone, so a companion of the wrong
598	// architecture, or one linking a symbol version this node lacks, used to be
599	// discovered by its unit failing to start — during the install loop below,
600	// which runs AFTER the swap. That is the expensive side of the line these
601	// guards exist to stay on: the promote fails either way, but with the server
602	// already restarted onto the new release.
603	//
604	// The companion bytes are present and verified by now: they arrived in the
605	// same rsync and `manifest_verify_script` above covers the whole release
606	// directory, `companions/` included. So there is nothing to wait for.
607	for c in &node.companions {
608	let src = companion_src(&release_dir, &c.name);
609	run_checked(
610	executor,
611	&arch_guard_script(&src),
612	"verifying companion arch matches node",
613	)
614	.await
615	.with_context(\|\| {
616	format!(
617	"companion {} architecture does not match the target node",
618	c.name
619	)
620	})
621	.context(FailureStage::BeforeSwap)?;
622	run_checked(
623	executor,
624	&ldd_guard_script(&src),
625	"verifying the node can resolve the companion's dynamic dependencies",
626	)
627	.await
628	.with_context(\|\| {
629	format!(
630	"the target node cannot satisfy companion {}'s dynamic dependencies",
631	c.name
632	)
633	})
634	.context(FailureStage::BeforeSwap)?;
635	}
636
637	// Config-drift guard (opt-in per node). Runs the freshly-rsynced binary in
638	// config-only mode with the node's env sourced, BEFORE the swap, so a
639	// required var missing on this node fails here — service still intact —
640	// rather than after the restart, which would crash-loop it (how testnot
641	// went down on a missing CDN_BASE_URL). Skipped unless the node sets
642	// `config_check_env_file`.
643	if let Some(env_file) = node.config_check_env_file.as_deref() {
644	tracing::info!(node = %node.name, version, "deploy: pre-swap config check");
645	check_target_config(executor, &deployed_bin, env_file)
646	.await
647	.context("pre-swap config check failed")
648	.context(FailureStage::BeforeSwap)?;
649	}
650
651	tracing::info!(node = %node.name, version, "deploy: symlink swap + service reload");
652	let restart_cmd = format!(
653	"sudo /bin/systemctl reload-or-restart {}",
654	sh_quote(service)
655	);
656	let swap_and_restart = swap_and_restart_script(release_root, release_id, &restart_cmd);
657	run_checked(
658	executor,
659	&swap_and_restart,
660	"symlink swap + systemctl reload-or-restart",
661	)
662	.await
663	.context(FailureStage::AtOrAfterSwap)?;
664
665	// Companion services (opt-in per node): install each from the just-rsynced
666	// bundle and restart its unit via the node-side wrapper, AFTER the server is
667	// up (mnw-cli is `After=makenotwork.service`). They shipped from the SAME sha
668	// in this SAME bundle — the lockstep guarantee. A failure here fails the
669	// promote: a companion is part of the deploy, not a best-effort side effect.
670	for c in &node.companions {
671	let src = companion_src(&release_dir, &c.name);
672	tracing::info!(node = %node.name, companion = %c.name, "deploy: install companion + restart");
673	let cmd = install_companion_cmd(&src, &c.install_path, &c.service_name);
674	run_checked(executor, &cmd, "install companion + restart")
675	.await
676	.with_context(\|\| {
677	format!(
678	"companion {} deploy failed (server already swapped)",
679	c.name
680	)
681	})
682	.context(FailureStage::AtOrAfterSwap)?;
683	}
684
685	// No pinned set means the caller could not determine what is referenced,
686	// and a gc that cannot tell is the exact failure this parameter exists to
687	// stop. Skipping costs disk on the node; running blind costs the artifact a
688	// rollback resolves to. `finalize_local_release` takes the same position by
689	// refusing to publish at all when the pin query fails.
690	match pinned {
691	Some(pinned) => {
692	if let Err(e) = gc_remote_releases(executor, release_root, pinned).await {
693	tracing::warn!(error = %e, "remote release GC failed (non-fatal)");
694	}
695	}
696	None => tracing::warn!(
697	node = %node.name,
698	"remote release GC skipped: the pinned set is unknown, and a gc that \
699	cannot see what is referenced is what stranded the host store twice"
700	),
701	}
702
703	Ok(PathBuf::from(release_root)
704	.join("releases")
705	.join(release_id))
706	}
707
708	/// Where a companion's binary sits inside the staged release directory.
709	///
710	/// One function because two places need it and they must not drift: the guards
711	/// that run before the swap check this path, and the installer after the swap
712	/// reads it. A guard that checked a path the installer did not use would be a
713	/// check of nothing, and would look exactly like a passing check.
714	fn companion_src(release_dir: &str, name: &str) -> String {
715	format!("{release_dir}/companions/{name}")
716	}
717
718	/// Absolute path of the node-side companion installer (shipped once per node;
719	/// granted to the deploy user by a single scoped sudoers line). It installs the
720	/// staged binary to its `ExecStart` path and restarts the unit — keeping the
721	/// sudo grant to one script rather than a broad `install`/`systemctl` grant.
722	const COMPANION_INSTALLER: &str = "/usr/local/lib/mnw/install-companion.sh";
723
724	/// Command run on the node to install a staged companion binary and restart its
725	/// unit, via the wrapper. Pure builder so it can be unit-tested; all three args
726	/// are shell-quoted (paths/unit names, operator config — but quoted regardless).
727	fn install_companion_cmd(src: &str, install_path: &str, service: &str) -> String {
728	format!(
729	"sudo {installer} {src} {dst} {svc}",
730	installer = sh_quote(COMPANION_INSTALLER),
731	src = sh_quote(src),
732	dst = sh_quote(install_path),
733	svc = sh_quote(service),
734	)
735	}
736
737	/// Pre-swap config-drift check: load the node's env file the way systemd loads
738	/// it, then run the freshly-deployed binary in `MNW_CHECK_CONFIG=1` mode (loads
739	/// config, exits 0/1, no DB/migrations/bind). A non-zero exit — a required var
740	/// missing — is surfaced by `run_checked` as an error, failing the promote
741	/// before the swap.
742	///
743	/// Bounded by a timeout as a backstop: a binary predating `MNW_CHECK_CONFIG`
744	/// would ignore the var and try to start normally, which must not hang the
745	/// deploy. A timeout is reported as a failure (fail closed) — the operator only
746	/// opts a node in once a check-capable version is deployed, so a timeout means
747	/// something is wrong, not a routine older binary.
748	async fn check_target_config(
749	executor: &dyn Executor,
750	deployed_bin: &str,
751	env_file: &str,
752	) -> Result<()> {
753	// Readability first, as its own step with its own message.
754	//
755	// The env file is read by this check AS THE DEPLOY USER, and it is the only
756	// thing that does. systemd loads `EnvironmentFile=` as root before dropping
757	// to `User=`, so the running service does not care about the mode — which
758	// means a file rewritten 0600 breaks the next deploy while the current one
759	// keeps serving, and the breakage is invisible until someone ships. That is
760	// exactly how prod deploy 0.11.3 failed on 2026-08-01.
761	//
762	// Without this step the operator gets `bash: line 9: <file>: Permission
763	// denied` out of a generated script and has to reverse-engineer which user
764	// and which file. Naming the user, the mode and the owner turns that into a
765	// one-line read.
766	let probe = readability_probe_script(env_file);
767	if let Ok(Err(e)) = tokio::time::timeout(
768	std::time::Duration::from_secs(20),
769	run_checked(executor, &probe, "env file readability"),
770	)
771	.await
772	{
773	return Err(e).context(format!(
774	"the deploy user cannot read {env_file}. systemd reads EnvironmentFile= as root, so \
775	the running service is unaffected and this breaks only deploys. Expected mode 0640 \
776	owned root:<service user> (see sando/deploy/bootstrap-node.sh); something that \
777	rewrote the file likely did so with a 077 umask"
778	));
779	}
780
781	let script = config_check_script(env_file, deployed_bin);
782	let fut = run_checked(executor, &script, "pre-swap config check");
783	match tokio::time::timeout(std::time::Duration::from_secs(20), fut).await {
784	Ok(result) => result.map(\|_\| ()),
785	Err(_) => anyhow::bail!(
786	"pre-swap config check timed out after 20s — the binary may predate \
787	MNW_CHECK_CONFIG or the check hung; refusing to swap"
788	),
789	}
790	}
791
792	/// Assert the deploy user can read `env_file`, reporting who it is and what the
793	/// file actually looks like when it cannot.
794	///
795	/// `stat` output is best-effort: a node without it (or a file that does not
796	/// exist) still gets the identity line, which is the half an operator cannot
797	/// derive from the failure on their own.
798	fn readability_probe_script(env_file: &str) -> String {
799	format!(
800	"if [ ! -e {env} ]; then\n\
801	\techo \"{env_disp}: does not exist on this node\" >&2; exit 1\n\
802	fi\n\
803	if [ ! -r {env} ]; then\n\
804	\techo \"cannot read {env_disp} as $(id -un) (groups: $(id -Gn))\" >&2\n\
805	\tstat -c 'actual: mode %a owner %U:%G' {env} >&2 2>/dev/null \|\| true\n\
806	\texit 1\n\
807	fi\n",
808	env = sh_quote(env_file),
809	env_disp = env_file,
810	)
811	}
812
813	/// Shell that loads `env_file` with systemd `EnvironmentFile=` semantics, then
814	/// runs `bin` under `MNW_CHECK_CONFIG=1`.
815	///
816	/// Load the file line by line and `export` each `KEY=VALUE` verbatim rather than
817	/// `. env_file`. Dot-sourcing runs the file as a script, so any shell
818	/// metacharacter in a value (`$`, backticks, `;`, `&`, a glob, whitespace) is
819	/// expanded or word-split — a DB URL carrying a password silently dropped
820	/// `DATABASE_URL` to empty on our nodes, which would fail the check (and thus
821	/// every deploy) even though systemd starts the service fine. `export "$line"`
822	/// assigns the already-expanded word literally, matching systemd's "no variable
823	/// expansion" rule. Comments and blank lines are skipped; the `\|\| [ -n "$line" ]`
824	/// guard processes a final line with no trailing newline. (Quoted values —
825	/// `KEY="v"` — aren't unquoted here the way systemd would, but our env files use
826	/// bare `KEY=VALUE`, and a stray quote can only make the check stricter, never
827	/// wave a bad config through.)
828	fn config_check_script(env_file: &str, bin: &str) -> String {
829	format!(
830	"set -eu\n\
831	while IFS= read -r __sando_l \|\| [ -n \"$__sando_l\" ]; do\n\
832	\tcase \"$__sando_l\" in ''\|'#'*) continue ;; esac\n\
833	\texport \"$__sando_l\"\n\
834	done < {env}\n\
835	MNW_CHECK_CONFIG=1 {bin}\n",
836	env = sh_quote(env_file),
837	bin = sh_quote(bin),
838	)
839	}
840
841	/// Build the swap-and-restart shell script for a remote node.
842	///
843	/// The symlink swap is atomic via `mv -T` of a freshly-created symlink over the
844	/// old one (the rename(2) is the atomic step; `ln -sfn` alone does
845	/// unlink+symlink, which has a window). The load-bearing part: if `restart_cmd`
846	/// fails after the flip, `current` is rolled back to its prior target before
847	/// the script exits non-zero. Otherwise a failed restart would leave `current`
848	/// pointing at the new, un-activated release while the service still runs the
849	/// old one — and a later reboot/cron restart would then silently bring up the
850	/// release the deploy reported as failed. Best-effort re-restart of the prior
851	/// version keeps the running service consistent with the restored symlink.
852	///
853	/// `restart_cmd` is injected (rather than hardcoded) so tests can drive the
854	/// failure and success paths with a `false`/`true` stand-in.
855	fn swap_and_restart_script(release_root: &str, release_id: &str, restart_cmd: &str) -> String {
856	format!(
857	"set -e\n\
858	cd {root}\n\
859	prev=$(readlink current 2>/dev/null \|\| true)\n\
860	ln -sfn releases/{rel} current.new\n\
861	mv -Tf current.new current\n\
862	if ! {restart}; then\n\
863	if [ -n \"$prev\" ]; then\n\
864	ln -sfn \"$prev\" current.rollback\n\
865	mv -Tf current.rollback current\n\
866	{restart} \|\| true\n\
867	fi\n\
868	echo \"deploy: restart failed; rolled symlink back to ${{prev:-<none>}}\" >&2\n\
869	exit 1\n\
870	fi\n",
871	root = sh_quote(release_root),
872	rel = sh_quote(release_id),
873	restart = restart_cmd,
874	)
875	}
876
877	/// Shell that re-hashes the rsynced bundle on the node against its shipped
878	/// `MANIFEST` and aborts (exit 1) if any file drifted (invariant 3, wiki note
879	/// `release-artifact-identity`). The `MANIFEST` is `sha256sum` check format,
880	/// so `sha256sum -c` verifies every listed file with node-native tooling and
881	/// names the one that failed. `--strict` fails on a malformed manifest line;
882	/// `--quiet` drops the per-file OK spam and keeps only failures.
883	///
884	/// A bundle staged by a pre-identity build carries no `MANIFEST`; that is not an
885	/// error — it logs a skip and exits 0, so a mid-migration deploy of a legacy
886	/// artifact still ships. Once every tier has cycled once, every bundle has one.
887	fn manifest_verify_script(release_dir: &str) -> String {
888	format!(
889	"set -e\n\
890	cd {dir}\n\
891	if [ ! -f MANIFEST ]; then\n\
892	echo \"deploy: no MANIFEST in bundle; skipping digest verification (legacy artifact)\" >&2\n\
893	exit 0\n\
894	fi\n\
895	sha256sum --quiet --strict -c MANIFEST\n",
896	dir = sh_quote(release_dir),
897	)
898	}
899
900	/// Shell that aborts (exit 1) if `bin`'s ELF architecture doesn't match the
901	/// node it's running on. Reads the ELF `e_machine` field (2 bytes LE at offset
902	/// 18) and compares it to the value implied by `uname -m`. An arch we don't have
903	/// a mapping for logs and proceeds — the guard exists to catch the concrete
904	/// x86_64-vs-aarch64 confusion, not to gate genuinely-new targets.
905	fn arch_guard_script(bin: &str) -> String {
906	format!(
907	"set -e\n\
908	bin={bin}\n\
909	arch=$(uname -m)\n\
910	machine=$(od -An -tx1 -j18 -N2 \"$bin\" 2>/dev/null \| tr -d ' \\n')\n\
911	case \"$arch\" in\n\
912	x86_64\|amd64) want=3e00 ;;\n\
913	aarch64\|arm64) want=b700 ;;\n\
914	*) echo \"deploy: arch check skipped (unmapped node arch $arch)\" >&2; want= ;;\n\
915	esac\n\
916	if [ -n \"$want\" ] && [ \"$machine\" != \"$want\" ]; then\n\
917	echo \"deploy: arch mismatch — node $arch expects e_machine $want but binary has ${{machine:-<unreadable>}}\" >&2\n\
918	exit 1\n\
919	fi\n",
920	bin = sh_quote(bin),
921	)
922	}
923
924	/// Refuse a binary whose dynamic dependencies the node cannot satisfy, before
925	/// the symlink swap.
926	///
927	/// The sibling of [`arch_guard_script`], and it exists because the boundary took
928	/// the check away from the builder. Bento's recipe used to compare the binary's
929	/// highest `GLIBC_` symbol against `ldd --version` on the service host, which it
930	/// could only do while it held a `[[deploy]]` entry naming that host. A
931	/// handed-off service has none: which machine runs the bytes is environment
932	/// knowledge, which is Sando's half. So the check moves here, where the node and
933	/// the artifact are already in the same value.
934	///
935	/// It asks the stronger question, because here it can. Bento compared two
936	/// version numbers from two machines; this runs the node's own loader against
937	/// the bytes that were just rsynced onto it. That covers every shared library
938	/// and every symbol version, not glibc alone, and it answers "will this exec
939	/// here" rather than "is this number smaller than that one".
940	///
941	/// Three outcomes, and only one of them fails:
942	///
943	/// - `not found` in `ldd` output — a missing library or an unsatisfiable symbol
944	/// version. This is the failure, and it is exactly what would otherwise be
945	/// discovered by the unit failing to start after the swap.
946	/// - not a dynamic executable — `ldd` exits non-zero and says so. A static
947	/// binary has nothing to resolve, so it passes.
948	/// - no `ldd` on the node — nothing to check with. Logged and passed: "cannot
949	/// verify" is not "known bad", the same call `arch_guard_script` makes for an
950	/// unmapped arch.
951	///
952	/// `ldd` runs the loader, which for an arbitrary binary is code execution. These
953	/// bytes are ours, already verified against their MANIFEST on this node, and
954	/// about to be exec'd by the service unit a second later.
955	fn ldd_guard_script(bin: &str) -> String {
956	format!(
957	"set -e\n\
958	bin={bin}\n\
959	command -v ldd >/dev/null 2>&1 \|\| {{ echo \"deploy: ldd check skipped (no ldd on node)\" >&2; exit 0; }}\n\
960	out=$(ldd \"$bin\" 2>&1) \|\| {{ \n\
961	case \"$out\" in\n\
962	\"not a dynamic executable\") echo \"deploy: ldd check passed (static binary)\" >&2; exit 0 ;;\n\
963	*) echo \"deploy: ldd failed on $bin: $out\" >&2; exit 1 ;;\n\
964	esac\n\
965	}}\n\
966	if printf '%s' \"$out\" \| grep -q 'not found'; then\n\
967	echo \"deploy: this node cannot satisfy the binary's dynamic dependencies:\" >&2\n\
968	printf '%s\\n' \"$out\" \| grep 'not found' >&2\n\
969	exit 1\n\
970	fi\n",
971	bin = sh_quote(bin),
972	)
973	}
974
975	/// Trim `releases/` to the pinned set plus the [`RELEASES_TO_KEEP`] newest of
976	/// what is left.
977	///
978	/// Pinning is applied before the count, so a referenced artifact cannot be aged
979	/// out by rebuilds of a newer version — the failure that stranded production
980	/// twice. See [`crate::retention`].
981	async fn gc_local_releases(release_root: &Path, pinned: &PinnedReleases) -> Result<()> {
982	let releases = release_root.join("releases");
983	if !releases.exists() {
984	return Ok(());
985	}
986	let mut entries = Vec::new();
987	let mut rd = tokio::fs::read_dir(&releases).await?;
988	while let Some(entry) = rd.next_entry().await? {
989	if !entry.file_type().await?.is_dir() {
990	continue;
991	}
992	// Set aside before anything is ordered or counted: a pinned dir is not
993	// a candidate, so it can never occupy one of the count's slots either.
994	if entry
995	.file_name()
996	.to_str()
997	.is_some_and(\|n\| pinned.contains(n))
998	{
999	continue;
1000	}
1001	let meta = entry.metadata().await?;
1002	entries.push((entry.path(), meta.modified()?));
1003	}
1004	entries.sort_by_key(\|e\| std::cmp::Reverse(e.1));
1005	for (path, _) in entries.into_iter().skip(RELEASES_TO_KEEP) {
1006	if let Err(e) = tokio::fs::remove_dir_all(&path).await {
1007	tracing::warn!(path = %path.display(), error = %e, "gc: rm failed");
1008	} else {
1009	tracing::debug!(path = %path.display(), "gc: removed old release");
1010	}
1011	}
1012	Ok(())
1013	}
1014
1015	/// Trim a node's `releases/` to the pinned set plus the [`RELEASES_TO_KEEP`]
1016	/// newest of what is left.
1017	///
1018	/// The node mirrors the host's directory name (see [`deploy_node`]), so the same
1019	/// [`PinnedReleases`] the host gc subtracts is the right set here — nothing has
1020	/// to be recomputed per node.
1021	///
1022	/// Weaker instance of the host defect: every promote and rollback rsyncs from
1023	/// the host store, so a node directory evicted here is re-pushed rather than
1024	/// lost. What it costs is a full rsync of a bundle the node already had, at the
1025	/// worst moment — mid-rollback, with a tier already failing.
1026	///
1027	/// Pinning is applied before the count, matching [`gc_local_releases`]: a pinned
1028	/// directory is not a candidate, so it cannot occupy one of the count's slots.
1029	async fn gc_remote_releases(
1030	executor: &dyn Executor,
1031	release_root: &str,
1032	pinned: &PinnedReleases,
1033	) -> Result<()> {
1034	run_checked(
1035	executor,
1036	&remote_gc_script(release_root, pinned),
1037	"remote release gc",
1038	)
1039	.await
1040	.map(\|_\| ())
1041	}
1042
1043	/// The remote gc as shell.
1044	///
1045	/// Split out so the script is testable without an executor: it is the half of
1046	/// this that can be wrong in a way `rm -rf` makes expensive.
1047	///
1048	/// `ls -1t` orders by mtime desc. Pinned names are carried in the positional
1049	/// parameters rather than a here-doc piped through `grep -v`, because `grep`
1050	/// exits 1 when it selects no lines — which happens on both edges that matter
1051	/// (nothing pinned, or everything pinned) and would abort the script under
1052	/// `set -e` for the two cases that are perfectly normal. Comparing with `case`
1053	/// has no exit status to trip over, and matches whole names rather than
1054	/// substrings, which a `grep -F` without `-x` would not.
1055	fn remote_gc_script(release_root: &str, pinned: &PinnedReleases) -> String {
1056	let pins = pinned
1057	.sorted_names()
1058	.into_iter()
1059	.map(sh_quote)
1060	.collect::<Vec<_>>()
1061	.join(" ");
1062	// `set --` with no operands unsets the positional parameters, which is
1063	// exactly what a nothing-pinned gc wants: the `for` below then iterates zero
1064	// times and every directory is a candidate. Written as one branch because
1065	// `set -- ` with an empty expansion is the same statement.
1066	let set_pins = format!("set -- {pins}");
1067	format!(
1068	"set -e; cd {root}/releases 2>/dev/null \|\| exit 0; \
1069	{set_pins}; \
1070	n=0; \
1071	ls -1t \| while IFS= read -r d; do \
1072	for p in \"$@\"; do \
1073	if [ \"$d\" = \"$p\" ]; then continue 2; fi; \
1074	done; \
1075	n=$((n+1)); \
1076	if [ \"$n\" -le {keep} ]; then continue; fi; \
1077	rm -rf -- \"$d\"; \
1078	done",
1079	root = sh_quote(release_root),
1080	keep = RELEASES_TO_KEEP,
1081	)
1082	}
1083
1084	#[cfg(test)]
1085	mod tests {
1086	use super::*;
1087
1088	/// Nothing deployed, so nothing pinned: the tests that exercise the count
1089	/// alone pass this, and the ones that exercise pinning build their own set.
1090	fn no_pins() -> PinnedReleases {
1091	PinnedReleases::none()
1092	}
1093
1094	use crate::topology::NodeCompanion;
1095	use ops_exec::{CapabilitySet, LocalExec, SshExec};
1096	use std::os::unix::process::ExitStatusExt;
1097	use std::sync::{Arc, Mutex as StdMutex};
1098	use std::time::SystemTime;
1099
1100	// ---- placement ----
1101	//
1102	// The whole table, because the interesting cases are the two where one side
1103	// said nothing. Treating silence as agreement is how a wrong-architecture
1104	// deploy would get through, and it is the shape a "check it before you call"
1105	// guard tends to end up with.
1106
1107	fn node_on(platform: Option<&str>) -> Node {
1108	Node {
1109	name: crate::domain::NodeId::new("n1"),
1110	ssh_target: "deploy@n1".into(),
1111	release_root: "/opt/x".into(),
1112	platform: platform.map(\|p\| Platform::parse(p).unwrap()),
1113	base_image: None,
1114	libc: None,
1115	service_name: "x.service".into(),
1116	config_check_env_file: None,
1117	actuate: crate::topology::default_actuate(),
1118	observe: crate::topology::default_observe(),
1119	health_url: None,
1120	companions: Vec::new(),
1121	}
1122	}
1123
1124	/// A node that declares a glibc older than the bundle needs is refused
1125	/// before the rsync, and the message names both numbers so the operator
1126	/// knows which side to fix.
1127	#[tokio::test]
1128	async fn a_bundle_above_the_node_s_declared_glibc_is_refused_before_the_rsync() {
1129	let dir = tempfile::tempdir().unwrap();
1130	let exe = std::fs::read(std::env::current_exe().unwrap()).unwrap();
1131	std::fs::write(dir.path().join("bin"), &exe).unwrap();
1132	let Some(floor) = crate::elf::glibc_floor(&exe) else {
1133	return; // a static test binary states no floor; nothing to compare
1134	};
1135
1136	let mut node = node_on(None);
1137	node.libc = Some("2.0".into()); // older than anything real
1138	let err = check_bundle_fits_node(&node, dir.path())
1139	.await
1140	.expect_err("a bundle above the node's glibc must be refused");
1141	// `{:#}` walks the context chain: the outermost context is the
1142	// `FailureStage`, whose Display is the operator-facing "nothing moved"
1143	// line, and the cause below it is the reason.
1144	let msg = format!("{err:#}");
1145	assert!(
1146	msg.contains(&floor.to_string()) && msg.contains("2.0"),
1147	"the refusal must name both numbers: {msg}"
1148	);
1149	assert_eq!(
1150	stage_of(&err),
1151	Some(FailureStage::BeforeSwap),
1152	"refusing here must be recoverable: nothing has moved yet"
1153	);
1154	}
1155
1156	#[tokio::test]
1157	async fn a_bundle_within_the_node_s_declared_glibc_passes() {
1158	let dir = tempfile::tempdir().unwrap();
1159	let exe = std::fs::read(std::env::current_exe().unwrap()).unwrap();
1160	std::fs::write(dir.path().join("bin"), &exe).unwrap();
1161
1162	let mut node = node_on(None);
1163	node.libc = Some("99.0".into()); // newer than anything real
1164	check_bundle_fits_node(&node, dir.path())
1165	.await
1166	.expect("a bundle the node can load must pass");
1167	}
1168
1169	/// The three ways there is nothing to compare. All three pass, because
1170	/// "cannot verify" is not "known bad" — the same call `arch_guard_script`
1171	/// makes for an unmapped architecture.
1172	#[tokio::test]
1173	async fn nothing_to_compare_is_a_pass_not_a_refusal() {
1174	let dir = tempfile::tempdir().unwrap();
1175	let exe = std::fs::read(std::env::current_exe().unwrap()).unwrap();
1176	std::fs::write(dir.path().join("bin"), &exe).unwrap();
1177
1178	// 1. The node declares no libc.
1179	let node = node_on(None);
1180	check_bundle_fits_node(&node, dir.path()).await.unwrap();
1181
1182	// 2. The node's declared libc is not a version (a config typo).
1183	let mut typo = node_on(None);
1184	typo.libc = Some("noble".into());
1185	check_bundle_fits_node(&typo, dir.path()).await.unwrap();
1186
1187	// 3. The bundle holds no ELF, so it states no floor.
1188	let empty = tempfile::tempdir().unwrap();
1189	std::fs::write(empty.path().join("style.css"), b"body{}").unwrap();
1190	let mut strict = node_on(None);
1191	strict.libc = Some("2.0".into());
1192	check_bundle_fits_node(&strict, empty.path())
1193	.await
1194	.expect("a bundle with no binaries has no floor to exceed");
1195	}
1196
1197	#[test]
1198	fn matching_platforms_are_placeable() {
1199	let node = node_on(Some("linux/aarch64"));
1200	let art = Platform::parse("linux/aarch64").unwrap();
1201	let p = Placement::check(&node, Path::new("/r/abc"), Some(&art)).expect("a match places");
1202	assert_eq!(p.bundle(), Path::new("/r/abc"));
1203	assert_eq!(p.node().name.as_str(), "n1");
1204	}
1205
1206	#[test]
1207	fn a_different_architecture_is_refused() {
1208	// The failure this type exists for: pom's aarch64 bundle reaching the
1209	// x86_64 box, which execs nothing and takes the watcher down.
1210	let node = node_on(Some("linux/x86_64"));
1211	let art = Platform::parse("linux/aarch64").unwrap();
1212	let err = Placement::check(&node, Path::new("/r/abc"), Some(&art)).unwrap_err();
1213	assert!(
1214	matches!(err, PlacementError::Mismatch { .. }),
1215	"expected a mismatch, got {err}"
1216	);
1217	// The message has to name both, or an operator cannot tell which half
1218	// is wrong.
1219	let msg = err.to_string();
1220	assert!(
1221	msg.contains("linux/x86_64") && msg.contains("linux/aarch64"),
1222	"{msg}"
1223	);
1224	}
1225
1226	#[test]
1227	fn a_silent_node_refuses_a_stated_artifact() {
1228	// Not "the node probably runs it". A node that never said what it is
1229	// cannot vouch for a bundle that did, and the pairing that looks
1230	// harmless here is exactly the one that ships the wrong half of a
1231	// two-architecture release.
1232	let node = node_on(None);
1233	let art = Platform::parse("linux/aarch64").unwrap();
1234	assert!(matches!(
1235	Placement::check(&node, Path::new("/r/abc"), Some(&art)),
1236	Err(PlacementError::NodeSilent { .. })
1237	));
1238	}
1239
1240	#[test]
1241	fn a_stated_node_refuses_a_silent_artifact() {
1242	let node = node_on(Some("linux/aarch64"));
1243	assert!(matches!(
1244	Placement::check(&node, Path::new("/r/abc"), None),
1245	Err(PlacementError::ArtifactSilent { .. })
1246	));
1247	}
1248
1249	#[test]
1250	fn both_silent_is_the_single_platform_world_and_still_places() {
1251	// MNW is here and stays here. Its nodes declare nothing and its builds
1252	// record nothing, which is the truth about a product with one build host
1253	// and one architecture. The moment either side starts stating, the other
1254	// has to as well — that is the forcing function, and it is why this cell
1255	// is the only admissible non-match.
1256	let node = node_on(None);
1257	Placement::check(&node, Path::new("/r/abc"), None).expect("the pre-pom world still ships");
1258	}
1259
1260	#[test]
1261	fn platform_parsing_is_a_shape_not_a_spelling() {
1262	assert_eq!(
1263	Platform::parse("Linux/AArch64").unwrap(),
1264	Platform::parse("linux/aarch64").unwrap(),
1265	"case is not a distinction between two machines"
1266	);
1267	for bad in ["linux", "linux/", "/aarch64", "linux/aarch64/gnu", ""] {
1268	assert!(Platform::parse(bad).is_err(), "{bad:?} should not parse");
1269	}
1270	}
1271
1272	// ---- failure stage ----
1273	//
1274	// The 2026-08-01 prod deploy failed its pre-swap config check, and the
1275	// rollback then failed the same way — which left the node safely on the old
1276	// version, and was reported as "it remains on the new version, manual
1277	// intervention needed". These pin the distinction the reporting layer now
1278	// depends on.
1279
1280	#[test]
1281	fn a_pre_swap_failure_is_recoverable_as_such() {
1282	let e = anyhow::anyhow!("Permission denied")
1283	.context("pre-swap config check failed")
1284	.context(FailureStage::BeforeSwap);
1285	assert_eq!(stage_of(&e), Some(FailureStage::BeforeSwap));
1286	// The reason survives alongside the stage; the stage does not replace it.
1287	let rendered = format!("{e:#}");
1288	assert!(
1289	rendered.contains("pre-swap config check failed"),
1290	"{rendered}"
1291	);
1292	assert!(rendered.contains("Permission denied"), "{rendered}");
1293	}
1294
1295	#[test]
1296	fn a_post_swap_failure_is_recoverable_as_such() {
1297	let e = anyhow::anyhow!("unit failed to start")
1298	.context("companion x deploy failed (server already swapped)")
1299	.context(FailureStage::AtOrAfterSwap);
1300	assert_eq!(stage_of(&e), Some(FailureStage::AtOrAfterSwap));
1301	}
1302
1303	#[test]
1304	fn an_unannotated_failure_has_no_stage() {
1305	// Must be None, not a default. A caller seeing None has to treat the
1306	// node as indeterminate; inferring "before the swap" would reintroduce
1307	// the original bug pointing the other way, which is the dangerous way.
1308	let e = anyhow::anyhow!("something older, from before stages existed");
1309	assert_eq!(stage_of(&e), None);
1310	}
1311
1312	// ---- env file readability probe ----
1313
1314	#[tokio::test]
1315	async fn readability_probe_passes_on_a_readable_file() {
1316	let tmp = tempfile::tempdir().unwrap();
1317	let f = tmp.path().join("ok.env");
1318	tokio::fs::write(&f, "A=1\n").await.unwrap();
1319	let script = readability_probe_script(&f.to_string_lossy());
1320	let out = run_checked(&local_executor(), &script, "probe").await;
1321	assert!(out.is_ok(), "{:?}", out.err().map(\|e\| format!("{e:#}")));
1322	}
1323
1324	#[tokio::test]
1325	async fn readability_probe_names_the_user_and_mode_when_unreadable() {
1326	// Root can read anything, so a mode-based test would pass spuriously
1327	// there. Skip rather than assert something false. No libc dependency
1328	// for one probe: a 0-mode temp file is readable iff we are root.
1329	let probe_dir = tempfile::tempdir().unwrap();
1330	let probe_file = probe_dir.path().join("root-check");
1331	tokio::fs::write(&probe_file, "x").await.unwrap();
1332	tokio::fs::set_permissions(
1333	&probe_file,
1334	std::os::unix::fs::PermissionsExt::from_mode(0o000),
1335	)
1336	.await
1337	.unwrap();
1338	if tokio::fs::read(&probe_file).await.is_ok() {
1339	return; // running as root
1340	}
1341	let tmp = tempfile::tempdir().unwrap();
1342	let f = tmp.path().join("locked.env");
1343	tokio::fs::write(&f, "A=1\n").await.unwrap();
1344	tokio::fs::set_permissions(&f, std::os::unix::fs::PermissionsExt::from_mode(0o000))
1345	.await
1346	.unwrap();
1347
1348	let script = readability_probe_script(&f.to_string_lossy());
1349	let err = run_checked(&local_executor(), &script, "probe")
1350	.await
1351	.expect_err("an unreadable file must fail the probe");
1352	let msg = format!("{err:#}");
1353	// The two things the raw bash error does not tell you.
1354	assert!(msg.contains("cannot read"), "{msg}");
1355	assert!(msg.contains("mode 0") \|\| msg.contains("mode "), "{msg}");
1356	}
1357
1358	#[tokio::test]
1359	async fn readability_probe_distinguishes_missing_from_unreadable() {
1360	let tmp = tempfile::tempdir().unwrap();
1361	let missing = tmp.path().join("nope.env");
1362	let script = readability_probe_script(&missing.to_string_lossy());
1363	let err = run_checked(&local_executor(), &script, "probe")
1364	.await
1365	.expect_err("a missing file must fail the probe");
1366	let msg = format!("{err:#}");
1367	assert!(msg.contains("does not exist"), "{msg}");
1368	}
1369
1370	#[test]
1371	fn the_two_stages_read_differently() {
1372	// These strings end up in an operator's terminal during an incident.
1373	let before = FailureStage::BeforeSwap.to_string();
1374	let after = FailureStage::AtOrAfterSwap.to_string();
1375	assert!(before.contains("previous version"), "{before}");
1376	assert!(after.contains("indeterminate"), "{after}");
1377	assert_ne!(before, after);
1378	}
1379
1380	/// A LocalExec granted the default node capabilities (deploy + restart).
1381	fn local_executor() -> LocalExec {
1382	LocalExec::new(CapabilitySet::from_tokens(
1383	["deploy", "restart"],
1384	["health"],
1385	))
1386	}
1387
1388	#[tokio::test]
1389	async fn deploy_local_copies_multiple_binaries_and_swaps_symlink() {
1390	let tmp = tempfile::tempdir().unwrap();
1391	let root = tmp.path();
1392
1393	let src_dir = root.join("src");
1394	tokio::fs::create_dir_all(&src_dir).await.unwrap();
1395	let primary = src_dir.join("makenotwork");
1396	let admin = src_dir.join("mnw-admin");
1397	tokio::fs::write(&primary, b"PRIMARY").await.unwrap();
1398	tokio::fs::write(&admin, b"ADMIN").await.unwrap();
1399
1400	let release_root = root.join("releases-root");
1401	tokio::fs::create_dir_all(&release_root).await.unwrap();
1402
1403	// Stage into staging/<build_id> (no publish yet).
1404	let staging = stage_local_bundle(&release_root, 42, &[primary.clone(), admin.clone()])
1405	.await
1406	.expect("stage_local_bundle should succeed");
1407	assert_eq!(staging, release_root.join("staging").join("42"));
1408	assert!(
1409	!release_root.join("current").exists(),
1410	"staging must not publish or flip current"
1411	);
1412
1413	// Publish content-addressed at releases/<digest16>.
1414	let released =
1415	finalize_local_release(&release_root, &staging, "deadbeefcafe0000", &no_pins())
1416	.await
1417	.expect("finalize_local_release should succeed");
1418	assert_eq!(
1419	released,
1420	release_root.join("releases").join("deadbeefcafe0000")
1421	);
1422	assert!(
1423	!staging.exists(),
1424	"staging dir is consumed by the publish rename"
1425	);
1426	assert_eq!(
1427	tokio::fs::read(released.join("makenotwork")).await.unwrap(),
1428	b"PRIMARY"
1429	);
1430	assert_eq!(
1431	tokio::fs::read(released.join("mnw-admin")).await.unwrap(),
1432	b"ADMIN"
1433	);
1434
1435	let current = release_root.join("current");
1436	let target = tokio::fs::read_link(&current).await.unwrap();
1437	assert_eq!(target.to_string_lossy(), "releases/deadbeefcafe0000");
1438	let via_current = tokio::fs::read(current.join("makenotwork")).await.unwrap();
1439	assert_eq!(via_current, b"PRIMARY");
1440	}
1441
1442	#[tokio::test]
1443	async fn finalize_second_release_swaps_symlink_and_keeps_old_dir() {
1444	let tmp = tempfile::tempdir().unwrap();
1445	let root = tmp.path();
1446	let src_dir = root.join("src");
1447	tokio::fs::create_dir_all(&src_dir).await.unwrap();
1448	let bin = src_dir.join("server");
1449	tokio::fs::write(&bin, b"V1").await.unwrap();
1450
1451	let release_root = root.join("rr");
1452	tokio::fs::create_dir_all(&release_root).await.unwrap();
1453
1454	// Two builds, distinct digests (distinct content) -> two release dirs.
1455	let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1456	.await
1457	.unwrap();
1458	finalize_local_release(&release_root, &s1, "1111111111111111", &no_pins())
1459	.await
1460	.unwrap();
1461	tokio::fs::write(&bin, b"V2").await.unwrap();
1462	let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1463	.await
1464	.unwrap();
1465	finalize_local_release(&release_root, &s2, "2222222222222222", &no_pins())
1466	.await
1467	.unwrap();
1468
1469	assert!(
1470	release_root
1471	.join("releases/1111111111111111/server")
1472	.exists()
1473	);
1474	assert!(
1475	release_root
1476	.join("releases/2222222222222222/server")
1477	.exists()
1478	);
1479	let target = tokio::fs::read_link(release_root.join("current"))
1480	.await
1481	.unwrap();
1482	assert_eq!(target.to_string_lossy(), "releases/2222222222222222");
1483	let via_current = tokio::fs::read(release_root.join("current/server"))
1484	.await
1485	.unwrap();
1486	assert_eq!(via_current, b"V2");
1487	}
1488
1489	#[tokio::test]
1490	async fn finalize_reuses_an_existing_release_of_the_same_digest() {
1491	let tmp = tempfile::tempdir().unwrap();
1492	let root = tmp.path();
1493	let bin = root.join("server");
1494	tokio::fs::write(&bin, b"BYTES").await.unwrap();
1495	let release_root = root.join("rr");
1496	tokio::fs::create_dir_all(&release_root).await.unwrap();
1497
1498	let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1499	.await
1500	.unwrap();
1501	finalize_local_release(&release_root, &s1, "abc123abc123abc1", &no_pins())
1502	.await
1503	.unwrap();
1504	// Same digest rebuilt (e.g. a re-run at the same content): finalize must
1505	// reuse the existing release and drop the redundant staging dir, not error.
1506	let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1507	.await
1508	.unwrap();
1509	let released = finalize_local_release(&release_root, &s2, "abc123abc123abc1", &no_pins())
1510	.await
1511	.expect("finalize is idempotent on a repeated digest");
1512	assert_eq!(released, release_root.join("releases/abc123abc123abc1"));
1513	assert!(!s2.exists(), "redundant staging dropped");
1514	}
1515
1516	#[tokio::test]
1517	async fn manifest_verify_script_passes_on_match_fails_on_drift_and_skips_when_absent() {
1518	// The node-side verification is a shell running `sha256sum -c MANIFEST`;
1519	// drive the real script through bash to prove it accepts a good bundle,
1520	// rejects a tampered one, and no-ops on a legacy (MANIFEST-less) bundle.
1521	let dir = tempfile::tempdir().unwrap();
1522	tokio::fs::write(dir.path().join("server"), b"BINARY")
1523	.await
1524	.unwrap();
1525	tokio::fs::create_dir(dir.path().join("static"))
1526	.await
1527	.unwrap();
1528	tokio::fs::write(dir.path().join("static/app.css"), b"body{}")
1529	.await
1530	.unwrap();
1531	let digest = crate::bundle::digest_dir(dir.path()).await.unwrap();
1532	tokio::fs::write(dir.path().join("MANIFEST"), digest.manifest.as_bytes())
1533	.await
1534	.unwrap();
1535
1536	let run = \|d: &std::path::Path\| {
1537	let script = manifest_verify_script(d.to_str().unwrap());
1538	async move {
1539	Command::new("bash")
1540	.arg("-c")
1541	.arg(&script)
1542	.output()
1543	.await
1544	.unwrap()
1545	}
1546	};
1547
1548	let ok = run(dir.path()).await;
1549	assert!(
1550	ok.status.success(),
1551	"matching bundle verifies: {}",
1552	String::from_utf8_lossy(&ok.stderr)
1553	);
1554
1555	// Drift one file: sha256sum -c must fail (current symlink left intact).
1556	tokio::fs::write(dir.path().join("static/app.css"), b"TAMPERED")
1557	.await
1558	.unwrap();
1559	let bad = run(dir.path()).await;
1560	assert!(!bad.status.success(), "a drifted file fails verification");
1561
1562	// Legacy bundle with no MANIFEST: skip, not fail.
1563	let legacy = tempfile::tempdir().unwrap();
1564	tokio::fs::write(legacy.path().join("server"), b"x")
1565	.await
1566	.unwrap();
1567	let skip = run(legacy.path()).await;
1568	assert!(
1569	skip.status.success(),
1570	"a bundle without a MANIFEST skips verification rather than failing"
1571	);
1572	}
1573
1574	#[tokio::test]
1575	async fn gc_local_releases_keeps_last_n_by_mtime() {
1576	let tmp = tempfile::tempdir().unwrap();
1577	let root = tmp.path();
1578	let releases = root.join("releases");
1579	tokio::fs::create_dir_all(&releases).await.unwrap();
1580
1581	let total = RELEASES_TO_KEEP + 3;
1582	let mut names = Vec::new();
1583	for i in 0..total {
1584	let name = format!("v{i:02}");
1585	let dir = releases.join(&name);
1586	tokio::fs::create_dir(&dir).await.unwrap();
1587	let f = std::fs::File::open(&dir).unwrap();
1588	let when =
1589	SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
1590	let times = std::fs::FileTimes::new().set_modified(when);
1591	f.set_times(times).unwrap();
1592	names.push(name);
1593	}
1594
1595	gc_local_releases(root, &no_pins()).await.unwrap();
1596
1597	let surviving_expected: Vec<_> = names
1598	.iter()
1599	.skip(total - RELEASES_TO_KEEP)
1600	.cloned()
1601	.collect();
1602	for name in &surviving_expected {
1603	assert!(releases.join(name).exists(), "expected to survive: {name}");
1604	}
1605	for name in names.iter().take(total - RELEASES_TO_KEEP) {
1606	assert!(
1607	!releases.join(name).exists(),
1608	"expected to be pruned: {name}"
1609	);
1610	}
1611	}
1612
1613	#[tokio::test]
1614	async fn gc_local_releases_never_evicts_a_pinned_dir() {
1615	// The 2026-08-25 shape exactly: the oldest dir is the one production is
1616	// running, and enough newer rebuilds exist to push it past the count.
1617	// Under the count alone it was the first thing deleted.
1618	let tmp = tempfile::tempdir().unwrap();
1619	let root = tmp.path();
1620	let releases = root.join("releases");
1621	tokio::fs::create_dir_all(&releases).await.unwrap();
1622
1623	let total = RELEASES_TO_KEEP + 3;
1624	let mut names = Vec::new();
1625	for i in 0..total {
1626	let name = format!("v{i:02}");
1627	let dir = releases.join(&name);
1628	tokio::fs::create_dir(&dir).await.unwrap();
1629	let f = std::fs::File::open(&dir).unwrap();
1630	let when =
1631	SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
1632	f.set_times(std::fs::FileTimes::new().set_modified(when))
1633	.unwrap();
1634	names.push(name);
1635	}
1636
1637	// The two oldest: what a tier is running and what it would roll back to.
1638	let pinned: PinnedReleases = [names[0].clone(), names[1].clone()].into_iter().collect();
1639	gc_local_releases(root, &pinned).await.unwrap();
1640
1641	for name in [&names[0], &names[1]] {
1642	assert!(
1643	releases.join(name).exists(),
1644	"a referenced artifact was evicted: {name}"
1645	);
1646	}
1647	// And the count still applies to everything else, from a floor that the
1648	// pinned pair did not eat into: the newest RELEASES_TO_KEEP unpinned
1649	// dirs survive, so pinning two costs two extra slots rather than two of
1650	// the five.
1651	let unpinned: Vec<_> = names.iter().filter(\|n\| !pinned.contains(n)).collect();
1652	let cut = unpinned.len() - RELEASES_TO_KEEP;
1653	for name in unpinned.iter().take(cut) {
1654	assert!(
1655	!releases.join(name).exists(),
1656	"expected to be pruned: {name}"
1657	);
1658	}
1659	for name in unpinned.iter().skip(cut) {
1660	assert!(releases.join(name).exists(), "expected to survive: {name}");
1661	}
1662	}
1663
1664	#[tokio::test]
1665	async fn gc_local_releases_keeps_a_pinned_dir_that_is_not_even_present() {
1666	// A pinned name with nothing on disk must not disturb the count. This is
1667	// the state the bug leaves behind, and gc runs again while it holds.
1668	let tmp = tempfile::tempdir().unwrap();
1669	let root = tmp.path();
1670	let releases = root.join("releases");
1671	tokio::fs::create_dir_all(&releases).await.unwrap();
1672	for i in 0..=RELEASES_TO_KEEP {
1673	tokio::fs::create_dir(releases.join(format!("v{i}")))
1674	.await
1675	.unwrap();
1676	}
1677	let pinned: PinnedReleases = ["gone-already".to_string()].into_iter().collect();
1678	gc_local_releases(root, &pinned).await.unwrap();
1679
1680	let left = std::fs::read_dir(&releases).unwrap().count();
1681	assert_eq!(left, RELEASES_TO_KEEP);
1682	}
1683
1684	#[tokio::test]
1685	async fn gc_local_releases_noop_when_below_threshold() {
1686	let tmp = tempfile::tempdir().unwrap();
1687	let root = tmp.path();
1688	let releases = root.join("releases");
1689	tokio::fs::create_dir_all(&releases).await.unwrap();
1690	for i in 0..3 {
1691	tokio::fs::create_dir(releases.join(format!("v{i}")))
1692	.await
1693	.unwrap();
1694	}
1695	gc_local_releases(root, &no_pins()).await.unwrap();
1696	for i in 0..3 {
1697	assert!(releases.join(format!("v{i}")).exists());
1698	}
1699	}
1700
1701	// ---- remote gc ----
1702	//
1703	// Driven through `LocalExec`, so these run the real shell the node runs
1704	// rather than asserting on the script's text. The script is the half of the
1705	// remote gc that can be wrong, and it is wrong with `rm -rf`.
1706
1707	/// `releases/` with `total` dirs named `v00..`, oldest first by mtime.
1708	async fn releases_by_age(root: &Path, total: usize) -> Vec<String> {
1709	let releases = root.join("releases");
1710	tokio::fs::create_dir_all(&releases).await.unwrap();
1711	let mut names = Vec::new();
1712	for i in 0..total {
1713	let name = format!("v{i:02}");
1714	let dir = releases.join(&name);
1715	tokio::fs::create_dir(&dir).await.unwrap();
1716	// A file inside, so a deletion is visible as more than an empty dir.
1717	tokio::fs::write(dir.join("makenotwork"), b"x")
1718	.await
1719	.unwrap();
1720	let f = std::fs::File::open(&dir).unwrap();
1721	let when =
1722	SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
1723	f.set_times(std::fs::FileTimes::new().set_modified(when))
1724	.unwrap();
1725	names.push(name);
1726	}
1727	names
1728	}
1729
1730	/// Parity with the count-only script this replaced: nothing pinned, newest
1731	/// `RELEASES_TO_KEEP` survive. If this drifts the change was not a
1732	/// refinement of the old behaviour but a replacement of it.
1733	#[tokio::test]
1734	async fn gc_remote_releases_keeps_last_n_by_mtime_when_nothing_is_pinned() {
1735	let tmp = tempfile::tempdir().unwrap();
1736	let root = tmp.path();
1737	let total = RELEASES_TO_KEEP + 3;
1738	let names = releases_by_age(root, total).await;
1739
1740	gc_remote_releases(&local_executor(), root.to_str().unwrap(), &no_pins())
1741	.await
1742	.unwrap();
1743
1744	let releases = root.join("releases");
1745	for name in names.iter().take(total - RELEASES_TO_KEEP) {
1746	assert!(!releases.join(name).exists(), "expected pruned: {name}");
1747	}
1748	for name in names.iter().skip(total - RELEASES_TO_KEEP) {
1749	assert!(releases.join(name).exists(), "expected to survive: {name}");
1750	}
1751	}
1752
1753	/// The done condition, on the node: the dirs a tier's current and previous
1754	/// artifacts name survive even when they are the oldest on disk and well
1755	/// past the count. Same shape as the host-store test above, which is the
1756	/// point — the two stores now answer the same question the same way.
1757	#[tokio::test]
1758	async fn gc_remote_releases_never_evicts_a_pinned_dir() {
1759	let tmp = tempfile::tempdir().unwrap();
1760	let root = tmp.path();
1761	let total = RELEASES_TO_KEEP + 3;
1762	let names = releases_by_age(root, total).await;
1763
1764	let pinned: PinnedReleases = [names[0].clone(), names[1].clone()].into_iter().collect();
1765	gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
1766	.await
1767	.unwrap();
1768
1769	let releases = root.join("releases");
1770	for name in [&names[0], &names[1]] {
1771	assert!(
1772	releases.join(name).exists(),
1773	"a referenced artifact was evicted from the node: {name}"
1774	);
1775	}
1776	// And pinning does not spend the count's slots, again matching the host.
1777	let unpinned: Vec<_> = names.iter().filter(\|n\| !pinned.contains(n)).collect();
1778	let cut = unpinned.len() - RELEASES_TO_KEEP;
1779	for name in unpinned.iter().take(cut) {
1780	assert!(!releases.join(name).exists(), "expected pruned: {name}");
1781	}
1782	for name in unpinned.iter().skip(cut) {
1783	assert!(releases.join(name).exists(), "expected to survive: {name}");
1784	}
1785	}
1786
1787	/// Every dir pinned means the loop deletes nothing and the script still
1788	/// exits 0. Worth its own test because the obvious implementation of this
1789	/// filter is `grep -v`, which exits 1 when it selects no lines and would
1790	/// have failed the deploy here under `set -e`.
1791	#[tokio::test]
1792	async fn gc_remote_releases_succeeds_when_everything_is_pinned() {
1793	let tmp = tempfile::tempdir().unwrap();
1794	let root = tmp.path();
1795	let names = releases_by_age(root, RELEASES_TO_KEEP + 3).await;
1796	let pinned: PinnedReleases = names.iter().cloned().collect();
1797
1798	gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
1799	.await
1800	.unwrap();
1801
1802	let releases = root.join("releases");
1803	for name in &names {
1804	assert!(releases.join(name).exists(), "expected to survive: {name}");
1805	}
1806	}
1807
1808	/// A `releases/` that does not exist is not an error: a node's first deploy
1809	/// creates the dir, and gc runs on the same path.
1810	#[tokio::test]
1811	async fn gc_remote_releases_is_a_noop_when_the_store_is_missing() {
1812	let tmp = tempfile::tempdir().unwrap();
1813	gc_remote_releases(&local_executor(), tmp.path().to_str().unwrap(), &no_pins())
1814	.await
1815	.unwrap();
1816	}
1817
1818	/// Names reach the script as positional parameters, so a name that looks
1819	/// like shell must be compared whole rather than expanded or split. None of
1820	/// these can be a digest16, but the pre-identity names are version strings
1821	/// and the pinned set is data read out of a database.
1822	#[tokio::test]
1823	async fn gc_remote_releases_quotes_pinned_names() {
1824	let tmp = tempfile::tempdir().unwrap();
1825	let root = tmp.path();
1826	let releases = root.join("releases");
1827	tokio::fs::create_dir_all(&releases).await.unwrap();
1828	let awkward = ["a b", "x'y", "*"];
1829	for name in awkward {
1830	tokio::fs::create_dir(releases.join(name)).await.unwrap();
1831	}
1832	// Enough newer dirs that the count alone would evict all three.
1833	let filler: Vec<String> = (0..=RELEASES_TO_KEEP).map(\|i\| format!("f{i}")).collect();
1834	for name in &filler {
1835	tokio::fs::create_dir(releases.join(name)).await.unwrap();
1836	}
1837
1838	let pinned: PinnedReleases = awkward.iter().map(\|s\| (*s).to_string()).collect();
1839	gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
1840	.await
1841	.unwrap();
1842
1843	for name in awkward {
1844	assert!(releases.join(name).exists(), "expected to survive: {name}");
1845	}
1846	}
1847
1848	/// A pinned name matches a whole directory name, never a prefix of one.
1849	/// `case`-with-globbing or a `grep -F` without `-x` would keep `v0` and
1850	/// `v01` both because one contains the other, quietly widening the pinned
1851	/// set past what the database said.
1852	#[tokio::test]
1853	async fn gc_remote_releases_matches_whole_names_not_prefixes() {
1854	let tmp = tempfile::tempdir().unwrap();
1855	let root = tmp.path();
1856	let names = releases_by_age(root, RELEASES_TO_KEEP + 3).await;
1857
1858	// Pin the oldest by an exact name; its neighbours share the prefix.
1859	let pinned: PinnedReleases = [names[0].clone()].into_iter().collect();
1860	gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
1861	.await
1862	.unwrap();
1863
1864	let releases = root.join("releases");
1865	assert!(
1866	releases.join(&names[0]).exists(),
1867	"the pinned dir was evicted"
1868	);
1869	assert!(
1870	!releases.join(&names[1]).exists(),
1871	"a dir sharing the pinned name's prefix was treated as pinned"
1872	);
1873	}
1874
1875	#[tokio::test]
1876	async fn gc_local_releases_noop_when_releases_dir_missing() {
1877	let tmp = tempfile::tempdir().unwrap();
1878	gc_local_releases(tmp.path(), &no_pins()).await.unwrap();
1879	}
1880
1881	#[tokio::test]
1882	async fn deploy_remote_fails_cleanly_when_host_unreachable() {
1883	// 192.0.2.0/24 is reserved for documentation and routes nowhere.
1884	// ConnectTimeout=10 limits the test wallclock to ~10s worst case.
1885	let tmp = tempfile::tempdir().unwrap();
1886	let staged = tmp.path().join("releases").join("0.0.1");
1887	tokio::fs::create_dir_all(&staged).await.unwrap();
1888	tokio::fs::write(staged.join("server"), b"x").await.unwrap();
1889
1890	let node = crate::topology::Node {
1891	platform: None,
1892	base_image: None,
1893	libc: None,
1894	name: "unreachable".into(),
1895	ssh_target: "deploy@192.0.2.1".into(),
1896	release_root: "/opt/never".into(),
1897	service_name: "makenotwork.service".into(),
1898	health_url: None,
1899	config_check_env_file: None,
1900	actuate: crate::topology::default_actuate(),
1901	observe: crate::topology::default_observe(),
1902	companions: Vec::new(),
1903	};
1904	let executor = SshExec::new(
1905	node.ssh_target.clone(),
1906	CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1907	);
1908
1909	let placement = Placement::check(&node, &staged, None).expect("both sides silent");
1910	let result = deploy_node(&executor, placement, "0.0.1", "server", Some(&no_pins())).await;
1911	let err = result.expect_err("deploy to unreachable host should fail");
1912	let msg = format!("{err:#}");
1913	// Don't pin exact wording, just that the failure is attributed (ssh /
1914	// rsync / connection) and that no panic / hang happened.
1915	assert!(
1916	msg.contains("ssh")
1917	\|\| msg.contains("rsync")
1918	\|\| msg.contains("connection")
1919	\|\| msg.contains("Connection"),
1920	"unexpected error: {msg}"
1921	);
1922	}
1923
1924	#[tokio::test]
1925	async fn deploy_node_with_local_ssh_target_swaps_symlink() {
1926	// ssh_target="local" routes to the local fast-path: just a symlink
1927	// swap, no remote calls.
1928	let tmp = tempfile::tempdir().unwrap();
1929	let release_root = tmp.path().to_path_buf();
1930	let staged = release_root.join("releases").join("0.0.1");
1931	tokio::fs::create_dir_all(&staged).await.unwrap();
1932	tokio::fs::write(staged.join("server"), b"x").await.unwrap();
1933
1934	let node = crate::topology::Node {
1935	platform: None,
1936	base_image: None,
1937	libc: None,
1938	name: "local-dev".into(),
1939	ssh_target: "local".into(),
1940	release_root: release_root.to_string_lossy().into_owned(),
1941	service_name: "makenotwork.service".into(),
1942	health_url: None,
1943	config_check_env_file: None,
1944	actuate: crate::topology::default_actuate(),
1945	observe: crate::topology::default_observe(),
1946	companions: Vec::new(),
1947	};
1948	let executor = local_executor();
1949
1950	let out = deploy_node(
1951	&executor,
1952	Placement::check(&node, &staged, None).unwrap(),
1953	"0.0.1",
1954	"server",
1955	Some(&no_pins()),
1956	)
1957	.await
1958	.unwrap();
1959	assert_eq!(out, staged);
1960	let target = tokio::fs::read_link(release_root.join("current"))
1961	.await
1962	.unwrap();
1963	assert_eq!(target.to_string_lossy(), "releases/0.0.1");
1964	}
1965
1966	// ---- swap_and_restart_script: symlink/restart consistency ----
1967
1968	async fn run_script(script: &str) -> std::process::Output {
1969	Command::new("sh")
1970	.arg("-c")
1971	.arg(script)
1972	.output()
1973	.await
1974	.unwrap()
1975	}
1976
1977	async fn setup_release_root(with_current: bool) -> tempfile::TempDir {
1978	let tmp = tempfile::tempdir().unwrap();
1979	let root = tmp.path();
1980	tokio::fs::create_dir_all(root.join("releases/old"))
1981	.await
1982	.unwrap();
1983	tokio::fs::create_dir_all(root.join("releases/new"))
1984	.await
1985	.unwrap();
1986	if with_current {
1987	std::os::unix::fs::symlink("releases/old", root.join("current")).unwrap();
1988	}
1989	tmp
1990	}
1991
1992	#[tokio::test]
1993	async fn swap_and_restart_keeps_new_symlink_when_restart_succeeds() {
1994	let tmp = setup_release_root(true).await;
1995	let root = tmp.path().to_string_lossy().into_owned();
1996	let out = run_script(&swap_and_restart_script(&root, "new", "true")).await;
1997	assert!(
1998	out.status.success(),
1999	"script should succeed when restart succeeds"
2000	);
2001	let target = tokio::fs::read_link(tmp.path().join("current"))
2002	.await
2003	.unwrap();
2004	assert_eq!(
2005	target.to_string_lossy(),
2006	"releases/new",
2007	"symlink advanced to new"
2008	);
2009	}
2010
2011	#[tokio::test]
2012	async fn swap_and_restart_rolls_symlink_back_when_restart_fails() {
2013	// The bug: a restart failure after the flip must NOT leave `current`
2014	// pointing at the new (un-activated) release.
2015	let tmp = setup_release_root(true).await;
2016	let root = tmp.path().to_string_lossy().into_owned();
2017	let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
2018	assert!(!out.status.success(), "script must fail when restart fails");
2019	let target = tokio::fs::read_link(tmp.path().join("current"))
2020	.await
2021	.unwrap();
2022	assert_eq!(
2023	target.to_string_lossy(),
2024	"releases/old",
2025	"symlink rolled back to prev so a later restart can't silently activate new",
2026	);
2027	}
2028
2029	// ---- arch_guard_script: wrong-arch artifacts fail closed ----
2030
2031	/// A 20-byte stub whose ELF e_machine field (offset 18, 2 bytes LE) is set.
2032	fn elf_stub_with_machine(b18: u8, b19: u8) -> tempfile::NamedTempFile {
2033	let mut data = vec![0u8; 20];
2034	data[18] = b18;
2035	data[19] = b19;
2036	let f = tempfile::NamedTempFile::new().unwrap();
2037	std::fs::write(f.path(), &data).unwrap();
2038	f
2039	}
2040
2041	/// e_machine low byte for the host running the test, if mapped.
2042	fn host_machine_lo() -> Option<u8> {
2043	match std::env::consts::ARCH {
2044	"x86_64" => Some(0x3e),
2045	"aarch64" => Some(0xb7),
2046	_ => None,
2047	}
2048	}
2049
2050	#[tokio::test]
2051	async fn arch_guard_passes_for_matching_binary() {
2052	let Some(lo) = host_machine_lo() else { return };
2053	let f = elf_stub_with_machine(lo, 0x00);
2054	let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
2055	assert!(
2056	out.status.success(),
2057	"matching arch must pass: {}",
2058	String::from_utf8_lossy(&out.stderr),
2059	);
2060	}
2061
2062	#[tokio::test]
2063	async fn arch_guard_fails_closed_for_wrong_binary() {
2064	// Use the other arch's e_machine so it can't match the host.
2065	let wrong = match std::env::consts::ARCH {
2066	"x86_64" => 0xb7, // aarch64 binary on an x86_64 node
2067	"aarch64" => 0x3e, // x86_64 binary on an aarch64 node
2068	_ => return,
2069	};
2070	let f = elf_stub_with_machine(wrong, 0x00);
2071	let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
2072	assert!(
2073	!out.status.success(),
2074	"wrong-arch binary must fail closed before the symlink swap"
2075	);
2076	}
2077
2078	// ---- ldd_guard_script: a binary this node cannot resolve fails closed ----
2079
2080	/// A fake `ldd` on PATH that prints `body` and exits `code`, so the guard's
2081	/// three outcomes can be exercised without a binary that genuinely fails to
2082	/// link. The real `ldd` cannot be made to produce a `not found` on demand.
2083	async fn run_ldd_guard_with_fake(body: &str, code: i32) -> std::process::Output {
2084	let dir = tempfile::tempdir().unwrap();
2085	let fake = dir.path().join("ldd");
2086	std::fs::write(
2087	&fake,
2088	format!("#!/bin/sh\ncat <<'EOF'\n{body}\nEOF\nexit {code}\n"),
2089	)
2090	.unwrap();
2091	let mut perms = std::fs::metadata(&fake).unwrap().permissions();
2092	std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
2093	std::fs::set_permissions(&fake, perms).unwrap();
2094	let bin = dir.path().join("subject");
2095	std::fs::write(&bin, b"x").unwrap();
2096	Command::new("sh")
2097	.arg("-c")
2098	.arg(ldd_guard_script(&bin.to_string_lossy()))
2099	.env("PATH", format!("{}:/usr/bin:/bin", dir.path().display()))
2100	.output()
2101	.await
2102	.unwrap()
2103	}
2104
2105	#[tokio::test]
2106	async fn ldd_guard_fails_closed_on_an_unsatisfiable_symbol_version() {
2107	// The exact failure Bento's glibc_check used to catch at build time, and
2108	// the reason this guard exists: right arch, resolves every library, and
2109	// still cannot exec because the node's glibc is older than the build
2110	// host's.
2111	let out = run_ldd_guard_with_fake(
2112	"\tlinux-vdso.so.1 (0x00007fff)\n\
2113	\t/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.40' not found (required by ./pom)\n\
2114	\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
2115	0,
2116	)
2117	.await;
2118	assert!(
2119	!out.status.success(),
2120	"an unsatisfiable symbol version must fail before the symlink swap"
2121	);
2122	let stderr = String::from_utf8_lossy(&out.stderr);
2123	assert!(
2124	stderr.contains("GLIBC_2.40"),
2125	"the offending line must reach the operator, not just a verdict: {stderr}"
2126	);
2127	}
2128
2129	#[tokio::test]
2130	async fn ldd_guard_fails_closed_on_a_missing_library() {
2131	let out = run_ldd_guard_with_fake("\tlibfoo.so.1 => not found", 0).await;
2132	assert!(!out.status.success(), "a missing library must fail closed");
2133	}
2134
2135	#[tokio::test]
2136	async fn ldd_guard_passes_a_resolvable_binary() {
2137	let out = run_ldd_guard_with_fake(
2138	"\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
2139	0,
2140	)
2141	.await;
2142	assert!(
2143	out.status.success(),
2144	"a fully resolved binary must pass: {}",
2145	String::from_utf8_lossy(&out.stderr),
2146	);
2147	}
2148
2149	#[tokio::test]
2150	async fn ldd_guard_passes_a_static_binary() {
2151	// ldd exits non-zero for these. Nothing to resolve is not a failure.
2152	let out = run_ldd_guard_with_fake("\tnot a dynamic executable", 1).await;
2153	assert!(
2154	out.status.success(),
2155	"a static binary has no dependencies to satisfy: {}",
2156	String::from_utf8_lossy(&out.stderr),
2157	);
2158	}
2159
2160	#[tokio::test]
2161	async fn ldd_guard_fails_when_ldd_errors_for_another_reason() {
2162	// Not the static case: ldd said something else and exited non-zero. We
2163	// do not know the binary is fine, so we do not say it is.
2164	let out = run_ldd_guard_with_fake("ldd: cannot read file", 1).await;
2165	assert!(
2166	!out.status.success(),
2167	"an unexplained ldd failure must not read as a pass"
2168	);
2169	}
2170
2171	#[tokio::test]
2172	async fn ldd_guard_skips_when_the_node_has_no_ldd() {
2173	// Cannot verify is not known bad, matching arch_guard's unmapped-arch
2174	// call. PATH holds nothing, so `command -v ldd` finds none.
2175	let dir = tempfile::tempdir().unwrap();
2176	let bin = dir.path().join("subject");
2177	std::fs::write(&bin, b"x").unwrap();
2178	// Absolute path to the shell: PATH is what this test empties, so
2179	// resolving `sh` through it would fail before the script ever ran.
2180	let out = Command::new("/bin/sh")
2181	.arg("-c")
2182	.arg(ldd_guard_script(&bin.to_string_lossy()))
2183	.env("PATH", dir.path().display().to_string())
2184	.output()
2185	.await
2186	.unwrap();
2187	assert!(
2188	out.status.success(),
2189	"a node with no ldd must not fail the deploy: {}",
2190	String::from_utf8_lossy(&out.stderr),
2191	);
2192	}
2193
2194	#[tokio::test]
2195	async fn swap_and_restart_first_deploy_failure_has_no_prev_to_restore() {
2196	// No prior `current`. A restart failure leaves `current` at new (the only
2197	// version) and still reports failure — documented degenerate case.
2198	let tmp = setup_release_root(false).await;
2199	let root = tmp.path().to_string_lossy().into_owned();
2200	let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
2201	assert!(!out.status.success(), "script must fail when restart fails");
2202	let target = tokio::fs::read_link(tmp.path().join("current"))
2203	.await
2204	.unwrap();
2205	assert_eq!(
2206	target.to_string_lossy(),
2207	"releases/new",
2208	"no prev existed to roll back to"
2209	);
2210	}
2211
2212	// ---- config_check_script: systemd-faithful env loading ----
2213
2214	#[tokio::test]
2215	async fn config_check_script_loads_values_with_shell_metachars() {
2216	// The bug: `. env_file` expands/word-splits values, so a URL or a
2217	// password containing a shell metacharacter is mangled — it dropped
2218	// DATABASE_URL to empty on a real node, which would fail every deploy.
2219	// The export-loop must load such a value intact. The "binary" is a
2220	// checker script (a real path, like a deployed binary) that exits 0 only
2221	// if the var arrived byte-for-byte — it compares against the expected
2222	// value read from a file, so nothing re-interprets the metacharacters.
2223	let tricky = "postgres://u:p$ss;w&rd@h/db `x` $(y)";
2224	// Plain files in a tempdir: no lingering write fd, so the checker can be
2225	// exec'd (a NamedTempFile stays open and would ETXTBSY).
2226	let dir = tempfile::tempdir().unwrap();
2227	let expected_path = dir.path().join("expected");
2228	std::fs::write(&expected_path, tricky).unwrap(); // no trailing newline
2229
2230	let env_path = dir.path().join("node.env");
2231	std::fs::write(
2232	&env_path,
2233	format!(
2234	"# a comment\n\nDATABASE_URL={tricky}\nOTHER=plain\nEXPECTED_FILE={ef}\n",
2235	ef = expected_path.display(),
2236	),
2237	)
2238	.unwrap();
2239
2240	let checker_path = dir.path().join("checker.sh");
2241	std::fs::write(
2242	&checker_path,
2243	"#!/bin/sh\nwant=$(cat \"$EXPECTED_FILE\")\n\
2244	[ \"$DATABASE_URL\" = \"$want\" ] \|\| { echo \"DB [$DATABASE_URL] != [$want]\" >&2; exit 1; }\n\
2245	[ \"$OTHER\" = plain ] \|\| { echo \"OTHER [$OTHER]\" >&2; exit 1; }\n",
2246	)
2247	.unwrap();
2248	std::fs::set_permissions(
2249	&checker_path,
2250	std::os::unix::fs::PermissionsExt::from_mode(0o755),
2251	)
2252	.unwrap();
2253
2254	let script =
2255	config_check_script(&env_path.to_string_lossy(), &checker_path.to_string_lossy());
2256	let out = run_script(&script).await;
2257	assert!(
2258	out.status.success(),
2259	"value with shell metachars must load intact; stderr: {}",
2260	String::from_utf8_lossy(&out.stderr),
2261	);
2262	}
2263
2264	// ---- install-companion.sh: the node-side guard rails ----
2265
2266	/// Run the shipped installer script with three args; returns its exit code.
2267	/// Exercises the real file rather than a copy of its logic, because the
2268	/// script is the ONLY control on a NOPASSWD sudo grant.
2269	fn run_installer(src: &str, dst: &str, service: &str) -> i32 {
2270	let script =
2271	std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../deploy/install-companion.sh");
2272	std::process::Command::new("bash")
2273	.arg(&script)
2274	.args([src, dst, service])
2275	.output()
2276	.expect("running install-companion.sh")
2277	.status
2278	.code()
2279	.expect("script exited via signal")
2280	}
2281
2282	// Guards run before any filesystem write, so these never install anything.
2283	// Exit 3 = refused by a guard; exit 4 = guards passed, src simply absent.
2284	const REFUSED: i32 = 3;
2285	const PASSED_GUARDS: i32 = 4;
2286
2287	#[test]
2288	fn installer_refuses_a_dst_that_escapes_opt_via_dotdot() {
2289	// `/opt/../etc/...` matches a bare `/opt/*` glob. With the sudoers
2290	// wildcard that meant `install -m 0755` as root to anywhere, plus a
2291	// restart of any unit — so the path must be normalised before the test.
2292	assert_eq!(
2293	run_installer(
2294	"/opt/mnw/releases/1.0.0/companions/mnw-cli",
2295	"/opt/../etc/systemd/system/evil.service",
2296	"mnw-cli.service",
2297	),
2298	REFUSED,
2299	);
2300	}
2301
2302	#[test]
2303	fn installer_refuses_a_src_that_escapes_the_bundle_via_dotdot() {
2304	assert_eq!(
2305	run_installer(
2306	"/opt/mnw/releases/1.0.0/companions/../../../../../etc/shadow",
2307	"/opt/mnw-cli/mnw-cli",
2308	"mnw-cli.service",
2309	),
2310	REFUSED,
2311	);
2312	}
2313
2314	#[test]
2315	fn installer_accepts_the_real_companion_paths() {
2316	// The guards must not have been tightened into uselessness: the shape
2317	// Sando actually sends has to get past them. It stops at the missing
2318	// src (exit 4), which is proof the guards accepted it.
2319	assert_eq!(
2320	run_installer(
2321	"/opt/mnw/releases/1.0.0/companions/mnw-cli",
2322	"/opt/mnw-cli/mnw-cli",
2323	"mnw-cli.service",
2324	),
2325	PASSED_GUARDS,
2326	);
2327	}
2328
2329	#[test]
2330	fn installer_refuses_a_service_name_with_a_path_separator() {
2331	assert_eq!(
2332	run_installer(
2333	"/opt/mnw/releases/1.0.0/companions/mnw-cli",
2334	"/opt/mnw-cli/mnw-cli",
2335	"../../etc/evil.service",
2336	),
2337	REFUSED,
2338	);
2339	}
2340
2341	// ---- install_companion_cmd: shape + quoting ----
2342
2343	#[test]
2344	fn install_companion_cmd_shape_and_quoting() {
2345	let cmd = install_companion_cmd(
2346	"/opt/mnw/releases/0.10.14/companions/mnw-cli",
2347	"/opt/mnw-cli/mnw-cli",
2348	"mnw-cli.service",
2349	);
2350	// Routes through the wrapper (single sudoers grant), sudo-invoked, with
2351	// src, dst, service in that order.
2352	assert!(cmd.starts_with("sudo "), "must be sudo-invoked: {cmd}");
2353	assert!(
2354	cmd.contains("/usr/local/lib/mnw/install-companion.sh"),
2355	"{cmd}"
2356	);
2357	let installer_pos = cmd.find("install-companion.sh").unwrap();
2358	let src_pos = cmd.find("companions/mnw-cli").unwrap();
2359	let dst_pos = cmd.find("/opt/mnw-cli/mnw-cli").unwrap();
2360	let svc_pos = cmd.find("mnw-cli.service").unwrap();
2361	assert!(
2362	installer_pos < src_pos && src_pos < dst_pos && dst_pos < svc_pos,
2363	"arg order: {cmd}"
2364	);
2365	}
2366
2367	#[test]
2368	fn install_companion_cmd_quotes_metachars() {
2369	// A path with a space/quote must be shell-safe (defense in depth even
2370	// though these come from operator config).
2371	let cmd = install_companion_cmd("/a b/src", "/dst'x", "u.service");
2372	let out = std::process::Command::new("sh")
2373	.arg("-c")
2374	.arg(format!(
2375	"set -- {}; echo \"$#\"",
2376	cmd.strip_prefix("sudo ").unwrap()
2377	))
2378	.output()
2379	.unwrap();
2380	// installer + 3 args = 4 positional words after quoting.
2381	assert_eq!(
2382	String::from_utf8_lossy(&out.stdout).trim(),
2383	"4",
2384	"quoting split wrong: {cmd}"
2385	);
2386	}
2387
2388	#[tokio::test]
2389	async fn config_check_script_propagates_binary_failure() {
2390	// A required var missing (the binary exits non-zero) must fail the check.
2391	let env = tempfile::NamedTempFile::new().unwrap();
2392	std::fs::write(env.path(), "FOO=bar\n").unwrap();
2393	let script = config_check_script(&env.path().to_string_lossy(), "false");
2394	let out = run_script(&script).await;
2395	assert!(
2396	!out.status.success(),
2397	"a non-zero MNW_CHECK_CONFIG exit must fail the check"
2398	);
2399	}
2400
2401	#[tokio::test]
2402	async fn deploy_node_denied_when_executor_lacks_deploy_grant() {
2403	// Defense in depth: an executor without the deploy grant refuses the
2404	// step before any filesystem / ssh action.
2405	let tmp = tempfile::tempdir().unwrap();
2406	let release_root = tmp.path().to_path_buf();
2407	let staged = release_root.join("releases").join("0.0.1");
2408	tokio::fs::create_dir_all(&staged).await.unwrap();
2409
2410	let node = crate::topology::Node {
2411	platform: None,
2412	base_image: None,
2413	libc: None,
2414	name: "local-dev".into(),
2415	ssh_target: "local".into(),
2416	release_root: release_root.to_string_lossy().into_owned(),
2417	service_name: "makenotwork.service".into(),
2418	health_url: None,
2419	config_check_env_file: None,
2420	actuate: vec!["restart".into()], // no deploy
2421	observe: vec![],
2422	companions: Vec::new(),
2423	};
2424	let executor = LocalExec::new(CapabilitySet::from_tokens(["restart"], Vec::<&str>::new()));
2425	let err = deploy_node(
2426	&executor,
2427	Placement::check(&node, &staged, None).unwrap(),
2428	"0.0.1",
2429	"server",
2430	Some(&no_pins()),
2431	)
2432	.await
2433	.unwrap_err();
2434	assert!(
2435	format!("{err:#}").contains("capability denied"),
2436	"expected capability denial"
2437	);
2438	}
2439
2440	// ---- FakeExec: the deploy_remote choreography without a real host ----
2441	//
2442	// deploy_node's local fast-path is covered above with a real LocalExec, but
2443	// the remote path (rsync + arch guard + config-drift + swap + companions +
2444	// gc) short-circuits on `ssh_target != "local"` and so never ran under test
2445	// without a reachable node. FakeExec records every executor call in order
2446	// and can be told to fail one shell step (matched by substring) or the rsync
2447	// push, so the ordering and the fail-closed-before-swap contract are
2448	// assertable in-process.
2449
2450	struct FakeExec {
2451	caps: CapabilitySet,
2452	calls: Arc<StdMutex<Vec<String>>>,
2453	/// The first `run_streaming` whose script contains this substring exits
2454	/// non-zero (a failed shell step), e.g. the arch guard.
2455	fail_run_matching: Option<String>,
2456	/// `push_dir` (the rsync) returns an error.
2457	fail_push_dir: bool,
2458	}
2459
2460	impl FakeExec {
2461	fn new() -> Self {
2462	Self {
2463	caps: CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
2464	calls: Arc::new(StdMutex::new(Vec::new())),
2465	fail_run_matching: None,
2466	fail_push_dir: false,
2467	}
2468	}
2469	fn log(&self) -> Vec<String> {
2470	self.calls.lock().unwrap().clone()
2471	}
2472	}
2473
2474	#[async_trait]
2475	impl Executor for FakeExec {
2476	async fn run_streaming(&self, step: &Step, _sink: &mut dyn LogSink) -> Result<RunOutput> {
2477	// Every deploy step is a `Step::shell`, so the script is argv's tail.
2478	let script = step.argv.last().cloned().unwrap_or_default();
2479	self.calls.lock().unwrap().push(format!("run:{script}"));
2480	let fail = self
2481	.fail_run_matching
2482	.as_deref()
2483	.is_some_and(\|m\| script.contains(m));
2484	Ok(RunOutput {
2485	status: std::process::ExitStatus::from_raw(if fail { 1 << 8 } else { 0 }),
2486	stdout: Vec::new(),
2487	stderr: if fail {
2488	b"fake step failure".to_vec()
2489	} else {
2490	Vec::new()
2491	},
2492	})
2493	}
2494	async fn pull_file(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
2495	self.calls.lock().unwrap().push("pull_file".into());
2496	Ok(())
2497	}
2498	async fn pull_dir(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
2499	self.calls.lock().unwrap().push("pull_dir".into());
2500	Ok(())
2501	}
2502	async fn pull_glob(&self, _glob: &str, _local: &Path, _opts: &SyncOpts) -> Result<()> {
2503	self.calls.lock().unwrap().push("pull_glob".into());
2504	Ok(())
2505	}
2506	async fn push_dir(&self, _local: &Path, remote: &Path, _opts: &SyncOpts) -> Result<()> {
2507	self.calls
2508	.lock()
2509	.unwrap()
2510	.push(format!("push_dir:{}", remote.display()));
2511	if self.fail_push_dir {
2512	anyhow::bail!("fake rsync failure");
2513	}
2514	Ok(())
2515	}
2516	fn capabilities(&self) -> &CapabilitySet {
2517	&self.caps
2518	}
2519	}
2520
2521	fn remote_node(config_check: bool, companions: Vec<NodeCompanion>) -> Node {
2522	Node {
2523	platform: None,
2524	base_image: None,
2525	libc: None,
2526	name: "web-a".into(),
2527	ssh_target: "deploy@web-a".into(),
2528	release_root: "/opt/mnw".into(),
2529	service_name: "makenotwork.service".into(),
2530	health_url: None,
2531	config_check_env_file: config_check.then(\|\| "/etc/mnw/node.env".to_string()),
2532	actuate: crate::topology::default_actuate(),
2533	observe: crate::topology::default_observe(),
2534	companions,
2535	}
2536	}
2537
2538	fn companion() -> NodeCompanion {
2539	NodeCompanion {
2540	name: "mnw-cli".into(),
2541	install_path: "/opt/mnw-cli/mnw-cli".into(),
2542	service_name: "mnw-cli.service".into(),
2543	}
2544	}
2545
2546	/// Index of the first recorded call whose text contains `needle` (panics if
2547	/// absent — the assertion message names what was missing).
2548	fn pos(log: &[String], needle: &str) -> usize {
2549	log.iter()
2550	.position(\|c\| c.contains(needle))
2551	.unwrap_or_else(\|\| panic!("no call matched {needle:?} in {log:#?}"))
2552	}
2553
2554	#[tokio::test]
2555	async fn deploy_remote_runs_the_full_choreography_in_order() {
2556	// A node opted into the config-drift check and carrying one companion:
2557	// mkdir -> rsync -> arch guard -> config check -> swap+restart ->
2558	// companion install -> gc, in that order.
2559	let tmp = tempfile::tempdir().unwrap();
2560	let staged = tmp.path().join("releases").join("0.9.0");
2561	tokio::fs::create_dir_all(&staged).await.unwrap();
2562
2563	let node = remote_node(true, vec![companion()]);
2564	let exec = FakeExec::new();
2565	let out = deploy_node(
2566	&exec,
2567	Placement::check(&node, &staged, None).unwrap(),
2568	"0.9.0",
2569	"makenotwork",
2570	Some(&no_pins()),
2571	)
2572	.await
2573	.expect("deploy_remote should succeed against the fake");
2574	assert_eq!(out, PathBuf::from("/opt/mnw/releases/0.9.0"));
2575
2576	let log = exec.log();
2577	let mkdir = pos(&log, "mkdir -p");
2578	let rsync = pos(&log, "push_dir:/opt/mnw/releases/0.9.0");
2579	let arch = pos(&log, "e_machine");
2580	let cfg = pos(&log, "MNW_CHECK_CONFIG=1");
2581	let swap = pos(&log, "reload-or-restart");
2582	let comp = pos(&log, "install-companion.sh");
2583	let gc = pos(&log, "ls -1t");
2584	assert!(
2585	mkdir < rsync && rsync < arch && arch < cfg && cfg < swap && swap < comp && comp < gc,
2586	"deploy steps out of order: {log:#?}"
2587	);
2588	}
2589
2590	#[tokio::test]
2591	async fn deploy_remote_aborts_before_swap_when_rsync_fails() {
2592	// The rsync failing must fail the deploy BEFORE the symlink swap — the
2593	// "current symlink left intact" contract. Assert the swap never ran.
2594	let tmp = tempfile::tempdir().unwrap();
2595	let staged = tmp.path().join("releases").join("0.9.0");
2596	tokio::fs::create_dir_all(&staged).await.unwrap();
2597
2598	let node = remote_node(false, Vec::new());
2599	let mut exec = FakeExec::new();
2600	exec.fail_push_dir = true;
2601	let err = deploy_node(
2602	&exec,
2603	Placement::check(&node, &staged, None).unwrap(),
2604	"0.9.0",
2605	"makenotwork",
2606	Some(&no_pins()),
2607	)
2608	.await
2609	.expect_err("rsync failure must fail the deploy");
2610	assert!(
2611	format!("{err:#}").contains("rsync"),
2612	"error should attribute the rsync: {err:#}"
2613	);
2614	let log = exec.log();
2615	assert!(
2616	!log.iter().any(\|c\| c.contains("reload-or-restart")),
2617	"swap must not run after a failed rsync: {log:#?}"
2618	);
2619	}
2620
2621	#[tokio::test]
2622	async fn deploy_remote_aborts_before_swap_when_arch_guard_fails() {
2623	// A wrong-arch binary must fail closed before the swap. The fake fails
2624	// the arch-guard shell step; the swap must not follow.
2625	let tmp = tempfile::tempdir().unwrap();
2626	let staged = tmp.path().join("releases").join("0.9.0");
2627	tokio::fs::create_dir_all(&staged).await.unwrap();
2628
2629	let node = remote_node(false, Vec::new());
2630	let mut exec = FakeExec::new();
2631	exec.fail_run_matching = Some("e_machine".into());
2632	let err = deploy_node(
2633	&exec,
2634	Placement::check(&node, &staged, None).unwrap(),
2635	"0.9.0",
2636	"makenotwork",
2637	Some(&no_pins()),
2638	)
2639	.await
2640	.expect_err("arch mismatch must fail the deploy");
2641	assert!(
2642	format!("{err:#}").contains("architecture"),
2643	"error should mention the arch check: {err:#}"
2644	);
2645	let log = exec.log();
2646	assert!(
2647	!log.iter().any(\|c\| c.contains("reload-or-restart")),
2648	"swap must not run after a failed arch guard: {log:#?}"
2649	);
2650	}
2651
2652	#[tokio::test]
2653	async fn deploy_remote_skips_config_check_when_node_opts_out() {
2654	// No config_check_env_file => the pre-swap config check is skipped, but
2655	// the rest of the choreography (including the swap) still runs.
2656	let tmp = tempfile::tempdir().unwrap();
2657	let staged = tmp.path().join("releases").join("0.9.0");
2658	tokio::fs::create_dir_all(&staged).await.unwrap();
2659
2660	let node = remote_node(false, Vec::new());
2661	let exec = FakeExec::new();
2662	deploy_node(
2663	&exec,
2664	Placement::check(&node, &staged, None).unwrap(),
2665	"0.9.0",
2666	"makenotwork",
2667	Some(&no_pins()),
2668	)
2669	.await
2670	.unwrap();
2671	let log = exec.log();
2672	assert!(
2673	!log.iter().any(\|c\| c.contains("MNW_CHECK_CONFIG=1")),
2674	"config check must be skipped when the node opts out: {log:#?}"
2675	);
2676	assert!(
2677	log.iter().any(\|c\| c.contains("reload-or-restart")),
2678	"the swap must still run: {log:#?}"
2679	);
2680	}
2681
2682	/// A companion is guarded on the same terms as the primary, and BEFORE the
2683	/// swap. It used to be checked by nothing at all, so the first thing that
2684	/// noticed a bad companion was its unit failing to start during the install
2685	/// loop, which runs after the server has already been restarted.
2686	#[tokio::test]
2687	async fn companions_are_guarded_before_the_swap() {
2688	let tmp = tempfile::tempdir().unwrap();
2689	let staged = tmp.path().join("releases").join("0.9.0");
2690	tokio::fs::create_dir_all(&staged).await.unwrap();
2691
2692	let node = remote_node(false, vec![companion()]);
2693	let exec = FakeExec::new();
2694	deploy_node(
2695	&exec,
2696	Placement::check(&node, &staged, None).unwrap(),
2697	"0.9.0",
2698	"makenotwork",
2699	Some(&no_pins()),
2700	)
2701	.await
2702	.unwrap();
2703
2704	let log = exec.log();
2705	// The companion's own arch and loader checks, named by its path so they
2706	// cannot be confused with the primary's.
2707	let guard = pos(&log, "companions/mnw-cli");
2708	let swap = pos(&log, "reload-or-restart");
2709	let install = pos(&log, "install-companion.sh");
2710	assert!(
2711	guard < swap && swap < install,
2712	"a companion must be guarded before the swap and installed after it: {log:#?}"
2713	);
2714	let companion_guards = log
2715	.iter()
2716	.filter(\|c\| c.contains("companions/mnw-cli") && !c.contains("install-companion.sh"))
2717	.count();
2718	assert_eq!(
2719	companion_guards, 2,
2720	"both guards must run against the companion, not just one: {log:#?}"
2721	);
2722	}
2723
2724	/// And failing one of them fails the promote with the service intact, which
2725	/// is the whole point of moving the check ahead of the swap.
2726	#[tokio::test]
2727	async fn a_companion_failing_its_guard_aborts_before_the_swap() {
2728	let tmp = tempfile::tempdir().unwrap();
2729	let staged = tmp.path().join("releases").join("0.9.0");
2730	tokio::fs::create_dir_all(&staged).await.unwrap();
2731
2732	let node = remote_node(false, vec![companion()]);
2733	let mut exec = FakeExec::new();
2734	// Fails the first script naming the companion, which is its arch guard.
2735	// The primary's guards name the primary and are unaffected.
2736	exec.fail_run_matching = Some("companions/mnw-cli".into());
2737	let err = deploy_node(
2738	&exec,
2739	Placement::check(&node, &staged, None).unwrap(),
2740	"0.9.0",
2741	"makenotwork",
2742	Some(&no_pins()),
2743	)
2744	.await
2745	.expect_err("a bad companion must fail the deploy");
2746
2747	let msg = format!("{err:#}");
2748	assert!(
2749	msg.contains("mnw-cli"),
2750	"the refusal must name which companion: {msg}"
2751	);
2752	assert_eq!(
2753	stage_of(&err),
2754	Some(FailureStage::BeforeSwap),
2755	"a companion guard failing must leave the service intact: {msg}"
2756	);
2757	let log = exec.log();
2758	assert!(
2759	!log.iter().any(\|c\| c.contains("reload-or-restart")),
2760	"swap must not run after a failed companion guard: {log:#?}"
2761	);
2762	assert!(
2763	!log.iter().any(\|c\| c.contains("install-companion.sh")),
2764	"nothing should be installed after a failed companion guard: {log:#?}"
2765	);
2766	}
2767
2768	/// The guards and the installer must read the same path. A guard checking a
2769	/// path the installer does not use is a check of nothing, and passes.
2770	#[test]
2771	fn the_guarded_companion_path_is_the_one_installed() {
2772	let release_dir = "/opt/mnw/releases/0.9.0";
2773	let src = companion_src(release_dir, "mnw-cli");
2774	assert_eq!(src, "/opt/mnw/releases/0.9.0/companions/mnw-cli");
2775	let cmd = install_companion_cmd(&src, "/opt/mnw-cli/mnw-cli", "mnw-cli.service");
2776	assert!(
2777	cmd.contains(&src),
2778	"the installer must read the path the guards checked: {cmd}"
2779	);
2780	}
2781
2782	#[tokio::test]
2783	async fn deploy_remote_installs_companion_after_the_swap() {
2784	// Companions are After= the server: their install must land after the
2785	// symlink swap + service restart, never before.
2786	let tmp = tempfile::tempdir().unwrap();
2787	let staged = tmp.path().join("releases").join("0.9.0");
2788	tokio::fs::create_dir_all(&staged).await.unwrap();
2789
2790	let node = remote_node(false, vec![companion()]);
2791	let exec = FakeExec::new();
2792	deploy_node(
2793	&exec,
2794	Placement::check(&node, &staged, None).unwrap(),
2795	"0.9.0",
2796	"makenotwork",
2797	Some(&no_pins()),
2798	)
2799	.await
2800	.unwrap();
2801	let log = exec.log();
2802	assert!(
2803	pos(&log, "reload-or-restart") < pos(&log, "install-companion.sh"),
2804	"companion install must follow the swap: {log:#?}"
2805	);
2806	}
2807	}
2808