Skip to main content

max / makenotwork

60.9 KB · 1721 lines History Blame Raw
1 //! Tests for [`super`].
2
3 use super::*;
4
5 /// Nothing deployed, so nothing pinned: the tests that exercise the count
6 /// alone pass this, and the ones that exercise pinning build their own set.
7 fn no_pins() -> PinnedReleases {
8 PinnedReleases::none()
9 }
10
11 use crate::topology::NodeCompanion;
12 use async_trait::async_trait;
13 use ops_exec::{CapabilitySet, LocalExec, LogSink, SshExec};
14 use std::os::unix::process::ExitStatusExt;
15 use std::sync::{Arc, Mutex as StdMutex};
16 use std::time::SystemTime;
17
18 // ---- placement ----
19 //
20 // The whole table, because the interesting cases are the two where one side
21 // said nothing. Treating silence as agreement is how a wrong-architecture
22 // deploy would get through, and it is the shape a "check it before you call"
23 // guard tends to end up with.
24
25 fn node_on(platform: Option<&str>) -> Node {
26 Node {
27 name: crate::domain::NodeId::new("n1"),
28 ssh_target: "deploy@n1".into(),
29 release_root: "/opt/x".into(),
30 platform: platform.map(|p| Platform::parse(p).unwrap()),
31 base_image: None,
32 libc: None,
33 service_name: "x.service".into(),
34 config_check_env_file: None,
35 actuate: crate::topology::default_actuate(),
36 observe: crate::topology::default_observe(),
37 health_url: None,
38 companions: Vec::new(),
39 }
40 }
41
42 /// A node that declares a glibc older than the bundle needs is refused
43 /// before the rsync, and the message names both numbers so the operator
44 /// knows which side to fix.
45 #[tokio::test]
46 async fn a_bundle_above_the_node_s_declared_glibc_is_refused_before_the_rsync() {
47 let dir = tempfile::tempdir().unwrap();
48 let exe = std::fs::read(std::env::current_exe().unwrap()).unwrap();
49 std::fs::write(dir.path().join("bin"), &exe).unwrap();
50 let Some(floor) = crate::elf::glibc_floor(&exe) else {
51 return; // a static test binary states no floor; nothing to compare
52 };
53
54 let mut node = node_on(None);
55 node.libc = Some("2.0".into()); // older than anything real
56 let err = check_bundle_fits_node(&node, dir.path())
57 .await
58 .expect_err("a bundle above the node's glibc must be refused");
59 // `{:#}` walks the context chain: the outermost context is the
60 // `FailureStage`, whose Display is the operator-facing "nothing moved"
61 // line, and the cause below it is the reason.
62 let msg = format!("{err:#}");
63 assert!(
64 msg.contains(&floor.to_string()) && msg.contains("2.0"),
65 "the refusal must name both numbers: {msg}"
66 );
67 assert_eq!(
68 stage_of(&err),
69 Some(FailureStage::BeforeSwap),
70 "refusing here must be recoverable: nothing has moved yet"
71 );
72 }
73
74 #[tokio::test]
75 async fn a_bundle_within_the_node_s_declared_glibc_passes() {
76 let dir = tempfile::tempdir().unwrap();
77 let exe = std::fs::read(std::env::current_exe().unwrap()).unwrap();
78 std::fs::write(dir.path().join("bin"), &exe).unwrap();
79
80 let mut node = node_on(None);
81 node.libc = Some("99.0".into()); // newer than anything real
82 check_bundle_fits_node(&node, dir.path())
83 .await
84 .expect("a bundle the node can load must pass");
85 }
86
87 /// The three ways there is nothing to compare. All three pass, because
88 /// "cannot verify" is not "known bad" — the same call `arch_guard_script`
89 /// makes for an unmapped architecture.
90 #[tokio::test]
91 async fn nothing_to_compare_is_a_pass_not_a_refusal() {
92 let dir = tempfile::tempdir().unwrap();
93 let exe = std::fs::read(std::env::current_exe().unwrap()).unwrap();
94 std::fs::write(dir.path().join("bin"), &exe).unwrap();
95
96 // 1. The node declares no libc.
97 let node = node_on(None);
98 check_bundle_fits_node(&node, dir.path()).await.unwrap();
99
100 // 2. The node's declared libc is not a version (a config typo).
101 let mut typo = node_on(None);
102 typo.libc = Some("noble".into());
103 check_bundle_fits_node(&typo, dir.path()).await.unwrap();
104
105 // 3. The bundle holds no ELF, so it states no floor.
106 let empty = tempfile::tempdir().unwrap();
107 std::fs::write(empty.path().join("style.css"), b"body{}").unwrap();
108 let mut strict = node_on(None);
109 strict.libc = Some("2.0".into());
110 check_bundle_fits_node(&strict, empty.path())
111 .await
112 .expect("a bundle with no binaries has no floor to exceed");
113 }
114
115 #[test]
116 fn matching_platforms_are_placeable() {
117 let node = node_on(Some("linux/aarch64"));
118 let art = Platform::parse("linux/aarch64").unwrap();
119 let p = Placement::check(&node, Path::new("/r/abc"), Some(&art)).expect("a match places");
120 assert_eq!(p.bundle(), Path::new("/r/abc"));
121 assert_eq!(p.node().name.as_str(), "n1");
122 }
123
124 #[test]
125 fn a_different_architecture_is_refused() {
126 // The failure this type exists for: pom's aarch64 bundle reaching the
127 // x86_64 box, which execs nothing and takes the watcher down.
128 let node = node_on(Some("linux/x86_64"));
129 let art = Platform::parse("linux/aarch64").unwrap();
130 let err = Placement::check(&node, Path::new("/r/abc"), Some(&art)).unwrap_err();
131 assert!(
132 matches!(err, PlacementError::Mismatch { .. }),
133 "expected a mismatch, got {err}"
134 );
135 // The message has to name both, or an operator cannot tell which half
136 // is wrong.
137 let msg = err.to_string();
138 assert!(
139 msg.contains("linux/x86_64") && msg.contains("linux/aarch64"),
140 "{msg}"
141 );
142 }
143
144 #[test]
145 fn a_silent_node_refuses_a_stated_artifact() {
146 // Not "the node probably runs it". A node that never said what it is
147 // cannot vouch for a bundle that did, and the pairing that looks
148 // harmless here is exactly the one that ships the wrong half of a
149 // two-architecture release.
150 let node = node_on(None);
151 let art = Platform::parse("linux/aarch64").unwrap();
152 assert!(matches!(
153 Placement::check(&node, Path::new("/r/abc"), Some(&art)),
154 Err(PlacementError::NodeSilent { .. })
155 ));
156 }
157
158 #[test]
159 fn a_stated_node_refuses_a_silent_artifact() {
160 let node = node_on(Some("linux/aarch64"));
161 assert!(matches!(
162 Placement::check(&node, Path::new("/r/abc"), None),
163 Err(PlacementError::ArtifactSilent { .. })
164 ));
165 }
166
167 #[test]
168 fn both_silent_is_the_single_platform_world_and_still_places() {
169 // MNW is here and stays here. Its nodes declare nothing and its builds
170 // record nothing, which is the truth about a product with one build host
171 // and one architecture. The moment either side starts stating, the other
172 // has to as well — that is the forcing function, and it is why this cell
173 // is the only admissible non-match.
174 let node = node_on(None);
175 Placement::check(&node, Path::new("/r/abc"), None).expect("the pre-pom world still ships");
176 }
177
178 #[test]
179 fn platform_parsing_is_a_shape_not_a_spelling() {
180 assert_eq!(
181 Platform::parse("Linux/AArch64").unwrap(),
182 Platform::parse("linux/aarch64").unwrap(),
183 "case is not a distinction between two machines"
184 );
185 for bad in ["linux", "linux/", "/aarch64", "linux/aarch64/gnu", ""] {
186 assert!(Platform::parse(bad).is_err(), "{bad:?} should not parse");
187 }
188 }
189
190 // ---- failure stage ----
191 //
192 // The 2026-08-01 prod deploy failed its pre-swap config check, and the
193 // rollback then failed the same way — which left the node safely on the old
194 // version, and was reported as "it remains on the new version, manual
195 // intervention needed". These pin the distinction the reporting layer now
196 // depends on.
197
198 #[test]
199 fn a_pre_swap_failure_is_recoverable_as_such() {
200 let e = anyhow::anyhow!("Permission denied")
201 .context("pre-swap config check failed")
202 .context(FailureStage::BeforeSwap);
203 assert_eq!(stage_of(&e), Some(FailureStage::BeforeSwap));
204 // The reason survives alongside the stage; the stage does not replace it.
205 let rendered = format!("{e:#}");
206 assert!(
207 rendered.contains("pre-swap config check failed"),
208 "{rendered}"
209 );
210 assert!(rendered.contains("Permission denied"), "{rendered}");
211 }
212
213 #[test]
214 fn a_post_swap_failure_is_recoverable_as_such() {
215 let e = anyhow::anyhow!("unit failed to start")
216 .context("companion x deploy failed (server already swapped)")
217 .context(FailureStage::AtOrAfterSwap);
218 assert_eq!(stage_of(&e), Some(FailureStage::AtOrAfterSwap));
219 }
220
221 #[test]
222 fn an_unannotated_failure_has_no_stage() {
223 // Must be None, not a default. A caller seeing None has to treat the
224 // node as indeterminate; inferring "before the swap" would reintroduce
225 // the original bug pointing the other way, which is the dangerous way.
226 let e = anyhow::anyhow!("something older, from before stages existed");
227 assert_eq!(stage_of(&e), None);
228 }
229
230 // ---- env file readability probe ----
231
232 #[tokio::test]
233 async fn readability_probe_passes_on_a_readable_file() {
234 let tmp = tempfile::tempdir().unwrap();
235 let f = tmp.path().join("ok.env");
236 tokio::fs::write(&f, "A=1\n").await.unwrap();
237 let script = readability_probe_script(&f.to_string_lossy());
238 let out = run_checked(&local_executor(), &script, "probe").await;
239 assert!(out.is_ok(), "{:?}", out.err().map(|e| format!("{e:#}")));
240 }
241
242 #[tokio::test]
243 async fn readability_probe_names_the_user_and_mode_when_unreadable() {
244 // Root can read anything, so a mode-based test would pass spuriously
245 // there. Skip rather than assert something false. No libc dependency
246 // for one probe: a 0-mode temp file is readable iff we are root.
247 let probe_dir = tempfile::tempdir().unwrap();
248 let probe_file = probe_dir.path().join("root-check");
249 tokio::fs::write(&probe_file, "x").await.unwrap();
250 tokio::fs::set_permissions(
251 &probe_file,
252 std::os::unix::fs::PermissionsExt::from_mode(0o000),
253 )
254 .await
255 .unwrap();
256 if tokio::fs::read(&probe_file).await.is_ok() {
257 return; // running as root
258 }
259 let tmp = tempfile::tempdir().unwrap();
260 let f = tmp.path().join("locked.env");
261 tokio::fs::write(&f, "A=1\n").await.unwrap();
262 tokio::fs::set_permissions(&f, std::os::unix::fs::PermissionsExt::from_mode(0o000))
263 .await
264 .unwrap();
265
266 let script = readability_probe_script(&f.to_string_lossy());
267 let err = run_checked(&local_executor(), &script, "probe")
268 .await
269 .expect_err("an unreadable file must fail the probe");
270 let msg = format!("{err:#}");
271 // The two things the raw bash error does not tell you.
272 assert!(msg.contains("cannot read"), "{msg}");
273 assert!(msg.contains("mode 0") || msg.contains("mode "), "{msg}");
274 }
275
276 #[tokio::test]
277 async fn readability_probe_distinguishes_missing_from_unreadable() {
278 let tmp = tempfile::tempdir().unwrap();
279 let missing = tmp.path().join("nope.env");
280 let script = readability_probe_script(&missing.to_string_lossy());
281 let err = run_checked(&local_executor(), &script, "probe")
282 .await
283 .expect_err("a missing file must fail the probe");
284 let msg = format!("{err:#}");
285 assert!(msg.contains("does not exist"), "{msg}");
286 }
287
288 #[test]
289 fn the_two_stages_read_differently() {
290 // These strings end up in an operator's terminal during an incident.
291 let before = FailureStage::BeforeSwap.to_string();
292 let after = FailureStage::AtOrAfterSwap.to_string();
293 assert!(before.contains("previous version"), "{before}");
294 assert!(after.contains("indeterminate"), "{after}");
295 assert_ne!(before, after);
296 }
297
298 /// A LocalExec granted the default node capabilities (deploy + restart).
299 fn local_executor() -> LocalExec {
300 LocalExec::new(CapabilitySet::from_tokens(
301 ["deploy", "restart"],
302 ["health"],
303 ))
304 }
305
306 #[tokio::test]
307 async fn deploy_local_copies_multiple_binaries_and_swaps_symlink() {
308 let tmp = tempfile::tempdir().unwrap();
309 let root = tmp.path();
310
311 let src_dir = root.join("src");
312 tokio::fs::create_dir_all(&src_dir).await.unwrap();
313 let primary = src_dir.join("makenotwork");
314 let admin = src_dir.join("mnw-admin");
315 tokio::fs::write(&primary, b"PRIMARY").await.unwrap();
316 tokio::fs::write(&admin, b"ADMIN").await.unwrap();
317
318 let release_root = root.join("releases-root");
319 tokio::fs::create_dir_all(&release_root).await.unwrap();
320
321 // Stage into staging/<build_id> (no publish yet).
322 let staging = stage_local_bundle(&release_root, 42, &[primary.clone(), admin.clone()])
323 .await
324 .expect("stage_local_bundle should succeed");
325 assert_eq!(staging, release_root.join("staging").join("42"));
326 assert!(
327 !release_root.join("current").exists(),
328 "staging must not publish or flip current"
329 );
330
331 // Publish content-addressed at releases/<digest16>.
332 let released = finalize_local_release(&release_root, &staging, "deadbeefcafe0000", &no_pins())
333 .await
334 .expect("finalize_local_release should succeed");
335 assert_eq!(
336 released,
337 release_root.join("releases").join("deadbeefcafe0000")
338 );
339 assert!(
340 !staging.exists(),
341 "staging dir is consumed by the publish rename"
342 );
343 assert_eq!(
344 tokio::fs::read(released.join("makenotwork")).await.unwrap(),
345 b"PRIMARY"
346 );
347 assert_eq!(
348 tokio::fs::read(released.join("mnw-admin")).await.unwrap(),
349 b"ADMIN"
350 );
351
352 let current = release_root.join("current");
353 let target = tokio::fs::read_link(&current).await.unwrap();
354 assert_eq!(target.to_string_lossy(), "releases/deadbeefcafe0000");
355 let via_current = tokio::fs::read(current.join("makenotwork")).await.unwrap();
356 assert_eq!(via_current, b"PRIMARY");
357 }
358
359 #[tokio::test]
360 async fn finalize_second_release_swaps_symlink_and_keeps_old_dir() {
361 let tmp = tempfile::tempdir().unwrap();
362 let root = tmp.path();
363 let src_dir = root.join("src");
364 tokio::fs::create_dir_all(&src_dir).await.unwrap();
365 let bin = src_dir.join("server");
366 tokio::fs::write(&bin, b"V1").await.unwrap();
367
368 let release_root = root.join("rr");
369 tokio::fs::create_dir_all(&release_root).await.unwrap();
370
371 // Two builds, distinct digests (distinct content) -> two release dirs.
372 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
373 .await
374 .unwrap();
375 finalize_local_release(&release_root, &s1, "1111111111111111", &no_pins())
376 .await
377 .unwrap();
378 tokio::fs::write(&bin, b"V2").await.unwrap();
379 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
380 .await
381 .unwrap();
382 finalize_local_release(&release_root, &s2, "2222222222222222", &no_pins())
383 .await
384 .unwrap();
385
386 assert!(
387 release_root
388 .join("releases/1111111111111111/server")
389 .exists()
390 );
391 assert!(
392 release_root
393 .join("releases/2222222222222222/server")
394 .exists()
395 );
396 let target = tokio::fs::read_link(release_root.join("current"))
397 .await
398 .unwrap();
399 assert_eq!(target.to_string_lossy(), "releases/2222222222222222");
400 let via_current = tokio::fs::read(release_root.join("current/server"))
401 .await
402 .unwrap();
403 assert_eq!(via_current, b"V2");
404 }
405
406 #[tokio::test]
407 async fn finalize_reuses_an_existing_release_of_the_same_digest() {
408 let tmp = tempfile::tempdir().unwrap();
409 let root = tmp.path();
410 let bin = root.join("server");
411 tokio::fs::write(&bin, b"BYTES").await.unwrap();
412 let release_root = root.join("rr");
413 tokio::fs::create_dir_all(&release_root).await.unwrap();
414
415 let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
416 .await
417 .unwrap();
418 finalize_local_release(&release_root, &s1, "abc123abc123abc1", &no_pins())
419 .await
420 .unwrap();
421 // Same digest rebuilt (e.g. a re-run at the same content): finalize must
422 // reuse the existing release and drop the redundant staging dir, not error.
423 let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
424 .await
425 .unwrap();
426 let released = finalize_local_release(&release_root, &s2, "abc123abc123abc1", &no_pins())
427 .await
428 .expect("finalize is idempotent on a repeated digest");
429 assert_eq!(released, release_root.join("releases/abc123abc123abc1"));
430 assert!(!s2.exists(), "redundant staging dropped");
431 }
432
433 #[tokio::test]
434 async fn manifest_verify_script_passes_on_match_fails_on_drift_and_skips_when_absent() {
435 // The node-side verification is a shell running `sha256sum -c MANIFEST`;
436 // drive the real script through bash to prove it accepts a good bundle,
437 // rejects a tampered one, and no-ops on a legacy (MANIFEST-less) bundle.
438 let dir = tempfile::tempdir().unwrap();
439 tokio::fs::write(dir.path().join("server"), b"BINARY")
440 .await
441 .unwrap();
442 tokio::fs::create_dir(dir.path().join("static"))
443 .await
444 .unwrap();
445 tokio::fs::write(dir.path().join("static/app.css"), b"body{}")
446 .await
447 .unwrap();
448 let digest = crate::bundle::digest_dir(dir.path()).await.unwrap();
449 tokio::fs::write(dir.path().join("MANIFEST"), digest.manifest.as_bytes())
450 .await
451 .unwrap();
452
453 let run = |d: &std::path::Path| {
454 let script = manifest_verify_script(d.to_str().unwrap());
455 async move {
456 Command::new("bash")
457 .arg("-c")
458 .arg(&script)
459 .output()
460 .await
461 .unwrap()
462 }
463 };
464
465 let ok = run(dir.path()).await;
466 assert!(
467 ok.status.success(),
468 "matching bundle verifies: {}",
469 String::from_utf8_lossy(&ok.stderr)
470 );
471
472 // Drift one file: sha256sum -c must fail (current symlink left intact).
473 tokio::fs::write(dir.path().join("static/app.css"), b"TAMPERED")
474 .await
475 .unwrap();
476 let bad = run(dir.path()).await;
477 assert!(!bad.status.success(), "a drifted file fails verification");
478
479 // Legacy bundle with no MANIFEST: skip, not fail.
480 let legacy = tempfile::tempdir().unwrap();
481 tokio::fs::write(legacy.path().join("server"), b"x")
482 .await
483 .unwrap();
484 let skip = run(legacy.path()).await;
485 assert!(
486 skip.status.success(),
487 "a bundle without a MANIFEST skips verification rather than failing"
488 );
489 }
490
491 #[tokio::test]
492 async fn gc_local_releases_keeps_last_n_by_mtime() {
493 let tmp = tempfile::tempdir().unwrap();
494 let root = tmp.path();
495 let releases = root.join("releases");
496 tokio::fs::create_dir_all(&releases).await.unwrap();
497
498 let total = RELEASES_TO_KEEP + 3;
499 let mut names = Vec::new();
500 for i in 0..total {
501 let name = format!("v{i:02}");
502 let dir = releases.join(&name);
503 tokio::fs::create_dir(&dir).await.unwrap();
504 let f = std::fs::File::open(&dir).unwrap();
505 let when =
506 SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
507 let times = std::fs::FileTimes::new().set_modified(when);
508 f.set_times(times).unwrap();
509 names.push(name);
510 }
511
512 gc_local_releases(root, &no_pins()).await.unwrap();
513
514 let surviving_expected: Vec<_> = names
515 .iter()
516 .skip(total - RELEASES_TO_KEEP)
517 .cloned()
518 .collect();
519 for name in &surviving_expected {
520 assert!(releases.join(name).exists(), "expected to survive: {name}");
521 }
522 for name in names.iter().take(total - RELEASES_TO_KEEP) {
523 assert!(
524 !releases.join(name).exists(),
525 "expected to be pruned: {name}"
526 );
527 }
528 }
529
530 #[tokio::test]
531 async fn gc_local_releases_never_evicts_a_pinned_dir() {
532 // The 2026-08-25 shape exactly: the oldest dir is the one production is
533 // running, and enough newer rebuilds exist to push it past the count.
534 // Under the count alone it was the first thing deleted.
535 let tmp = tempfile::tempdir().unwrap();
536 let root = tmp.path();
537 let releases = root.join("releases");
538 tokio::fs::create_dir_all(&releases).await.unwrap();
539
540 let total = RELEASES_TO_KEEP + 3;
541 let mut names = Vec::new();
542 for i in 0..total {
543 let name = format!("v{i:02}");
544 let dir = releases.join(&name);
545 tokio::fs::create_dir(&dir).await.unwrap();
546 let f = std::fs::File::open(&dir).unwrap();
547 let when =
548 SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
549 f.set_times(std::fs::FileTimes::new().set_modified(when))
550 .unwrap();
551 names.push(name);
552 }
553
554 // The two oldest: what a tier is running and what it would roll back to.
555 let pinned: PinnedReleases = [names[0].clone(), names[1].clone()].into_iter().collect();
556 gc_local_releases(root, &pinned).await.unwrap();
557
558 for name in [&names[0], &names[1]] {
559 assert!(
560 releases.join(name).exists(),
561 "a referenced artifact was evicted: {name}"
562 );
563 }
564 // And the count still applies to everything else, from a floor that the
565 // pinned pair did not eat into: the newest RELEASES_TO_KEEP unpinned
566 // dirs survive, so pinning two costs two extra slots rather than two of
567 // the five.
568 let unpinned: Vec<_> = names.iter().filter(|n| !pinned.contains(n)).collect();
569 let cut = unpinned.len() - RELEASES_TO_KEEP;
570 for name in unpinned.iter().take(cut) {
571 assert!(
572 !releases.join(name).exists(),
573 "expected to be pruned: {name}"
574 );
575 }
576 for name in unpinned.iter().skip(cut) {
577 assert!(releases.join(name).exists(), "expected to survive: {name}");
578 }
579 }
580
581 #[tokio::test]
582 async fn gc_local_releases_keeps_a_pinned_dir_that_is_not_even_present() {
583 // A pinned name with nothing on disk must not disturb the count. This is
584 // the state the bug leaves behind, and gc runs again while it holds.
585 let tmp = tempfile::tempdir().unwrap();
586 let root = tmp.path();
587 let releases = root.join("releases");
588 tokio::fs::create_dir_all(&releases).await.unwrap();
589 for i in 0..=RELEASES_TO_KEEP {
590 tokio::fs::create_dir(releases.join(format!("v{i}")))
591 .await
592 .unwrap();
593 }
594 let pinned: PinnedReleases = ["gone-already".to_string()].into_iter().collect();
595 gc_local_releases(root, &pinned).await.unwrap();
596
597 let left = std::fs::read_dir(&releases).unwrap().count();
598 assert_eq!(left, RELEASES_TO_KEEP);
599 }
600
601 #[tokio::test]
602 async fn gc_local_releases_noop_when_below_threshold() {
603 let tmp = tempfile::tempdir().unwrap();
604 let root = tmp.path();
605 let releases = root.join("releases");
606 tokio::fs::create_dir_all(&releases).await.unwrap();
607 for i in 0..3 {
608 tokio::fs::create_dir(releases.join(format!("v{i}")))
609 .await
610 .unwrap();
611 }
612 gc_local_releases(root, &no_pins()).await.unwrap();
613 for i in 0..3 {
614 assert!(releases.join(format!("v{i}")).exists());
615 }
616 }
617
618 // ---- remote gc ----
619 //
620 // Driven through `LocalExec`, so these run the real shell the node runs
621 // rather than asserting on the script's text. The script is the half of the
622 // remote gc that can be wrong, and it is wrong with `rm -rf`.
623
624 /// `releases/` with `total` dirs named `v00..`, oldest first by mtime.
625 async fn releases_by_age(root: &Path, total: usize) -> Vec<String> {
626 let releases = root.join("releases");
627 tokio::fs::create_dir_all(&releases).await.unwrap();
628 let mut names = Vec::new();
629 for i in 0..total {
630 let name = format!("v{i:02}");
631 let dir = releases.join(&name);
632 tokio::fs::create_dir(&dir).await.unwrap();
633 // A file inside, so a deletion is visible as more than an empty dir.
634 tokio::fs::write(dir.join("makenotwork"), b"x")
635 .await
636 .unwrap();
637 let f = std::fs::File::open(&dir).unwrap();
638 let when =
639 SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
640 f.set_times(std::fs::FileTimes::new().set_modified(when))
641 .unwrap();
642 names.push(name);
643 }
644 names
645 }
646
647 /// Nothing pinned: the newest `RELEASES_TO_KEEP` survive.
648 #[tokio::test]
649 async fn gc_remote_releases_keeps_last_n_by_mtime_when_nothing_is_pinned() {
650 let tmp = tempfile::tempdir().unwrap();
651 let root = tmp.path();
652 let total = RELEASES_TO_KEEP + 3;
653 let names = releases_by_age(root, total).await;
654
655 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &no_pins())
656 .await
657 .unwrap();
658
659 let releases = root.join("releases");
660 for name in names.iter().take(total - RELEASES_TO_KEEP) {
661 assert!(!releases.join(name).exists(), "expected pruned: {name}");
662 }
663 for name in names.iter().skip(total - RELEASES_TO_KEEP) {
664 assert!(releases.join(name).exists(), "expected to survive: {name}");
665 }
666 }
667
668 /// The done condition, on the node: the dirs a tier's current and previous
669 /// artifacts name survive even when they are the oldest on disk and well
670 /// past the count. Same shape as the host-store test above, which is the
671 /// point — the two stores now answer the same question the same way.
672 #[tokio::test]
673 async fn gc_remote_releases_never_evicts_a_pinned_dir() {
674 let tmp = tempfile::tempdir().unwrap();
675 let root = tmp.path();
676 let total = RELEASES_TO_KEEP + 3;
677 let names = releases_by_age(root, total).await;
678
679 let pinned: PinnedReleases = [names[0].clone(), names[1].clone()].into_iter().collect();
680 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
681 .await
682 .unwrap();
683
684 let releases = root.join("releases");
685 for name in [&names[0], &names[1]] {
686 assert!(
687 releases.join(name).exists(),
688 "a referenced artifact was evicted from the node: {name}"
689 );
690 }
691 // And pinning does not spend the count's slots, again matching the host.
692 let unpinned: Vec<_> = names.iter().filter(|n| !pinned.contains(n)).collect();
693 let cut = unpinned.len() - RELEASES_TO_KEEP;
694 for name in unpinned.iter().take(cut) {
695 assert!(!releases.join(name).exists(), "expected pruned: {name}");
696 }
697 for name in unpinned.iter().skip(cut) {
698 assert!(releases.join(name).exists(), "expected to survive: {name}");
699 }
700 }
701
702 /// Every dir pinned means the loop deletes nothing and the script still
703 /// exits 0. Worth its own test because the obvious implementation of this
704 /// filter is `grep -v`, which exits 1 when it selects no lines and would
705 /// have failed the deploy here under `set -e`.
706 #[tokio::test]
707 async fn gc_remote_releases_succeeds_when_everything_is_pinned() {
708 let tmp = tempfile::tempdir().unwrap();
709 let root = tmp.path();
710 let names = releases_by_age(root, RELEASES_TO_KEEP + 3).await;
711 let pinned: PinnedReleases = names.iter().cloned().collect();
712
713 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
714 .await
715 .unwrap();
716
717 let releases = root.join("releases");
718 for name in &names {
719 assert!(releases.join(name).exists(), "expected to survive: {name}");
720 }
721 }
722
723 /// A `releases/` that does not exist is not an error: a node's first deploy
724 /// creates the dir, and gc runs on the same path.
725 #[tokio::test]
726 async fn gc_remote_releases_is_a_noop_when_the_store_is_missing() {
727 let tmp = tempfile::tempdir().unwrap();
728 gc_remote_releases(&local_executor(), tmp.path().to_str().unwrap(), &no_pins())
729 .await
730 .unwrap();
731 }
732
733 /// Names reach the script as positional parameters, so a name that looks
734 /// like shell must be compared whole rather than expanded or split. None of
735 /// these can be a digest16, but the pre-identity names are version strings
736 /// and the pinned set is data read out of a database.
737 #[tokio::test]
738 async fn gc_remote_releases_quotes_pinned_names() {
739 let tmp = tempfile::tempdir().unwrap();
740 let root = tmp.path();
741 let releases = root.join("releases");
742 tokio::fs::create_dir_all(&releases).await.unwrap();
743 let awkward = ["a b", "x'y", "*"];
744 for name in awkward {
745 tokio::fs::create_dir(releases.join(name)).await.unwrap();
746 }
747 // Enough newer dirs that the count alone would evict all three.
748 let filler: Vec<String> = (0..=RELEASES_TO_KEEP).map(|i| format!("f{i}")).collect();
749 for name in &filler {
750 tokio::fs::create_dir(releases.join(name)).await.unwrap();
751 }
752
753 let pinned: PinnedReleases = awkward.iter().map(|s| (*s).to_string()).collect();
754 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
755 .await
756 .unwrap();
757
758 for name in awkward {
759 assert!(releases.join(name).exists(), "expected to survive: {name}");
760 }
761 }
762
763 /// A pinned name matches a whole directory name, never a prefix of one.
764 /// `case`-with-globbing or a `grep -F` without `-x` would keep `v0` and
765 /// `v01` both because one contains the other, quietly widening the pinned
766 /// set past what the database said.
767 #[tokio::test]
768 async fn gc_remote_releases_matches_whole_names_not_prefixes() {
769 let tmp = tempfile::tempdir().unwrap();
770 let root = tmp.path();
771 let names = releases_by_age(root, RELEASES_TO_KEEP + 3).await;
772
773 // Pin the oldest by an exact name; its neighbours share the prefix.
774 let pinned: PinnedReleases = [names[0].clone()].into_iter().collect();
775 gc_remote_releases(&local_executor(), root.to_str().unwrap(), &pinned)
776 .await
777 .unwrap();
778
779 let releases = root.join("releases");
780 assert!(
781 releases.join(&names[0]).exists(),
782 "the pinned dir was evicted"
783 );
784 assert!(
785 !releases.join(&names[1]).exists(),
786 "a dir sharing the pinned name's prefix was treated as pinned"
787 );
788 }
789
790 #[tokio::test]
791 async fn gc_local_releases_noop_when_releases_dir_missing() {
792 let tmp = tempfile::tempdir().unwrap();
793 gc_local_releases(tmp.path(), &no_pins()).await.unwrap();
794 }
795
796 #[tokio::test]
797 async fn deploy_remote_fails_cleanly_when_host_unreachable() {
798 // 192.0.2.0/24 is reserved for documentation and routes nowhere.
799 // ConnectTimeout=10 limits the test wallclock to ~10s worst case.
800 let tmp = tempfile::tempdir().unwrap();
801 let staged = tmp.path().join("releases").join("0.0.1");
802 tokio::fs::create_dir_all(&staged).await.unwrap();
803 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
804
805 let node = crate::topology::Node {
806 platform: None,
807 base_image: None,
808 libc: None,
809 name: "unreachable".into(),
810 ssh_target: "deploy@192.0.2.1".into(),
811 release_root: "/opt/never".into(),
812 service_name: "makenotwork.service".into(),
813 health_url: None,
814 config_check_env_file: None,
815 actuate: crate::topology::default_actuate(),
816 observe: crate::topology::default_observe(),
817 companions: Vec::new(),
818 };
819 let executor = SshExec::new(
820 node.ssh_target.clone(),
821 CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
822 );
823
824 let placement = Placement::check(&node, &staged, None).expect("both sides silent");
825 let result = deploy_node(&executor, placement, "0.0.1", "server", Some(&no_pins())).await;
826 let err = result.expect_err("deploy to unreachable host should fail");
827 let msg = format!("{err:#}");
828 // Don't pin exact wording, just that the failure is attributed (ssh /
829 // rsync / connection) and that no panic / hang happened.
830 assert!(
831 msg.contains("ssh")
832 || msg.contains("rsync")
833 || msg.contains("connection")
834 || msg.contains("Connection"),
835 "unexpected error: {msg}"
836 );
837 }
838
839 #[tokio::test]
840 async fn deploy_node_with_local_ssh_target_swaps_symlink() {
841 // ssh_target="local" routes to the local fast-path: just a symlink
842 // swap, no remote calls.
843 let tmp = tempfile::tempdir().unwrap();
844 let release_root = tmp.path().to_path_buf();
845 let staged = release_root.join("releases").join("0.0.1");
846 tokio::fs::create_dir_all(&staged).await.unwrap();
847 tokio::fs::write(staged.join("server"), b"x").await.unwrap();
848
849 let node = crate::topology::Node {
850 platform: None,
851 base_image: None,
852 libc: None,
853 name: "local-dev".into(),
854 ssh_target: "local".into(),
855 release_root: release_root.to_string_lossy().into_owned(),
856 service_name: "makenotwork.service".into(),
857 health_url: None,
858 config_check_env_file: None,
859 actuate: crate::topology::default_actuate(),
860 observe: crate::topology::default_observe(),
861 companions: Vec::new(),
862 };
863 let executor = local_executor();
864
865 let out = deploy_node(
866 &executor,
867 Placement::check(&node, &staged, None).unwrap(),
868 "0.0.1",
869 "server",
870 Some(&no_pins()),
871 )
872 .await
873 .unwrap();
874 assert_eq!(out, staged);
875 let target = tokio::fs::read_link(release_root.join("current"))
876 .await
877 .unwrap();
878 assert_eq!(target.to_string_lossy(), "releases/0.0.1");
879 }
880
881 // ---- swap_and_restart_script: symlink/restart consistency ----
882
883 async fn run_script(script: &str) -> std::process::Output {
884 Command::new("sh")
885 .arg("-c")
886 .arg(script)
887 .output()
888 .await
889 .unwrap()
890 }
891
892 async fn setup_release_root(with_current: bool) -> tempfile::TempDir {
893 let tmp = tempfile::tempdir().unwrap();
894 let root = tmp.path();
895 tokio::fs::create_dir_all(root.join("releases/old"))
896 .await
897 .unwrap();
898 tokio::fs::create_dir_all(root.join("releases/new"))
899 .await
900 .unwrap();
901 if with_current {
902 std::os::unix::fs::symlink("releases/old", root.join("current")).unwrap();
903 }
904 tmp
905 }
906
907 #[tokio::test]
908 async fn swap_and_restart_keeps_new_symlink_when_restart_succeeds() {
909 let tmp = setup_release_root(true).await;
910 let root = tmp.path().to_string_lossy().into_owned();
911 let out = run_script(&swap_and_restart_script(&root, "new", "true")).await;
912 assert!(
913 out.status.success(),
914 "script should succeed when restart succeeds"
915 );
916 let target = tokio::fs::read_link(tmp.path().join("current"))
917 .await
918 .unwrap();
919 assert_eq!(
920 target.to_string_lossy(),
921 "releases/new",
922 "symlink advanced to new"
923 );
924 }
925
926 #[tokio::test]
927 async fn swap_and_restart_rolls_symlink_back_when_restart_fails() {
928 // The bug: a restart failure after the flip must NOT leave `current`
929 // pointing at the new (un-activated) release.
930 let tmp = setup_release_root(true).await;
931 let root = tmp.path().to_string_lossy().into_owned();
932 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
933 assert!(!out.status.success(), "script must fail when restart fails");
934 let target = tokio::fs::read_link(tmp.path().join("current"))
935 .await
936 .unwrap();
937 assert_eq!(
938 target.to_string_lossy(),
939 "releases/old",
940 "symlink rolled back to prev so a later restart can't silently activate new",
941 );
942 }
943
944 // ---- arch_guard_script: wrong-arch artifacts fail closed ----
945
946 /// A 20-byte stub whose ELF e_machine field (offset 18, 2 bytes LE) is set.
947 fn elf_stub_with_machine(b18: u8, b19: u8) -> tempfile::NamedTempFile {
948 let mut data = vec![0u8; 20];
949 data[18] = b18;
950 data[19] = b19;
951 let f = tempfile::NamedTempFile::new().unwrap();
952 std::fs::write(f.path(), &data).unwrap();
953 f
954 }
955
956 /// e_machine low byte for the host running the test, if mapped.
957 fn host_machine_lo() -> Option<u8> {
958 match std::env::consts::ARCH {
959 "x86_64" => Some(0x3e),
960 "aarch64" => Some(0xb7),
961 _ => None,
962 }
963 }
964
965 #[tokio::test]
966 async fn arch_guard_passes_for_matching_binary() {
967 let Some(lo) = host_machine_lo() else { return };
968 let f = elf_stub_with_machine(lo, 0x00);
969 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
970 assert!(
971 out.status.success(),
972 "matching arch must pass: {}",
973 String::from_utf8_lossy(&out.stderr),
974 );
975 }
976
977 #[tokio::test]
978 async fn arch_guard_fails_closed_for_wrong_binary() {
979 // Use the other arch's e_machine so it can't match the host.
980 let wrong = match std::env::consts::ARCH {
981 "x86_64" => 0xb7, // aarch64 binary on an x86_64 node
982 "aarch64" => 0x3e, // x86_64 binary on an aarch64 node
983 _ => return,
984 };
985 let f = elf_stub_with_machine(wrong, 0x00);
986 let out = run_script(&arch_guard_script(&f.path().to_string_lossy())).await;
987 assert!(
988 !out.status.success(),
989 "wrong-arch binary must fail closed before the symlink swap"
990 );
991 }
992
993 // ---- ldd_guard_script: a binary this node cannot resolve fails closed ----
994
995 /// A fake `ldd` on PATH that prints `body` and exits `code`, so the guard's
996 /// three outcomes can be exercised without a binary that genuinely fails to
997 /// link. The real `ldd` cannot be made to produce a `not found` on demand.
998 async fn run_ldd_guard_with_fake(body: &str, code: i32) -> std::process::Output {
999 let dir = tempfile::tempdir().unwrap();
1000 let fake = dir.path().join("ldd");
1001 std::fs::write(
1002 &fake,
1003 format!("#!/bin/sh\ncat <<'EOF'\n{body}\nEOF\nexit {code}\n"),
1004 )
1005 .unwrap();
1006 let mut perms = std::fs::metadata(&fake).unwrap().permissions();
1007 std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
1008 std::fs::set_permissions(&fake, perms).unwrap();
1009 let bin = dir.path().join("subject");
1010 std::fs::write(&bin, b"x").unwrap();
1011 Command::new("sh")
1012 .arg("-c")
1013 .arg(ldd_guard_script(&bin.to_string_lossy()))
1014 .env("PATH", format!("{}:/usr/bin:/bin", dir.path().display()))
1015 .output()
1016 .await
1017 .unwrap()
1018 }
1019
1020 #[tokio::test]
1021 async fn ldd_guard_fails_closed_on_an_unsatisfiable_symbol_version() {
1022 // The exact failure Bento's glibc_check used to catch at build time, and
1023 // the reason this guard exists: right arch, resolves every library, and
1024 // still cannot exec because the node's glibc is older than the build
1025 // host's.
1026 let out = run_ldd_guard_with_fake(
1027 "\tlinux-vdso.so.1 (0x00007fff)\n\
1028 \t/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.40' not found (required by ./pom)\n\
1029 \tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
1030 0,
1031 )
1032 .await;
1033 assert!(
1034 !out.status.success(),
1035 "an unsatisfiable symbol version must fail before the symlink swap"
1036 );
1037 let stderr = String::from_utf8_lossy(&out.stderr);
1038 assert!(
1039 stderr.contains("GLIBC_2.40"),
1040 "the offending line must reach the operator, not just a verdict: {stderr}"
1041 );
1042 }
1043
1044 #[tokio::test]
1045 async fn ldd_guard_fails_closed_on_a_missing_library() {
1046 let out = run_ldd_guard_with_fake("\tlibfoo.so.1 => not found", 0).await;
1047 assert!(!out.status.success(), "a missing library must fail closed");
1048 }
1049
1050 #[tokio::test]
1051 async fn ldd_guard_passes_a_resolvable_binary() {
1052 let out = run_ldd_guard_with_fake(
1053 "\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f00)",
1054 0,
1055 )
1056 .await;
1057 assert!(
1058 out.status.success(),
1059 "a fully resolved binary must pass: {}",
1060 String::from_utf8_lossy(&out.stderr),
1061 );
1062 }
1063
1064 #[tokio::test]
1065 async fn ldd_guard_passes_a_static_binary() {
1066 // ldd exits non-zero for these. Nothing to resolve is not a failure.
1067 let out = run_ldd_guard_with_fake("\tnot a dynamic executable", 1).await;
1068 assert!(
1069 out.status.success(),
1070 "a static binary has no dependencies to satisfy: {}",
1071 String::from_utf8_lossy(&out.stderr),
1072 );
1073 }
1074
1075 #[tokio::test]
1076 async fn ldd_guard_fails_when_ldd_errors_for_another_reason() {
1077 // Not the static case: ldd said something else and exited non-zero. We
1078 // do not know the binary is fine, so we do not say it is.
1079 let out = run_ldd_guard_with_fake("ldd: cannot read file", 1).await;
1080 assert!(
1081 !out.status.success(),
1082 "an unexplained ldd failure must not read as a pass"
1083 );
1084 }
1085
1086 #[tokio::test]
1087 async fn ldd_guard_skips_when_the_node_has_no_ldd() {
1088 // Cannot verify is not known bad, matching arch_guard's unmapped-arch
1089 // call. PATH holds nothing, so `command -v ldd` finds none.
1090 let dir = tempfile::tempdir().unwrap();
1091 let bin = dir.path().join("subject");
1092 std::fs::write(&bin, b"x").unwrap();
1093 // Absolute path to the shell: PATH is what this test empties, so
1094 // resolving `sh` through it would fail before the script ever ran.
1095 let out = Command::new("/bin/sh")
1096 .arg("-c")
1097 .arg(ldd_guard_script(&bin.to_string_lossy()))
1098 .env("PATH", dir.path().display().to_string())
1099 .output()
1100 .await
1101 .unwrap();
1102 assert!(
1103 out.status.success(),
1104 "a node with no ldd must not fail the deploy: {}",
1105 String::from_utf8_lossy(&out.stderr),
1106 );
1107 }
1108
1109 #[tokio::test]
1110 async fn swap_and_restart_first_deploy_failure_has_no_prev_to_restore() {
1111 // No prior `current`. A restart failure leaves `current` at new (the only
1112 // version) and still reports failure — documented degenerate case.
1113 let tmp = setup_release_root(false).await;
1114 let root = tmp.path().to_string_lossy().into_owned();
1115 let out = run_script(&swap_and_restart_script(&root, "new", "false")).await;
1116 assert!(!out.status.success(), "script must fail when restart fails");
1117 let target = tokio::fs::read_link(tmp.path().join("current"))
1118 .await
1119 .unwrap();
1120 assert_eq!(
1121 target.to_string_lossy(),
1122 "releases/new",
1123 "no prev existed to roll back to"
1124 );
1125 }
1126
1127 // ---- config_check_script: systemd-faithful env loading ----
1128
1129 #[tokio::test]
1130 async fn config_check_script_loads_values_with_shell_metachars() {
1131 // The bug: `. env_file` expands/word-splits values, so a URL or a
1132 // password containing a shell metacharacter is mangled — it dropped
1133 // DATABASE_URL to empty on a real node, which would fail every deploy.
1134 // The export-loop must load such a value intact. The "binary" is a
1135 // checker script (a real path, like a deployed binary) that exits 0 only
1136 // if the var arrived byte-for-byte — it compares against the expected
1137 // value read from a file, so nothing re-interprets the metacharacters.
1138 let tricky = "postgres://u:p$ss;w&rd@h/db `x` $(y)";
1139 // Plain files in a tempdir: no lingering write fd, so the checker can be
1140 // exec'd (a NamedTempFile stays open and would ETXTBSY).
1141 let dir = tempfile::tempdir().unwrap();
1142 let expected_path = dir.path().join("expected");
1143 std::fs::write(&expected_path, tricky).unwrap(); // no trailing newline
1144
1145 let env_path = dir.path().join("node.env");
1146 std::fs::write(
1147 &env_path,
1148 format!(
1149 "# a comment\n\nDATABASE_URL={tricky}\nOTHER=plain\nEXPECTED_FILE={ef}\n",
1150 ef = expected_path.display(),
1151 ),
1152 )
1153 .unwrap();
1154
1155 let checker_path = dir.path().join("checker.sh");
1156 std::fs::write(
1157 &checker_path,
1158 "#!/bin/sh\nwant=$(cat \"$EXPECTED_FILE\")\n\
1159 [ \"$DATABASE_URL\" = \"$want\" ] || { echo \"DB [$DATABASE_URL] != [$want]\" >&2; exit 1; }\n\
1160 [ \"$OTHER\" = plain ] || { echo \"OTHER [$OTHER]\" >&2; exit 1; }\n",
1161 )
1162 .unwrap();
1163 std::fs::set_permissions(
1164 &checker_path,
1165 std::os::unix::fs::PermissionsExt::from_mode(0o755),
1166 )
1167 .unwrap();
1168
1169 let script = config_check_script(&env_path.to_string_lossy(), &checker_path.to_string_lossy());
1170 let out = run_script(&script).await;
1171 assert!(
1172 out.status.success(),
1173 "value with shell metachars must load intact; stderr: {}",
1174 String::from_utf8_lossy(&out.stderr),
1175 );
1176 }
1177
1178 // ---- install-companion.sh: the node-side guard rails ----
1179
1180 /// Run the shipped installer script with three args; returns its exit code.
1181 /// Exercises the real file rather than a copy of its logic, because the
1182 /// script is the ONLY control on a NOPASSWD sudo grant.
1183 fn run_installer(src: &str, dst: &str, service: &str) -> i32 {
1184 let script =
1185 std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../deploy/install-companion.sh");
1186 std::process::Command::new("bash")
1187 .arg(&script)
1188 .args([src, dst, service])
1189 .output()
1190 .expect("running install-companion.sh")
1191 .status
1192 .code()
1193 .expect("script exited via signal")
1194 }
1195
1196 // Guards run before any filesystem write, so these never install anything.
1197 // Exit 3 = refused by a guard; exit 4 = guards passed, src simply absent.
1198 const REFUSED: i32 = 3;
1199 const PASSED_GUARDS: i32 = 4;
1200
1201 #[test]
1202 fn installer_refuses_a_dst_that_escapes_opt_via_dotdot() {
1203 // `/opt/../etc/...` matches a bare `/opt/*` glob. With the sudoers
1204 // wildcard that meant `install -m 0755` as root to anywhere, plus a
1205 // restart of any unit — so the path must be normalised before the test.
1206 assert_eq!(
1207 run_installer(
1208 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1209 "/opt/../etc/systemd/system/evil.service",
1210 "mnw-cli.service",
1211 ),
1212 REFUSED,
1213 );
1214 }
1215
1216 #[test]
1217 fn installer_refuses_a_src_that_escapes_the_bundle_via_dotdot() {
1218 assert_eq!(
1219 run_installer(
1220 "/opt/mnw/releases/1.0.0/companions/../../../../../etc/shadow",
1221 "/opt/mnw-cli/mnw-cli",
1222 "mnw-cli.service",
1223 ),
1224 REFUSED,
1225 );
1226 }
1227
1228 #[test]
1229 fn installer_accepts_the_real_companion_paths() {
1230 // The guards must not have been tightened into uselessness: the shape
1231 // Sando actually sends has to get past them. It stops at the missing
1232 // src (exit 4), which is proof the guards accepted it.
1233 assert_eq!(
1234 run_installer(
1235 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1236 "/opt/mnw-cli/mnw-cli",
1237 "mnw-cli.service",
1238 ),
1239 PASSED_GUARDS,
1240 );
1241 }
1242
1243 #[test]
1244 fn installer_refuses_a_service_name_with_a_path_separator() {
1245 assert_eq!(
1246 run_installer(
1247 "/opt/mnw/releases/1.0.0/companions/mnw-cli",
1248 "/opt/mnw-cli/mnw-cli",
1249 "../../etc/evil.service",
1250 ),
1251 REFUSED,
1252 );
1253 }
1254
1255 // ---- install_companion_cmd: shape + quoting ----
1256
1257 #[test]
1258 fn install_companion_cmd_shape_and_quoting() {
1259 let cmd = install_companion_cmd(
1260 "/opt/mnw/releases/0.10.14/companions/mnw-cli",
1261 "/opt/mnw-cli/mnw-cli",
1262 "mnw-cli.service",
1263 );
1264 // Routes through the wrapper (single sudoers grant), sudo-invoked, with
1265 // src, dst, service in that order.
1266 assert!(cmd.starts_with("sudo "), "must be sudo-invoked: {cmd}");
1267 assert!(
1268 cmd.contains("/usr/local/lib/mnw/install-companion.sh"),
1269 "{cmd}"
1270 );
1271 let installer_pos = cmd.find("install-companion.sh").unwrap();
1272 let src_pos = cmd.find("companions/mnw-cli").unwrap();
1273 let dst_pos = cmd.find("/opt/mnw-cli/mnw-cli").unwrap();
1274 let svc_pos = cmd.find("mnw-cli.service").unwrap();
1275 assert!(
1276 installer_pos < src_pos && src_pos < dst_pos && dst_pos < svc_pos,
1277 "arg order: {cmd}"
1278 );
1279 }
1280
1281 #[test]
1282 fn install_companion_cmd_quotes_metachars() {
1283 // A path with a space/quote must be shell-safe (defense in depth even
1284 // though these come from operator config).
1285 let cmd = install_companion_cmd("/a b/src", "/dst'x", "u.service");
1286 let out = std::process::Command::new("sh")
1287 .arg("-c")
1288 .arg(format!(
1289 "set -- {}; echo \"$#\"",
1290 cmd.strip_prefix("sudo ").unwrap()
1291 ))
1292 .output()
1293 .unwrap();
1294 // installer + 3 args = 4 positional words after quoting.
1295 assert_eq!(
1296 String::from_utf8_lossy(&out.stdout).trim(),
1297 "4",
1298 "quoting split wrong: {cmd}"
1299 );
1300 }
1301
1302 #[tokio::test]
1303 async fn config_check_script_propagates_binary_failure() {
1304 // A required var missing (the binary exits non-zero) must fail the check.
1305 let env = tempfile::NamedTempFile::new().unwrap();
1306 std::fs::write(env.path(), "FOO=bar\n").unwrap();
1307 let script = config_check_script(&env.path().to_string_lossy(), "false");
1308 let out = run_script(&script).await;
1309 assert!(
1310 !out.status.success(),
1311 "a non-zero MNW_CHECK_CONFIG exit must fail the check"
1312 );
1313 }
1314
1315 #[tokio::test]
1316 async fn deploy_node_denied_when_executor_lacks_deploy_grant() {
1317 // Defense in depth: an executor without the deploy grant refuses the
1318 // step before any filesystem / ssh action.
1319 let tmp = tempfile::tempdir().unwrap();
1320 let release_root = tmp.path().to_path_buf();
1321 let staged = release_root.join("releases").join("0.0.1");
1322 tokio::fs::create_dir_all(&staged).await.unwrap();
1323
1324 let node = crate::topology::Node {
1325 platform: None,
1326 base_image: None,
1327 libc: None,
1328 name: "local-dev".into(),
1329 ssh_target: "local".into(),
1330 release_root: release_root.to_string_lossy().into_owned(),
1331 service_name: "makenotwork.service".into(),
1332 health_url: None,
1333 config_check_env_file: None,
1334 actuate: vec!["restart".into()], // no deploy
1335 observe: vec![],
1336 companions: Vec::new(),
1337 };
1338 let executor = LocalExec::new(CapabilitySet::from_tokens(["restart"], Vec::<&str>::new()));
1339 let err = deploy_node(
1340 &executor,
1341 Placement::check(&node, &staged, None).unwrap(),
1342 "0.0.1",
1343 "server",
1344 Some(&no_pins()),
1345 )
1346 .await
1347 .unwrap_err();
1348 assert!(
1349 format!("{err:#}").contains("capability denied"),
1350 "expected capability denial"
1351 );
1352 }
1353
1354 // ---- FakeExec: the deploy_remote choreography without a real host ----
1355 //
1356 // deploy_node's local fast-path is covered above with a real LocalExec, but
1357 // the remote path (rsync + arch guard + config-drift + swap + companions +
1358 // gc) short-circuits on `ssh_target != "local"` and so never ran under test
1359 // without a reachable node. FakeExec records every executor call in order
1360 // and can be told to fail one shell step (matched by substring) or the rsync
1361 // push, so the ordering and the fail-closed-before-swap contract are
1362 // assertable in-process.
1363
1364 struct FakeExec {
1365 caps: CapabilitySet,
1366 calls: Arc<StdMutex<Vec<String>>>,
1367 /// The first `run_streaming` whose script contains this substring exits
1368 /// non-zero (a failed shell step), e.g. the arch guard.
1369 fail_run_matching: Option<String>,
1370 /// `push_dir` (the rsync) returns an error.
1371 fail_push_dir: bool,
1372 }
1373
1374 impl FakeExec {
1375 fn new() -> Self {
1376 Self {
1377 caps: CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1378 calls: Arc::new(StdMutex::new(Vec::new())),
1379 fail_run_matching: None,
1380 fail_push_dir: false,
1381 }
1382 }
1383 fn log(&self) -> Vec<String> {
1384 self.calls.lock().unwrap().clone()
1385 }
1386 }
1387
1388 #[async_trait]
1389 impl Executor for FakeExec {
1390 async fn run_streaming(&self, step: &Step, _sink: &mut dyn LogSink) -> Result<RunOutput> {
1391 // Every deploy step is a `Step::shell`, so the script is argv's tail.
1392 let script = step.argv.last().cloned().unwrap_or_default();
1393 self.calls.lock().unwrap().push(format!("run:{script}"));
1394 let fail = self
1395 .fail_run_matching
1396 .as_deref()
1397 .is_some_and(|m| script.contains(m));
1398 Ok(RunOutput {
1399 status: std::process::ExitStatus::from_raw(if fail { 1 << 8 } else { 0 }),
1400 stdout: Vec::new(),
1401 stderr: if fail {
1402 b"fake step failure".to_vec()
1403 } else {
1404 Vec::new()
1405 },
1406 })
1407 }
1408 async fn pull_file(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1409 self.calls.lock().unwrap().push("pull_file".into());
1410 Ok(())
1411 }
1412 async fn pull_dir(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1413 self.calls.lock().unwrap().push("pull_dir".into());
1414 Ok(())
1415 }
1416 async fn pull_glob(&self, _glob: &str, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1417 self.calls.lock().unwrap().push("pull_glob".into());
1418 Ok(())
1419 }
1420 async fn push_dir(&self, _local: &Path, remote: &Path, _opts: &SyncOpts) -> Result<()> {
1421 self.calls
1422 .lock()
1423 .unwrap()
1424 .push(format!("push_dir:{}", remote.display()));
1425 if self.fail_push_dir {
1426 anyhow::bail!("fake rsync failure");
1427 }
1428 Ok(())
1429 }
1430 fn capabilities(&self) -> &CapabilitySet {
1431 &self.caps
1432 }
1433 }
1434
1435 fn remote_node(config_check: bool, companions: Vec<NodeCompanion>) -> Node {
1436 Node {
1437 platform: None,
1438 base_image: None,
1439 libc: None,
1440 name: "web-a".into(),
1441 ssh_target: "deploy@web-a".into(),
1442 release_root: "/opt/mnw".into(),
1443 service_name: "makenotwork.service".into(),
1444 health_url: None,
1445 config_check_env_file: config_check.then(|| "/etc/mnw/node.env".to_string()),
1446 actuate: crate::topology::default_actuate(),
1447 observe: crate::topology::default_observe(),
1448 companions,
1449 }
1450 }
1451
1452 fn companion() -> NodeCompanion {
1453 NodeCompanion {
1454 name: "mnw-cli".into(),
1455 install_path: "/opt/mnw-cli/mnw-cli".into(),
1456 service_name: "mnw-cli.service".into(),
1457 }
1458 }
1459
1460 /// Index of the first recorded call whose text contains `needle` (panics if
1461 /// absent — the assertion message names what was missing).
1462 fn pos(log: &[String], needle: &str) -> usize {
1463 log.iter()
1464 .position(|c| c.contains(needle))
1465 .unwrap_or_else(|| panic!("no call matched {needle:?} in {log:#?}"))
1466 }
1467
1468 #[tokio::test]
1469 async fn deploy_remote_runs_the_full_choreography_in_order() {
1470 // A node opted into the config-drift check and carrying one companion:
1471 // mkdir -> rsync -> arch guard -> config check -> swap+restart ->
1472 // companion install -> gc, in that order.
1473 let tmp = tempfile::tempdir().unwrap();
1474 let staged = tmp.path().join("releases").join("0.9.0");
1475 tokio::fs::create_dir_all(&staged).await.unwrap();
1476
1477 let node = remote_node(true, vec![companion()]);
1478 let exec = FakeExec::new();
1479 let out = deploy_node(
1480 &exec,
1481 Placement::check(&node, &staged, None).unwrap(),
1482 "0.9.0",
1483 "makenotwork",
1484 Some(&no_pins()),
1485 )
1486 .await
1487 .expect("deploy_remote should succeed against the fake");
1488 assert_eq!(out, PathBuf::from("/opt/mnw/releases/0.9.0"));
1489
1490 let log = exec.log();
1491 let mkdir = pos(&log, "mkdir -p");
1492 let rsync = pos(&log, "push_dir:/opt/mnw/releases/0.9.0");
1493 let arch = pos(&log, "e_machine");
1494 let cfg = pos(&log, "MNW_CHECK_CONFIG=1");
1495 let swap = pos(&log, "reload-or-restart");
1496 let comp = pos(&log, "install-companion.sh");
1497 let gc = pos(&log, "ls -1t");
1498 assert!(
1499 mkdir < rsync && rsync < arch && arch < cfg && cfg < swap && swap < comp && comp < gc,
1500 "deploy steps out of order: {log:#?}"
1501 );
1502 }
1503
1504 #[tokio::test]
1505 async fn deploy_remote_aborts_before_swap_when_rsync_fails() {
1506 // The rsync failing must fail the deploy BEFORE the symlink swap — the
1507 // "current symlink left intact" contract. Assert the swap never ran.
1508 let tmp = tempfile::tempdir().unwrap();
1509 let staged = tmp.path().join("releases").join("0.9.0");
1510 tokio::fs::create_dir_all(&staged).await.unwrap();
1511
1512 let node = remote_node(false, Vec::new());
1513 let mut exec = FakeExec::new();
1514 exec.fail_push_dir = true;
1515 let err = deploy_node(
1516 &exec,
1517 Placement::check(&node, &staged, None).unwrap(),
1518 "0.9.0",
1519 "makenotwork",
1520 Some(&no_pins()),
1521 )
1522 .await
1523 .expect_err("rsync failure must fail the deploy");
1524 assert!(
1525 format!("{err:#}").contains("rsync"),
1526 "error should attribute the rsync: {err:#}"
1527 );
1528 let log = exec.log();
1529 assert!(
1530 !log.iter().any(|c| c.contains("reload-or-restart")),
1531 "swap must not run after a failed rsync: {log:#?}"
1532 );
1533 }
1534
1535 #[tokio::test]
1536 async fn deploy_remote_aborts_before_swap_when_arch_guard_fails() {
1537 // A wrong-arch binary must fail closed before the swap. The fake fails
1538 // the arch-guard shell step; the swap must not follow.
1539 let tmp = tempfile::tempdir().unwrap();
1540 let staged = tmp.path().join("releases").join("0.9.0");
1541 tokio::fs::create_dir_all(&staged).await.unwrap();
1542
1543 let node = remote_node(false, Vec::new());
1544 let mut exec = FakeExec::new();
1545 exec.fail_run_matching = Some("e_machine".into());
1546 let err = deploy_node(
1547 &exec,
1548 Placement::check(&node, &staged, None).unwrap(),
1549 "0.9.0",
1550 "makenotwork",
1551 Some(&no_pins()),
1552 )
1553 .await
1554 .expect_err("arch mismatch must fail the deploy");
1555 assert!(
1556 format!("{err:#}").contains("architecture"),
1557 "error should mention the arch check: {err:#}"
1558 );
1559 let log = exec.log();
1560 assert!(
1561 !log.iter().any(|c| c.contains("reload-or-restart")),
1562 "swap must not run after a failed arch guard: {log:#?}"
1563 );
1564 }
1565
1566 #[tokio::test]
1567 async fn deploy_remote_skips_config_check_when_node_opts_out() {
1568 // No config_check_env_file => the pre-swap config check is skipped, but
1569 // the rest of the choreography (including the swap) still runs.
1570 let tmp = tempfile::tempdir().unwrap();
1571 let staged = tmp.path().join("releases").join("0.9.0");
1572 tokio::fs::create_dir_all(&staged).await.unwrap();
1573
1574 let node = remote_node(false, Vec::new());
1575 let exec = FakeExec::new();
1576 deploy_node(
1577 &exec,
1578 Placement::check(&node, &staged, None).unwrap(),
1579 "0.9.0",
1580 "makenotwork",
1581 Some(&no_pins()),
1582 )
1583 .await
1584 .unwrap();
1585 let log = exec.log();
1586 assert!(
1587 !log.iter().any(|c| c.contains("MNW_CHECK_CONFIG=1")),
1588 "config check must be skipped when the node opts out: {log:#?}"
1589 );
1590 assert!(
1591 log.iter().any(|c| c.contains("reload-or-restart")),
1592 "the swap must still run: {log:#?}"
1593 );
1594 }
1595
1596 /// A companion is guarded on the same terms as the primary, and BEFORE the
1597 /// swap. Unguarded, the first thing to notice a bad companion is its unit
1598 /// failing to start during the install loop, which runs after the server has
1599 /// already been restarted.
1600 #[tokio::test]
1601 async fn companions_are_guarded_before_the_swap() {
1602 let tmp = tempfile::tempdir().unwrap();
1603 let staged = tmp.path().join("releases").join("0.9.0");
1604 tokio::fs::create_dir_all(&staged).await.unwrap();
1605
1606 let node = remote_node(false, vec![companion()]);
1607 let exec = FakeExec::new();
1608 deploy_node(
1609 &exec,
1610 Placement::check(&node, &staged, None).unwrap(),
1611 "0.9.0",
1612 "makenotwork",
1613 Some(&no_pins()),
1614 )
1615 .await
1616 .unwrap();
1617
1618 let log = exec.log();
1619 // The companion's own arch and loader checks, named by its path so they
1620 // cannot be confused with the primary's.
1621 let guard = pos(&log, "companions/mnw-cli");
1622 let swap = pos(&log, "reload-or-restart");
1623 let install = pos(&log, "install-companion.sh");
1624 assert!(
1625 guard < swap && swap < install,
1626 "a companion must be guarded before the swap and installed after it: {log:#?}"
1627 );
1628 let companion_guards = log
1629 .iter()
1630 .filter(|c| c.contains("companions/mnw-cli") && !c.contains("install-companion.sh"))
1631 .count();
1632 assert_eq!(
1633 companion_guards, 2,
1634 "both guards must run against the companion, not just one: {log:#?}"
1635 );
1636 }
1637
1638 /// And failing one of them fails the promote with the service intact, which
1639 /// is the whole point of moving the check ahead of the swap.
1640 #[tokio::test]
1641 async fn a_companion_failing_its_guard_aborts_before_the_swap() {
1642 let tmp = tempfile::tempdir().unwrap();
1643 let staged = tmp.path().join("releases").join("0.9.0");
1644 tokio::fs::create_dir_all(&staged).await.unwrap();
1645
1646 let node = remote_node(false, vec![companion()]);
1647 let mut exec = FakeExec::new();
1648 // Fails the first script naming the companion, which is its arch guard.
1649 // The primary's guards name the primary and are unaffected.
1650 exec.fail_run_matching = Some("companions/mnw-cli".into());
1651 let err = deploy_node(
1652 &exec,
1653 Placement::check(&node, &staged, None).unwrap(),
1654 "0.9.0",
1655 "makenotwork",
1656 Some(&no_pins()),
1657 )
1658 .await
1659 .expect_err("a bad companion must fail the deploy");
1660
1661 let msg = format!("{err:#}");
1662 assert!(
1663 msg.contains("mnw-cli"),
1664 "the refusal must name which companion: {msg}"
1665 );
1666 assert_eq!(
1667 stage_of(&err),
1668 Some(FailureStage::BeforeSwap),
1669 "a companion guard failing must leave the service intact: {msg}"
1670 );
1671 let log = exec.log();
1672 assert!(
1673 !log.iter().any(|c| c.contains("reload-or-restart")),
1674 "swap must not run after a failed companion guard: {log:#?}"
1675 );
1676 assert!(
1677 !log.iter().any(|c| c.contains("install-companion.sh")),
1678 "nothing should be installed after a failed companion guard: {log:#?}"
1679 );
1680 }
1681
1682 /// The guards and the installer must read the same path. A guard checking a
1683 /// path the installer does not use is a check of nothing, and passes.
1684 #[test]
1685 fn the_guarded_companion_path_is_the_one_installed() {
1686 let release_dir = "/opt/mnw/releases/0.9.0";
1687 let src = companion_src(release_dir, "mnw-cli");
1688 assert_eq!(src, "/opt/mnw/releases/0.9.0/companions/mnw-cli");
1689 let cmd = install_companion_cmd(&src, "/opt/mnw-cli/mnw-cli", "mnw-cli.service");
1690 assert!(
1691 cmd.contains(&src),
1692 "the installer must read the path the guards checked: {cmd}"
1693 );
1694 }
1695
1696 #[tokio::test]
1697 async fn deploy_remote_installs_companion_after_the_swap() {
1698 // Companions are After= the server: their install must land after the
1699 // symlink swap + service restart, never before.
1700 let tmp = tempfile::tempdir().unwrap();
1701 let staged = tmp.path().join("releases").join("0.9.0");
1702 tokio::fs::create_dir_all(&staged).await.unwrap();
1703
1704 let node = remote_node(false, vec![companion()]);
1705 let exec = FakeExec::new();
1706 deploy_node(
1707 &exec,
1708 Placement::check(&node, &staged, None).unwrap(),
1709 "0.9.0",
1710 "makenotwork",
1711 Some(&no_pins()),
1712 )
1713 .await
1714 .unwrap();
1715 let log = exec.log();
1716 assert!(
1717 pos(&log, "reload-or-restart") < pos(&log, "install-companion.sh"),
1718 "companion install must follow the swap: {log:#?}"
1719 );
1720 }
1721