| 567 |
567 |
|
.context("the target node cannot satisfy the deployed binary's dynamic dependencies")
|
| 568 |
568 |
|
.context(FailureStage::BeforeSwap)?;
|
| 569 |
569 |
|
|
|
570 |
+ |
// The same two guards for every companion, and for the same reason. Both
|
|
571 |
+ |
// checks above take the primary binary alone, so a companion of the wrong
|
|
572 |
+ |
// architecture, or one linking a symbol version this node lacks, used to be
|
|
573 |
+ |
// discovered by its unit failing to start — during the install loop below,
|
|
574 |
+ |
// which runs AFTER the swap. That is the expensive side of the line these
|
|
575 |
+ |
// guards exist to stay on: the promote fails either way, but with the server
|
|
576 |
+ |
// already restarted onto the new release.
|
|
577 |
+ |
//
|
|
578 |
+ |
// The companion bytes are present and verified by now: they arrived in the
|
|
579 |
+ |
// same rsync and `manifest_verify_script` above covers the whole release
|
|
580 |
+ |
// directory, `companions/` included. So there is nothing to wait for.
|
|
581 |
+ |
for c in &node.companions {
|
|
582 |
+ |
let src = companion_src(&release_dir, &c.name);
|
|
583 |
+ |
run_checked(
|
|
584 |
+ |
executor,
|
|
585 |
+ |
&arch_guard_script(&src),
|
|
586 |
+ |
"verifying companion arch matches node",
|
|
587 |
+ |
)
|
|
588 |
+ |
.await
|
|
589 |
+ |
.with_context(|| {
|
|
590 |
+ |
format!(
|
|
591 |
+ |
"companion {} architecture does not match the target node",
|
|
592 |
+ |
c.name
|
|
593 |
+ |
)
|
|
594 |
+ |
})
|
|
595 |
+ |
.context(FailureStage::BeforeSwap)?;
|
|
596 |
+ |
run_checked(
|
|
597 |
+ |
executor,
|
|
598 |
+ |
&ldd_guard_script(&src),
|
|
599 |
+ |
"verifying the node can resolve the companion's dynamic dependencies",
|
|
600 |
+ |
)
|
|
601 |
+ |
.await
|
|
602 |
+ |
.with_context(|| {
|
|
603 |
+ |
format!(
|
|
604 |
+ |
"the target node cannot satisfy companion {}'s dynamic dependencies",
|
|
605 |
+ |
c.name
|
|
606 |
+ |
)
|
|
607 |
+ |
})
|
|
608 |
+ |
.context(FailureStage::BeforeSwap)?;
|
|
609 |
+ |
}
|
|
610 |
+ |
|
| 570 |
611 |
|
// Config-drift guard (opt-in per node). Runs the freshly-rsynced binary in
|
| 571 |
612 |
|
// config-only mode with the node's env sourced, BEFORE the swap, so a
|
| 572 |
613 |
|
// required var missing on this node fails here — service still intact —
|
| 601 |
642 |
|
// in this SAME bundle — the lockstep guarantee. A failure here fails the
|
| 602 |
643 |
|
// promote: a companion is part of the deploy, not a best-effort side effect.
|
| 603 |
644 |
|
for c in &node.companions {
|
| 604 |
|
- |
let src = format!(
|
| 605 |
|
- |
"{release_root}/releases/{release_id}/companions/{name}",
|
| 606 |
|
- |
name = c.name,
|
| 607 |
|
- |
);
|
|
645 |
+ |
let src = companion_src(&release_dir, &c.name);
|
| 608 |
646 |
|
tracing::info!(node = %node.name, companion = %c.name, "deploy: install companion + restart");
|
| 609 |
647 |
|
let cmd = install_companion_cmd(&src, &c.install_path, &c.service_name);
|
| 610 |
648 |
|
run_checked(executor, &cmd, "install companion + restart")
|
| 627 |
665 |
|
.join(release_id))
|
| 628 |
666 |
|
}
|
| 629 |
667 |
|
|
|
668 |
+ |
/// Where a companion's binary sits inside the staged release directory.
|
|
669 |
+ |
///
|
|
670 |
+ |
/// One function because two places need it and they must not drift: the guards
|
|
671 |
+ |
/// that run before the swap check this path, and the installer after the swap
|
|
672 |
+ |
/// reads it. A guard that checked a path the installer did not use would be a
|
|
673 |
+ |
/// check of nothing, and would look exactly like a passing check.
|
|
674 |
+ |
fn companion_src(release_dir: &str, name: &str) -> String {
|
|
675 |
+ |
format!("{release_dir}/companions/{name}")
|
|
676 |
+ |
}
|
|
677 |
+ |
|
| 630 |
678 |
|
/// Absolute path of the node-side companion installer (shipped once per node;
|
| 631 |
679 |
|
/// granted to the deploy user by a single scoped sudoers line). It installs the
|
| 632 |
680 |
|
/// staged binary to its `ExecStart` path and restarts the unit — keeping the
|
| 2262 |
2310 |
|
);
|
| 2263 |
2311 |
|
}
|
| 2264 |
2312 |
|
|
|
2313 |
+ |
/// A companion is guarded on the same terms as the primary, and BEFORE the
|
|
2314 |
+ |
/// swap. It used to be checked by nothing at all, so the first thing that
|
|
2315 |
+ |
/// noticed a bad companion was its unit failing to start during the install
|
|
2316 |
+ |
/// loop, which runs after the server has already been restarted.
|
|
2317 |
+ |
#[tokio::test]
|
|
2318 |
+ |
async fn companions_are_guarded_before_the_swap() {
|
|
2319 |
+ |
let tmp = tempfile::tempdir().unwrap();
|
|
2320 |
+ |
let staged = tmp.path().join("releases").join("0.9.0");
|
|
2321 |
+ |
tokio::fs::create_dir_all(&staged).await.unwrap();
|
|
2322 |
+ |
|
|
2323 |
+ |
let node = remote_node(false, vec![companion()]);
|
|
2324 |
+ |
let exec = FakeExec::new();
|
|
2325 |
+ |
deploy_node(
|
|
2326 |
+ |
&exec,
|
|
2327 |
+ |
Placement::check(&node, &staged, None).unwrap(),
|
|
2328 |
+ |
"0.9.0",
|
|
2329 |
+ |
"makenotwork",
|
|
2330 |
+ |
)
|
|
2331 |
+ |
.await
|
|
2332 |
+ |
.unwrap();
|
|
2333 |
+ |
|
|
2334 |
+ |
let log = exec.log();
|
|
2335 |
+ |
// The companion's own arch and loader checks, named by its path so they
|
|
2336 |
+ |
// cannot be confused with the primary's.
|
|
2337 |
+ |
let guard = pos(&log, "companions/mnw-cli");
|
|
2338 |
+ |
let swap = pos(&log, "reload-or-restart");
|
|
2339 |
+ |
let install = pos(&log, "install-companion.sh");
|
|
2340 |
+ |
assert!(
|
|
2341 |
+ |
guard < swap && swap < install,
|
|
2342 |
+ |
"a companion must be guarded before the swap and installed after it: {log:#?}"
|
|
2343 |
+ |
);
|
|
2344 |
+ |
let companion_guards = log
|
|
2345 |
+ |
.iter()
|
|
2346 |
+ |
.filter(|c| c.contains("companions/mnw-cli") && !c.contains("install-companion.sh"))
|
|
2347 |
+ |
.count();
|
|
2348 |
+ |
assert_eq!(
|
|
2349 |
+ |
companion_guards, 2,
|
|
2350 |
+ |
"both guards must run against the companion, not just one: {log:#?}"
|
|
2351 |
+ |
);
|
|
2352 |
+ |
}
|
|
2353 |
+ |
|
|
2354 |
+ |
/// And failing one of them fails the promote with the service intact, which
|
|
2355 |
+ |
/// is the whole point of moving the check ahead of the swap.
|
|
2356 |
+ |
#[tokio::test]
|
|
2357 |
+ |
async fn a_companion_failing_its_guard_aborts_before_the_swap() {
|
|
2358 |
+ |
let tmp = tempfile::tempdir().unwrap();
|
|
2359 |
+ |
let staged = tmp.path().join("releases").join("0.9.0");
|
|
2360 |
+ |
tokio::fs::create_dir_all(&staged).await.unwrap();
|
|
2361 |
+ |
|
|
2362 |
+ |
let node = remote_node(false, vec![companion()]);
|
|
2363 |
+ |
let mut exec = FakeExec::new();
|
|
2364 |
+ |
// Fails the first script naming the companion, which is its arch guard.
|
|
2365 |
+ |
// The primary's guards name the primary and are unaffected.
|
|
2366 |
+ |
exec.fail_run_matching = Some("companions/mnw-cli".into());
|
|
2367 |
+ |
let err = deploy_node(
|
|
2368 |
+ |
&exec,
|
|
2369 |
+ |
Placement::check(&node, &staged, None).unwrap(),
|
|
2370 |
+ |
"0.9.0",
|
|
2371 |
+ |
"makenotwork",
|
|
2372 |
+ |
)
|
|
2373 |
+ |
.await
|
|
2374 |
+ |
.expect_err("a bad companion must fail the deploy");
|
|
2375 |
+ |
|
|
2376 |
+ |
let msg = format!("{err:#}");
|
|
2377 |
+ |
assert!(
|
|
2378 |
+ |
msg.contains("mnw-cli"),
|
|
2379 |
+ |
"the refusal must name which companion: {msg}"
|
|
2380 |
+ |
);
|
|
2381 |
+ |
assert_eq!(
|
|
2382 |
+ |
stage_of(&err),
|
|
2383 |
+ |
Some(FailureStage::BeforeSwap),
|
|
2384 |
+ |
"a companion guard failing must leave the service intact: {msg}"
|
|
2385 |
+ |
);
|
|
2386 |
+ |
let log = exec.log();
|
|
2387 |
+ |
assert!(
|
|
2388 |
+ |
!log.iter().any(|c| c.contains("reload-or-restart")),
|
|
2389 |
+ |
"swap must not run after a failed companion guard: {log:#?}"
|
|
2390 |
+ |
);
|
|
2391 |
+ |
assert!(
|
|
2392 |
+ |
!log.iter().any(|c| c.contains("install-companion.sh")),
|
|
2393 |
+ |
"nothing should be installed after a failed companion guard: {log:#?}"
|
|
2394 |
+ |
);
|
|
2395 |
+ |
}
|
|
2396 |
+ |
|
|
2397 |
+ |
/// The guards and the installer must read the same path. A guard checking a
|
|
2398 |
+ |
/// path the installer does not use is a check of nothing, and passes.
|
|
2399 |
+ |
#[test]
|
|
2400 |
+ |
fn the_guarded_companion_path_is_the_one_installed() {
|
|
2401 |
+ |
let release_dir = "/opt/mnw/releases/0.9.0";
|
|
2402 |
+ |
let src = companion_src(release_dir, "mnw-cli");
|
|
2403 |
+ |
assert_eq!(src, "/opt/mnw/releases/0.9.0/companions/mnw-cli");
|
|
2404 |
+ |
let cmd = install_companion_cmd(&src, "/opt/mnw-cli/mnw-cli", "mnw-cli.service");
|
|
2405 |
+ |
assert!(
|
|
2406 |
+ |
cmd.contains(&src),
|
|
2407 |
+ |
"the installer must read the path the guards checked: {cmd}"
|
|
2408 |
+ |
);
|
|
2409 |
+ |
}
|
|
2410 |
+ |
|
| 2265 |
2411 |
|
#[tokio::test]
|
| 2266 |
2412 |
|
async fn deploy_remote_installs_companion_after_the_swap() {
|
| 2267 |
2413 |
|
// Companions are After= the server: their install must land after the
|