Skip to main content

max / makenotwork

47.6 KB · 1258 lines History Blame Raw
1 use crate::domain::{GateKind, NodeId, TierId};
2 use anyhow::{Context, Result};
3 use serde::{Deserialize, Serialize};
4 use std::path::Path;
5
6 #[derive(Debug, Clone, Serialize, Deserialize)]
7 pub struct Topology {
8 /// The repo Sando checks out to build this product.
9 ///
10 /// `None` for an intake-only product: pom is built by Bento on two
11 /// machines and handed over as finished bytes, so Sando fetches no source
12 /// for it and there is no bare repo on this host to name. A topology that
13 /// declares no repo cannot be `/rebuild`-ed, which is the same statement
14 /// [`AppConfig::build_host`](crate::config::AppConfig::build_host) makes
15 /// from the other side.
16 #[serde(default)]
17 pub repo: Option<RepoConfig>,
18 /// Prod dumps `/backup/fetch` pulls, one per database `migration_dry_run`
19 /// has a check for. A list because the repo ships more than one service
20 /// with its own database and its own `sqlx::migrate!()` at boot: the server
21 /// migrates `makenotwork`, multithreaded migrates `multithreaded`, and a
22 /// gate that restores only the first proves nothing about the second.
23 ///
24 /// Accepts both the historical single `[backup]` table and a `[[backup]]`
25 /// list, so a deployed `sando.toml` keeps working unedited (the single form
26 /// deserializes to a one-entry list named `server`).
27 #[serde(deserialize_with = "one_or_many_backup")]
28 pub backup: Vec<BackupConfig>,
29 #[serde(rename = "tier")]
30 pub tiers: Vec<Tier>,
31 /// Extra repos to fetch and check out beside the main worktree before a
32 /// build, so a path dependency that reaches across the repo split resolves.
33 /// Empty (default) keeps an existing `sando.toml` working unedited. See
34 /// [`AuxRepo`] and [`crate::build::checkout_aux_repos`].
35 #[serde(default, rename = "aux_repo")]
36 pub aux_repos: Vec<AuxRepo>,
37 }
38
39 /// An auxiliary repo checked out beside the per-sha worktree so cross-repo path
40 /// dependencies resolve at build time.
41 ///
42 /// The concrete case (2026-07-24): `mnw-cli` — a companion built from the MNW
43 /// worktree — carries `synckit-client = { path = "../../synckit/synckit-client" }`
44 /// after synckit moved to its own repo. From the companion crate at
45 /// `<workdir>/<sha>/mnw-cli`, that path resolves to `<workdir>/synckit`, a sibling
46 /// of the per-sha worktree that Sando otherwise never creates, so the companion
47 /// build failed with "No such file or directory". An `aux_repo` named to land at
48 /// `checkout_dir = "synckit"` puts the synckit source exactly there.
49 ///
50 /// The checkout is at the FIXED `<workdir>/<checkout_dir>`, not per-sha: the path
51 /// dependency resolves to that spot regardless of the MNW sha, and builds
52 /// serialize, so a single shared checkout refreshed to `branch` HEAD each build
53 /// is correct. Because a path dep has no lockfile pin, "branch HEAD" is the honest
54 /// resolution — the same contract as the dev working copy.
55 #[derive(Debug, Clone, Serialize, Deserialize)]
56 pub struct AuxRepo {
57 /// Human label for logs and errors (e.g. `synckit`).
58 pub name: String,
59 /// Bare repo Sando fetches into and worktrees from, e.g.
60 /// `/srv/sando/synckit.git`. Auto-created (hookless) on first build.
61 pub bare_path: String,
62 /// Canonical git remote fetched before checkout. Like [`RepoConfig::upstream`]
63 /// but required here: an aux repo is pull-based (nobody pushes to its bare).
64 pub upstream: String,
65 /// Branch whose HEAD is checked out.
66 pub branch: String,
67 /// Where the worktree lands, relative to `cfg.workdir`. May name a nested
68 /// location (`Libraries/docengine`), because a path dep resolves to wherever
69 /// the dev tree keeps the crate and Sando has to match that shape. Every
70 /// component must be a plain name — no leading slash, no `..` — so the
71 /// checkout stays under the workdir.
72 pub checkout_dir: String,
73 }
74
75 #[derive(Debug, Clone, Serialize, Deserialize)]
76 pub struct RepoConfig {
77 pub bare_path: String,
78 pub branch: String,
79 /// Optional canonical git remote. When set, `/rebuild` fetches the deploy
80 /// branch from here into the bare repo before worktree-ing the target sha,
81 /// so a freshly-pushed commit is resolvable without anyone pushing to the
82 /// bare repo directly (pull-based deploys). When unset, Sando relies on the
83 /// bare repo already containing the sha (push-based / hook-driven).
84 #[serde(default)]
85 pub upstream: Option<String>,
86 }
87
88 #[derive(Debug, Clone, Serialize, Deserialize)]
89 pub struct BackupConfig {
90 /// Which database this dump is of, as referenced by a daemon-config
91 /// `[[migration_check]]`'s `backup` key and recorded in the `backups`
92 /// table's `name` column. Defaults to `server` so the historical single
93 /// `[backup]` table needs no edit — and so the pre-existing rows, which the
94 /// state-DB migration backfills to `server`, keep matching it.
95 #[serde(default = "default_backup_name")]
96 pub name: String,
97 pub source: String,
98 pub local_path: String,
99 }
100
101 fn default_backup_name() -> String {
102 "server".into()
103 }
104
105 /// Accept `[backup]` (one table) or `[[backup]]` (a list) for the same key.
106 /// Serde cannot express "table or sequence" on a `Vec` field on its own, and
107 /// the alternative — renaming the key — would break every deployed
108 /// `sando.toml` at startup, on the box whose whole job is deploying.
109 fn one_or_many_backup<'de, D>(de: D) -> std::result::Result<Vec<BackupConfig>, D::Error>
110 where
111 D: serde::Deserializer<'de>,
112 {
113 #[derive(Deserialize)]
114 #[serde(untagged)]
115 enum OneOrMany {
116 One(BackupConfig),
117 Many(Vec<BackupConfig>),
118 }
119 Ok(match OneOrMany::deserialize(de)? {
120 OneOrMany::One(b) => vec![b],
121 OneOrMany::Many(v) => v,
122 })
123 }
124
125 #[derive(Debug, Clone, Serialize, Deserialize)]
126 pub struct Tier {
127 pub name: TierId,
128 #[serde(default)]
129 pub provisioned: bool,
130 pub gates: Vec<Gate>,
131 #[serde(default)]
132 pub canary: CanaryPolicy,
133 #[serde(default, rename = "node")]
134 pub nodes: Vec<Node>,
135 /// Where the public reaches this tier, CDN included.
136 ///
137 /// Deliberately not derivable from a node's `ssh_target`. The whole value of
138 /// [`Gate::PageSmoke`] is that it goes the way a visitor goes -- through
139 /// Cloudflare, against the hostname in the browser's address bar -- and a
140 /// URL computed from the node would reach the origin and inherit exactly the
141 /// blind spot the gate exists to close. So it is stated, or the gate is not
142 /// available.
143 #[serde(default)]
144 pub public_url: Option<String>,
145 }
146
147 #[derive(Debug, Clone, Serialize, Deserialize)]
148 pub struct Node {
149 pub name: NodeId,
150 pub ssh_target: String,
151 pub release_root: String,
152 /// What this machine runs, as `os/arch` (e.g. `linux/aarch64`).
153 ///
154 /// Compared against the bundle's own platform before anything is pushed;
155 /// see [`crate::deploy::Placement`]. Optional, and a node that declares it
156 /// can only be given a bundle that declares a matching one — silence on
157 /// either side is a refusal, not a pass. MNW's nodes declare nothing and
158 /// keep the single-platform behavior they have always had; pom's declare
159 /// theirs, because pom is the product where one version is two bundles.
160 #[serde(default)]
161 pub platform: Option<crate::domain::Platform>,
162 /// systemd unit name to reload-or-restart after the symlink swap.
163 /// Defaults to "makenotwork.service" because that's MNW's prod unit.
164 #[serde(default = "default_service_name")]
165 pub service_name: String,
166 /// Opt-in config-drift guard. When set to the node's env-file path (e.g.
167 /// `/etc/mnw/makenotwork.env`), the deploy sources that file and runs the
168 /// freshly-rsynced binary in `MNW_CHECK_CONFIG=1` mode BEFORE the symlink
169 /// swap. A required var missing on this node (a var added upstream but never
170 /// added to the node's env — how testnot crash-looped on CDN_BASE_URL) then
171 /// fails the promote with the running service still intact, instead of after
172 /// the swap+restart. Unset (default) skips the check, so an existing
173 /// `sando.toml` keeps working and it never runs against a binary too old to
174 /// support the mode; enable it once a check-capable version is deployed.
175 #[serde(default)]
176 pub config_check_env_file: Option<String>,
177 /// Capability grant for this node's executor (see `ops_exec`). Defaults to
178 /// the current behavior of every Sando node — actuate deploy+restart,
179 /// observe health — so an existing `sando.toml` keeps working unedited.
180 #[serde(default = "default_actuate")]
181 pub actuate: Vec<String>,
182 #[serde(default = "default_observe")]
183 pub observe: Vec<String>,
184 /// Optional HTTP readiness URL the `node_health` gate curls on the node (over
185 /// its executor) after `systemctl is-active`. Typically a loopback address
186 /// the service binds, e.g. `http://127.0.0.1:8080/health`. When unset, the
187 /// gate proves the unit is active post-restart but does not HTTP-probe; set
188 /// it for a full readiness check. Kept optional so an existing `sando.toml`
189 /// needs no edit.
190 #[serde(default)]
191 pub health_url: Option<String>,
192 /// Companion services this node installs from the release bundle after the
193 /// server is swapped and restarted. Each entry names a `[[companion]]` the
194 /// daemon builds (see `Config::companions`) and says where its binary lands
195 /// and which unit to restart — so a contract-coupled service (mnw-cli) ships
196 /// in the same promote as the server instead of drifting. Empty (default) =
197 /// server-only node, so an existing `sando.toml` keeps working unedited.
198 #[serde(default, rename = "companion")]
199 pub companions: Vec<NodeCompanion>,
200 }
201
202 /// Where a node installs a built companion binary and which unit to bounce.
203 #[derive(Debug, Clone, Serialize, Deserialize)]
204 pub struct NodeCompanion {
205 /// Must match a `Config::companions[].name` — the bundle subdir to read from.
206 pub name: String,
207 /// Absolute path the companion binary is installed to on the node
208 /// (the unit's `ExecStart`), e.g. `/opt/mnw-cli/mnw-cli`.
209 pub install_path: String,
210 /// systemd unit restarted after the binary is installed, e.g.
211 /// `mnw-cli.service`.
212 pub service_name: String,
213 }
214
215 fn default_service_name() -> String {
216 "makenotwork.service".into()
217 }
218
219 /// The capability set every pre-existing Sando node implicitly had: it deploys
220 /// and restarts, and is health-observed. Keeping these as the defaults is what
221 /// lets `sando.toml` stay unchanged through the executor refactor.
222 pub fn default_actuate() -> Vec<String> {
223 vec!["deploy".into(), "restart".into()]
224 }
225 pub fn default_observe() -> Vec<String> {
226 vec!["health".into()]
227 }
228
229 #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
230 #[serde(rename_all = "snake_case")]
231 pub enum CanaryPolicy {
232 #[default]
233 Sequential,
234 Parallel,
235 }
236
237 impl CanaryPolicy {
238 pub fn as_str(self) -> &'static str {
239 match self {
240 CanaryPolicy::Sequential => "sequential",
241 CanaryPolicy::Parallel => "parallel",
242 }
243 }
244 }
245
246 #[derive(Debug, Clone, Serialize, Deserialize)]
247 #[serde(tag = "kind", rename_all = "snake_case")]
248 pub enum Gate {
249 CargoTest,
250 HardeningTest,
251 Clippy,
252 Fmt,
253 CargoAudit,
254 CargoDeny,
255 MigrationDryRun,
256 CodeSmoke,
257 BootSmoke,
258 NodeHealth,
259 /// Post-deploy, in a real browser, over the tier's public URL.
260 ///
261 /// The one gate that crosses the CDN. `boot_smoke` runs on the build host
262 /// and `node_health` reaches a node over its executor, so between them the
263 /// edge was never watched -- and on 2026-08-14 testnot served a page whose
264 /// JavaScript did not run for hours behind nine green gates, because the
265 /// CDN held a stale module that failed to link against a fresh one.
266 ///
267 /// Needs `public_url` on the tier. A tier without one cannot run it, which
268 /// `validate` refuses at load rather than at promote time.
269 PageSmoke,
270 BurnIn {
271 hours: u32,
272 },
273 ManualConfirm,
274 }
275
276 impl Gate {
277 /// The discriminant — the identifier we use in events, schema columns,
278 /// and the TUI. Gate parameters (e.g. `BurnIn.hours`) stay with `Gate`
279 /// and are not carried into `gate_runs` history.
280 pub fn kind(&self) -> GateKind {
281 match self {
282 Gate::CargoTest => GateKind::CargoTest,
283 Gate::HardeningTest => GateKind::HardeningTest,
284 Gate::Clippy => GateKind::Clippy,
285 Gate::Fmt => GateKind::Fmt,
286 Gate::CargoAudit => GateKind::CargoAudit,
287 Gate::CargoDeny => GateKind::CargoDeny,
288 Gate::MigrationDryRun => GateKind::MigrationDryRun,
289 Gate::CodeSmoke => GateKind::CodeSmoke,
290 Gate::BootSmoke => GateKind::BootSmoke,
291 Gate::NodeHealth => GateKind::NodeHealth,
292 Gate::PageSmoke => GateKind::PageSmoke,
293 Gate::BurnIn { .. } => GateKind::BurnIn,
294 Gate::ManualConfirm => GateKind::ManualConfirm,
295 }
296 }
297
298 /// Gates that execute against a tier's freshly-deployed nodes, recording a
299 /// `gate_runs` row, at the end of a successful promote to that tier — the
300 /// evidence the *next* promote checks. Only `node_health`: it probes the
301 /// nodes the deploy just shipped to. `boot_smoke` is NOT post-deploy — it is
302 /// a build-time gate that boots the staged artifact on the build host and
303 /// proves nothing about a deployed node (the Run-2 SERIOUS-3 blind spot:
304 /// boot_smoke used to re-run locally at promote time and the next promote
305 /// trusted a binary that never touched the node).
306 pub fn runs_post_deploy(&self) -> bool {
307 matches!(self, Gate::NodeHealth | Gate::PageSmoke)
308 }
309
310 /// Gates evaluated at promote time against the deployed nodes or the
311 /// operator, as opposed to build-time gates (`cargo_test`,
312 /// `migration_dry_run`, `code_smoke`, `boot_smoke`) that run once on the build host and
313 /// prove nothing about a promote. A serving tier whose gate list contains
314 /// none of these would wave every promote straight through; `Topology
315 /// ::validate` refuses to load such a config (the structural form of CF1's
316 /// fail-closed default). `boot_smoke` no longer counts here — a serving tier
317 /// must declare a real node-level / operator gate, not a host smoke test.
318 pub fn guards_promotion(&self) -> bool {
319 matches!(
320 self,
321 Gate::NodeHealth | Gate::PageSmoke | Gate::BurnIn { .. } | Gate::ManualConfirm
322 )
323 }
324 }
325
326 impl Topology {
327 pub fn load(path: &Path) -> Result<Self> {
328 let raw = std::fs::read_to_string(path)
329 .with_context(|| format!("reading topology at {}", path.display()))?;
330 let topo: Topology = toml::from_str(&raw)?;
331 topo.validate()?;
332 Ok(topo)
333 }
334
335 /// Defense-in-depth for the build-host guard: a host that serves a
336 /// provisioned tier must never be designated the builder. The runtime
337 /// hostname check in `build::run` is the real guard; this catches a
338 /// misconfiguration (build_host pointed at a prod node's name/ssh_target) at
339 /// startup, before the daemon ever accepts a build. Called from `main` once
340 /// config + topology are both loaded.
341 pub fn ensure_build_host_not_serving(&self, build_host: &str) -> Result<()> {
342 for t in &self.tiers {
343 if !t.provisioned || t.name.as_str() == "host" {
344 continue;
345 }
346 for n in &t.nodes {
347 if n.name.as_str() == build_host || n.ssh_target == build_host {
348 anyhow::bail!(
349 "build_host {build_host:?} is also node {} (ssh {}) in serving tier {} — \
350 the builder must not be a prod/serving node",
351 n.name,
352 n.ssh_target,
353 t.name
354 );
355 }
356 }
357 }
358 Ok(())
359 }
360
361 #[cfg(test)]
362 pub(crate) fn validate_for_test(&self) -> Result<()> {
363 self.validate()
364 }
365
366 /// Every `[[migration_check]]` in the daemon config must name a dump this
367 /// topology declares. The two files are separate — daemon config is
368 /// per-host, topology is per-project — so nothing but this catches a check
369 /// pointing at a backup nobody fetches. Left uncaught it surfaces as a
370 /// permanently `Blocked` gate the first time someone promotes, which reads
371 /// like a missed fetch rather than a config typo. Called from `main` once
372 /// both are loaded, and so under `--check-config`.
373 pub fn ensure_migration_checks_have_backups(
374 &self,
375 checks: &[crate::config::MigrationCheck],
376 ) -> Result<()> {
377 // A product no tier dry-runs migrations for owes no dumps. `checks` is
378 // never empty — the config defaults it to MNW's `server` entry — so
379 // without this, an intake-only product with no postgres anywhere is
380 // asked to declare a prod dump for a gate it does not configure.
381 if !self
382 .tiers
383 .iter()
384 .flat_map(|t| &t.gates)
385 .any(|g| g.kind() == GateKind::MigrationDryRun)
386 {
387 return Ok(());
388 }
389 for c in checks {
390 anyhow::ensure!(
391 self.backup_named(&c.backup).is_some(),
392 "migration_check {} restores backup {:?}, which no [[backup]] in {} declares \
393 (have: {})",
394 c.dir.display(),
395 c.backup,
396 "the topology",
397 self.backup
398 .iter()
399 .map(|b| b.name.as_str())
400 .collect::<Vec<_>>()
401 .join(", "),
402 );
403 }
404 Ok(())
405 }
406
407 /// Every `[[test_target]]` with an `aux_repo` must name a repo this topology
408 /// checks out. Third of the cross-file checks, and the one whose absence has
409 /// already cost coverage once: an unresolvable target is a warn-and-skip, by
410 /// design, so that a config describing the tip can still build an older sha.
411 /// That makes a typo here indistinguishable from a legitimate bisect skip —
412 /// a green gate that ran one crate fewer than it says it does.
413 pub fn ensure_test_target_aux_repos_exist(
414 &self,
415 targets: &[crate::config::TestTarget],
416 ) -> Result<()> {
417 for t in targets {
418 let Some(name) = t.aux_repo.as_deref() else {
419 continue;
420 };
421 anyhow::ensure!(
422 self.aux_repos.iter().any(|a| a.name == name),
423 "test_target {} names aux_repo {:?}, which no [[aux_repo]] in the topology \
424 checks out, so the gate would skip it as absent (have: {})",
425 t.label(),
426 name,
427 if self.aux_repos.is_empty() {
428 "none".to_string()
429 } else {
430 self.aux_repos
431 .iter()
432 .map(|a| a.name.as_str())
433 .collect::<Vec<_>>()
434 .join(", ")
435 },
436 );
437 }
438 Ok(())
439 }
440
441 /// Every `[[tier.node.companion]]` must name a companion the daemon config
442 /// actually builds. Same two-file split as the migration checks above, and
443 /// the same class of typo, but a worse landing: companions are installed
444 /// AFTER the symlink swap (`deploy::deploy_remote`), so a name that stages
445 /// nothing fails a promote with the server already live on the new version
446 /// — `FailureStage::AtOrAfterSwap`, the case that needs a human to go look.
447 /// Catching it at load turns that into a startup error on the build host.
448 pub fn ensure_node_companions_are_built(
449 &self,
450 built: &[crate::config::Companion],
451 ) -> Result<()> {
452 for t in &self.tiers {
453 for n in &t.nodes {
454 for c in &n.companions {
455 anyhow::ensure!(
456 built.iter().any(|b| b.name == c.name),
457 "tier {} node {} installs companion {:?}, which no [[companion]] in the \
458 daemon config builds, so nothing would be staged under \
459 companions/{} (have: {})",
460 t.name,
461 n.name,
462 c.name,
463 c.name,
464 if built.is_empty() {
465 "none".to_string()
466 } else {
467 built
468 .iter()
469 .map(|b| b.name.as_str())
470 .collect::<Vec<_>>()
471 .join(", ")
472 },
473 );
474 }
475 }
476 }
477 Ok(())
478 }
479
480 /// The configured dump for `name`, or `None` when nothing declares it.
481 pub fn backup_named(&self, name: &str) -> Option<&BackupConfig> {
482 self.backup.iter().find(|b| b.name == name)
483 }
484
485 fn validate(&self) -> Result<()> {
486 // A dump is only owed by a product that actually dry-runs migrations.
487 // The unconditional form of this asserted something about every
488 // product's tiers from a fact about one: pom configures no
489 // `migration_dry_run` anywhere (it has no postgres schema at all), so
490 // requiring it to declare a prod dump would be demanding a fixture for
491 // a gate it never runs.
492 let dry_runs_migrations = self
493 .tiers
494 .iter()
495 .flat_map(|t| &t.gates)
496 .any(|g| g.kind() == GateKind::MigrationDryRun);
497 anyhow::ensure!(
498 !dry_runs_migrations || !self.backup.is_empty(),
499 "a tier configures migration_dry_run but the topology declares no [backup]; \
500 the gate would have nothing to restore"
501 );
502 // page_smoke has one input and it is not optional. Caught here rather
503 // than at promote time, because a gate that cannot run is a gate that
504 // would be discovered red halfway through a deploy, which is the worst
505 // moment to learn a URL is missing.
506 for t in &self.tiers {
507 let smokes = t.gates.iter().any(|g| g.kind() == GateKind::PageSmoke);
508 anyhow::ensure!(
509 !smokes || t.public_url.is_some(),
510 "tier {} configures page_smoke but declares no public_url; the gate has to \
511 request the site the way a visitor does, and a URL derived from a node would \
512 reach the origin and miss the CDN it exists to watch",
513 t.name,
514 );
515 }
516 for (i, b) in self.backup.iter().enumerate() {
517 anyhow::ensure!(
518 !b.name.is_empty()
519 && b.name
520 .bytes()
521 .all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-'),
522 "backup name {:?} must be non-empty and match [A-Za-z0-9_-]+; it keys the \
523 `backups` table and a daemon-config migration_check",
524 b.name,
525 );
526 anyhow::ensure!(
527 !b.source.is_empty() && !b.local_path.is_empty(),
528 "backup {} has an empty source/local_path",
529 b.name,
530 );
531 // Two dumps sharing a name would interleave in `backups`, so the
532 // freshness check and the plausibility floor would each read the
533 // other's row. Two sharing a `local_path` would overwrite each
534 // other on disk, and whichever fetched last would be restored for
535 // both — green, and proving nothing about one of the databases.
536 for prior in &self.backup[..i] {
537 anyhow::ensure!(
538 prior.name != b.name,
539 "two backup entries share the name {:?}",
540 b.name,
541 );
542 anyhow::ensure!(
543 prior.local_path != b.local_path,
544 "backups {:?} and {:?} share local_path {:?}; they would overwrite each other",
545 prior.name,
546 b.name,
547 b.local_path,
548 );
549 }
550 }
551 anyhow::ensure!(
552 !self.tiers.is_empty(),
553 "topology must declare at least one tier"
554 );
555 for t in &self.tiers {
556 // The `host` tier is the build tier (cargo_test / migration_dry_run /
557 // code_smoke / boot_smoke run once on the host); every other tier serves an
558 // artifact to nodes, so it is exempted from the node and
559 // promotion-gate checks the same way.
560 let is_build_tier = t.name.as_str() == "host";
561 if t.provisioned && t.nodes.is_empty() && !is_build_tier {
562 anyhow::bail!("tier {} is provisioned but has no nodes", t.name);
563 }
564 // Fail closed by default: a provisioned serving tier must declare at
565 // least one gate that actually guards a promote. Without this, an
566 // empty (or build-time-only) gate list waves every promote through
567 // because `unsatisfied_gates` finds nothing to check (CF1 root cause).
568 if t.provisioned && !is_build_tier && !t.gates.iter().any(Gate::guards_promotion) {
569 anyhow::bail!(
570 "tier {} is provisioned to serve but declares no promotion gate \
571 (need at least one of node_health / burn_in / manual_confirm)",
572 t.name
573 );
574 }
575 }
576 let mut seen_dirs: Vec<Vec<&str>> = Vec::new();
577 for aux in &self.aux_repos {
578 anyhow::ensure!(
579 !aux.name.is_empty() && !aux.bare_path.is_empty() && !aux.branch.is_empty(),
580 "aux_repo entry has an empty name/bare_path/branch"
581 );
582 // `checkout_dir` becomes a `workdir.join(..)`. Nesting is allowed —
583 // docengine lives at `Libraries/docengine` in the dev tree and the
584 // path dep resolves to that shape — but every component must be a
585 // plain name so an aux repo can never write outside the workdir.
586 let dir = &aux.checkout_dir;
587 let parts: Vec<&str> = dir.split('/').collect();
588 anyhow::ensure!(
589 !dir.is_empty()
590 && !dir.contains('\\')
591 && parts
592 .iter()
593 .all(|c| !c.is_empty() && *c != "." && *c != ".."),
594 "aux_repo {} has an unsafe checkout_dir {dir:?} (must be a relative path of \
595 plain components: no leading slash, empty segments, or dot-dot)",
596 aux.name,
597 );
598 // Two checkouts may not share a dir, and neither may sit inside the
599 // other: `git worktree add` into a path under a live worktree buries
600 // one checkout in the other's tree, and whichever builds second wins.
601 for prior in &seen_dirs {
602 let common = prior.len().min(parts.len());
603 anyhow::ensure!(
604 prior[..common] != parts[..common],
605 "two aux_repo entries share or nest checkout_dir {dir:?}; \
606 they would clobber each other"
607 );
608 }
609 seen_dirs.push(parts);
610 }
611 Ok(())
612 }
613 }
614
615 #[cfg(test)]
616 mod tests {
617 use super::*;
618
619 /// A minimal topology with one serving tier whose gate block is `gates`.
620 fn topo_with_serving_gates(provisioned: bool, gates: &str) -> Topology {
621 let raw = format!(
622 r#"
623 [repo]
624 bare_path = "/tmp/repo.git"
625 branch = "main"
626
627 [backup]
628 source = "ssh://prod/dump.sql.gz"
629 local_path = "/tmp/dump.sql.gz"
630
631 [[tier]]
632 name = "b"
633 provisioned = {provisioned}
634 gates = [{gates}]
635 [[tier.node]]
636 name = "prod-1"
637 ssh_target = "prod-1"
638 release_root = "/srv/mnw"
639 "#
640 );
641 toml::from_str(&raw).expect("parse test topology")
642 }
643
644 #[test]
645 fn page_smoke_without_a_public_url_is_rejected_at_load() {
646 // The gate's whole value is that it requests the site the way a visitor
647 // does. Without a URL it cannot, and a gate that cannot run must not be
648 // discovered halfway through a promote.
649 let topo = topo_with_serving_gates(true, r#"{ kind = "page_smoke" }"#);
650 let err = topo.validate_for_test().expect_err("must refuse");
651 assert!(
652 err.to_string().contains("public_url"),
653 "error should name the missing field: {err}"
654 );
655 }
656
657 #[test]
658 fn page_smoke_with_a_public_url_loads() {
659 let raw = r#"
660 [repo]
661 bare_path = "/tmp/repo.git"
662 branch = "main"
663
664 [backup]
665 source = "ssh://prod/dump.sql.gz"
666 local_path = "/tmp/dump.sql.gz"
667
668 [[tier]]
669 name = "a"
670 provisioned = true
671 public_url = "https://testnot.work"
672 gates = [{ kind = "page_smoke" }]
673 [[tier.node]]
674 name = "testnot-1"
675 ssh_target = "testnot-1"
676 release_root = "/srv/mnw"
677 "#;
678 let topo: Topology = toml::from_str(raw).expect("parse");
679 topo.validate_for_test().expect("valid");
680 assert_eq!(
681 topo.tiers[0].public_url.as_deref(),
682 Some("https://testnot.work")
683 );
684 // It guards promotion out of its tier, and it runs after the deploy
685 // rather than on the build host -- both are the point of it.
686 assert!(topo.tiers[0].gates[0].guards_promotion());
687 assert!(topo.tiers[0].gates[0].runs_post_deploy());
688 }
689
690 #[test]
691 fn provisioned_serving_tier_with_no_gates_is_rejected() {
692 let topo = topo_with_serving_gates(true, "");
693 let err = topo.validate_for_test().unwrap_err().to_string();
694 assert!(err.contains("no promotion gate"), "{err}");
695 }
696
697 #[test]
698 fn provisioned_serving_tier_with_only_build_gates_is_rejected() {
699 // cargo_test / migration_dry_run are build-time and prove nothing about a
700 // promote, so a serving tier carrying only them still fails closed.
701 let topo = topo_with_serving_gates(
702 true,
703 r#"{ kind = "cargo_test" }, { kind = "migration_dry_run" }"#,
704 );
705 let err = topo.validate_for_test().unwrap_err().to_string();
706 assert!(err.contains("no promotion gate"), "{err}");
707 }
708
709 #[test]
710 fn provisioned_serving_tier_with_a_promotion_gate_is_accepted() {
711 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
712 assert!(topo.validate_for_test().is_ok());
713 }
714
715 #[test]
716 fn provisioned_serving_tier_with_only_boot_smoke_is_rejected() {
717 // boot_smoke is a build-host gate now; it proves nothing about a node, so
718 // a serving tier carrying only boot_smoke must fail closed exactly like an
719 // empty gate list (Run-2 SERIOUS-3 structural close).
720 let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
721 let err = topo.validate_for_test().unwrap_err().to_string();
722 assert!(err.contains("no promotion gate"), "{err}");
723 }
724
725 #[test]
726 fn unprovisioned_tier_with_empty_gates_is_skipped() {
727 // A declared-but-not-yet-provisioned tier (e.g. tier c) carries no
728 // promote authority, so the gate requirement does not apply yet.
729 let topo = topo_with_serving_gates(false, "");
730 assert!(topo.validate_for_test().is_ok());
731 }
732
733 #[test]
734 fn build_host_matching_a_serving_node_is_rejected() {
735 // prod-1 is a node in the provisioned serving tier built above; naming it
736 // as the builder must fail closed.
737 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
738 let err = topo
739 .ensure_build_host_not_serving("prod-1")
740 .unwrap_err()
741 .to_string();
742 assert!(err.contains("must not be a prod/serving node"), "{err}");
743 }
744
745 #[test]
746 fn build_host_distinct_from_serving_nodes_is_accepted() {
747 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
748 assert!(topo.ensure_build_host_not_serving("fw13").is_ok());
749 }
750
751 #[test]
752 fn node_companions_default_empty_and_parse_when_present() {
753 // A node without [[tier.node.companion]] is server-only.
754 let plain = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
755 assert!(plain.tiers[0].nodes[0].companions.is_empty());
756
757 // A node that declares a companion carries its install target + unit.
758 let raw = r#"
759 [repo]
760 bare_path = "/tmp/repo.git"
761 branch = "main"
762 [backup]
763 source = "s"
764 local_path = "/tmp/d"
765 [[tier]]
766 name = "b"
767 provisioned = true
768 gates = [{ kind = "node_health" }]
769 [[tier.node]]
770 name = "prod-1"
771 ssh_target = "makenotwork@alpha-west-1"
772 release_root = "/opt/mnw"
773 [[tier.node.companion]]
774 name = "mnw-cli"
775 install_path = "/opt/mnw-cli/mnw-cli"
776 service_name = "mnw-cli.service"
777 "#;
778 let topo: Topology = toml::from_str(raw).expect("parse companion topology");
779 let c = &topo.tiers[0].nodes[0].companions;
780 assert_eq!(c.len(), 1);
781 assert_eq!(c[0].name, "mnw-cli");
782 assert_eq!(c[0].install_path, "/opt/mnw-cli/mnw-cli");
783 assert_eq!(c[0].service_name, "mnw-cli.service");
784 }
785
786 /// A topology whose one node installs the named companions.
787 fn topo_installing(names: &[&str]) -> Topology {
788 let mut blocks = String::new();
789 for n in names {
790 use std::fmt::Write;
791 let _ = write!(
792 blocks,
793 "[[tier.node.companion]]\nname = \"{n}\"\n\
794 install_path = \"/opt/{n}/{n}\"\nservice_name = \"{n}.service\"\n"
795 );
796 }
797 let raw = format!(
798 r#"
799 [repo]
800 bare_path = "/tmp/repo.git"
801 branch = "main"
802 [backup]
803 source = "s"
804 local_path = "/tmp/d"
805 [[tier]]
806 name = "b"
807 provisioned = true
808 gates = [{{ kind = "node_health" }}]
809 [[tier.node]]
810 name = "prod-1"
811 ssh_target = "makenotwork@alpha-west-1"
812 release_root = "/opt/mnw"
813 {blocks}"#
814 );
815 toml::from_str(&raw).expect("parse topology")
816 }
817
818 fn built(names: &[&str]) -> Vec<crate::config::Companion> {
819 names
820 .iter()
821 .map(|n| crate::config::Companion {
822 name: (*n).to_string(),
823 manifest_dir: (*n).into(),
824 bin: (*n).to_string(),
825 })
826 .collect()
827 }
828
829 fn test_target(dir: &str, aux_repo: Option<&str>) -> crate::config::TestTarget {
830 crate::config::TestTarget {
831 dir: dir.into(),
832 aux_repo: aux_repo.map(str::to_string),
833 features: Vec::new(),
834 all_features: false,
835 scratch_db: false,
836 }
837 }
838
839 #[test]
840 fn a_test_target_naming_a_checked_out_aux_repo_is_accepted() {
841 let topo = topo_with_aux(
842 "[[aux_repo]]\nname = \"docengine\"\nbare_path = \"/tmp/d.git\"\n\
843 upstream = \"git@h:max/d.git\"\nbranch = \"main\"\ncheckout_dir = \"Libraries/docengine\"\n",
844 )
845 .expect("parse");
846 assert!(
847 topo.ensure_test_target_aux_repos_exist(&[
848 test_target("server", None),
849 test_target("", Some("docengine")),
850 ])
851 .is_ok()
852 );
853 }
854
855 #[test]
856 fn a_test_target_naming_an_unknown_aux_repo_is_rejected_at_load() {
857 // The failure this exists to prevent is silent: an unresolvable target
858 // is a warn-and-skip (bisect), so the gate stays green having run one
859 // crate fewer than the config claims.
860 let topo = topo_with_aux(
861 "[[aux_repo]]\nname = \"synckit\"\nbare_path = \"/tmp/s.git\"\n\
862 upstream = \"git@h:max/s.git\"\nbranch = \"main\"\ncheckout_dir = \"synckit\"\n",
863 )
864 .expect("parse");
865 let err = topo
866 .ensure_test_target_aux_repos_exist(&[test_target("", Some("docengine"))])
867 .unwrap_err()
868 .to_string();
869 assert!(err.contains("docengine"), "{err}");
870 assert!(err.contains("have: synckit"), "{err}");
871 }
872
873 #[test]
874 fn test_targets_without_an_aux_repo_need_no_aux_repos_declared() {
875 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
876 assert!(
877 topo.ensure_test_target_aux_repos_exist(&[test_target("server", None)])
878 .is_ok()
879 );
880 }
881
882 #[test]
883 fn a_node_companion_the_daemon_builds_is_accepted() {
884 let topo = topo_installing(&["mnw-cli", "multithreaded"]);
885 assert!(
886 topo.ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
887 .is_ok()
888 );
889 }
890
891 #[test]
892 fn a_node_companion_nothing_builds_is_rejected_at_load() {
893 // Left uncaught this fails during the post-swap install on prod, with
894 // the server already live on the new version.
895 let topo = topo_installing(&["mnw-cli", "multithreadd"]);
896 let err = topo
897 .ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
898 .unwrap_err()
899 .to_string();
900 assert!(err.contains("multithreadd"), "{err}");
901 assert!(err.contains("no [[companion]]"), "{err}");
902 // The message names what IS available, so the typo is obvious.
903 assert!(err.contains("mnw-cli, multithreaded"), "{err}");
904 }
905
906 #[test]
907 fn a_node_companion_with_no_companions_configured_at_all_is_rejected() {
908 let topo = topo_installing(&["multithreaded"]);
909 let err = topo
910 .ensure_node_companions_are_built(&[])
911 .unwrap_err()
912 .to_string();
913 assert!(err.contains("have: none"), "{err}");
914 }
915
916 #[test]
917 fn a_topology_installing_no_companions_is_fine_with_none_built() {
918 let topo = topo_installing(&[]);
919 assert!(topo.ensure_node_companions_are_built(&[]).is_ok());
920 }
921
922 fn topo_with_aux(aux_block: &str) -> Result<Topology> {
923 let raw = format!(
924 r#"
925 [repo]
926 bare_path = "/tmp/repo.git"
927 branch = "main"
928 [backup]
929 source = "s"
930 local_path = "/tmp/d"
931 [[tier]]
932 name = "b"
933 provisioned = true
934 gates = [{{ kind = "node_health" }}]
935 [[tier.node]]
936 name = "prod-1"
937 ssh_target = "prod-1"
938 release_root = "/srv/mnw"
939 {aux_block}
940 "#
941 );
942 let topo: Topology = toml::from_str(&raw)?;
943 topo.validate_for_test()?;
944 Ok(topo)
945 }
946
947 #[test]
948 fn aux_repos_default_empty() {
949 let topo = topo_with_aux("").expect("no aux_repo block is fine");
950 assert!(topo.aux_repos.is_empty());
951 }
952
953 #[test]
954 fn aux_repo_parses_all_fields() {
955 let topo = topo_with_aux(
956 r#"
957 [[aux_repo]]
958 name = "synckit"
959 bare_path = "/srv/sando/synckit.git"
960 upstream = "git@ssh.makenot.work:max/synckit.git"
961 branch = "main"
962 checkout_dir = "synckit""#,
963 )
964 .expect("valid aux_repo parses");
965 assert_eq!(topo.aux_repos.len(), 1);
966 let a = &topo.aux_repos[0];
967 assert_eq!(a.name, "synckit");
968 assert_eq!(a.bare_path, "/srv/sando/synckit.git");
969 assert_eq!(a.upstream, "git@ssh.makenot.work:max/synckit.git");
970 assert_eq!(a.branch, "main");
971 assert_eq!(a.checkout_dir, "synckit");
972 }
973
974 #[test]
975 fn aux_repo_with_nested_checkout_dir_is_accepted() {
976 let topo = topo_with_aux(
977 r#"
978 [[aux_repo]]
979 name = "docengine"
980 bare_path = "/srv/sando/docengine.git"
981 upstream = "git@ssh.makenot.work:max/docengine.git"
982 branch = "main"
983 checkout_dir = "Libraries/docengine""#,
984 )
985 .expect("a nested checkout_dir is a valid location");
986 assert_eq!(topo.aux_repos[0].checkout_dir, "Libraries/docengine");
987 }
988
989 #[test]
990 fn aux_repo_with_traversing_checkout_dir_is_rejected() {
991 for bad in [
992 "../escape",
993 "a/../../escape",
994 "a/./b",
995 "a//b",
996 "/abs",
997 "a/",
998 "..",
999 ".",
1000 ] {
1001 let err = topo_with_aux(&format!(
1002 r#"
1003 [[aux_repo]]
1004 name = "x"
1005 bare_path = "/srv/sando/x.git"
1006 upstream = "u"
1007 branch = "main"
1008 checkout_dir = "{bad}""#,
1009 ))
1010 .unwrap_err()
1011 .to_string();
1012 assert!(err.contains("unsafe checkout_dir"), "for {bad:?}: {err}");
1013 }
1014 }
1015
1016 #[test]
1017 fn aux_repos_sharing_a_checkout_dir_are_rejected() {
1018 let err = topo_with_aux(
1019 r#"
1020 [[aux_repo]]
1021 name = "one"
1022 bare_path = "/srv/sando/one.git"
1023 upstream = "u"
1024 branch = "main"
1025 checkout_dir = "shared"
1026 [[aux_repo]]
1027 name = "two"
1028 bare_path = "/srv/sando/two.git"
1029 upstream = "u"
1030 branch = "main"
1031 checkout_dir = "shared""#,
1032 )
1033 .unwrap_err()
1034 .to_string();
1035 assert!(err.contains("share or nest checkout_dir"), "{err}");
1036 }
1037
1038 #[test]
1039 fn aux_repo_nested_inside_another_checkout_dir_is_rejected() {
1040 let err = topo_with_aux(
1041 r#"
1042 [[aux_repo]]
1043 name = "outer"
1044 bare_path = "/srv/sando/outer.git"
1045 upstream = "u"
1046 branch = "main"
1047 checkout_dir = "Libraries"
1048 [[aux_repo]]
1049 name = "inner"
1050 bare_path = "/srv/sando/inner.git"
1051 upstream = "u"
1052 branch = "main"
1053 checkout_dir = "Libraries/docengine""#,
1054 )
1055 .unwrap_err()
1056 .to_string();
1057 assert!(err.contains("share or nest checkout_dir"), "{err}");
1058 }
1059
1060 /// A topology whose `[backup]`/`[[backup]]` section is `backup_block`.
1061 fn topo_with_backup_block(backup_block: &str) -> Result<Topology> {
1062 let raw = format!(
1063 r#"
1064 [repo]
1065 bare_path = "/tmp/repo.git"
1066 branch = "main"
1067 {backup_block}
1068 [[tier]]
1069 name = "b"
1070 provisioned = true
1071 # migration_dry_run is what makes the backup rules apply at all: a product no
1072 # tier dry-runs migrations for owes no dumps, so a fixture exercising those rules
1073 # has to configure the gate.
1074 gates = [{{ kind = "node_health" }}, {{ kind = "migration_dry_run" }}]
1075 [[tier.node]]
1076 name = "prod-1"
1077 ssh_target = "prod-1"
1078 release_root = "/srv/mnw"
1079 "#
1080 );
1081 let topo: Topology = toml::from_str(&raw)?;
1082 topo.validate_for_test()?;
1083 Ok(topo)
1084 }
1085
1086 #[test]
1087 fn a_single_backup_table_still_parses_as_one_named_server() {
1088 // Back-compat is the point: every deployed sando.toml uses the single
1089 // `[backup]` form, and the box this config lives on is the one whose job
1090 // is deploying — it must not need an edit to start.
1091 let topo = topo_with_backup_block(
1092 r#"
1093 [backup]
1094 source = "ssh://prod/dump.sql.gz"
1095 local_path = "/tmp/dump.sql.gz""#,
1096 )
1097 .expect("the single-table form must still load");
1098 assert_eq!(topo.backup.len(), 1);
1099 assert_eq!(topo.backup[0].name, "server");
1100 assert!(topo.backup_named("server").is_some());
1101 }
1102
1103 #[test]
1104 fn a_backup_list_parses_and_keeps_its_names() {
1105 let topo = topo_with_backup_block(
1106 r#"
1107 [[backup]]
1108 name = "server"
1109 source = "ssh://prod/makenotwork/latest.sql.gz"
1110 local_path = "/tmp/server.sql.gz"
1111 [[backup]]
1112 name = "multithreaded"
1113 source = "ssh://prod/multithreaded/latest.sql.gz"
1114 local_path = "/tmp/mt.sql.gz""#,
1115 )
1116 .expect("the list form must load");
1117 assert_eq!(topo.backup.len(), 2);
1118 assert_eq!(
1119 topo.backup_named("multithreaded").unwrap().local_path,
1120 "/tmp/mt.sql.gz"
1121 );
1122 assert!(topo.backup_named("nope").is_none());
1123 }
1124
1125 #[test]
1126 fn two_backups_sharing_a_name_are_rejected() {
1127 // They would interleave in `backups`, so the freshness check and the
1128 // plausibility floor would each read the other's row.
1129 let err = topo_with_backup_block(
1130 r#"
1131 [[backup]]
1132 name = "server"
1133 source = "a"
1134 local_path = "/tmp/a.sql.gz"
1135 [[backup]]
1136 name = "server"
1137 source = "b"
1138 local_path = "/tmp/b.sql.gz""#,
1139 )
1140 .unwrap_err()
1141 .to_string();
1142 assert!(err.contains("share the name"), "{err}");
1143 }
1144
1145 #[test]
1146 fn two_backups_sharing_a_local_path_are_rejected() {
1147 // Whichever fetched last would be restored for both checks — green, and
1148 // proving nothing about one of the two databases.
1149 let err = topo_with_backup_block(
1150 r#"
1151 [[backup]]
1152 name = "server"
1153 source = "a"
1154 local_path = "/tmp/same.sql.gz"
1155 [[backup]]
1156 name = "multithreaded"
1157 source = "b"
1158 local_path = "/tmp/same.sql.gz""#,
1159 )
1160 .unwrap_err()
1161 .to_string();
1162 assert!(err.contains("share local_path"), "{err}");
1163 }
1164
1165 #[test]
1166 fn a_migration_check_naming_an_undeclared_backup_is_rejected_at_startup() {
1167 // Daemon config and topology are separate files, so nothing but this
1168 // cross-check catches the typo. Uncaught it surfaces as a permanently
1169 // Blocked gate on the next promote, which reads like a missed fetch.
1170 let topo = topo_with_backup_block(
1171 r#"
1172 [backup]
1173 source = "s"
1174 local_path = "/tmp/d""#,
1175 )
1176 .unwrap();
1177 let checks = vec![crate::config::MigrationCheck {
1178 dir: std::path::PathBuf::from("multithreaded/migrations"),
1179 backup: "multithreaded".into(),
1180 scratch_db: Some("sando_scratch_mt".into()),
1181 owner_role: Some("multithreaded".into()),
1182 }];
1183 let err = topo
1184 .ensure_migration_checks_have_backups(&checks)
1185 .unwrap_err()
1186 .to_string();
1187 assert!(err.contains("which no [[backup]]"), "{err}");
1188
1189 // And the shipped pair agree, which is the case that actually ships.
1190 let shipped_checks = crate::config::default_migration_checks_for_test();
1191 shipped()
1192 .ensure_migration_checks_have_backups(&shipped_checks)
1193 .expect("the default server check resolves against the shipped topology");
1194 }
1195
1196 #[test]
1197 fn real_sando_toml_loads_clean() {
1198 // The shipped topology must satisfy the invariant — guards against a
1199 // regression that would lock sandod out of its own config.
1200 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
1201 Topology::load(&path).expect("shipped sando.toml must validate");
1202 }
1203
1204 fn shipped() -> Topology {
1205 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
1206 Topology::load(&path).expect("shipped sando.toml must validate")
1207 }
1208
1209 #[test]
1210 fn shipping_to_the_last_provisioned_tier_needs_an_operator_signoff() {
1211 // A tier's gates guard promotion *out* of it, which is the subtlety that
1212 // made manual_confirm inert: it sat on tier b, guarding b -> c, and c is
1213 // not provisioned. So the ship to production was cleared by node_health
1214 // + burn_in alone — and `hotfix: true` skips burn_in.
1215 //
1216 // The gate that matters therefore belongs on the PREDECESSOR of the last
1217 // provisioned tier. Asserted structurally so re-provisioning tiers cannot
1218 // silently strand the sign-off again.
1219 let topo = shipped();
1220 let last = topo
1221 .tiers
1222 .iter()
1223 .rposition(|t| t.provisioned)
1224 .expect("some tier must be provisioned");
1225 assert!(last > 0, "the production tier cannot be the first tier");
1226 let guard = &topo.tiers[last - 1];
1227 assert!(
1228 guard.gates.iter().any(|g| matches!(g, Gate::ManualConfirm)),
1229 "tier {} guards promotion into the last provisioned tier ({}), so it must require an operator sign-off; its gates are {:?}",
1230 guard.name,
1231 topo.tiers[last].name,
1232 guard
1233 .gates
1234 .iter()
1235 .map(|g| g.kind().as_str())
1236 .collect::<Vec<_>>(),
1237 );
1238 }
1239
1240 #[test]
1241 fn every_serving_node_has_a_readiness_probe() {
1242 // Without health_url, node_health degrades to `systemctl is-active`,
1243 // which a crash-looping binary satisfies between restarts — exactly what
1244 // the 0.10.14 CDN_BASE_URL crash-loop did on prod-1.
1245 let topo = shipped();
1246 for tier in topo.tiers.iter().filter(|t| t.provisioned) {
1247 for node in &tier.nodes {
1248 assert!(
1249 node.health_url.is_some(),
1250 "node {} on tier {} has no health_url, so node_health proves only that systemd thinks the unit is running",
1251 node.name,
1252 tier.name,
1253 );
1254 }
1255 }
1256 }
1257 }
1258