Skip to main content

max / makenotwork

52.2 KB · 1367 lines History Blame Raw
1 use crate::domain::{GateKind, NodeId, TierId};
2 use anyhow::{Context, Result};
3 use serde::{Deserialize, Serialize};
4 use std::path::Path;
5
6 #[derive(Debug, Clone, Serialize, Deserialize)]
7 pub struct Topology {
8 /// The repo Sando checks out to build this product.
9 ///
10 /// `None` for an intake-only product: pom is built by Bento on two
11 /// machines and handed over as finished bytes, so Sando fetches no source
12 /// for it and there is no bare repo on this host to name. A topology that
13 /// declares no repo cannot be `/rebuild`-ed, which is the same statement
14 /// [`AppConfig::build_host`](crate::config::AppConfig::build_host) makes
15 /// from the other side.
16 #[serde(default)]
17 pub repo: Option<RepoConfig>,
18 /// Prod dumps `/backup/fetch` pulls, one per database `migration_dry_run`
19 /// has a check for. A list because the repo ships more than one service
20 /// with its own database and its own `sqlx::migrate!()` at boot: the server
21 /// migrates `makenotwork`, multithreaded migrates `multithreaded`, and a
22 /// gate that restores only the first proves nothing about the second.
23 ///
24 /// Accepts both the historical single `[backup]` table and a `[[backup]]`
25 /// list, so a deployed `sando.toml` keeps working unedited (the single form
26 /// deserializes to a one-entry list named `server`).
27 #[serde(deserialize_with = "one_or_many_backup")]
28 pub backup: Vec<BackupConfig>,
29 #[serde(rename = "tier")]
30 pub tiers: Vec<Tier>,
31 /// Extra repos to fetch and check out beside the main worktree before a
32 /// build, so a path dependency that reaches across the repo split resolves.
33 /// Empty (default) keeps an existing `sando.toml` working unedited. See
34 /// [`AuxRepo`] and [`crate::build::checkout_aux_repos`].
35 #[serde(default, rename = "aux_repo")]
36 pub aux_repos: Vec<AuxRepo>,
37 }
38
39 /// An auxiliary repo checked out beside the per-sha worktree so cross-repo path
40 /// dependencies resolve at build time.
41 ///
42 /// The concrete case (2026-07-24): `mnw-cli` — a companion built from the MNW
43 /// worktree — carries `synckit-client = { path = "../../synckit/synckit-client" }`
44 /// after synckit moved to its own repo. From the companion crate at
45 /// `<workdir>/<sha>/mnw-cli`, that path resolves to `<workdir>/synckit`, a sibling
46 /// of the per-sha worktree that Sando otherwise never creates, so the companion
47 /// build failed with "No such file or directory". An `aux_repo` named to land at
48 /// `checkout_dir = "synckit"` puts the synckit source exactly there.
49 ///
50 /// The checkout is at the FIXED `<workdir>/<checkout_dir>`, not per-sha: the path
51 /// dependency resolves to that spot regardless of the MNW sha, and builds
52 /// serialize, so a single shared checkout refreshed to `branch` HEAD each build
53 /// is correct. Because a path dep has no lockfile pin, "branch HEAD" is the honest
54 /// resolution — the same contract as the dev working copy.
55 #[derive(Debug, Clone, Serialize, Deserialize)]
56 pub struct AuxRepo {
57 /// Human label for logs and errors (e.g. `synckit`).
58 pub name: String,
59 /// Bare repo Sando fetches into and worktrees from, e.g.
60 /// `/srv/sando/synckit.git`. Auto-created (hookless) on first build.
61 pub bare_path: String,
62 /// Canonical git remote fetched before checkout. Like [`RepoConfig::upstream`]
63 /// but required here: an aux repo is pull-based (nobody pushes to its bare).
64 pub upstream: String,
65 /// Branch whose HEAD is checked out.
66 pub branch: String,
67 /// Where the worktree lands, relative to `cfg.workdir`. May name a nested
68 /// location (`Libraries/docengine`), because a path dep resolves to wherever
69 /// the dev tree keeps the crate and Sando has to match that shape. Every
70 /// component must be a plain name — no leading slash, no `..` — so the
71 /// checkout stays under the workdir.
72 pub checkout_dir: String,
73 }
74
75 #[derive(Debug, Clone, Serialize, Deserialize)]
76 pub struct RepoConfig {
77 pub bare_path: String,
78 pub branch: String,
79 /// Optional canonical git remote. When set, `/rebuild` fetches the deploy
80 /// branch from here into the bare repo before worktree-ing the target sha,
81 /// so a freshly-pushed commit is resolvable without anyone pushing to the
82 /// bare repo directly (pull-based deploys). When unset, Sando relies on the
83 /// bare repo already containing the sha (push-based / hook-driven).
84 #[serde(default)]
85 pub upstream: Option<String>,
86 }
87
88 #[derive(Debug, Clone, Serialize, Deserialize)]
89 pub struct BackupConfig {
90 /// Which database this dump is of, as referenced by a daemon-config
91 /// `[[migration_check]]`'s `backup` key and recorded in the `backups`
92 /// table's `name` column. Defaults to `server` so the historical single
93 /// `[backup]` table needs no edit — and so the pre-existing rows, which the
94 /// state-DB migration backfills to `server`, keep matching it.
95 #[serde(default = "default_backup_name")]
96 pub name: String,
97 pub source: String,
98 pub local_path: String,
99 }
100
101 fn default_backup_name() -> String {
102 "server".into()
103 }
104
105 /// Accept `[backup]` (one table) or `[[backup]]` (a list) for the same key.
106 /// Serde cannot express "table or sequence" on a `Vec` field on its own, and
107 /// the alternative — renaming the key — would break every deployed
108 /// `sando.toml` at startup, on the box whose whole job is deploying.
109 fn one_or_many_backup<'de, D>(de: D) -> std::result::Result<Vec<BackupConfig>, D::Error>
110 where
111 D: serde::Deserializer<'de>,
112 {
113 #[derive(Deserialize)]
114 #[serde(untagged)]
115 enum OneOrMany {
116 One(BackupConfig),
117 Many(Vec<BackupConfig>),
118 }
119 Ok(match OneOrMany::deserialize(de)? {
120 OneOrMany::One(b) => vec![b],
121 OneOrMany::Many(v) => v,
122 })
123 }
124
125 #[derive(Debug, Clone, Serialize, Deserialize)]
126 pub struct Tier {
127 pub name: TierId,
128 #[serde(default)]
129 pub provisioned: bool,
130 pub gates: Vec<Gate>,
131 #[serde(default)]
132 pub canary: CanaryPolicy,
133 #[serde(default, rename = "node")]
134 pub nodes: Vec<Node>,
135 /// Where the public reaches this tier, CDN included.
136 ///
137 /// Deliberately not derivable from a node's `ssh_target`. The whole value of
138 /// [`Gate::PageSmoke`] is that it goes the way a visitor goes -- through
139 /// Cloudflare, against the hostname in the browser's address bar -- and a
140 /// URL computed from the node would reach the origin and inherit exactly the
141 /// blind spot the gate exists to close. So it is stated, or the gate is not
142 /// available.
143 #[serde(default)]
144 pub public_url: Option<String>,
145 }
146
147 #[derive(Debug, Clone, Serialize, Deserialize)]
148 pub struct Node {
149 pub name: NodeId,
150 pub ssh_target: String,
151 pub release_root: String,
152 /// What this machine runs, as `os/arch` (e.g. `linux/aarch64`).
153 ///
154 /// Compared against the bundle's own platform before anything is pushed;
155 /// see [`crate::deploy::Placement`]. Optional, and a node that declares it
156 /// can only be given a bundle that declares a matching one — silence on
157 /// either side is a refusal, not a pass. MNW's nodes declare nothing and
158 /// keep the single-platform behavior they have always had; pom's declare
159 /// theirs, because pom is the product where one version is two bundles.
160 #[serde(default)]
161 pub platform: Option<crate::domain::Platform>,
162 /// What this machine IS, as `id/version` from `/etc/os-release`
163 /// (e.g. `ubuntu/24.04`, `alloy/0.1`).
164 ///
165 /// Verified against the node before anything is pushed to it, so a box that
166 /// was rebuilt into something else fails the promote with the running
167 /// service intact instead of being discovered by a binary that will not
168 /// start. Optional: a node that declares nothing is not checked, and the
169 /// deploy log says it was not. See [`ops_core::base_image`] for why silence
170 /// is a skip here and a refusal in [`crate::deploy::Placement`].
171 #[serde(default)]
172 pub base_image: Option<ops_core::base_image::BaseImage>,
173 /// The glibc version this node has, as `ldd --version` reports it
174 /// (e.g. `2.39`). Checked independently of [`Self::base_image`], because a
175 /// point release moves under a pinned base and it is this number that
176 /// decides whether a binary loads.
177 ///
178 /// Declaring it does NOT replace the pre-swap `ldd` guard, which compares
179 /// the actual binary against the actual node. It makes the node's floor a
180 /// stated fact that can be compared before bytes are built or moved.
181 #[serde(default)]
182 pub libc: Option<String>,
183 /// systemd unit name to reload-or-restart after the symlink swap.
184 /// Defaults to "makenotwork.service" because that's MNW's prod unit.
185 #[serde(default = "default_service_name")]
186 pub service_name: String,
187 /// Opt-in config-drift guard. When set to the node's env-file path (e.g.
188 /// `/etc/mnw/makenotwork.env`), the deploy sources that file and runs the
189 /// freshly-rsynced binary in `MNW_CHECK_CONFIG=1` mode BEFORE the symlink
190 /// swap. A required var missing on this node (a var added upstream but never
191 /// added to the node's env — how testnot crash-looped on CDN_BASE_URL) then
192 /// fails the promote with the running service still intact, instead of after
193 /// the swap+restart. Unset (default) skips the check, so an existing
194 /// `sando.toml` keeps working and it never runs against a binary too old to
195 /// support the mode; enable it once a check-capable version is deployed.
196 #[serde(default)]
197 pub config_check_env_file: Option<String>,
198 /// Capability grant for this node's executor (see `ops_exec`). Defaults to
199 /// the current behavior of every Sando node — actuate deploy+restart,
200 /// observe health — so an existing `sando.toml` keeps working unedited.
201 #[serde(default = "default_actuate")]
202 pub actuate: Vec<String>,
203 #[serde(default = "default_observe")]
204 pub observe: Vec<String>,
205 /// Optional HTTP readiness URL the `node_health` gate curls on the node (over
206 /// its executor) after `systemctl is-active`. Typically a loopback address
207 /// the service binds, e.g. `http://127.0.0.1:8080/health`. When unset, the
208 /// gate proves the unit is active post-restart but does not HTTP-probe; set
209 /// it for a full readiness check. Kept optional so an existing `sando.toml`
210 /// needs no edit.
211 #[serde(default)]
212 pub health_url: Option<String>,
213 /// Companion services this node installs from the release bundle after the
214 /// server is swapped and restarted. Each entry names a `[[companion]]` the
215 /// daemon builds (see `Config::companions`) and says where its binary lands
216 /// and which unit to restart — so a contract-coupled service (mnw-cli) ships
217 /// in the same promote as the server instead of drifting. Empty (default) =
218 /// server-only node, so an existing `sando.toml` keeps working unedited.
219 #[serde(default, rename = "companion")]
220 pub companions: Vec<NodeCompanion>,
221 }
222
223 /// Where a node installs a built companion binary and which unit to bounce.
224 #[derive(Debug, Clone, Serialize, Deserialize)]
225 pub struct NodeCompanion {
226 /// Must match a `Config::companions[].name` — the bundle subdir to read from.
227 pub name: String,
228 /// Absolute path the companion binary is installed to on the node
229 /// (the unit's `ExecStart`), e.g. `/opt/mnw-cli/mnw-cli`.
230 pub install_path: String,
231 /// systemd unit restarted after the binary is installed, e.g.
232 /// `mnw-cli.service`.
233 pub service_name: String,
234 }
235
236 fn default_service_name() -> String {
237 "makenotwork.service".into()
238 }
239
240 /// The capability set every pre-existing Sando node implicitly had: it deploys
241 /// and restarts, and is health-observed. Keeping these as the defaults is what
242 /// lets `sando.toml` stay unchanged through the executor refactor.
243 pub fn default_actuate() -> Vec<String> {
244 vec!["deploy".into(), "restart".into()]
245 }
246 pub fn default_observe() -> Vec<String> {
247 vec!["health".into()]
248 }
249
250 #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
251 #[serde(rename_all = "snake_case")]
252 pub enum CanaryPolicy {
253 #[default]
254 Sequential,
255 Parallel,
256 }
257
258 impl CanaryPolicy {
259 pub fn as_str(self) -> &'static str {
260 match self {
261 CanaryPolicy::Sequential => "sequential",
262 CanaryPolicy::Parallel => "parallel",
263 }
264 }
265 }
266
267 #[derive(Debug, Clone, Serialize, Deserialize)]
268 #[serde(tag = "kind", rename_all = "snake_case")]
269 pub enum Gate {
270 CargoTest,
271 HardeningTest,
272 Clippy,
273 Fmt,
274 CargoAudit,
275 CargoDeny,
276 MigrationDryRun,
277 CodeSmoke,
278 BootSmoke,
279 NodeHealth,
280 /// Post-deploy, in a real browser, over the tier's public URL.
281 ///
282 /// The one gate that crosses the CDN. `boot_smoke` runs on the build host
283 /// and `node_health` reaches a node over its executor, so between them the
284 /// edge was never watched -- and on 2026-08-14 testnot served a page whose
285 /// JavaScript did not run for hours behind nine green gates, because the
286 /// CDN held a stale module that failed to link against a fresh one.
287 ///
288 /// Needs `public_url` on the tier. A tier without one cannot run it, which
289 /// `validate` refuses at load rather than at promote time.
290 PageSmoke,
291 BurnIn {
292 hours: u32,
293 },
294 ManualConfirm,
295 }
296
297 impl Gate {
298 /// The discriminant — the identifier we use in events, schema columns,
299 /// and the TUI. Gate parameters (e.g. `BurnIn.hours`) stay with `Gate`
300 /// and are not carried into `gate_runs` history.
301 pub fn kind(&self) -> GateKind {
302 match self {
303 Gate::CargoTest => GateKind::CargoTest,
304 Gate::HardeningTest => GateKind::HardeningTest,
305 Gate::Clippy => GateKind::Clippy,
306 Gate::Fmt => GateKind::Fmt,
307 Gate::CargoAudit => GateKind::CargoAudit,
308 Gate::CargoDeny => GateKind::CargoDeny,
309 Gate::MigrationDryRun => GateKind::MigrationDryRun,
310 Gate::CodeSmoke => GateKind::CodeSmoke,
311 Gate::BootSmoke => GateKind::BootSmoke,
312 Gate::NodeHealth => GateKind::NodeHealth,
313 Gate::PageSmoke => GateKind::PageSmoke,
314 Gate::BurnIn { .. } => GateKind::BurnIn,
315 Gate::ManualConfirm => GateKind::ManualConfirm,
316 }
317 }
318
319 /// Gates that execute against a tier's freshly-deployed nodes, recording a
320 /// `gate_runs` row, at the end of a successful promote to that tier — the
321 /// evidence the *next* promote checks. Only `node_health`: it probes the
322 /// nodes the deploy just shipped to. `boot_smoke` is NOT post-deploy — it is
323 /// a build-time gate that boots the staged artifact on the build host and
324 /// proves nothing about a deployed node (the Run-2 SERIOUS-3 blind spot:
325 /// boot_smoke used to re-run locally at promote time and the next promote
326 /// trusted a binary that never touched the node).
327 pub fn runs_post_deploy(&self) -> bool {
328 matches!(self, Gate::NodeHealth | Gate::PageSmoke)
329 }
330
331 /// Gates evaluated at promote time against the deployed nodes or the
332 /// operator, as opposed to build-time gates (`cargo_test`,
333 /// `migration_dry_run`, `code_smoke`, `boot_smoke`) that run once on the build host and
334 /// prove nothing about a promote. A serving tier whose gate list contains
335 /// none of these would wave every promote straight through; `Topology
336 /// ::validate` refuses to load such a config (the structural form of CF1's
337 /// fail-closed default). `boot_smoke` no longer counts here — a serving tier
338 /// must declare a real node-level / operator gate, not a host smoke test.
339 pub fn guards_promotion(&self) -> bool {
340 matches!(
341 self,
342 Gate::NodeHealth | Gate::PageSmoke | Gate::BurnIn { .. } | Gate::ManualConfirm
343 )
344 }
345 }
346
347 impl Topology {
348 pub fn load(path: &Path) -> Result<Self> {
349 let raw = std::fs::read_to_string(path)
350 .with_context(|| format!("reading topology at {}", path.display()))?;
351 let topo: Topology = toml::from_str(&raw)?;
352 topo.validate()?;
353 Ok(topo)
354 }
355
356 /// Defense-in-depth for the build-host guard: a host that serves a
357 /// provisioned tier must never be designated the builder. The runtime
358 /// hostname check in `build::run` is the real guard; this catches a
359 /// misconfiguration (build_host pointed at a prod node's name/ssh_target) at
360 /// startup, before the daemon ever accepts a build. Called from `main` once
361 /// config + topology are both loaded.
362 pub fn ensure_build_host_not_serving(&self, build_host: &str) -> Result<()> {
363 for t in &self.tiers {
364 if !t.provisioned || t.name.as_str() == "host" {
365 continue;
366 }
367 for n in &t.nodes {
368 if n.name.as_str() == build_host || n.ssh_target == build_host {
369 anyhow::bail!(
370 "build_host {build_host:?} is also node {} (ssh {}) in serving tier {} — \
371 the builder must not be a prod/serving node",
372 n.name,
373 n.ssh_target,
374 t.name
375 );
376 }
377 }
378 }
379 Ok(())
380 }
381
382 #[cfg(test)]
383 pub(crate) fn validate_for_test(&self) -> Result<()> {
384 self.validate()
385 }
386
387 /// Every `[[migration_check]]` in the daemon config must name a dump this
388 /// topology declares. The two files are separate — daemon config is
389 /// per-host, topology is per-project — so nothing but this catches a check
390 /// pointing at a backup nobody fetches. Left uncaught it surfaces as a
391 /// permanently `Blocked` gate the first time someone promotes, which reads
392 /// like a missed fetch rather than a config typo. Called from `main` once
393 /// both are loaded, and so under `--check-config`.
394 pub fn ensure_migration_checks_have_backups(
395 &self,
396 checks: &[crate::config::MigrationCheck],
397 ) -> Result<()> {
398 // A product no tier dry-runs migrations for owes no dumps. `checks` is
399 // never empty — the config defaults it to MNW's `server` entry — so
400 // without this, an intake-only product with no postgres anywhere is
401 // asked to declare a prod dump for a gate it does not configure.
402 if !self
403 .tiers
404 .iter()
405 .flat_map(|t| &t.gates)
406 .any(|g| g.kind() == GateKind::MigrationDryRun)
407 {
408 return Ok(());
409 }
410 for c in checks {
411 anyhow::ensure!(
412 self.backup_named(&c.backup).is_some(),
413 "migration_check {} restores backup {:?}, which no [[backup]] in {} declares \
414 (have: {})",
415 c.dir.display(),
416 c.backup,
417 "the topology",
418 self.backup
419 .iter()
420 .map(|b| b.name.as_str())
421 .collect::<Vec<_>>()
422 .join(", "),
423 );
424 }
425 Ok(())
426 }
427
428 /// Every `[[test_target]]` with an `aux_repo` must name a repo this topology
429 /// checks out. Third of the cross-file checks, and the one whose absence has
430 /// already cost coverage once: an unresolvable target is a warn-and-skip, by
431 /// design, so that a config describing the tip can still build an older sha.
432 /// That makes a typo here indistinguishable from a legitimate bisect skip —
433 /// a green gate that ran one crate fewer than it says it does.
434 pub fn ensure_test_target_aux_repos_exist(
435 &self,
436 targets: &[crate::config::TestTarget],
437 ) -> Result<()> {
438 for t in targets {
439 let Some(name) = t.aux_repo.as_deref() else {
440 continue;
441 };
442 anyhow::ensure!(
443 self.aux_repos.iter().any(|a| a.name == name),
444 "test_target {} names aux_repo {:?}, which no [[aux_repo]] in the topology \
445 checks out, so the gate would skip it as absent (have: {})",
446 t.label(),
447 name,
448 if self.aux_repos.is_empty() {
449 "none".to_string()
450 } else {
451 self.aux_repos
452 .iter()
453 .map(|a| a.name.as_str())
454 .collect::<Vec<_>>()
455 .join(", ")
456 },
457 );
458 }
459 Ok(())
460 }
461
462 /// Every `[[tier.node.companion]]` must name a companion the daemon config
463 /// actually builds. Same two-file split as the migration checks above, and
464 /// the same class of typo, but a worse landing: companions are installed
465 /// AFTER the symlink swap (`deploy::deploy_remote`), so a name that stages
466 /// nothing fails a promote with the server already live on the new version
467 /// — `FailureStage::AtOrAfterSwap`, the case that needs a human to go look.
468 /// Catching it at load turns that into a startup error on the build host.
469 pub fn ensure_node_companions_are_built(
470 &self,
471 built: &[crate::config::Companion],
472 ) -> Result<()> {
473 for t in &self.tiers {
474 for n in &t.nodes {
475 for c in &n.companions {
476 anyhow::ensure!(
477 built.iter().any(|b| b.name == c.name),
478 "tier {} node {} installs companion {:?}, which no [[companion]] in the \
479 daemon config builds, so nothing would be staged under \
480 companions/{} (have: {})",
481 t.name,
482 n.name,
483 c.name,
484 c.name,
485 if built.is_empty() {
486 "none".to_string()
487 } else {
488 built
489 .iter()
490 .map(|b| b.name.as_str())
491 .collect::<Vec<_>>()
492 .join(", ")
493 },
494 );
495 }
496 }
497 }
498 Ok(())
499 }
500
501 /// The configured dump for `name`, or `None` when nothing declares it.
502 pub fn backup_named(&self, name: &str) -> Option<&BackupConfig> {
503 self.backup.iter().find(|b| b.name == name)
504 }
505
506 fn validate(&self) -> Result<()> {
507 // A dump is only owed by a product that actually dry-runs migrations.
508 // The unconditional form of this asserted something about every
509 // product's tiers from a fact about one: pom configures no
510 // `migration_dry_run` anywhere (it has no postgres schema at all), so
511 // requiring it to declare a prod dump would be demanding a fixture for
512 // a gate it never runs.
513 let dry_runs_migrations = self
514 .tiers
515 .iter()
516 .flat_map(|t| &t.gates)
517 .any(|g| g.kind() == GateKind::MigrationDryRun);
518 anyhow::ensure!(
519 !dry_runs_migrations || !self.backup.is_empty(),
520 "a tier configures migration_dry_run but the topology declares no [backup]; \
521 the gate would have nothing to restore"
522 );
523 // page_smoke has one input and it is not optional. Caught here rather
524 // than at promote time, because a gate that cannot run is a gate that
525 // would be discovered red halfway through a deploy, which is the worst
526 // moment to learn a URL is missing.
527 for t in &self.tiers {
528 let smokes = t.gates.iter().any(|g| g.kind() == GateKind::PageSmoke);
529 anyhow::ensure!(
530 !smokes || t.public_url.is_some(),
531 "tier {} configures page_smoke but declares no public_url; the gate has to \
532 request the site the way a visitor does, and a URL derived from a node would \
533 reach the origin and miss the CDN it exists to watch",
534 t.name,
535 );
536 }
537 for (i, b) in self.backup.iter().enumerate() {
538 anyhow::ensure!(
539 !b.name.is_empty()
540 && b.name
541 .bytes()
542 .all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-'),
543 "backup name {:?} must be non-empty and match [A-Za-z0-9_-]+; it keys the \
544 `backups` table and a daemon-config migration_check",
545 b.name,
546 );
547 anyhow::ensure!(
548 !b.source.is_empty() && !b.local_path.is_empty(),
549 "backup {} has an empty source/local_path",
550 b.name,
551 );
552 // Two dumps sharing a name would interleave in `backups`, so the
553 // freshness check and the plausibility floor would each read the
554 // other's row. Two sharing a `local_path` would overwrite each
555 // other on disk, and whichever fetched last would be restored for
556 // both — green, and proving nothing about one of the databases.
557 for prior in &self.backup[..i] {
558 anyhow::ensure!(
559 prior.name != b.name,
560 "two backup entries share the name {:?}",
561 b.name,
562 );
563 anyhow::ensure!(
564 prior.local_path != b.local_path,
565 "backups {:?} and {:?} share local_path {:?}; they would overwrite each other",
566 prior.name,
567 b.name,
568 b.local_path,
569 );
570 }
571 }
572 anyhow::ensure!(
573 !self.tiers.is_empty(),
574 "topology must declare at least one tier"
575 );
576 for t in &self.tiers {
577 // The `host` tier is the build tier (cargo_test / migration_dry_run /
578 // code_smoke / boot_smoke run once on the host); every other tier serves an
579 // artifact to nodes, so it is exempted from the node and
580 // promotion-gate checks the same way.
581 let is_build_tier = t.name.as_str() == "host";
582 if t.provisioned && t.nodes.is_empty() && !is_build_tier {
583 anyhow::bail!("tier {} is provisioned but has no nodes", t.name);
584 }
585 // Fail closed by default: a provisioned serving tier must declare at
586 // least one gate that actually guards a promote. Without this, an
587 // empty (or build-time-only) gate list waves every promote through
588 // because `unsatisfied_gates` finds nothing to check (CF1 root cause).
589 if t.provisioned && !is_build_tier && !t.gates.iter().any(Gate::guards_promotion) {
590 anyhow::bail!(
591 "tier {} is provisioned to serve but declares no promotion gate \
592 (need at least one of node_health / burn_in / manual_confirm)",
593 t.name
594 );
595 }
596 }
597 let mut seen_dirs: Vec<Vec<&str>> = Vec::new();
598 for aux in &self.aux_repos {
599 anyhow::ensure!(
600 !aux.name.is_empty() && !aux.bare_path.is_empty() && !aux.branch.is_empty(),
601 "aux_repo entry has an empty name/bare_path/branch"
602 );
603 // `checkout_dir` becomes a `workdir.join(..)`. Nesting is allowed —
604 // docengine lives at `Libraries/docengine` in the dev tree and the
605 // path dep resolves to that shape — but every component must be a
606 // plain name so an aux repo can never write outside the workdir.
607 let dir = &aux.checkout_dir;
608 let parts: Vec<&str> = dir.split('/').collect();
609 anyhow::ensure!(
610 !dir.is_empty()
611 && !dir.contains('\\')
612 && parts
613 .iter()
614 .all(|c| !c.is_empty() && *c != "." && *c != ".."),
615 "aux_repo {} has an unsafe checkout_dir {dir:?} (must be a relative path of \
616 plain components: no leading slash, empty segments, or dot-dot)",
617 aux.name,
618 );
619 // Two checkouts may not share a dir, and neither may sit inside the
620 // other: `git worktree add` into a path under a live worktree buries
621 // one checkout in the other's tree, and whichever builds second wins.
622 for prior in &seen_dirs {
623 let common = prior.len().min(parts.len());
624 anyhow::ensure!(
625 prior[..common] != parts[..common],
626 "two aux_repo entries share or nest checkout_dir {dir:?}; \
627 they would clobber each other"
628 );
629 }
630 seen_dirs.push(parts);
631 }
632 Ok(())
633 }
634 }
635
636 #[cfg(test)]
637 mod tests {
638 /// The shipped `sando.toml` must parse with the base-image declarations in
639 /// it, and the values must be the ones measured on the boxes. A declaration
640 /// that silently fails to deserialize is worse than none: the node would be
641 /// treated as undeclared, skipped, and the deploy log would say so in a line
642 /// nobody reads.
643 #[test]
644 fn the_shipped_topology_declares_what_each_node_is() {
645 let raw = std::fs::read_to_string(
646 std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml"),
647 )
648 .expect("the repo's sando.toml must be readable from the daemon crate");
649 let topo: Topology = toml::from_str(&raw).expect("sando.toml must parse");
650
651 let nodes: Vec<&Node> = topo.tiers.iter().flat_map(|t| t.nodes.iter()).collect();
652 let by = |name: &str| {
653 *nodes
654 .iter()
655 .find(|n| n.name.as_str() == name)
656 .unwrap_or_else(|| panic!("`{name}` must be in the topology"))
657 };
658 let image = |n: &Node| n.base_image.as_ref().map(ToString::to_string);
659
660 // Measured 2026-08-25. Staging is deliberately NOT the same base as
661 // production, and the assertion below says so out loud: tier A does not
662 // rehearse tier B on the axis these fields exist for.
663 let testnot = by("testnot-1");
664 assert_eq!(image(testnot).as_deref(), Some("ubuntu/26.04"));
665 assert_eq!(testnot.libc.as_deref(), Some("2.43"));
666
667 let prod = by("prod-1");
668 assert_eq!(image(prod).as_deref(), Some("ubuntu/24.04"));
669 assert_eq!(prod.libc.as_deref(), Some("2.39"));
670
671 assert_ne!(
672 testnot.base_image, prod.base_image,
673 "if these ever match, delete this assertion and the comment in \
674 sando.toml that explains why they do not"
675 );
676 }
677
678 use super::*;
679
680 /// A minimal topology with one serving tier whose gate block is `gates`.
681 fn topo_with_serving_gates(provisioned: bool, gates: &str) -> Topology {
682 let raw = format!(
683 r#"
684 [repo]
685 bare_path = "/tmp/repo.git"
686 branch = "main"
687
688 [backup]
689 source = "ssh://prod/dump.sql.gz"
690 local_path = "/tmp/dump.sql.gz"
691
692 [[tier]]
693 name = "b"
694 provisioned = {provisioned}
695 gates = [{gates}]
696 [[tier.node]]
697 name = "prod-1"
698 ssh_target = "prod-1"
699 release_root = "/srv/mnw"
700 "#
701 );
702 toml::from_str(&raw).expect("parse test topology")
703 }
704
705 #[test]
706 fn page_smoke_without_a_public_url_is_rejected_at_load() {
707 // The gate's whole value is that it requests the site the way a visitor
708 // does. Without a URL it cannot, and a gate that cannot run must not be
709 // discovered halfway through a promote.
710 let topo = topo_with_serving_gates(true, r#"{ kind = "page_smoke" }"#);
711 let err = topo.validate_for_test().expect_err("must refuse");
712 assert!(
713 err.to_string().contains("public_url"),
714 "error should name the missing field: {err}"
715 );
716 }
717
718 #[test]
719 fn page_smoke_with_a_public_url_loads() {
720 let raw = r#"
721 [repo]
722 bare_path = "/tmp/repo.git"
723 branch = "main"
724
725 [backup]
726 source = "ssh://prod/dump.sql.gz"
727 local_path = "/tmp/dump.sql.gz"
728
729 [[tier]]
730 name = "a"
731 provisioned = true
732 public_url = "https://testnot.work"
733 gates = [{ kind = "page_smoke" }]
734 [[tier.node]]
735 name = "testnot-1"
736 ssh_target = "testnot-1"
737 release_root = "/srv/mnw"
738 "#;
739 let topo: Topology = toml::from_str(raw).expect("parse");
740 topo.validate_for_test().expect("valid");
741 assert_eq!(
742 topo.tiers[0].public_url.as_deref(),
743 Some("https://testnot.work")
744 );
745 // It guards promotion out of its tier, and it runs after the deploy
746 // rather than on the build host -- both are the point of it.
747 assert!(topo.tiers[0].gates[0].guards_promotion());
748 assert!(topo.tiers[0].gates[0].runs_post_deploy());
749 }
750
751 #[test]
752 fn provisioned_serving_tier_with_no_gates_is_rejected() {
753 let topo = topo_with_serving_gates(true, "");
754 let err = topo.validate_for_test().unwrap_err().to_string();
755 assert!(err.contains("no promotion gate"), "{err}");
756 }
757
758 #[test]
759 fn provisioned_serving_tier_with_only_build_gates_is_rejected() {
760 // cargo_test / migration_dry_run are build-time and prove nothing about a
761 // promote, so a serving tier carrying only them still fails closed.
762 let topo = topo_with_serving_gates(
763 true,
764 r#"{ kind = "cargo_test" }, { kind = "migration_dry_run" }"#,
765 );
766 let err = topo.validate_for_test().unwrap_err().to_string();
767 assert!(err.contains("no promotion gate"), "{err}");
768 }
769
770 #[test]
771 fn provisioned_serving_tier_with_a_promotion_gate_is_accepted() {
772 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
773 assert!(topo.validate_for_test().is_ok());
774 }
775
776 #[test]
777 fn provisioned_serving_tier_with_only_boot_smoke_is_rejected() {
778 // boot_smoke is a build-host gate now; it proves nothing about a node, so
779 // a serving tier carrying only boot_smoke must fail closed exactly like an
780 // empty gate list (Run-2 SERIOUS-3 structural close).
781 let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
782 let err = topo.validate_for_test().unwrap_err().to_string();
783 assert!(err.contains("no promotion gate"), "{err}");
784 }
785
786 #[test]
787 fn unprovisioned_tier_with_empty_gates_is_skipped() {
788 // A declared-but-not-yet-provisioned tier (e.g. tier c) carries no
789 // promote authority, so the gate requirement does not apply yet.
790 let topo = topo_with_serving_gates(false, "");
791 assert!(topo.validate_for_test().is_ok());
792 }
793
794 #[test]
795 fn build_host_matching_a_serving_node_is_rejected() {
796 // prod-1 is a node in the provisioned serving tier built above; naming it
797 // as the builder must fail closed.
798 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
799 let err = topo
800 .ensure_build_host_not_serving("prod-1")
801 .unwrap_err()
802 .to_string();
803 assert!(err.contains("must not be a prod/serving node"), "{err}");
804 }
805
806 #[test]
807 fn build_host_distinct_from_serving_nodes_is_accepted() {
808 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
809 assert!(topo.ensure_build_host_not_serving("fw13").is_ok());
810 }
811
812 #[test]
813 fn node_companions_default_empty_and_parse_when_present() {
814 // A node without [[tier.node.companion]] is server-only.
815 let plain = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
816 assert!(plain.tiers[0].nodes[0].companions.is_empty());
817
818 // A node that declares a companion carries its install target + unit.
819 let raw = r#"
820 [repo]
821 bare_path = "/tmp/repo.git"
822 branch = "main"
823 [backup]
824 source = "s"
825 local_path = "/tmp/d"
826 [[tier]]
827 name = "b"
828 provisioned = true
829 gates = [{ kind = "node_health" }]
830 [[tier.node]]
831 name = "prod-1"
832 ssh_target = "makenotwork@alpha-west-1"
833 release_root = "/opt/mnw"
834 [[tier.node.companion]]
835 name = "mnw-cli"
836 install_path = "/opt/mnw-cli/mnw-cli"
837 service_name = "mnw-cli.service"
838 "#;
839 let topo: Topology = toml::from_str(raw).expect("parse companion topology");
840 let c = &topo.tiers[0].nodes[0].companions;
841 assert_eq!(c.len(), 1);
842 assert_eq!(c[0].name, "mnw-cli");
843 assert_eq!(c[0].install_path, "/opt/mnw-cli/mnw-cli");
844 assert_eq!(c[0].service_name, "mnw-cli.service");
845 }
846
847 /// A topology whose one node installs the named companions.
848 fn topo_installing(names: &[&str]) -> Topology {
849 let mut blocks = String::new();
850 for n in names {
851 use std::fmt::Write;
852 let _ = write!(
853 blocks,
854 "[[tier.node.companion]]\nname = \"{n}\"\n\
855 install_path = \"/opt/{n}/{n}\"\nservice_name = \"{n}.service\"\n"
856 );
857 }
858 let raw = format!(
859 r#"
860 [repo]
861 bare_path = "/tmp/repo.git"
862 branch = "main"
863 [backup]
864 source = "s"
865 local_path = "/tmp/d"
866 [[tier]]
867 name = "b"
868 provisioned = true
869 gates = [{{ kind = "node_health" }}]
870 [[tier.node]]
871 name = "prod-1"
872 ssh_target = "makenotwork@alpha-west-1"
873 release_root = "/opt/mnw"
874 {blocks}"#
875 );
876 toml::from_str(&raw).expect("parse topology")
877 }
878
879 fn built(names: &[&str]) -> Vec<crate::config::Companion> {
880 names
881 .iter()
882 .map(|n| crate::config::Companion {
883 name: (*n).to_string(),
884 manifest_dir: (*n).into(),
885 bin: (*n).to_string(),
886 })
887 .collect()
888 }
889
890 fn test_target(dir: &str, aux_repo: Option<&str>) -> crate::config::TestTarget {
891 crate::config::TestTarget {
892 dir: dir.into(),
893 aux_repo: aux_repo.map(str::to_string),
894 features: Vec::new(),
895 all_features: false,
896 scratch_db: false,
897 }
898 }
899
900 #[test]
901 fn a_test_target_naming_a_checked_out_aux_repo_is_accepted() {
902 let topo = topo_with_aux(
903 "[[aux_repo]]\nname = \"docengine\"\nbare_path = \"/tmp/d.git\"\n\
904 upstream = \"git@h:max/d.git\"\nbranch = \"main\"\ncheckout_dir = \"Libraries/docengine\"\n",
905 )
906 .expect("parse");
907 assert!(
908 topo.ensure_test_target_aux_repos_exist(&[
909 test_target("server", None),
910 test_target("", Some("docengine")),
911 ])
912 .is_ok()
913 );
914 }
915
916 #[test]
917 fn a_test_target_naming_an_unknown_aux_repo_is_rejected_at_load() {
918 // The failure this exists to prevent is silent: an unresolvable target
919 // is a warn-and-skip (bisect), so the gate stays green having run one
920 // crate fewer than the config claims.
921 let topo = topo_with_aux(
922 "[[aux_repo]]\nname = \"synckit\"\nbare_path = \"/tmp/s.git\"\n\
923 upstream = \"git@h:max/s.git\"\nbranch = \"main\"\ncheckout_dir = \"synckit\"\n",
924 )
925 .expect("parse");
926 let err = topo
927 .ensure_test_target_aux_repos_exist(&[test_target("", Some("docengine"))])
928 .unwrap_err()
929 .to_string();
930 assert!(err.contains("docengine"), "{err}");
931 assert!(err.contains("have: synckit"), "{err}");
932 }
933
934 #[test]
935 fn test_targets_without_an_aux_repo_need_no_aux_repos_declared() {
936 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
937 assert!(
938 topo.ensure_test_target_aux_repos_exist(&[test_target("server", None)])
939 .is_ok()
940 );
941 }
942
943 #[test]
944 fn a_node_companion_the_daemon_builds_is_accepted() {
945 let topo = topo_installing(&["mnw-cli", "multithreaded"]);
946 assert!(
947 topo.ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
948 .is_ok()
949 );
950 }
951
952 #[test]
953 fn a_node_companion_nothing_builds_is_rejected_at_load() {
954 // Left uncaught this fails during the post-swap install on prod, with
955 // the server already live on the new version.
956 let topo = topo_installing(&["mnw-cli", "multithreadd"]);
957 let err = topo
958 .ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
959 .unwrap_err()
960 .to_string();
961 assert!(err.contains("multithreadd"), "{err}");
962 assert!(err.contains("no [[companion]]"), "{err}");
963 // The message names what IS available, so the typo is obvious.
964 assert!(err.contains("mnw-cli, multithreaded"), "{err}");
965 }
966
967 #[test]
968 fn a_node_companion_with_no_companions_configured_at_all_is_rejected() {
969 let topo = topo_installing(&["multithreaded"]);
970 let err = topo
971 .ensure_node_companions_are_built(&[])
972 .unwrap_err()
973 .to_string();
974 assert!(err.contains("have: none"), "{err}");
975 }
976
977 #[test]
978 fn a_topology_installing_no_companions_is_fine_with_none_built() {
979 let topo = topo_installing(&[]);
980 assert!(topo.ensure_node_companions_are_built(&[]).is_ok());
981 }
982
983 fn topo_with_aux(aux_block: &str) -> Result<Topology> {
984 let raw = format!(
985 r#"
986 [repo]
987 bare_path = "/tmp/repo.git"
988 branch = "main"
989 [backup]
990 source = "s"
991 local_path = "/tmp/d"
992 [[tier]]
993 name = "b"
994 provisioned = true
995 gates = [{{ kind = "node_health" }}]
996 [[tier.node]]
997 name = "prod-1"
998 ssh_target = "prod-1"
999 release_root = "/srv/mnw"
1000 {aux_block}
1001 "#
1002 );
1003 let topo: Topology = toml::from_str(&raw)?;
1004 topo.validate_for_test()?;
1005 Ok(topo)
1006 }
1007
1008 #[test]
1009 fn aux_repos_default_empty() {
1010 let topo = topo_with_aux("").expect("no aux_repo block is fine");
1011 assert!(topo.aux_repos.is_empty());
1012 }
1013
1014 #[test]
1015 fn aux_repo_parses_all_fields() {
1016 let topo = topo_with_aux(
1017 r#"
1018 [[aux_repo]]
1019 name = "synckit"
1020 bare_path = "/srv/sando/synckit.git"
1021 upstream = "git@ssh.makenot.work:max/synckit.git"
1022 branch = "main"
1023 checkout_dir = "synckit""#,
1024 )
1025 .expect("valid aux_repo parses");
1026 assert_eq!(topo.aux_repos.len(), 1);
1027 let a = &topo.aux_repos[0];
1028 assert_eq!(a.name, "synckit");
1029 assert_eq!(a.bare_path, "/srv/sando/synckit.git");
1030 assert_eq!(a.upstream, "git@ssh.makenot.work:max/synckit.git");
1031 assert_eq!(a.branch, "main");
1032 assert_eq!(a.checkout_dir, "synckit");
1033 }
1034
1035 #[test]
1036 fn aux_repo_with_nested_checkout_dir_is_accepted() {
1037 let topo = topo_with_aux(
1038 r#"
1039 [[aux_repo]]
1040 name = "docengine"
1041 bare_path = "/srv/sando/docengine.git"
1042 upstream = "git@ssh.makenot.work:max/docengine.git"
1043 branch = "main"
1044 checkout_dir = "Libraries/docengine""#,
1045 )
1046 .expect("a nested checkout_dir is a valid location");
1047 assert_eq!(topo.aux_repos[0].checkout_dir, "Libraries/docengine");
1048 }
1049
1050 #[test]
1051 fn aux_repo_with_traversing_checkout_dir_is_rejected() {
1052 for bad in [
1053 "../escape",
1054 "a/../../escape",
1055 "a/./b",
1056 "a//b",
1057 "/abs",
1058 "a/",
1059 "..",
1060 ".",
1061 ] {
1062 let err = topo_with_aux(&format!(
1063 r#"
1064 [[aux_repo]]
1065 name = "x"
1066 bare_path = "/srv/sando/x.git"
1067 upstream = "u"
1068 branch = "main"
1069 checkout_dir = "{bad}""#,
1070 ))
1071 .unwrap_err()
1072 .to_string();
1073 assert!(err.contains("unsafe checkout_dir"), "for {bad:?}: {err}");
1074 }
1075 }
1076
1077 #[test]
1078 fn aux_repos_sharing_a_checkout_dir_are_rejected() {
1079 let err = topo_with_aux(
1080 r#"
1081 [[aux_repo]]
1082 name = "one"
1083 bare_path = "/srv/sando/one.git"
1084 upstream = "u"
1085 branch = "main"
1086 checkout_dir = "shared"
1087 [[aux_repo]]
1088 name = "two"
1089 bare_path = "/srv/sando/two.git"
1090 upstream = "u"
1091 branch = "main"
1092 checkout_dir = "shared""#,
1093 )
1094 .unwrap_err()
1095 .to_string();
1096 assert!(err.contains("share or nest checkout_dir"), "{err}");
1097 }
1098
1099 #[test]
1100 fn aux_repo_nested_inside_another_checkout_dir_is_rejected() {
1101 let err = topo_with_aux(
1102 r#"
1103 [[aux_repo]]
1104 name = "outer"
1105 bare_path = "/srv/sando/outer.git"
1106 upstream = "u"
1107 branch = "main"
1108 checkout_dir = "Libraries"
1109 [[aux_repo]]
1110 name = "inner"
1111 bare_path = "/srv/sando/inner.git"
1112 upstream = "u"
1113 branch = "main"
1114 checkout_dir = "Libraries/docengine""#,
1115 )
1116 .unwrap_err()
1117 .to_string();
1118 assert!(err.contains("share or nest checkout_dir"), "{err}");
1119 }
1120
1121 /// A topology whose `[backup]`/`[[backup]]` section is `backup_block`.
1122 fn topo_with_backup_block(backup_block: &str) -> Result<Topology> {
1123 let raw = format!(
1124 r#"
1125 [repo]
1126 bare_path = "/tmp/repo.git"
1127 branch = "main"
1128 {backup_block}
1129 [[tier]]
1130 name = "b"
1131 provisioned = true
1132 # migration_dry_run is what makes the backup rules apply at all: a product no
1133 # tier dry-runs migrations for owes no dumps, so a fixture exercising those rules
1134 # has to configure the gate.
1135 gates = [{{ kind = "node_health" }}, {{ kind = "migration_dry_run" }}]
1136 [[tier.node]]
1137 name = "prod-1"
1138 ssh_target = "prod-1"
1139 release_root = "/srv/mnw"
1140 "#
1141 );
1142 let topo: Topology = toml::from_str(&raw)?;
1143 topo.validate_for_test()?;
1144 Ok(topo)
1145 }
1146
1147 /// A topology with no `[backup]` at all, so the dump rules are exercised
1148 /// by what its gates ask for rather than by what it declares.
1149 fn topo_without_backup(gates: &str) -> Result<Topology> {
1150 let raw = format!(
1151 r#"
1152 backup = []
1153
1154 [repo]
1155 bare_path = "/tmp/repo.git"
1156 branch = "main"
1157
1158 [[tier]]
1159 name = "b"
1160 provisioned = true
1161 gates = [{gates}]
1162 [[tier.node]]
1163 name = "prod-1"
1164 ssh_target = "prod-1"
1165 release_root = "/srv/mnw"
1166 "#
1167 );
1168 let topo: Topology = toml::from_str(&raw)?;
1169 topo.validate_for_test()?;
1170 Ok(topo)
1171 }
1172
1173 #[test]
1174 fn a_product_that_never_dry_runs_migrations_owes_no_dump() {
1175 // pom is this product: no postgres schema, so no tier configures
1176 // migration_dry_run and demanding a prod dump would be demanding a
1177 // fixture for a gate that never runs.
1178 let topo = topo_without_backup(r#"{ kind = "node_health" }"#)
1179 .expect("a topology with no migration gate loads without a [backup]");
1180 assert!(topo.backup.is_empty());
1181 }
1182
1183 #[test]
1184 fn a_migration_dry_run_with_no_backup_is_rejected_at_load() {
1185 // The gate restores a dump into the scratch database. With nothing
1186 // declared it would have nothing to restore, and the discovery would
1187 // come mid-promote.
1188 let err =
1189 topo_without_backup(r#"{ kind = "node_health" }, { kind = "migration_dry_run" }"#)
1190 .expect_err("a dry-run gate with no dump declared must not load")
1191 .to_string();
1192 assert!(err.contains("declares no [backup]"), "{err}");
1193 }
1194
1195 #[test]
1196 fn a_single_backup_table_still_parses_as_one_named_server() {
1197 // Back-compat is the point: every deployed sando.toml uses the single
1198 // `[backup]` form, and the box this config lives on is the one whose job
1199 // is deploying — it must not need an edit to start.
1200 let topo = topo_with_backup_block(
1201 r#"
1202 [backup]
1203 source = "ssh://prod/dump.sql.gz"
1204 local_path = "/tmp/dump.sql.gz""#,
1205 )
1206 .expect("the single-table form must still load");
1207 assert_eq!(topo.backup.len(), 1);
1208 assert_eq!(topo.backup[0].name, "server");
1209 assert!(topo.backup_named("server").is_some());
1210 }
1211
1212 #[test]
1213 fn a_backup_list_parses_and_keeps_its_names() {
1214 let topo = topo_with_backup_block(
1215 r#"
1216 [[backup]]
1217 name = "server"
1218 source = "ssh://prod/makenotwork/latest.sql.gz"
1219 local_path = "/tmp/server.sql.gz"
1220 [[backup]]
1221 name = "multithreaded"
1222 source = "ssh://prod/multithreaded/latest.sql.gz"
1223 local_path = "/tmp/mt.sql.gz""#,
1224 )
1225 .expect("the list form must load");
1226 assert_eq!(topo.backup.len(), 2);
1227 assert_eq!(
1228 topo.backup_named("multithreaded").unwrap().local_path,
1229 "/tmp/mt.sql.gz"
1230 );
1231 assert!(topo.backup_named("nope").is_none());
1232 }
1233
1234 #[test]
1235 fn two_backups_sharing_a_name_are_rejected() {
1236 // They would interleave in `backups`, so the freshness check and the
1237 // plausibility floor would each read the other's row.
1238 let err = topo_with_backup_block(
1239 r#"
1240 [[backup]]
1241 name = "server"
1242 source = "a"
1243 local_path = "/tmp/a.sql.gz"
1244 [[backup]]
1245 name = "server"
1246 source = "b"
1247 local_path = "/tmp/b.sql.gz""#,
1248 )
1249 .unwrap_err()
1250 .to_string();
1251 assert!(err.contains("share the name"), "{err}");
1252 }
1253
1254 #[test]
1255 fn two_backups_sharing_a_local_path_are_rejected() {
1256 // Whichever fetched last would be restored for both checks — green, and
1257 // proving nothing about one of the two databases.
1258 let err = topo_with_backup_block(
1259 r#"
1260 [[backup]]
1261 name = "server"
1262 source = "a"
1263 local_path = "/tmp/same.sql.gz"
1264 [[backup]]
1265 name = "multithreaded"
1266 source = "b"
1267 local_path = "/tmp/same.sql.gz""#,
1268 )
1269 .unwrap_err()
1270 .to_string();
1271 assert!(err.contains("share local_path"), "{err}");
1272 }
1273
1274 #[test]
1275 fn a_migration_check_naming_an_undeclared_backup_is_rejected_at_startup() {
1276 // Daemon config and topology are separate files, so nothing but this
1277 // cross-check catches the typo. Uncaught it surfaces as a permanently
1278 // Blocked gate on the next promote, which reads like a missed fetch.
1279 let topo = topo_with_backup_block(
1280 r#"
1281 [backup]
1282 source = "s"
1283 local_path = "/tmp/d""#,
1284 )
1285 .unwrap();
1286 let checks = vec![crate::config::MigrationCheck {
1287 dir: std::path::PathBuf::from("multithreaded/migrations"),
1288 backup: "multithreaded".into(),
1289 scratch_db: Some("sando_scratch_mt".into()),
1290 owner_role: Some("multithreaded".into()),
1291 }];
1292 let err = topo
1293 .ensure_migration_checks_have_backups(&checks)
1294 .unwrap_err()
1295 .to_string();
1296 assert!(err.contains("which no [[backup]]"), "{err}");
1297
1298 // And the shipped pair agree, which is the case that actually ships.
1299 let shipped_checks = crate::config::default_migration_checks_for_test();
1300 shipped()
1301 .ensure_migration_checks_have_backups(&shipped_checks)
1302 .expect("the default server check resolves against the shipped topology");
1303 }
1304
1305 #[test]
1306 fn real_sando_toml_loads_clean() {
1307 // The shipped topology must satisfy the invariant — guards against a
1308 // regression that would lock sandod out of its own config.
1309 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
1310 Topology::load(&path).expect("shipped sando.toml must validate");
1311 }
1312
1313 fn shipped() -> Topology {
1314 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
1315 Topology::load(&path).expect("shipped sando.toml must validate")
1316 }
1317
1318 #[test]
1319 fn shipping_to_the_last_provisioned_tier_needs_an_operator_signoff() {
1320 // A tier's gates guard promotion *out* of it, which is the subtlety that
1321 // made manual_confirm inert: it sat on tier b, guarding b -> c, and c is
1322 // not provisioned. So the ship to production was cleared by node_health
1323 // + burn_in alone — and `hotfix: true` skips burn_in.
1324 //
1325 // The gate that matters therefore belongs on the PREDECESSOR of the last
1326 // provisioned tier. Asserted structurally so re-provisioning tiers cannot
1327 // silently strand the sign-off again.
1328 let topo = shipped();
1329 let last = topo
1330 .tiers
1331 .iter()
1332 .rposition(|t| t.provisioned)
1333 .expect("some tier must be provisioned");
1334 assert!(last > 0, "the production tier cannot be the first tier");
1335 let guard = &topo.tiers[last - 1];
1336 assert!(
1337 guard.gates.iter().any(|g| matches!(g, Gate::ManualConfirm)),
1338 "tier {} guards promotion into the last provisioned tier ({}), so it must require an operator sign-off; its gates are {:?}",
1339 guard.name,
1340 topo.tiers[last].name,
1341 guard
1342 .gates
1343 .iter()
1344 .map(|g| g.kind().as_str())
1345 .collect::<Vec<_>>(),
1346 );
1347 }
1348
1349 #[test]
1350 fn every_serving_node_has_a_readiness_probe() {
1351 // Without health_url, node_health degrades to `systemctl is-active`,
1352 // which a crash-looping binary satisfies between restarts — exactly what
1353 // the 0.10.14 CDN_BASE_URL crash-loop did on prod-1.
1354 let topo = shipped();
1355 for tier in topo.tiers.iter().filter(|t| t.provisioned) {
1356 for node in &tier.nodes {
1357 assert!(
1358 node.health_url.is_some(),
1359 "node {} on tier {} has no health_url, so node_health proves only that systemd thinks the unit is running",
1360 node.name,
1361 tier.name,
1362 );
1363 }
1364 }
1365 }
1366 }
1367