Skip to main content

max / makenotwork

51.5 KB · 1361 lines History Blame Raw
1 use crate::domain::{GateKind, NodeId, TierId};
2 use anyhow::{Context, Result};
3 use serde::{Deserialize, Serialize};
4 use std::path::Path;
5
6 #[derive(Debug, Clone, Serialize, Deserialize)]
7 pub struct Topology {
8 /// The repo Sando checks out to build this product.
9 ///
10 /// `None` for an intake-only product: pom is built by Bento on two
11 /// machines and handed over as finished bytes, so Sando fetches no source
12 /// for it and there is no bare repo on this host to name. A topology that
13 /// declares no repo cannot be `/rebuild`-ed, which is the same statement
14 /// [`AppConfig::build_host`](crate::config::AppConfig::build_host) makes
15 /// from the other side.
16 #[serde(default)]
17 pub repo: Option<RepoConfig>,
18 /// Prod dumps `/backup/fetch` pulls, one per database `migration_dry_run`
19 /// has a check for. A list because the repo ships more than one service
20 /// with its own database and its own `sqlx::migrate!()` at boot: the server
21 /// migrates `makenotwork`, multithreaded migrates `multithreaded`, and a
22 /// gate that restores only the first proves nothing about the second.
23 ///
24 /// Accepts both a single `[backup]` table and a `[[backup]]` list; the
25 /// single form deserializes to a one-entry list named `server`.
26 #[serde(deserialize_with = "one_or_many_backup")]
27 pub backup: Vec<BackupConfig>,
28 #[serde(rename = "tier")]
29 pub tiers: Vec<Tier>,
30 /// Extra repos to fetch and check out beside the main worktree before a
31 /// build, so a path dependency that reaches across the repo split resolves.
32 /// Empty (the default) fetches nothing extra. See
33 /// [`AuxRepo`] and [`crate::build::checkout_aux_repos`].
34 #[serde(default, rename = "aux_repo")]
35 pub aux_repos: Vec<AuxRepo>,
36 }
37
38 /// An auxiliary repo checked out beside the per-sha worktree so cross-repo path
39 /// dependencies resolve at build time.
40 ///
41 /// The concrete case: `mnw-cli`, a companion built from the MNW worktree,
42 /// carries `synckit-client = { path = "../../synckit/synckit-client" }`. From the
43 /// companion crate at `<workdir>/<sha>/mnw-cli` that path resolves to
44 /// `<workdir>/synckit`, a sibling of the per-sha worktree Sando otherwise never
45 /// creates, so the companion build fails with "No such file or directory". An
46 /// `aux_repo` named to land at `checkout_dir = "synckit"` puts the synckit source
47 /// exactly there.
48 ///
49 /// The checkout is at the FIXED `<workdir>/<checkout_dir>`, not per-sha: the path
50 /// dependency resolves to that spot regardless of the MNW sha, and builds
51 /// serialize, so a single shared checkout refreshed to `branch` HEAD each build
52 /// is correct. Because a path dep has no lockfile pin, "branch HEAD" is the honest
53 /// resolution — the same contract as the dev working copy.
54 #[derive(Debug, Clone, Serialize, Deserialize)]
55 pub struct AuxRepo {
56 /// Human label for logs and errors (e.g. `synckit`).
57 pub name: String,
58 /// Bare repo Sando fetches into and worktrees from, e.g.
59 /// `/srv/sando/synckit.git`. Auto-created (hookless) on first build.
60 pub bare_path: String,
61 /// Canonical git remote fetched before checkout. Like [`RepoConfig::upstream`]
62 /// but required here: an aux repo is pull-based (nobody pushes to its bare).
63 pub upstream: String,
64 /// Branch whose HEAD is checked out.
65 pub branch: String,
66 /// Where the worktree lands, relative to `cfg.workdir`. May name a nested
67 /// location (`Libraries/docengine`), because a path dep resolves to wherever
68 /// the dev tree keeps the crate and Sando has to match that shape. Every
69 /// component must be a plain name — no leading slash, no `..` — so the
70 /// checkout stays under the workdir.
71 pub checkout_dir: String,
72 }
73
74 #[derive(Debug, Clone, Serialize, Deserialize)]
75 pub struct RepoConfig {
76 pub bare_path: String,
77 pub branch: String,
78 /// Optional canonical git remote. When set, `/rebuild` fetches the deploy
79 /// branch from here into the bare repo before worktree-ing the target sha,
80 /// so a freshly-pushed commit is resolvable without anyone pushing to the
81 /// bare repo directly (pull-based deploys). When unset, Sando relies on the
82 /// bare repo already containing the sha (push-based / hook-driven).
83 #[serde(default)]
84 pub upstream: Option<String>,
85 }
86
87 #[derive(Debug, Clone, Serialize, Deserialize)]
88 pub struct BackupConfig {
89 /// Which database this dump is of, as referenced by a daemon-config
90 /// `[[migration_check]]`'s `backup` key and recorded in the `backups`
91 /// table's `name` column. Defaults to `server`, which is also what the
92 /// state-DB migration backfills pre-existing rows to.
93 #[serde(default = "default_backup_name")]
94 pub name: String,
95 pub source: String,
96 pub local_path: String,
97 }
98
99 fn default_backup_name() -> String {
100 "server".into()
101 }
102
103 /// Accept `[backup]` (one table) or `[[backup]]` (a list) for the same key.
104 /// Serde cannot express "table or sequence" on a `Vec` field on its own, and
105 /// the alternative — renaming the key — would break every deployed
106 /// `sando.toml` at startup, on the box whose whole job is deploying.
107 fn one_or_many_backup<'de, D>(de: D) -> std::result::Result<Vec<BackupConfig>, D::Error>
108 where
109 D: serde::Deserializer<'de>,
110 {
111 #[derive(Deserialize)]
112 #[serde(untagged)]
113 enum OneOrMany {
114 One(BackupConfig),
115 Many(Vec<BackupConfig>),
116 }
117 Ok(match OneOrMany::deserialize(de)? {
118 OneOrMany::One(b) => vec![b],
119 OneOrMany::Many(v) => v,
120 })
121 }
122
123 #[derive(Debug, Clone, Serialize, Deserialize)]
124 pub struct Tier {
125 pub name: TierId,
126 #[serde(default)]
127 pub provisioned: bool,
128 pub gates: Vec<Gate>,
129 #[serde(default)]
130 pub canary: CanaryPolicy,
131 #[serde(default, rename = "node")]
132 pub nodes: Vec<Node>,
133 /// Where the public reaches this tier, CDN included.
134 ///
135 /// Deliberately not derivable from a node's `ssh_target`. The whole value of
136 /// [`Gate::PageSmoke`] is that it goes the way a visitor goes -- through
137 /// Cloudflare, against the hostname in the browser's address bar -- and a
138 /// URL computed from the node would reach the origin and inherit exactly the
139 /// blind spot the gate exists to close. So it is stated, or the gate is not
140 /// available.
141 #[serde(default)]
142 pub public_url: Option<String>,
143 }
144
145 #[derive(Debug, Clone, Serialize, Deserialize)]
146 pub struct Node {
147 pub name: NodeId,
148 pub ssh_target: String,
149 pub release_root: String,
150 /// What this machine runs, as `os/arch` (e.g. `linux/aarch64`).
151 ///
152 /// Compared against the bundle's own platform before anything is pushed;
153 /// see [`crate::deploy::Placement`]. Optional, and a node that declares it
154 /// can only be given a bundle that declares a matching one — silence on
155 /// either side is a refusal, not a pass. MNW's nodes declare nothing and
156 /// keep the single-platform behavior they have always had; pom's declare
157 /// theirs, because pom is the product where one version is two bundles.
158 #[serde(default)]
159 pub platform: Option<crate::domain::Platform>,
160 /// What this machine IS, as `id/version` from `/etc/os-release`
161 /// (e.g. `ubuntu/24.04`, `alloy/0.1`).
162 ///
163 /// Verified against the node before anything is pushed to it, so a box that
164 /// was rebuilt into something else fails the promote with the running
165 /// service intact instead of being discovered by a binary that will not
166 /// start. Optional: a node that declares nothing is not checked, and the
167 /// deploy log says it was not. See [`ops_core::base_image`] for why silence
168 /// is a skip here and a refusal in [`crate::deploy::Placement`].
169 #[serde(default)]
170 pub base_image: Option<ops_core::base_image::BaseImage>,
171 /// The glibc version this node has, as `ldd --version` reports it
172 /// (e.g. `2.39`). Checked independently of [`Self::base_image`], because a
173 /// point release moves under a pinned base and it is this number that
174 /// decides whether a binary loads.
175 ///
176 /// Declaring it does NOT replace the pre-swap `ldd` guard, which compares
177 /// the actual binary against the actual node. It makes the node's floor a
178 /// stated fact that can be compared before bytes are built or moved.
179 #[serde(default)]
180 pub libc: Option<String>,
181 /// systemd unit name to reload-or-restart after the symlink swap.
182 /// Defaults to "makenotwork.service" because that's MNW's prod unit.
183 #[serde(default = "default_service_name")]
184 pub service_name: String,
185 /// Opt-in config-drift guard. When set to the node's env-file path (e.g.
186 /// `/etc/mnw/makenotwork.env`), the deploy sources that file and runs the
187 /// freshly-rsynced binary in `MNW_CHECK_CONFIG=1` mode BEFORE the symlink
188 /// swap. A required var missing on this node (a var added upstream but never
189 /// added to the node's env — how testnot crash-looped on CDN_BASE_URL) then
190 /// fails the promote with the running service still intact, instead of after
191 /// the swap+restart. Unset (default) skips the check, so an existing
192 /// `sando.toml` keeps working and it never runs against a binary too old to
193 /// support the mode; enable it once a check-capable version is deployed.
194 #[serde(default)]
195 pub config_check_env_file: Option<String>,
196 /// Capability grant for this node's executor (see `ops_exec`). Defaults to
197 /// the current behavior of every Sando node — actuate deploy+restart,
198 /// observe health — so an existing `sando.toml` keeps working unedited.
199 #[serde(default = "default_actuate")]
200 pub actuate: Vec<String>,
201 #[serde(default = "default_observe")]
202 pub observe: Vec<String>,
203 /// Optional HTTP readiness URL the `node_health` gate curls on the node (over
204 /// its executor) after `systemctl is-active`. Typically a loopback address
205 /// the service binds, e.g. `http://127.0.0.1:8080/health`. When unset, the
206 /// gate proves the unit is active post-restart but does not HTTP-probe; set
207 /// it for a full readiness check. Kept optional so an existing `sando.toml`
208 /// needs no edit.
209 #[serde(default)]
210 pub health_url: Option<String>,
211 /// Companion services this node installs from the release bundle after the
212 /// server is swapped and restarted. Each entry names a `[[companion]]` the
213 /// daemon builds (see `Config::companions`) and says where its binary lands
214 /// and which unit to restart — so a contract-coupled service (mnw-cli) ships
215 /// in the same promote as the server instead of drifting. Empty (default) =
216 /// server-only node, so an existing `sando.toml` keeps working unedited.
217 #[serde(default, rename = "companion")]
218 pub companions: Vec<NodeCompanion>,
219 }
220
221 /// Where a node installs a built companion binary and which unit to bounce.
222 #[derive(Debug, Clone, Serialize, Deserialize)]
223 pub struct NodeCompanion {
224 /// Must match a `Config::companions[].name` — the bundle subdir to read from.
225 pub name: String,
226 /// Absolute path the companion binary is installed to on the node
227 /// (the unit's `ExecStart`), e.g. `/opt/mnw-cli/mnw-cli`.
228 pub install_path: String,
229 /// systemd unit restarted after the binary is installed, e.g.
230 /// `mnw-cli.service`.
231 pub service_name: String,
232 }
233
234 fn default_service_name() -> String {
235 "makenotwork.service".into()
236 }
237
238 /// The capability set every pre-existing Sando node implicitly had: it deploys
239 /// and restarts, and is health-observed. Keeping these as the defaults is what
240 /// lets `sando.toml` stay unchanged through the executor refactor.
241 pub fn default_actuate() -> Vec<String> {
242 vec!["deploy".into(), "restart".into()]
243 }
244 pub fn default_observe() -> Vec<String> {
245 vec!["health".into()]
246 }
247
248 #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
249 #[serde(rename_all = "snake_case")]
250 pub enum CanaryPolicy {
251 #[default]
252 Sequential,
253 Parallel,
254 }
255
256 impl CanaryPolicy {
257 pub fn as_str(self) -> &'static str {
258 match self {
259 CanaryPolicy::Sequential => "sequential",
260 CanaryPolicy::Parallel => "parallel",
261 }
262 }
263 }
264
265 #[derive(Debug, Clone, Serialize, Deserialize)]
266 #[serde(tag = "kind", rename_all = "snake_case")]
267 pub enum Gate {
268 CargoTest,
269 HardeningTest,
270 Clippy,
271 Fmt,
272 CargoAudit,
273 CargoDeny,
274 MigrationDryRun,
275 CodeSmoke,
276 BootSmoke,
277 NodeHealth,
278 /// Post-deploy, in a real browser, over the tier's public URL.
279 ///
280 /// The one gate that crosses the CDN. `boot_smoke` runs on the build host
281 /// and `node_health` reaches a node over its executor, so neither would
282 /// notice a stale module held at the CDN failing to link against a fresh
283 /// one.
284 ///
285 /// Needs `public_url` on the tier. A tier without one cannot run it, which
286 /// `validate` refuses at load rather than at promote time.
287 PageSmoke,
288 BurnIn {
289 hours: u32,
290 },
291 ManualConfirm,
292 }
293
294 impl Gate {
295 /// The discriminant: the identifier used in events, schema columns, and the
296 /// TUI. Gate parameters (e.g. `BurnIn.hours`) stay with `Gate`
297 /// and are not carried into `gate_runs` history.
298 pub fn kind(&self) -> GateKind {
299 match self {
300 Gate::CargoTest => GateKind::CargoTest,
301 Gate::HardeningTest => GateKind::HardeningTest,
302 Gate::Clippy => GateKind::Clippy,
303 Gate::Fmt => GateKind::Fmt,
304 Gate::CargoAudit => GateKind::CargoAudit,
305 Gate::CargoDeny => GateKind::CargoDeny,
306 Gate::MigrationDryRun => GateKind::MigrationDryRun,
307 Gate::CodeSmoke => GateKind::CodeSmoke,
308 Gate::BootSmoke => GateKind::BootSmoke,
309 Gate::NodeHealth => GateKind::NodeHealth,
310 Gate::PageSmoke => GateKind::PageSmoke,
311 Gate::BurnIn { .. } => GateKind::BurnIn,
312 Gate::ManualConfirm => GateKind::ManualConfirm,
313 }
314 }
315
316 /// Gates that execute against a tier's freshly-deployed nodes, recording a
317 /// `gate_runs` row, at the end of a successful promote to that tier, as the
318 /// evidence the *next* promote checks. `boot_smoke` is NOT post-deploy: it
319 /// is a build-time gate that boots the staged artifact on the build host and
320 /// proves nothing about a deployed node.
321 pub fn runs_post_deploy(&self) -> bool {
322 matches!(self, Gate::NodeHealth | Gate::PageSmoke)
323 }
324
325 /// Gates evaluated at promote time against the deployed nodes or the
326 /// operator, as opposed to build-time gates (`cargo_test`,
327 /// `migration_dry_run`, `code_smoke`, `boot_smoke`) that run once on the build host and
328 /// prove nothing about a promote. A serving tier whose gate list contains
329 /// none of these would wave every promote straight through; `Topology
330 /// ::validate` refuses to load such a config. `boot_smoke` does not count
331 /// here: a serving tier must declare a real node-level or operator gate, not
332 /// a host smoke test.
333 pub fn guards_promotion(&self) -> bool {
334 matches!(
335 self,
336 Gate::NodeHealth | Gate::PageSmoke | Gate::BurnIn { .. } | Gate::ManualConfirm
337 )
338 }
339 }
340
341 impl Topology {
342 pub fn load(path: &Path) -> Result<Self> {
343 let raw = std::fs::read_to_string(path)
344 .with_context(|| format!("reading topology at {}", path.display()))?;
345 let topo: Topology = toml::from_str(&raw)?;
346 topo.validate()?;
347 Ok(topo)
348 }
349
350 /// Defense-in-depth for the build-host guard: a host that serves a
351 /// provisioned tier must never be designated the builder. The runtime
352 /// hostname check in `build::run` is the real guard; this catches a
353 /// misconfiguration (build_host pointed at a prod node's name/ssh_target) at
354 /// startup, before the daemon ever accepts a build. Called from `main` once
355 /// config + topology are both loaded.
356 pub fn ensure_build_host_not_serving(&self, build_host: &str) -> Result<()> {
357 for t in &self.tiers {
358 if !t.provisioned || t.name.as_str() == "host" {
359 continue;
360 }
361 for n in &t.nodes {
362 if n.name.as_str() == build_host || n.ssh_target == build_host {
363 anyhow::bail!(
364 "build_host {build_host:?} is also node {} (ssh {}) in serving tier {} — \
365 the builder must not be a prod/serving node",
366 n.name,
367 n.ssh_target,
368 t.name
369 );
370 }
371 }
372 }
373 Ok(())
374 }
375
376 #[cfg(test)]
377 pub(crate) fn validate_for_test(&self) -> Result<()> {
378 self.validate()
379 }
380
381 /// Every `[[migration_check]]` in the daemon config must name a dump this
382 /// topology declares. The two files are separate — daemon config is
383 /// per-host, topology is per-project — so nothing but this catches a check
384 /// pointing at a backup nobody fetches. Left uncaught it surfaces as a
385 /// permanently `Blocked` gate the first time someone promotes, which reads
386 /// like a missed fetch rather than a config typo. Called from `main` once
387 /// both are loaded, and so under `--check-config`.
388 pub fn ensure_migration_checks_have_backups(
389 &self,
390 checks: &[crate::config::MigrationCheck],
391 ) -> Result<()> {
392 // A product no tier dry-runs migrations for owes no dumps. `checks` is
393 // never empty — the config defaults it to MNW's `server` entry — so
394 // without this, an intake-only product with no postgres anywhere is
395 // asked to declare a prod dump for a gate it does not configure.
396 if !self
397 .tiers
398 .iter()
399 .flat_map(|t| &t.gates)
400 .any(|g| g.kind() == GateKind::MigrationDryRun)
401 {
402 return Ok(());
403 }
404 for c in checks {
405 anyhow::ensure!(
406 self.backup_named(&c.backup).is_some(),
407 "migration_check {} restores backup {:?}, which no [[backup]] in {} declares \
408 (have: {})",
409 c.dir.display(),
410 c.backup,
411 "the topology",
412 self.backup
413 .iter()
414 .map(|b| b.name.as_str())
415 .collect::<Vec<_>>()
416 .join(", "),
417 );
418 }
419 Ok(())
420 }
421
422 /// Every `[[test_target]]` with an `aux_repo` must name a repo this topology
423 /// checks out. Third of the cross-file checks, and the one whose absence has
424 /// already cost coverage once: an unresolvable target is a warn-and-skip, by
425 /// design, so that a config describing the tip can still build an older sha.
426 /// That makes a typo here indistinguishable from a legitimate bisect skip —
427 /// a green gate that ran one crate fewer than it says it does.
428 pub fn ensure_test_target_aux_repos_exist(
429 &self,
430 targets: &[crate::config::TestTarget],
431 ) -> Result<()> {
432 for t in targets {
433 let Some(name) = t.aux_repo.as_deref() else {
434 continue;
435 };
436 anyhow::ensure!(
437 self.aux_repos.iter().any(|a| a.name == name),
438 "test_target {} names aux_repo {:?}, which no [[aux_repo]] in the topology \
439 checks out, so the gate would skip it as absent (have: {})",
440 t.label(),
441 name,
442 if self.aux_repos.is_empty() {
443 "none".to_string()
444 } else {
445 self.aux_repos
446 .iter()
447 .map(|a| a.name.as_str())
448 .collect::<Vec<_>>()
449 .join(", ")
450 },
451 );
452 }
453 Ok(())
454 }
455
456 /// Every `[[tier.node.companion]]` must name a companion the daemon config
457 /// actually builds. Same two-file split as the migration checks above, and
458 /// the same class of typo, but a worse landing: companions are installed
459 /// AFTER the symlink swap (`deploy::deploy_remote`), so a name that stages
460 /// nothing fails a promote with the server already live on the new version
461 /// — `FailureStage::AtOrAfterSwap`, the case that needs a human to go look.
462 /// Catching it at load turns that into a startup error on the build host.
463 pub fn ensure_node_companions_are_built(
464 &self,
465 built: &[crate::config::Companion],
466 ) -> Result<()> {
467 for t in &self.tiers {
468 for n in &t.nodes {
469 for c in &n.companions {
470 anyhow::ensure!(
471 built.iter().any(|b| b.name == c.name),
472 "tier {} node {} installs companion {:?}, which no [[companion]] in the \
473 daemon config builds, so nothing would be staged under \
474 companions/{} (have: {})",
475 t.name,
476 n.name,
477 c.name,
478 c.name,
479 if built.is_empty() {
480 "none".to_string()
481 } else {
482 built
483 .iter()
484 .map(|b| b.name.as_str())
485 .collect::<Vec<_>>()
486 .join(", ")
487 },
488 );
489 }
490 }
491 }
492 Ok(())
493 }
494
495 /// The configured dump for `name`, or `None` when nothing declares it.
496 pub fn backup_named(&self, name: &str) -> Option<&BackupConfig> {
497 self.backup.iter().find(|b| b.name == name)
498 }
499
500 fn validate(&self) -> Result<()> {
501 // A dump is only owed by a product that actually dry-runs migrations.
502 // The unconditional form of this asserted something about every
503 // product's tiers from a fact about one: pom configures no
504 // `migration_dry_run` anywhere (it has no postgres schema at all), so
505 // requiring it to declare a prod dump would be demanding a fixture for
506 // a gate it never runs.
507 let dry_runs_migrations = self
508 .tiers
509 .iter()
510 .flat_map(|t| &t.gates)
511 .any(|g| g.kind() == GateKind::MigrationDryRun);
512 anyhow::ensure!(
513 !dry_runs_migrations || !self.backup.is_empty(),
514 "a tier configures migration_dry_run but the topology declares no [backup]; \
515 the gate would have nothing to restore"
516 );
517 // page_smoke has one input and it is not optional. Caught here rather
518 // than at promote time, because a gate that cannot run is a gate that
519 // would be discovered red halfway through a deploy, which is the worst
520 // moment to learn a URL is missing.
521 for t in &self.tiers {
522 let smokes = t.gates.iter().any(|g| g.kind() == GateKind::PageSmoke);
523 anyhow::ensure!(
524 !smokes || t.public_url.is_some(),
525 "tier {} configures page_smoke but declares no public_url; the gate has to \
526 request the site the way a visitor does, and a URL derived from a node would \
527 reach the origin and miss the CDN it exists to watch",
528 t.name,
529 );
530 }
531 for (i, b) in self.backup.iter().enumerate() {
532 anyhow::ensure!(
533 !b.name.is_empty()
534 && b.name
535 .bytes()
536 .all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-'),
537 "backup name {:?} must be non-empty and match [A-Za-z0-9_-]+; it keys the \
538 `backups` table and a daemon-config migration_check",
539 b.name,
540 );
541 anyhow::ensure!(
542 !b.source.is_empty() && !b.local_path.is_empty(),
543 "backup {} has an empty source/local_path",
544 b.name,
545 );
546 // Two dumps sharing a name would interleave in `backups`, so the
547 // freshness check and the plausibility floor would each read the
548 // other's row. Two sharing a `local_path` would overwrite each
549 // other on disk, and whichever fetched last would be restored for
550 // both — green, and proving nothing about one of the databases.
551 for prior in &self.backup[..i] {
552 anyhow::ensure!(
553 prior.name != b.name,
554 "two backup entries share the name {:?}",
555 b.name,
556 );
557 anyhow::ensure!(
558 prior.local_path != b.local_path,
559 "backups {:?} and {:?} share local_path {:?}; they would overwrite each other",
560 prior.name,
561 b.name,
562 b.local_path,
563 );
564 }
565 }
566 anyhow::ensure!(
567 !self.tiers.is_empty(),
568 "topology must declare at least one tier"
569 );
570 for t in &self.tiers {
571 // The `host` tier is the build tier (cargo_test / migration_dry_run /
572 // code_smoke / boot_smoke run once on the host); every other tier serves an
573 // artifact to nodes, so it is exempted from the node and
574 // promotion-gate checks the same way.
575 let is_build_tier = t.name.as_str() == "host";
576 if t.provisioned && t.nodes.is_empty() && !is_build_tier {
577 anyhow::bail!("tier {} is provisioned but has no nodes", t.name);
578 }
579 // Fail closed by default: a provisioned serving tier must declare at
580 // least one gate that actually guards a promote. Without this, an
581 // empty (or build-time-only) gate list waves every promote through
582 // because `unsatisfied_gates` finds nothing to check (CF1 root cause).
583 if t.provisioned && !is_build_tier && !t.gates.iter().any(Gate::guards_promotion) {
584 anyhow::bail!(
585 "tier {} is provisioned to serve but declares no promotion gate \
586 (need at least one of node_health / burn_in / manual_confirm)",
587 t.name
588 );
589 }
590 }
591 let mut seen_dirs: Vec<Vec<&str>> = Vec::new();
592 for aux in &self.aux_repos {
593 anyhow::ensure!(
594 !aux.name.is_empty() && !aux.bare_path.is_empty() && !aux.branch.is_empty(),
595 "aux_repo entry has an empty name/bare_path/branch"
596 );
597 // `checkout_dir` becomes a `workdir.join(..)`. Nesting is allowed —
598 // docengine lives at `Libraries/docengine` in the dev tree and the
599 // path dep resolves to that shape — but every component must be a
600 // plain name so an aux repo can never write outside the workdir.
601 let dir = &aux.checkout_dir;
602 let parts: Vec<&str> = dir.split('/').collect();
603 anyhow::ensure!(
604 !dir.is_empty()
605 && !dir.contains('\\')
606 && parts
607 .iter()
608 .all(|c| !c.is_empty() && *c != "." && *c != ".."),
609 "aux_repo {} has an unsafe checkout_dir {dir:?} (must be a relative path of \
610 plain components: no leading slash, empty segments, or dot-dot)",
611 aux.name,
612 );
613 // Two checkouts may not share a dir, and neither may sit inside the
614 // other: `git worktree add` into a path under a live worktree buries
615 // one checkout in the other's tree, and whichever builds second wins.
616 for prior in &seen_dirs {
617 let common = prior.len().min(parts.len());
618 anyhow::ensure!(
619 prior[..common] != parts[..common],
620 "two aux_repo entries share or nest checkout_dir {dir:?}; \
621 they would clobber each other"
622 );
623 }
624 seen_dirs.push(parts);
625 }
626 Ok(())
627 }
628 }
629
630 #[cfg(test)]
631 mod tests {
632 /// The shipped `sando.toml` must parse with the base-image declarations in
633 /// it, and the values must be the ones measured on the boxes. A declaration
634 /// that silently fails to deserialize is worse than none: the node would be
635 /// treated as undeclared, skipped, and the deploy log would say so in a line
636 /// nobody reads.
637 #[test]
638 fn the_shipped_topology_declares_what_each_node_is() {
639 let raw = std::fs::read_to_string(
640 std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml"),
641 )
642 .expect("the repo's sando.toml must be readable from the daemon crate");
643 let topo: Topology = toml::from_str(&raw).expect("sando.toml must parse");
644
645 let nodes: Vec<&Node> = topo.tiers.iter().flat_map(|t| t.nodes.iter()).collect();
646 let by = |name: &str| {
647 *nodes
648 .iter()
649 .find(|n| n.name.as_str() == name)
650 .unwrap_or_else(|| panic!("`{name}` must be in the topology"))
651 };
652 let image = |n: &Node| n.base_image.as_ref().map(ToString::to_string);
653
654 // Measured 2026-08-25. Staging is deliberately NOT the same base as
655 // production, and the assertion below says so out loud: tier A does not
656 // rehearse tier B on the axis these fields exist for.
657 let testnot = by("testnot-1");
658 assert_eq!(image(testnot).as_deref(), Some("ubuntu/26.04"));
659 assert_eq!(testnot.libc.as_deref(), Some("2.43"));
660
661 let prod = by("prod-1");
662 assert_eq!(image(prod).as_deref(), Some("ubuntu/24.04"));
663 assert_eq!(prod.libc.as_deref(), Some("2.39"));
664
665 assert_ne!(
666 testnot.base_image, prod.base_image,
667 "if these ever match, delete this assertion and the comment in \
668 sando.toml that explains why they do not"
669 );
670 }
671
672 use super::*;
673
674 /// A minimal topology with one serving tier whose gate block is `gates`.
675 fn topo_with_serving_gates(provisioned: bool, gates: &str) -> Topology {
676 let raw = format!(
677 r#"
678 [repo]
679 bare_path = "/tmp/repo.git"
680 branch = "main"
681
682 [backup]
683 source = "ssh://prod/dump.sql.gz"
684 local_path = "/tmp/dump.sql.gz"
685
686 [[tier]]
687 name = "b"
688 provisioned = {provisioned}
689 gates = [{gates}]
690 [[tier.node]]
691 name = "prod-1"
692 ssh_target = "prod-1"
693 release_root = "/srv/mnw"
694 "#
695 );
696 toml::from_str(&raw).expect("parse test topology")
697 }
698
699 #[test]
700 fn page_smoke_without_a_public_url_is_rejected_at_load() {
701 // The gate's whole value is that it requests the site the way a visitor
702 // does. Without a URL it cannot, and a gate that cannot run must not be
703 // discovered halfway through a promote.
704 let topo = topo_with_serving_gates(true, r#"{ kind = "page_smoke" }"#);
705 let err = topo.validate_for_test().expect_err("must refuse");
706 assert!(
707 err.to_string().contains("public_url"),
708 "error should name the missing field: {err}"
709 );
710 }
711
712 #[test]
713 fn page_smoke_with_a_public_url_loads() {
714 let raw = r#"
715 [repo]
716 bare_path = "/tmp/repo.git"
717 branch = "main"
718
719 [backup]
720 source = "ssh://prod/dump.sql.gz"
721 local_path = "/tmp/dump.sql.gz"
722
723 [[tier]]
724 name = "a"
725 provisioned = true
726 public_url = "https://testnot.work"
727 gates = [{ kind = "page_smoke" }]
728 [[tier.node]]
729 name = "testnot-1"
730 ssh_target = "testnot-1"
731 release_root = "/srv/mnw"
732 "#;
733 let topo: Topology = toml::from_str(raw).expect("parse");
734 topo.validate_for_test().expect("valid");
735 assert_eq!(
736 topo.tiers[0].public_url.as_deref(),
737 Some("https://testnot.work")
738 );
739 // It guards promotion out of its tier, and it runs after the deploy
740 // rather than on the build host -- both are the point of it.
741 assert!(topo.tiers[0].gates[0].guards_promotion());
742 assert!(topo.tiers[0].gates[0].runs_post_deploy());
743 }
744
745 #[test]
746 fn provisioned_serving_tier_with_no_gates_is_rejected() {
747 let topo = topo_with_serving_gates(true, "");
748 let err = topo.validate_for_test().unwrap_err().to_string();
749 assert!(err.contains("no promotion gate"), "{err}");
750 }
751
752 #[test]
753 fn provisioned_serving_tier_with_only_build_gates_is_rejected() {
754 // cargo_test / migration_dry_run are build-time and prove nothing about a
755 // promote, so a serving tier carrying only them still fails closed.
756 let topo = topo_with_serving_gates(
757 true,
758 r#"{ kind = "cargo_test" }, { kind = "migration_dry_run" }"#,
759 );
760 let err = topo.validate_for_test().unwrap_err().to_string();
761 assert!(err.contains("no promotion gate"), "{err}");
762 }
763
764 #[test]
765 fn provisioned_serving_tier_with_a_promotion_gate_is_accepted() {
766 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
767 assert!(topo.validate_for_test().is_ok());
768 }
769
770 #[test]
771 fn provisioned_serving_tier_with_only_boot_smoke_is_rejected() {
772 // boot_smoke is a build-host gate now; it proves nothing about a node, so
773 // a serving tier carrying only boot_smoke must fail closed exactly like an
774 // empty gate list (Run-2 SERIOUS-3 structural close).
775 let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
776 let err = topo.validate_for_test().unwrap_err().to_string();
777 assert!(err.contains("no promotion gate"), "{err}");
778 }
779
780 #[test]
781 fn unprovisioned_tier_with_empty_gates_is_skipped() {
782 // A declared-but-not-yet-provisioned tier (e.g. tier c) carries no
783 // promote authority, so the gate requirement does not apply yet.
784 let topo = topo_with_serving_gates(false, "");
785 assert!(topo.validate_for_test().is_ok());
786 }
787
788 #[test]
789 fn build_host_matching_a_serving_node_is_rejected() {
790 // prod-1 is a node in the provisioned serving tier built above; naming it
791 // as the builder must fail closed.
792 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
793 let err = topo
794 .ensure_build_host_not_serving("prod-1")
795 .unwrap_err()
796 .to_string();
797 assert!(err.contains("must not be a prod/serving node"), "{err}");
798 }
799
800 #[test]
801 fn build_host_distinct_from_serving_nodes_is_accepted() {
802 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
803 assert!(topo.ensure_build_host_not_serving("fw13").is_ok());
804 }
805
806 #[test]
807 fn node_companions_default_empty_and_parse_when_present() {
808 // A node without [[tier.node.companion]] is server-only.
809 let plain = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
810 assert!(plain.tiers[0].nodes[0].companions.is_empty());
811
812 // A node that declares a companion carries its install target + unit.
813 let raw = r#"
814 [repo]
815 bare_path = "/tmp/repo.git"
816 branch = "main"
817 [backup]
818 source = "s"
819 local_path = "/tmp/d"
820 [[tier]]
821 name = "b"
822 provisioned = true
823 gates = [{ kind = "node_health" }]
824 [[tier.node]]
825 name = "prod-1"
826 ssh_target = "makenotwork@alpha-west-1"
827 release_root = "/opt/mnw"
828 [[tier.node.companion]]
829 name = "mnw-cli"
830 install_path = "/opt/mnw-cli/mnw-cli"
831 service_name = "mnw-cli.service"
832 "#;
833 let topo: Topology = toml::from_str(raw).expect("parse companion topology");
834 let c = &topo.tiers[0].nodes[0].companions;
835 assert_eq!(c.len(), 1);
836 assert_eq!(c[0].name, "mnw-cli");
837 assert_eq!(c[0].install_path, "/opt/mnw-cli/mnw-cli");
838 assert_eq!(c[0].service_name, "mnw-cli.service");
839 }
840
841 /// A topology whose one node installs the named companions.
842 fn topo_installing(names: &[&str]) -> Topology {
843 let mut blocks = String::new();
844 for n in names {
845 use std::fmt::Write;
846 let _ = write!(
847 blocks,
848 "[[tier.node.companion]]\nname = \"{n}\"\n\
849 install_path = \"/opt/{n}/{n}\"\nservice_name = \"{n}.service\"\n"
850 );
851 }
852 let raw = format!(
853 r#"
854 [repo]
855 bare_path = "/tmp/repo.git"
856 branch = "main"
857 [backup]
858 source = "s"
859 local_path = "/tmp/d"
860 [[tier]]
861 name = "b"
862 provisioned = true
863 gates = [{{ kind = "node_health" }}]
864 [[tier.node]]
865 name = "prod-1"
866 ssh_target = "makenotwork@alpha-west-1"
867 release_root = "/opt/mnw"
868 {blocks}"#
869 );
870 toml::from_str(&raw).expect("parse topology")
871 }
872
873 fn built(names: &[&str]) -> Vec<crate::config::Companion> {
874 names
875 .iter()
876 .map(|n| crate::config::Companion {
877 name: (*n).to_string(),
878 manifest_dir: (*n).into(),
879 bin: (*n).to_string(),
880 })
881 .collect()
882 }
883
884 fn test_target(dir: &str, aux_repo: Option<&str>) -> crate::config::TestTarget {
885 crate::config::TestTarget {
886 dir: dir.into(),
887 aux_repo: aux_repo.map(str::to_string),
888 features: Vec::new(),
889 all_features: false,
890 scratch_db: false,
891 }
892 }
893
894 #[test]
895 fn a_test_target_naming_a_checked_out_aux_repo_is_accepted() {
896 let topo = topo_with_aux(
897 "[[aux_repo]]\nname = \"docengine\"\nbare_path = \"/tmp/d.git\"\n\
898 upstream = \"git@h:max/d.git\"\nbranch = \"main\"\ncheckout_dir = \"Libraries/docengine\"\n",
899 )
900 .expect("parse");
901 assert!(
902 topo.ensure_test_target_aux_repos_exist(&[
903 test_target("server", None),
904 test_target("", Some("docengine")),
905 ])
906 .is_ok()
907 );
908 }
909
910 #[test]
911 fn a_test_target_naming_an_unknown_aux_repo_is_rejected_at_load() {
912 // The failure this exists to prevent is silent: an unresolvable target
913 // is a warn-and-skip (bisect), so the gate stays green having run one
914 // crate fewer than the config claims.
915 let topo = topo_with_aux(
916 "[[aux_repo]]\nname = \"synckit\"\nbare_path = \"/tmp/s.git\"\n\
917 upstream = \"git@h:max/s.git\"\nbranch = \"main\"\ncheckout_dir = \"synckit\"\n",
918 )
919 .expect("parse");
920 let err = topo
921 .ensure_test_target_aux_repos_exist(&[test_target("", Some("docengine"))])
922 .unwrap_err()
923 .to_string();
924 assert!(err.contains("docengine"), "{err}");
925 assert!(err.contains("have: synckit"), "{err}");
926 }
927
928 #[test]
929 fn test_targets_without_an_aux_repo_need_no_aux_repos_declared() {
930 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
931 assert!(
932 topo.ensure_test_target_aux_repos_exist(&[test_target("server", None)])
933 .is_ok()
934 );
935 }
936
937 #[test]
938 fn a_node_companion_the_daemon_builds_is_accepted() {
939 let topo = topo_installing(&["mnw-cli", "multithreaded"]);
940 assert!(
941 topo.ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
942 .is_ok()
943 );
944 }
945
946 #[test]
947 fn a_node_companion_nothing_builds_is_rejected_at_load() {
948 // Left uncaught this fails during the post-swap install on prod, with
949 // the server already live on the new version.
950 let topo = topo_installing(&["mnw-cli", "multithreadd"]);
951 let err = topo
952 .ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
953 .unwrap_err()
954 .to_string();
955 assert!(err.contains("multithreadd"), "{err}");
956 assert!(err.contains("no [[companion]]"), "{err}");
957 // The message names what IS available, so the typo is obvious.
958 assert!(err.contains("mnw-cli, multithreaded"), "{err}");
959 }
960
961 #[test]
962 fn a_node_companion_with_no_companions_configured_at_all_is_rejected() {
963 let topo = topo_installing(&["multithreaded"]);
964 let err = topo
965 .ensure_node_companions_are_built(&[])
966 .unwrap_err()
967 .to_string();
968 assert!(err.contains("have: none"), "{err}");
969 }
970
971 #[test]
972 fn a_topology_installing_no_companions_is_fine_with_none_built() {
973 let topo = topo_installing(&[]);
974 assert!(topo.ensure_node_companions_are_built(&[]).is_ok());
975 }
976
977 fn topo_with_aux(aux_block: &str) -> Result<Topology> {
978 let raw = format!(
979 r#"
980 [repo]
981 bare_path = "/tmp/repo.git"
982 branch = "main"
983 [backup]
984 source = "s"
985 local_path = "/tmp/d"
986 [[tier]]
987 name = "b"
988 provisioned = true
989 gates = [{{ kind = "node_health" }}]
990 [[tier.node]]
991 name = "prod-1"
992 ssh_target = "prod-1"
993 release_root = "/srv/mnw"
994 {aux_block}
995 "#
996 );
997 let topo: Topology = toml::from_str(&raw)?;
998 topo.validate_for_test()?;
999 Ok(topo)
1000 }
1001
1002 #[test]
1003 fn aux_repos_default_empty() {
1004 let topo = topo_with_aux("").expect("no aux_repo block is fine");
1005 assert!(topo.aux_repos.is_empty());
1006 }
1007
1008 #[test]
1009 fn aux_repo_parses_all_fields() {
1010 let topo = topo_with_aux(
1011 r#"
1012 [[aux_repo]]
1013 name = "synckit"
1014 bare_path = "/srv/sando/synckit.git"
1015 upstream = "git@ssh.makenot.work:max/synckit.git"
1016 branch = "main"
1017 checkout_dir = "synckit""#,
1018 )
1019 .expect("valid aux_repo parses");
1020 assert_eq!(topo.aux_repos.len(), 1);
1021 let a = &topo.aux_repos[0];
1022 assert_eq!(a.name, "synckit");
1023 assert_eq!(a.bare_path, "/srv/sando/synckit.git");
1024 assert_eq!(a.upstream, "git@ssh.makenot.work:max/synckit.git");
1025 assert_eq!(a.branch, "main");
1026 assert_eq!(a.checkout_dir, "synckit");
1027 }
1028
1029 #[test]
1030 fn aux_repo_with_nested_checkout_dir_is_accepted() {
1031 let topo = topo_with_aux(
1032 r#"
1033 [[aux_repo]]
1034 name = "docengine"
1035 bare_path = "/srv/sando/docengine.git"
1036 upstream = "git@ssh.makenot.work:max/docengine.git"
1037 branch = "main"
1038 checkout_dir = "Libraries/docengine""#,
1039 )
1040 .expect("a nested checkout_dir is a valid location");
1041 assert_eq!(topo.aux_repos[0].checkout_dir, "Libraries/docengine");
1042 }
1043
1044 #[test]
1045 fn aux_repo_with_traversing_checkout_dir_is_rejected() {
1046 for bad in [
1047 "../escape",
1048 "a/../../escape",
1049 "a/./b",
1050 "a//b",
1051 "/abs",
1052 "a/",
1053 "..",
1054 ".",
1055 ] {
1056 let err = topo_with_aux(&format!(
1057 r#"
1058 [[aux_repo]]
1059 name = "x"
1060 bare_path = "/srv/sando/x.git"
1061 upstream = "u"
1062 branch = "main"
1063 checkout_dir = "{bad}""#,
1064 ))
1065 .unwrap_err()
1066 .to_string();
1067 assert!(err.contains("unsafe checkout_dir"), "for {bad:?}: {err}");
1068 }
1069 }
1070
1071 #[test]
1072 fn aux_repos_sharing_a_checkout_dir_are_rejected() {
1073 let err = topo_with_aux(
1074 r#"
1075 [[aux_repo]]
1076 name = "one"
1077 bare_path = "/srv/sando/one.git"
1078 upstream = "u"
1079 branch = "main"
1080 checkout_dir = "shared"
1081 [[aux_repo]]
1082 name = "two"
1083 bare_path = "/srv/sando/two.git"
1084 upstream = "u"
1085 branch = "main"
1086 checkout_dir = "shared""#,
1087 )
1088 .unwrap_err()
1089 .to_string();
1090 assert!(err.contains("share or nest checkout_dir"), "{err}");
1091 }
1092
1093 #[test]
1094 fn aux_repo_nested_inside_another_checkout_dir_is_rejected() {
1095 let err = topo_with_aux(
1096 r#"
1097 [[aux_repo]]
1098 name = "outer"
1099 bare_path = "/srv/sando/outer.git"
1100 upstream = "u"
1101 branch = "main"
1102 checkout_dir = "Libraries"
1103 [[aux_repo]]
1104 name = "inner"
1105 bare_path = "/srv/sando/inner.git"
1106 upstream = "u"
1107 branch = "main"
1108 checkout_dir = "Libraries/docengine""#,
1109 )
1110 .unwrap_err()
1111 .to_string();
1112 assert!(err.contains("share or nest checkout_dir"), "{err}");
1113 }
1114
1115 /// A topology whose `[backup]`/`[[backup]]` section is `backup_block`.
1116 fn topo_with_backup_block(backup_block: &str) -> Result<Topology> {
1117 let raw = format!(
1118 r#"
1119 [repo]
1120 bare_path = "/tmp/repo.git"
1121 branch = "main"
1122 {backup_block}
1123 [[tier]]
1124 name = "b"
1125 provisioned = true
1126 # migration_dry_run is what makes the backup rules apply at all: a product no
1127 # tier dry-runs migrations for owes no dumps, so a fixture exercising those rules
1128 # has to configure the gate.
1129 gates = [{{ kind = "node_health" }}, {{ kind = "migration_dry_run" }}]
1130 [[tier.node]]
1131 name = "prod-1"
1132 ssh_target = "prod-1"
1133 release_root = "/srv/mnw"
1134 "#
1135 );
1136 let topo: Topology = toml::from_str(&raw)?;
1137 topo.validate_for_test()?;
1138 Ok(topo)
1139 }
1140
1141 /// A topology with no `[backup]` at all, so the dump rules are exercised
1142 /// by what its gates ask for rather than by what it declares.
1143 fn topo_without_backup(gates: &str) -> Result<Topology> {
1144 let raw = format!(
1145 r#"
1146 backup = []
1147
1148 [repo]
1149 bare_path = "/tmp/repo.git"
1150 branch = "main"
1151
1152 [[tier]]
1153 name = "b"
1154 provisioned = true
1155 gates = [{gates}]
1156 [[tier.node]]
1157 name = "prod-1"
1158 ssh_target = "prod-1"
1159 release_root = "/srv/mnw"
1160 "#
1161 );
1162 let topo: Topology = toml::from_str(&raw)?;
1163 topo.validate_for_test()?;
1164 Ok(topo)
1165 }
1166
1167 #[test]
1168 fn a_product_that_never_dry_runs_migrations_owes_no_dump() {
1169 // pom is this product: no postgres schema, so no tier configures
1170 // migration_dry_run and demanding a prod dump would be demanding a
1171 // fixture for a gate that never runs.
1172 let topo = topo_without_backup(r#"{ kind = "node_health" }"#)
1173 .expect("a topology with no migration gate loads without a [backup]");
1174 assert!(topo.backup.is_empty());
1175 }
1176
1177 #[test]
1178 fn a_migration_dry_run_with_no_backup_is_rejected_at_load() {
1179 // The gate restores a dump into the scratch database. With nothing
1180 // declared it would have nothing to restore, and the discovery would
1181 // come mid-promote.
1182 let err =
1183 topo_without_backup(r#"{ kind = "node_health" }, { kind = "migration_dry_run" }"#)
1184 .expect_err("a dry-run gate with no dump declared must not load")
1185 .to_string();
1186 assert!(err.contains("declares no [backup]"), "{err}");
1187 }
1188
1189 #[test]
1190 fn a_single_backup_table_still_parses_as_one_named_server() {
1191 // Back-compat is the point: every deployed sando.toml uses the single
1192 // `[backup]` form, and the box this config lives on is the one whose job
1193 // is deploying — it must not need an edit to start.
1194 let topo = topo_with_backup_block(
1195 r#"
1196 [backup]
1197 source = "ssh://prod/dump.sql.gz"
1198 local_path = "/tmp/dump.sql.gz""#,
1199 )
1200 .expect("the single-table form must still load");
1201 assert_eq!(topo.backup.len(), 1);
1202 assert_eq!(topo.backup[0].name, "server");
1203 assert!(topo.backup_named("server").is_some());
1204 }
1205
1206 #[test]
1207 fn a_backup_list_parses_and_keeps_its_names() {
1208 let topo = topo_with_backup_block(
1209 r#"
1210 [[backup]]
1211 name = "server"
1212 source = "ssh://prod/makenotwork/latest.sql.gz"
1213 local_path = "/tmp/server.sql.gz"
1214 [[backup]]
1215 name = "multithreaded"
1216 source = "ssh://prod/multithreaded/latest.sql.gz"
1217 local_path = "/tmp/mt.sql.gz""#,
1218 )
1219 .expect("the list form must load");
1220 assert_eq!(topo.backup.len(), 2);
1221 assert_eq!(
1222 topo.backup_named("multithreaded").unwrap().local_path,
1223 "/tmp/mt.sql.gz"
1224 );
1225 assert!(topo.backup_named("nope").is_none());
1226 }
1227
1228 #[test]
1229 fn two_backups_sharing_a_name_are_rejected() {
1230 // They would interleave in `backups`, so the freshness check and the
1231 // plausibility floor would each read the other's row.
1232 let err = topo_with_backup_block(
1233 r#"
1234 [[backup]]
1235 name = "server"
1236 source = "a"
1237 local_path = "/tmp/a.sql.gz"
1238 [[backup]]
1239 name = "server"
1240 source = "b"
1241 local_path = "/tmp/b.sql.gz""#,
1242 )
1243 .unwrap_err()
1244 .to_string();
1245 assert!(err.contains("share the name"), "{err}");
1246 }
1247
1248 #[test]
1249 fn two_backups_sharing_a_local_path_are_rejected() {
1250 // Whichever fetched last would be restored for both checks — green, and
1251 // proving nothing about one of the two databases.
1252 let err = topo_with_backup_block(
1253 r#"
1254 [[backup]]
1255 name = "server"
1256 source = "a"
1257 local_path = "/tmp/same.sql.gz"
1258 [[backup]]
1259 name = "multithreaded"
1260 source = "b"
1261 local_path = "/tmp/same.sql.gz""#,
1262 )
1263 .unwrap_err()
1264 .to_string();
1265 assert!(err.contains("share local_path"), "{err}");
1266 }
1267
1268 #[test]
1269 fn a_migration_check_naming_an_undeclared_backup_is_rejected_at_startup() {
1270 // Daemon config and topology are separate files, so nothing but this
1271 // cross-check catches the typo. Uncaught it surfaces as a permanently
1272 // Blocked gate on the next promote, which reads like a missed fetch.
1273 let topo = topo_with_backup_block(
1274 r#"
1275 [backup]
1276 source = "s"
1277 local_path = "/tmp/d""#,
1278 )
1279 .unwrap();
1280 let checks = vec![crate::config::MigrationCheck {
1281 dir: std::path::PathBuf::from("multithreaded/migrations"),
1282 backup: "multithreaded".into(),
1283 scratch_db: Some("sando_scratch_mt".into()),
1284 owner_role: Some("multithreaded".into()),
1285 }];
1286 let err = topo
1287 .ensure_migration_checks_have_backups(&checks)
1288 .unwrap_err()
1289 .to_string();
1290 assert!(err.contains("which no [[backup]]"), "{err}");
1291
1292 // And the shipped pair agree, which is the case that actually ships.
1293 let shipped_checks = crate::config::default_migration_checks_for_test();
1294 shipped()
1295 .ensure_migration_checks_have_backups(&shipped_checks)
1296 .expect("the default server check resolves against the shipped topology");
1297 }
1298
1299 #[test]
1300 fn real_sando_toml_loads_clean() {
1301 // The shipped topology must satisfy the invariant — guards against a
1302 // regression that would lock sandod out of its own config.
1303 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
1304 Topology::load(&path).expect("shipped sando.toml must validate");
1305 }
1306
1307 fn shipped() -> Topology {
1308 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
1309 Topology::load(&path).expect("shipped sando.toml must validate")
1310 }
1311
1312 #[test]
1313 fn shipping_to_the_last_provisioned_tier_needs_an_operator_signoff() {
1314 // A tier's gates guard promotion *out* of it, which is the subtlety that
1315 // made manual_confirm inert: it sat on tier b, guarding b -> c, and c is
1316 // not provisioned. So the ship to production was cleared by node_health
1317 // + burn_in alone — and `hotfix: true` skips burn_in.
1318 //
1319 // The gate that matters therefore belongs on the PREDECESSOR of the last
1320 // provisioned tier. Asserted structurally so re-provisioning tiers cannot
1321 // silently strand the sign-off again.
1322 let topo = shipped();
1323 let last = topo
1324 .tiers
1325 .iter()
1326 .rposition(|t| t.provisioned)
1327 .expect("some tier must be provisioned");
1328 assert!(last > 0, "the production tier cannot be the first tier");
1329 let guard = &topo.tiers[last - 1];
1330 assert!(
1331 guard.gates.iter().any(|g| matches!(g, Gate::ManualConfirm)),
1332 "tier {} guards promotion into the last provisioned tier ({}), so it must require an operator sign-off; its gates are {:?}",
1333 guard.name,
1334 topo.tiers[last].name,
1335 guard
1336 .gates
1337 .iter()
1338 .map(|g| g.kind().as_str())
1339 .collect::<Vec<_>>(),
1340 );
1341 }
1342
1343 #[test]
1344 fn every_serving_node_has_a_readiness_probe() {
1345 // Without health_url, node_health degrades to `systemctl is-active`,
1346 // which a crash-looping binary satisfies between restarts — exactly what
1347 // the 0.10.14 CDN_BASE_URL crash-loop did on prod-1.
1348 let topo = shipped();
1349 for tier in topo.tiers.iter().filter(|t| t.provisioned) {
1350 for node in &tier.nodes {
1351 assert!(
1352 node.health_url.is_some(),
1353 "node {} on tier {} has no health_url, so node_health proves only that systemd thinks the unit is running",
1354 node.name,
1355 tier.name,
1356 );
1357 }
1358 }
1359 }
1360 }
1361