Skip to main content

max / makenotwork

44.2 KB · 1174 lines History Blame Raw
1 use crate::domain::{GateKind, NodeId, TierId};
2 use anyhow::{Context, Result};
3 use serde::{Deserialize, Serialize};
4 use std::path::Path;
5
6 #[derive(Debug, Clone, Serialize, Deserialize)]
7 pub struct Topology {
8 /// The repo Sando checks out to build this product.
9 ///
10 /// `None` for an intake-only product: pom is built by Bento on two
11 /// machines and handed over as finished bytes, so Sando fetches no source
12 /// for it and there is no bare repo on this host to name. A topology that
13 /// declares no repo cannot be `/rebuild`-ed, which is the same statement
14 /// [`AppConfig::build_host`](crate::config::AppConfig::build_host) makes
15 /// from the other side.
16 #[serde(default)]
17 pub repo: Option<RepoConfig>,
18 /// Prod dumps `/backup/fetch` pulls, one per database `migration_dry_run`
19 /// has a check for. A list because the repo ships more than one service
20 /// with its own database and its own `sqlx::migrate!()` at boot: the server
21 /// migrates `makenotwork`, multithreaded migrates `multithreaded`, and a
22 /// gate that restores only the first proves nothing about the second.
23 ///
24 /// Accepts both the historical single `[backup]` table and a `[[backup]]`
25 /// list, so a deployed `sando.toml` keeps working unedited (the single form
26 /// deserializes to a one-entry list named `server`).
27 #[serde(deserialize_with = "one_or_many_backup")]
28 pub backup: Vec<BackupConfig>,
29 #[serde(rename = "tier")]
30 pub tiers: Vec<Tier>,
31 /// Extra repos to fetch and check out beside the main worktree before a
32 /// build, so a path dependency that reaches across the repo split resolves.
33 /// Empty (default) keeps an existing `sando.toml` working unedited. See
34 /// [`AuxRepo`] and [`crate::build::checkout_aux_repos`].
35 #[serde(default, rename = "aux_repo")]
36 pub aux_repos: Vec<AuxRepo>,
37 }
38
39 /// An auxiliary repo checked out beside the per-sha worktree so cross-repo path
40 /// dependencies resolve at build time.
41 ///
42 /// The concrete case (2026-07-24): `mnw-cli` — a companion built from the MNW
43 /// worktree — carries `synckit-client = { path = "../../synckit/synckit-client" }`
44 /// after synckit moved to its own repo. From the companion crate at
45 /// `<workdir>/<sha>/mnw-cli`, that path resolves to `<workdir>/synckit`, a sibling
46 /// of the per-sha worktree that Sando otherwise never creates, so the companion
47 /// build failed with "No such file or directory". An `aux_repo` named to land at
48 /// `checkout_dir = "synckit"` puts the synckit source exactly there.
49 ///
50 /// The checkout is at the FIXED `<workdir>/<checkout_dir>`, not per-sha: the path
51 /// dependency resolves to that spot regardless of the MNW sha, and builds
52 /// serialize, so a single shared checkout refreshed to `branch` HEAD each build
53 /// is correct. Because a path dep has no lockfile pin, "branch HEAD" is the honest
54 /// resolution — the same contract as the dev working copy.
55 #[derive(Debug, Clone, Serialize, Deserialize)]
56 pub struct AuxRepo {
57 /// Human label for logs and errors (e.g. `synckit`).
58 pub name: String,
59 /// Bare repo Sando fetches into and worktrees from, e.g.
60 /// `/srv/sando/synckit.git`. Auto-created (hookless) on first build.
61 pub bare_path: String,
62 /// Canonical git remote fetched before checkout. Like [`RepoConfig::upstream`]
63 /// but required here: an aux repo is pull-based (nobody pushes to its bare).
64 pub upstream: String,
65 /// Branch whose HEAD is checked out.
66 pub branch: String,
67 /// Where the worktree lands, relative to `cfg.workdir`. May name a nested
68 /// location (`Libraries/docengine`), because a path dep resolves to wherever
69 /// the dev tree keeps the crate and Sando has to match that shape. Every
70 /// component must be a plain name — no leading slash, no `..` — so the
71 /// checkout stays under the workdir.
72 pub checkout_dir: String,
73 }
74
75 #[derive(Debug, Clone, Serialize, Deserialize)]
76 pub struct RepoConfig {
77 pub bare_path: String,
78 pub branch: String,
79 /// Optional canonical git remote. When set, `/rebuild` fetches the deploy
80 /// branch from here into the bare repo before worktree-ing the target sha,
81 /// so a freshly-pushed commit is resolvable without anyone pushing to the
82 /// bare repo directly (pull-based deploys). When unset, Sando relies on the
83 /// bare repo already containing the sha (push-based / hook-driven).
84 #[serde(default)]
85 pub upstream: Option<String>,
86 }
87
88 #[derive(Debug, Clone, Serialize, Deserialize)]
89 pub struct BackupConfig {
90 /// Which database this dump is of, as referenced by a daemon-config
91 /// `[[migration_check]]`'s `backup` key and recorded in the `backups`
92 /// table's `name` column. Defaults to `server` so the historical single
93 /// `[backup]` table needs no edit — and so the pre-existing rows, which the
94 /// state-DB migration backfills to `server`, keep matching it.
95 #[serde(default = "default_backup_name")]
96 pub name: String,
97 pub source: String,
98 pub local_path: String,
99 }
100
101 fn default_backup_name() -> String {
102 "server".into()
103 }
104
105 /// Accept `[backup]` (one table) or `[[backup]]` (a list) for the same key.
106 /// Serde cannot express "table or sequence" on a `Vec` field on its own, and
107 /// the alternative — renaming the key — would break every deployed
108 /// `sando.toml` at startup, on the box whose whole job is deploying.
109 fn one_or_many_backup<'de, D>(de: D) -> std::result::Result<Vec<BackupConfig>, D::Error>
110 where
111 D: serde::Deserializer<'de>,
112 {
113 #[derive(Deserialize)]
114 #[serde(untagged)]
115 enum OneOrMany {
116 One(BackupConfig),
117 Many(Vec<BackupConfig>),
118 }
119 Ok(match OneOrMany::deserialize(de)? {
120 OneOrMany::One(b) => vec![b],
121 OneOrMany::Many(v) => v,
122 })
123 }
124
125 #[derive(Debug, Clone, Serialize, Deserialize)]
126 pub struct Tier {
127 pub name: TierId,
128 #[serde(default)]
129 pub provisioned: bool,
130 pub gates: Vec<Gate>,
131 #[serde(default)]
132 pub canary: CanaryPolicy,
133 #[serde(default, rename = "node")]
134 pub nodes: Vec<Node>,
135 }
136
137 #[derive(Debug, Clone, Serialize, Deserialize)]
138 pub struct Node {
139 pub name: NodeId,
140 pub ssh_target: String,
141 pub release_root: String,
142 /// What this machine runs, as `os/arch` (e.g. `linux/aarch64`).
143 ///
144 /// Compared against the bundle's own platform before anything is pushed;
145 /// see [`crate::deploy::Placement`]. Optional, and a node that declares it
146 /// can only be given a bundle that declares a matching one — silence on
147 /// either side is a refusal, not a pass. MNW's nodes declare nothing and
148 /// keep the single-platform behavior they have always had; pom's declare
149 /// theirs, because pom is the product where one version is two bundles.
150 #[serde(default)]
151 pub platform: Option<crate::domain::Platform>,
152 /// systemd unit name to reload-or-restart after the symlink swap.
153 /// Defaults to "makenotwork.service" because that's MNW's prod unit.
154 #[serde(default = "default_service_name")]
155 pub service_name: String,
156 /// Opt-in config-drift guard. When set to the node's env-file path (e.g.
157 /// `/etc/mnw/makenotwork.env`), the deploy sources that file and runs the
158 /// freshly-rsynced binary in `MNW_CHECK_CONFIG=1` mode BEFORE the symlink
159 /// swap. A required var missing on this node (a var added upstream but never
160 /// added to the node's env — how testnot crash-looped on CDN_BASE_URL) then
161 /// fails the promote with the running service still intact, instead of after
162 /// the swap+restart. Unset (default) skips the check, so an existing
163 /// `sando.toml` keeps working and it never runs against a binary too old to
164 /// support the mode; enable it once a check-capable version is deployed.
165 #[serde(default)]
166 pub config_check_env_file: Option<String>,
167 /// Capability grant for this node's executor (see `ops_exec`). Defaults to
168 /// the current behavior of every Sando node — actuate deploy+restart,
169 /// observe health — so an existing `sando.toml` keeps working unedited.
170 #[serde(default = "default_actuate")]
171 pub actuate: Vec<String>,
172 #[serde(default = "default_observe")]
173 pub observe: Vec<String>,
174 /// Optional HTTP readiness URL the `node_health` gate curls on the node (over
175 /// its executor) after `systemctl is-active`. Typically a loopback address
176 /// the service binds, e.g. `http://127.0.0.1:8080/health`. When unset, the
177 /// gate proves the unit is active post-restart but does not HTTP-probe; set
178 /// it for a full readiness check. Kept optional so an existing `sando.toml`
179 /// needs no edit.
180 #[serde(default)]
181 pub health_url: Option<String>,
182 /// Companion services this node installs from the release bundle after the
183 /// server is swapped and restarted. Each entry names a `[[companion]]` the
184 /// daemon builds (see `Config::companions`) and says where its binary lands
185 /// and which unit to restart — so a contract-coupled service (mnw-cli) ships
186 /// in the same promote as the server instead of drifting. Empty (default) =
187 /// server-only node, so an existing `sando.toml` keeps working unedited.
188 #[serde(default, rename = "companion")]
189 pub companions: Vec<NodeCompanion>,
190 }
191
192 /// Where a node installs a built companion binary and which unit to bounce.
193 #[derive(Debug, Clone, Serialize, Deserialize)]
194 pub struct NodeCompanion {
195 /// Must match a `Config::companions[].name` — the bundle subdir to read from.
196 pub name: String,
197 /// Absolute path the companion binary is installed to on the node
198 /// (the unit's `ExecStart`), e.g. `/opt/mnw-cli/mnw-cli`.
199 pub install_path: String,
200 /// systemd unit restarted after the binary is installed, e.g.
201 /// `mnw-cli.service`.
202 pub service_name: String,
203 }
204
205 fn default_service_name() -> String {
206 "makenotwork.service".into()
207 }
208
209 /// The capability set every pre-existing Sando node implicitly had: it deploys
210 /// and restarts, and is health-observed. Keeping these as the defaults is what
211 /// lets `sando.toml` stay unchanged through the executor refactor.
212 pub fn default_actuate() -> Vec<String> {
213 vec!["deploy".into(), "restart".into()]
214 }
215 pub fn default_observe() -> Vec<String> {
216 vec!["health".into()]
217 }
218
219 #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
220 #[serde(rename_all = "snake_case")]
221 pub enum CanaryPolicy {
222 #[default]
223 Sequential,
224 Parallel,
225 }
226
227 impl CanaryPolicy {
228 pub fn as_str(self) -> &'static str {
229 match self {
230 CanaryPolicy::Sequential => "sequential",
231 CanaryPolicy::Parallel => "parallel",
232 }
233 }
234 }
235
236 #[derive(Debug, Clone, Serialize, Deserialize)]
237 #[serde(tag = "kind", rename_all = "snake_case")]
238 pub enum Gate {
239 CargoTest,
240 HardeningTest,
241 Clippy,
242 Fmt,
243 CargoAudit,
244 CargoDeny,
245 MigrationDryRun,
246 CodeSmoke,
247 BootSmoke,
248 NodeHealth,
249 BurnIn { hours: u32 },
250 ManualConfirm,
251 }
252
253 impl Gate {
254 /// The discriminant — the identifier we use in events, schema columns,
255 /// and the TUI. Gate parameters (e.g. `BurnIn.hours`) stay with `Gate`
256 /// and are not carried into `gate_runs` history.
257 pub fn kind(&self) -> GateKind {
258 match self {
259 Gate::CargoTest => GateKind::CargoTest,
260 Gate::HardeningTest => GateKind::HardeningTest,
261 Gate::Clippy => GateKind::Clippy,
262 Gate::Fmt => GateKind::Fmt,
263 Gate::CargoAudit => GateKind::CargoAudit,
264 Gate::CargoDeny => GateKind::CargoDeny,
265 Gate::MigrationDryRun => GateKind::MigrationDryRun,
266 Gate::CodeSmoke => GateKind::CodeSmoke,
267 Gate::BootSmoke => GateKind::BootSmoke,
268 Gate::NodeHealth => GateKind::NodeHealth,
269 Gate::BurnIn { .. } => GateKind::BurnIn,
270 Gate::ManualConfirm => GateKind::ManualConfirm,
271 }
272 }
273
274 /// Gates that execute against a tier's freshly-deployed nodes, recording a
275 /// `gate_runs` row, at the end of a successful promote to that tier — the
276 /// evidence the *next* promote checks. Only `node_health`: it probes the
277 /// nodes the deploy just shipped to. `boot_smoke` is NOT post-deploy — it is
278 /// a build-time gate that boots the staged artifact on the build host and
279 /// proves nothing about a deployed node (the Run-2 SERIOUS-3 blind spot:
280 /// boot_smoke used to re-run locally at promote time and the next promote
281 /// trusted a binary that never touched the node).
282 pub fn runs_post_deploy(&self) -> bool {
283 matches!(self, Gate::NodeHealth)
284 }
285
286 /// Gates evaluated at promote time against the deployed nodes or the
287 /// operator, as opposed to build-time gates (`cargo_test`,
288 /// `migration_dry_run`, `code_smoke`, `boot_smoke`) that run once on the build host and
289 /// prove nothing about a promote. A serving tier whose gate list contains
290 /// none of these would wave every promote straight through; `Topology
291 /// ::validate` refuses to load such a config (the structural form of CF1's
292 /// fail-closed default). `boot_smoke` no longer counts here — a serving tier
293 /// must declare a real node-level / operator gate, not a host smoke test.
294 pub fn guards_promotion(&self) -> bool {
295 matches!(
296 self,
297 Gate::NodeHealth | Gate::BurnIn { .. } | Gate::ManualConfirm
298 )
299 }
300 }
301
302 impl Topology {
303 pub fn load(path: &Path) -> Result<Self> {
304 let raw = std::fs::read_to_string(path)
305 .with_context(|| format!("reading topology at {}", path.display()))?;
306 let topo: Topology = toml::from_str(&raw)?;
307 topo.validate()?;
308 Ok(topo)
309 }
310
311 /// Defense-in-depth for the build-host guard: a host that serves a
312 /// provisioned tier must never be designated the builder. The runtime
313 /// hostname check in `build::run` is the real guard; this catches a
314 /// misconfiguration (build_host pointed at a prod node's name/ssh_target) at
315 /// startup, before the daemon ever accepts a build. Called from `main` once
316 /// config + topology are both loaded.
317 pub fn ensure_build_host_not_serving(&self, build_host: &str) -> Result<()> {
318 for t in &self.tiers {
319 if !t.provisioned || t.name.as_str() == "host" {
320 continue;
321 }
322 for n in &t.nodes {
323 if n.name.as_str() == build_host || n.ssh_target == build_host {
324 anyhow::bail!(
325 "build_host {build_host:?} is also node {} (ssh {}) in serving tier {} — \
326 the builder must not be a prod/serving node",
327 n.name,
328 n.ssh_target,
329 t.name
330 );
331 }
332 }
333 }
334 Ok(())
335 }
336
337 #[cfg(test)]
338 pub(crate) fn validate_for_test(&self) -> Result<()> {
339 self.validate()
340 }
341
342 /// Every `[[migration_check]]` in the daemon config must name a dump this
343 /// topology declares. The two files are separate — daemon config is
344 /// per-host, topology is per-project — so nothing but this catches a check
345 /// pointing at a backup nobody fetches. Left uncaught it surfaces as a
346 /// permanently `Blocked` gate the first time someone promotes, which reads
347 /// like a missed fetch rather than a config typo. Called from `main` once
348 /// both are loaded, and so under `--check-config`.
349 pub fn ensure_migration_checks_have_backups(
350 &self,
351 checks: &[crate::config::MigrationCheck],
352 ) -> Result<()> {
353 // A product no tier dry-runs migrations for owes no dumps. `checks` is
354 // never empty — the config defaults it to MNW's `server` entry — so
355 // without this, an intake-only product with no postgres anywhere is
356 // asked to declare a prod dump for a gate it does not configure.
357 if !self
358 .tiers
359 .iter()
360 .flat_map(|t| &t.gates)
361 .any(|g| g.kind() == GateKind::MigrationDryRun)
362 {
363 return Ok(());
364 }
365 for c in checks {
366 anyhow::ensure!(
367 self.backup_named(&c.backup).is_some(),
368 "migration_check {} restores backup {:?}, which no [[backup]] in {} declares \
369 (have: {})",
370 c.dir.display(),
371 c.backup,
372 "the topology",
373 self.backup
374 .iter()
375 .map(|b| b.name.as_str())
376 .collect::<Vec<_>>()
377 .join(", "),
378 );
379 }
380 Ok(())
381 }
382
383 /// Every `[[test_target]]` with an `aux_repo` must name a repo this topology
384 /// checks out. Third of the cross-file checks, and the one whose absence has
385 /// already cost coverage once: an unresolvable target is a warn-and-skip, by
386 /// design, so that a config describing the tip can still build an older sha.
387 /// That makes a typo here indistinguishable from a legitimate bisect skip —
388 /// a green gate that ran one crate fewer than it says it does.
389 pub fn ensure_test_target_aux_repos_exist(
390 &self,
391 targets: &[crate::config::TestTarget],
392 ) -> Result<()> {
393 for t in targets {
394 let Some(name) = t.aux_repo.as_deref() else {
395 continue;
396 };
397 anyhow::ensure!(
398 self.aux_repos.iter().any(|a| a.name == name),
399 "test_target {} names aux_repo {:?}, which no [[aux_repo]] in the topology \
400 checks out, so the gate would skip it as absent (have: {})",
401 t.label(),
402 name,
403 if self.aux_repos.is_empty() {
404 "none".to_string()
405 } else {
406 self.aux_repos
407 .iter()
408 .map(|a| a.name.as_str())
409 .collect::<Vec<_>>()
410 .join(", ")
411 },
412 );
413 }
414 Ok(())
415 }
416
417 /// Every `[[tier.node.companion]]` must name a companion the daemon config
418 /// actually builds. Same two-file split as the migration checks above, and
419 /// the same class of typo, but a worse landing: companions are installed
420 /// AFTER the symlink swap (`deploy::deploy_remote`), so a name that stages
421 /// nothing fails a promote with the server already live on the new version
422 /// — `FailureStage::AtOrAfterSwap`, the case that needs a human to go look.
423 /// Catching it at load turns that into a startup error on the build host.
424 pub fn ensure_node_companions_are_built(
425 &self,
426 built: &[crate::config::Companion],
427 ) -> Result<()> {
428 for t in &self.tiers {
429 for n in &t.nodes {
430 for c in &n.companions {
431 anyhow::ensure!(
432 built.iter().any(|b| b.name == c.name),
433 "tier {} node {} installs companion {:?}, which no [[companion]] in the \
434 daemon config builds, so nothing would be staged under \
435 companions/{} (have: {})",
436 t.name,
437 n.name,
438 c.name,
439 c.name,
440 if built.is_empty() {
441 "none".to_string()
442 } else {
443 built
444 .iter()
445 .map(|b| b.name.as_str())
446 .collect::<Vec<_>>()
447 .join(", ")
448 },
449 );
450 }
451 }
452 }
453 Ok(())
454 }
455
456 /// The configured dump for `name`, or `None` when nothing declares it.
457 pub fn backup_named(&self, name: &str) -> Option<&BackupConfig> {
458 self.backup.iter().find(|b| b.name == name)
459 }
460
461 fn validate(&self) -> Result<()> {
462 // A dump is only owed by a product that actually dry-runs migrations.
463 // The unconditional form of this asserted something about every
464 // product's tiers from a fact about one: pom configures no
465 // `migration_dry_run` anywhere (it has no postgres schema at all), so
466 // requiring it to declare a prod dump would be demanding a fixture for
467 // a gate it never runs.
468 let dry_runs_migrations = self
469 .tiers
470 .iter()
471 .flat_map(|t| &t.gates)
472 .any(|g| g.kind() == GateKind::MigrationDryRun);
473 anyhow::ensure!(
474 !dry_runs_migrations || !self.backup.is_empty(),
475 "a tier configures migration_dry_run but the topology declares no [backup]; \
476 the gate would have nothing to restore"
477 );
478 for (i, b) in self.backup.iter().enumerate() {
479 anyhow::ensure!(
480 !b.name.is_empty()
481 && b.name
482 .bytes()
483 .all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-'),
484 "backup name {:?} must be non-empty and match [A-Za-z0-9_-]+; it keys the \
485 `backups` table and a daemon-config migration_check",
486 b.name,
487 );
488 anyhow::ensure!(
489 !b.source.is_empty() && !b.local_path.is_empty(),
490 "backup {} has an empty source/local_path",
491 b.name,
492 );
493 // Two dumps sharing a name would interleave in `backups`, so the
494 // freshness check and the plausibility floor would each read the
495 // other's row. Two sharing a `local_path` would overwrite each
496 // other on disk, and whichever fetched last would be restored for
497 // both — green, and proving nothing about one of the databases.
498 for prior in &self.backup[..i] {
499 anyhow::ensure!(
500 prior.name != b.name,
501 "two backup entries share the name {:?}",
502 b.name,
503 );
504 anyhow::ensure!(
505 prior.local_path != b.local_path,
506 "backups {:?} and {:?} share local_path {:?}; they would overwrite each other",
507 prior.name,
508 b.name,
509 b.local_path,
510 );
511 }
512 }
513 anyhow::ensure!(
514 !self.tiers.is_empty(),
515 "topology must declare at least one tier"
516 );
517 for t in &self.tiers {
518 // The `host` tier is the build tier (cargo_test / migration_dry_run /
519 // code_smoke / boot_smoke run once on the host); every other tier serves an
520 // artifact to nodes, so it is exempted from the node and
521 // promotion-gate checks the same way.
522 let is_build_tier = t.name.as_str() == "host";
523 if t.provisioned && t.nodes.is_empty() && !is_build_tier {
524 anyhow::bail!("tier {} is provisioned but has no nodes", t.name);
525 }
526 // Fail closed by default: a provisioned serving tier must declare at
527 // least one gate that actually guards a promote. Without this, an
528 // empty (or build-time-only) gate list waves every promote through
529 // because `unsatisfied_gates` finds nothing to check (CF1 root cause).
530 if t.provisioned && !is_build_tier && !t.gates.iter().any(Gate::guards_promotion) {
531 anyhow::bail!(
532 "tier {} is provisioned to serve but declares no promotion gate \
533 (need at least one of node_health / burn_in / manual_confirm)",
534 t.name
535 );
536 }
537 }
538 let mut seen_dirs: Vec<Vec<&str>> = Vec::new();
539 for aux in &self.aux_repos {
540 anyhow::ensure!(
541 !aux.name.is_empty() && !aux.bare_path.is_empty() && !aux.branch.is_empty(),
542 "aux_repo entry has an empty name/bare_path/branch"
543 );
544 // `checkout_dir` becomes a `workdir.join(..)`. Nesting is allowed —
545 // docengine lives at `Libraries/docengine` in the dev tree and the
546 // path dep resolves to that shape — but every component must be a
547 // plain name so an aux repo can never write outside the workdir.
548 let dir = &aux.checkout_dir;
549 let parts: Vec<&str> = dir.split('/').collect();
550 anyhow::ensure!(
551 !dir.is_empty()
552 && !dir.contains('\\')
553 && parts
554 .iter()
555 .all(|c| !c.is_empty() && *c != "." && *c != ".."),
556 "aux_repo {} has an unsafe checkout_dir {dir:?} (must be a relative path of \
557 plain components: no leading slash, empty segments, or dot-dot)",
558 aux.name,
559 );
560 // Two checkouts may not share a dir, and neither may sit inside the
561 // other: `git worktree add` into a path under a live worktree buries
562 // one checkout in the other's tree, and whichever builds second wins.
563 for prior in &seen_dirs {
564 let common = prior.len().min(parts.len());
565 anyhow::ensure!(
566 prior[..common] != parts[..common],
567 "two aux_repo entries share or nest checkout_dir {dir:?}; \
568 they would clobber each other"
569 );
570 }
571 seen_dirs.push(parts);
572 }
573 Ok(())
574 }
575 }
576
577 #[cfg(test)]
578 mod tests {
579 use super::*;
580
581 /// A minimal topology with one serving tier whose gate block is `gates`.
582 fn topo_with_serving_gates(provisioned: bool, gates: &str) -> Topology {
583 let raw = format!(
584 r#"
585 [repo]
586 bare_path = "/tmp/repo.git"
587 branch = "main"
588
589 [backup]
590 source = "ssh://prod/dump.sql.gz"
591 local_path = "/tmp/dump.sql.gz"
592
593 [[tier]]
594 name = "b"
595 provisioned = {provisioned}
596 gates = [{gates}]
597 [[tier.node]]
598 name = "prod-1"
599 ssh_target = "prod-1"
600 release_root = "/srv/mnw"
601 "#
602 );
603 toml::from_str(&raw).expect("parse test topology")
604 }
605
606 #[test]
607 fn provisioned_serving_tier_with_no_gates_is_rejected() {
608 let topo = topo_with_serving_gates(true, "");
609 let err = topo.validate_for_test().unwrap_err().to_string();
610 assert!(err.contains("no promotion gate"), "{err}");
611 }
612
613 #[test]
614 fn provisioned_serving_tier_with_only_build_gates_is_rejected() {
615 // cargo_test / migration_dry_run are build-time and prove nothing about a
616 // promote, so a serving tier carrying only them still fails closed.
617 let topo = topo_with_serving_gates(
618 true,
619 r#"{ kind = "cargo_test" }, { kind = "migration_dry_run" }"#,
620 );
621 let err = topo.validate_for_test().unwrap_err().to_string();
622 assert!(err.contains("no promotion gate"), "{err}");
623 }
624
625 #[test]
626 fn provisioned_serving_tier_with_a_promotion_gate_is_accepted() {
627 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
628 assert!(topo.validate_for_test().is_ok());
629 }
630
631 #[test]
632 fn provisioned_serving_tier_with_only_boot_smoke_is_rejected() {
633 // boot_smoke is a build-host gate now; it proves nothing about a node, so
634 // a serving tier carrying only boot_smoke must fail closed exactly like an
635 // empty gate list (Run-2 SERIOUS-3 structural close).
636 let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
637 let err = topo.validate_for_test().unwrap_err().to_string();
638 assert!(err.contains("no promotion gate"), "{err}");
639 }
640
641 #[test]
642 fn unprovisioned_tier_with_empty_gates_is_skipped() {
643 // A declared-but-not-yet-provisioned tier (e.g. tier c) carries no
644 // promote authority, so the gate requirement does not apply yet.
645 let topo = topo_with_serving_gates(false, "");
646 assert!(topo.validate_for_test().is_ok());
647 }
648
649 #[test]
650 fn build_host_matching_a_serving_node_is_rejected() {
651 // prod-1 is a node in the provisioned serving tier built above; naming it
652 // as the builder must fail closed.
653 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
654 let err = topo
655 .ensure_build_host_not_serving("prod-1")
656 .unwrap_err()
657 .to_string();
658 assert!(err.contains("must not be a prod/serving node"), "{err}");
659 }
660
661 #[test]
662 fn build_host_distinct_from_serving_nodes_is_accepted() {
663 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
664 assert!(topo.ensure_build_host_not_serving("fw13").is_ok());
665 }
666
667 #[test]
668 fn node_companions_default_empty_and_parse_when_present() {
669 // A node without [[tier.node.companion]] is server-only.
670 let plain = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
671 assert!(plain.tiers[0].nodes[0].companions.is_empty());
672
673 // A node that declares a companion carries its install target + unit.
674 let raw = r#"
675 [repo]
676 bare_path = "/tmp/repo.git"
677 branch = "main"
678 [backup]
679 source = "s"
680 local_path = "/tmp/d"
681 [[tier]]
682 name = "b"
683 provisioned = true
684 gates = [{ kind = "node_health" }]
685 [[tier.node]]
686 name = "prod-1"
687 ssh_target = "makenotwork@alpha-west-1"
688 release_root = "/opt/mnw"
689 [[tier.node.companion]]
690 name = "mnw-cli"
691 install_path = "/opt/mnw-cli/mnw-cli"
692 service_name = "mnw-cli.service"
693 "#;
694 let topo: Topology = toml::from_str(raw).expect("parse companion topology");
695 let c = &topo.tiers[0].nodes[0].companions;
696 assert_eq!(c.len(), 1);
697 assert_eq!(c[0].name, "mnw-cli");
698 assert_eq!(c[0].install_path, "/opt/mnw-cli/mnw-cli");
699 assert_eq!(c[0].service_name, "mnw-cli.service");
700 }
701
702 /// A topology whose one node installs the named companions.
703 fn topo_installing(names: &[&str]) -> Topology {
704 let mut blocks = String::new();
705 for n in names {
706 use std::fmt::Write;
707 let _ = write!(
708 blocks,
709 "[[tier.node.companion]]\nname = \"{n}\"\n\
710 install_path = \"/opt/{n}/{n}\"\nservice_name = \"{n}.service\"\n"
711 );
712 }
713 let raw = format!(
714 r#"
715 [repo]
716 bare_path = "/tmp/repo.git"
717 branch = "main"
718 [backup]
719 source = "s"
720 local_path = "/tmp/d"
721 [[tier]]
722 name = "b"
723 provisioned = true
724 gates = [{{ kind = "node_health" }}]
725 [[tier.node]]
726 name = "prod-1"
727 ssh_target = "makenotwork@alpha-west-1"
728 release_root = "/opt/mnw"
729 {blocks}"#
730 );
731 toml::from_str(&raw).expect("parse topology")
732 }
733
734 fn built(names: &[&str]) -> Vec<crate::config::Companion> {
735 names
736 .iter()
737 .map(|n| crate::config::Companion {
738 name: (*n).to_string(),
739 manifest_dir: (*n).into(),
740 bin: (*n).to_string(),
741 })
742 .collect()
743 }
744
745 fn test_target(dir: &str, aux_repo: Option<&str>) -> crate::config::TestTarget {
746 crate::config::TestTarget {
747 dir: dir.into(),
748 aux_repo: aux_repo.map(str::to_string),
749 features: Vec::new(),
750 all_features: false,
751 scratch_db: false,
752 }
753 }
754
755 #[test]
756 fn a_test_target_naming_a_checked_out_aux_repo_is_accepted() {
757 let topo = topo_with_aux(
758 "[[aux_repo]]\nname = \"docengine\"\nbare_path = \"/tmp/d.git\"\n\
759 upstream = \"git@h:max/d.git\"\nbranch = \"main\"\ncheckout_dir = \"Libraries/docengine\"\n",
760 )
761 .expect("parse");
762 assert!(
763 topo.ensure_test_target_aux_repos_exist(&[
764 test_target("server", None),
765 test_target("", Some("docengine")),
766 ])
767 .is_ok()
768 );
769 }
770
771 #[test]
772 fn a_test_target_naming_an_unknown_aux_repo_is_rejected_at_load() {
773 // The failure this exists to prevent is silent: an unresolvable target
774 // is a warn-and-skip (bisect), so the gate stays green having run one
775 // crate fewer than the config claims.
776 let topo = topo_with_aux(
777 "[[aux_repo]]\nname = \"synckit\"\nbare_path = \"/tmp/s.git\"\n\
778 upstream = \"git@h:max/s.git\"\nbranch = \"main\"\ncheckout_dir = \"synckit\"\n",
779 )
780 .expect("parse");
781 let err = topo
782 .ensure_test_target_aux_repos_exist(&[test_target("", Some("docengine"))])
783 .unwrap_err()
784 .to_string();
785 assert!(err.contains("docengine"), "{err}");
786 assert!(err.contains("have: synckit"), "{err}");
787 }
788
789 #[test]
790 fn test_targets_without_an_aux_repo_need_no_aux_repos_declared() {
791 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
792 assert!(
793 topo.ensure_test_target_aux_repos_exist(&[test_target("server", None)])
794 .is_ok()
795 );
796 }
797
798 #[test]
799 fn a_node_companion_the_daemon_builds_is_accepted() {
800 let topo = topo_installing(&["mnw-cli", "multithreaded"]);
801 assert!(
802 topo.ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
803 .is_ok()
804 );
805 }
806
807 #[test]
808 fn a_node_companion_nothing_builds_is_rejected_at_load() {
809 // Left uncaught this fails during the post-swap install on prod, with
810 // the server already live on the new version.
811 let topo = topo_installing(&["mnw-cli", "multithreadd"]);
812 let err = topo
813 .ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
814 .unwrap_err()
815 .to_string();
816 assert!(err.contains("multithreadd"), "{err}");
817 assert!(err.contains("no [[companion]]"), "{err}");
818 // The message names what IS available, so the typo is obvious.
819 assert!(err.contains("mnw-cli, multithreaded"), "{err}");
820 }
821
822 #[test]
823 fn a_node_companion_with_no_companions_configured_at_all_is_rejected() {
824 let topo = topo_installing(&["multithreaded"]);
825 let err = topo
826 .ensure_node_companions_are_built(&[])
827 .unwrap_err()
828 .to_string();
829 assert!(err.contains("have: none"), "{err}");
830 }
831
832 #[test]
833 fn a_topology_installing_no_companions_is_fine_with_none_built() {
834 let topo = topo_installing(&[]);
835 assert!(topo.ensure_node_companions_are_built(&[]).is_ok());
836 }
837
838 fn topo_with_aux(aux_block: &str) -> Result<Topology> {
839 let raw = format!(
840 r#"
841 [repo]
842 bare_path = "/tmp/repo.git"
843 branch = "main"
844 [backup]
845 source = "s"
846 local_path = "/tmp/d"
847 [[tier]]
848 name = "b"
849 provisioned = true
850 gates = [{{ kind = "node_health" }}]
851 [[tier.node]]
852 name = "prod-1"
853 ssh_target = "prod-1"
854 release_root = "/srv/mnw"
855 {aux_block}
856 "#
857 );
858 let topo: Topology = toml::from_str(&raw)?;
859 topo.validate_for_test()?;
860 Ok(topo)
861 }
862
863 #[test]
864 fn aux_repos_default_empty() {
865 let topo = topo_with_aux("").expect("no aux_repo block is fine");
866 assert!(topo.aux_repos.is_empty());
867 }
868
869 #[test]
870 fn aux_repo_parses_all_fields() {
871 let topo = topo_with_aux(
872 r#"
873 [[aux_repo]]
874 name = "synckit"
875 bare_path = "/srv/sando/synckit.git"
876 upstream = "git@ssh.makenot.work:max/synckit.git"
877 branch = "main"
878 checkout_dir = "synckit""#,
879 )
880 .expect("valid aux_repo parses");
881 assert_eq!(topo.aux_repos.len(), 1);
882 let a = &topo.aux_repos[0];
883 assert_eq!(a.name, "synckit");
884 assert_eq!(a.bare_path, "/srv/sando/synckit.git");
885 assert_eq!(a.upstream, "git@ssh.makenot.work:max/synckit.git");
886 assert_eq!(a.branch, "main");
887 assert_eq!(a.checkout_dir, "synckit");
888 }
889
890 #[test]
891 fn aux_repo_with_nested_checkout_dir_is_accepted() {
892 let topo = topo_with_aux(
893 r#"
894 [[aux_repo]]
895 name = "docengine"
896 bare_path = "/srv/sando/docengine.git"
897 upstream = "git@ssh.makenot.work:max/docengine.git"
898 branch = "main"
899 checkout_dir = "Libraries/docengine""#,
900 )
901 .expect("a nested checkout_dir is a valid location");
902 assert_eq!(topo.aux_repos[0].checkout_dir, "Libraries/docengine");
903 }
904
905 #[test]
906 fn aux_repo_with_traversing_checkout_dir_is_rejected() {
907 for bad in [
908 "../escape",
909 "a/../../escape",
910 "a/./b",
911 "a//b",
912 "/abs",
913 "a/",
914 "..",
915 ".",
916 ] {
917 let err = topo_with_aux(&format!(
918 r#"
919 [[aux_repo]]
920 name = "x"
921 bare_path = "/srv/sando/x.git"
922 upstream = "u"
923 branch = "main"
924 checkout_dir = "{bad}""#,
925 ))
926 .unwrap_err()
927 .to_string();
928 assert!(err.contains("unsafe checkout_dir"), "for {bad:?}: {err}");
929 }
930 }
931
932 #[test]
933 fn aux_repos_sharing_a_checkout_dir_are_rejected() {
934 let err = topo_with_aux(
935 r#"
936 [[aux_repo]]
937 name = "one"
938 bare_path = "/srv/sando/one.git"
939 upstream = "u"
940 branch = "main"
941 checkout_dir = "shared"
942 [[aux_repo]]
943 name = "two"
944 bare_path = "/srv/sando/two.git"
945 upstream = "u"
946 branch = "main"
947 checkout_dir = "shared""#,
948 )
949 .unwrap_err()
950 .to_string();
951 assert!(err.contains("share or nest checkout_dir"), "{err}");
952 }
953
954 #[test]
955 fn aux_repo_nested_inside_another_checkout_dir_is_rejected() {
956 let err = topo_with_aux(
957 r#"
958 [[aux_repo]]
959 name = "outer"
960 bare_path = "/srv/sando/outer.git"
961 upstream = "u"
962 branch = "main"
963 checkout_dir = "Libraries"
964 [[aux_repo]]
965 name = "inner"
966 bare_path = "/srv/sando/inner.git"
967 upstream = "u"
968 branch = "main"
969 checkout_dir = "Libraries/docengine""#,
970 )
971 .unwrap_err()
972 .to_string();
973 assert!(err.contains("share or nest checkout_dir"), "{err}");
974 }
975
976 /// A topology whose `[backup]`/`[[backup]]` section is `backup_block`.
977 fn topo_with_backup_block(backup_block: &str) -> Result<Topology> {
978 let raw = format!(
979 r#"
980 [repo]
981 bare_path = "/tmp/repo.git"
982 branch = "main"
983 {backup_block}
984 [[tier]]
985 name = "b"
986 provisioned = true
987 # migration_dry_run is what makes the backup rules apply at all: a product no
988 # tier dry-runs migrations for owes no dumps, so a fixture exercising those rules
989 # has to configure the gate.
990 gates = [{{ kind = "node_health" }}, {{ kind = "migration_dry_run" }}]
991 [[tier.node]]
992 name = "prod-1"
993 ssh_target = "prod-1"
994 release_root = "/srv/mnw"
995 "#
996 );
997 let topo: Topology = toml::from_str(&raw)?;
998 topo.validate_for_test()?;
999 Ok(topo)
1000 }
1001
1002 #[test]
1003 fn a_single_backup_table_still_parses_as_one_named_server() {
1004 // Back-compat is the point: every deployed sando.toml uses the single
1005 // `[backup]` form, and the box this config lives on is the one whose job
1006 // is deploying — it must not need an edit to start.
1007 let topo = topo_with_backup_block(
1008 r#"
1009 [backup]
1010 source = "ssh://prod/dump.sql.gz"
1011 local_path = "/tmp/dump.sql.gz""#,
1012 )
1013 .expect("the single-table form must still load");
1014 assert_eq!(topo.backup.len(), 1);
1015 assert_eq!(topo.backup[0].name, "server");
1016 assert!(topo.backup_named("server").is_some());
1017 }
1018
1019 #[test]
1020 fn a_backup_list_parses_and_keeps_its_names() {
1021 let topo = topo_with_backup_block(
1022 r#"
1023 [[backup]]
1024 name = "server"
1025 source = "ssh://prod/makenotwork/latest.sql.gz"
1026 local_path = "/tmp/server.sql.gz"
1027 [[backup]]
1028 name = "multithreaded"
1029 source = "ssh://prod/multithreaded/latest.sql.gz"
1030 local_path = "/tmp/mt.sql.gz""#,
1031 )
1032 .expect("the list form must load");
1033 assert_eq!(topo.backup.len(), 2);
1034 assert_eq!(
1035 topo.backup_named("multithreaded").unwrap().local_path,
1036 "/tmp/mt.sql.gz"
1037 );
1038 assert!(topo.backup_named("nope").is_none());
1039 }
1040
1041 #[test]
1042 fn two_backups_sharing_a_name_are_rejected() {
1043 // They would interleave in `backups`, so the freshness check and the
1044 // plausibility floor would each read the other's row.
1045 let err = topo_with_backup_block(
1046 r#"
1047 [[backup]]
1048 name = "server"
1049 source = "a"
1050 local_path = "/tmp/a.sql.gz"
1051 [[backup]]
1052 name = "server"
1053 source = "b"
1054 local_path = "/tmp/b.sql.gz""#,
1055 )
1056 .unwrap_err()
1057 .to_string();
1058 assert!(err.contains("share the name"), "{err}");
1059 }
1060
1061 #[test]
1062 fn two_backups_sharing_a_local_path_are_rejected() {
1063 // Whichever fetched last would be restored for both checks — green, and
1064 // proving nothing about one of the two databases.
1065 let err = topo_with_backup_block(
1066 r#"
1067 [[backup]]
1068 name = "server"
1069 source = "a"
1070 local_path = "/tmp/same.sql.gz"
1071 [[backup]]
1072 name = "multithreaded"
1073 source = "b"
1074 local_path = "/tmp/same.sql.gz""#,
1075 )
1076 .unwrap_err()
1077 .to_string();
1078 assert!(err.contains("share local_path"), "{err}");
1079 }
1080
1081 #[test]
1082 fn a_migration_check_naming_an_undeclared_backup_is_rejected_at_startup() {
1083 // Daemon config and topology are separate files, so nothing but this
1084 // cross-check catches the typo. Uncaught it surfaces as a permanently
1085 // Blocked gate on the next promote, which reads like a missed fetch.
1086 let topo = topo_with_backup_block(
1087 r#"
1088 [backup]
1089 source = "s"
1090 local_path = "/tmp/d""#,
1091 )
1092 .unwrap();
1093 let checks = vec![crate::config::MigrationCheck {
1094 dir: std::path::PathBuf::from("multithreaded/migrations"),
1095 backup: "multithreaded".into(),
1096 scratch_db: Some("sando_scratch_mt".into()),
1097 owner_role: Some("multithreaded".into()),
1098 }];
1099 let err = topo
1100 .ensure_migration_checks_have_backups(&checks)
1101 .unwrap_err()
1102 .to_string();
1103 assert!(err.contains("which no [[backup]]"), "{err}");
1104
1105 // And the shipped pair agree, which is the case that actually ships.
1106 let shipped_checks = crate::config::default_migration_checks_for_test();
1107 shipped()
1108 .ensure_migration_checks_have_backups(&shipped_checks)
1109 .expect("the default server check resolves against the shipped topology");
1110 }
1111
1112 #[test]
1113 fn real_sando_toml_loads_clean() {
1114 // The shipped topology must satisfy the invariant — guards against a
1115 // regression that would lock sandod out of its own config.
1116 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
1117 Topology::load(&path).expect("shipped sando.toml must validate");
1118 }
1119
1120 fn shipped() -> Topology {
1121 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
1122 Topology::load(&path).expect("shipped sando.toml must validate")
1123 }
1124
1125 #[test]
1126 fn shipping_to_the_last_provisioned_tier_needs_an_operator_signoff() {
1127 // A tier's gates guard promotion *out* of it, which is the subtlety that
1128 // made manual_confirm inert: it sat on tier b, guarding b -> c, and c is
1129 // not provisioned. So the ship to production was cleared by node_health
1130 // + burn_in alone — and `hotfix: true` skips burn_in.
1131 //
1132 // The gate that matters therefore belongs on the PREDECESSOR of the last
1133 // provisioned tier. Asserted structurally so re-provisioning tiers cannot
1134 // silently strand the sign-off again.
1135 let topo = shipped();
1136 let last = topo
1137 .tiers
1138 .iter()
1139 .rposition(|t| t.provisioned)
1140 .expect("some tier must be provisioned");
1141 assert!(last > 0, "the production tier cannot be the first tier");
1142 let guard = &topo.tiers[last - 1];
1143 assert!(
1144 guard.gates.iter().any(|g| matches!(g, Gate::ManualConfirm)),
1145 "tier {} guards promotion into the last provisioned tier ({}), so it must require an operator sign-off; its gates are {:?}",
1146 guard.name,
1147 topo.tiers[last].name,
1148 guard
1149 .gates
1150 .iter()
1151 .map(|g| g.kind().as_str())
1152 .collect::<Vec<_>>(),
1153 );
1154 }
1155
1156 #[test]
1157 fn every_serving_node_has_a_readiness_probe() {
1158 // Without health_url, node_health degrades to `systemctl is-active`,
1159 // which a crash-looping binary satisfies between restarts — exactly what
1160 // the 0.10.14 CDN_BASE_URL crash-loop did on prod-1.
1161 let topo = shipped();
1162 for tier in topo.tiers.iter().filter(|t| t.provisioned) {
1163 for node in &tier.nodes {
1164 assert!(
1165 node.health_url.is_some(),
1166 "node {} on tier {} has no health_url, so node_health proves only that systemd thinks the unit is running",
1167 node.name,
1168 tier.name,
1169 );
1170 }
1171 }
1172 }
1173 }
1174