use crate::domain::{GateKind, NodeId, TierId}; use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use std::path::Path; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Topology { /// The repo Sando checks out to build this product. /// /// `None` for an intake-only product: pom is built by Bento on two /// machines and handed over as finished bytes, so Sando fetches no source /// for it and there is no bare repo on this host to name. A topology that /// declares no repo cannot be `/rebuild`-ed, which is the same statement /// [`AppConfig::build_host`](crate::config::AppConfig::build_host) makes /// from the other side. #[serde(default)] pub repo: Option, /// Prod dumps `/backup/fetch` pulls, one per database `migration_dry_run` /// has a check for. A list because the repo ships more than one service /// with its own database and its own `sqlx::migrate!()` at boot: the server /// migrates `makenotwork`, multithreaded migrates `multithreaded`, and a /// gate that restores only the first proves nothing about the second. /// /// Accepts both the historical single `[backup]` table and a `[[backup]]` /// list, so a deployed `sando.toml` keeps working unedited (the single form /// deserializes to a one-entry list named `server`). #[serde(deserialize_with = "one_or_many_backup")] pub backup: Vec, #[serde(rename = "tier")] pub tiers: Vec, /// Extra repos to fetch and check out beside the main worktree before a /// build, so a path dependency that reaches across the repo split resolves. /// Empty (default) keeps an existing `sando.toml` working unedited. See /// [`AuxRepo`] and [`crate::build::checkout_aux_repos`]. #[serde(default, rename = "aux_repo")] pub aux_repos: Vec, } /// An auxiliary repo checked out beside the per-sha worktree so cross-repo path /// dependencies resolve at build time. /// /// The concrete case (2026-07-24): `mnw-cli` — a companion built from the MNW /// worktree — carries `synckit-client = { path = "../../synckit/synckit-client" }` /// after synckit moved to its own repo. From the companion crate at /// `//mnw-cli`, that path resolves to `/synckit`, a sibling /// of the per-sha worktree that Sando otherwise never creates, so the companion /// build failed with "No such file or directory". An `aux_repo` named to land at /// `checkout_dir = "synckit"` puts the synckit source exactly there. /// /// The checkout is at the FIXED `/`, not per-sha: the path /// dependency resolves to that spot regardless of the MNW sha, and builds /// serialize, so a single shared checkout refreshed to `branch` HEAD each build /// is correct. Because a path dep has no lockfile pin, "branch HEAD" is the honest /// resolution — the same contract as the dev working copy. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AuxRepo { /// Human label for logs and errors (e.g. `synckit`). pub name: String, /// Bare repo Sando fetches into and worktrees from, e.g. /// `/srv/sando/synckit.git`. Auto-created (hookless) on first build. pub bare_path: String, /// Canonical git remote fetched before checkout. Like [`RepoConfig::upstream`] /// but required here: an aux repo is pull-based (nobody pushes to its bare). pub upstream: String, /// Branch whose HEAD is checked out. pub branch: String, /// Where the worktree lands, relative to `cfg.workdir`. May name a nested /// location (`Libraries/docengine`), because a path dep resolves to wherever /// the dev tree keeps the crate and Sando has to match that shape. Every /// component must be a plain name — no leading slash, no `..` — so the /// checkout stays under the workdir. pub checkout_dir: String, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RepoConfig { pub bare_path: String, pub branch: String, /// Optional canonical git remote. When set, `/rebuild` fetches the deploy /// branch from here into the bare repo before worktree-ing the target sha, /// so a freshly-pushed commit is resolvable without anyone pushing to the /// bare repo directly (pull-based deploys). When unset, Sando relies on the /// bare repo already containing the sha (push-based / hook-driven). #[serde(default)] pub upstream: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BackupConfig { /// Which database this dump is of, as referenced by a daemon-config /// `[[migration_check]]`'s `backup` key and recorded in the `backups` /// table's `name` column. Defaults to `server` so the historical single /// `[backup]` table needs no edit — and so the pre-existing rows, which the /// state-DB migration backfills to `server`, keep matching it. #[serde(default = "default_backup_name")] pub name: String, pub source: String, pub local_path: String, } fn default_backup_name() -> String { "server".into() } /// Accept `[backup]` (one table) or `[[backup]]` (a list) for the same key. /// Serde cannot express "table or sequence" on a `Vec` field on its own, and /// the alternative — renaming the key — would break every deployed /// `sando.toml` at startup, on the box whose whole job is deploying. fn one_or_many_backup<'de, D>(de: D) -> std::result::Result, D::Error> where D: serde::Deserializer<'de>, { #[derive(Deserialize)] #[serde(untagged)] enum OneOrMany { One(BackupConfig), Many(Vec), } Ok(match OneOrMany::deserialize(de)? { OneOrMany::One(b) => vec![b], OneOrMany::Many(v) => v, }) } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Tier { pub name: TierId, #[serde(default)] pub provisioned: bool, pub gates: Vec, #[serde(default)] pub canary: CanaryPolicy, #[serde(default, rename = "node")] pub nodes: Vec, /// Where the public reaches this tier, CDN included. /// /// Deliberately not derivable from a node's `ssh_target`. The whole value of /// [`Gate::PageSmoke`] is that it goes the way a visitor goes -- through /// Cloudflare, against the hostname in the browser's address bar -- and a /// URL computed from the node would reach the origin and inherit exactly the /// blind spot the gate exists to close. So it is stated, or the gate is not /// available. #[serde(default)] pub public_url: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Node { pub name: NodeId, pub ssh_target: String, pub release_root: String, /// What this machine runs, as `os/arch` (e.g. `linux/aarch64`). /// /// Compared against the bundle's own platform before anything is pushed; /// see [`crate::deploy::Placement`]. Optional, and a node that declares it /// can only be given a bundle that declares a matching one — silence on /// either side is a refusal, not a pass. MNW's nodes declare nothing and /// keep the single-platform behavior they have always had; pom's declare /// theirs, because pom is the product where one version is two bundles. #[serde(default)] pub platform: Option, /// What this machine IS, as `id/version` from `/etc/os-release` /// (e.g. `ubuntu/24.04`, `alloy/0.1`). /// /// Verified against the node before anything is pushed to it, so a box that /// was rebuilt into something else fails the promote with the running /// service intact instead of being discovered by a binary that will not /// start. Optional: a node that declares nothing is not checked, and the /// deploy log says it was not. See [`ops_core::base_image`] for why silence /// is a skip here and a refusal in [`crate::deploy::Placement`]. #[serde(default)] pub base_image: Option, /// The glibc version this node has, as `ldd --version` reports it /// (e.g. `2.39`). Checked independently of [`Self::base_image`], because a /// point release moves under a pinned base and it is this number that /// decides whether a binary loads. /// /// Declaring it does NOT replace the pre-swap `ldd` guard, which compares /// the actual binary against the actual node. It makes the node's floor a /// stated fact that can be compared before bytes are built or moved. #[serde(default)] pub libc: Option, /// systemd unit name to reload-or-restart after the symlink swap. /// Defaults to "makenotwork.service" because that's MNW's prod unit. #[serde(default = "default_service_name")] pub service_name: String, /// Opt-in config-drift guard. When set to the node's env-file path (e.g. /// `/etc/mnw/makenotwork.env`), the deploy sources that file and runs the /// freshly-rsynced binary in `MNW_CHECK_CONFIG=1` mode BEFORE the symlink /// swap. A required var missing on this node (a var added upstream but never /// added to the node's env — how testnot crash-looped on CDN_BASE_URL) then /// fails the promote with the running service still intact, instead of after /// the swap+restart. Unset (default) skips the check, so an existing /// `sando.toml` keeps working and it never runs against a binary too old to /// support the mode; enable it once a check-capable version is deployed. #[serde(default)] pub config_check_env_file: Option, /// Capability grant for this node's executor (see `ops_exec`). Defaults to /// the current behavior of every Sando node — actuate deploy+restart, /// observe health — so an existing `sando.toml` keeps working unedited. #[serde(default = "default_actuate")] pub actuate: Vec, #[serde(default = "default_observe")] pub observe: Vec, /// Optional HTTP readiness URL the `node_health` gate curls on the node (over /// its executor) after `systemctl is-active`. Typically a loopback address /// the service binds, e.g. `http://127.0.0.1:8080/health`. When unset, the /// gate proves the unit is active post-restart but does not HTTP-probe; set /// it for a full readiness check. Kept optional so an existing `sando.toml` /// needs no edit. #[serde(default)] pub health_url: Option, /// Companion services this node installs from the release bundle after the /// server is swapped and restarted. Each entry names a `[[companion]]` the /// daemon builds (see `Config::companions`) and says where its binary lands /// and which unit to restart — so a contract-coupled service (mnw-cli) ships /// in the same promote as the server instead of drifting. Empty (default) = /// server-only node, so an existing `sando.toml` keeps working unedited. #[serde(default, rename = "companion")] pub companions: Vec, } /// Where a node installs a built companion binary and which unit to bounce. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NodeCompanion { /// Must match a `Config::companions[].name` — the bundle subdir to read from. pub name: String, /// Absolute path the companion binary is installed to on the node /// (the unit's `ExecStart`), e.g. `/opt/mnw-cli/mnw-cli`. pub install_path: String, /// systemd unit restarted after the binary is installed, e.g. /// `mnw-cli.service`. pub service_name: String, } fn default_service_name() -> String { "makenotwork.service".into() } /// The capability set every pre-existing Sando node implicitly had: it deploys /// and restarts, and is health-observed. Keeping these as the defaults is what /// lets `sando.toml` stay unchanged through the executor refactor. pub fn default_actuate() -> Vec { vec!["deploy".into(), "restart".into()] } pub fn default_observe() -> Vec { vec!["health".into()] } #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)] #[serde(rename_all = "snake_case")] pub enum CanaryPolicy { #[default] Sequential, Parallel, } impl CanaryPolicy { pub fn as_str(self) -> &'static str { match self { CanaryPolicy::Sequential => "sequential", CanaryPolicy::Parallel => "parallel", } } } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum Gate { CargoTest, HardeningTest, Clippy, Fmt, CargoAudit, CargoDeny, MigrationDryRun, CodeSmoke, BootSmoke, NodeHealth, /// Post-deploy, in a real browser, over the tier's public URL. /// /// The one gate that crosses the CDN. `boot_smoke` runs on the build host /// and `node_health` reaches a node over its executor, so between them the /// edge was never watched -- and on 2026-08-14 testnot served a page whose /// JavaScript did not run for hours behind nine green gates, because the /// CDN held a stale module that failed to link against a fresh one. /// /// Needs `public_url` on the tier. A tier without one cannot run it, which /// `validate` refuses at load rather than at promote time. PageSmoke, BurnIn { hours: u32, }, ManualConfirm, } impl Gate { /// The discriminant — the identifier we use in events, schema columns, /// and the TUI. Gate parameters (e.g. `BurnIn.hours`) stay with `Gate` /// and are not carried into `gate_runs` history. pub fn kind(&self) -> GateKind { match self { Gate::CargoTest => GateKind::CargoTest, Gate::HardeningTest => GateKind::HardeningTest, Gate::Clippy => GateKind::Clippy, Gate::Fmt => GateKind::Fmt, Gate::CargoAudit => GateKind::CargoAudit, Gate::CargoDeny => GateKind::CargoDeny, Gate::MigrationDryRun => GateKind::MigrationDryRun, Gate::CodeSmoke => GateKind::CodeSmoke, Gate::BootSmoke => GateKind::BootSmoke, Gate::NodeHealth => GateKind::NodeHealth, Gate::PageSmoke => GateKind::PageSmoke, Gate::BurnIn { .. } => GateKind::BurnIn, Gate::ManualConfirm => GateKind::ManualConfirm, } } /// Gates that execute against a tier's freshly-deployed nodes, recording a /// `gate_runs` row, at the end of a successful promote to that tier — the /// evidence the *next* promote checks. Only `node_health`: it probes the /// nodes the deploy just shipped to. `boot_smoke` is NOT post-deploy — it is /// a build-time gate that boots the staged artifact on the build host and /// proves nothing about a deployed node (the Run-2 SERIOUS-3 blind spot: /// boot_smoke used to re-run locally at promote time and the next promote /// trusted a binary that never touched the node). pub fn runs_post_deploy(&self) -> bool { matches!(self, Gate::NodeHealth | Gate::PageSmoke) } /// Gates evaluated at promote time against the deployed nodes or the /// operator, as opposed to build-time gates (`cargo_test`, /// `migration_dry_run`, `code_smoke`, `boot_smoke`) that run once on the build host and /// prove nothing about a promote. A serving tier whose gate list contains /// none of these would wave every promote straight through; `Topology /// ::validate` refuses to load such a config (the structural form of CF1's /// fail-closed default). `boot_smoke` no longer counts here — a serving tier /// must declare a real node-level / operator gate, not a host smoke test. pub fn guards_promotion(&self) -> bool { matches!( self, Gate::NodeHealth | Gate::PageSmoke | Gate::BurnIn { .. } | Gate::ManualConfirm ) } } impl Topology { pub fn load(path: &Path) -> Result { let raw = std::fs::read_to_string(path) .with_context(|| format!("reading topology at {}", path.display()))?; let topo: Topology = toml::from_str(&raw)?; topo.validate()?; Ok(topo) } /// Defense-in-depth for the build-host guard: a host that serves a /// provisioned tier must never be designated the builder. The runtime /// hostname check in `build::run` is the real guard; this catches a /// misconfiguration (build_host pointed at a prod node's name/ssh_target) at /// startup, before the daemon ever accepts a build. Called from `main` once /// config + topology are both loaded. pub fn ensure_build_host_not_serving(&self, build_host: &str) -> Result<()> { for t in &self.tiers { if !t.provisioned || t.name.as_str() == "host" { continue; } for n in &t.nodes { if n.name.as_str() == build_host || n.ssh_target == build_host { anyhow::bail!( "build_host {build_host:?} is also node {} (ssh {}) in serving tier {} — \ the builder must not be a prod/serving node", n.name, n.ssh_target, t.name ); } } } Ok(()) } #[cfg(test)] pub(crate) fn validate_for_test(&self) -> Result<()> { self.validate() } /// Every `[[migration_check]]` in the daemon config must name a dump this /// topology declares. The two files are separate — daemon config is /// per-host, topology is per-project — so nothing but this catches a check /// pointing at a backup nobody fetches. Left uncaught it surfaces as a /// permanently `Blocked` gate the first time someone promotes, which reads /// like a missed fetch rather than a config typo. Called from `main` once /// both are loaded, and so under `--check-config`. pub fn ensure_migration_checks_have_backups( &self, checks: &[crate::config::MigrationCheck], ) -> Result<()> { // A product no tier dry-runs migrations for owes no dumps. `checks` is // never empty — the config defaults it to MNW's `server` entry — so // without this, an intake-only product with no postgres anywhere is // asked to declare a prod dump for a gate it does not configure. if !self .tiers .iter() .flat_map(|t| &t.gates) .any(|g| g.kind() == GateKind::MigrationDryRun) { return Ok(()); } for c in checks { anyhow::ensure!( self.backup_named(&c.backup).is_some(), "migration_check {} restores backup {:?}, which no [[backup]] in {} declares \ (have: {})", c.dir.display(), c.backup, "the topology", self.backup .iter() .map(|b| b.name.as_str()) .collect::>() .join(", "), ); } Ok(()) } /// Every `[[test_target]]` with an `aux_repo` must name a repo this topology /// checks out. Third of the cross-file checks, and the one whose absence has /// already cost coverage once: an unresolvable target is a warn-and-skip, by /// design, so that a config describing the tip can still build an older sha. /// That makes a typo here indistinguishable from a legitimate bisect skip — /// a green gate that ran one crate fewer than it says it does. pub fn ensure_test_target_aux_repos_exist( &self, targets: &[crate::config::TestTarget], ) -> Result<()> { for t in targets { let Some(name) = t.aux_repo.as_deref() else { continue; }; anyhow::ensure!( self.aux_repos.iter().any(|a| a.name == name), "test_target {} names aux_repo {:?}, which no [[aux_repo]] in the topology \ checks out, so the gate would skip it as absent (have: {})", t.label(), name, if self.aux_repos.is_empty() { "none".to_string() } else { self.aux_repos .iter() .map(|a| a.name.as_str()) .collect::>() .join(", ") }, ); } Ok(()) } /// Every `[[tier.node.companion]]` must name a companion the daemon config /// actually builds. Same two-file split as the migration checks above, and /// the same class of typo, but a worse landing: companions are installed /// AFTER the symlink swap (`deploy::deploy_remote`), so a name that stages /// nothing fails a promote with the server already live on the new version /// — `FailureStage::AtOrAfterSwap`, the case that needs a human to go look. /// Catching it at load turns that into a startup error on the build host. pub fn ensure_node_companions_are_built( &self, built: &[crate::config::Companion], ) -> Result<()> { for t in &self.tiers { for n in &t.nodes { for c in &n.companions { anyhow::ensure!( built.iter().any(|b| b.name == c.name), "tier {} node {} installs companion {:?}, which no [[companion]] in the \ daemon config builds, so nothing would be staged under \ companions/{} (have: {})", t.name, n.name, c.name, c.name, if built.is_empty() { "none".to_string() } else { built .iter() .map(|b| b.name.as_str()) .collect::>() .join(", ") }, ); } } } Ok(()) } /// The configured dump for `name`, or `None` when nothing declares it. pub fn backup_named(&self, name: &str) -> Option<&BackupConfig> { self.backup.iter().find(|b| b.name == name) } fn validate(&self) -> Result<()> { // A dump is only owed by a product that actually dry-runs migrations. // The unconditional form of this asserted something about every // product's tiers from a fact about one: pom configures no // `migration_dry_run` anywhere (it has no postgres schema at all), so // requiring it to declare a prod dump would be demanding a fixture for // a gate it never runs. let dry_runs_migrations = self .tiers .iter() .flat_map(|t| &t.gates) .any(|g| g.kind() == GateKind::MigrationDryRun); anyhow::ensure!( !dry_runs_migrations || !self.backup.is_empty(), "a tier configures migration_dry_run but the topology declares no [backup]; \ the gate would have nothing to restore" ); // page_smoke has one input and it is not optional. Caught here rather // than at promote time, because a gate that cannot run is a gate that // would be discovered red halfway through a deploy, which is the worst // moment to learn a URL is missing. for t in &self.tiers { let smokes = t.gates.iter().any(|g| g.kind() == GateKind::PageSmoke); anyhow::ensure!( !smokes || t.public_url.is_some(), "tier {} configures page_smoke but declares no public_url; the gate has to \ request the site the way a visitor does, and a URL derived from a node would \ reach the origin and miss the CDN it exists to watch", t.name, ); } for (i, b) in self.backup.iter().enumerate() { anyhow::ensure!( !b.name.is_empty() && b.name .bytes() .all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-'), "backup name {:?} must be non-empty and match [A-Za-z0-9_-]+; it keys the \ `backups` table and a daemon-config migration_check", b.name, ); anyhow::ensure!( !b.source.is_empty() && !b.local_path.is_empty(), "backup {} has an empty source/local_path", b.name, ); // Two dumps sharing a name would interleave in `backups`, so the // freshness check and the plausibility floor would each read the // other's row. Two sharing a `local_path` would overwrite each // other on disk, and whichever fetched last would be restored for // both — green, and proving nothing about one of the databases. for prior in &self.backup[..i] { anyhow::ensure!( prior.name != b.name, "two backup entries share the name {:?}", b.name, ); anyhow::ensure!( prior.local_path != b.local_path, "backups {:?} and {:?} share local_path {:?}; they would overwrite each other", prior.name, b.name, b.local_path, ); } } anyhow::ensure!( !self.tiers.is_empty(), "topology must declare at least one tier" ); for t in &self.tiers { // The `host` tier is the build tier (cargo_test / migration_dry_run / // code_smoke / boot_smoke run once on the host); every other tier serves an // artifact to nodes, so it is exempted from the node and // promotion-gate checks the same way. let is_build_tier = t.name.as_str() == "host"; if t.provisioned && t.nodes.is_empty() && !is_build_tier { anyhow::bail!("tier {} is provisioned but has no nodes", t.name); } // Fail closed by default: a provisioned serving tier must declare at // least one gate that actually guards a promote. Without this, an // empty (or build-time-only) gate list waves every promote through // because `unsatisfied_gates` finds nothing to check (CF1 root cause). if t.provisioned && !is_build_tier && !t.gates.iter().any(Gate::guards_promotion) { anyhow::bail!( "tier {} is provisioned to serve but declares no promotion gate \ (need at least one of node_health / burn_in / manual_confirm)", t.name ); } } let mut seen_dirs: Vec> = Vec::new(); for aux in &self.aux_repos { anyhow::ensure!( !aux.name.is_empty() && !aux.bare_path.is_empty() && !aux.branch.is_empty(), "aux_repo entry has an empty name/bare_path/branch" ); // `checkout_dir` becomes a `workdir.join(..)`. Nesting is allowed — // docengine lives at `Libraries/docengine` in the dev tree and the // path dep resolves to that shape — but every component must be a // plain name so an aux repo can never write outside the workdir. let dir = &aux.checkout_dir; let parts: Vec<&str> = dir.split('/').collect(); anyhow::ensure!( !dir.is_empty() && !dir.contains('\\') && parts .iter() .all(|c| !c.is_empty() && *c != "." && *c != ".."), "aux_repo {} has an unsafe checkout_dir {dir:?} (must be a relative path of \ plain components: no leading slash, empty segments, or dot-dot)", aux.name, ); // Two checkouts may not share a dir, and neither may sit inside the // other: `git worktree add` into a path under a live worktree buries // one checkout in the other's tree, and whichever builds second wins. for prior in &seen_dirs { let common = prior.len().min(parts.len()); anyhow::ensure!( prior[..common] != parts[..common], "two aux_repo entries share or nest checkout_dir {dir:?}; \ they would clobber each other" ); } seen_dirs.push(parts); } Ok(()) } } #[cfg(test)] mod tests { /// The shipped `sando.toml` must parse with the base-image declarations in /// it, and the values must be the ones measured on the boxes. A declaration /// that silently fails to deserialize is worse than none: the node would be /// treated as undeclared, skipped, and the deploy log would say so in a line /// nobody reads. #[test] fn the_shipped_topology_declares_what_each_node_is() { let raw = std::fs::read_to_string( std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml"), ) .expect("the repo's sando.toml must be readable from the daemon crate"); let topo: Topology = toml::from_str(&raw).expect("sando.toml must parse"); let nodes: Vec<&Node> = topo.tiers.iter().flat_map(|t| t.nodes.iter()).collect(); let by = |name: &str| { *nodes .iter() .find(|n| n.name.as_str() == name) .unwrap_or_else(|| panic!("`{name}` must be in the topology")) }; let image = |n: &Node| n.base_image.as_ref().map(ToString::to_string); // Measured 2026-08-25. Staging is deliberately NOT the same base as // production, and the assertion below says so out loud: tier A does not // rehearse tier B on the axis these fields exist for. let testnot = by("testnot-1"); assert_eq!(image(testnot).as_deref(), Some("ubuntu/26.04")); assert_eq!(testnot.libc.as_deref(), Some("2.43")); let prod = by("prod-1"); assert_eq!(image(prod).as_deref(), Some("ubuntu/24.04")); assert_eq!(prod.libc.as_deref(), Some("2.39")); assert_ne!( testnot.base_image, prod.base_image, "if these ever match, delete this assertion and the comment in \ sando.toml that explains why they do not" ); } use super::*; /// A minimal topology with one serving tier whose gate block is `gates`. fn topo_with_serving_gates(provisioned: bool, gates: &str) -> Topology { let raw = format!( r#" [repo] bare_path = "/tmp/repo.git" branch = "main" [backup] source = "ssh://prod/dump.sql.gz" local_path = "/tmp/dump.sql.gz" [[tier]] name = "b" provisioned = {provisioned} gates = [{gates}] [[tier.node]] name = "prod-1" ssh_target = "prod-1" release_root = "/srv/mnw" "# ); toml::from_str(&raw).expect("parse test topology") } #[test] fn page_smoke_without_a_public_url_is_rejected_at_load() { // The gate's whole value is that it requests the site the way a visitor // does. Without a URL it cannot, and a gate that cannot run must not be // discovered halfway through a promote. let topo = topo_with_serving_gates(true, r#"{ kind = "page_smoke" }"#); let err = topo.validate_for_test().expect_err("must refuse"); assert!( err.to_string().contains("public_url"), "error should name the missing field: {err}" ); } #[test] fn page_smoke_with_a_public_url_loads() { let raw = r#" [repo] bare_path = "/tmp/repo.git" branch = "main" [backup] source = "ssh://prod/dump.sql.gz" local_path = "/tmp/dump.sql.gz" [[tier]] name = "a" provisioned = true public_url = "https://testnot.work" gates = [{ kind = "page_smoke" }] [[tier.node]] name = "testnot-1" ssh_target = "testnot-1" release_root = "/srv/mnw" "#; let topo: Topology = toml::from_str(raw).expect("parse"); topo.validate_for_test().expect("valid"); assert_eq!( topo.tiers[0].public_url.as_deref(), Some("https://testnot.work") ); // It guards promotion out of its tier, and it runs after the deploy // rather than on the build host -- both are the point of it. assert!(topo.tiers[0].gates[0].guards_promotion()); assert!(topo.tiers[0].gates[0].runs_post_deploy()); } #[test] fn provisioned_serving_tier_with_no_gates_is_rejected() { let topo = topo_with_serving_gates(true, ""); let err = topo.validate_for_test().unwrap_err().to_string(); assert!(err.contains("no promotion gate"), "{err}"); } #[test] fn provisioned_serving_tier_with_only_build_gates_is_rejected() { // cargo_test / migration_dry_run are build-time and prove nothing about a // promote, so a serving tier carrying only them still fails closed. let topo = topo_with_serving_gates( true, r#"{ kind = "cargo_test" }, { kind = "migration_dry_run" }"#, ); let err = topo.validate_for_test().unwrap_err().to_string(); assert!(err.contains("no promotion gate"), "{err}"); } #[test] fn provisioned_serving_tier_with_a_promotion_gate_is_accepted() { let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#); assert!(topo.validate_for_test().is_ok()); } #[test] fn provisioned_serving_tier_with_only_boot_smoke_is_rejected() { // boot_smoke is a build-host gate now; it proves nothing about a node, so // a serving tier carrying only boot_smoke must fail closed exactly like an // empty gate list (Run-2 SERIOUS-3 structural close). let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#); let err = topo.validate_for_test().unwrap_err().to_string(); assert!(err.contains("no promotion gate"), "{err}"); } #[test] fn unprovisioned_tier_with_empty_gates_is_skipped() { // A declared-but-not-yet-provisioned tier (e.g. tier c) carries no // promote authority, so the gate requirement does not apply yet. let topo = topo_with_serving_gates(false, ""); assert!(topo.validate_for_test().is_ok()); } #[test] fn build_host_matching_a_serving_node_is_rejected() { // prod-1 is a node in the provisioned serving tier built above; naming it // as the builder must fail closed. let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#); let err = topo .ensure_build_host_not_serving("prod-1") .unwrap_err() .to_string(); assert!(err.contains("must not be a prod/serving node"), "{err}"); } #[test] fn build_host_distinct_from_serving_nodes_is_accepted() { let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#); assert!(topo.ensure_build_host_not_serving("fw13").is_ok()); } #[test] fn node_companions_default_empty_and_parse_when_present() { // A node without [[tier.node.companion]] is server-only. let plain = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#); assert!(plain.tiers[0].nodes[0].companions.is_empty()); // A node that declares a companion carries its install target + unit. let raw = r#" [repo] bare_path = "/tmp/repo.git" branch = "main" [backup] source = "s" local_path = "/tmp/d" [[tier]] name = "b" provisioned = true gates = [{ kind = "node_health" }] [[tier.node]] name = "prod-1" ssh_target = "makenotwork@alpha-west-1" release_root = "/opt/mnw" [[tier.node.companion]] name = "mnw-cli" install_path = "/opt/mnw-cli/mnw-cli" service_name = "mnw-cli.service" "#; let topo: Topology = toml::from_str(raw).expect("parse companion topology"); let c = &topo.tiers[0].nodes[0].companions; assert_eq!(c.len(), 1); assert_eq!(c[0].name, "mnw-cli"); assert_eq!(c[0].install_path, "/opt/mnw-cli/mnw-cli"); assert_eq!(c[0].service_name, "mnw-cli.service"); } /// A topology whose one node installs the named companions. fn topo_installing(names: &[&str]) -> Topology { let mut blocks = String::new(); for n in names { use std::fmt::Write; let _ = write!( blocks, "[[tier.node.companion]]\nname = \"{n}\"\n\ install_path = \"/opt/{n}/{n}\"\nservice_name = \"{n}.service\"\n" ); } let raw = format!( r#" [repo] bare_path = "/tmp/repo.git" branch = "main" [backup] source = "s" local_path = "/tmp/d" [[tier]] name = "b" provisioned = true gates = [{{ kind = "node_health" }}] [[tier.node]] name = "prod-1" ssh_target = "makenotwork@alpha-west-1" release_root = "/opt/mnw" {blocks}"# ); toml::from_str(&raw).expect("parse topology") } fn built(names: &[&str]) -> Vec { names .iter() .map(|n| crate::config::Companion { name: (*n).to_string(), manifest_dir: (*n).into(), bin: (*n).to_string(), }) .collect() } fn test_target(dir: &str, aux_repo: Option<&str>) -> crate::config::TestTarget { crate::config::TestTarget { dir: dir.into(), aux_repo: aux_repo.map(str::to_string), features: Vec::new(), all_features: false, scratch_db: false, } } #[test] fn a_test_target_naming_a_checked_out_aux_repo_is_accepted() { let topo = topo_with_aux( "[[aux_repo]]\nname = \"docengine\"\nbare_path = \"/tmp/d.git\"\n\ upstream = \"git@h:max/d.git\"\nbranch = \"main\"\ncheckout_dir = \"Libraries/docengine\"\n", ) .expect("parse"); assert!( topo.ensure_test_target_aux_repos_exist(&[ test_target("server", None), test_target("", Some("docengine")), ]) .is_ok() ); } #[test] fn a_test_target_naming_an_unknown_aux_repo_is_rejected_at_load() { // The failure this exists to prevent is silent: an unresolvable target // is a warn-and-skip (bisect), so the gate stays green having run one // crate fewer than the config claims. let topo = topo_with_aux( "[[aux_repo]]\nname = \"synckit\"\nbare_path = \"/tmp/s.git\"\n\ upstream = \"git@h:max/s.git\"\nbranch = \"main\"\ncheckout_dir = \"synckit\"\n", ) .expect("parse"); let err = topo .ensure_test_target_aux_repos_exist(&[test_target("", Some("docengine"))]) .unwrap_err() .to_string(); assert!(err.contains("docengine"), "{err}"); assert!(err.contains("have: synckit"), "{err}"); } #[test] fn test_targets_without_an_aux_repo_need_no_aux_repos_declared() { let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#); assert!( topo.ensure_test_target_aux_repos_exist(&[test_target("server", None)]) .is_ok() ); } #[test] fn a_node_companion_the_daemon_builds_is_accepted() { let topo = topo_installing(&["mnw-cli", "multithreaded"]); assert!( topo.ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"])) .is_ok() ); } #[test] fn a_node_companion_nothing_builds_is_rejected_at_load() { // Left uncaught this fails during the post-swap install on prod, with // the server already live on the new version. let topo = topo_installing(&["mnw-cli", "multithreadd"]); let err = topo .ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"])) .unwrap_err() .to_string(); assert!(err.contains("multithreadd"), "{err}"); assert!(err.contains("no [[companion]]"), "{err}"); // The message names what IS available, so the typo is obvious. assert!(err.contains("mnw-cli, multithreaded"), "{err}"); } #[test] fn a_node_companion_with_no_companions_configured_at_all_is_rejected() { let topo = topo_installing(&["multithreaded"]); let err = topo .ensure_node_companions_are_built(&[]) .unwrap_err() .to_string(); assert!(err.contains("have: none"), "{err}"); } #[test] fn a_topology_installing_no_companions_is_fine_with_none_built() { let topo = topo_installing(&[]); assert!(topo.ensure_node_companions_are_built(&[]).is_ok()); } fn topo_with_aux(aux_block: &str) -> Result { let raw = format!( r#" [repo] bare_path = "/tmp/repo.git" branch = "main" [backup] source = "s" local_path = "/tmp/d" [[tier]] name = "b" provisioned = true gates = [{{ kind = "node_health" }}] [[tier.node]] name = "prod-1" ssh_target = "prod-1" release_root = "/srv/mnw" {aux_block} "# ); let topo: Topology = toml::from_str(&raw)?; topo.validate_for_test()?; Ok(topo) } #[test] fn aux_repos_default_empty() { let topo = topo_with_aux("").expect("no aux_repo block is fine"); assert!(topo.aux_repos.is_empty()); } #[test] fn aux_repo_parses_all_fields() { let topo = topo_with_aux( r#" [[aux_repo]] name = "synckit" bare_path = "/srv/sando/synckit.git" upstream = "git@ssh.makenot.work:max/synckit.git" branch = "main" checkout_dir = "synckit""#, ) .expect("valid aux_repo parses"); assert_eq!(topo.aux_repos.len(), 1); let a = &topo.aux_repos[0]; assert_eq!(a.name, "synckit"); assert_eq!(a.bare_path, "/srv/sando/synckit.git"); assert_eq!(a.upstream, "git@ssh.makenot.work:max/synckit.git"); assert_eq!(a.branch, "main"); assert_eq!(a.checkout_dir, "synckit"); } #[test] fn aux_repo_with_nested_checkout_dir_is_accepted() { let topo = topo_with_aux( r#" [[aux_repo]] name = "docengine" bare_path = "/srv/sando/docengine.git" upstream = "git@ssh.makenot.work:max/docengine.git" branch = "main" checkout_dir = "Libraries/docengine""#, ) .expect("a nested checkout_dir is a valid location"); assert_eq!(topo.aux_repos[0].checkout_dir, "Libraries/docengine"); } #[test] fn aux_repo_with_traversing_checkout_dir_is_rejected() { for bad in [ "../escape", "a/../../escape", "a/./b", "a//b", "/abs", "a/", "..", ".", ] { let err = topo_with_aux(&format!( r#" [[aux_repo]] name = "x" bare_path = "/srv/sando/x.git" upstream = "u" branch = "main" checkout_dir = "{bad}""#, )) .unwrap_err() .to_string(); assert!(err.contains("unsafe checkout_dir"), "for {bad:?}: {err}"); } } #[test] fn aux_repos_sharing_a_checkout_dir_are_rejected() { let err = topo_with_aux( r#" [[aux_repo]] name = "one" bare_path = "/srv/sando/one.git" upstream = "u" branch = "main" checkout_dir = "shared" [[aux_repo]] name = "two" bare_path = "/srv/sando/two.git" upstream = "u" branch = "main" checkout_dir = "shared""#, ) .unwrap_err() .to_string(); assert!(err.contains("share or nest checkout_dir"), "{err}"); } #[test] fn aux_repo_nested_inside_another_checkout_dir_is_rejected() { let err = topo_with_aux( r#" [[aux_repo]] name = "outer" bare_path = "/srv/sando/outer.git" upstream = "u" branch = "main" checkout_dir = "Libraries" [[aux_repo]] name = "inner" bare_path = "/srv/sando/inner.git" upstream = "u" branch = "main" checkout_dir = "Libraries/docengine""#, ) .unwrap_err() .to_string(); assert!(err.contains("share or nest checkout_dir"), "{err}"); } /// A topology whose `[backup]`/`[[backup]]` section is `backup_block`. fn topo_with_backup_block(backup_block: &str) -> Result { let raw = format!( r#" [repo] bare_path = "/tmp/repo.git" branch = "main" {backup_block} [[tier]] name = "b" provisioned = true # migration_dry_run is what makes the backup rules apply at all: a product no # tier dry-runs migrations for owes no dumps, so a fixture exercising those rules # has to configure the gate. gates = [{{ kind = "node_health" }}, {{ kind = "migration_dry_run" }}] [[tier.node]] name = "prod-1" ssh_target = "prod-1" release_root = "/srv/mnw" "# ); let topo: Topology = toml::from_str(&raw)?; topo.validate_for_test()?; Ok(topo) } /// A topology with no `[backup]` at all, so the dump rules are exercised /// by what its gates ask for rather than by what it declares. fn topo_without_backup(gates: &str) -> Result { let raw = format!( r#" backup = [] [repo] bare_path = "/tmp/repo.git" branch = "main" [[tier]] name = "b" provisioned = true gates = [{gates}] [[tier.node]] name = "prod-1" ssh_target = "prod-1" release_root = "/srv/mnw" "# ); let topo: Topology = toml::from_str(&raw)?; topo.validate_for_test()?; Ok(topo) } #[test] fn a_product_that_never_dry_runs_migrations_owes_no_dump() { // pom is this product: no postgres schema, so no tier configures // migration_dry_run and demanding a prod dump would be demanding a // fixture for a gate that never runs. let topo = topo_without_backup(r#"{ kind = "node_health" }"#) .expect("a topology with no migration gate loads without a [backup]"); assert!(topo.backup.is_empty()); } #[test] fn a_migration_dry_run_with_no_backup_is_rejected_at_load() { // The gate restores a dump into the scratch database. With nothing // declared it would have nothing to restore, and the discovery would // come mid-promote. let err = topo_without_backup(r#"{ kind = "node_health" }, { kind = "migration_dry_run" }"#) .expect_err("a dry-run gate with no dump declared must not load") .to_string(); assert!(err.contains("declares no [backup]"), "{err}"); } #[test] fn a_single_backup_table_still_parses_as_one_named_server() { // Back-compat is the point: every deployed sando.toml uses the single // `[backup]` form, and the box this config lives on is the one whose job // is deploying — it must not need an edit to start. let topo = topo_with_backup_block( r#" [backup] source = "ssh://prod/dump.sql.gz" local_path = "/tmp/dump.sql.gz""#, ) .expect("the single-table form must still load"); assert_eq!(topo.backup.len(), 1); assert_eq!(topo.backup[0].name, "server"); assert!(topo.backup_named("server").is_some()); } #[test] fn a_backup_list_parses_and_keeps_its_names() { let topo = topo_with_backup_block( r#" [[backup]] name = "server" source = "ssh://prod/makenotwork/latest.sql.gz" local_path = "/tmp/server.sql.gz" [[backup]] name = "multithreaded" source = "ssh://prod/multithreaded/latest.sql.gz" local_path = "/tmp/mt.sql.gz""#, ) .expect("the list form must load"); assert_eq!(topo.backup.len(), 2); assert_eq!( topo.backup_named("multithreaded").unwrap().local_path, "/tmp/mt.sql.gz" ); assert!(topo.backup_named("nope").is_none()); } #[test] fn two_backups_sharing_a_name_are_rejected() { // They would interleave in `backups`, so the freshness check and the // plausibility floor would each read the other's row. let err = topo_with_backup_block( r#" [[backup]] name = "server" source = "a" local_path = "/tmp/a.sql.gz" [[backup]] name = "server" source = "b" local_path = "/tmp/b.sql.gz""#, ) .unwrap_err() .to_string(); assert!(err.contains("share the name"), "{err}"); } #[test] fn two_backups_sharing_a_local_path_are_rejected() { // Whichever fetched last would be restored for both checks — green, and // proving nothing about one of the two databases. let err = topo_with_backup_block( r#" [[backup]] name = "server" source = "a" local_path = "/tmp/same.sql.gz" [[backup]] name = "multithreaded" source = "b" local_path = "/tmp/same.sql.gz""#, ) .unwrap_err() .to_string(); assert!(err.contains("share local_path"), "{err}"); } #[test] fn a_migration_check_naming_an_undeclared_backup_is_rejected_at_startup() { // Daemon config and topology are separate files, so nothing but this // cross-check catches the typo. Uncaught it surfaces as a permanently // Blocked gate on the next promote, which reads like a missed fetch. let topo = topo_with_backup_block( r#" [backup] source = "s" local_path = "/tmp/d""#, ) .unwrap(); let checks = vec![crate::config::MigrationCheck { dir: std::path::PathBuf::from("multithreaded/migrations"), backup: "multithreaded".into(), scratch_db: Some("sando_scratch_mt".into()), owner_role: Some("multithreaded".into()), }]; let err = topo .ensure_migration_checks_have_backups(&checks) .unwrap_err() .to_string(); assert!(err.contains("which no [[backup]]"), "{err}"); // And the shipped pair agree, which is the case that actually ships. let shipped_checks = crate::config::default_migration_checks_for_test(); shipped() .ensure_migration_checks_have_backups(&shipped_checks) .expect("the default server check resolves against the shipped topology"); } #[test] fn real_sando_toml_loads_clean() { // The shipped topology must satisfy the invariant — guards against a // regression that would lock sandod out of its own config. let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml"); Topology::load(&path).expect("shipped sando.toml must validate"); } fn shipped() -> Topology { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml"); Topology::load(&path).expect("shipped sando.toml must validate") } #[test] fn shipping_to_the_last_provisioned_tier_needs_an_operator_signoff() { // A tier's gates guard promotion *out* of it, which is the subtlety that // made manual_confirm inert: it sat on tier b, guarding b -> c, and c is // not provisioned. So the ship to production was cleared by node_health // + burn_in alone — and `hotfix: true` skips burn_in. // // The gate that matters therefore belongs on the PREDECESSOR of the last // provisioned tier. Asserted structurally so re-provisioning tiers cannot // silently strand the sign-off again. let topo = shipped(); let last = topo .tiers .iter() .rposition(|t| t.provisioned) .expect("some tier must be provisioned"); assert!(last > 0, "the production tier cannot be the first tier"); let guard = &topo.tiers[last - 1]; assert!( guard.gates.iter().any(|g| matches!(g, Gate::ManualConfirm)), "tier {} guards promotion into the last provisioned tier ({}), so it must require an operator sign-off; its gates are {:?}", guard.name, topo.tiers[last].name, guard .gates .iter() .map(|g| g.kind().as_str()) .collect::>(), ); } #[test] fn every_serving_node_has_a_readiness_probe() { // Without health_url, node_health degrades to `systemctl is-active`, // which a crash-looping binary satisfies between restarts — exactly what // the 0.10.14 CDN_BASE_URL crash-loop did on prod-1. let topo = shipped(); for tier in topo.tiers.iter().filter(|t| t.provisioned) { for node in &tier.nodes { assert!( node.health_url.is_some(), "node {} on tier {} has no health_url, so node_health proves only that systemd thinks the unit is running", node.name, tier.name, ); } } } }