Skip to main content

max / makenotwork

25.0 KB · 672 lines History Blame Raw
1 use crate::domain::{GateKind, NodeId, TierId};
2 use anyhow::{Context, Result};
3 use serde::{Deserialize, Serialize};
4 use std::path::Path;
5
6 #[derive(Debug, Clone, Serialize, Deserialize)]
7 pub struct Topology {
8 pub repo: RepoConfig,
9 pub backup: BackupConfig,
10 #[serde(rename = "tier")]
11 pub tiers: Vec<Tier>,
12 /// Extra repos to fetch and check out beside the main worktree before a
13 /// build, so a path dependency that reaches across the repo split resolves.
14 /// Empty (default) keeps an existing `sando.toml` working unedited. See
15 /// [`AuxRepo`] and [`crate::build::checkout_aux_repos`].
16 #[serde(default, rename = "aux_repo")]
17 pub aux_repos: Vec<AuxRepo>,
18 }
19
20 /// An auxiliary repo checked out beside the per-sha worktree so cross-repo path
21 /// dependencies resolve at build time.
22 ///
23 /// The concrete case (2026-07-24): `mnw-cli` — a companion built from the MNW
24 /// worktree — carries `synckit-client = { path = "../../synckit/synckit-client" }`
25 /// after synckit moved to its own repo. From the companion crate at
26 /// `<workdir>/<sha>/mnw-cli`, that path resolves to `<workdir>/synckit`, a sibling
27 /// of the per-sha worktree that Sando otherwise never creates, so the companion
28 /// build failed with "No such file or directory". An `aux_repo` named to land at
29 /// `checkout_dir = "synckit"` puts the synckit source exactly there.
30 ///
31 /// The checkout is at the FIXED `<workdir>/<checkout_dir>`, not per-sha: the path
32 /// dependency resolves to that spot regardless of the MNW sha, and builds
33 /// serialize, so a single shared checkout refreshed to `branch` HEAD each build
34 /// is correct. Because a path dep has no lockfile pin, "branch HEAD" is the honest
35 /// resolution — the same contract as the dev working copy.
36 #[derive(Debug, Clone, Serialize, Deserialize)]
37 pub struct AuxRepo {
38 /// Human label for logs and errors (e.g. `synckit`).
39 pub name: String,
40 /// Bare repo Sando fetches into and worktrees from, e.g.
41 /// `/srv/sando/synckit.git`. Auto-created (hookless) on first build.
42 pub bare_path: String,
43 /// Canonical git remote fetched before checkout. Like [`RepoConfig::upstream`]
44 /// but required here: an aux repo is pull-based (nobody pushes to its bare).
45 pub upstream: String,
46 /// Branch whose HEAD is checked out.
47 pub branch: String,
48 /// Where the worktree lands, relative to `cfg.workdir`. May name a nested
49 /// location (`Libraries/docengine`), because a path dep resolves to wherever
50 /// the dev tree keeps the crate and Sando has to match that shape. Every
51 /// component must be a plain name — no leading slash, no `..` — so the
52 /// checkout stays under the workdir.
53 pub checkout_dir: String,
54 }
55
56 #[derive(Debug, Clone, Serialize, Deserialize)]
57 pub struct RepoConfig {
58 pub bare_path: String,
59 pub branch: String,
60 /// Optional canonical git remote. When set, `/rebuild` fetches the deploy
61 /// branch from here into the bare repo before worktree-ing the target sha,
62 /// so a freshly-pushed commit is resolvable without anyone pushing to the
63 /// bare repo directly (pull-based deploys). When unset, Sando relies on the
64 /// bare repo already containing the sha (push-based / hook-driven).
65 #[serde(default)]
66 pub upstream: Option<String>,
67 }
68
69 #[derive(Debug, Clone, Serialize, Deserialize)]
70 pub struct BackupConfig {
71 pub source: String,
72 pub local_path: String,
73 }
74
75 #[derive(Debug, Clone, Serialize, Deserialize)]
76 pub struct Tier {
77 pub name: TierId,
78 #[serde(default)]
79 pub provisioned: bool,
80 pub gates: Vec<Gate>,
81 #[serde(default)]
82 pub canary: CanaryPolicy,
83 #[serde(default, rename = "node")]
84 pub nodes: Vec<Node>,
85 }
86
87 #[derive(Debug, Clone, Serialize, Deserialize)]
88 pub struct Node {
89 pub name: NodeId,
90 pub ssh_target: String,
91 pub release_root: String,
92 /// systemd unit name to reload-or-restart after the symlink swap.
93 /// Defaults to "makenotwork.service" because that's MNW's prod unit.
94 #[serde(default = "default_service_name")]
95 pub service_name: String,
96 /// Opt-in config-drift guard. When set to the node's env-file path (e.g.
97 /// `/etc/mnw/makenotwork.env`), the deploy sources that file and runs the
98 /// freshly-rsynced binary in `MNW_CHECK_CONFIG=1` mode BEFORE the symlink
99 /// swap. A required var missing on this node (a var added upstream but never
100 /// added to the node's env — how testnot crash-looped on CDN_BASE_URL) then
101 /// fails the promote with the running service still intact, instead of after
102 /// the swap+restart. Unset (default) skips the check, so an existing
103 /// `sando.toml` keeps working and it never runs against a binary too old to
104 /// support the mode; enable it once a check-capable version is deployed.
105 #[serde(default)]
106 pub config_check_env_file: Option<String>,
107 /// Capability grant for this node's executor (see `ops_exec`). Defaults to
108 /// the current behavior of every Sando node — actuate deploy+restart,
109 /// observe health — so an existing `sando.toml` keeps working unedited.
110 #[serde(default = "default_actuate")]
111 pub actuate: Vec<String>,
112 #[serde(default = "default_observe")]
113 pub observe: Vec<String>,
114 /// Optional HTTP readiness URL the `node_health` gate curls on the node (over
115 /// its executor) after `systemctl is-active`. Typically a loopback address
116 /// the service binds, e.g. `http://127.0.0.1:8080/health`. When unset, the
117 /// gate proves the unit is active post-restart but does not HTTP-probe; set
118 /// it for a full readiness check. Kept optional so an existing `sando.toml`
119 /// needs no edit.
120 #[serde(default)]
121 pub health_url: Option<String>,
122 /// Companion services this node installs from the release bundle after the
123 /// server is swapped and restarted. Each entry names a `[[companion]]` the
124 /// daemon builds (see `Config::companions`) and says where its binary lands
125 /// and which unit to restart — so a contract-coupled service (mnw-cli) ships
126 /// in the same promote as the server instead of drifting. Empty (default) =
127 /// server-only node, so an existing `sando.toml` keeps working unedited.
128 #[serde(default, rename = "companion")]
129 pub companions: Vec<NodeCompanion>,
130 }
131
132 /// Where a node installs a built companion binary and which unit to bounce.
133 #[derive(Debug, Clone, Serialize, Deserialize)]
134 pub struct NodeCompanion {
135 /// Must match a `Config::companions[].name` — the bundle subdir to read from.
136 pub name: String,
137 /// Absolute path the companion binary is installed to on the node
138 /// (the unit's `ExecStart`), e.g. `/opt/mnw-cli/mnw-cli`.
139 pub install_path: String,
140 /// systemd unit restarted after the binary is installed, e.g.
141 /// `mnw-cli.service`.
142 pub service_name: String,
143 }
144
145 fn default_service_name() -> String {
146 "makenotwork.service".into()
147 }
148
149 /// The capability set every pre-existing Sando node implicitly had: it deploys
150 /// and restarts, and is health-observed. Keeping these as the defaults is what
151 /// lets `sando.toml` stay unchanged through the executor refactor.
152 pub fn default_actuate() -> Vec<String> {
153 vec!["deploy".into(), "restart".into()]
154 }
155 pub fn default_observe() -> Vec<String> {
156 vec!["health".into()]
157 }
158
159 #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
160 #[serde(rename_all = "snake_case")]
161 pub enum CanaryPolicy {
162 #[default]
163 Sequential,
164 Parallel,
165 }
166
167 impl CanaryPolicy {
168 pub fn as_str(self) -> &'static str {
169 match self {
170 CanaryPolicy::Sequential => "sequential",
171 CanaryPolicy::Parallel => "parallel",
172 }
173 }
174 }
175
176 #[derive(Debug, Clone, Serialize, Deserialize)]
177 #[serde(tag = "kind", rename_all = "snake_case")]
178 pub enum Gate {
179 CargoTest,
180 HardeningTest,
181 Clippy,
182 Fmt,
183 CargoAudit,
184 CargoDeny,
185 MigrationDryRun,
186 CodeSmoke,
187 BootSmoke,
188 NodeHealth,
189 BurnIn { hours: u32 },
190 ManualConfirm,
191 }
192
193 impl Gate {
194 /// The discriminant — the identifier we use in events, schema columns,
195 /// and the TUI. Gate parameters (e.g. `BurnIn.hours`) stay with `Gate`
196 /// and are not carried into `gate_runs` history.
197 pub fn kind(&self) -> GateKind {
198 match self {
199 Gate::CargoTest => GateKind::CargoTest,
200 Gate::HardeningTest => GateKind::HardeningTest,
201 Gate::Clippy => GateKind::Clippy,
202 Gate::Fmt => GateKind::Fmt,
203 Gate::CargoAudit => GateKind::CargoAudit,
204 Gate::CargoDeny => GateKind::CargoDeny,
205 Gate::MigrationDryRun => GateKind::MigrationDryRun,
206 Gate::CodeSmoke => GateKind::CodeSmoke,
207 Gate::BootSmoke => GateKind::BootSmoke,
208 Gate::NodeHealth => GateKind::NodeHealth,
209 Gate::BurnIn { .. } => GateKind::BurnIn,
210 Gate::ManualConfirm => GateKind::ManualConfirm,
211 }
212 }
213
214 /// Gates that execute against a tier's freshly-deployed nodes, recording a
215 /// `gate_runs` row, at the end of a successful promote to that tier — the
216 /// evidence the *next* promote checks. Only `node_health`: it probes the
217 /// nodes the deploy just shipped to. `boot_smoke` is NOT post-deploy — it is
218 /// a build-time gate that boots the staged artifact on the build host and
219 /// proves nothing about a deployed node (the Run-2 SERIOUS-3 blind spot:
220 /// boot_smoke used to re-run locally at promote time and the next promote
221 /// trusted a binary that never touched the node).
222 pub fn runs_post_deploy(&self) -> bool {
223 matches!(self, Gate::NodeHealth)
224 }
225
226 /// Gates evaluated at promote time against the deployed nodes or the
227 /// operator, as opposed to build-time gates (`cargo_test`,
228 /// `migration_dry_run`, `code_smoke`, `boot_smoke`) that run once on the build host and
229 /// prove nothing about a promote. A serving tier whose gate list contains
230 /// none of these would wave every promote straight through; `Topology
231 /// ::validate` refuses to load such a config (the structural form of CF1's
232 /// fail-closed default). `boot_smoke` no longer counts here — a serving tier
233 /// must declare a real node-level / operator gate, not a host smoke test.
234 pub fn guards_promotion(&self) -> bool {
235 matches!(
236 self,
237 Gate::NodeHealth | Gate::BurnIn { .. } | Gate::ManualConfirm
238 )
239 }
240 }
241
242 impl Topology {
243 pub fn load(path: &Path) -> Result<Self> {
244 let raw = std::fs::read_to_string(path)
245 .with_context(|| format!("reading topology at {}", path.display()))?;
246 let topo: Topology = toml::from_str(&raw)?;
247 topo.validate()?;
248 Ok(topo)
249 }
250
251 /// Defense-in-depth for the build-host guard: a host that serves a
252 /// provisioned tier must never be designated the builder. The runtime
253 /// hostname check in `build::run` is the real guard; this catches a
254 /// misconfiguration (build_host pointed at a prod node's name/ssh_target) at
255 /// startup, before the daemon ever accepts a build. Called from `main` once
256 /// config + topology are both loaded.
257 pub fn ensure_build_host_not_serving(&self, build_host: &str) -> Result<()> {
258 for t in &self.tiers {
259 if !t.provisioned || t.name.as_str() == "host" {
260 continue;
261 }
262 for n in &t.nodes {
263 if n.name.as_str() == build_host || n.ssh_target == build_host {
264 anyhow::bail!(
265 "build_host {build_host:?} is also node {} (ssh {}) in serving tier {} — \
266 the builder must not be a prod/serving node",
267 n.name,
268 n.ssh_target,
269 t.name
270 );
271 }
272 }
273 }
274 Ok(())
275 }
276
277 #[cfg(test)]
278 pub(crate) fn validate_for_test(&self) -> Result<()> {
279 self.validate()
280 }
281
282 fn validate(&self) -> Result<()> {
283 anyhow::ensure!(
284 !self.tiers.is_empty(),
285 "topology must declare at least one tier"
286 );
287 for t in &self.tiers {
288 // The `host` tier is the build tier (cargo_test / migration_dry_run /
289 // code_smoke / boot_smoke run once on the host); every other tier serves an
290 // artifact to nodes, so it is exempted from the node and
291 // promotion-gate checks the same way.
292 let is_build_tier = t.name.as_str() == "host";
293 if t.provisioned && t.nodes.is_empty() && !is_build_tier {
294 anyhow::bail!("tier {} is provisioned but has no nodes", t.name);
295 }
296 // Fail closed by default: a provisioned serving tier must declare at
297 // least one gate that actually guards a promote. Without this, an
298 // empty (or build-time-only) gate list waves every promote through
299 // because `unsatisfied_gates` finds nothing to check (CF1 root cause).
300 if t.provisioned && !is_build_tier && !t.gates.iter().any(Gate::guards_promotion) {
301 anyhow::bail!(
302 "tier {} is provisioned to serve but declares no promotion gate \
303 (need at least one of node_health / burn_in / manual_confirm)",
304 t.name
305 );
306 }
307 }
308 let mut seen_dirs: Vec<Vec<&str>> = Vec::new();
309 for aux in &self.aux_repos {
310 anyhow::ensure!(
311 !aux.name.is_empty() && !aux.bare_path.is_empty() && !aux.branch.is_empty(),
312 "aux_repo entry has an empty name/bare_path/branch"
313 );
314 // `checkout_dir` becomes a `workdir.join(..)`. Nesting is allowed —
315 // docengine lives at `Libraries/docengine` in the dev tree and the
316 // path dep resolves to that shape — but every component must be a
317 // plain name so an aux repo can never write outside the workdir.
318 let dir = &aux.checkout_dir;
319 let parts: Vec<&str> = dir.split('/').collect();
320 anyhow::ensure!(
321 !dir.is_empty()
322 && !dir.contains('\\')
323 && parts
324 .iter()
325 .all(|c| !c.is_empty() && *c != "." && *c != ".."),
326 "aux_repo {} has an unsafe checkout_dir {dir:?} (must be a relative path of \
327 plain components: no leading slash, empty segments, or dot-dot)",
328 aux.name,
329 );
330 // Two checkouts may not share a dir, and neither may sit inside the
331 // other: `git worktree add` into a path under a live worktree buries
332 // one checkout in the other's tree, and whichever builds second wins.
333 for prior in &seen_dirs {
334 let common = prior.len().min(parts.len());
335 anyhow::ensure!(
336 prior[..common] != parts[..common],
337 "two aux_repo entries share or nest checkout_dir {dir:?}; \
338 they would clobber each other"
339 );
340 }
341 seen_dirs.push(parts);
342 }
343 Ok(())
344 }
345 }
346
347 #[cfg(test)]
348 mod tests {
349 use super::*;
350
351 /// A minimal topology with one serving tier whose gate block is `gates`.
352 fn topo_with_serving_gates(provisioned: bool, gates: &str) -> Topology {
353 let raw = format!(
354 r#"
355 [repo]
356 bare_path = "/tmp/repo.git"
357 branch = "main"
358
359 [backup]
360 source = "ssh://prod/dump.sql.gz"
361 local_path = "/tmp/dump.sql.gz"
362
363 [[tier]]
364 name = "b"
365 provisioned = {provisioned}
366 gates = [{gates}]
367 [[tier.node]]
368 name = "prod-1"
369 ssh_target = "prod-1"
370 release_root = "/srv/mnw"
371 "#
372 );
373 toml::from_str(&raw).expect("parse test topology")
374 }
375
376 #[test]
377 fn provisioned_serving_tier_with_no_gates_is_rejected() {
378 let topo = topo_with_serving_gates(true, "");
379 let err = topo.validate_for_test().unwrap_err().to_string();
380 assert!(err.contains("no promotion gate"), "{err}");
381 }
382
383 #[test]
384 fn provisioned_serving_tier_with_only_build_gates_is_rejected() {
385 // cargo_test / migration_dry_run are build-time and prove nothing about a
386 // promote, so a serving tier carrying only them still fails closed.
387 let topo = topo_with_serving_gates(
388 true,
389 r#"{ kind = "cargo_test" }, { kind = "migration_dry_run" }"#,
390 );
391 let err = topo.validate_for_test().unwrap_err().to_string();
392 assert!(err.contains("no promotion gate"), "{err}");
393 }
394
395 #[test]
396 fn provisioned_serving_tier_with_a_promotion_gate_is_accepted() {
397 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
398 assert!(topo.validate_for_test().is_ok());
399 }
400
401 #[test]
402 fn provisioned_serving_tier_with_only_boot_smoke_is_rejected() {
403 // boot_smoke is a build-host gate now; it proves nothing about a node, so
404 // a serving tier carrying only boot_smoke must fail closed exactly like an
405 // empty gate list (Run-2 SERIOUS-3 structural close).
406 let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
407 let err = topo.validate_for_test().unwrap_err().to_string();
408 assert!(err.contains("no promotion gate"), "{err}");
409 }
410
411 #[test]
412 fn unprovisioned_tier_with_empty_gates_is_skipped() {
413 // A declared-but-not-yet-provisioned tier (e.g. tier c) carries no
414 // promote authority, so the gate requirement does not apply yet.
415 let topo = topo_with_serving_gates(false, "");
416 assert!(topo.validate_for_test().is_ok());
417 }
418
419 #[test]
420 fn build_host_matching_a_serving_node_is_rejected() {
421 // prod-1 is a node in the provisioned serving tier built above; naming it
422 // as the builder must fail closed.
423 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
424 let err = topo
425 .ensure_build_host_not_serving("prod-1")
426 .unwrap_err()
427 .to_string();
428 assert!(err.contains("must not be a prod/serving node"), "{err}");
429 }
430
431 #[test]
432 fn build_host_distinct_from_serving_nodes_is_accepted() {
433 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
434 assert!(topo.ensure_build_host_not_serving("fw13").is_ok());
435 }
436
437 #[test]
438 fn node_companions_default_empty_and_parse_when_present() {
439 // A node without [[tier.node.companion]] is server-only.
440 let plain = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
441 assert!(plain.tiers[0].nodes[0].companions.is_empty());
442
443 // A node that declares a companion carries its install target + unit.
444 let raw = r#"
445 [repo]
446 bare_path = "/tmp/repo.git"
447 branch = "main"
448 [backup]
449 source = "s"
450 local_path = "/tmp/d"
451 [[tier]]
452 name = "b"
453 provisioned = true
454 gates = [{ kind = "node_health" }]
455 [[tier.node]]
456 name = "prod-1"
457 ssh_target = "makenotwork@alpha-west-1"
458 release_root = "/opt/mnw"
459 [[tier.node.companion]]
460 name = "mnw-cli"
461 install_path = "/opt/mnw-cli/mnw-cli"
462 service_name = "mnw-cli.service"
463 "#;
464 let topo: Topology = toml::from_str(raw).expect("parse companion topology");
465 let c = &topo.tiers[0].nodes[0].companions;
466 assert_eq!(c.len(), 1);
467 assert_eq!(c[0].name, "mnw-cli");
468 assert_eq!(c[0].install_path, "/opt/mnw-cli/mnw-cli");
469 assert_eq!(c[0].service_name, "mnw-cli.service");
470 }
471
472 fn topo_with_aux(aux_block: &str) -> Result<Topology> {
473 let raw = format!(
474 r#"
475 [repo]
476 bare_path = "/tmp/repo.git"
477 branch = "main"
478 [backup]
479 source = "s"
480 local_path = "/tmp/d"
481 [[tier]]
482 name = "b"
483 provisioned = true
484 gates = [{{ kind = "node_health" }}]
485 [[tier.node]]
486 name = "prod-1"
487 ssh_target = "prod-1"
488 release_root = "/srv/mnw"
489 {aux_block}
490 "#
491 );
492 let topo: Topology = toml::from_str(&raw)?;
493 topo.validate_for_test()?;
494 Ok(topo)
495 }
496
497 #[test]
498 fn aux_repos_default_empty() {
499 let topo = topo_with_aux("").expect("no aux_repo block is fine");
500 assert!(topo.aux_repos.is_empty());
501 }
502
503 #[test]
504 fn aux_repo_parses_all_fields() {
505 let topo = topo_with_aux(
506 r#"
507 [[aux_repo]]
508 name = "synckit"
509 bare_path = "/srv/sando/synckit.git"
510 upstream = "git@ssh.makenot.work:max/synckit.git"
511 branch = "main"
512 checkout_dir = "synckit""#,
513 )
514 .expect("valid aux_repo parses");
515 assert_eq!(topo.aux_repos.len(), 1);
516 let a = &topo.aux_repos[0];
517 assert_eq!(a.name, "synckit");
518 assert_eq!(a.bare_path, "/srv/sando/synckit.git");
519 assert_eq!(a.upstream, "git@ssh.makenot.work:max/synckit.git");
520 assert_eq!(a.branch, "main");
521 assert_eq!(a.checkout_dir, "synckit");
522 }
523
524 #[test]
525 fn aux_repo_with_nested_checkout_dir_is_accepted() {
526 let topo = topo_with_aux(
527 r#"
528 [[aux_repo]]
529 name = "docengine"
530 bare_path = "/srv/sando/docengine.git"
531 upstream = "git@ssh.makenot.work:max/docengine.git"
532 branch = "main"
533 checkout_dir = "Libraries/docengine""#,
534 )
535 .expect("a nested checkout_dir is a valid location");
536 assert_eq!(topo.aux_repos[0].checkout_dir, "Libraries/docengine");
537 }
538
539 #[test]
540 fn aux_repo_with_traversing_checkout_dir_is_rejected() {
541 for bad in [
542 "../escape",
543 "a/../../escape",
544 "a/./b",
545 "a//b",
546 "/abs",
547 "a/",
548 "..",
549 ".",
550 ] {
551 let err = topo_with_aux(&format!(
552 r#"
553 [[aux_repo]]
554 name = "x"
555 bare_path = "/srv/sando/x.git"
556 upstream = "u"
557 branch = "main"
558 checkout_dir = "{bad}""#,
559 ))
560 .unwrap_err()
561 .to_string();
562 assert!(err.contains("unsafe checkout_dir"), "for {bad:?}: {err}");
563 }
564 }
565
566 #[test]
567 fn aux_repos_sharing_a_checkout_dir_are_rejected() {
568 let err = topo_with_aux(
569 r#"
570 [[aux_repo]]
571 name = "one"
572 bare_path = "/srv/sando/one.git"
573 upstream = "u"
574 branch = "main"
575 checkout_dir = "shared"
576 [[aux_repo]]
577 name = "two"
578 bare_path = "/srv/sando/two.git"
579 upstream = "u"
580 branch = "main"
581 checkout_dir = "shared""#,
582 )
583 .unwrap_err()
584 .to_string();
585 assert!(err.contains("share or nest checkout_dir"), "{err}");
586 }
587
588 #[test]
589 fn aux_repo_nested_inside_another_checkout_dir_is_rejected() {
590 let err = topo_with_aux(
591 r#"
592 [[aux_repo]]
593 name = "outer"
594 bare_path = "/srv/sando/outer.git"
595 upstream = "u"
596 branch = "main"
597 checkout_dir = "Libraries"
598 [[aux_repo]]
599 name = "inner"
600 bare_path = "/srv/sando/inner.git"
601 upstream = "u"
602 branch = "main"
603 checkout_dir = "Libraries/docengine""#,
604 )
605 .unwrap_err()
606 .to_string();
607 assert!(err.contains("share or nest checkout_dir"), "{err}");
608 }
609
610 #[test]
611 fn real_sando_toml_loads_clean() {
612 // The shipped topology must satisfy the invariant — guards against a
613 // regression that would lock sandod out of its own config.
614 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
615 Topology::load(&path).expect("shipped sando.toml must validate");
616 }
617
618 fn shipped() -> Topology {
619 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
620 Topology::load(&path).expect("shipped sando.toml must validate")
621 }
622
623 #[test]
624 fn shipping_to_the_last_provisioned_tier_needs_an_operator_signoff() {
625 // A tier's gates guard promotion *out* of it, which is the subtlety that
626 // made manual_confirm inert: it sat on tier b, guarding b -> c, and c is
627 // not provisioned. So the ship to production was cleared by node_health
628 // + burn_in alone — and `hotfix: true` skips burn_in.
629 //
630 // The gate that matters therefore belongs on the PREDECESSOR of the last
631 // provisioned tier. Asserted structurally so re-provisioning tiers cannot
632 // silently strand the sign-off again.
633 let topo = shipped();
634 let last = topo
635 .tiers
636 .iter()
637 .rposition(|t| t.provisioned)
638 .expect("some tier must be provisioned");
639 assert!(last > 0, "the production tier cannot be the first tier");
640 let guard = &topo.tiers[last - 1];
641 assert!(
642 guard.gates.iter().any(|g| matches!(g, Gate::ManualConfirm)),
643 "tier {} guards promotion into the last provisioned tier ({}), so it must require an operator sign-off; its gates are {:?}",
644 guard.name,
645 topo.tiers[last].name,
646 guard
647 .gates
648 .iter()
649 .map(|g| g.kind().as_str())
650 .collect::<Vec<_>>(),
651 );
652 }
653
654 #[test]
655 fn every_serving_node_has_a_readiness_probe() {
656 // Without health_url, node_health degrades to `systemctl is-active`,
657 // which a crash-looping binary satisfies between restarts — exactly what
658 // the 0.10.14 CDN_BASE_URL crash-loop did on prod-1.
659 let topo = shipped();
660 for tier in topo.tiers.iter().filter(|t| t.provisioned) {
661 for node in &tier.nodes {
662 assert!(
663 node.health_url.is_some(),
664 "node {} on tier {} has no health_url, so node_health proves only that systemd thinks the unit is running",
665 node.name,
666 tier.name,
667 );
668 }
669 }
670 }
671 }
672