Skip to main content

max / makenotwork

22.0 KB · 730 lines History Blame Raw
1 //! Tests for [`super`].
2
3 /// The shipped `sando.toml` must parse with the base-image declarations in
4 /// it, and the values must be the ones measured on the boxes. A declaration
5 /// that silently fails to deserialize is worse than none: the node would be
6 /// treated as undeclared, skipped, and the deploy log would say so in a line
7 /// nobody reads.
8 #[test]
9 fn the_shipped_topology_declares_what_each_node_is() {
10 let raw = std::fs::read_to_string(
11 std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml"),
12 )
13 .expect("the repo's sando.toml must be readable from the daemon crate");
14 let topo: Topology = toml::from_str(&raw).expect("sando.toml must parse");
15
16 let nodes: Vec<&Node> = topo.tiers.iter().flat_map(|t| t.nodes.iter()).collect();
17 let by = |name: &str| {
18 *nodes
19 .iter()
20 .find(|n| n.name.as_str() == name)
21 .unwrap_or_else(|| panic!("`{name}` must be in the topology"))
22 };
23 let image = |n: &Node| n.base_image.as_ref().map(ToString::to_string);
24
25 // Measured 2026-08-25. Staging is deliberately NOT the same base as
26 // production, and the assertion below says so out loud: tier A does not
27 // rehearse tier B on the axis these fields exist for.
28 let testnot = by("testnot-1");
29 assert_eq!(image(testnot).as_deref(), Some("ubuntu/26.04"));
30 assert_eq!(testnot.libc.as_deref(), Some("2.43"));
31
32 let prod = by("prod-1");
33 assert_eq!(image(prod).as_deref(), Some("ubuntu/24.04"));
34 assert_eq!(prod.libc.as_deref(), Some("2.39"));
35
36 assert_ne!(
37 testnot.base_image, prod.base_image,
38 "if these ever match, delete this assertion and the comment in \
39 sando.toml that explains why they do not"
40 );
41 }
42
43 use super::*;
44
45 /// A minimal topology with one serving tier whose gate block is `gates`.
46 fn topo_with_serving_gates(provisioned: bool, gates: &str) -> Topology {
47 let raw = format!(
48 r#"
49 [repo]
50 bare_path = "/tmp/repo.git"
51 branch = "main"
52
53 [backup]
54 source = "ssh://prod/dump.sql.gz"
55 local_path = "/tmp/dump.sql.gz"
56
57 [[tier]]
58 name = "b"
59 provisioned = {provisioned}
60 gates = [{gates}]
61 [[tier.node]]
62 name = "prod-1"
63 ssh_target = "prod-1"
64 release_root = "/srv/mnw"
65 "#
66 );
67 toml::from_str(&raw).expect("parse test topology")
68 }
69
70 #[test]
71 fn page_smoke_without_a_public_url_is_rejected_at_load() {
72 // The gate's whole value is that it requests the site the way a visitor
73 // does. Without a URL it cannot, and a gate that cannot run must not be
74 // discovered halfway through a promote.
75 let topo = topo_with_serving_gates(true, r#"{ kind = "page_smoke" }"#);
76 let err = topo.validate_for_test().expect_err("must refuse");
77 assert!(
78 err.to_string().contains("public_url"),
79 "error should name the missing field: {err}"
80 );
81 }
82
83 #[test]
84 fn page_smoke_with_a_public_url_loads() {
85 let raw = r#"
86 [repo]
87 bare_path = "/tmp/repo.git"
88 branch = "main"
89
90 [backup]
91 source = "ssh://prod/dump.sql.gz"
92 local_path = "/tmp/dump.sql.gz"
93
94 [[tier]]
95 name = "a"
96 provisioned = true
97 public_url = "https://testnot.work"
98 gates = [{ kind = "page_smoke" }]
99 [[tier.node]]
100 name = "testnot-1"
101 ssh_target = "testnot-1"
102 release_root = "/srv/mnw"
103 "#;
104 let topo: Topology = toml::from_str(raw).expect("parse");
105 topo.validate_for_test().expect("valid");
106 assert_eq!(
107 topo.tiers[0].public_url.as_deref(),
108 Some("https://testnot.work")
109 );
110 // It guards promotion out of its tier, and it runs after the deploy
111 // rather than on the build host -- both are the point of it.
112 assert!(topo.tiers[0].gates[0].guards_promotion());
113 assert!(topo.tiers[0].gates[0].runs_post_deploy());
114 }
115
116 #[test]
117 fn provisioned_serving_tier_with_no_gates_is_rejected() {
118 let topo = topo_with_serving_gates(true, "");
119 let err = topo.validate_for_test().unwrap_err().to_string();
120 assert!(err.contains("no promotion gate"), "{err}");
121 }
122
123 #[test]
124 fn provisioned_serving_tier_with_only_build_gates_is_rejected() {
125 // cargo_test / migration_dry_run are build-time and prove nothing about a
126 // promote, so a serving tier carrying only them still fails closed.
127 let topo = topo_with_serving_gates(
128 true,
129 r#"{ kind = "cargo_test" }, { kind = "migration_dry_run" }"#,
130 );
131 let err = topo.validate_for_test().unwrap_err().to_string();
132 assert!(err.contains("no promotion gate"), "{err}");
133 }
134
135 #[test]
136 fn provisioned_serving_tier_with_a_promotion_gate_is_accepted() {
137 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
138 assert!(topo.validate_for_test().is_ok());
139 }
140
141 #[test]
142 fn provisioned_serving_tier_with_only_boot_smoke_is_rejected() {
143 // boot_smoke is a build-host gate now; it proves nothing about a node, so
144 // a serving tier carrying only boot_smoke must fail closed exactly like an
145 // empty gate list (Run-2 SERIOUS-3 structural close).
146 let topo = topo_with_serving_gates(true, r#"{ kind = "boot_smoke" }"#);
147 let err = topo.validate_for_test().unwrap_err().to_string();
148 assert!(err.contains("no promotion gate"), "{err}");
149 }
150
151 #[test]
152 fn unprovisioned_tier_with_empty_gates_is_skipped() {
153 // A declared-but-not-yet-provisioned tier (e.g. tier c) carries no
154 // promote authority, so the gate requirement does not apply yet.
155 let topo = topo_with_serving_gates(false, "");
156 assert!(topo.validate_for_test().is_ok());
157 }
158
159 #[test]
160 fn build_host_matching_a_serving_node_is_rejected() {
161 // prod-1 is a node in the provisioned serving tier built above; naming it
162 // as the builder must fail closed.
163 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
164 let err = topo
165 .ensure_build_host_not_serving("prod-1")
166 .unwrap_err()
167 .to_string();
168 assert!(err.contains("must not be a prod/serving node"), "{err}");
169 }
170
171 #[test]
172 fn build_host_distinct_from_serving_nodes_is_accepted() {
173 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
174 assert!(topo.ensure_build_host_not_serving("fw13").is_ok());
175 }
176
177 #[test]
178 fn node_companions_default_empty_and_parse_when_present() {
179 // A node without [[tier.node.companion]] is server-only.
180 let plain = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
181 assert!(plain.tiers[0].nodes[0].companions.is_empty());
182
183 // A node that declares a companion carries its install target + unit.
184 let raw = r#"
185 [repo]
186 bare_path = "/tmp/repo.git"
187 branch = "main"
188 [backup]
189 source = "s"
190 local_path = "/tmp/d"
191 [[tier]]
192 name = "b"
193 provisioned = true
194 gates = [{ kind = "node_health" }]
195 [[tier.node]]
196 name = "prod-1"
197 ssh_target = "makenotwork@alpha-west-1"
198 release_root = "/opt/mnw"
199 [[tier.node.companion]]
200 name = "mnw-cli"
201 install_path = "/opt/mnw-cli/mnw-cli"
202 service_name = "mnw-cli.service"
203 "#;
204 let topo: Topology = toml::from_str(raw).expect("parse companion topology");
205 let c = &topo.tiers[0].nodes[0].companions;
206 assert_eq!(c.len(), 1);
207 assert_eq!(c[0].name, "mnw-cli");
208 assert_eq!(c[0].install_path, "/opt/mnw-cli/mnw-cli");
209 assert_eq!(c[0].service_name, "mnw-cli.service");
210 }
211
212 /// A topology whose one node installs the named companions.
213 fn topo_installing(names: &[&str]) -> Topology {
214 let mut blocks = String::new();
215 for n in names {
216 use std::fmt::Write;
217 let _ = write!(
218 blocks,
219 "[[tier.node.companion]]\nname = \"{n}\"\n\
220 install_path = \"/opt/{n}/{n}\"\nservice_name = \"{n}.service\"\n"
221 );
222 }
223 let raw = format!(
224 r#"
225 [repo]
226 bare_path = "/tmp/repo.git"
227 branch = "main"
228 [backup]
229 source = "s"
230 local_path = "/tmp/d"
231 [[tier]]
232 name = "b"
233 provisioned = true
234 gates = [{{ kind = "node_health" }}]
235 [[tier.node]]
236 name = "prod-1"
237 ssh_target = "makenotwork@alpha-west-1"
238 release_root = "/opt/mnw"
239 {blocks}"#
240 );
241 toml::from_str(&raw).expect("parse topology")
242 }
243
244 fn built(names: &[&str]) -> Vec<crate::config::Companion> {
245 names
246 .iter()
247 .map(|n| crate::config::Companion {
248 name: (*n).to_string(),
249 manifest_dir: (*n).into(),
250 bin: (*n).to_string(),
251 })
252 .collect()
253 }
254
255 fn test_target(dir: &str, aux_repo: Option<&str>) -> crate::config::TestTarget {
256 crate::config::TestTarget {
257 dir: dir.into(),
258 aux_repo: aux_repo.map(str::to_string),
259 features: Vec::new(),
260 all_features: false,
261 scratch_db: false,
262 }
263 }
264
265 #[test]
266 fn a_test_target_naming_a_checked_out_aux_repo_is_accepted() {
267 let topo = topo_with_aux(
268 "[[aux_repo]]\nname = \"docengine\"\nbare_path = \"/tmp/d.git\"\n\
269 upstream = \"git@h:max/d.git\"\nbranch = \"main\"\ncheckout_dir = \"Libraries/docengine\"\n",
270 )
271 .expect("parse");
272 assert!(
273 topo.ensure_test_target_aux_repos_exist(&[
274 test_target("server", None),
275 test_target("", Some("docengine")),
276 ])
277 .is_ok()
278 );
279 }
280
281 #[test]
282 fn a_test_target_naming_an_unknown_aux_repo_is_rejected_at_load() {
283 // The failure this exists to prevent is silent: an unresolvable target
284 // is a warn-and-skip (bisect), so the gate stays green having run one
285 // crate fewer than the config claims.
286 let topo = topo_with_aux(
287 "[[aux_repo]]\nname = \"synckit\"\nbare_path = \"/tmp/s.git\"\n\
288 upstream = \"git@h:max/s.git\"\nbranch = \"main\"\ncheckout_dir = \"synckit\"\n",
289 )
290 .expect("parse");
291 let err = topo
292 .ensure_test_target_aux_repos_exist(&[test_target("", Some("docengine"))])
293 .unwrap_err()
294 .to_string();
295 assert!(err.contains("docengine"), "{err}");
296 assert!(err.contains("have: synckit"), "{err}");
297 }
298
299 #[test]
300 fn test_targets_without_an_aux_repo_need_no_aux_repos_declared() {
301 let topo = topo_with_serving_gates(true, r#"{ kind = "node_health" }"#);
302 assert!(
303 topo.ensure_test_target_aux_repos_exist(&[test_target("server", None)])
304 .is_ok()
305 );
306 }
307
308 #[test]
309 fn a_node_companion_the_daemon_builds_is_accepted() {
310 let topo = topo_installing(&["mnw-cli", "multithreaded"]);
311 assert!(
312 topo.ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
313 .is_ok()
314 );
315 }
316
317 #[test]
318 fn a_node_companion_nothing_builds_is_rejected_at_load() {
319 // Left uncaught this fails during the post-swap install on prod, with
320 // the server already live on the new version.
321 let topo = topo_installing(&["mnw-cli", "multithreadd"]);
322 let err = topo
323 .ensure_node_companions_are_built(&built(&["mnw-cli", "multithreaded"]))
324 .unwrap_err()
325 .to_string();
326 assert!(err.contains("multithreadd"), "{err}");
327 assert!(err.contains("no [[companion]]"), "{err}");
328 // The message names what IS available, so the typo is obvious.
329 assert!(err.contains("mnw-cli, multithreaded"), "{err}");
330 }
331
332 #[test]
333 fn a_node_companion_with_no_companions_configured_at_all_is_rejected() {
334 let topo = topo_installing(&["multithreaded"]);
335 let err = topo
336 .ensure_node_companions_are_built(&[])
337 .unwrap_err()
338 .to_string();
339 assert!(err.contains("have: none"), "{err}");
340 }
341
342 #[test]
343 fn a_topology_installing_no_companions_is_fine_with_none_built() {
344 let topo = topo_installing(&[]);
345 assert!(topo.ensure_node_companions_are_built(&[]).is_ok());
346 }
347
348 fn topo_with_aux(aux_block: &str) -> Result<Topology> {
349 let raw = format!(
350 r#"
351 [repo]
352 bare_path = "/tmp/repo.git"
353 branch = "main"
354 [backup]
355 source = "s"
356 local_path = "/tmp/d"
357 [[tier]]
358 name = "b"
359 provisioned = true
360 gates = [{{ kind = "node_health" }}]
361 [[tier.node]]
362 name = "prod-1"
363 ssh_target = "prod-1"
364 release_root = "/srv/mnw"
365 {aux_block}
366 "#
367 );
368 let topo: Topology = toml::from_str(&raw)?;
369 topo.validate_for_test()?;
370 Ok(topo)
371 }
372
373 #[test]
374 fn aux_repos_default_empty() {
375 let topo = topo_with_aux("").expect("no aux_repo block is fine");
376 assert!(topo.aux_repos.is_empty());
377 }
378
379 #[test]
380 fn aux_repo_parses_all_fields() {
381 let topo = topo_with_aux(
382 r#"
383 [[aux_repo]]
384 name = "synckit"
385 bare_path = "/srv/sando/synckit.git"
386 upstream = "git@ssh.makenot.work:max/synckit.git"
387 branch = "main"
388 checkout_dir = "synckit""#,
389 )
390 .expect("valid aux_repo parses");
391 assert_eq!(topo.aux_repos.len(), 1);
392 let a = &topo.aux_repos[0];
393 assert_eq!(a.name, "synckit");
394 assert_eq!(a.bare_path, "/srv/sando/synckit.git");
395 assert_eq!(a.upstream, "git@ssh.makenot.work:max/synckit.git");
396 assert_eq!(a.branch, "main");
397 assert_eq!(a.checkout_dir, "synckit");
398 }
399
400 #[test]
401 fn aux_repo_with_nested_checkout_dir_is_accepted() {
402 let topo = topo_with_aux(
403 r#"
404 [[aux_repo]]
405 name = "docengine"
406 bare_path = "/srv/sando/docengine.git"
407 upstream = "git@ssh.makenot.work:max/docengine.git"
408 branch = "main"
409 checkout_dir = "Libraries/docengine""#,
410 )
411 .expect("a nested checkout_dir is a valid location");
412 assert_eq!(topo.aux_repos[0].checkout_dir, "Libraries/docengine");
413 }
414
415 #[test]
416 fn aux_repo_with_traversing_checkout_dir_is_rejected() {
417 for bad in [
418 "../escape",
419 "a/../../escape",
420 "a/./b",
421 "a//b",
422 "/abs",
423 "a/",
424 "..",
425 ".",
426 ] {
427 let err = topo_with_aux(&format!(
428 r#"
429 [[aux_repo]]
430 name = "x"
431 bare_path = "/srv/sando/x.git"
432 upstream = "u"
433 branch = "main"
434 checkout_dir = "{bad}""#,
435 ))
436 .unwrap_err()
437 .to_string();
438 assert!(err.contains("unsafe checkout_dir"), "for {bad:?}: {err}");
439 }
440 }
441
442 #[test]
443 fn aux_repos_sharing_a_checkout_dir_are_rejected() {
444 let err = topo_with_aux(
445 r#"
446 [[aux_repo]]
447 name = "one"
448 bare_path = "/srv/sando/one.git"
449 upstream = "u"
450 branch = "main"
451 checkout_dir = "shared"
452 [[aux_repo]]
453 name = "two"
454 bare_path = "/srv/sando/two.git"
455 upstream = "u"
456 branch = "main"
457 checkout_dir = "shared""#,
458 )
459 .unwrap_err()
460 .to_string();
461 assert!(err.contains("share or nest checkout_dir"), "{err}");
462 }
463
464 #[test]
465 fn aux_repo_nested_inside_another_checkout_dir_is_rejected() {
466 let err = topo_with_aux(
467 r#"
468 [[aux_repo]]
469 name = "outer"
470 bare_path = "/srv/sando/outer.git"
471 upstream = "u"
472 branch = "main"
473 checkout_dir = "Libraries"
474 [[aux_repo]]
475 name = "inner"
476 bare_path = "/srv/sando/inner.git"
477 upstream = "u"
478 branch = "main"
479 checkout_dir = "Libraries/docengine""#,
480 )
481 .unwrap_err()
482 .to_string();
483 assert!(err.contains("share or nest checkout_dir"), "{err}");
484 }
485
486 /// A topology whose `[backup]`/`[[backup]]` section is `backup_block`.
487 fn topo_with_backup_block(backup_block: &str) -> Result<Topology> {
488 let raw = format!(
489 r#"
490 [repo]
491 bare_path = "/tmp/repo.git"
492 branch = "main"
493 {backup_block}
494 [[tier]]
495 name = "b"
496 provisioned = true
497 # migration_dry_run is what makes the backup rules apply at all: a product no
498 # tier dry-runs migrations for owes no dumps, so a fixture exercising those rules
499 # has to configure the gate.
500 gates = [{{ kind = "node_health" }}, {{ kind = "migration_dry_run" }}]
501 [[tier.node]]
502 name = "prod-1"
503 ssh_target = "prod-1"
504 release_root = "/srv/mnw"
505 "#
506 );
507 let topo: Topology = toml::from_str(&raw)?;
508 topo.validate_for_test()?;
509 Ok(topo)
510 }
511
512 /// A topology with no `[backup]` at all, so the dump rules are exercised
513 /// by what its gates ask for rather than by what it declares.
514 fn topo_without_backup(gates: &str) -> Result<Topology> {
515 let raw = format!(
516 r#"
517 backup = []
518
519 [repo]
520 bare_path = "/tmp/repo.git"
521 branch = "main"
522
523 [[tier]]
524 name = "b"
525 provisioned = true
526 gates = [{gates}]
527 [[tier.node]]
528 name = "prod-1"
529 ssh_target = "prod-1"
530 release_root = "/srv/mnw"
531 "#
532 );
533 let topo: Topology = toml::from_str(&raw)?;
534 topo.validate_for_test()?;
535 Ok(topo)
536 }
537
538 #[test]
539 fn a_product_that_never_dry_runs_migrations_owes_no_dump() {
540 // pom is this product: no postgres schema, so no tier configures
541 // migration_dry_run and demanding a prod dump would be demanding a
542 // fixture for a gate that never runs.
543 let topo = topo_without_backup(r#"{ kind = "node_health" }"#)
544 .expect("a topology with no migration gate loads without a [backup]");
545 assert!(topo.backup.is_empty());
546 }
547
548 #[test]
549 fn a_migration_dry_run_with_no_backup_is_rejected_at_load() {
550 // The gate restores a dump into the scratch database. With nothing
551 // declared it would have nothing to restore, and the discovery would
552 // come mid-promote.
553 let err = topo_without_backup(r#"{ kind = "node_health" }, { kind = "migration_dry_run" }"#)
554 .expect_err("a dry-run gate with no dump declared must not load")
555 .to_string();
556 assert!(err.contains("declares no [backup]"), "{err}");
557 }
558
559 #[test]
560 fn a_single_backup_table_still_parses_as_one_named_server() {
561 // Back-compat is the point: every deployed sando.toml uses the single
562 // `[backup]` form, and the box this config lives on is the one whose job
563 // is deploying — it must not need an edit to start.
564 let topo = topo_with_backup_block(
565 r#"
566 [backup]
567 source = "ssh://prod/dump.sql.gz"
568 local_path = "/tmp/dump.sql.gz""#,
569 )
570 .expect("the single-table form must still load");
571 assert_eq!(topo.backup.len(), 1);
572 assert_eq!(topo.backup[0].name, "server");
573 assert!(topo.backup_named("server").is_some());
574 }
575
576 #[test]
577 fn a_backup_list_parses_and_keeps_its_names() {
578 let topo = topo_with_backup_block(
579 r#"
580 [[backup]]
581 name = "server"
582 source = "ssh://prod/makenotwork/latest.sql.gz"
583 local_path = "/tmp/server.sql.gz"
584 [[backup]]
585 name = "multithreaded"
586 source = "ssh://prod/multithreaded/latest.sql.gz"
587 local_path = "/tmp/mt.sql.gz""#,
588 )
589 .expect("the list form must load");
590 assert_eq!(topo.backup.len(), 2);
591 assert_eq!(
592 topo.backup_named("multithreaded").unwrap().local_path,
593 "/tmp/mt.sql.gz"
594 );
595 assert!(topo.backup_named("nope").is_none());
596 }
597
598 #[test]
599 fn two_backups_sharing_a_name_are_rejected() {
600 // They would interleave in `backups`, so the freshness check and the
601 // plausibility floor would each read the other's row.
602 let err = topo_with_backup_block(
603 r#"
604 [[backup]]
605 name = "server"
606 source = "a"
607 local_path = "/tmp/a.sql.gz"
608 [[backup]]
609 name = "server"
610 source = "b"
611 local_path = "/tmp/b.sql.gz""#,
612 )
613 .unwrap_err()
614 .to_string();
615 assert!(err.contains("share the name"), "{err}");
616 }
617
618 #[test]
619 fn two_backups_sharing_a_local_path_are_rejected() {
620 // Whichever fetched last would be restored for both checks — green, and
621 // proving nothing about one of the two databases.
622 let err = topo_with_backup_block(
623 r#"
624 [[backup]]
625 name = "server"
626 source = "a"
627 local_path = "/tmp/same.sql.gz"
628 [[backup]]
629 name = "multithreaded"
630 source = "b"
631 local_path = "/tmp/same.sql.gz""#,
632 )
633 .unwrap_err()
634 .to_string();
635 assert!(err.contains("share local_path"), "{err}");
636 }
637
638 #[test]
639 fn a_migration_check_naming_an_undeclared_backup_is_rejected_at_startup() {
640 // Daemon config and topology are separate files, so nothing but this
641 // cross-check catches the typo. Uncaught it surfaces as a permanently
642 // Blocked gate on the next promote, which reads like a missed fetch.
643 let topo = topo_with_backup_block(
644 r#"
645 [backup]
646 source = "s"
647 local_path = "/tmp/d""#,
648 )
649 .unwrap();
650 let checks = vec![crate::config::MigrationCheck {
651 dir: std::path::PathBuf::from("multithreaded/migrations"),
652 backup: "multithreaded".into(),
653 scratch_db: Some("sando_scratch_mt".into()),
654 owner_role: Some("multithreaded".into()),
655 }];
656 let err = topo
657 .ensure_migration_checks_have_backups(&checks)
658 .unwrap_err()
659 .to_string();
660 assert!(err.contains("which no [[backup]]"), "{err}");
661
662 // And the shipped pair agree, which is the case that actually ships.
663 let shipped_checks = crate::config::default_migration_checks_for_test();
664 shipped()
665 .ensure_migration_checks_have_backups(&shipped_checks)
666 .expect("the default server check resolves against the shipped topology");
667 }
668
669 #[test]
670 fn real_sando_toml_loads_clean() {
671 // The shipped topology must satisfy the invariant — guards against a
672 // regression that would lock sandod out of its own config.
673 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
674 Topology::load(&path).expect("shipped sando.toml must validate");
675 }
676
677 fn shipped() -> Topology {
678 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
679 Topology::load(&path).expect("shipped sando.toml must validate")
680 }
681
682 #[test]
683 fn shipping_to_the_last_provisioned_tier_needs_an_operator_signoff() {
684 // A tier's gates guard promotion *out* of it, which is the subtlety that
685 // made manual_confirm inert: it sat on tier b, guarding b -> c, and c is
686 // not provisioned. So the ship to production was cleared by node_health
687 // + burn_in alone — and `hotfix: true` skips burn_in.
688 //
689 // The gate that matters therefore belongs on the PREDECESSOR of the last
690 // provisioned tier. Asserted structurally so re-provisioning tiers cannot
691 // silently strand the sign-off again.
692 let topo = shipped();
693 let last = topo
694 .tiers
695 .iter()
696 .rposition(|t| t.provisioned)
697 .expect("some tier must be provisioned");
698 assert!(last > 0, "the production tier cannot be the first tier");
699 let guard = &topo.tiers[last - 1];
700 assert!(
701 guard.gates.iter().any(|g| matches!(g, Gate::ManualConfirm)),
702 "tier {} guards promotion into the last provisioned tier ({}), so it must require an operator sign-off; its gates are {:?}",
703 guard.name,
704 topo.tiers[last].name,
705 guard
706 .gates
707 .iter()
708 .map(|g| g.kind().as_str())
709 .collect::<Vec<_>>(),
710 );
711 }
712
713 #[test]
714 fn every_serving_node_has_a_readiness_probe() {
715 // Without health_url, node_health degrades to `systemctl is-active`,
716 // which a crash-looping binary satisfies between restarts — exactly what
717 // the 0.10.14 CDN_BASE_URL crash-loop did on prod-1.
718 let topo = shipped();
719 for tier in topo.tiers.iter().filter(|t| t.provisioned) {
720 for node in &tier.nodes {
721 assert!(
722 node.health_url.is_some(),
723 "node {} on tier {} has no health_url, so node_health proves only that systemd thinks the unit is running",
724 node.name,
725 tier.name,
726 );
727 }
728 }
729 }
730