Skip to main content

max / makenotwork

100.6 KB · 2994 lines History Blame Raw
1 //! Tests for [`super`].
2
3 use super::promotion::{
4 Evidence, PromotedBuild, RollbackReport, rollback_deployed_nodes, unsatisfied_gates,
5 };
6 use super::*;
7 use crate::config::AppConfig;
8 use crate::topology::{BackupConfig, CanaryPolicy, Gate, Node, RepoConfig, Tier, Topology};
9 use async_trait::async_trait;
10 use axum::body::Body;
11 use axum::http::{Request, StatusCode};
12 use http_body_util::BodyExt;
13 use ops_exec::{CapabilitySet, Executor, LogSink, RunOutput, Step, SyncOpts};
14 use sqlx::SqlitePool;
15 use sqlx::sqlite::SqlitePoolOptions;
16 use std::collections::BTreeMap;
17 use std::os::unix::process::ExitStatusExt;
18 use std::path::PathBuf;
19 use std::sync::{Arc, Mutex as StdMutex};
20 use tower::ServiceExt;
21
22 async fn fresh_pool() -> SqlitePool {
23 let pool = SqlitePoolOptions::new()
24 .max_connections(1)
25 .connect("sqlite::memory:")
26 .await
27 .unwrap();
28 sqlx::migrate!("./migrations").run(&pool).await.unwrap();
29 pool
30 }
31
32 /// Two-tier topology used by the route tests: mm (provisioned, no nodes)
33 /// → a (provisioned, one local node). Mirrors the production shape
34 /// without involving real ssh / postgres.
35 fn test_topo() -> Topology {
36 Topology {
37 repo: Some(RepoConfig {
38 bare_path: "/tmp/test.git".into(),
39 branch: "main".into(),
40 upstream: None,
41 }),
42 backup: vec![BackupConfig {
43 name: "server".into(),
44 source: "file:///tmp/test-backup.sql".into(),
45 local_path: "/tmp/local-backup.sql".into(),
46 }],
47 tiers: vec![
48 Tier {
49 public_url: None,
50 name: "host".into(),
51 provisioned: true,
52 gates: vec![],
53 canary: CanaryPolicy::Sequential,
54 nodes: vec![],
55 },
56 Tier {
57 public_url: None,
58 name: "a".into(),
59 provisioned: true,
60 gates: vec![Gate::BootSmoke],
61 canary: CanaryPolicy::Sequential,
62 nodes: vec![Node {
63 platform: None,
64 base_image: None,
65 libc: None,
66 name: "a-local".into(),
67 ssh_target: "local".into(),
68 release_root: "/tmp/a-node".into(),
69 service_name: "makenotwork.service".into(),
70 health_url: None,
71 config_check_env_file: None,
72 actuate: crate::topology::default_actuate(),
73 observe: crate::topology::default_observe(),
74 companions: Vec::new(),
75 }],
76 },
77 ],
78 aux_repos: Vec::new(),
79 }
80 }
81
82 fn test_cfg() -> AppConfig {
83 AppConfig {
84 page_smoke_cmd: None,
85 platform: None,
86 code_smoke_env: BTreeMap::default(),
87 id: crate::domain::AppId::default(),
88 topology_path: PathBuf::from("/tmp/test-sando.toml"),
89 build_host: Some("test-host".into()),
90 workdir: PathBuf::from("/tmp/sando-work"),
91 release_root: PathBuf::from("/tmp/sando-releases"),
92 scratch_db_url: None,
93 scratch_owner_role: "makenotwork".into(),
94 boot_smoke_port: 18181,
95 code_smoke_port: 18182,
96 bin_names: vec!["makenotwork".into()],
97 logs_root: PathBuf::from("/tmp/sando-logs"),
98 release_contents: vec![],
99 cargo_target_dir: None,
100 gate_timeout_secs: 2400,
101 companions: Vec::new(),
102 test_targets: vec![crate::config::TestTarget {
103 dir: PathBuf::from("server"),
104 aux_repo: None,
105 features: vec!["fast-tests".into()],
106 all_features: false,
107 scratch_db: true,
108 }],
109 migration_checks: vec![],
110 frontend_builds: vec![],
111 backup_max_age_hours: 48,
112 }
113 }
114
115 /// Two products mounted on one daemon address different state.
116 ///
117 /// The mount-per-product shape is what makes this true: each router carries
118 /// its own product's config, so `/apps/pom/state` cannot answer from MNW's
119 /// tiers even if a handler forgets the product exists. The root mount keeps
120 /// meaning the default product, which is what the runbook and the TUI call.
121 #[tokio::test]
122 async fn each_app_is_addressable_and_the_root_stays_the_default() {
123 let pool = fresh_pool().await;
124 // MNW ships host + a; pom ships one tier of its own, named differently
125 // so the response says which product answered.
126 for (app, tiers) in [("mnw", vec!["host", "a"]), ("pom", vec!["pom-host"])] {
127 for (i, name) in tiers.iter().enumerate() {
128 sqlx::query("INSERT INTO tiers (app, name, ord, provisioned) VALUES (?, ?, ?, 1)")
129 .bind(app)
130 .bind(name)
131 .bind(i as i64)
132 .execute(&pool)
133 .await
134 .unwrap();
135 sqlx::query("INSERT INTO tier_state (app, tier) VALUES (?, ?)")
136 .bind(app)
137 .bind(name)
138 .execute(&pool)
139 .await
140 .unwrap();
141 }
142 }
143
144 let mnw_topo = Arc::new(test_topo());
145 let mut pom_topo = test_topo();
146 pom_topo.tiers = vec![crate::topology::Tier {
147 public_url: None,
148 name: "pom-host".into(),
149 provisioned: true,
150 gates: vec![],
151 canary: crate::topology::CanaryPolicy::Sequential,
152 nodes: vec![],
153 }];
154 let pom_topo = Arc::new(pom_topo);
155
156 let mnw_cfg = Arc::new(test_cfg());
157 let mut pom = test_cfg();
158 pom.id = crate::domain::AppId::new("pom");
159 let pom_cfg = Arc::new(pom);
160
161 let mut apps = crate::state::AppMap::new();
162 for (id, cfg, topo) in [
163 (mnw_cfg.id.clone(), mnw_cfg.clone(), mnw_topo.clone()),
164 (pom_cfg.id.clone(), pom_cfg.clone(), pom_topo.clone()),
165 ] {
166 let executors = Arc::new(crate::state::build_executors(&topo));
167 apps.insert(
168 id,
169 Arc::new(crate::state::App {
170 cfg,
171 topo,
172 executors,
173 }),
174 );
175 }
176 let state = AppState {
177 pool,
178 apps: Arc::new(apps),
179 default_app: mnw_cfg.id.clone(),
180 topo: mnw_topo,
181 cfg: mnw_cfg,
182 active_build: Arc::new(tokio::sync::Mutex::new(None)),
183 deploy_lock: Arc::new(tokio::sync::Mutex::new(())),
184 events: crate::events::channel(),
185 executors: Arc::new(std::collections::HashMap::new()),
186 api_token: None,
187 };
188
189 let get = async |uri: &str| -> String {
190 let resp = router_for_apps(state.clone())
191 .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
192 .await
193 .unwrap();
194 assert_eq!(resp.status(), StatusCode::OK, "GET {uri}");
195 body_string(resp).await
196 };
197
198 // The root is the default product.
199 let root = get("/state").await;
200 assert!(root.contains("\"host\""), "root /state: {root}");
201 assert!(!root.contains("pom-host"), "root must not show pom: {root}");
202
203 // Each product answers under its own mount.
204 let mnw = get("/apps/mnw/state").await;
205 assert_eq!(mnw, root, "the default mount and the root are one product");
206 let pom = get("/apps/pom/state").await;
207 assert!(pom.contains("pom-host"), "/apps/pom/state: {pom}");
208 assert!(
209 !pom.contains("\"host\""),
210 "pom must not see mnw's tiers: {pom}"
211 );
212
213 // And the index says what is mounted.
214 let index = get("/apps").await;
215 assert!(
216 index.contains("\"mnw\"") && index.contains("\"pom\""),
217 "{index}"
218 );
219 assert!(index.contains("\"default_app\":\"mnw\""), "{index}");
220
221 // An unconfigured product is not a route.
222 let resp = router_for_apps(state.clone())
223 .oneshot(
224 Request::builder()
225 .uri("/apps/nope/state")
226 .body(Body::empty())
227 .unwrap(),
228 )
229 .await
230 .unwrap();
231 assert_eq!(resp.status(), StatusCode::NOT_FOUND);
232 }
233
234 async fn test_state() -> AppState {
235 let pool = fresh_pool().await;
236 // Seed tier rows so FKs on tier_state / gate_runs are satisfied.
237 for (i, name) in ["host", "a"].iter().enumerate() {
238 sqlx::query(
239 "INSERT INTO tiers (name, ord, provisioned, canary) VALUES (?, ?, 1, 'sequential')",
240 )
241 .bind(name)
242 .bind(i as i64)
243 .execute(&pool)
244 .await
245 .unwrap();
246 sqlx::query("INSERT INTO tier_state (tier) VALUES (?)")
247 .bind(name)
248 .execute(&pool)
249 .await
250 .unwrap();
251 }
252 // Don't call install_recorder in tests — it touches a process-global
253 // and conflicts when tests run in parallel.
254 let topo = test_topo();
255 let executors = Arc::new(crate::state::build_executors(&topo));
256 let topo = Arc::new(topo);
257 let cfg = Arc::new(test_cfg());
258 let (apps, default_app) = crate::state::one_app(cfg.clone(), topo.clone(), executors.clone());
259 AppState {
260 pool,
261 apps,
262 default_app,
263 topo,
264 cfg,
265 active_build: Arc::new(tokio::sync::Mutex::new(None)),
266 deploy_lock: Arc::new(tokio::sync::Mutex::new(())),
267 events: crate::events::channel(),
268 executors,
269 api_token: None,
270 }
271 }
272
273 async fn body_string(resp: axum::response::Response) -> String {
274 let bytes = resp.into_body().collect().await.unwrap().to_bytes();
275 String::from_utf8(bytes.to_vec()).unwrap()
276 }
277
278 /// Insert the FK prerequisites for inserting gate_runs/tier_state rows.
279 async fn seed(pool: &SqlitePool, tier: &str, version: &str) {
280 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES (?, 0, 1, 'sequential') ON CONFLICT DO NOTHING")
281 .bind(tier).execute(pool).await.unwrap();
282 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), '/tmp/x') ON CONFLICT DO NOTHING")
283 .bind(version).execute(pool).await.unwrap();
284 sqlx::query(
285 "INSERT INTO tier_state (tier, current_version) VALUES (?, NULL) ON CONFLICT DO NOTHING",
286 )
287 .bind(tier)
288 .execute(pool)
289 .await
290 .unwrap();
291 }
292
293 async fn insert_gate(pool: &SqlitePool, tier: &str, version: &str, kind: &str, passed: i64) {
294 let status = if passed == 1 { "passed" } else { "failed" };
295 sqlx::query(
296 "INSERT INTO gate_runs (version, tier, gate_kind, started_at, finished_at, status) \
297 VALUES (?, ?, ?, datetime('now'), datetime('now'), ?)",
298 )
299 .bind(version)
300 .bind(tier)
301 .bind(kind)
302 .bind(status)
303 .execute(pool)
304 .await
305 .unwrap();
306 }
307
308 /// `/state` says when a tier's artifact is gone, and stays quiet when it is
309 /// not.
310 ///
311 /// The gap this closes: `versions`/`build_runs` keep naming a path long
312 /// after gc removed the bytes, and nothing reconciled the two. Both times it
313 /// happened, the tier read green until an rsync failed mid-promote.
314 #[tokio::test]
315 async fn state_reports_a_referenced_artifact_whose_bytes_are_gone() {
316 let tmp = tempfile::tempdir().unwrap();
317 let present = tmp.path().join("releases").join("1111111111111111");
318 tokio::fs::create_dir_all(&present).await.unwrap();
319
320 let state = test_state().await;
321 let bin = state.cfg.primary_bin().to_string();
322 tokio::fs::write(present.join(&bin), b"bin").await.unwrap();
323
324 seed(&state.pool, "a", "0.11.20").await;
325 let build = seed_build(
326 &state.pool,
327 "2a53c900",
328 "0.11.20",
329 present.to_string_lossy().as_ref(),
330 )
331 .await;
332 sqlx::query("UPDATE tier_state SET current_version = ?, current_build_id = ? WHERE tier = 'a'")
333 .bind("0.11.20")
334 .bind(build)
335 .execute(&state.pool)
336 .await
337 .unwrap();
338
339 let tier_a = |v: &StateView| {
340 v.tiers
341 .iter()
342 .find(|t| t.name == "a")
343 .expect("tier a")
344 .missing_artifact
345 .clone()
346 };
347
348 assert_eq!(
349 tier_a(&state_view(&state).await.unwrap()),
350 None,
351 "the artifact is on disk; nothing to report"
352 );
353
354 // gc takes it, as it did twice in production. Nothing in the database
355 // changes, which is the whole defect.
356 tokio::fs::remove_dir_all(&present).await.unwrap();
357
358 let said = tier_a(&state_view(&state).await.unwrap()).expect("a missing-artifact report");
359 assert!(said.contains("0.11.20"), "{said}");
360 assert!(said.contains("current"), "{said}");
361 }
362
363 /// A rebuild at an unchanged version must not move what a tier reports.
364 ///
365 /// Gate rows keyed on (tier, version) let two runs of one version interleave,
366 /// with each gate showing whichever had written it last: two reads of the
367 /// same tier a minute apart disagree about whether `hardening_test` passed or
368 /// never ran.
369 #[tokio::test]
370 async fn state_reports_the_build_the_tier_is_running() {
371 let state = test_state().await;
372 seed(&state.pool, "a", "0.11.20").await;
373 let first = seed_build(&state.pool, "2a53c900", "0.11.20", "/rel/1111111111111111").await;
374 let second = seed_build(&state.pool, "adf56cd9", "0.11.20", "/rel/2222222222222222").await;
375 // The tier is running the first build, and it went green there.
376 insert_gate_build(&state.pool, "a", "0.11.20", "hardening_test", 1, first).await;
377 sqlx::query("UPDATE tier_state SET current_version = ?, current_build_id = ? WHERE tier = 'a'")
378 .bind("0.11.20")
379 .bind(first)
380 .execute(&state.pool)
381 .await
382 .unwrap();
383
384 let before = state_view(&state).await.unwrap();
385 let gates_of = |v: &StateView| {
386 v.tiers
387 .iter()
388 .find(|t| t.name == "a")
389 .unwrap()
390 .gates
391 .iter()
392 .map(|g| (g.kind.clone(), g.status.clone()))
393 .collect::<Vec<_>>()
394 };
395 assert_eq!(
396 gates_of(&before),
397 vec![("hardening_test".to_string(), Some("passed".to_string()))],
398 );
399
400 // A retry of the same version fails the same gate. The tier still runs
401 // the first build, so what it reports is unchanged.
402 insert_gate_build(&state.pool, "a", "0.11.20", "hardening_test", 0, second).await;
403 assert_eq!(
404 gates_of(&state_view(&state).await.unwrap()),
405 gates_of(&before),
406 "a sibling rebuild rewrote a tier it was never deployed to",
407 );
408 }
409
410 // ---- unsatisfied_gates ----
411
412 fn tid(s: &str) -> crate::domain::TierId {
413 crate::domain::TierId::new(s)
414 }
415
416 #[tokio::test]
417 async fn unsatisfied_gates_empty_when_no_configured_gates() {
418 // A tier that configures no gates has nothing to satisfy.
419 let pool = fresh_pool().await;
420 seed(&pool, "host", "0.8.12").await;
421 let pending = unsatisfied_gates(
422 &pool,
423 &crate::domain::AppId::default(),
424 &tid("host"),
425 &[],
426 &Evidence {
427 version: "0.8.12",
428 builds: &[],
429 tier_build: None,
430 },
431 false,
432 )
433 .await
434 .unwrap();
435 assert_eq!(pending, Vec::<String>::new());
436 }
437
438 #[tokio::test]
439 async fn unsatisfied_gates_flags_configured_gate_that_never_ran() {
440 // THE CF1 FIX: a configured gate with no gate_runs row is unsatisfied
441 // (fail closed), NOT silently treated as green. Before this, an A tier
442 // whose boot_smoke never executed exposed zero rows and waved promotion
443 // straight through to prod.
444 let pool = fresh_pool().await;
445 seed(&pool, "a", "0.8.12").await;
446 let pending = unsatisfied_gates(
447 &pool,
448 &crate::domain::AppId::default(),
449 &tid("a"),
450 &[Gate::BootSmoke],
451 &Evidence {
452 version: "0.8.12",
453 builds: &[],
454 tier_build: None,
455 },
456 false,
457 )
458 .await
459 .unwrap();
460 assert_eq!(pending, vec!["boot_smoke".to_string()]);
461 }
462
463 #[tokio::test]
464 async fn unsatisfied_gates_flags_failed_kind() {
465 let pool = fresh_pool().await;
466 seed(&pool, "host", "0.8.12").await;
467 insert_gate(&pool, "host", "0.8.12", "cargo_test", 0).await;
468 insert_gate(&pool, "host", "0.8.12", "boot_smoke", 1).await;
469 let pending = unsatisfied_gates(
470 &pool,
471 &crate::domain::AppId::default(),
472 &tid("host"),
473 &[Gate::CargoTest, Gate::BootSmoke],
474 &Evidence {
475 version: "0.8.12",
476 builds: &[],
477 tier_build: None,
478 },
479 false,
480 )
481 .await
482 .unwrap();
483 assert_eq!(pending, vec!["cargo_test".to_string()]);
484 }
485
486 #[tokio::test]
487 async fn unsatisfied_gates_latest_row_wins() {
488 // Two runs of the same gate; only the latest counts. A flap from
489 // red to green should clear the pending entry.
490 let pool = fresh_pool().await;
491 seed(&pool, "host", "0.8.12").await;
492 insert_gate(&pool, "host", "0.8.12", "cargo_test", 0).await;
493 insert_gate(&pool, "host", "0.8.12", "cargo_test", 1).await;
494 let pending = unsatisfied_gates(
495 &pool,
496 &crate::domain::AppId::default(),
497 &tid("host"),
498 &[Gate::CargoTest],
499 &Evidence {
500 version: "0.8.12",
501 builds: &[],
502 tier_build: None,
503 },
504 false,
505 )
506 .await
507 .unwrap();
508 assert!(pending.is_empty());
509 }
510
511 async fn insert_confirm(
512 pool: &SqlitePool,
513 tier: &str,
514 version: &str,
515 at: chrono::DateTime<chrono::Utc>,
516 ) {
517 sqlx::query(
518 "INSERT INTO gate_runs (version, tier, gate_kind, started_at, finished_at, status) \
519 VALUES (?, ?, 'manual_confirm', ?, ?, 'passed')",
520 )
521 .bind(version)
522 .bind(tier)
523 .bind(at.to_rfc3339())
524 .bind(at.to_rfc3339())
525 .execute(pool)
526 .await
527 .unwrap();
528 }
529
530 #[tokio::test]
531 async fn unsatisfied_gates_manual_confirm_requires_fresh_confirmation() {
532 // A confirmation only satisfies the gate if it post-dates the version's
533 // current landing on the tier (burn_in_started_at). A stale confirm left
534 // over from before a rollback + rollback-forward must NOT wave it through.
535 let pool = fresh_pool().await;
536 seed(&pool, "a", "0.8.12").await;
537 let landed = chrono::Utc::now();
538 sqlx::query("UPDATE tier_state SET burn_in_started_at = ? WHERE tier = 'a'")
539 .bind(landed.to_rfc3339())
540 .execute(&pool)
541 .await
542 .unwrap();
543
544 // Stale confirmation (recorded before this landing) -> unsatisfied.
545 insert_confirm(&pool, "a", "0.8.12", landed - chrono::Duration::hours(1)).await;
546 let pending = unsatisfied_gates(
547 &pool,
548 &crate::domain::AppId::default(),
549 &tid("a"),
550 &[Gate::ManualConfirm],
551 &Evidence {
552 version: "0.8.12",
553 builds: &[],
554 tier_build: None,
555 },
556 false,
557 )
558 .await
559 .unwrap();
560 assert_eq!(
561 pending,
562 vec!["manual_confirm".to_string()],
563 "stale confirm must not satisfy"
564 );
565
566 // Fresh confirmation (after this landing) -> satisfied.
567 insert_confirm(&pool, "a", "0.8.12", landed + chrono::Duration::minutes(5)).await;
568 let pending = unsatisfied_gates(
569 &pool,
570 &crate::domain::AppId::default(),
571 &tid("a"),
572 &[Gate::ManualConfirm],
573 &Evidence {
574 version: "0.8.12",
575 builds: &[],
576 tier_build: None,
577 },
578 false,
579 )
580 .await
581 .unwrap();
582 assert!(pending.is_empty(), "fresh confirm satisfies");
583 }
584
585 #[tokio::test]
586 async fn unsatisfied_gates_manual_confirm_fails_closed_without_baseline() {
587 // No landing clock (burn_in_started_at NULL) -> a passed confirm row is
588 // not provably fresh, so fail closed and require a new confirmation.
589 let pool = fresh_pool().await;
590 seed(&pool, "a", "0.8.12").await; // leaves burn_in_started_at NULL
591 insert_confirm(&pool, "a", "0.8.12", chrono::Utc::now()).await;
592 let pending = unsatisfied_gates(
593 &pool,
594 &crate::domain::AppId::default(),
595 &tid("a"),
596 &[Gate::ManualConfirm],
597 &Evidence {
598 version: "0.8.12",
599 builds: &[],
600 tier_build: None,
601 },
602 false,
603 )
604 .await
605 .unwrap();
606 assert_eq!(
607 pending,
608 vec!["manual_confirm".to_string()],
609 "no baseline -> fail closed"
610 );
611 }
612
613 #[tokio::test]
614 async fn unsatisfied_gates_hotfix_skips_only_burn_in() {
615 // burn_in is evaluated live (no clock started -> not elapsed); cargo_test
616 // has a failing row. Normal: both unsatisfied, in configured order.
617 // hotfix: burn_in suppressed, cargo_test still flagged. Lock the semantic
618 // so a future change doesn't widen the hotfix bypass.
619 let pool = fresh_pool().await;
620 seed(&pool, "a", "0.8.12").await;
621 insert_gate(&pool, "a", "0.8.12", "cargo_test", 0).await;
622 let gates = [Gate::BurnIn { hours: 48 }, Gate::CargoTest];
623
624 let normal = unsatisfied_gates(
625 &pool,
626 &crate::domain::AppId::default(),
627 &tid("a"),
628 &gates,
629 &Evidence {
630 version: "0.8.12",
631 builds: &[],
632 tier_build: None,
633 },
634 false,
635 )
636 .await
637 .unwrap();
638 assert_eq!(
639 normal,
640 vec!["burn_in".to_string(), "cargo_test".to_string()]
641 );
642
643 let with_hotfix = unsatisfied_gates(
644 &pool,
645 &crate::domain::AppId::default(),
646 &tid("a"),
647 &gates,
648 &Evidence {
649 version: "0.8.12",
650 builds: &[],
651 tier_build: None,
652 },
653 true,
654 )
655 .await
656 .unwrap();
657 assert_eq!(with_hotfix, vec!["cargo_test".to_string()]);
658 }
659
660 #[tokio::test]
661 async fn unsatisfied_gates_burn_in_passes_when_window_elapsed() {
662 // A burn-in clock started far enough in the past satisfies the gate
663 // live — no gate_runs row needed.
664 let pool = fresh_pool().await;
665 seed(&pool, "a", "0.8.12").await;
666 sqlx::query("UPDATE tier_state SET burn_in_started_at = ? WHERE tier = 'a'")
667 .bind((chrono::Utc::now() - chrono::Duration::hours(50)).to_rfc3339())
668 .execute(&pool)
669 .await
670 .unwrap();
671 let pending = unsatisfied_gates(
672 &pool,
673 &crate::domain::AppId::default(),
674 &tid("a"),
675 &[Gate::BurnIn { hours: 48 }],
676 &Evidence {
677 version: "0.8.12",
678 builds: &[],
679 tier_build: None,
680 },
681 false,
682 )
683 .await
684 .unwrap();
685 assert!(pending.is_empty(), "50h elapsed satisfies a 48h burn-in");
686 }
687
688 #[tokio::test]
689 async fn unsatisfied_gates_ignores_other_tiers_and_versions() {
690 let pool = fresh_pool().await;
691 seed(&pool, "host", "0.8.12").await;
692 seed(&pool, "host", "0.8.11").await;
693 seed(&pool, "a", "0.8.12").await;
694 // Mark host/0.8.12 cargo_test failing, but unrelated tiers/versions
695 // shouldn't pollute the query.
696 insert_gate(&pool, "host", "0.8.12", "cargo_test", 0).await;
697 insert_gate(&pool, "a", "0.8.12", "cargo_test", 0).await;
698 insert_gate(&pool, "host", "0.8.11", "cargo_test", 0).await;
699
700 let pending = unsatisfied_gates(
701 &pool,
702 &crate::domain::AppId::default(),
703 &tid("host"),
704 &[Gate::CargoTest],
705 &Evidence {
706 version: "0.8.12",
707 builds: &[],
708 tier_build: None,
709 },
710 false,
711 )
712 .await
713 .unwrap();
714 assert_eq!(pending, vec!["cargo_test".to_string()]);
715 }
716
717 #[tokio::test]
718 async fn unsatisfied_gates_null_status_is_treated_as_failing() {
719 // An in-flight gate (started_at set, finished_at + status NULL)
720 // should NOT be treated as green. Otherwise a race could promote
721 // before the gate concludes.
722 let pool = fresh_pool().await;
723 seed(&pool, "host", "0.8.12").await;
724 sqlx::query(
725 "INSERT INTO gate_runs (version, tier, gate_kind, started_at) \
726 VALUES ('0.8.12', 'host', 'cargo_test', datetime('now'))",
727 )
728 .execute(&pool)
729 .await
730 .unwrap();
731
732 let pending = unsatisfied_gates(
733 &pool,
734 &crate::domain::AppId::default(),
735 &tid("host"),
736 &[Gate::CargoTest],
737 &Evidence {
738 version: "0.8.12",
739 builds: &[],
740 tier_build: None,
741 },
742 false,
743 )
744 .await
745 .unwrap();
746 assert_eq!(pending, vec!["cargo_test".to_string()]);
747 }
748
749 // ---- /confirm/{tier} ----
750
751 #[tokio::test]
752 async fn confirm_rejects_when_tier_has_no_current_version() {
753 // tier_state.a.current_version is NULL by default. /confirm has
754 // nothing to confirm against → GateBlocked (400).
755 let state = test_state().await;
756 let app = router(state.clone());
757 let resp = app
758 .oneshot(
759 Request::builder()
760 .method("POST")
761 .uri("/confirm/a")
762 .body(Body::empty())
763 .unwrap(),
764 )
765 .await
766 .unwrap();
767 assert_eq!(resp.status(), StatusCode::CONFLICT);
768 let body = body_string(resp).await;
769 assert!(body.contains("no current_version"), "got: {body}");
770 }
771
772 #[tokio::test]
773 async fn confirm_accepts_when_current_version_set_and_inserts_row() {
774 let state = test_state().await;
775 // Seed a version + advance tier a's state to it.
776 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.8.12','sha',datetime('now'),'/tmp/x')")
777 .execute(&state.pool).await.unwrap();
778 sqlx::query("UPDATE tier_state SET current_version = '0.8.12' WHERE tier = 'a'")
779 .execute(&state.pool)
780 .await
781 .unwrap();
782
783 let app = router(state.clone());
784 let resp = app
785 .oneshot(
786 Request::builder()
787 .method("POST")
788 .uri("/confirm/a")
789 .body(Body::empty())
790 .unwrap(),
791 )
792 .await
793 .unwrap();
794 assert_eq!(resp.status(), StatusCode::OK);
795 let body = body_string(resp).await;
796 assert!(body.contains("\"tier\":\"a\""));
797 assert!(body.contains("\"version\":\"0.8.12\""));
798
799 // A passing gate_runs row was inserted.
800 let count: (i64,) = sqlx::query_as(
801 "SELECT COUNT(*) FROM gate_runs WHERE tier='a' AND gate_kind='manual_confirm' AND status='passed'",
802 )
803 .fetch_one(&state.pool)
804 .await
805 .unwrap();
806 assert_eq!(count.0, 1);
807 }
808
809 #[tokio::test]
810 async fn confirm_404s_for_unknown_tier() {
811 let state = test_state().await;
812 let app = router(state);
813 let resp = app
814 .oneshot(
815 Request::builder()
816 .method("POST")
817 .uri("/confirm/zzzz")
818 .body(Body::empty())
819 .unwrap(),
820 )
821 .await
822 .unwrap();
823 assert_eq!(resp.status(), StatusCode::NOT_FOUND);
824 }
825
826 #[tokio::test]
827 async fn get_run_404s_for_unknown_id() {
828 let state = test_state().await;
829 let app = router(state);
830 let resp = app
831 .oneshot(
832 Request::builder()
833 .uri("/runs/999")
834 .body(Body::empty())
835 .unwrap(),
836 )
837 .await
838 .unwrap();
839 assert_eq!(resp.status(), StatusCode::NOT_FOUND);
840 }
841
842 #[tokio::test]
843 async fn get_run_returns_view_with_gates() {
844 let state = test_state().await;
845 // A run that reached version 0.10.2 and ran two host gates (one red).
846 let run_id = crate::runs::create(&state.pool, &state.cfg.id, "abc1234def")
847 .await
848 .unwrap();
849 let ver: crate::domain::Version = "0.10.2".parse().unwrap();
850 seed(&state.pool, "host", "0.10.2").await;
851 crate::runs::set_version(&state.pool, run_id, &ver)
852 .await
853 .unwrap();
854 // Keyed on the run, not on the version: these are the rows that carry
855 // this run's build id, and a sibling rebuild of 0.10.2 writing its own
856 // rows must not change what this run reports.
857 insert_gate_build(&state.pool, "host", "0.10.2", "cargo_test", 0, run_id.0).await;
858 insert_gate_build(&state.pool, "host", "0.10.2", "boot_smoke", 1, run_id.0).await;
859
860 let app = router(state);
861 let resp = app
862 .oneshot(
863 Request::builder()
864 .uri(format!("/runs/{}", run_id.0))
865 .body(Body::empty())
866 .unwrap(),
867 )
868 .await
869 .unwrap();
870 assert_eq!(resp.status(), StatusCode::OK);
871 let v: serde_json::Value = serde_json::from_str(&body_string(resp).await).unwrap();
872 assert_eq!(v["run_id"], run_id.0);
873 assert_eq!(v["sha"], "abc1234def");
874 assert_eq!(v["version"], "0.10.2");
875 assert_eq!(v["result"], "building");
876 // Both host gates surface, latest-per-kind, alphabetized by kind.
877 assert_eq!(v["gates"].as_array().unwrap().len(), 2);
878 assert_eq!(v["gates"][0]["kind"], "boot_smoke");
879 assert_eq!(v["gates"][0]["status"], "passed");
880 assert_eq!(v["gates"][1]["kind"], "cargo_test");
881 assert_eq!(v["gates"][1]["status"], "failed");
882 }
883
884 #[tokio::test]
885 async fn get_run_ignores_a_sibling_rebuild_of_the_same_version() {
886 // A rebuild at an unchanged version is the normal way to retry a red
887 // build. Keyed on (tier, version), the two runs' rows interleaved and
888 // each gate reported whichever run wrote it last — a reader of /runs/{id}
889 // could watch a passed gate become "not run" a minute later with nothing
890 // touched. Runs 60-62 of mnw-server 0.11.20, 2026-08-19.
891 let state = test_state().await;
892 seed(&state.pool, "host", "0.11.20").await;
893 let ver: crate::domain::Version = "0.11.20".parse().unwrap();
894
895 let first = crate::runs::create(&state.pool, &state.cfg.id, "2a53c900")
896 .await
897 .unwrap();
898 crate::runs::set_version(&state.pool, first, &ver)
899 .await
900 .unwrap();
901 insert_gate_build(&state.pool, "host", "0.11.20", "cargo_deny", 0, first.0).await;
902
903 let second = crate::runs::create(&state.pool, &state.cfg.id, "adf56cd9")
904 .await
905 .unwrap();
906 crate::runs::set_version(&state.pool, second, &ver)
907 .await
908 .unwrap();
909 insert_gate_build(&state.pool, "host", "0.11.20", "cargo_deny", 1, second.0).await;
910
911 // The later run passing does not turn the earlier run green, and the
912 // earlier run failing does not follow the later one.
913 for (run, want) in [(first, "failed"), (second, "passed")] {
914 let app = router(state.clone());
915 let resp = app
916 .oneshot(
917 Request::builder()
918 .uri(format!("/runs/{}", run.0))
919 .body(Body::empty())
920 .unwrap(),
921 )
922 .await
923 .unwrap();
924 assert_eq!(resp.status(), StatusCode::OK);
925 let v: serde_json::Value = serde_json::from_str(&body_string(resp).await).unwrap();
926 assert_eq!(v["gates"].as_array().unwrap().len(), 1, "run {}", run.0);
927 assert_eq!(v["gates"][0]["kind"], "cargo_deny");
928 assert_eq!(v["gates"][0]["status"], want, "run {}", run.0);
929 }
930 }
931
932 #[tokio::test]
933 async fn get_run_wait_returns_immediately_when_settled() {
934 let state = test_state().await;
935 let run_id = crate::runs::create(&state.pool, &state.cfg.id, "abc1234def")
936 .await
937 .unwrap();
938 crate::runs::mark_passed(&state.pool, run_id).await.unwrap();
939
940 let app = router(state);
941 // Generous timeout, but an already-settled run must not wait for it.
942 let resp = app
943 .oneshot(
944 Request::builder()
945 .uri(format!("/runs/{}/wait?timeout_ms=60000", run_id.0))
946 .body(Body::empty())
947 .unwrap(),
948 )
949 .await
950 .unwrap();
951 assert_eq!(resp.status(), StatusCode::OK);
952 let v: serde_json::Value = serde_json::from_str(&body_string(resp).await).unwrap();
953 assert_eq!(v["result"], "passed");
954 }
955
956 #[tokio::test]
957 async fn get_run_wait_returns_building_at_timeout() {
958 let state = test_state().await;
959 let run_id = crate::runs::create(&state.pool, &state.cfg.id, "abc1234def")
960 .await
961 .unwrap();
962
963 let app = router(state);
964 // timeout_ms=0 → deadline is now → the first poll returns the
965 // still-building run rather than blocking.
966 let resp = app
967 .oneshot(
968 Request::builder()
969 .uri(format!("/runs/{}/wait?timeout_ms=0", run_id.0))
970 .body(Body::empty())
971 .unwrap(),
972 )
973 .await
974 .unwrap();
975 assert_eq!(resp.status(), StatusCode::OK);
976 let v: serde_json::Value = serde_json::from_str(&body_string(resp).await).unwrap();
977 assert_eq!(v["result"], "building");
978 }
979
980 #[tokio::test]
981 async fn get_run_wait_404s_for_unknown_id() {
982 let state = test_state().await;
983 let app = router(state);
984 let resp = app
985 .oneshot(
986 Request::builder()
987 .uri("/runs/999/wait?timeout_ms=0")
988 .body(Body::empty())
989 .unwrap(),
990 )
991 .await
992 .unwrap();
993 assert_eq!(resp.status(), StatusCode::NOT_FOUND);
994 }
995
996 #[test]
997 fn self_update_unit_maps_sha_to_instance() {
998 let sha = crate::domain::GitSha::parse("abc1234def5678").unwrap();
999 assert_eq!(
1000 self_update_unit(&sha),
1001 "sando-update@abc1234def5678.service"
1002 );
1003 }
1004
1005 #[tokio::test]
1006 async fn self_update_rejects_bad_sha_with_400() {
1007 // A malformed sha is a client error and must be rejected *before* any
1008 // privileged unit is triggered (so this test never shells out).
1009 let state = test_state().await;
1010 let app = router(state);
1011 let resp = app
1012 .oneshot(
1013 Request::builder()
1014 .method("POST")
1015 .uri("/self-update")
1016 .header("Content-Type", "application/json")
1017 .body(Body::from(r#"{"sha":"not-a-sha!"}"#))
1018 .unwrap(),
1019 )
1020 .await
1021 .unwrap();
1022 assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
1023 }
1024
1025 // ---- /promote/{tier} default-version resolution ----
1026
1027 #[tokio::test]
1028 async fn promote_to_first_tier_is_rejected() {
1029 // tier 0 is host — you /rebuild, not /promote.
1030 let state = test_state().await;
1031 let app = router(state);
1032 let resp = app
1033 .oneshot(
1034 Request::builder()
1035 .method("POST")
1036 .uri("/promote/host")
1037 .body(Body::empty())
1038 .unwrap(),
1039 )
1040 .await
1041 .unwrap();
1042 assert_eq!(resp.status(), StatusCode::CONFLICT);
1043 let body = body_string(resp).await;
1044 assert!(
1045 body.contains("cannot /promote to the first tier"),
1046 "got: {body}"
1047 );
1048 }
1049
1050 #[tokio::test]
1051 async fn promote_without_body_and_no_predecessor_version_errors() {
1052 // tier a has no body version supplied AND its predecessor mm has
1053 // current_version=NULL. Should fail before any deploy.
1054 let state = test_state().await;
1055 let app = router(state);
1056 let resp = app
1057 .oneshot(
1058 Request::builder()
1059 .method("POST")
1060 .uri("/promote/a")
1061 .body(Body::empty())
1062 .unwrap(),
1063 )
1064 .await
1065 .unwrap();
1066 assert_eq!(resp.status(), StatusCode::CONFLICT);
1067 let body = body_string(resp).await;
1068 assert!(
1069 body.contains("no version specified") || body.contains("no current_version"),
1070 "got: {body}"
1071 );
1072 }
1073
1074 #[tokio::test]
1075 async fn promote_blocked_when_predecessor_gate_never_ran() {
1076 // End-to-end CF1: the host tier configures boot_smoke but it never ran
1077 // (no gate_runs row). Promoting host -> a must be GateBlocked, citing the
1078 // unsatisfied gate, instead of waving through on zero evidence. A real
1079 // `versions` row is present so the ONLY thing that can block is the gate.
1080 let pool = fresh_pool().await;
1081 for (i, name) in ["host", "a"].iter().enumerate() {
1082 sqlx::query(
1083 "INSERT INTO tiers (name, ord, provisioned, canary) VALUES (?, ?, 1, 'sequential')",
1084 )
1085 .bind(name)
1086 .bind(i as i64)
1087 .execute(&pool)
1088 .await
1089 .unwrap();
1090 sqlx::query("INSERT INTO tier_state (tier) VALUES (?)")
1091 .bind(name)
1092 .execute(&pool)
1093 .await
1094 .unwrap();
1095 }
1096 let mut topo = test_topo();
1097 topo.tiers[0].gates = vec![Gate::BootSmoke]; // host configures a gate...
1098 let executors = Arc::new(crate::state::build_executors(&topo));
1099 let topo = Arc::new(topo);
1100 let cfg = Arc::new(test_cfg());
1101 let (apps, default_app) = crate::state::one_app(cfg.clone(), topo.clone(), executors.clone());
1102 let state = AppState {
1103 pool,
1104 apps,
1105 default_app,
1106 topo,
1107 cfg,
1108 active_build: Arc::new(tokio::sync::Mutex::new(None)),
1109 deploy_lock: Arc::new(tokio::sync::Mutex::new(())),
1110 events: crate::events::channel(),
1111 executors,
1112 api_token: None,
1113 };
1114 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.8.12','sha',datetime('now'),'/tmp/x')")
1115 .execute(&state.pool).await.unwrap();
1116 sqlx::query("UPDATE tier_state SET current_version = '0.8.12' WHERE tier = 'host'")
1117 .execute(&state.pool)
1118 .await
1119 .unwrap();
1120
1121 let app = router(state);
1122 let resp = app
1123 .oneshot(
1124 Request::builder()
1125 .method("POST")
1126 .uri("/promote/a")
1127 .body(Body::empty())
1128 .unwrap(),
1129 )
1130 .await
1131 .unwrap();
1132 assert_eq!(resp.status(), StatusCode::CONFLICT);
1133 let body = body_string(resp).await;
1134 assert!(
1135 body.contains("boot_smoke"),
1136 "expected boot_smoke to block; got: {body}"
1137 );
1138 }
1139
1140 #[tokio::test]
1141 async fn migration_bearing_promote_requires_fresh_confirm() {
1142 // The predecessor (host) configures NO gates, so nothing would normally
1143 // block host -> a. A `bears_migration` promote must still be blocked on a
1144 // fresh `manual_confirm` it does not have: rollback restores the binary
1145 // only, so the one-way advance needs a conscious operator sign-off.
1146 let pool = fresh_pool().await;
1147 for (i, name) in ["host", "a"].iter().enumerate() {
1148 sqlx::query(
1149 "INSERT INTO tiers (name, ord, provisioned, canary) VALUES (?, ?, 1, 'sequential')",
1150 )
1151 .bind(name)
1152 .bind(i as i64)
1153 .execute(&pool)
1154 .await
1155 .unwrap();
1156 sqlx::query("INSERT INTO tier_state (tier) VALUES (?)")
1157 .bind(name)
1158 .execute(&pool)
1159 .await
1160 .unwrap();
1161 }
1162 let mut topo = test_topo();
1163 topo.tiers[0].gates = vec![]; // host configures no gates at all
1164 let executors = Arc::new(crate::state::build_executors(&topo));
1165 let topo = Arc::new(topo);
1166 let cfg = Arc::new(test_cfg());
1167 let (apps, default_app) = crate::state::one_app(cfg.clone(), topo.clone(), executors.clone());
1168 let state = AppState {
1169 pool,
1170 apps,
1171 default_app,
1172 topo,
1173 cfg,
1174 active_build: Arc::new(tokio::sync::Mutex::new(None)),
1175 deploy_lock: Arc::new(tokio::sync::Mutex::new(())),
1176 events: crate::events::channel(),
1177 executors,
1178 api_token: None,
1179 };
1180 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.8.12','sha',datetime('now'),'/tmp/x')")
1181 .execute(&state.pool).await.unwrap();
1182 sqlx::query("UPDATE tier_state SET current_version = '0.8.12' WHERE tier = 'host'")
1183 .execute(&state.pool)
1184 .await
1185 .unwrap();
1186
1187 let app = router(state);
1188 let resp = app
1189 .oneshot(
1190 Request::builder()
1191 .method("POST")
1192 .uri("/promote/a")
1193 .header("content-type", "application/json")
1194 .body(Body::from(r#"{"bears_migration": true}"#))
1195 .unwrap(),
1196 )
1197 .await
1198 .unwrap();
1199 assert_eq!(resp.status(), StatusCode::CONFLICT);
1200 let body = body_string(resp).await;
1201 assert!(
1202 body.contains("manual_confirm"),
1203 "expected the migration promote to block on manual_confirm; got: {body}"
1204 );
1205 }
1206
1207 #[tokio::test]
1208 async fn tier_state_advance_is_atomic_previous_from_old_current() {
1209 // CF3: the promote advance is a single UPDATE where previous_version is
1210 // set from the row's *old* current_version (SQLite evaluates RHS against
1211 // the original row). No read-modify-write to lose under concurrency.
1212 let pool = fresh_pool().await;
1213 seed(&pool, "a", "1.0.0").await;
1214 // current_version FKs into versions, so the target must exist too.
1215 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('2.0.0','sha',datetime('now'),'/tmp/x')")
1216 .execute(&pool).await.unwrap();
1217 sqlx::query("UPDATE tier_state SET current_version = '1.0.0' WHERE tier = 'a'")
1218 .execute(&pool)
1219 .await
1220 .unwrap();
1221
1222 // Exercise the sealed forward-advance primitive itself — the same op
1223 // /promote and the host build path both call (S1), not a copy of its SQL.
1224 let v = crate::domain::Version::parse("2.0.0").unwrap();
1225 crate::runs::advance_tier(&pool, &crate::domain::AppId::default(), "a", &v, None)
1226 .await
1227 .unwrap();
1228
1229 let (cur, prev): (Option<String>, Option<String>) =
1230 sqlx::query_as("SELECT current_version, previous_version FROM tier_state WHERE tier = 'a'")
1231 .fetch_one(&pool)
1232 .await
1233 .unwrap();
1234 assert_eq!(cur.as_deref(), Some("2.0.0"));
1235 assert_eq!(
1236 prev.as_deref(),
1237 Some("1.0.0"),
1238 "previous = the pre-update current, atomically"
1239 );
1240 }
1241
1242 #[tokio::test]
1243 async fn canary_rollback_restores_deployed_nodes_to_previous_version() {
1244 use crate::topology::{Node, default_actuate, default_observe};
1245 let tmp = tempfile::tempdir().unwrap();
1246
1247 // Two local nodes, each pre-seeded as if a promote had flipped them to
1248 // 2.0.0 (current -> releases/2.0.0), with the prior 1.0.0 still on disk.
1249 let mut nodes = Vec::new();
1250 for name in ["n1", "n2"] {
1251 let rr = tmp.path().join(name);
1252 for v in ["1.0.0", "2.0.0"] {
1253 tokio::fs::create_dir_all(rr.join("releases").join(v))
1254 .await
1255 .unwrap();
1256 }
1257 tokio::fs::symlink("releases/2.0.0", rr.join("current"))
1258 .await
1259 .unwrap();
1260 nodes.push(Node {
1261 platform: None,
1262 base_image: None,
1263 libc: None,
1264 name: name.into(),
1265 ssh_target: "local".into(),
1266 release_root: rr.to_string_lossy().into_owned(),
1267 service_name: "x.service".into(),
1268 health_url: None,
1269 config_check_env_file: None,
1270 actuate: default_actuate(),
1271 observe: default_observe(),
1272 companions: Vec::new(),
1273 });
1274 }
1275
1276 let mut state = test_state().await;
1277 // The rollback target needs a versions row. The release dir name comes
1278 // from the artifact_path's parent (legacy layout: releases/<version>).
1279 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('1.0.0','sha',datetime('now'),'/tmp/staged/releases/1.0.0/makenotwork')")
1280 .execute(&state.pool).await.unwrap();
1281 let execs: crate::state::ExecutorMap = nodes
1282 .iter()
1283 .map(|n| (n.name.clone(), crate::state::build_executor(n)))
1284 .collect();
1285 state.executors = std::sync::Arc::new(execs);
1286
1287 let refs: Vec<&Node> = nodes.iter().collect();
1288 let report = rollback_deployed_nodes(&state, &tid("a"), &refs, "1.0.0").await;
1289 assert_eq!(report.restored, 2, "both deployed nodes should be restored");
1290 assert!(
1291 report.is_consistent(),
1292 "nothing should be indeterminate: {report:?}"
1293 );
1294
1295 for n in &nodes {
1296 let cur = tokio::fs::read_link(std::path::Path::new(&n.release_root).join("current"))
1297 .await
1298 .unwrap();
1299 assert_eq!(
1300 cur.to_string_lossy(),
1301 "releases/1.0.0",
1302 "node {} rolled back",
1303 n.name
1304 );
1305 }
1306 }
1307
1308 #[tokio::test]
1309 async fn canary_rollback_is_noop_without_a_previous_artifact() {
1310 use crate::topology::{Node, default_actuate, default_observe};
1311 let tmp = tempfile::tempdir().unwrap();
1312 let rr = tmp.path().join("n1");
1313 tokio::fs::create_dir_all(&rr).await.unwrap();
1314 let node = Node {
1315 platform: None,
1316 base_image: None,
1317 libc: None,
1318 name: "n1".into(),
1319 ssh_target: "local".into(),
1320 release_root: rr.to_string_lossy().into_owned(),
1321 service_name: "x.service".into(),
1322 health_url: None,
1323 config_check_env_file: None,
1324 actuate: default_actuate(),
1325 observe: default_observe(),
1326 companions: Vec::new(),
1327 };
1328 let state = test_state().await; // no versions row for "9.9.9"
1329 let report = rollback_deployed_nodes(&state, &tid("a"), &[&node], "9.9.9").await;
1330 assert_eq!(
1331 report.restored, 0,
1332 "no artifact to roll back to -> nothing restored, no panic"
1333 );
1334 assert_eq!(
1335 report.touched(),
1336 1,
1337 "the node must be accounted for somewhere"
1338 );
1339 // No rollback could even be attempted, so the node's version is not
1340 // knowable here. That must read as indeterminate, not as safe.
1341 assert_eq!(report.indeterminate, 1);
1342 assert!(!report.is_consistent());
1343 }
1344
1345 /// A rollback that fails before the symlink swap leaves the node on the
1346 /// version it was already running, the one being rolled back to. Counting
1347 /// that as "not restored" would report `restored=0 of=1` and send an
1348 /// operator to inspect a production box that is entirely fine.
1349 #[test]
1350 fn a_rollback_that_failed_before_the_swap_is_not_an_incident() {
1351 let report = RollbackReport {
1352 restored: 0,
1353 already_on_previous: 1,
1354 indeterminate: 0,
1355 };
1356 assert_eq!(report.touched(), 1);
1357 assert!(
1358 report.is_consistent(),
1359 "a node that never left the previous version is not split-brain"
1360 );
1361
1362 // Contrast: the same zero restored, but the swap had run. This one does
1363 // warrant a human, and the two must not report the same way.
1364 let real = RollbackReport {
1365 restored: 0,
1366 already_on_previous: 0,
1367 indeterminate: 1,
1368 };
1369 assert_eq!(real.touched(), 1);
1370 assert!(!real.is_consistent());
1371 }
1372
1373 /// A genuine split-brain still reports as one: some nodes back on the old
1374 /// version, one stranded.
1375 #[test]
1376 fn a_mixed_outcome_is_inconsistent_if_any_node_is_unknown() {
1377 let report = RollbackReport {
1378 restored: 2,
1379 already_on_previous: 1,
1380 indeterminate: 1,
1381 };
1382 assert_eq!(report.touched(), 4);
1383 assert!(!report.is_consistent());
1384 }
1385
1386 // ---- FleetFake: a multi-node promote across recorded fake executors ----
1387 //
1388 // The route-level promote tests until now ran a single local node against a
1389 // real LocalExec, so the sequential-canary fan-out, the cross-node deploy
1390 // ordering, and the mid-canary rollback of already-flipped nodes had no
1391 // coverage. FleetFake records every deploy op (tagged by node) into one
1392 // shared log so ordering is visible across the fleet, and can fail any op
1393 // whose shell script or rsync target contains a marker — used to fail a
1394 // node's forward deploy of the new version while its rollback to the prior
1395 // version (a different `releases/<v>` path) still succeeds.
1396
1397 struct FleetFake {
1398 tag: String,
1399 caps: CapabilitySet,
1400 log: Arc<StdMutex<Vec<String>>>,
1401 fail_if_contains: Option<String>,
1402 }
1403
1404 impl FleetFake {
1405 fn fails(&self, text: &str) -> bool {
1406 self.fail_if_contains
1407 .as_deref()
1408 .is_some_and(|m| text.contains(m))
1409 }
1410 }
1411
1412 #[async_trait]
1413 impl Executor for FleetFake {
1414 async fn run_streaming(
1415 &self,
1416 step: &Step,
1417 _sink: &mut dyn LogSink,
1418 ) -> anyhow::Result<RunOutput> {
1419 let script = step.argv.last().cloned().unwrap_or_default();
1420 self.log
1421 .lock()
1422 .unwrap()
1423 .push(format!("{}:run:{script}", self.tag));
1424 Ok(RunOutput {
1425 status: std::process::ExitStatus::from_raw(if self.fails(&script) {
1426 1 << 8
1427 } else {
1428 0
1429 }),
1430 stdout: Vec::new(),
1431 stderr: Vec::new(),
1432 })
1433 }
1434 async fn pull_file(
1435 &self,
1436 _r: &std::path::Path,
1437 _l: &std::path::Path,
1438 _o: &SyncOpts,
1439 ) -> anyhow::Result<()> {
1440 Ok(())
1441 }
1442 async fn pull_dir(
1443 &self,
1444 _r: &std::path::Path,
1445 _l: &std::path::Path,
1446 _o: &SyncOpts,
1447 ) -> anyhow::Result<()> {
1448 Ok(())
1449 }
1450 async fn pull_glob(&self, _g: &str, _l: &std::path::Path, _o: &SyncOpts) -> anyhow::Result<()> {
1451 Ok(())
1452 }
1453 async fn push_dir(
1454 &self,
1455 _local: &std::path::Path,
1456 remote: &std::path::Path,
1457 _o: &SyncOpts,
1458 ) -> anyhow::Result<()> {
1459 let dst = remote.display().to_string();
1460 self.log
1461 .lock()
1462 .unwrap()
1463 .push(format!("{}:push:{dst}", self.tag));
1464 if self.fails(&dst) {
1465 anyhow::bail!("fake rsync failure on {}", self.tag);
1466 }
1467 Ok(())
1468 }
1469 fn capabilities(&self) -> &CapabilitySet {
1470 &self.caps
1471 }
1472 }
1473
1474 /// Rebuild tier "a" with `names` as remote fake nodes sharing one op log,
1475 /// seed the version/tier_state prerequisites for a promote of 3.0.0 up from
1476 /// `host` (tier "a" starts on 2.0.0 with 1.0.0 behind it), and optionally
1477 /// make `fail_node` fail any op containing `fail_marker`. Returns the state
1478 /// and the shared log.
1479 async fn fleet_fixture(
1480 names: &[&str],
1481 fail_node: Option<&str>,
1482 fail_marker: &str,
1483 ) -> (AppState, Arc<StdMutex<Vec<String>>>) {
1484 use crate::topology::{default_actuate, default_observe};
1485 let mut state = test_state().await;
1486 let log = Arc::new(StdMutex::new(Vec::<String>::new()));
1487
1488 let nodes: Vec<Node> = names
1489 .iter()
1490 .map(|name| Node {
1491 platform: None,
1492 base_image: None,
1493 libc: None,
1494 name: (*name).into(),
1495 ssh_target: format!("deploy@{name}"),
1496 release_root: format!("/tmp/fleet/{name}"),
1497 service_name: "makenotwork.service".into(),
1498 health_url: None,
1499 config_check_env_file: None,
1500 actuate: default_actuate(),
1501 observe: default_observe(),
1502 companions: Vec::new(),
1503 })
1504 .collect();
1505
1506 let mut topo = (*state.topo).clone();
1507 topo.tiers[1].nodes = nodes.clone();
1508 // Drop the tier's post-deploy gate: the deploy fan-out is the subject
1509 // here, and node_health would need its own probe wiring.
1510 topo.tiers[1].gates = vec![];
1511 state.topo = Arc::new(topo);
1512
1513 let execs: crate::state::ExecutorMap = nodes
1514 .iter()
1515 .map(|n| {
1516 let nm = n.name.to_string();
1517 let fail = fail_node == Some(nm.as_str());
1518 let exec: Arc<dyn Executor> = Arc::new(FleetFake {
1519 tag: nm,
1520 caps: CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1521 log: log.clone(),
1522 fail_if_contains: fail.then(|| fail_marker.to_string()),
1523 });
1524 (n.name.clone(), exec)
1525 })
1526 .collect();
1527 state.executors = Arc::new(execs);
1528
1529 for n in &nodes {
1530 // deploys.node FKs into `nodes`.
1531 sqlx::query(
1532 "INSERT INTO nodes (name, tier, ssh_target, release_root) VALUES (?, 'a', ?, ?)",
1533 )
1534 .bind(&n.name)
1535 .bind(&n.ssh_target)
1536 .bind(&n.release_root)
1537 .execute(&state.pool)
1538 .await
1539 .unwrap();
1540 }
1541 for v in ["1.0.0", "2.0.0", "3.0.0"] {
1542 // Legacy (pre-identity) artifact_path is `releases/<version>/<bin>`,
1543 // so the release dir the node mirrors is named for the version. The
1544 // fixture reflects that real layout (parent basename == version).
1545 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), ?)")
1546 .bind(v).bind(format!("/tmp/staged/releases/{v}/makenotwork")).execute(&state.pool).await.unwrap();
1547 }
1548 sqlx::query("UPDATE tier_state SET current_version = '3.0.0' WHERE tier = 'host'")
1549 .execute(&state.pool)
1550 .await
1551 .unwrap();
1552 sqlx::query(
1553 "UPDATE tier_state SET current_version = '2.0.0', previous_version = '1.0.0' WHERE tier = 'a'",
1554 )
1555 .execute(&state.pool)
1556 .await
1557 .unwrap();
1558 (state, log)
1559 }
1560
1561 /// Index of the first logged op belonging to `node` (panics if the node was
1562 /// never touched — the message names it).
1563 fn first_touch(log: &[String], node: &str) -> usize {
1564 let prefix = format!("{node}:");
1565 log.iter()
1566 .position(|e| e.starts_with(&prefix))
1567 .unwrap_or_else(|| panic!("node {node:?} was never deployed to; log: {log:#?}"))
1568 }
1569
1570 #[tokio::test]
1571 async fn promote_deploys_every_node_in_tier_order_and_advances() {
1572 let (state, log) = fleet_fixture(&["a1", "a2", "a3"], None, "").await;
1573 let pool = state.pool.clone();
1574
1575 let Json(body) = promote_inner(state, "a".into(), PromoteBody::default())
1576 .await
1577 .expect("all nodes deploy, so the promote succeeds");
1578 assert_eq!(
1579 body["nodes_deployed"],
1580 serde_json::json!(["a1", "a2", "a3"]),
1581 "the response names every node the promote reached",
1582 );
1583
1584 let (cur, prev) = tier_versions(&pool, "a").await;
1585 assert_eq!(cur.as_deref(), Some("3.0.0"));
1586 assert_eq!(prev.as_deref(), Some("2.0.0"));
1587
1588 // Sequential canary: a1 is fully touched before a2, a2 before a3.
1589 let log = log.lock().unwrap().clone();
1590 assert!(
1591 first_touch(&log, "a1") < first_touch(&log, "a2")
1592 && first_touch(&log, "a2") < first_touch(&log, "a3"),
1593 "nodes must deploy in tier order: {log:#?}",
1594 );
1595
1596 // Every node has a green deploy row for the promoted version.
1597 let ok: i64 = sqlx::query_scalar(
1598 "SELECT COUNT(*) FROM deploys WHERE version = '3.0.0' AND outcome = 'ok'",
1599 )
1600 .fetch_one(&pool)
1601 .await
1602 .unwrap();
1603 assert_eq!(ok, 3, "one ok deploy row per node");
1604 }
1605
1606 #[tokio::test]
1607 async fn a_mid_canary_deploy_failure_rolls_touched_nodes_back_and_does_not_advance() {
1608 // a2 fails its forward deploy of 3.0.0; a1 was already flipped, a3 is
1609 // never reached. The touched nodes (a1, a2) roll back to 2.0.0 — their
1610 // rollback ops target `releases/2.0.0`, which the marker does not match —
1611 // and tier_state must NOT advance.
1612 let (state, log) = fleet_fixture(&["a1", "a2", "a3"], Some("a2"), "releases/3.0.0").await;
1613 let pool = state.pool.clone();
1614
1615 let err = promote_inner(state, "a".into(), PromoteBody::default())
1616 .await
1617 .expect_err("a mid-canary deploy failure must fail the promote");
1618 assert!(
1619 matches!(err, crate::error::Error::Other(_)),
1620 "a deploy failure propagates as Other, got: {err:?}",
1621 );
1622
1623 // tier_state untouched: the failure returns before advance_tier.
1624 let (cur, prev) = tier_versions(&pool, "a").await;
1625 assert_eq!(
1626 cur.as_deref(),
1627 Some("2.0.0"),
1628 "a failed rollout must not advance"
1629 );
1630 assert_eq!(prev.as_deref(), Some("1.0.0"));
1631
1632 let log = log.lock().unwrap().clone();
1633 // a3 sits after the failed a2 in the sequence and is never touched.
1634 assert!(
1635 !log.iter().any(|e| e.starts_with("a3:")),
1636 "nodes after the failure must not be deployed to: {log:#?}",
1637 );
1638 // Both touched nodes were rolled back to the prior version.
1639 for n in ["a1", "a2"] {
1640 assert!(
1641 log.iter()
1642 .any(|e| e.starts_with(&format!("{n}:")) && e.contains("releases/2.0.0")),
1643 "touched node {n} must be restored to 2.0.0: {log:#?}",
1644 );
1645 }
1646
1647 // The forward attempt is on the record: a1 ok, a2 failed.
1648 let a1: String =
1649 sqlx::query_scalar("SELECT outcome FROM deploys WHERE version = '3.0.0' AND node = 'a1'")
1650 .fetch_one(&pool)
1651 .await
1652 .unwrap();
1653 assert_eq!(a1, "ok");
1654 let a2: String =
1655 sqlx::query_scalar("SELECT outcome FROM deploys WHERE version = '3.0.0' AND node = 'a2'")
1656 .fetch_one(&pool)
1657 .await
1658 .unwrap();
1659 assert_eq!(a2, "failed");
1660
1661 // Both touched nodes restored => the tier is consistent on 2.0.0, so the
1662 // partial flag is cleared, not set.
1663 let reason: Option<String> =
1664 sqlx::query_scalar("SELECT partial_reason FROM tier_state WHERE tier = 'a'")
1665 .fetch_one(&pool)
1666 .await
1667 .unwrap();
1668 assert_eq!(
1669 reason, None,
1670 "a fully-restored canary leaves the tier consistent, not partial",
1671 );
1672 }
1673
1674 /// Build a one-node tier "a" on a tempdir release root, pre-seeded as if a
1675 /// promote had flipped it to `current` with `prev` still staged on disk.
1676 /// Returns the state (topology rewired to the tempdir node) and the tempdir,
1677 /// which the caller must keep alive.
1678 async fn rollback_fixture(prev: &str, current: &str) -> (AppState, tempfile::TempDir) {
1679 use crate::topology::{Node, default_actuate, default_observe};
1680 let tmp = tempfile::tempdir().unwrap();
1681 let rr = tmp.path().join("a-local");
1682 for v in [prev, current] {
1683 tokio::fs::create_dir_all(rr.join("releases").join(v))
1684 .await
1685 .unwrap();
1686 }
1687 tokio::fs::symlink(format!("releases/{current}"), rr.join("current"))
1688 .await
1689 .unwrap();
1690 let node = Node {
1691 platform: None,
1692 base_image: None,
1693 libc: None,
1694 name: "a-local".into(),
1695 ssh_target: "local".into(),
1696 release_root: rr.to_string_lossy().into_owned(),
1697 service_name: "x.service".into(),
1698 health_url: None,
1699 config_check_env_file: None,
1700 actuate: default_actuate(),
1701 observe: default_observe(),
1702 companions: Vec::new(),
1703 };
1704
1705 let mut state = test_state().await;
1706 let mut topo = (*state.topo).clone();
1707 topo.tiers[1].nodes = vec![node];
1708 state.executors = Arc::new(crate::state::build_executors(&topo));
1709 state.topo = Arc::new(topo);
1710
1711 // deploys.node FKs into `nodes`, so the promote path needs the row.
1712 sqlx::query("INSERT INTO nodes (name, tier, ssh_target, release_root) VALUES ('a-local', 'a', 'local', ?)")
1713 .bind(rr.to_string_lossy().into_owned())
1714 .execute(&state.pool).await.unwrap();
1715 for v in [prev, current] {
1716 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), ?)")
1717 .bind(v).bind(format!("/tmp/staged/releases/{v}/makenotwork")).execute(&state.pool).await.unwrap();
1718 }
1719 sqlx::query("UPDATE tier_state SET current_version = ?, previous_version = ? WHERE tier = 'a'")
1720 .bind(current)
1721 .bind(prev)
1722 .execute(&state.pool)
1723 .await
1724 .unwrap();
1725 (state, tmp)
1726 }
1727
1728 async fn tier_versions(pool: &SqlitePool, tier: &str) -> (Option<String>, Option<String>) {
1729 sqlx::query_as("SELECT current_version, previous_version FROM tier_state WHERE tier = ?")
1730 .bind(tier)
1731 .fetch_one(pool)
1732 .await
1733 .unwrap()
1734 }
1735
1736 #[tokio::test]
1737 async fn rollback_clears_previous_version_rather_than_swapping_it() {
1738 // The swap bug: writing the version we just rolled OFF into
1739 // previous_version made a second /rollback roll FORWARD onto the broken
1740 // build the operator was escaping. previous_version must go NULL.
1741 let (state, _tmp) = rollback_fixture("1.0.0", "2.0.0").await;
1742 let pool = state.pool.clone();
1743
1744 let _ = rollback(State(state), Path("a".to_string())).await.unwrap();
1745
1746 let (cur, prev) = tier_versions(&pool, "a").await;
1747 assert_eq!(
1748 cur.as_deref(),
1749 Some("1.0.0"),
1750 "rolled back to the previous version"
1751 );
1752 assert_eq!(
1753 prev, None,
1754 "the version we rolled off must NOT become the rollback target"
1755 );
1756 }
1757
1758 #[tokio::test]
1759 async fn second_rollback_refuses_instead_of_rolling_forward() {
1760 let (state, _tmp) = rollback_fixture("1.0.0", "2.0.0").await;
1761 let pool = state.pool.clone();
1762 let release_root = state.topo.tiers[1].nodes[0].release_root.clone();
1763
1764 let _ = rollback(State(state.clone()), Path("a".to_string()))
1765 .await
1766 .unwrap();
1767 let err = rollback(State(state), Path("a".to_string()))
1768 .await
1769 .expect_err("only one step of history is tracked; a second rollback must refuse");
1770 assert!(
1771 matches!(&err, crate::error::Error::GateBlocked(m) if m.contains("no previous_version")),
1772 "expected a loud refusal, got: {err:?}",
1773 );
1774
1775 // The refusal is total: neither the DB nor the node moved back to 2.0.0.
1776 let (cur, _) = tier_versions(&pool, "a").await;
1777 assert_eq!(cur.as_deref(), Some("1.0.0"));
1778 let link = tokio::fs::read_link(std::path::Path::new(&release_root).join("current"))
1779 .await
1780 .unwrap();
1781 assert_eq!(
1782 link.to_string_lossy(),
1783 "releases/1.0.0",
1784 "node stays on the rolled-back version"
1785 );
1786 }
1787
1788 #[tokio::test]
1789 async fn promote_refuses_an_unprovisioned_tier() {
1790 // Every step of a promote to a node-less tier is a silent no-op that
1791 // still reports success: the deploy loop iterates nothing and
1792 // advance_tier records a current_version the tier never received.
1793 let (mut state, _tmp) = rollback_fixture("1.0.0", "2.0.0").await;
1794 let mut topo = (*state.topo).clone();
1795 topo.tiers[1].provisioned = false;
1796 topo.tiers[1].nodes.clear();
1797 state.topo = Arc::new(topo);
1798 let pool = state.pool.clone();
1799 sqlx::query("UPDATE tier_state SET current_version = '2.0.0' WHERE tier = 'host'")
1800 .execute(&pool)
1801 .await
1802 .unwrap();
1803
1804 let err = promote_inner(state, "a".into(), PromoteBody::default())
1805 .await
1806 .expect_err("promoting to an unprovisioned tier must be refused");
1807 assert!(
1808 matches!(&err, crate::error::Error::GateBlocked(m) if m.contains("not provisioned")),
1809 "got: {err:?}",
1810 );
1811
1812 // And nothing was recorded: the tier keeps whatever it had before.
1813 let (cur, _) = tier_versions(&pool, "a").await;
1814 assert_eq!(
1815 cur.as_deref(),
1816 Some("2.0.0"),
1817 "a refused promote must not advance tier_state",
1818 );
1819 }
1820
1821 #[tokio::test]
1822 async fn promote_advances_the_tier_and_flips_the_symlink_when_gates_are_green() {
1823 // The happy path. Every other promote test asserts a refusal or a red
1824 // outcome, so nothing pinned what a *successful* promote actually does:
1825 // deploy reaches the node, the `current` symlink flips, tier_state
1826 // advances with previous_version = the version we came off, any stale
1827 // partial flag clears, and the handler reports the nodes it touched.
1828 use crate::topology::Gate;
1829 let (mut state, _tmp) = rollback_fixture("1.0.0", "2.0.0").await;
1830
1831 // Source tier `host` gates the promote on cargo_test. Satisfying it is
1832 // the point of the test: the sibling case asserts an unsatisfied gate
1833 // blocks, this one asserts a satisfied gate lets the promote through.
1834 let mut topo = (*state.topo).clone();
1835 topo.tiers[0].gates = vec![Gate::CargoTest];
1836 state.topo = Arc::new(topo);
1837 let pool = state.pool.clone();
1838 let release_root = state.topo.tiers[1].nodes[0].release_root.clone();
1839
1840 // 3.0.0 is staged on the build host and green on `host`.
1841 tokio::fs::create_dir_all(
1842 std::path::Path::new(&release_root)
1843 .join("releases")
1844 .join("3.0.0"),
1845 )
1846 .await
1847 .unwrap();
1848 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('3.0.0','sha',datetime('now'),'/tmp/staged/releases/3.0.0/makenotwork')")
1849 .execute(&pool).await.unwrap();
1850 sqlx::query("UPDATE tier_state SET current_version = '3.0.0' WHERE tier = 'host'")
1851 .execute(&pool)
1852 .await
1853 .unwrap();
1854 insert_gate(&pool, "host", "3.0.0", "cargo_test", 1).await;
1855
1856 // A stale flag from an earlier incident, which a clean rollout clears.
1857 set_partial(&state, &tid("a"), "left over from a previous canary").await;
1858
1859 let Json(body) = promote_inner(state, "a".into(), PromoteBody::default())
1860 .await
1861 .expect("gates are green and the node deploys, so the promote succeeds");
1862
1863 assert_eq!(body["tier"], "a");
1864 assert_eq!(body["version"], "3.0.0");
1865 assert_eq!(
1866 body["nodes_deployed"],
1867 serde_json::json!(["a-local"]),
1868 "the response names every node the promote reached",
1869 );
1870
1871 let (cur, prev) = tier_versions(&pool, "a").await;
1872 assert_eq!(cur.as_deref(), Some("3.0.0"));
1873 assert_eq!(
1874 prev.as_deref(),
1875 Some("2.0.0"),
1876 "previous_version is the version we came off, so a rollback aims at it",
1877 );
1878
1879 // The node genuinely moved: the promote is not just bookkeeping.
1880 let link = tokio::fs::read_link(std::path::Path::new(&release_root).join("current"))
1881 .await
1882 .unwrap();
1883 assert_eq!(link.to_string_lossy(), "releases/3.0.0");
1884
1885 let reason: Option<String> =
1886 sqlx::query_scalar("SELECT partial_reason FROM tier_state WHERE tier = 'a'")
1887 .fetch_one(&pool)
1888 .await
1889 .unwrap();
1890 assert_eq!(
1891 reason, None,
1892 "a clean full rollout clears a stale partial flag",
1893 );
1894
1895 // The deploy is on the record as having succeeded, which is what the
1896 // next promote's gate check and /state both read.
1897 let (node, outcome): (String, String) =
1898 sqlx::query_as("SELECT node, outcome FROM deploys WHERE version = '3.0.0'")
1899 .fetch_one(&pool)
1900 .await
1901 .unwrap();
1902 assert_eq!(node, "a-local");
1903 assert_eq!(outcome, "ok");
1904 }
1905
1906 #[tokio::test]
1907 async fn promote_fails_and_flags_the_tier_when_post_deploy_gates_are_red() {
1908 // The deploy reached every node, so tier_state advances (a stale
1909 // current_version would aim a later rollback at the wrong artifact), but
1910 // the promote must NOT report success: the tier is flagged partial and
1911 // the handler returns the gate failure.
1912 use crate::topology::Gate;
1913 let (mut state, _tmp) = rollback_fixture("1.0.0", "2.0.0").await;
1914 // node_health is the tier's only gate, and it fails closed with no
1915 // probes — an empty executor map gives it nothing to probe while
1916 // deploy_node still falls back to a built executor and succeeds.
1917 let mut topo = (*state.topo).clone();
1918 topo.tiers[1].gates = vec![Gate::NodeHealth];
1919 state.topo = Arc::new(topo);
1920 state.executors = Arc::new(crate::state::ExecutorMap::new());
1921 let pool = state.pool.clone();
1922 // Promote 3.0.0 up from host, which configures no gates of its own.
1923 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('3.0.0','sha',datetime('now'),'/tmp/staged/releases/3.0.0/makenotwork')")
1924 .execute(&pool).await.unwrap();
1925 sqlx::query("UPDATE tier_state SET current_version = '3.0.0' WHERE tier = 'host'")
1926 .execute(&pool)
1927 .await
1928 .unwrap();
1929
1930 let err = promote_inner(state, "a".into(), PromoteBody::default())
1931 .await
1932 .expect_err("red post-deploy gates must fail the promote");
1933 assert!(
1934 matches!(&err, crate::error::Error::GateBlocked(m) if m.contains("node_health")),
1935 "the failure must name the red gate, got: {err:?}",
1936 );
1937
1938 let (cur, _) = tier_versions(&pool, "a").await;
1939 assert_eq!(
1940 cur.as_deref(),
1941 Some("3.0.0"),
1942 "tier_state tracks what the nodes actually run"
1943 );
1944 let reason: Option<String> =
1945 sqlx::query_scalar("SELECT partial_reason FROM tier_state WHERE tier = 'a'")
1946 .fetch_one(&pool)
1947 .await
1948 .unwrap();
1949 assert!(
1950 reason.as_deref().is_some_and(|r| r.contains("node_health")),
1951 "the tier must be flagged for /state and the TUI, got: {reason:?}",
1952 );
1953 }
1954
1955 #[tokio::test]
1956 async fn set_partial_then_clear_roundtrips() {
1957 let state = test_state().await;
1958 let read = || async {
1959 sqlx::query_scalar::<_, Option<String>>(
1960 "SELECT partial_reason FROM tier_state WHERE tier = 'a'",
1961 )
1962 .fetch_one(&state.pool)
1963 .await
1964 .unwrap()
1965 };
1966 assert_eq!(read().await, None, "consistent tier starts clean");
1967 set_partial(&state, &tid("a"), "canary rollback incomplete: 1/2").await;
1968 assert_eq!(
1969 read().await.as_deref(),
1970 Some("canary rollback incomplete: 1/2")
1971 );
1972 clear_partial(&state, &tid("a")).await;
1973 assert_eq!(read().await, None, "clear nulls it back out");
1974 }
1975
1976 #[tokio::test]
1977 async fn state_surfaces_partial_reason() {
1978 use axum::extract::State;
1979 let state = test_state().await;
1980 set_partial(
1981 &state,
1982 &tid("a"),
1983 "first-deploy canary failed: 1 node(s) on 2.0.0",
1984 )
1985 .await;
1986 let Json(view) = get_state(State(state)).await.unwrap();
1987 let a = view.tiers.iter().find(|t| t.name == "a").unwrap();
1988 assert_eq!(
1989 a.partial_reason.as_deref(),
1990 Some("first-deploy canary failed: 1 node(s) on 2.0.0"),
1991 );
1992 let host = view.tiers.iter().find(|t| t.name == "host").unwrap();
1993 assert_eq!(
1994 host.partial_reason, None,
1995 "untouched tier stays clean in /state"
1996 );
1997 }
1998
1999 #[tokio::test]
2000 async fn state_build_is_null_until_first_rebuild_then_surfaces_latest() {
2001 use axum::extract::State;
2002 let state = test_state().await;
2003 // No build runs yet → build is null, so /state doesn't pretend a build
2004 // is happening.
2005 let Json(view) = get_state(State(state.clone())).await.unwrap();
2006 assert!(view.build.is_none());
2007
2008 // A failed run must surface its cause in /state, not just in /runs.
2009 let run_id = crate::runs::create(&state.pool, &state.cfg.id, "deadbeef")
2010 .await
2011 .unwrap();
2012 crate::runs::mark_failed(&state.pool, run_id, "cargo_test: 3 test(s) failed")
2013 .await
2014 .unwrap();
2015 let Json(view) = get_state(State(state)).await.unwrap();
2016 let b = view.build.expect("build surfaced");
2017 assert_eq!(b.run_id, run_id.0);
2018 assert_eq!(b.result, "failed");
2019 assert_eq!(
2020 b.failure_summary.as_deref(),
2021 Some("cargo_test: 3 test(s) failed")
2022 );
2023 }
2024
2025 #[tokio::test]
2026 async fn status_json_serves_the_shared_payload_over_the_real_router() {
2027 // The mapping itself is tested in `crate::status`. This asserts the
2028 // route is wired, serves valid JSON, and stays internally consistent
2029 // (no dangling child or action references) against a real topology
2030 // rather than a hand-built fixture.
2031 let state = test_state().await;
2032 set_partial(&state, &tid("a"), "canary rollback incomplete: 1/2").await;
2033
2034 let resp = router(state)
2035 .oneshot(
2036 Request::builder()
2037 .uri("/status.json")
2038 .body(Body::empty())
2039 .unwrap(),
2040 )
2041 .await
2042 .unwrap();
2043 assert_eq!(resp.status(), StatusCode::OK);
2044
2045 let body = http_body_util::BodyExt::collect(resp.into_body())
2046 .await
2047 .unwrap()
2048 .to_bytes();
2049 let payload: ops_status::Payload = serde_json::from_slice(&body).unwrap();
2050
2051 assert_eq!(payload.source, crate::status::SOURCE);
2052 assert_eq!(payload.schema, ops_status::SCHEMA_VERSION);
2053 assert_eq!(payload.validate(), Ok(()));
2054 assert_eq!(
2055 payload.node("tier:a").unwrap().status,
2056 ops_status::Status::Failed,
2057 "a partial tier must surface as failed over the wire"
2058 );
2059 assert_eq!(payload.worst_status(), ops_status::Status::Failed);
2060 }
2061
2062 #[tokio::test]
2063 async fn promote_with_explicit_version_but_missing_artifact_404s() {
2064 // Explicit version supplied, gates trivially pass (mm has none in
2065 // test_topo), but `versions` table has no row → 404.
2066 let state = test_state().await;
2067 let app = router(state);
2068 let resp = app
2069 .oneshot(
2070 Request::builder()
2071 .method("POST")
2072 .uri("/promote/a")
2073 .header("content-type", "application/json")
2074 .body(Body::from(r#"{"version":"9.9.9"}"#))
2075 .unwrap(),
2076 )
2077 .await
2078 .unwrap();
2079 assert_eq!(resp.status(), StatusCode::NOT_FOUND);
2080 }
2081
2082 // ---- GET /logs/{version}/{gate} ----
2083
2084 async fn state_with_logs_root(logs_root: PathBuf) -> AppState {
2085 let mut s = test_state().await;
2086 let mut cfg = (*s.cfg).clone();
2087 cfg.logs_root = logs_root;
2088 s.cfg = Arc::new(cfg);
2089 s
2090 }
2091
2092 #[tokio::test]
2093 async fn get_gate_log_returns_file_contents() {
2094 let tmp = tempfile::tempdir().unwrap();
2095 let dir = tmp.path().join("0.9.5");
2096 tokio::fs::create_dir_all(&dir).await.unwrap();
2097 tokio::fs::write(dir.join("cargo_test.log"), b"hello sandod\n")
2098 .await
2099 .unwrap();
2100
2101 let state = state_with_logs_root(tmp.path().to_path_buf()).await;
2102 let app = router(state);
2103 let resp = app
2104 .oneshot(
2105 Request::builder()
2106 .uri("/logs/0.9.5/cargo_test")
2107 .body(Body::empty())
2108 .unwrap(),
2109 )
2110 .await
2111 .unwrap();
2112 assert_eq!(resp.status(), StatusCode::OK);
2113 assert_eq!(body_string(resp).await, "hello sandod\n");
2114 }
2115
2116 #[tokio::test]
2117 async fn get_gate_log_accepts_a_log_ref_verbatim() {
2118 // `log_ref` on a gate row is `<build_id>/<gate>.log`; appending it to
2119 // `/logs/` must work as-is, so following the ref is the path of least
2120 // effort. Guessing from the version is what returns another run's output.
2121 let tmp = tempfile::tempdir().unwrap();
2122 tokio::fs::create_dir_all(tmp.path().join("62"))
2123 .await
2124 .unwrap();
2125 tokio::fs::write(tmp.path().join("62/cargo_deny.log"), b"run 62 only")
2126 .await
2127 .unwrap();
2128 let state = state_with_logs_root(tmp.path().to_path_buf()).await;
2129 let app = router(state);
2130 let resp = app
2131 .oneshot(
2132 Request::builder()
2133 .uri("/logs/62/cargo_deny.log")
2134 .body(Body::empty())
2135 .unwrap(),
2136 )
2137 .await
2138 .unwrap();
2139 assert_eq!(resp.status(), StatusCode::OK);
2140 assert_eq!(body_string(resp).await, "run 62 only");
2141 }
2142
2143 #[tokio::test]
2144 async fn get_gate_log_404s_when_missing() {
2145 let tmp = tempfile::tempdir().unwrap();
2146 let state = state_with_logs_root(tmp.path().to_path_buf()).await;
2147 let app = router(state);
2148 let resp = app
2149 .oneshot(
2150 Request::builder()
2151 .uri("/logs/0.9.5/cargo_test")
2152 .body(Body::empty())
2153 .unwrap(),
2154 )
2155 .await
2156 .unwrap();
2157 assert_eq!(resp.status(), StatusCode::NOT_FOUND);
2158 }
2159
2160 // ---- CF2: bearer-token auth on deploy mutators ----
2161
2162 #[tokio::test]
2163 async fn mutating_route_requires_bearer_when_token_set() {
2164 let mut state = test_state().await;
2165 state.api_token = Some(std::sync::Arc::from("s3cr3t"));
2166 let app = router(state);
2167
2168 // No Authorization header -> 401, before any deploy logic runs.
2169 let resp = app
2170 .clone()
2171 .oneshot(
2172 Request::builder()
2173 .method("POST")
2174 .uri("/promote/a")
2175 .body(Body::empty())
2176 .unwrap(),
2177 )
2178 .await
2179 .unwrap();
2180 assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
2181
2182 // Wrong token -> 401.
2183 let resp = app
2184 .clone()
2185 .oneshot(
2186 Request::builder()
2187 .method("POST")
2188 .uri("/promote/a")
2189 .header("authorization", "Bearer nope")
2190 .body(Body::empty())
2191 .unwrap(),
2192 )
2193 .await
2194 .unwrap();
2195 assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
2196
2197 // Correct token -> passes auth (then blocked downstream by gates /
2198 // missing predecessor version, but specifically NOT 401).
2199 let resp = app
2200 .clone()
2201 .oneshot(
2202 Request::builder()
2203 .method("POST")
2204 .uri("/promote/a")
2205 .header("authorization", "Bearer s3cr3t")
2206 .body(Body::empty())
2207 .unwrap(),
2208 )
2209 .await
2210 .unwrap();
2211 assert_ne!(resp.status(), StatusCode::UNAUTHORIZED);
2212 }
2213
2214 #[tokio::test]
2215 async fn read_routes_require_token_when_set() {
2216 // Reads expose prod state (versions, SHAs, gate logs, the event stream),
2217 // so they are bearer-gated too — not just the mutators. A tailnet peer
2218 // without the token gets 401; the TUI presents the token and gets 200.
2219 let mut state = test_state().await;
2220 state.api_token = Some(std::sync::Arc::from("s3cr3t"));
2221 let app = router(state);
2222
2223 // No token -> 401 on a read.
2224 let resp = app
2225 .clone()
2226 .oneshot(
2227 Request::builder()
2228 .uri("/state")
2229 .body(Body::empty())
2230 .unwrap(),
2231 )
2232 .await
2233 .unwrap();
2234 assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
2235
2236 // Correct token -> 200.
2237 let resp = app
2238 .oneshot(
2239 Request::builder()
2240 .uri("/state")
2241 .header("authorization", "Bearer s3cr3t")
2242 .body(Body::empty())
2243 .unwrap(),
2244 )
2245 .await
2246 .unwrap();
2247 assert_eq!(resp.status(), StatusCode::OK);
2248 }
2249
2250 #[tokio::test]
2251 async fn read_routes_open_without_a_token() {
2252 // The loopback/dev posture: no token configured, reads pass through so a
2253 // local TUI needs no credential.
2254 let state = test_state().await;
2255 let app = router(state);
2256 let resp = app
2257 .oneshot(
2258 Request::builder()
2259 .uri("/state")
2260 .body(Body::empty())
2261 .unwrap(),
2262 )
2263 .await
2264 .unwrap();
2265 assert_eq!(resp.status(), StatusCode::OK);
2266 }
2267
2268 #[tokio::test]
2269 async fn self_update_malformed_body_is_400_not_422() {
2270 // TypedBody funnels a JSON deserialize failure through the Error envelope
2271 // (400), not axum's raw 422 — keeping every mutator on one error contract.
2272 let state = test_state().await; // no token -> auth passes, body is the gate
2273 let app = router(state);
2274 let resp = app
2275 .oneshot(
2276 Request::builder()
2277 .method("POST")
2278 .uri("/self-update")
2279 .header("content-type", "application/json")
2280 .body(Body::from("{ not valid json"))
2281 .unwrap(),
2282 )
2283 .await
2284 .unwrap();
2285 assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
2286 }
2287
2288 #[tokio::test]
2289 async fn gate_log_rejects_unknown_gate_kind() {
2290 // The gate segment is an allowlisted GateKind, not a free-form filename:
2291 // an unknown kind is a 404, so `*.log` basenames can't be probed.
2292 let state = test_state().await;
2293 let app = router(state);
2294 let resp = app
2295 .oneshot(
2296 Request::builder()
2297 .uri("/logs/0.9.6/passwd")
2298 .body(Body::empty())
2299 .unwrap(),
2300 )
2301 .await
2302 .unwrap();
2303 assert_eq!(resp.status(), StatusCode::NOT_FOUND);
2304 }
2305
2306 /// Path-traversal guard: a `..` segment must not escape logs_root.
2307 /// axum's `{name}` param already rejects a literal `/` in the value, but
2308 /// `..` as a whole segment is structurally valid and must be blocked at
2309 /// the handler.
2310 #[tokio::test]
2311 async fn get_gate_log_rejects_dotdot_segments() {
2312 let tmp = tempfile::tempdir().unwrap();
2313 let state = state_with_logs_root(tmp.path().to_path_buf()).await;
2314 let app = router(state);
2315 let resp = app
2316 .oneshot(
2317 Request::builder()
2318 .uri("/logs/../etc/passwd")
2319 .body(Body::empty())
2320 .unwrap(),
2321 )
2322 .await
2323 .unwrap();
2324 assert_eq!(resp.status(), StatusCode::NOT_FOUND);
2325 }
2326
2327 // ---- build-identity path (wiki release-artifact-identity) ----
2328
2329 /// Insert a settled (passed) build_runs row with content identity and return
2330 /// its id. `staged_path` is the content-addressed release dir the bundle was
2331 /// published to (`releases/<digest16>`).
2332 async fn seed_build(pool: &SqlitePool, sha: &str, version: &str, staged_path: &str) -> i64 {
2333 sqlx::query_scalar(
2334 "INSERT INTO build_runs (sha, version, phase, result, started_at, bundle_digest, staged_path)
2335 VALUES (?, ?, 'done', 'passed', datetime('now'), ?, ?) RETURNING id",
2336 )
2337 .bind(sha)
2338 .bind(version)
2339 .bind(format!("{sha}-digest"))
2340 .bind(staged_path)
2341 .fetch_one(pool)
2342 .await
2343 .unwrap()
2344 }
2345
2346 async fn insert_gate_build(
2347 pool: &SqlitePool,
2348 tier: &str,
2349 version: &str,
2350 kind: &str,
2351 passed: i64,
2352 build_id: i64,
2353 ) {
2354 let status = if passed == 1 { "passed" } else { "failed" };
2355 sqlx::query(
2356 "INSERT INTO gate_runs (version, tier, gate_kind, started_at, finished_at, status, build_id) \
2357 VALUES (?, ?, ?, datetime('now'), datetime('now'), ?, ?)",
2358 )
2359 .bind(version)
2360 .bind(tier)
2361 .bind(kind)
2362 .bind(status)
2363 .bind(build_id)
2364 .execute(pool)
2365 .await
2366 .unwrap();
2367 }
2368
2369 #[tokio::test]
2370 async fn unsatisfied_gates_keys_build_evidence_on_build_id() {
2371 let pool = fresh_pool().await;
2372 seed(&pool, "a", "3.0.0").await;
2373 // Two builds of the SAME version string. Only b1's gate ran.
2374 let b1 = seed_build(&pool, "sha1", "3.0.0", "/rel/1111111111111111").await;
2375 let b2 = seed_build(&pool, "sha2", "3.0.0", "/rel/2222222222222222").await;
2376 insert_gate_build(&pool, "a", "3.0.0", "cargo_test", 1, b1).await;
2377
2378 // b1's own evidence satisfies.
2379 let ok = unsatisfied_gates(
2380 &pool,
2381 &crate::domain::AppId::default(),
2382 &tid("a"),
2383 &[Gate::CargoTest],
2384 &Evidence {
2385 version: "3.0.0",
2386 builds: &[PromotedBuild {
2387 platform: None,
2388 build_id: Some(b1),
2389 }],
2390 tier_build: None,
2391 },
2392 false,
2393 )
2394 .await
2395 .unwrap();
2396 assert!(ok.is_empty(), "the build that passed its gate is satisfied");
2397
2398 // b2 shares the version but has no gate row of its own: fail closed. This
2399 // is the hole — a rebuild reusing the version must not ride b1's evidence.
2400 let bad = unsatisfied_gates(
2401 &pool,
2402 &crate::domain::AppId::default(),
2403 &tid("a"),
2404 &[Gate::CargoTest],
2405 &Evidence {
2406 version: "3.0.0",
2407 builds: &[PromotedBuild {
2408 platform: None,
2409 build_id: Some(b2),
2410 }],
2411 tier_build: None,
2412 },
2413 false,
2414 )
2415 .await
2416 .unwrap();
2417 assert_eq!(
2418 bad,
2419 vec!["cargo_test".to_string()],
2420 "a different build of the same version does not inherit the evidence"
2421 );
2422
2423 // Legacy (pre-identity) callers still resolve by version string.
2424 let legacy = unsatisfied_gates(
2425 &pool,
2426 &crate::domain::AppId::default(),
2427 &tid("a"),
2428 &[Gate::CargoTest],
2429 &Evidence {
2430 version: "3.0.0",
2431 builds: &[],
2432 tier_build: None,
2433 },
2434 false,
2435 )
2436 .await
2437 .unwrap();
2438 assert!(legacy.is_empty(), "version-keyed legacy path unchanged");
2439 }
2440
2441 fn plat(s: &str) -> crate::domain::Platform {
2442 crate::domain::Platform::parse(s).unwrap()
2443 }
2444
2445 /// The structural block that made a cross-architecture promote impossible.
2446 ///
2447 /// A pom astra -> hetzner promote ships the x86_64 bundle. astra is aarch64
2448 /// and runs only the aarch64 one, so a `node_health` looked up against the
2449 /// x86_64 build finds nothing and fail-closed refuses, while `/state`
2450 /// correctly reports astra's node_health as passed off astra's own row. No
2451 /// amount of re-running or re-confirming clears that: astra will never run
2452 /// that bundle.
2453 ///
2454 /// node_health is evidence about the tier, so it is keyed on the tier's own
2455 /// build. cargo_test is evidence about the bytes, so it stays per shipped
2456 /// build. The assertion below holds both halves at once.
2457 #[tokio::test]
2458 async fn tier_gates_key_on_the_tier_s_build_not_on_what_it_ships_onward() {
2459 let pool = fresh_pool().await;
2460 seed(&pool, "astra", "0.4.3").await;
2461 let arm = seed_build(&pool, "sha-arm", "0.4.3", "/rel/aaaaaaaaaaaaaaaa").await;
2462 let x86 = seed_build(&pool, "sha-x86", "0.4.3", "/rel/bbbbbbbbbbbbbbbb").await;
2463 // What astra actually has: its own node_health, and each bundle's own
2464 // artifact evidence from its own intake.
2465 insert_gate_build(&pool, "astra", "0.4.3", "node_health", 1, arm).await;
2466 insert_gate_build(&pool, "astra", "0.4.3", "cargo_test", 1, arm).await;
2467 insert_gate_build(&pool, "astra", "0.4.3", "cargo_test", 1, x86).await;
2468
2469 let ships_x86 = [PromotedBuild {
2470 platform: Some(plat("linux/x86_64")),
2471 build_id: Some(x86),
2472 }];
2473 let pending = unsatisfied_gates(
2474 &pool,
2475 &crate::domain::AppId::default(),
2476 &tid("astra"),
2477 &[Gate::NodeHealth, Gate::CargoTest],
2478 &Evidence {
2479 version: "0.4.3",
2480 builds: &ships_x86,
2481 tier_build: Some(arm),
2482 }, // astra's own build
2483 false,
2484 )
2485 .await
2486 .unwrap();
2487 assert!(
2488 pending.is_empty(),
2489 "astra's node_health vouches for astra, not for the bundle leaving it: {pending:?}"
2490 );
2491
2492 // And it is still a real gate: a tier whose own node_health never passed
2493 // is refused, however green the bundle it is shipping.
2494 seed(&pool, "hetzner", "0.4.3").await;
2495 insert_gate_build(&pool, "hetzner", "0.4.3", "cargo_test", 1, x86).await;
2496 let pending = unsatisfied_gates(
2497 &pool,
2498 &crate::domain::AppId::default(),
2499 &tid("hetzner"),
2500 &[Gate::NodeHealth, Gate::CargoTest],
2501 &Evidence {
2502 version: "0.4.3",
2503 builds: &ships_x86,
2504 tier_build: Some(x86),
2505 },
2506 false,
2507 )
2508 .await
2509 .unwrap();
2510 assert_eq!(pending, vec!["node_health".to_string()]);
2511 }
2512
2513 /// The looseness this closes: one pom version is two bundles with two
2514 /// digests, each accepted through its own intake. Checking only the build the
2515 /// source tier points at let the sibling ship on gate rows nobody read.
2516 /// Every build the promote will ship must show its own passed row.
2517 #[tokio::test]
2518 async fn every_shipped_build_must_show_its_own_gate_evidence() {
2519 let pool = fresh_pool().await;
2520 seed(&pool, "a", "4.0.0").await;
2521 let arm = seed_build(&pool, "sha-arm", "4.0.0", "/rel/aaaaaaaaaaaaaaaa").await;
2522 let x86 = seed_build(&pool, "sha-x86", "4.0.0", "/rel/bbbbbbbbbbbbbbbb").await;
2523 // Only the aarch64 half was gated. This is exactly the state a
2524 // two-architecture release passes through while the second build runs.
2525 insert_gate_build(&pool, "a", "4.0.0", "cargo_test", 1, arm).await;
2526
2527 let both = [
2528 PromotedBuild {
2529 platform: Some(plat("linux/aarch64")),
2530 build_id: Some(arm),
2531 },
2532 PromotedBuild {
2533 platform: Some(plat("linux/x86_64")),
2534 build_id: Some(x86),
2535 },
2536 ];
2537 let pending = unsatisfied_gates(
2538 &pool,
2539 &crate::domain::AppId::default(),
2540 &tid("a"),
2541 &[Gate::CargoTest],
2542 &Evidence {
2543 version: "4.0.0",
2544 builds: &both,
2545 tier_build: None,
2546 },
2547 false,
2548 )
2549 .await
2550 .unwrap();
2551 assert_eq!(
2552 pending,
2553 vec!["cargo_test (linux/x86_64)".to_string()],
2554 "the ungated half blocks the promote, and the message says which half"
2555 );
2556
2557 // Gate the sibling and the promote clears. Each architecture stands on
2558 // its own evidence; neither inherits the other's.
2559 insert_gate_build(&pool, "a", "4.0.0", "cargo_test", 1, x86).await;
2560 let pending = unsatisfied_gates(
2561 &pool,
2562 &crate::domain::AppId::default(),
2563 &tid("a"),
2564 &[Gate::CargoTest],
2565 &Evidence {
2566 version: "4.0.0",
2567 builds: &both,
2568 tier_build: None,
2569 },
2570 false,
2571 )
2572 .await
2573 .unwrap();
2574 assert!(pending.is_empty(), "both halves gated: {pending:?}");
2575 }
2576
2577 /// A single-platform product's error message is exactly what it always was.
2578 /// The platform qualifier is for telling two halves apart, so adding it to a
2579 /// product that has one half would be noise in the one message an operator
2580 /// reads under pressure.
2581 #[tokio::test]
2582 async fn one_shipped_build_reports_an_unqualified_gate_name() {
2583 let pool = fresh_pool().await;
2584 seed(&pool, "a", "5.0.0").await;
2585 let only = seed_build(&pool, "sha-one", "5.0.0", "/rel/cccccccccccccccc").await;
2586
2587 let pending = unsatisfied_gates(
2588 &pool,
2589 &crate::domain::AppId::default(),
2590 &tid("a"),
2591 &[Gate::CargoTest],
2592 &Evidence {
2593 version: "5.0.0",
2594 builds: &[PromotedBuild {
2595 platform: Some(plat("linux/x86_64")),
2596 build_id: Some(only),
2597 }],
2598 tier_build: None,
2599 },
2600 false,
2601 )
2602 .await
2603 .unwrap();
2604 assert_eq!(pending, vec!["cargo_test".to_string()]);
2605 }
2606
2607 /// `burn_in` is keyed on the tier's clock, not on a build, so a
2608 /// two-architecture promote must ask about it once rather than name it twice
2609 /// in the failure.
2610 #[tokio::test]
2611 async fn a_tier_scoped_gate_is_reported_once_across_several_builds() {
2612 let pool = fresh_pool().await;
2613 seed(&pool, "a", "6.0.0").await;
2614 let arm = seed_build(&pool, "sha-arm6", "6.0.0", "/rel/dddddddddddddddd").await;
2615 let x86 = seed_build(&pool, "sha-x866", "6.0.0", "/rel/eeeeeeeeeeeeeeee").await;
2616
2617 let pending = unsatisfied_gates(
2618 &pool,
2619 &crate::domain::AppId::default(),
2620 &tid("a"),
2621 &[Gate::BurnIn { hours: 48 }],
2622 &Evidence {
2623 version: "6.0.0",
2624 builds: &[
2625 PromotedBuild {
2626 platform: Some(plat("linux/aarch64")),
2627 build_id: Some(arm),
2628 },
2629 PromotedBuild {
2630 platform: Some(plat("linux/x86_64")),
2631 build_id: Some(x86),
2632 },
2633 ],
2634 tier_build: None,
2635 },
2636 false,
2637 )
2638 .await
2639 .unwrap();
2640 assert_eq!(
2641 pending,
2642 vec!["burn_in".to_string()],
2643 "a tier-scoped gate belongs to the tier, not to each build"
2644 );
2645 }
2646
2647 /// A tier is usually several nodes on one architecture. Deduplicating means
2648 /// three x86_64 nodes ask about one build once, rather than repeating the
2649 /// same gate name three times in the error.
2650 #[test]
2651 fn distinct_builds_collapses_nodes_that_share_a_build() {
2652 use super::promotion::distinct_builds;
2653 let node = Node {
2654 platform: None,
2655 base_image: None,
2656 libc: None,
2657 name: "n1".into(),
2658 ssh_target: "local".into(),
2659 release_root: "/tmp/n1".into(),
2660 service_name: "makenotwork.service".into(),
2661 health_url: None,
2662 config_check_env_file: None,
2663 actuate: crate::topology::default_actuate(),
2664 observe: crate::topology::default_observe(),
2665 companions: Vec::new(),
2666 };
2667 let bundles = vec![
2668 (
2669 &node,
2670 std::path::PathBuf::from("/rel/a"),
2671 Some(plat("linux/x86_64")),
2672 Some(7),
2673 ),
2674 (
2675 &node,
2676 std::path::PathBuf::from("/rel/a"),
2677 Some(plat("linux/x86_64")),
2678 Some(7),
2679 ),
2680 (
2681 &node,
2682 std::path::PathBuf::from("/rel/b"),
2683 Some(plat("linux/aarch64")),
2684 Some(8),
2685 ),
2686 ];
2687 let builds = distinct_builds(&bundles);
2688 assert_eq!(builds.len(), 2);
2689 assert_eq!(builds[0].build_id, Some(7));
2690 assert_eq!(builds[1].build_id, Some(8));
2691 }
2692
2693 #[tokio::test]
2694 async fn promote_rejects_an_explicit_version_that_is_not_the_source_build() {
2695 // The burn-in hole: `promote --version Y` used to check the SOURCE tier's
2696 // clock/evidence (which belong to whatever is current there), letting Y
2697 // inherit another build's 48h. Now promote resolves the source's current
2698 // build and refuses an explicit version that isn't it.
2699 let state = test_state().await;
2700 seed_version(&state.pool, "3.0.0").await;
2701 let b = seed_build(&state.pool, "shaB", "3.0.0", "/rel/deadbeefdeadbeef").await;
2702 sqlx::query(
2703 "UPDATE tier_state SET current_version='3.0.0', current_build_id=? WHERE tier='host'",
2704 )
2705 .bind(b)
2706 .execute(&state.pool)
2707 .await
2708 .unwrap();
2709
2710 let err = promote_inner(
2711 state,
2712 "a".into(),
2713 PromoteBody {
2714 version: Some("2.0.0".into()),
2715 ..Default::default()
2716 },
2717 )
2718 .await
2719 .expect_err("promoting a version other than the source build must be refused");
2720 assert!(
2721 matches!(&err, crate::error::Error::GateBlocked(m) if m.contains("vouched for")),
2722 "the refusal must name the mismatch, got: {err:?}",
2723 );
2724 }
2725
2726 #[tokio::test]
2727 async fn promote_identity_path_deploys_the_source_build_and_advances_build_id() {
2728 // Full promote through the identity path: source tier points at a build,
2729 // promote resolves the artifact through it, deploys build_runs.staged_path
2730 // (content-addressed), and advances the target's current_build_id.
2731 let mut state = test_state().await;
2732 let node_root = tempfile::tempdir().unwrap();
2733 // Point the a-local node at a real tempdir so the symlink swap is checkable.
2734 let mut topo = (*state.topo).clone();
2735 topo.tiers[1].nodes[0].release_root = node_root.path().to_string_lossy().into_owned();
2736 topo.tiers[1].gates = vec![]; // isolate the deploy fan-out
2737 state.topo = Arc::new(topo);
2738 state.executors = Arc::new(crate::state::build_executors(&state.topo));
2739
2740 // deploys.node FKs into `nodes`, so the promote path needs the row.
2741 sqlx::query("INSERT INTO nodes (name, tier, ssh_target, release_root) VALUES ('a-local', 'a', 'local', ?)")
2742 .bind(node_root.path().to_string_lossy().into_owned())
2743 .execute(&state.pool)
2744 .await
2745 .unwrap();
2746
2747 seed_version(&state.pool, "3.0.0").await;
2748 let staged = format!(
2749 "{}/releases/abc123abc123abc1",
2750 node_root.path().to_string_lossy()
2751 );
2752 let b = seed_build(&state.pool, "shaB", "3.0.0", &staged).await;
2753 sqlx::query(
2754 "UPDATE tier_state SET current_version='3.0.0', current_build_id=? WHERE tier='host'",
2755 )
2756 .bind(b)
2757 .execute(&state.pool)
2758 .await
2759 .unwrap();
2760
2761 let pool = state.pool.clone();
2762 let Json(body) = promote_inner(state, "a".into(), PromoteBody::default())
2763 .await
2764 .expect("identity-path promote succeeds");
2765 assert_eq!(body["version"], "3.0.0");
2766
2767 // Target tier advanced with the build identity, not just the version.
2768 let (cur_v, cur_b): (Option<String>, Option<i64>) =
2769 sqlx::query_as("SELECT current_version, current_build_id FROM tier_state WHERE tier = 'a'")
2770 .fetch_one(&pool)
2771 .await
2772 .unwrap();
2773 assert_eq!(cur_v.as_deref(), Some("3.0.0"));
2774 assert_eq!(cur_b, Some(b), "target records the promoted build id");
2775
2776 // The deploy row is attributed to the build.
2777 let deploy_b: Option<i64> = sqlx::query_scalar(
2778 "SELECT build_id FROM deploys WHERE tier = 'a' ORDER BY id DESC LIMIT 1",
2779 )
2780 .fetch_one(&pool)
2781 .await
2782 .unwrap();
2783 assert_eq!(deploy_b, Some(b));
2784
2785 // The node's `current` points at the content-addressed release dir, whose
2786 // name is the staged_path's basename — not the version.
2787 let link = tokio::fs::read_link(node_root.path().join("current"))
2788 .await
2789 .unwrap();
2790 assert_eq!(link.to_string_lossy(), "releases/abc123abc123abc1");
2791 }
2792
2793 /// Insert a bare `versions` row (FK target for tier_state.current_version).
2794 async fn seed_version(pool: &SqlitePool, version: &str) {
2795 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), '/tmp/x') ON CONFLICT DO NOTHING")
2796 .bind(version).execute(pool).await.unwrap();
2797 }
2798
2799 // ---- per-node bundle resolution (wiki release-artifact-identity) ----
2800
2801 /// A settled build of `version` recorded as `platform`'s half of it.
2802 async fn seed_platform_build(
2803 pool: &SqlitePool,
2804 sha: &str,
2805 version: &str,
2806 platform: &str,
2807 staged_path: &str,
2808 ) -> i64 {
2809 sqlx::query_scalar(
2810 "INSERT INTO build_runs (sha, version, phase, result, started_at, bundle_digest, staged_path, platform)
2811 VALUES (?, ?, 'done', 'passed', datetime('now'), ?, ?, ?) RETURNING id",
2812 )
2813 .bind(sha)
2814 .bind(version)
2815 .bind(format!("{sha}-digest"))
2816 .bind(staged_path)
2817 .bind(platform)
2818 .fetch_one(pool)
2819 .await
2820 .unwrap()
2821 }
2822
2823 fn node_on(name: &str, platform: Option<&str>) -> Node {
2824 Node {
2825 platform: platform.map(plat),
2826 base_image: None,
2827 libc: None,
2828 name: name.into(),
2829 ssh_target: "local".into(),
2830 release_root: format!("/tmp/{name}"),
2831 service_name: "makenotwork.service".into(),
2832 health_url: None,
2833 config_check_env_file: None,
2834 actuate: crate::topology::default_actuate(),
2835 observe: crate::topology::default_observe(),
2836 companions: Vec::new(),
2837 }
2838 }
2839
2840 /// One pom version is two bundles with two digests. The node states which
2841 /// architecture it can run, and the sibling bundle is resolved out of
2842 /// `build_runs` rather than the caller's own half being shipped everywhere.
2843 #[tokio::test]
2844 async fn a_node_on_the_other_architecture_gets_its_own_bundle() {
2845 let state = test_state().await;
2846 seed_version(&state.pool, "5.0.0").await;
2847 let arm = seed_platform_build(
2848 &state.pool,
2849 "sha-arm",
2850 "5.0.0",
2851 "linux/aarch64",
2852 "/rel/aaaaaaaaaaaaaaaa",
2853 )
2854 .await;
2855 let x86 = seed_platform_build(
2856 &state.pool,
2857 "sha-x86",
2858 "5.0.0",
2859 "linux/x86_64",
2860 "/rel/bbbbbbbbbbbbbbbb",
2861 )
2862 .await;
2863
2864 let n_arm = node_on("astra", Some("linux/aarch64"));
2865 let n_x86 = node_on("hetzner", Some("linux/x86_64"));
2866 let bundles = super::promotion::bundles_for_nodes(
2867 &state,
2868 "5.0.0",
2869 &[&n_arm, &n_x86],
2870 std::path::Path::new("/rel/aaaaaaaaaaaaaaaa"),
2871 Some(&plat("linux/aarch64")),
2872 Some(arm),
2873 )
2874 .await
2875 .expect("both architectures have a green bundle at this version");
2876
2877 assert_eq!(bundles.len(), 2);
2878 assert_eq!(
2879 bundles[0].1,
2880 std::path::PathBuf::from("/rel/aaaaaaaaaaaaaaaa")
2881 );
2882 assert_eq!(bundles[0].2, Some(plat("linux/aarch64")));
2883 assert_eq!(bundles[0].3, Some(arm));
2884 // The one the caller never held: resolved by platform, and it carries
2885 // the sibling's own build id so its own gate evidence is what gets
2886 // checked.
2887 assert_eq!(
2888 bundles[1].1,
2889 std::path::PathBuf::from("/rel/bbbbbbbbbbbbbbbb")
2890 );
2891 assert_eq!(bundles[1].2, Some(plat("linux/x86_64")));
2892 assert_eq!(bundles[1].3, Some(x86));
2893 }
2894
2895 /// A version with no green bundle for the node's architecture fails the
2896 /// whole resolution, before any node is touched.
2897 #[tokio::test]
2898 async fn a_missing_architecture_half_refuses_the_promote_rather_than_defaulting() {
2899 let state = test_state().await;
2900 seed_version(&state.pool, "5.0.0").await;
2901 seed_platform_build(
2902 &state.pool,
2903 "sha-arm",
2904 "5.0.0",
2905 "linux/aarch64",
2906 "/rel/aaaaaaaaaaaaaaaa",
2907 )
2908 .await;
2909 let n_x86 = node_on("hetzner", Some("linux/x86_64"));
2910 let err = super::promotion::bundles_for_nodes(
2911 &state,
2912 "5.0.0",
2913 &[&n_x86],
2914 std::path::Path::new("/rel/aaaaaaaaaaaaaaaa"),
2915 Some(&plat("linux/aarch64")),
2916 None,
2917 )
2918 .await
2919 .expect_err("the x86_64 half was never built, so there is nothing to ship");
2920 assert!(
2921 format!("{err:?}").contains("no green linux/x86_64 bundle"),
2922 "{err:?}"
2923 );
2924 }
2925
2926 /// When the node and the caller name the same platform, the caller's bundle
2927 /// is the answer and no lookup happens. A single-platform product whose
2928 /// nodes have started stating a platform still has no `build_runs.platform`
2929 /// row to find, so a lookup here would refuse a promote that is fine.
2930 #[tokio::test]
2931 async fn a_node_that_agrees_with_the_caller_takes_the_callers_bundle() {
2932 let state = test_state().await;
2933 seed_version(&state.pool, "5.0.0").await;
2934 let node = node_on("hetzner", Some("linux/x86_64"));
2935 let bundles = super::promotion::bundles_for_nodes(
2936 &state,
2937 "5.0.0",
2938 &[&node],
2939 std::path::Path::new("/rel/legacy"),
2940 Some(&plat("linux/x86_64")),
2941 None,
2942 )
2943 .await
2944 .expect("the caller already holds the bundle this node wants");
2945 assert_eq!(bundles[0].1, std::path::PathBuf::from("/rel/legacy"));
2946 assert_eq!(bundles[0].2, Some(plat("linux/x86_64")));
2947 assert_eq!(bundles[0].3, None);
2948 }
2949
2950 /// The previous version is two bundles too. If the one this node runs
2951 /// cannot be resolved, nothing is attempted anywhere and every touched node
2952 /// is indeterminate — which is the truth, not a default.
2953 #[tokio::test]
2954 async fn a_rollback_that_cannot_resolve_a_bundle_reports_every_node_indeterminate() {
2955 let state = test_state().await;
2956 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('1.0.0','sha',datetime('now'),'/tmp/staged/releases/1.0.0/makenotwork')")
2957 .execute(&state.pool).await.unwrap();
2958 // No build_runs row for either node's architecture at 1.0.0.
2959 let n1 = node_on("n1", Some("linux/aarch64"));
2960 let n2 = node_on("n2", Some("linux/aarch64"));
2961 let report = rollback_deployed_nodes(&state, &tid("a"), &[&n1, &n2], "1.0.0").await;
2962 assert_eq!(report.restored, 0);
2963 assert_eq!(
2964 report.indeterminate, 2,
2965 "one per node the rollback never reached: {report:?}"
2966 );
2967 assert!(!report.is_consistent());
2968 }
2969
2970 /// A rollback that fails at the symlink swap on one node of several leaves
2971 /// that node unknown and the rest restored. The count has to be per node:
2972 /// a tier reported wholesale indeterminate sends an operator to inspect
2973 /// boxes that are fine, and a tier reported wholesale restored hides the
2974 /// one that is not.
2975 #[tokio::test]
2976 async fn a_rollback_failing_on_one_node_counts_only_that_node_indeterminate() {
2977 // The marker matches the swap-and-restart script, which is the only op
2978 // annotated AtOrAfterSwap; a1's rollback therefore lands in the
2979 // indeterminate arm rather than the already-on-previous one.
2980 let (state, _log) = fleet_fixture(&["a1", "a2"], Some("a1"), "reload-or-restart").await;
2981 let nodes: Vec<Node> = state.topo.tiers[1].nodes.clone();
2982 let refs: Vec<&Node> = nodes.iter().collect();
2983
2984 let report = rollback_deployed_nodes(&state, &tid("a"), &refs, "1.0.0").await;
2985 assert_eq!(report.restored, 1, "a2 came back: {report:?}");
2986 assert_eq!(
2987 report.indeterminate, 1,
2988 "only the node whose swap failed is unknown: {report:?}"
2989 );
2990 assert_eq!(report.already_on_previous, 0);
2991 assert_eq!(report.touched(), 2);
2992 assert!(!report.is_consistent());
2993 }
2994