Skip to main content

max / makenotwork

Check every build a promote ships, not just the one the tier points at One pom version is two bundles with two digests, each built natively and accepted through its own intake. `unsatisfied_gates` took a single `build_id` — the source tier's `current_build_id` — so a two-architecture promote evaluated one architecture's gate rows and shipped both. The bytes were still covered, because `bundle_for_platform` only resolves a sibling whose own build settled green, but a green build row is not a green gate run: the build says the bytes compiled, the gate run says the tier vouched for them. The sibling shipped on evidence nobody read. It now takes the set of builds the promote will actually ship, and every one has to show its own passed row. Getting that set means resolving the per-node bundles BEFORE the gate check rather than after, which is also the better order independently: a version missing its x86_64 half now fails before any gate work instead of after it. `bundle_for_platform` returns the build id alongside the path, so the set is read off the same resolver that decides what ships. Deriving it separately would be a second implementation of "which bundle goes where", which is the class of gap this whole area keeps closing. Failures are reported qualified by platform — `cargo_test (linux/x86_64)` — only when there is more than one build to tell apart. A single-platform product's message is byte-for-byte what it was, because the qualifier exists to disambiguate and there is nothing to disambiguate. `burn_in` and `manual_confirm` are still evaluated once. Both key on the tier's own clock rather than on a build, so asking per build would ask the same question twice and answer it twice in the one message an operator reads under pressure. One quiet failure written out rather than left to an invariant: a provisioned tier with no nodes resolves no bundles, and an empty set would fall through to the version-string lookup — the pre-identity path, weaker than what the tier is owed. The `provisioned` guard makes it unreachable today. It falls back to the source build explicitly, because the alternative degrades in the direction of less evidence. Rollback is unchanged and passes no build id: it re-runs no gates, and the bytes are ones the tier already ran.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-08-07 02:18 UTC
Signed with PGP, not checked
Commit: 9c1968227a27a1e884f05d4c0f7a35427eb8ff93
Parent: 9bfe22d
2 files changed, +383 insertions, -92 deletions
@@ -403,9 +403,12 @@
403 403 // own. Resolved before any node is touched, so a version whose other half
404 404 // was never accepted fails the rollback whole rather than partway.
405 405 let target_nodes: Vec<&crate::topology::Node> = target.nodes.iter().collect();
406 + // No fallback build id: a rollback re-runs no gates, so there is no evidence
407 + // to key on. These are bytes the tier already ran.
406 408 let bundles =
407 - promotion::bundles_for_nodes(&s, &previous_str, &target_nodes, &staged_dir, None).await?;
408 - for (i, (node, node_bundle, node_bundle_platform)) in bundles.iter().enumerate() {
409 + promotion::bundles_for_nodes(&s, &previous_str, &target_nodes, &staged_dir, None, None)
410 + .await?;
411 + for (i, (node, node_bundle, node_bundle_platform, _)) in bundles.iter().enumerate() {
409 412 let executor = s
410 413 .executors
411 414 .get(&node.name)
@@ -1064,7 +1067,9 @@
1064 1067
1065 1068 #[cfg(test)]
1066 1069 mod tests {
1067 - use super::promotion::{RollbackReport, rollback_deployed_nodes, unsatisfied_gates};
1070 + use super::promotion::{
1071 + PromotedBuild, RollbackReport, rollback_deployed_nodes, unsatisfied_gates,
1072 + };
1068 1073 use super::*;
1069 1074 use crate::config::AppConfig;
1070 1075 use crate::topology::{BackupConfig, CanaryPolicy, Gate, Node, RepoConfig, Tier, Topology};
@@ -1372,7 +1377,7 @@
1372 1377 &tid("host"),
1373 1378 &[],
1374 1379 "0.8.12",
1375 - None,
1380 + &[],
1376 1381 false,
1377 1382 )
1378 1383 .await
@@ -1394,7 +1399,7 @@
1394 1399 &tid("a"),
1395 1400 &[Gate::BootSmoke],
1396 1401 "0.8.12",
1397 - None,
1402 + &[],
1398 1403 false,
1399 1404 )
1400 1405 .await
@@ -1414,7 +1419,7 @@
1414 1419 &tid("host"),
1415 1420 &[Gate::CargoTest, Gate::BootSmoke],
1416 1421 "0.8.12",
1417 - None,
1422 + &[],
1418 1423 false,
1419 1424 )
1420 1425 .await
@@ -1436,7 +1441,7 @@
1436 1441 &tid("host"),
1437 1442 &[Gate::CargoTest],
1438 1443 "0.8.12",
1439 - None,
1444 + &[],
1440 1445 false,
1441 1446 )
1442 1447 .await
@@ -1485,7 +1490,7 @@
1485 1490 &tid("a"),
1486 1491 &[Gate::ManualConfirm],
1487 1492 "0.8.12",
1488 - None,
1493 + &[],
1489 1494 false,
1490 1495 )
1491 1496 .await
@@ -1504,7 +1509,7 @@
1504 1509 &tid("a"),
1505 1510 &[Gate::ManualConfirm],
1506 1511 "0.8.12",
1507 - None,
1512 + &[],
1508 1513 false,
1509 1514 )
1510 1515 .await
@@ -1525,7 +1530,7 @@
1525 1530 &tid("a"),
1526 1531 &[Gate::ManualConfirm],
1527 1532 "0.8.12",
1528 - None,
1533 + &[],
1529 1534 false,
1530 1535 )
1531 1536 .await
@@ -1554,7 +1559,7 @@
1554 1559 &tid("a"),
1555 1560 &gates,
1556 1561 "0.8.12",
1557 - None,
1562 + &[],
1558 1563 false,
1559 1564 )
1560 1565 .await
@@ -1570,7 +1575,7 @@
1570 1575 &tid("a"),
1571 1576 &gates,
1572 1577 "0.8.12",
1573 - None,
1578 + &[],
1574 1579 true,
1575 1580 )
1576 1581 .await
@@ -1595,7 +1600,7 @@
1595 1600 &tid("a"),
1596 1601 &[Gate::BurnIn { hours: 48 }],
1597 1602 "0.8.12",
1598 - None,
1603 + &[],
1599 1604 false,
1600 1605 )
1601 1606 .await
@@ -1621,7 +1626,7 @@
1621 1626 &tid("host"),
1622 1627 &[Gate::CargoTest],
1623 1628 "0.8.12",
1624 - None,
1629 + &[],
1625 1630 false,
1626 1631 )
1627 1632 .await
@@ -1650,7 +1655,7 @@
1650 1655 &tid("host"),
1651 1656 &[Gate::CargoTest],
1652 1657 "0.8.12",
1653 - None,
1658 + &[],
1654 1659 false,
1655 1660 )
1656 1661 .await
@@ -3223,7 +3228,10 @@
3223 3228 &tid("a"),
3224 3229 &[Gate::CargoTest],
3225 3230 "3.0.0",
3226 - Some(b1),
3231 + &[PromotedBuild {
3232 + platform: None,
3233 + build_id: Some(b1),
3234 + }],
3227 3235 false,
3228 3236 )
3229 3237 .await
@@ -3238,7 +3246,10 @@
3238 3246 &tid("a"),
3239 3247 &[Gate::CargoTest],
3240 3248 "3.0.0",
3241 - Some(b2),
3249 + &[PromotedBuild {
3250 + platform: None,
3251 + build_id: Some(b2),
3252 + }],
3242 3253 false,
3243 3254 )
3244 3255 .await
@@ -3256,7 +3267,7 @@
3256 3267 &tid("a"),
3257 3268 &[Gate::CargoTest],
3258 3269 "3.0.0",
3259 - None,
3270 + &[],
3260 3271 false,
3261 3272 )
3262 3273 .await
@@ -3264,6 +3275,176 @@
3264 3275 assert!(legacy.is_empty(), "version-keyed legacy path unchanged");
3265 3276 }
3266 3277
3278 + fn plat(s: &str) -> crate::domain::Platform {
3279 + crate::domain::Platform::parse(s).unwrap()
3280 + }
3281 +
3282 + /// The looseness this closes: one pom version is two bundles with two
3283 + /// digests, each accepted through its own intake. Checking only the build the
3284 + /// source tier points at let the sibling ship on gate rows nobody read.
3285 + /// Every build the promote will ship must show its own passed row.
3286 + #[tokio::test]
3287 + async fn every_shipped_build_must_show_its_own_gate_evidence() {
3288 + let pool = fresh_pool().await;
3289 + seed(&pool, "a", "4.0.0").await;
3290 + let arm = seed_build(&pool, "sha-arm", "4.0.0", "/rel/aaaaaaaaaaaaaaaa").await;
3291 + let x86 = seed_build(&pool, "sha-x86", "4.0.0", "/rel/bbbbbbbbbbbbbbbb").await;
3292 + // Only the aarch64 half was gated. This is exactly the state a
3293 + // two-architecture release passes through while the second build runs.
3294 + insert_gate_build(&pool, "a", "4.0.0", "cargo_test", 1, arm).await;
3295 +
3296 + let both = [
3297 + PromotedBuild {
3298 + platform: Some(plat("linux/aarch64")),
3299 + build_id: Some(arm),
3300 + },
3301 + PromotedBuild {
3302 + platform: Some(plat("linux/x86_64")),
3303 + build_id: Some(x86),
3304 + },
3305 + ];
3306 + let pending = unsatisfied_gates(
3307 + &pool,
3308 + &crate::domain::AppId::default(),
3309 + &tid("a"),
3310 + &[Gate::CargoTest],
3311 + "4.0.0",
3312 + &both,
3313 + false,
3314 + )
3315 + .await
3316 + .unwrap();
3317 + assert_eq!(
3318 + pending,
3319 + vec!["cargo_test (linux/x86_64)".to_string()],
3320 + "the ungated half blocks the promote, and the message says which half"
3321 + );
3322 +
3323 + // Gate the sibling and the promote clears. Each architecture stands on
3324 + // its own evidence; neither inherits the other's.
3325 + insert_gate_build(&pool, "a", "4.0.0", "cargo_test", 1, x86).await;
3326 + let pending = unsatisfied_gates(
3327 + &pool,
3328 + &crate::domain::AppId::default(),
3329 + &tid("a"),
3330 + &[Gate::CargoTest],
3331 + "4.0.0",
3332 + &both,
3333 + false,
3334 + )
3335 + .await
3336 + .unwrap();
3337 + assert!(pending.is_empty(), "both halves gated: {pending:?}");
3338 + }
3339 +
3340 + /// A single-platform product's error message is exactly what it always was.
3341 + /// The platform qualifier is for telling two halves apart, so adding it to a
3342 + /// product that has one half would be noise in the one message an operator
3343 + /// reads under pressure.
3344 + #[tokio::test]
3345 + async fn one_shipped_build_reports_an_unqualified_gate_name() {
3346 + let pool = fresh_pool().await;
3347 + seed(&pool, "a", "5.0.0").await;
3348 + let only = seed_build(&pool, "sha-one", "5.0.0", "/rel/cccccccccccccccc").await;
3349 +
3350 + let pending = unsatisfied_gates(
3351 + &pool,
3352 + &crate::domain::AppId::default(),
3353 + &tid("a"),
3354 + &[Gate::CargoTest],
3355 + "5.0.0",
3356 + &[PromotedBuild {
3357 + platform: Some(plat("linux/x86_64")),
3358 + build_id: Some(only),
3359 + }],
3360 + false,
3361 + )
3362 + .await
3363 + .unwrap();
3364 + assert_eq!(pending, vec!["cargo_test".to_string()]);
3365 + }
3366 +
3367 + /// `burn_in` is keyed on the tier's clock, not on a build, so a
3368 + /// two-architecture promote must ask about it once rather than name it twice
3369 + /// in the failure.
3370 + #[tokio::test]
3371 + async fn a_tier_scoped_gate_is_reported_once_across_several_builds() {
3372 + let pool = fresh_pool().await;
3373 + seed(&pool, "a", "6.0.0").await;
3374 + let arm = seed_build(&pool, "sha-arm6", "6.0.0", "/rel/dddddddddddddddd").await;
3375 + let x86 = seed_build(&pool, "sha-x866", "6.0.0", "/rel/eeeeeeeeeeeeeeee").await;
3376 +
3377 + let pending = unsatisfied_gates(
3378 + &pool,
3379 + &crate::domain::AppId::default(),
3380 + &tid("a"),
3381 + &[Gate::BurnIn { hours: 48 }],
3382 + "6.0.0",
3383 + &[
3384 + PromotedBuild {
3385 + platform: Some(plat("linux/aarch64")),
3386 + build_id: Some(arm),
3387 + },
3388 + PromotedBuild {
3389 + platform: Some(plat("linux/x86_64")),
3390 + build_id: Some(x86),
3391 + },
3392 + ],
3393 + false,
3394 + )
3395 + .await
3396 + .unwrap();
3397 + assert_eq!(
3398 + pending,
3399 + vec!["burn_in".to_string()],
3400 + "a tier-scoped gate belongs to the tier, not to each build"
3401 + );
3402 + }
3403 +
3404 + /// A tier is usually several nodes on one architecture. Deduplicating means
3405 + /// three x86_64 nodes ask about one build once, rather than repeating the
3406 + /// same gate name three times in the error.
3407 + #[test]
3408 + fn distinct_builds_collapses_nodes_that_share_a_build() {
3409 + use super::promotion::distinct_builds;
3410 + let node = Node {
3411 + platform: None,
3412 + name: "n1".into(),
3413 + ssh_target: "local".into(),
3414 + release_root: "/tmp/n1".into(),
3415 + service_name: "makenotwork.service".into(),
3416 + health_url: None,
3417 + config_check_env_file: None,
3418 + actuate: crate::topology::default_actuate(),
3419 + observe: crate::topology::default_observe(),
3420 + companions: Vec::new(),
3421 + };
3422 + let bundles = vec![
3423 + (
3424 + &node,
3425 + std::path::PathBuf::from("/rel/a"),
3426 + Some(plat("linux/x86_64")),
3427 + Some(7),
3428 + ),
3429 + (
3430 + &node,
3431 + std::path::PathBuf::from("/rel/a"),
3432 + Some(plat("linux/x86_64")),
3433 + Some(7),
3434 + ),
3435 + (
3436 + &node,
3437 + std::path::PathBuf::from("/rel/b"),
3438 + Some(plat("linux/aarch64")),
3439 + Some(8),
3440 + ),
3441 + ];
3442 + let builds = distinct_builds(&bundles);
3443 + assert_eq!(builds.len(), 2);
3444 + assert_eq!(builds[0].build_id, Some(7));
3445 + assert_eq!(builds[1].build_id, Some(8));
3446 + }
3447 +
3267 3448 #[tokio::test]
3268 3449 async fn promote_rejects_an_explicit_version_that_is_not_the_source_build() {
3269 3450 // The burn-in hole: `promote --version Y` used to check the SOURCE tier's
@@ -157,13 +157,44 @@
157 157 {
158 158 effective_gates.push(crate::topology::Gate::ManualConfirm);
159 159 }
160 + // Which bytes each node gets is resolved FIRST, because it is what says
161 + // how many builds the gate check has to cover. A two-architecture promote
162 + // ships two bundles with two sets of evidence, and checking only the one
163 + // the source tier points at would wave the sibling through on gate rows
164 + // nobody read. It also means a version missing its other half fails here,
165 + // before any gate work, rather than halfway down the rollout.
166 + let target_nodes: Vec<&crate::topology::Node> = target.nodes.iter().collect();
167 + let bundles = bundles_for_nodes(
168 + &s,
169 + &version_str,
170 + &target_nodes,
171 + &staged_dir,
172 + source_platform.as_ref(),
173 + build_id,
174 + )
175 + .await?;
176 + let mut promoted_builds = distinct_builds(&bundles);
177 + if promoted_builds.is_empty() {
178 + // A provisioned tier with no nodes ships nothing, so there is nothing to
179 + // resolve — but the gate check still has to be keyed on the source build.
180 + // Left empty it would fall through to the version-string lookup, which is
181 + // the pre-identity path and weaker than what this tier is owed. The
182 + // `provisioned` guard above makes this unreachable today; it is written
183 + // out because the alternative fails quietly in the direction of less
184 + // evidence.
185 + promoted_builds.push(PromotedBuild {
186 + platform: source_platform.clone(),
187 + build_id,
188 + });
189 + }
190 +
160 191 let pending = unsatisfied_gates(
161 192 &s.pool,
162 193 &s.cfg.id,
163 194 &source.name,
164 195 &effective_gates,
165 196 &version_str,
166 - build_id,
197 + &promoted_builds,
167 198 body.hotfix,
168 199 )
169 200 .await?;
@@ -187,30 +218,14 @@
187 218 .map_err(crate::error::Error::Db)?
188 219 .flatten();
189 220
190 - // 2b. Decide which bytes each node gets, before any of them are touched.
191 - //
192 - // A single-platform product has one answer: the bundle the source tier
193 - // vouched for. A product whose one version is several bundles — pom, on
194 - // aarch64 and x86_64 — has one per architecture, and the node is what
195 - // says which. Resolving up front means a version that is missing its
196 - // x86_64 half fails the promote before the aarch64 half has been pushed
197 - // anywhere, rather than halfway down the tier.
198 - let target_nodes: Vec<&crate::topology::Node> = target.nodes.iter().collect();
199 - let bundles = bundles_for_nodes(
200 - &s,
201 - &version_str,
202 - &target_nodes,
203 - &staged_dir,
204 - source_platform.as_ref(),
205 - )
206 - .await?;
207 -
208 221 // 3. Deploy to each node. Sequential canary is the only policy
209 222 // implemented in v0; parallel is a one-line change once we trust the
210 223 // sequential path. Track the nodes already flipped to the new version so
211 224 // a mid-rollout failure can roll them back (canary rollback).
212 225 let mut deployed: Vec<&crate::topology::Node> = Vec::new();
213 - for (node, node_bundle, node_bundle_platform) in &bundles {
226 + // The build id is not read here: it did its work above, keying the gate
227 + // check to each bundle's own evidence. What ships is decided by `Placement`.
228 + for (node, node_bundle, node_bundle_platform, _) in &bundles {
214 229 // The proof that these bytes belong on this box. Built before the
215 230 // deploy row is written, so a mismatch never becomes an `in_progress`
216 231 // deploy that has to be reconciled.
@@ -574,13 +589,37 @@
574 589 }
575 590 }
576 591
577 - /// A node, the bundle it is to receive, and what that bundle runs on.
592 + /// A node, the bundle it is to receive, what that bundle runs on, and the build
593 + /// row that bundle came from.
594 + ///
595 + /// The build id is what makes the gate check per-platform: evidence is keyed on
596 + /// the build that produced the bytes, so a promote that ships two architectures
597 + /// has to look up two sets of gate rows, and this is where it learns which two.
598 + /// `None` is the legacy path (a bundle resolved by version string, from a tier
599 + /// with no `current_build_id`), where there is no build to key on.
578 600 pub(super) type NodeBundle<'a> = (
579 601 &'a crate::topology::Node,
580 602 std::path::PathBuf,
581 603 Option<crate::domain::Platform>,
604 + Option<i64>,
582 605 );
583 606
607 + /// One bundle a promote is about to ship, and the evidence key it is judged by.
608 + ///
609 + /// A single-platform product has exactly one of these and it is the source
610 + /// tier's own build, which is what every promote before two-architecture support
611 + /// checked. A product like pom has one per architecture, each standing on its
612 + /// own intake and its own gate run.
613 + #[derive(Debug, Clone, PartialEq, Eq)]
614 + pub(super) struct PromotedBuild {
615 + /// What the bundle runs on, when it says. Used only to qualify the name of a
616 + /// failing gate, so an operator reading "cargo_test" knows which half.
617 + pub platform: Option<crate::domain::Platform>,
618 + /// The `build_runs` row the evidence is keyed on; `None` falls back to the
619 + /// version-keyed lookup for a pre-identity tier.
620 + pub build_id: Option<i64>,
621 + }
622 +
584 623 /// Which bytes each node gets, resolved before any node is touched.
585 624 ///
586 625 /// A single-platform product answers with `fallback` for every node, which is
@@ -598,6 +637,7 @@
598 637 nodes: &[&'a crate::topology::Node],
599 638 fallback: &std::path::Path,
600 639 fallback_platform: Option<&crate::domain::Platform>,
640 + fallback_build_id: Option<i64>,
601 641 ) -> Result<Vec<NodeBundle<'a>>> {
602 642 let mut out = Vec::with_capacity(nodes.len());
603 643 for node in nodes.iter().copied() {
@@ -605,23 +645,50 @@
605 645 // The node states nothing, so there is nothing to resolve against;
606 646 // it gets the caller's bundle, and `Placement::check` decides
607 647 // whether that pairing is admissible at all.
608 - None => (fallback.to_path_buf(), fallback_platform.cloned()),
648 + None => (
649 + fallback.to_path_buf(),
650 + fallback_platform.cloned(),
651 + fallback_build_id,
652 + ),
609 653 // The node states a platform. Give it the bundle recorded for that
610 654 // platform at this version — the caller's own when they agree, its
611 655 // sibling when they do not.
612 - Some(want) if fallback_platform == Some(want) => {
613 - (fallback.to_path_buf(), fallback_platform.cloned())
614 - }
615 - Some(want) => (
616 - bundle_for_platform(s, version, want).await?,
617 - Some(want.clone()),
656 + Some(want) if fallback_platform == Some(want) => (
657 + fallback.to_path_buf(),
658 + fallback_platform.cloned(),
659 + fallback_build_id,
618 660 ),
661 + Some(want) => {
662 + let (build_id, path) = bundle_for_platform(s, version, want).await?;
663 + (path, Some(want.clone()), build_id)
664 + }
619 665 };
620 - out.push((node, resolved.0, resolved.1));
666 + out.push((node, resolved.0, resolved.1, resolved.2));
621 667 }
622 668 Ok(out)
623 669 }
624 670
671 + /// The distinct builds `bundles` will ship, in a stable order.
672 + ///
673 + /// Deduplicated because a tier is usually several nodes on one architecture, and
674 + /// evaluating one build's gates once per node would say the same thing three
675 + /// times in the error an operator reads. Keyed on the whole entry rather than on
676 + /// the build id alone, so the legacy `None` case does not collapse two
677 + /// version-resolved bundles into one.
678 + pub(super) fn distinct_builds(bundles: &[NodeBundle<'_>]) -> Vec<PromotedBuild> {
679 + let mut out: Vec<PromotedBuild> = Vec::new();
680 + for (_, _, platform, build_id) in bundles {
681 + let entry = PromotedBuild {
682 + platform: platform.clone(),
683 + build_id: *build_id,
684 + };
685 + if !out.contains(&entry) {
686 + out.push(entry);
687 + }
688 + }
689 + out
690 + }
691 +
625 692 /// The bundle recorded for `version` on `platform`.
626 693 ///
627 694 /// This is what makes a two-architecture product promotable. One pom version is
@@ -637,9 +704,9 @@
637 704 s: &AppState,
638 705 version: &str,
639 706 platform: &crate::domain::Platform,
640 - ) -> Result<std::path::PathBuf> {
641 - let row: Option<(Option<String>,)> = sqlx::query_as(
642 - "SELECT staged_path FROM build_runs
707 + ) -> Result<(Option<i64>, std::path::PathBuf)> {
708 + let row: Option<(i64, Option<String>)> = sqlx::query_as(
709 + "SELECT id, staged_path FROM build_runs
643 710 WHERE app = ? AND version = ? AND platform = ? AND result = 'passed'
644 711 ORDER BY id DESC LIMIT 1",
645 712 )
@@ -651,8 +718,12 @@
651 718 .map_err(crate::error::Error::Db)?;
652 719
653 720 match row {
654 - Some((Some(path),)) => Ok(std::path::PathBuf::from(path)),
655 - Some((None,)) => Err(crate::error::Error::Other(anyhow::anyhow!(
721 + // The id comes back alongside the path so the caller can check this
722 + // build's own gate evidence rather than the source tier's. A green build
723 + // row is not a green gate run: the build says the bytes compiled, the
724 + // gate run says the tier vouched for them.
725 + Some((id, Some(path))) => Ok((Some(id), std::path::PathBuf::from(path))),
726 + Some((_, None)) => Err(crate::error::Error::Other(anyhow::anyhow!(
656 727 "the {platform} build of {version} has no staged_path; cannot promote it"
657 728 ))),
658 729 None => Err(crate::error::Error::GateBlocked(format!(
@@ -724,7 +795,10 @@
724 795 // propagated (the promote is already failing), so it reports the node as
725 796 // indeterminate — which is the truth: nothing was attempted and the node's
726 797 // version is whatever the failed deploy left.
727 - let bundles = match bundles_for_nodes(s, prev_version, nodes, &staged_dir, None).await {
798 + // No fallback build id: a rollback resolves the previous version's bundles
799 + // by version string and re-runs no gates, so there is no evidence to key.
800 + // The bytes are ones this tier already ran.
801 + let bundles = match bundles_for_nodes(s, prev_version, nodes, &staged_dir, None, None).await {
728 802 Ok(b) => b,
729 803 Err(e) => {
730 804 tracing::error!(tier = %tier, prev = prev_version, error = %e,
@@ -737,7 +811,7 @@
737 811 };
738 812
739 813 let mut report = RollbackReport::default();
740 - for (node, node_bundle, node_bundle_platform) in &bundles {
814 + for (node, node_bundle, node_bundle_platform, _) in &bundles {
741 815 let executor = s
742 816 .executors
743 817 .get(&node.name)
@@ -848,22 +922,35 @@
848 922 /// - every other kind requires a `passed` row for (tier, version) — a missing
849 923 /// or non-passed latest row counts as unsatisfied.
850 924 ///
851 - /// `build_id` is the identity of the artifact being promoted (wiki note
852 - /// `release-artifact-identity`). When `Some`, the deterministic build-evidence
853 - /// gates are checked against the row that vouched for *that build* rather than
854 - /// against any row that happens to carry the version string — this is what stops
855 - /// a `promote --version Y` from riding on gate rows a different build left under
856 - /// the same version. `None` is the legacy/pre-identity path: fall back to the
857 - /// version-keyed lookup so a mid-migration tier (NULL `current_build_id`) still
858 - /// promotes. `burn_in` is clock-based either way — the clock is reset on every
859 - /// advance, so it belongs to the build now current on the tier.
925 + /// `builds` are the artifacts this promote will actually ship (wiki note
926 + /// `release-artifact-identity`). Each carries a `build_id`, and the deterministic
927 + /// build-evidence gates are checked against the rows that vouched for *that
928 + /// build* rather than against any row that happens to carry the version string —
929 + /// this is what stops a `promote --version Y` from riding on gate rows a
930 + /// different build left under the same version. A `None` build id is the
931 + /// legacy/pre-identity path: fall back to the version-keyed lookup so a
932 + /// mid-migration tier (NULL `current_build_id`) still promotes.
933 + ///
934 + /// **One promote can ship several builds**, and every one of them is checked.
935 + /// A pom version is two bundles with two digests, each built natively and
936 + /// accepted through its own intake, so the source tier's evidence for one
937 + /// architecture says nothing about the other. Checking only the build the tier
938 + /// points at would let an x86_64 node take bytes whose `cargo_test` row nobody
939 + /// looked at. When there is more than one, a failing gate is reported qualified
940 + /// by platform, because "cargo_test not satisfied" is not actionable if the
941 + /// operator cannot tell which half it is about.
942 + ///
943 + /// `burn_in` and `manual_confirm` are evaluated once regardless: both are keyed
944 + /// on the tier's own clock (`tier_state.burn_in_started_at`), not on a build, so
945 + /// asking per build would ask the same question N times and answer it N times in
946 + /// the error.
860 947 pub(super) async fn unsatisfied_gates(
861 948 pool: &sqlx::SqlitePool,
862 949 app: &crate::domain::AppId,
863 950 tier: &crate::domain::TierId,
864 951 gates: &[crate::topology::Gate],
865 952 version: &str,
866 - build_id: Option<i64>,
953 + builds: &[PromotedBuild],
867 954 hotfix: bool,
868 955 ) -> std::result::Result<Vec<String>, crate::error::Error> {
869 956 use crate::topology::Gate;
@@ -946,34 +1033,57 @@
946 1033 | Gate::NodeHealth => {
947 1034 // Latest row for this configured gate kind; NULL/missing/any
948 1035 // non-'passed' status all count as unsatisfied (fail closed).
949 - // Keyed on build_id when the artifact has an identity (the
950 - // evidence must be for *this* build), else on the version string.
951 - let status: Option<String> = match build_id {
952 - Some(bid) => sqlx::query_scalar(
953 - "SELECT status FROM gate_runs
954 - WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
955 - ORDER BY id DESC LIMIT 1",
956 - )
957 - .bind(app)
958 - .bind(tier.as_str())
959 - .bind(bid)
960 - .bind(kind.as_str()),
961 - None => sqlx::query_scalar(
962 - "SELECT status FROM gate_runs
963 - WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
964 - ORDER BY id DESC LIMIT 1",
965 - )
966 - .bind(app)
967 - .bind(tier.as_str())
968 - .bind(version)
969 - .bind(kind.as_str()),
970 - }
971 - .fetch_optional(pool)
972 - .await
973 - .map_err(crate::error::Error::Db)?
974 - .flatten();
975 - if status.as_deref() != Some("passed") {
976 - bad.push(kind.as_str().to_string());
1036 + // Every build this promote ships has to show its own passed row.
1037 + // An empty `builds` is the caller saying "no identities to key
1038 + // on"; one version-keyed lookup is the pre-identity behaviour.
1039 + let lookups: &[PromotedBuild] = if builds.is_empty() {
1040 + &[PromotedBuild {
1041 + platform: None,
1042 + build_id: None,
1043 + }]
1044 + } else {
1045 + builds
1046 + };
1047 + let qualify = lookups.len() > 1;
1048 + for b in lookups {
1049 + // Keyed on build_id when the artifact has an identity (the
1050 + // evidence must be for *this* build), else on the version.
1051 + let status: Option<String> = match b.build_id {
1052 + Some(bid) => sqlx::query_scalar(
1053 + "SELECT status FROM gate_runs
1054 + WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
1055 + ORDER BY id DESC LIMIT 1",
1056 + )
1057 + .bind(app)
1058 + .bind(tier.as_str())
1059 + .bind(bid)
1060 + .bind(kind.as_str()),
1061 + None => sqlx::query_scalar(
1062 + "SELECT status FROM gate_runs
1063 + WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
1064 + ORDER BY id DESC LIMIT 1",
1065 + )
1066 + .bind(app)
1067 + .bind(tier.as_str())
1068 + .bind(version)
1069 + .bind(kind.as_str()),
1070 + }
1071 + .fetch_optional(pool)
1072 + .await
1073 + .map_err(crate::error::Error::Db)?
1074 + .flatten();
1075 + if status.as_deref() != Some("passed") {
1076 + // Qualified only when there is more than one build to
1077 + // tell apart, so a single-platform product's message is
1078 + // exactly what it always was.
1079 + let name = match (&b.platform, qualify) {
1080 + (Some(p), true) => format!("{} ({p})", kind.as_str()),
1081 + _ => kind.as_str().to_string(),
1082 + };
1083 + if !bad.contains(&name) {
1084 + bad.push(name);
1085 + }
1086 + }
977 1087 }
978 1088 }
979 1089 }