| 157 |
157 |
|
{
|
| 158 |
158 |
|
effective_gates.push(crate::topology::Gate::ManualConfirm);
|
| 159 |
159 |
|
}
|
|
160 |
+ |
// Which bytes each node gets is resolved FIRST, because it is what says
|
|
161 |
+ |
// how many builds the gate check has to cover. A two-architecture promote
|
|
162 |
+ |
// ships two bundles with two sets of evidence, and checking only the one
|
|
163 |
+ |
// the source tier points at would wave the sibling through on gate rows
|
|
164 |
+ |
// nobody read. It also means a version missing its other half fails here,
|
|
165 |
+ |
// before any gate work, rather than halfway down the rollout.
|
|
166 |
+ |
let target_nodes: Vec<&crate::topology::Node> = target.nodes.iter().collect();
|
|
167 |
+ |
let bundles = bundles_for_nodes(
|
|
168 |
+ |
&s,
|
|
169 |
+ |
&version_str,
|
|
170 |
+ |
&target_nodes,
|
|
171 |
+ |
&staged_dir,
|
|
172 |
+ |
source_platform.as_ref(),
|
|
173 |
+ |
build_id,
|
|
174 |
+ |
)
|
|
175 |
+ |
.await?;
|
|
176 |
+ |
let mut promoted_builds = distinct_builds(&bundles);
|
|
177 |
+ |
if promoted_builds.is_empty() {
|
|
178 |
+ |
// A provisioned tier with no nodes ships nothing, so there is nothing to
|
|
179 |
+ |
// resolve — but the gate check still has to be keyed on the source build.
|
|
180 |
+ |
// Left empty it would fall through to the version-string lookup, which is
|
|
181 |
+ |
// the pre-identity path and weaker than what this tier is owed. The
|
|
182 |
+ |
// `provisioned` guard above makes this unreachable today; it is written
|
|
183 |
+ |
// out because the alternative fails quietly in the direction of less
|
|
184 |
+ |
// evidence.
|
|
185 |
+ |
promoted_builds.push(PromotedBuild {
|
|
186 |
+ |
platform: source_platform.clone(),
|
|
187 |
+ |
build_id,
|
|
188 |
+ |
});
|
|
189 |
+ |
}
|
|
190 |
+ |
|
| 160 |
191 |
|
let pending = unsatisfied_gates(
|
| 161 |
192 |
|
&s.pool,
|
| 162 |
193 |
|
&s.cfg.id,
|
| 163 |
194 |
|
&source.name,
|
| 164 |
195 |
|
&effective_gates,
|
| 165 |
196 |
|
&version_str,
|
| 166 |
|
- |
build_id,
|
|
197 |
+ |
&promoted_builds,
|
| 167 |
198 |
|
body.hotfix,
|
| 168 |
199 |
|
)
|
| 169 |
200 |
|
.await?;
|
| 187 |
218 |
|
.map_err(crate::error::Error::Db)?
|
| 188 |
219 |
|
.flatten();
|
| 189 |
220 |
|
|
| 190 |
|
- |
// 2b. Decide which bytes each node gets, before any of them are touched.
|
| 191 |
|
- |
//
|
| 192 |
|
- |
// A single-platform product has one answer: the bundle the source tier
|
| 193 |
|
- |
// vouched for. A product whose one version is several bundles — pom, on
|
| 194 |
|
- |
// aarch64 and x86_64 — has one per architecture, and the node is what
|
| 195 |
|
- |
// says which. Resolving up front means a version that is missing its
|
| 196 |
|
- |
// x86_64 half fails the promote before the aarch64 half has been pushed
|
| 197 |
|
- |
// anywhere, rather than halfway down the tier.
|
| 198 |
|
- |
let target_nodes: Vec<&crate::topology::Node> = target.nodes.iter().collect();
|
| 199 |
|
- |
let bundles = bundles_for_nodes(
|
| 200 |
|
- |
&s,
|
| 201 |
|
- |
&version_str,
|
| 202 |
|
- |
&target_nodes,
|
| 203 |
|
- |
&staged_dir,
|
| 204 |
|
- |
source_platform.as_ref(),
|
| 205 |
|
- |
)
|
| 206 |
|
- |
.await?;
|
| 207 |
|
- |
|
| 208 |
221 |
|
// 3. Deploy to each node. Sequential canary is the only policy
|
| 209 |
222 |
|
// implemented in v0; parallel is a one-line change once we trust the
|
| 210 |
223 |
|
// sequential path. Track the nodes already flipped to the new version so
|
| 211 |
224 |
|
// a mid-rollout failure can roll them back (canary rollback).
|
| 212 |
225 |
|
let mut deployed: Vec<&crate::topology::Node> = Vec::new();
|
| 213 |
|
- |
for (node, node_bundle, node_bundle_platform) in &bundles {
|
|
226 |
+ |
// The build id is not read here: it did its work above, keying the gate
|
|
227 |
+ |
// check to each bundle's own evidence. What ships is decided by `Placement`.
|
|
228 |
+ |
for (node, node_bundle, node_bundle_platform, _) in &bundles {
|
| 214 |
229 |
|
// The proof that these bytes belong on this box. Built before the
|
| 215 |
230 |
|
// deploy row is written, so a mismatch never becomes an `in_progress`
|
| 216 |
231 |
|
// deploy that has to be reconciled.
|
| 574 |
589 |
|
}
|
| 575 |
590 |
|
}
|
| 576 |
591 |
|
|
| 577 |
|
- |
/// A node, the bundle it is to receive, and what that bundle runs on.
|
|
592 |
+ |
/// A node, the bundle it is to receive, what that bundle runs on, and the build
|
|
593 |
+ |
/// row that bundle came from.
|
|
594 |
+ |
///
|
|
595 |
+ |
/// The build id is what makes the gate check per-platform: evidence is keyed on
|
|
596 |
+ |
/// the build that produced the bytes, so a promote that ships two architectures
|
|
597 |
+ |
/// has to look up two sets of gate rows, and this is where it learns which two.
|
|
598 |
+ |
/// `None` is the legacy path (a bundle resolved by version string, from a tier
|
|
599 |
+ |
/// with no `current_build_id`), where there is no build to key on.
|
| 578 |
600 |
|
pub(super) type NodeBundle<'a> = (
|
| 579 |
601 |
|
&'a crate::topology::Node,
|
| 580 |
602 |
|
std::path::PathBuf,
|
| 581 |
603 |
|
Option<crate::domain::Platform>,
|
|
604 |
+ |
Option<i64>,
|
| 582 |
605 |
|
);
|
| 583 |
606 |
|
|
|
607 |
+ |
/// One bundle a promote is about to ship, and the evidence key it is judged by.
|
|
608 |
+ |
///
|
|
609 |
+ |
/// A single-platform product has exactly one of these and it is the source
|
|
610 |
+ |
/// tier's own build, which is what every promote before two-architecture support
|
|
611 |
+ |
/// checked. A product like pom has one per architecture, each standing on its
|
|
612 |
+ |
/// own intake and its own gate run.
|
|
613 |
+ |
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
614 |
+ |
pub(super) struct PromotedBuild {
|
|
615 |
+ |
/// What the bundle runs on, when it says. Used only to qualify the name of a
|
|
616 |
+ |
/// failing gate, so an operator reading "cargo_test" knows which half.
|
|
617 |
+ |
pub platform: Option<crate::domain::Platform>,
|
|
618 |
+ |
/// The `build_runs` row the evidence is keyed on; `None` falls back to the
|
|
619 |
+ |
/// version-keyed lookup for a pre-identity tier.
|
|
620 |
+ |
pub build_id: Option<i64>,
|
|
621 |
+ |
}
|
|
622 |
+ |
|
| 584 |
623 |
|
/// Which bytes each node gets, resolved before any node is touched.
|
| 585 |
624 |
|
///
|
| 586 |
625 |
|
/// A single-platform product answers with `fallback` for every node, which is
|
| 598 |
637 |
|
nodes: &[&'a crate::topology::Node],
|
| 599 |
638 |
|
fallback: &std::path::Path,
|
| 600 |
639 |
|
fallback_platform: Option<&crate::domain::Platform>,
|
|
640 |
+ |
fallback_build_id: Option<i64>,
|
| 601 |
641 |
|
) -> Result<Vec<NodeBundle<'a>>> {
|
| 602 |
642 |
|
let mut out = Vec::with_capacity(nodes.len());
|
| 603 |
643 |
|
for node in nodes.iter().copied() {
|
| 605 |
645 |
|
// The node states nothing, so there is nothing to resolve against;
|
| 606 |
646 |
|
// it gets the caller's bundle, and `Placement::check` decides
|
| 607 |
647 |
|
// whether that pairing is admissible at all.
|
| 608 |
|
- |
None => (fallback.to_path_buf(), fallback_platform.cloned()),
|
|
648 |
+ |
None => (
|
|
649 |
+ |
fallback.to_path_buf(),
|
|
650 |
+ |
fallback_platform.cloned(),
|
|
651 |
+ |
fallback_build_id,
|
|
652 |
+ |
),
|
| 609 |
653 |
|
// The node states a platform. Give it the bundle recorded for that
|
| 610 |
654 |
|
// platform at this version — the caller's own when they agree, its
|
| 611 |
655 |
|
// sibling when they do not.
|
| 612 |
|
- |
Some(want) if fallback_platform == Some(want) => {
|
| 613 |
|
- |
(fallback.to_path_buf(), fallback_platform.cloned())
|
| 614 |
|
- |
}
|
| 615 |
|
- |
Some(want) => (
|
| 616 |
|
- |
bundle_for_platform(s, version, want).await?,
|
| 617 |
|
- |
Some(want.clone()),
|
|
656 |
+ |
Some(want) if fallback_platform == Some(want) => (
|
|
657 |
+ |
fallback.to_path_buf(),
|
|
658 |
+ |
fallback_platform.cloned(),
|
|
659 |
+ |
fallback_build_id,
|
| 618 |
660 |
|
),
|
|
661 |
+ |
Some(want) => {
|
|
662 |
+ |
let (build_id, path) = bundle_for_platform(s, version, want).await?;
|
|
663 |
+ |
(path, Some(want.clone()), build_id)
|
|
664 |
+ |
}
|
| 619 |
665 |
|
};
|
| 620 |
|
- |
out.push((node, resolved.0, resolved.1));
|
|
666 |
+ |
out.push((node, resolved.0, resolved.1, resolved.2));
|
| 621 |
667 |
|
}
|
| 622 |
668 |
|
Ok(out)
|
| 623 |
669 |
|
}
|
| 624 |
670 |
|
|
|
671 |
+ |
/// The distinct builds `bundles` will ship, in a stable order.
|
|
672 |
+ |
///
|
|
673 |
+ |
/// Deduplicated because a tier is usually several nodes on one architecture, and
|
|
674 |
+ |
/// evaluating one build's gates once per node would say the same thing three
|
|
675 |
+ |
/// times in the error an operator reads. Keyed on the whole entry rather than on
|
|
676 |
+ |
/// the build id alone, so the legacy `None` case does not collapse two
|
|
677 |
+ |
/// version-resolved bundles into one.
|
|
678 |
+ |
pub(super) fn distinct_builds(bundles: &[NodeBundle<'_>]) -> Vec<PromotedBuild> {
|
|
679 |
+ |
let mut out: Vec<PromotedBuild> = Vec::new();
|
|
680 |
+ |
for (_, _, platform, build_id) in bundles {
|
|
681 |
+ |
let entry = PromotedBuild {
|
|
682 |
+ |
platform: platform.clone(),
|
|
683 |
+ |
build_id: *build_id,
|
|
684 |
+ |
};
|
|
685 |
+ |
if !out.contains(&entry) {
|
|
686 |
+ |
out.push(entry);
|
|
687 |
+ |
}
|
|
688 |
+ |
}
|
|
689 |
+ |
out
|
|
690 |
+ |
}
|
|
691 |
+ |
|
| 625 |
692 |
|
/// The bundle recorded for `version` on `platform`.
|
| 626 |
693 |
|
///
|
| 627 |
694 |
|
/// This is what makes a two-architecture product promotable. One pom version is
|
| 637 |
704 |
|
s: &AppState,
|
| 638 |
705 |
|
version: &str,
|
| 639 |
706 |
|
platform: &crate::domain::Platform,
|
| 640 |
|
- |
) -> Result<std::path::PathBuf> {
|
| 641 |
|
- |
let row: Option<(Option<String>,)> = sqlx::query_as(
|
| 642 |
|
- |
"SELECT staged_path FROM build_runs
|
|
707 |
+ |
) -> Result<(Option<i64>, std::path::PathBuf)> {
|
|
708 |
+ |
let row: Option<(i64, Option<String>)> = sqlx::query_as(
|
|
709 |
+ |
"SELECT id, staged_path FROM build_runs
|
| 643 |
710 |
|
WHERE app = ? AND version = ? AND platform = ? AND result = 'passed'
|
| 644 |
711 |
|
ORDER BY id DESC LIMIT 1",
|
| 645 |
712 |
|
)
|
| 651 |
718 |
|
.map_err(crate::error::Error::Db)?;
|
| 652 |
719 |
|
|
| 653 |
720 |
|
match row {
|
| 654 |
|
- |
Some((Some(path),)) => Ok(std::path::PathBuf::from(path)),
|
| 655 |
|
- |
Some((None,)) => Err(crate::error::Error::Other(anyhow::anyhow!(
|
|
721 |
+ |
// The id comes back alongside the path so the caller can check this
|
|
722 |
+ |
// build's own gate evidence rather than the source tier's. A green build
|
|
723 |
+ |
// row is not a green gate run: the build says the bytes compiled, the
|
|
724 |
+ |
// gate run says the tier vouched for them.
|
|
725 |
+ |
Some((id, Some(path))) => Ok((Some(id), std::path::PathBuf::from(path))),
|
|
726 |
+ |
Some((_, None)) => Err(crate::error::Error::Other(anyhow::anyhow!(
|
| 656 |
727 |
|
"the {platform} build of {version} has no staged_path; cannot promote it"
|
| 657 |
728 |
|
))),
|
| 658 |
729 |
|
None => Err(crate::error::Error::GateBlocked(format!(
|
| 724 |
795 |
|
// propagated (the promote is already failing), so it reports the node as
|
| 725 |
796 |
|
// indeterminate — which is the truth: nothing was attempted and the node's
|
| 726 |
797 |
|
// version is whatever the failed deploy left.
|
| 727 |
|
- |
let bundles = match bundles_for_nodes(s, prev_version, nodes, &staged_dir, None).await {
|
|
798 |
+ |
// No fallback build id: a rollback resolves the previous version's bundles
|
|
799 |
+ |
// by version string and re-runs no gates, so there is no evidence to key.
|
|
800 |
+ |
// The bytes are ones this tier already ran.
|
|
801 |
+ |
let bundles = match bundles_for_nodes(s, prev_version, nodes, &staged_dir, None, None).await {
|
| 728 |
802 |
|
Ok(b) => b,
|
| 729 |
803 |
|
Err(e) => {
|
| 730 |
804 |
|
tracing::error!(tier = %tier, prev = prev_version, error = %e,
|
| 737 |
811 |
|
};
|
| 738 |
812 |
|
|
| 739 |
813 |
|
let mut report = RollbackReport::default();
|
| 740 |
|
- |
for (node, node_bundle, node_bundle_platform) in &bundles {
|
|
814 |
+ |
for (node, node_bundle, node_bundle_platform, _) in &bundles {
|
| 741 |
815 |
|
let executor = s
|
| 742 |
816 |
|
.executors
|
| 743 |
817 |
|
.get(&node.name)
|
| 848 |
922 |
|
/// - every other kind requires a `passed` row for (tier, version) — a missing
|
| 849 |
923 |
|
/// or non-passed latest row counts as unsatisfied.
|
| 850 |
924 |
|
///
|
| 851 |
|
- |
/// `build_id` is the identity of the artifact being promoted (wiki note
|
| 852 |
|
- |
/// `release-artifact-identity`). When `Some`, the deterministic build-evidence
|
| 853 |
|
- |
/// gates are checked against the row that vouched for *that build* rather than
|
| 854 |
|
- |
/// against any row that happens to carry the version string — this is what stops
|
| 855 |
|
- |
/// a `promote --version Y` from riding on gate rows a different build left under
|
| 856 |
|
- |
/// the same version. `None` is the legacy/pre-identity path: fall back to the
|
| 857 |
|
- |
/// version-keyed lookup so a mid-migration tier (NULL `current_build_id`) still
|
| 858 |
|
- |
/// promotes. `burn_in` is clock-based either way — the clock is reset on every
|
| 859 |
|
- |
/// advance, so it belongs to the build now current on the tier.
|
|
925 |
+ |
/// `builds` are the artifacts this promote will actually ship (wiki note
|
|
926 |
+ |
/// `release-artifact-identity`). Each carries a `build_id`, and the deterministic
|
|
927 |
+ |
/// build-evidence gates are checked against the rows that vouched for *that
|
|
928 |
+ |
/// build* rather than against any row that happens to carry the version string —
|
|
929 |
+ |
/// this is what stops a `promote --version Y` from riding on gate rows a
|
|
930 |
+ |
/// different build left under the same version. A `None` build id is the
|
|
931 |
+ |
/// legacy/pre-identity path: fall back to the version-keyed lookup so a
|
|
932 |
+ |
/// mid-migration tier (NULL `current_build_id`) still promotes.
|
|
933 |
+ |
///
|
|
934 |
+ |
/// **One promote can ship several builds**, and every one of them is checked.
|
|
935 |
+ |
/// A pom version is two bundles with two digests, each built natively and
|
|
936 |
+ |
/// accepted through its own intake, so the source tier's evidence for one
|
|
937 |
+ |
/// architecture says nothing about the other. Checking only the build the tier
|
|
938 |
+ |
/// points at would let an x86_64 node take bytes whose `cargo_test` row nobody
|
|
939 |
+ |
/// looked at. When there is more than one, a failing gate is reported qualified
|
|
940 |
+ |
/// by platform, because "cargo_test not satisfied" is not actionable if the
|
|
941 |
+ |
/// operator cannot tell which half it is about.
|
|
942 |
+ |
///
|
|
943 |
+ |
/// `burn_in` and `manual_confirm` are evaluated once regardless: both are keyed
|
|
944 |
+ |
/// on the tier's own clock (`tier_state.burn_in_started_at`), not on a build, so
|
|
945 |
+ |
/// asking per build would ask the same question N times and answer it N times in
|
|
946 |
+ |
/// the error.
|
| 860 |
947 |
|
pub(super) async fn unsatisfied_gates(
|
| 861 |
948 |
|
pool: &sqlx::SqlitePool,
|
| 862 |
949 |
|
app: &crate::domain::AppId,
|
| 863 |
950 |
|
tier: &crate::domain::TierId,
|
| 864 |
951 |
|
gates: &[crate::topology::Gate],
|
| 865 |
952 |
|
version: &str,
|
| 866 |
|
- |
build_id: Option<i64>,
|
|
953 |
+ |
builds: &[PromotedBuild],
|
| 867 |
954 |
|
hotfix: bool,
|
| 868 |
955 |
|
) -> std::result::Result<Vec<String>, crate::error::Error> {
|
| 869 |
956 |
|
use crate::topology::Gate;
|
| 946 |
1033 |
|
| Gate::NodeHealth => {
|
| 947 |
1034 |
|
// Latest row for this configured gate kind; NULL/missing/any
|
| 948 |
1035 |
|
// non-'passed' status all count as unsatisfied (fail closed).
|
| 949 |
|
- |
// Keyed on build_id when the artifact has an identity (the
|
| 950 |
|
- |
// evidence must be for *this* build), else on the version string.
|
| 951 |
|
- |
let status: Option<String> = match build_id {
|
| 952 |
|
- |
Some(bid) => sqlx::query_scalar(
|
| 953 |
|
- |
"SELECT status FROM gate_runs
|
| 954 |
|
- |
WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
|
| 955 |
|
- |
ORDER BY id DESC LIMIT 1",
|
| 956 |
|
- |
)
|
| 957 |
|
- |
.bind(app)
|
| 958 |
|
- |
.bind(tier.as_str())
|
| 959 |
|
- |
.bind(bid)
|
| 960 |
|
- |
.bind(kind.as_str()),
|
| 961 |
|
- |
None => sqlx::query_scalar(
|
| 962 |
|
- |
"SELECT status FROM gate_runs
|
| 963 |
|
- |
WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
|
| 964 |
|
- |
ORDER BY id DESC LIMIT 1",
|
| 965 |
|
- |
)
|
| 966 |
|
- |
.bind(app)
|
| 967 |
|
- |
.bind(tier.as_str())
|
| 968 |
|
- |
.bind(version)
|
| 969 |
|
- |
.bind(kind.as_str()),
|
| 970 |
|
- |
}
|
| 971 |
|
- |
.fetch_optional(pool)
|
| 972 |
|
- |
.await
|
| 973 |
|
- |
.map_err(crate::error::Error::Db)?
|
| 974 |
|
- |
.flatten();
|
| 975 |
|
- |
if status.as_deref() != Some("passed") {
|
| 976 |
|
- |
bad.push(kind.as_str().to_string());
|
|
1036 |
+ |
// Every build this promote ships has to show its own passed row.
|
|
1037 |
+ |
// An empty `builds` is the caller saying "no identities to key
|
|
1038 |
+ |
// on"; one version-keyed lookup is the pre-identity behaviour.
|
|
1039 |
+ |
let lookups: &[PromotedBuild] = if builds.is_empty() {
|
|
1040 |
+ |
&[PromotedBuild {
|
|
1041 |
+ |
platform: None,
|
|
1042 |
+ |
build_id: None,
|
|
1043 |
+ |
}]
|
|
1044 |
+ |
} else {
|
|
1045 |
+ |
builds
|
|
1046 |
+ |
};
|
|
1047 |
+ |
let qualify = lookups.len() > 1;
|
|
1048 |
+ |
for b in lookups {
|
|
1049 |
+ |
// Keyed on build_id when the artifact has an identity (the
|
|
1050 |
+ |
// evidence must be for *this* build), else on the version.
|
|
1051 |
+ |
let status: Option<String> = match b.build_id {
|
|
1052 |
+ |
Some(bid) => sqlx::query_scalar(
|
|
1053 |
+ |
"SELECT status FROM gate_runs
|
|
1054 |
+ |
WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
|
|
1055 |
+ |
ORDER BY id DESC LIMIT 1",
|
|
1056 |
+ |
)
|
|
1057 |
+ |
.bind(app)
|
|
1058 |
+ |
.bind(tier.as_str())
|
|
1059 |
+ |
.bind(bid)
|
|
1060 |
+ |
.bind(kind.as_str()),
|
|
1061 |
+ |
None => sqlx::query_scalar(
|
|
1062 |
+ |
"SELECT status FROM gate_runs
|
|
1063 |
+ |
WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
|
|
1064 |
+ |
ORDER BY id DESC LIMIT 1",
|
|
1065 |
+ |
)
|
|
1066 |
+ |
.bind(app)
|
|
1067 |
+ |
.bind(tier.as_str())
|
|
1068 |
+ |
.bind(version)
|
|
1069 |
+ |
.bind(kind.as_str()),
|
|
1070 |
+ |
}
|
|
1071 |
+ |
.fetch_optional(pool)
|
|
1072 |
+ |
.await
|
|
1073 |
+ |
.map_err(crate::error::Error::Db)?
|
|
1074 |
+ |
.flatten();
|
|
1075 |
+ |
if status.as_deref() != Some("passed") {
|
|
1076 |
+ |
// Qualified only when there is more than one build to
|
|
1077 |
+ |
// tell apart, so a single-platform product's message is
|
|
1078 |
+ |
// exactly what it always was.
|
|
1079 |
+ |
let name = match (&b.platform, qualify) {
|
|
1080 |
+ |
(Some(p), true) => format!("{} ({p})", kind.as_str()),
|
|
1081 |
+ |
_ => kind.as_str().to_string(),
|
|
1082 |
+ |
};
|
|
1083 |
+ |
if !bad.contains(&name) {
|
|
1084 |
+ |
bad.push(name);
|
|
1085 |
+ |
}
|
|
1086 |
+ |
}
|
| 977 |
1087 |
|
}
|
| 978 |
1088 |
|
}
|
| 979 |
1089 |
|
}
|