Skip to main content

max / makenotwork

57.5 KB · 1244 lines History Blame Raw
1 //! Promotion service: the deploy state machine lifted out of the HTTP
2 //! layer — promote orchestration, canary rollback of deployed nodes,
3 //! partial-state flags, and the promote-time gate-satisfaction check.
4 //! routes/mod.rs stays HTTP glue that calls into these.
5
6 use super::{AppState, Json, PromoteBody, Result};
7
8 pub(super) async fn promote_inner(
9 s: AppState,
10 tier: String,
11 body: PromoteBody,
12 ) -> Result<Json<serde_json::Value>> {
13 // Serialize the whole check -> deploy -> advance against any concurrent
14 // promote/rollback (CF3). Held for the entire task.
15 let _deploy_guard = s.deploy_lock.lock().await;
16 let tier = crate::domain::TierId::new(tier);
17 let idx = s
18 .topo
19 .tiers
20 .iter()
21 .position(|t| t.name == tier)
22 .ok_or(crate::error::Error::NotFound)?;
23 if idx == 0 {
24 return Err(crate::error::Error::GateBlocked(
25 "cannot /promote to the first tier; use /rebuild".into(),
26 ));
27 }
28 let target = &s.topo.tiers[idx];
29 let source = &s.topo.tiers[idx - 1];
30 // An unprovisioned tier has no nodes, so every step below is a no-op that
31 // still *looks* like success: the deploy loop iterates nothing, node_health
32 // returns Blocked, and advance_tier writes a current_version for a tier that
33 // has never received a byte. /state then reports tier c running a version it
34 // does not have. Refuse instead.
35 if !target.provisioned {
36 return Err(crate::error::Error::GateBlocked(format!(
37 "tier {} is not provisioned (no nodes); promoting to it would record a version it never received",
38 target.name,
39 )));
40 }
41
42 // Resolve the artifact through the SOURCE tier's evidence, not a version
43 // string (invariant 2, wiki [[release-artifact-identity]]). The build that is
44 // *current on the source tier* is the one that burned in and passed its gates
45 // there; promoting anything else lets a version string ride on another build's
46 // evidence and clock — the "promote --version Y inherits X's 48h" hole. A
47 // given `body.version` must name that same build.
48 let source_build: Option<(Option<i64>, Option<String>, Option<String>, Option<String>)> =
49 sqlx::query_as(
50 "SELECT ts.current_build_id, br.version, br.staged_path, br.platform
51 FROM tier_state ts
52 LEFT JOIN build_runs br ON br.id = ts.current_build_id
53 WHERE ts.app = ? AND ts.tier = ?",
54 )
55 .bind(&s.cfg.id)
56 .bind(&source.name)
57 .fetch_optional(&s.pool)
58 .await
59 .map_err(crate::error::Error::Db)?;
60
61 let (build_id, version_str, staged_dir, source_platform) = match source_build {
62 // Identity path: the source tier points at a recorded build.
63 Some((Some(bid), ver, staged_path, platform)) => {
64 let version_str = ver.ok_or_else(|| {
65 crate::error::Error::Other(anyhow::anyhow!(
66 "source build {bid} on tier {} has no recorded version",
67 source.name
68 ))
69 })?;
70 let staged_path = staged_path.ok_or_else(|| {
71 crate::error::Error::Other(anyhow::anyhow!(
72 "source build {bid} ({version_str}) has no staged_path; cannot promote"
73 ))
74 })?;
75 if let Some(req) = &body.version
76 && req != &version_str
77 {
78 return Err(crate::error::Error::GateBlocked(format!(
79 "tier {} is running build {bid} ({version_str}); refusing to promote an \
80 explicit version {req} that is not the build the tier vouched for",
81 source.name
82 )));
83 }
84 let platform = platform
85 .as_deref()
86 .map(crate::domain::Platform::parse)
87 .transpose()
88 .map_err(|e| crate::error::Error::Other(anyhow::anyhow!(e)))?;
89 (
90 Some(bid),
91 version_str,
92 std::path::PathBuf::from(staged_path),
93 platform,
94 )
95 }
96 // Legacy fallback: NULL current_build_id (a pre-identity tier). Resolve by
97 // version string through the versions table, exactly as before. One clean
98 // build-and-promote cycle re-anchors the tier onto the identity path.
99 _ => {
100 let version_str = match body.version.clone() {
101 Some(v) => v,
102 None => sqlx::query_scalar::<_, Option<String>>(
103 "SELECT current_version FROM tier_state WHERE app = ? AND tier = ?",
104 )
105 .bind(&s.cfg.id)
106 .bind(&source.name)
107 .fetch_optional(&s.pool)
108 .await
109 .map_err(crate::error::Error::Db)?
110 .flatten()
111 .ok_or_else(|| {
112 crate::error::Error::GateBlocked(format!(
113 "no version specified and tier {} has no current_version",
114 source.name
115 ))
116 })?,
117 };
118 let bin: Option<(String,)> =
119 sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
120 .bind(&s.cfg.id)
121 .bind(&version_str)
122 .fetch_optional(&s.pool)
123 .await
124 .map_err(crate::error::Error::Db)?;
125 let Some((bin,)) = bin else {
126 return Err(crate::error::Error::NotFound);
127 };
128 // `artifact_path` is the primary binary; the release dir is its parent.
129 let staged_dir = std::path::PathBuf::from(&bin)
130 .parent()
131 .ok_or_else(|| {
132 crate::error::Error::Other(anyhow::anyhow!("artifact_path has no parent"))
133 })?
134 .to_path_buf();
135 (None, version_str, staged_dir, None)
136 }
137 };
138 let version = crate::domain::Version::parse(&version_str)
139 .map_err(|e| crate::error::Error::Other(anyhow::anyhow!(e)))?;
140
141 // 1. Predecessor must have all of its configured gates satisfied for this
142 // build (with optional hotfix override that skips burn_in). Evaluated
143 // against the topology gate list, so a gate that never ran blocks the
144 // promote instead of being treated as green.
145 //
146 // A migration-bearing promote (`bears_migration`) forces a fresh
147 // `manual_confirm` on the predecessor even if the tier does not configure
148 // one: rollback restores the binary + release_contents only, never the
149 // schema, so the operator must consciously acknowledge the one-way advance
150 // (deploy/README.md "Rollback contract"). `hotfix` skips only `burn_in`, so
151 // it does not suppress this confirm.
152 let mut effective_gates = source.gates.clone();
153 if body.bears_migration
154 && !effective_gates
155 .iter()
156 .any(|g| matches!(g, crate::topology::Gate::ManualConfirm))
157 {
158 effective_gates.push(crate::topology::Gate::ManualConfirm);
159 }
160 // Which bytes each node gets is resolved FIRST, because it is what says
161 // how many builds the gate check has to cover. A two-architecture promote
162 // ships two bundles with two sets of evidence, and checking only the one
163 // the source tier points at would wave the sibling through on gate rows
164 // nobody read. It also means a version missing its other half fails here,
165 // before any gate work, rather than halfway down the rollout.
166 let target_nodes: Vec<&crate::topology::Node> = target.nodes.iter().collect();
167 let bundles = bundles_for_nodes(
168 &s,
169 &version_str,
170 &target_nodes,
171 &staged_dir,
172 source_platform.as_ref(),
173 build_id,
174 )
175 .await?;
176 let mut promoted_builds = distinct_builds(&bundles);
177 if promoted_builds.is_empty() {
178 // A provisioned tier with no nodes ships nothing, so there is nothing to
179 // resolve — but the gate check still has to be keyed on the source build.
180 // Left empty it would fall through to the version-string lookup, which is
181 // the pre-identity path and weaker than what this tier is owed. The
182 // `provisioned` guard above makes this unreachable today; it is written
183 // out because the alternative fails quietly in the direction of less
184 // evidence.
185 promoted_builds.push(PromotedBuild {
186 platform: source_platform.clone(),
187 build_id,
188 });
189 }
190
191 // The build the TARGET tier ends up running. The same as the source's for a
192 // single-platform product, and its cross-architecture sibling when the two
193 // tiers differ — promoting pom astra -> hetzner reads an aarch64 build from
194 // the source and ships the x86_64 one. Every row this promote writes about
195 // the target (its `deploys`, its post-deploy gate runs, its `tier_state`)
196 // keys on this, so the next promote out of that tier resolves through
197 // evidence for bytes that box actually ran.
198 //
199 // A tier whose nodes span platforms has no single current build, and NULL
200 // says so rather than naming one of them arbitrarily. No such tier exists
201 // today: pom's astra and hetzner tiers are one node each.
202 let target_build_id = match promoted_builds.as_slice() {
203 [only] => only.build_id,
204 _ => None,
205 };
206
207 let pending = unsatisfied_gates(
208 &s.pool,
209 &s.cfg.id,
210 &source.name,
211 &effective_gates,
212 &Evidence {
213 version: &version_str,
214 builds: &promoted_builds,
215 // The source tier's own build: what its node_health / page_smoke rows
216 // can possibly be about, which is not the same thing as what this
217 // promote ships when the two tiers are different architectures.
218 tier_build: build_id,
219 },
220 body.hotfix,
221 )
222 .await?;
223 if !pending.is_empty() {
224 return Err(crate::error::Error::GateBlocked(format!(
225 "{} gate(s) not satisfied on tier {}: {}",
226 pending.len(),
227 source.name,
228 pending.join(", "),
229 )));
230 }
231
232 // The version this tier was running before this promote — the rollback
233 // target if a canary node fails partway through a multi-node rollout.
234 let prev_version: Option<String> =
235 sqlx::query_scalar("SELECT current_version FROM tier_state WHERE app = ? AND tier = ?")
236 .bind(&s.cfg.id)
237 .bind(&target.name)
238 .fetch_optional(&s.pool)
239 .await
240 .map_err(crate::error::Error::Db)?
241 .flatten();
242
243 // Which release dirs the node gc must not touch. Read once, before the
244 // loop: it is the same set for every node (each mirrors the host's
245 // directory name) and reading it per node would let it change mid-rollout.
246 //
247 // Read BEFORE the first deploy on purpose. `tier_state` still holds the
248 // pre-promote current/previous here, which is exactly the pair a canary
249 // rollback needs to find on the node. The build being shipped now protects
250 // itself: it is the newest directory by mtime the moment it lands, so the
251 // count covers it until `tier_state` catches up.
252 //
253 // A failure to read it fails the promote, matching the host store's rule
254 // (`build::publish` propagates the same error rather than gc'ing blind).
255 // Nothing has been deployed at this point, so failing here is free.
256 let pinned = crate::retention::pinned_dirs(&s.pool, &s.cfg.id)
257 .await
258 .map_err(crate::error::Error::Other)?;
259
260 // 3. Deploy to each node. Sequential canary is the only policy
261 // implemented in v0; parallel is a one-line change once we trust the
262 // sequential path. Track the nodes already flipped to the new version so
263 // a mid-rollout failure can roll them back (canary rollback).
264 let mut deployed: Vec<&crate::topology::Node> = Vec::new();
265 for (node, node_bundle, node_bundle_platform, node_build_id) in &bundles {
266 // The proof that these bytes belong on this box. Built before the
267 // deploy row is written, so a mismatch never becomes an `in_progress`
268 // deploy that has to be reconciled.
269 let placement =
270 crate::deploy::Placement::check(node, node_bundle, node_bundle_platform.as_ref())
271 .map_err(|e| crate::error::Error::GateBlocked(e.to_string()))?;
272 let started = chrono::Utc::now().to_rfc3339();
273 crate::events::emit(
274 &s.events,
275 crate::events::Event::DeployStart {
276 tier: target.name.clone(),
277 node: node.name.clone(),
278 version: version.clone(),
279 },
280 );
281 let executor = s
282 .executors
283 .get(&node.name)
284 .cloned()
285 .unwrap_or_else(|| crate::state::build_executor(node));
286 // Record the deploy as `in_progress` BEFORE touching the node, so a crash
287 // between the node's symlink swap and this promote's `advance_tier` leaves
288 // a durable trace. Without the pre-write, a SIGKILL after the swap but
289 // before the row is inserted would land the new binary on the node with no
290 // DB evidence at all, and the startup reconcile — which reads `deploys` —
291 // could not see it. The row is finalized to ok/failed immediately below;
292 // any row still `in_progress` at startup is an orphan the reconcile settles
293 // and flags ([`crate::reconcile`]).
294 let deploy_id: i64 = sqlx::query_scalar(
295 "INSERT INTO deploys (app, version, tier, node, started_at, outcome, hotfix, reset_burn_in, build_id)
296 VALUES (?, ?, ?, ?, ?, 'in_progress', ?, ?, ?) RETURNING id",
297 )
298 .bind(&s.cfg.id)
299 .bind(&version).bind(&target.name).bind(&node.name)
300 .bind(&started)
301 .bind(body.hotfix as i64).bind(body.reset_burn_in as i64)
302 // The build THIS node received, which is not the source tier's when the
303 // two are different architectures: promoting pom astra -> hetzner reads
304 // an aarch64 build from the source and ships the x86_64 sibling. Binding
305 // the source's id here recorded the wrong bytes against the deploy, and
306 // `crate::reconcile` reads these rows.
307 .bind(node_build_id)
308 .fetch_one(&s.pool).await.map_err(crate::error::Error::Db)?;
309 let result = crate::deploy::deploy_node(
310 executor.as_ref(),
311 placement,
312 &version_str,
313 s.cfg.primary_bin(),
314 Some(&pinned),
315 )
316 .await;
317 let finished = chrono::Utc::now().to_rfc3339();
318 let (outcome_obj, err_for_propagation) = match result {
319 Ok(_) => (crate::outcome::DeployOutcome::ok(), None),
320 Err(e) => {
321 let msg = format!("{e:#}");
322 let kind = crate::classify::classify_deploy_error(&msg);
323 (crate::outcome::DeployOutcome::failed(kind), Some(e))
324 }
325 };
326 let outcome_json = serde_json::to_string(&outcome_obj)
327 .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
328 sqlx::query(
329 "UPDATE deploys SET finished_at = ?, outcome = ?, outcome_json = ? WHERE id = ?",
330 )
331 .bind(&finished)
332 .bind(outcome_obj.status_str())
333 .bind(&outcome_json)
334 .bind(deploy_id)
335 .execute(&s.pool)
336 .await
337 .map_err(crate::error::Error::Db)?;
338 if let Some(e) = err_for_propagation {
339 let crate::outcome::DeployStatus::Failed { failure } = outcome_obj.status else {
340 unreachable!("err_for_propagation is Some iff status is Failed");
341 };
342 tracing::error!(
343 tier = %target.name, node = %node.name, version = %version,
344 failure = failure.summary(),
345 "deploy failed; current symlink left intact, tier_state not advanced"
346 );
347 crate::events::emit(
348 &s.events,
349 crate::events::Event::DeployFailed {
350 tier: target.name.clone(),
351 node: node.name.clone(),
352 version: version.clone(),
353 failure,
354 },
355 );
356
357 // Canary rollback: restore every node this promote touched — the
358 // ones already flipped to the new version AND this failed node
359 // (whose state is indeterminate: the symlink swap may have landed
360 // before the restart failed) — back to the tier's prior version, so
361 // the fleet is left consistent on `prev` rather than split-brain.
362 // Nodes after this one were never touched and stay on `prev`.
363 deployed.push(node);
364 let touched = deployed.len();
365 match prev_version.as_deref() {
366 Some(prev) => {
367 let report = rollback_deployed_nodes(&s, &target.name, &deployed, prev).await;
368 // Say what is true, and say it differently when nothing is
369 // wrong. `restored=0 of=1` read as total failure when the
370 // accurate reading was "0 needed restoring" — the same
371 // confusion as the per-node message, one level up.
372 if report.is_consistent() {
373 tracing::warn!(
374 tier = %target.name,
375 restored = report.restored,
376 already_on_previous = report.already_on_previous,
377 of = report.touched(),
378 from = %version, to = prev,
379 "canary failed mid-rollout; every touched node is on the previous \
380 version and the tier is consistent",
381 );
382 } else {
383 tracing::error!(
384 tier = %target.name,
385 restored = report.restored,
386 already_on_previous = report.already_on_previous,
387 indeterminate = report.indeterminate,
388 of = report.touched(),
389 from = %version, to = prev,
390 "canary failed mid-rollout and the tier is NOT consistent; some nodes \
391 have an indeterminate version",
392 );
393 }
394 if report.restored > 0
395 && let Ok(prev_v) = crate::domain::Version::parse(prev)
396 {
397 crate::events::emit(
398 &s.events,
399 crate::events::Event::Rollback {
400 tier: target.name.clone(),
401 from: version.clone(),
402 to: prev_v,
403 },
404 );
405 }
406 // The tier is consistent when no node's version is unknown —
407 // which includes the case where a rollback "failed" before
408 // the swap and so left the node on `prev` already. Flagging
409 // that as partial would put a permanent scare on /state for
410 // a fleet that is entirely on one version.
411 if report.is_consistent() {
412 clear_partial(&s, &target.name).await;
413 } else {
414 set_partial(&s, &target.name, &format!(
415 "canary rollback incomplete: {indeterminate} of {touched} node(s) have an \
416 indeterminate version and may be on {version}; {restored} restored to \
417 {prev}, {already} were never swapped — manual check needed",
418 touched = report.touched(),
419 indeterminate = report.indeterminate,
420 restored = report.restored,
421 already = report.already_on_previous,
422 )).await;
423 }
424 }
425 None => {
426 tracing::error!(
427 tier = %target.name, count = touched, version = %version,
428 "canary failed on a first deploy (no previous version to restore to); \
429 touched nodes remain on the new version — manual cleanup needed",
430 );
431 set_partial(
432 &s,
433 &target.name,
434 &format!(
435 "first-deploy canary failed: {touched} node(s) left on {version}, \
436 no prior version to restore — manual cleanup needed",
437 ),
438 )
439 .await;
440 }
441 }
442 return Err(crate::error::Error::Other(e));
443 }
444 deployed.push(node);
445 crate::events::emit(
446 &s.events,
447 crate::events::Event::DeployOk {
448 tier: target.name.clone(),
449 node: node.name.clone(),
450 version: version.clone(),
451 },
452 );
453 }
454
455 // 3b. Run this tier's post-deploy gates (node_health) against the freshly
456 // deployed nodes and record their outcomes. These rows are the evidence
457 // the NEXT promote (this tier -> the following one) checks via
458 // `unsatisfied_gates`. Before CF1, only the host tier ran gates, so
459 // A/B/C had no evidence and promotion waved through; node_health now
460 // proves the deployed nodes are serving (Run-2 SERIOUS-3: boot_smoke
461 // used to re-run the staged binary locally and proved nothing about the
462 // node). burn_in / manual_confirm are not run here — they are evaluated
463 // live / by the operator at the next promote. A failed gate does not
464 // unwind this deploy (the artifact is already live on the tier); it
465 // blocks the next promote, which is the fail-closed behavior we want.
466 //
467 // It does, however, fail *this* promote's response. tier_state still
468 // advances below — the nodes genuinely run this version and a stale
469 // `current_version` would send a later rollback to the wrong artifact —
470 // but the tier is flagged partial and the handler returns the failure,
471 // so the operator learns at the promote instead of discovering it as an
472 // unexplained block on the next one.
473 let post_deploy: Vec<crate::topology::Gate> = target
474 .gates
475 .iter()
476 .filter(|g| g.runs_post_deploy())
477 .cloned()
478 .collect();
479 let mut post_deploy_failure: Option<String> = None;
480 if !post_deploy.is_empty() {
481 // node_health probes each node the deploy just shipped to, over the same
482 // executor the deploy used. Build the probe set from the tier's nodes and
483 // the startup executor map; a node missing an executor (shouldn't happen
484 // — both come from the same topology) is skipped, and an empty set makes
485 // node_health Blocked (fail closed).
486 let nodes: Vec<crate::gates::NodeProbe> = target
487 .nodes
488 .iter()
489 .filter_map(|n| {
490 s.executors
491 .get(&n.name)
492 .map(|exec| crate::gates::NodeProbe {
493 node: n.name.clone(),
494 service: n.service_name.clone(),
495 health_url: n.health_url.clone(),
496 executor: exec.clone(),
497 })
498 })
499 .collect();
500 let ctx = crate::gates::GateCtx {
501 pool: s.pool.clone(),
502 cfg: s.cfg.clone(),
503 tier: target.name.clone(),
504 version: version.clone(),
505 // No worktree at promote time; node_health works over executors, not
506 // a checkout. No bundle either: the gates here are about the nodes,
507 // not about bytes on this host.
508 worktree: None,
509 bundle: None,
510 events: s.events.clone(),
511 nodes,
512 // These post-deploy gate rows are the evidence the NEXT promote (this
513 // tier -> the following one) resolves through, so they must carry the
514 // build they vouch for — the one this tier just received, which is
515 // not the source's across an architecture boundary.
516 build_id: target_build_id,
517 // The hostname the public uses, straight from the tier. page_smoke
518 // has to request the site the way a visitor does; anything derived
519 // from a node would reach the origin and miss the CDN, which is the
520 // layer the gate exists to watch.
521 public_url: target.public_url.clone(),
522 // No checkout at promote time, so nothing to resolve against.
523 aux_dirs: std::collections::HashMap::new(),
524 };
525 post_deploy_failure = match crate::gates::run_all(&ctx, &post_deploy).await {
526 Ok(failed) if failed.is_empty() => None,
527 Ok(failed) => {
528 let names = failed
529 .iter()
530 .map(|k| k.as_str())
531 .collect::<Vec<_>>()
532 .join(", ");
533 tracing::warn!(
534 tier = %target.name, version = %version, gates = %names,
535 "post-deploy gate(s) failed; tier advanced but promotion to the next tier is blocked",
536 );
537 Some(format!(
538 "post-deploy gate(s) failed on {version}: {names}; \
539 the tier is serving {version} but cannot promote onward until they pass"
540 ))
541 }
542 Err(e) => {
543 tracing::error!(
544 tier = %target.name, version = %version, error = %e,
545 "post-deploy gate execution errored; promotion to the next tier is blocked",
546 );
547 Some(format!(
548 "post-deploy gate execution errored on {version}: {e}; \
549 the tier is serving {version} but cannot promote onward until the gates pass"
550 ))
551 }
552 };
553 }
554
555 // 4. Advance tier_state through the single sealed forward-advance op (atomic
556 // self-referential UPDATE; no read-modify-write to lose under concurrency,
557 // CF3). We hold deploy_lock for this whole handler, so the advance is
558 // serialized against rollback and the host build path's advance.
559 // reset_burn_in on the *source* tier nulls its clock only when the operator
560 // explicitly asked.
561 // `target_build_id`, not the source's: recording the source's would have
562 // left hetzner's `current_build_id` pointing at astra's aarch64 bundle,
563 // i.e. at bytes that box cannot execute.
564 crate::runs::advance_tier(
565 &s.pool,
566 &s.cfg.id,
567 target.name.as_str(),
568 &version,
569 target_build_id,
570 )
571 .await
572 .map_err(crate::error::Error::Db)?;
573
574 if body.reset_burn_in {
575 sqlx::query("UPDATE tier_state SET burn_in_started_at = NULL WHERE app = ? AND tier = ?")
576 .bind(&s.cfg.id)
577 .bind(&source.name)
578 .execute(&s.pool)
579 .await
580 .map_err(crate::error::Error::Db)?;
581 }
582
583 // Red post-deploy gates: the rollout itself reached every node, but the tier
584 // is not fit to promote onward. Flag it so /state and the TUI say so, and
585 // return the failure rather than a 200 the operator would read as "shipped,
586 // all good". tier_state has already advanced above — it describes what the
587 // nodes are running, not whether we're happy about it.
588 if let Some(reason) = post_deploy_failure {
589 set_partial(&s, &target.name, &reason).await;
590 return Err(crate::error::Error::GateBlocked(reason));
591 }
592
593 // A clean full rollout to every node clears any prior partial flag on this tier.
594 clear_partial(&s, &target.name).await;
595
596 crate::events::emit(
597 &s.events,
598 crate::events::Event::PromoteComplete {
599 tier: target.name.clone(),
600 version: version.clone(),
601 },
602 );
603 tracing::info!(
604 version = %version, tier = %target.name,
605 hotfix = body.hotfix, reset_burn_in = body.reset_burn_in,
606 "promote complete",
607 );
608
609 Ok(Json(serde_json::json!({
610 "tier": target.name,
611 "version": version,
612 "nodes_deployed": target.nodes.iter().map(|n| n.name.clone()).collect::<Vec<_>>(),
613 })))
614 }
615
616 /// What a canary rollback actually left behind, per node.
617 ///
618 /// Three outcomes, not two, and conflating the middle one with the last is the
619 /// bug this type exists to prevent: a rollback that fails *before* the symlink
620 /// swap leaves the node exactly where it already was, on the previous version.
621 /// Reporting that as "stranded on the new version, manual intervention needed"
622 /// sends an operator to do surgery on a healthy production box.
623 #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
624 pub(super) struct RollbackReport {
625 /// Put back on the previous version by a successful redeploy.
626 pub(super) restored: usize,
627 /// Rollback failed before the swap, so the node never left the previous
628 /// version. Nothing is stranded and nothing needs doing.
629 pub(super) already_on_previous: usize,
630 /// Rollback failed at or after the swap, or could not be attempted at all.
631 /// The node's version is not knowable from here. This is the only outcome
632 /// that warrants a human.
633 pub(super) indeterminate: usize,
634 }
635
636 impl RollbackReport {
637 /// Every touched node accounted for.
638 pub(super) fn touched(self) -> usize {
639 self.restored + self.already_on_previous + self.indeterminate
640 }
641
642 /// True when no node is in an unknown state, whether or not every rollback
643 /// "succeeded". A tier whose rollbacks all failed before the swap is
644 /// consistent on the previous version and is not an incident.
645 pub(super) fn is_consistent(self) -> bool {
646 self.indeterminate == 0
647 }
648 }
649
650 /// A node, the bundle it is to receive, what that bundle runs on, and the build
651 /// row that bundle came from.
652 ///
653 /// The build id is what makes the gate check per-platform: evidence is keyed on
654 /// the build that produced the bytes, so a promote that ships two architectures
655 /// has to look up two sets of gate rows, and this is where it learns which two.
656 /// `None` is the legacy path (a bundle resolved by version string, from a tier
657 /// with no `current_build_id`), where there is no build to key on.
658 pub(super) type NodeBundle<'a> = (
659 &'a crate::topology::Node,
660 std::path::PathBuf,
661 Option<crate::domain::Platform>,
662 Option<i64>,
663 );
664
665 /// One bundle a promote is about to ship, and the evidence key it is judged by.
666 ///
667 /// A single-platform product has exactly one of these and it is the source
668 /// tier's own build, which is what every promote before two-architecture support
669 /// checked. A product like pom has one per architecture, each standing on its
670 /// own intake and its own gate run.
671 #[derive(Debug, Clone, PartialEq, Eq)]
672 pub(super) struct PromotedBuild {
673 /// What the bundle runs on, when it says. Used only to qualify the name of a
674 /// failing gate, so an operator reading "cargo_test" knows which half.
675 pub platform: Option<crate::domain::Platform>,
676 /// The `build_runs` row the evidence is keyed on; `None` falls back to the
677 /// version-keyed lookup for a pre-identity tier.
678 pub build_id: Option<i64>,
679 }
680
681 /// What a promote is judged against, in one value: the release it names, the
682 /// artifacts it will ship, and the build the tier it is leaving is running.
683 ///
684 /// The three travel together because the gate check needs all three at once and
685 /// they answer different questions. `version` is the label and the pre-identity
686 /// fallback key. `builds` are the bytes going out, which artifact evidence is
687 /// keyed on. `tier_build` is what the source tier is itself running, which
688 /// post-deploy evidence is keyed on — and across an architecture boundary those
689 /// last two are different builds. See [`unsatisfied_gates`].
690 pub(super) struct Evidence<'a> {
691 pub version: &'a str,
692 pub builds: &'a [PromotedBuild],
693 pub tier_build: Option<i64>,
694 }
695
696 /// Which bytes each node gets, resolved before any node is touched.
697 ///
698 /// A single-platform product answers with `fallback` for every node, which is
699 /// the bundle the caller already resolved (the source tier's build on a promote,
700 /// the previous version's on a rollback). A product whose one version is several
701 /// bundles answers per node, because the node is the thing that says which
702 /// architecture it can run.
703 ///
704 /// Resolving the whole tier up front is deliberate: a version missing its
705 /// x86_64 half fails the promote before its aarch64 half has been pushed
706 /// anywhere, instead of halfway down a rollout with nodes already flipped.
707 pub(super) async fn bundles_for_nodes<'a>(
708 s: &AppState,
709 version: &str,
710 nodes: &[&'a crate::topology::Node],
711 fallback: &std::path::Path,
712 fallback_platform: Option<&crate::domain::Platform>,
713 fallback_build_id: Option<i64>,
714 ) -> Result<Vec<NodeBundle<'a>>> {
715 let mut out = Vec::with_capacity(nodes.len());
716 for node in nodes.iter().copied() {
717 let resolved = match &node.platform {
718 // The node states nothing, so there is nothing to resolve against;
719 // it gets the caller's bundle, and `Placement::check` decides
720 // whether that pairing is admissible at all.
721 None => (
722 fallback.to_path_buf(),
723 fallback_platform.cloned(),
724 fallback_build_id,
725 ),
726 // The node states a platform. Give it the bundle recorded for that
727 // platform at this version — the caller's own when they agree, its
728 // sibling when they do not.
729 Some(want) if fallback_platform == Some(want) => (
730 fallback.to_path_buf(),
731 fallback_platform.cloned(),
732 fallback_build_id,
733 ),
734 Some(want) => {
735 let (build_id, path) = bundle_for_platform(s, version, want).await?;
736 (path, Some(want.clone()), build_id)
737 }
738 };
739 out.push((node, resolved.0, resolved.1, resolved.2));
740 }
741 Ok(out)
742 }
743
744 /// The distinct builds `bundles` will ship, in a stable order.
745 ///
746 /// Deduplicated because a tier is usually several nodes on one architecture, and
747 /// evaluating one build's gates once per node would say the same thing three
748 /// times in the error an operator reads. Keyed on the whole entry rather than on
749 /// the build id alone, so the legacy `None` case does not collapse two
750 /// version-resolved bundles into one.
751 pub(super) fn distinct_builds(bundles: &[NodeBundle<'_>]) -> Vec<PromotedBuild> {
752 let mut out: Vec<PromotedBuild> = Vec::new();
753 for (_, _, platform, build_id) in bundles {
754 let entry = PromotedBuild {
755 platform: platform.clone(),
756 build_id: *build_id,
757 };
758 if !out.contains(&entry) {
759 out.push(entry);
760 }
761 }
762 out
763 }
764
765 /// The bundle recorded for `version` on `platform`.
766 ///
767 /// This is what makes a two-architecture product promotable. One pom version is
768 /// two bundles with two digests, each built natively by Bento and accepted
769 /// through its own intake; the tier ladder carries the version forward and this
770 /// answers "which of that version's bundles does this box take".
771 ///
772 /// Only a run that settled green qualifies. A sibling that failed its own host
773 /// gates is not a fallback for the one that passed — the source tier's evidence
774 /// says nothing about bytes it never saw, so each architecture's bundle stands
775 /// on its own intake and its own gate run.
776 async fn bundle_for_platform(
777 s: &AppState,
778 version: &str,
779 platform: &crate::domain::Platform,
780 ) -> Result<(Option<i64>, std::path::PathBuf)> {
781 let row: Option<(i64, Option<String>)> = sqlx::query_as(
782 "SELECT id, staged_path FROM build_runs
783 WHERE app = ? AND version = ? AND platform = ? AND result = 'passed'
784 ORDER BY id DESC LIMIT 1",
785 )
786 .bind(&s.cfg.id)
787 .bind(version)
788 .bind(platform.to_string())
789 .fetch_optional(&s.pool)
790 .await
791 .map_err(crate::error::Error::Db)?;
792
793 match row {
794 // The id comes back alongside the path so the caller can check this
795 // build's own gate evidence rather than the source tier's. A green build
796 // row is not a green gate run: the build says the bytes compiled, the
797 // gate run says the tier vouched for them.
798 Some((id, Some(path))) => Ok((Some(id), std::path::PathBuf::from(path))),
799 Some((_, None)) => Err(crate::error::Error::Other(anyhow::anyhow!(
800 "the {platform} build of {version} has no staged_path; cannot promote it"
801 ))),
802 None => Err(crate::error::Error::GateBlocked(format!(
803 "no green {platform} bundle recorded for {version}. Each architecture is \
804 a separate artifact with its own evidence, so this one has to be built \
805 and accepted before a {platform} node can take this version"
806 ))),
807 }
808 }
809
810 /// After a canary node fails mid-promote, restore the nodes already flipped to
811 /// the new version back to `prev_version`, leaving the tier consistent (all on
812 /// the old version) rather than split-brain. Best-effort: every node is
813 /// attempted; a per-node failure is logged but never propagated (the promote is
814 /// already failing).
815 ///
816 /// Returns a [`RollbackReport`] rather than a bare count, because "the rollback
817 /// failed" is not the same claim as "the node is on the new version" and the
818 /// caller has to be able to tell them apart. When the previous version has no
819 /// recorded artifact, no rollback can be attempted and every touched node is
820 /// reported indeterminate.
821 pub(super) async fn rollback_deployed_nodes(
822 s: &AppState,
823 tier: &crate::domain::TierId,
824 nodes: &[&crate::topology::Node],
825 prev_version: &str,
826 ) -> RollbackReport {
827 let bin: Option<(String,)> =
828 match sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
829 .bind(&s.cfg.id)
830 .bind(prev_version)
831 .fetch_optional(&s.pool)
832 .await
833 {
834 Ok(b) => b,
835 Err(e) => {
836 tracing::error!(tier = %tier, prev = prev_version, error = %e,
837 "canary rollback: looking up the previous artifact failed; no rollback attempted");
838 return RollbackReport {
839 indeterminate: nodes.len(),
840 ..Default::default()
841 };
842 }
843 };
844 let Some((bin,)) = bin else {
845 tracing::error!(tier = %tier, prev = prev_version, nodes = nodes.len(),
846 "canary rollback: previous version has no artifact_path; no rollback attempted");
847 return RollbackReport {
848 indeterminate: nodes.len(),
849 ..Default::default()
850 };
851 };
852 let Some(staged_dir) = std::path::PathBuf::from(&bin)
853 .parent()
854 .map(std::path::Path::to_path_buf)
855 else {
856 tracing::error!(tier = %tier, prev = prev_version,
857 "canary rollback: previous artifact_path has no parent dir; no rollback attempted");
858 return RollbackReport {
859 indeterminate: nodes.len(),
860 ..Default::default()
861 };
862 };
863
864 // A rollback ships bytes too, so it needs the same per-node resolution a
865 // promote does: on a two-architecture product the previous version is also
866 // two bundles, and restoring the wrong one would leave the node no better
867 // off than the failed canary did. A resolution failure here cannot be
868 // propagated (the promote is already failing), so it reports the node as
869 // indeterminate — which is the truth: nothing was attempted and the node's
870 // version is whatever the failed deploy left.
871 // No fallback build id: a rollback resolves the previous version's bundles
872 // by version string and re-runs no gates, so there is no evidence to key.
873 // The bytes are ones this tier already ran.
874 let bundles = match bundles_for_nodes(s, prev_version, nodes, &staged_dir, None, None).await {
875 Ok(b) => b,
876 Err(e) => {
877 tracing::error!(tier = %tier, prev = prev_version, error = %e,
878 "canary rollback: could not resolve a previous bundle per node; no rollback attempted");
879 return RollbackReport {
880 indeterminate: nodes.len(),
881 ..Default::default()
882 };
883 }
884 };
885
886 // The node gc's pinned set. Unlike the promote path this cannot fail the
887 // operation: a rollback is what saves a tier that is already failing, and
888 // refusing to run it because a SELECT failed would be the worse outcome by
889 // far. So a read failure degrades to `None`, which skips the node gc and
890 // deploys anyway.
891 let pinned = match crate::retention::pinned_dirs(&s.pool, &s.cfg.id).await {
892 Ok(p) => Some(p),
893 Err(e) => {
894 tracing::warn!(tier = %tier, error = %e,
895 "canary rollback: could not read the pinned set; rolling back with the node gc skipped");
896 None
897 }
898 };
899
900 let mut report = RollbackReport::default();
901 for (node, node_bundle, node_bundle_platform, _) in &bundles {
902 let executor = s
903 .executors
904 .get(&node.name)
905 .cloned()
906 .unwrap_or_else(|| crate::state::build_executor(node));
907 // Unreachable from here, and deliberately kept: `bundles_for_nodes`
908 // hands a stating node its own platform and a silent node the silent
909 // fallback (a rollback passes no fallback platform), so the pairing
910 // always matches. It stays because `Placement` is the only way to get a
911 // deployable bundle and the arm is what makes that hold if either side
912 // ever starts resolving differently. A mutation survivor here is that
913 // dead arm, not a coverage gap.
914 let placement =
915 match crate::deploy::Placement::check(node, node_bundle, node_bundle_platform.as_ref())
916 {
917 Ok(p) => p,
918 Err(e) => {
919 report.indeterminate += 1;
920 tracing::error!(tier = %tier, node = %node.name, error = %e,
921 "canary rollback: refused to place the previous bundle on this node");
922 continue;
923 }
924 };
925 match crate::deploy::deploy_node(
926 executor.as_ref(),
927 placement,
928 prev_version,
929 s.cfg.primary_bin(),
930 pinned.as_ref(),
931 )
932 .await
933 {
934 Ok(_) => {
935 report.restored += 1;
936 tracing::warn!(tier = %tier, node = %node.name, version = prev_version,
937 "canary rollback: node restored to the previous version");
938 }
939 // A rollback is itself a deploy, so it fails at a stage too. Failing
940 // before the swap means it never touched `current` — the node is
941 // still on the version it was already running, which is the one we
942 // were rolling back TO. That is the intended end state reached by a
943 // different route, not an incident.
944 Err(e) => match crate::deploy::stage_of(&e) {
945 Some(crate::deploy::FailureStage::BeforeSwap) => {
946 report.already_on_previous += 1;
947 tracing::warn!(
948 tier = %tier, node = %node.name, version = prev_version,
949 error = %format!("{e:#}"),
950 "canary rollback did not run, and did not need to: it failed before the \
951 symlink swap, so the node is already on the previous version",
952 );
953 }
954 // Unannotated errors land here deliberately. Guessing "safe"
955 // would reintroduce the original bug in the worse direction.
956 stage => {
957 report.indeterminate += 1;
958 tracing::error!(
959 tier = %tier, node = %node.name, version = prev_version,
960 error = %format!("{e:#}"),
961 stage = ?stage,
962 "canary rollback FAILED for node at or after the symlink swap; its version \
963 is indeterminate — manual intervention needed",
964 );
965 }
966 },
967 }
968 }
969 report
970 }
971
972 /// Flag a tier as left in a partial / mixed-version state, with a human-readable
973 /// reason surfaced through `/state` and the TUI. Best-effort: a failure to record
974 /// the flag is logged, never propagated — the caller is already on an error path
975 /// and the worse outcome is to mask the original failure with a bookkeeping one.
976 pub(super) async fn set_partial(s: &AppState, tier: &crate::domain::TierId, reason: &str) {
977 if let Err(e) =
978 sqlx::query("UPDATE tier_state SET partial_reason = ? WHERE app = ? AND tier = ?")
979 .bind(reason)
980 .bind(&s.cfg.id)
981 .bind(tier)
982 .execute(&s.pool)
983 .await
984 {
985 tracing::error!(tier = %tier, reason, error = %e,
986 "failed to record tier partial state; the fleet may be inconsistent without a /state flag");
987 }
988 }
989
990 /// Clear a tier's partial flag after a clean full promote or rollback. Errors are
991 /// logged but not propagated: the deploy itself succeeded, and a stale flag is a
992 /// visible nuisance, not a safety regression (the operator sees a partial marker
993 /// on a tier that is actually fine, and re-checks).
994 pub(super) async fn clear_partial(s: &AppState, tier: &crate::domain::TierId) {
995 if let Err(e) =
996 sqlx::query("UPDATE tier_state SET partial_reason = NULL WHERE app = ? AND tier = ?")
997 .bind(&s.cfg.id)
998 .bind(tier)
999 .execute(&s.pool)
1000 .await
1001 {
1002 tracing::warn!(tier = %tier, error = %e, "failed to clear tier partial flag");
1003 }
1004 }
1005
1006 /// Returns the kinds of `tier`'s *configured* gates that are not satisfied for
1007 /// `version`. `hotfix` suppresses the `burn_in` requirement only.
1008 ///
1009 /// Fail-closed against the topology gate list (the CF1 fix). The previous
1010 /// version inspected only existing `gate_runs` rows, so a configured gate that
1011 /// had *never run* produced no row and was invisibly treated as green — letting
1012 /// a promote wave through with zero evidence (it shipped 0.9.5 to prod with
1013 /// tier A's `boot_smoke` never recorded). Now every configured gate must show
1014 /// positive evidence:
1015 /// - `burn_in` is evaluated live against the tier's clock (a stored `blocked`
1016 /// row would otherwise never flip to passed as time elapses);
1017 /// - every other kind requires a `passed` row for (tier, version) — a missing
1018 /// or non-passed latest row counts as unsatisfied.
1019 ///
1020 /// `builds` are the artifacts this promote will actually ship (wiki note
1021 /// `release-artifact-identity`). Each carries a `build_id`, and the deterministic
1022 /// build-evidence gates are checked against the rows that vouched for *that
1023 /// build* rather than against any row that happens to carry the version string —
1024 /// this is what stops a `promote --version Y` from riding on gate rows a
1025 /// different build left under the same version. A `None` build id is the
1026 /// legacy/pre-identity path: fall back to the version-keyed lookup so a
1027 /// mid-migration tier (NULL `current_build_id`) still promotes.
1028 ///
1029 /// **One promote can ship several builds**, and every one of them is checked.
1030 /// A pom version is two bundles with two digests, each built natively and
1031 /// accepted through its own intake, so the source tier's evidence for one
1032 /// architecture says nothing about the other. Checking only the build the tier
1033 /// points at would let an x86_64 node take bytes whose `cargo_test` row nobody
1034 /// looked at. When there is more than one, a failing gate is reported qualified
1035 /// by platform, because "cargo_test not satisfied" is not actionable if the
1036 /// operator cannot tell which half it is about.
1037 ///
1038 /// `burn_in` and `manual_confirm` are evaluated once regardless: both are keyed
1039 /// on the tier's own clock (`tier_state.burn_in_started_at`), not on a build, so
1040 /// asking per build would ask the same question N times and answer it N times in
1041 /// the error.
1042 ///
1043 /// **So are `node_health` and `page_smoke`, and for a sharper reason: they are
1044 /// evidence about this tier, not about the bytes leaving it.** `tier_build` is
1045 /// the build this tier is itself running, and it is the only build those two can
1046 /// ever have a row for. Asking them per shipped build is a category error that
1047 /// made a cross-architecture promote structurally impossible: promoting pom
1048 /// 0.4.3 astra -> hetzner ships the **x86_64** bundle, astra has only ever run
1049 /// the **aarch64** one, and so `node_health` was looked up against a build astra
1050 /// never saw and never could — astra is aarch64. Fail-closed reported it
1051 /// unsatisfied, `/state` reported it passed (astra's own row, correctly), and the
1052 /// promote refused twice with the gate satisfied by every reading the operator
1053 /// had. It could not have been cleared by re-running anything.
1054 ///
1055 /// The artifact gates below stay per build and must: `cargo_test` on the aarch64
1056 /// bundle says nothing about the x86_64 one, and each stands on its own intake.
1057 /// What separates the two lists is whether the evidence travels with the bytes.
1058 /// A single-platform product is unaffected either way — its source tier's build
1059 /// *is* the build being shipped, so both readings ask the same question.
1060 pub(super) async fn unsatisfied_gates(
1061 pool: &sqlx::SqlitePool,
1062 app: &crate::domain::AppId,
1063 tier: &crate::domain::TierId,
1064 gates: &[crate::topology::Gate],
1065 evidence: &Evidence<'_>,
1066 hotfix: bool,
1067 ) -> std::result::Result<Vec<String>, crate::error::Error> {
1068 use crate::topology::Gate;
1069 let Evidence {
1070 version,
1071 builds,
1072 tier_build,
1073 } = *evidence;
1074 let mut bad = Vec::new();
1075 for gate in gates {
1076 let kind = gate.kind();
1077 match gate {
1078 Gate::BurnIn { hours } => {
1079 if hotfix {
1080 continue;
1081 }
1082 let ok = crate::gates::burn_in_satisfied(pool, app, tier, *hours)
1083 .await
1084 .map_err(crate::error::Error::Other)?;
1085 if !ok {
1086 bad.push(kind.as_str().to_string());
1087 }
1088 }
1089 Gate::ManualConfirm => {
1090 // A confirmation must be *fresh*: recorded at or after the
1091 // version's current landing on this tier (tier_state
1092 // .burn_in_started_at, the per-deploy clock). Without this a
1093 // confirmation row survives a rollback + rollback-forward and
1094 // waves a re-deploy of the same version through with no fresh
1095 // operator sign-off — weaker than burn_in, which is clock-based.
1096 // No baseline (NULL) => fail closed: require a fresh confirm.
1097 let confirmed_at: Option<String> = sqlx::query_scalar(
1098 "SELECT finished_at FROM gate_runs
1099 WHERE app = ?1 AND tier = ?2 AND version = ?3
1100 AND gate_kind = 'manual_confirm' AND status = 'passed'
1101 ORDER BY id DESC LIMIT 1",
1102 )
1103 .bind(app)
1104 .bind(tier.as_str())
1105 .bind(version)
1106 .fetch_optional(pool)
1107 .await
1108 .map_err(crate::error::Error::Db)?
1109 .flatten();
1110 let landed_at: Option<String> = sqlx::query_scalar(
1111 "SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?",
1112 )
1113 .bind(app)
1114 .bind(tier.as_str())
1115 .fetch_optional(pool)
1116 .await
1117 .map_err(crate::error::Error::Db)?
1118 .flatten();
1119 let fresh = match (confirmed_at, landed_at) {
1120 (Some(c), Some(l)) => {
1121 match (
1122 chrono::DateTime::parse_from_rfc3339(&c),
1123 chrono::DateTime::parse_from_rfc3339(&l),
1124 ) {
1125 (Ok(cd), Ok(ld)) => cd >= ld,
1126 _ => false, // unparseable timestamp -> fail closed
1127 }
1128 }
1129 _ => false,
1130 };
1131 if !fresh {
1132 bad.push(kind.as_str().to_string());
1133 }
1134 }
1135 // Post-deploy tier gates: evidence that THIS tier is healthy running
1136 // what it is running. Keyed on the tier's own build, once, because
1137 // that is the only build a row here can ever exist for — see the note
1138 // on `tier_build` above. `None` is the pre-identity tier, where the
1139 // version-keyed lookup is the legacy behaviour.
1140 Gate::NodeHealth | Gate::PageSmoke => {
1141 let status: Option<String> = match tier_build {
1142 Some(bid) => sqlx::query_scalar(
1143 "SELECT status FROM gate_runs
1144 WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
1145 ORDER BY id DESC LIMIT 1",
1146 )
1147 .bind(app)
1148 .bind(tier.as_str())
1149 .bind(bid)
1150 .bind(kind.as_str()),
1151 None => sqlx::query_scalar(
1152 "SELECT status FROM gate_runs
1153 WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
1154 ORDER BY id DESC LIMIT 1",
1155 )
1156 .bind(app)
1157 .bind(tier.as_str())
1158 .bind(version)
1159 .bind(kind.as_str()),
1160 }
1161 .fetch_optional(pool)
1162 .await
1163 .map_err(crate::error::Error::Db)?
1164 .flatten();
1165 if status.as_deref() != Some("passed") {
1166 bad.push(kind.as_str().to_string());
1167 }
1168 }
1169 // Artifact gates: evidence about the bytes, which travels with them.
1170 // The latest row for this (build, kind) must be `passed`, per build
1171 // this promote ships. Listed explicitly (no `_` catch-all) so adding
1172 // a new `Gate` variant is a compile error here until its promotion
1173 // semantics are decided — a transient-`blocked` kind silently falling
1174 // into "needs a passed row" would be permanently unsatisfiable, and
1175 // a post-deploy kind landing here is the defect the arm above fixes.
1176 Gate::CargoTest
1177 | Gate::HardeningTest
1178 | Gate::Clippy
1179 | Gate::Fmt
1180 | Gate::CargoAudit
1181 | Gate::CargoDeny
1182 | Gate::MigrationDryRun
1183 | Gate::CodeSmoke
1184 | Gate::BootSmoke => {
1185 // Latest row for this configured gate kind; NULL/missing/any
1186 // non-'passed' status all count as unsatisfied (fail closed).
1187 // Every build this promote ships has to show its own passed row.
1188 // An empty `builds` is the caller saying "no identities to key
1189 // on"; one version-keyed lookup is the pre-identity behaviour.
1190 let lookups: &[PromotedBuild] = if builds.is_empty() {
1191 &[PromotedBuild {
1192 platform: None,
1193 build_id: None,
1194 }]
1195 } else {
1196 builds
1197 };
1198 let qualify = lookups.len() > 1;
1199 for b in lookups {
1200 // Keyed on build_id when the artifact has an identity (the
1201 // evidence must be for *this* build), else on the version.
1202 let status: Option<String> = match b.build_id {
1203 Some(bid) => sqlx::query_scalar(
1204 "SELECT status FROM gate_runs
1205 WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
1206 ORDER BY id DESC LIMIT 1",
1207 )
1208 .bind(app)
1209 .bind(tier.as_str())
1210 .bind(bid)
1211 .bind(kind.as_str()),
1212 None => sqlx::query_scalar(
1213 "SELECT status FROM gate_runs
1214 WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
1215 ORDER BY id DESC LIMIT 1",
1216 )
1217 .bind(app)
1218 .bind(tier.as_str())
1219 .bind(version)
1220 .bind(kind.as_str()),
1221 }
1222 .fetch_optional(pool)
1223 .await
1224 .map_err(crate::error::Error::Db)?
1225 .flatten();
1226 if status.as_deref() != Some("passed") {
1227 // Qualified only when there is more than one build to
1228 // tell apart, so a single-platform product's message is
1229 // exactly what it always was.
1230 let name = match (&b.platform, qualify) {
1231 (Some(p), true) => format!("{} ({p})", kind.as_str()),
1232 _ => kind.as_str().to_string(),
1233 };
1234 if !bad.contains(&name) {
1235 bad.push(name);
1236 }
1237 }
1238 }
1239 }
1240 }
1241 }
1242 Ok(bad)
1243 }
1244