Skip to main content

max / makenotwork

50.0 KB · 1101 lines History Blame Raw
1 //! Promotion service: the deploy state machine lifted out of the HTTP
2 //! layer — promote orchestration, canary rollback of deployed nodes,
3 //! partial-state flags, and the promote-time gate-satisfaction check.
4 //! routes/mod.rs stays HTTP glue that calls into these.
5
6 use super::{AppState, Json, PromoteBody, Result};
7
8 pub(super) async fn promote_inner(
9 s: AppState,
10 tier: String,
11 body: PromoteBody,
12 ) -> Result<Json<serde_json::Value>> {
13 // Serialize the whole check -> deploy -> advance against any concurrent
14 // promote/rollback (CF3). Held for the entire task.
15 let _deploy_guard = s.deploy_lock.lock().await;
16 let tier = crate::domain::TierId::new(tier);
17 let idx = s
18 .topo
19 .tiers
20 .iter()
21 .position(|t| t.name == tier)
22 .ok_or(crate::error::Error::NotFound)?;
23 if idx == 0 {
24 return Err(crate::error::Error::GateBlocked(
25 "cannot /promote to the first tier; use /rebuild".into(),
26 ));
27 }
28 let target = &s.topo.tiers[idx];
29 let source = &s.topo.tiers[idx - 1];
30 // An unprovisioned tier has no nodes, so every step below is a no-op that
31 // still *looks* like success: the deploy loop iterates nothing, node_health
32 // returns Blocked, and advance_tier writes a current_version for a tier that
33 // has never received a byte. /state then reports tier c running a version it
34 // does not have. Refuse instead.
35 if !target.provisioned {
36 return Err(crate::error::Error::GateBlocked(format!(
37 "tier {} is not provisioned (no nodes); promoting to it would record a version it never received",
38 target.name,
39 )));
40 }
41
42 // Resolve the artifact through the SOURCE tier's evidence, not a version
43 // string (invariant 2, wiki [[release-artifact-identity]]). The build that is
44 // *current on the source tier* is the one that burned in and passed its gates
45 // there; promoting anything else lets a version string ride on another build's
46 // evidence and clock — the "promote --version Y inherits X's 48h" hole. A
47 // given `body.version` must name that same build.
48 let source_build: Option<(Option<i64>, Option<String>, Option<String>, Option<String>)> =
49 sqlx::query_as(
50 "SELECT ts.current_build_id, br.version, br.staged_path, br.platform
51 FROM tier_state ts
52 LEFT JOIN build_runs br ON br.id = ts.current_build_id
53 WHERE ts.app = ? AND ts.tier = ?",
54 )
55 .bind(&s.cfg.id)
56 .bind(&source.name)
57 .fetch_optional(&s.pool)
58 .await
59 .map_err(crate::error::Error::Db)?;
60
61 let (build_id, version_str, staged_dir, source_platform) = match source_build {
62 // Identity path: the source tier points at a recorded build.
63 Some((Some(bid), ver, staged_path, platform)) => {
64 let version_str = ver.ok_or_else(|| {
65 crate::error::Error::Other(anyhow::anyhow!(
66 "source build {bid} on tier {} has no recorded version",
67 source.name
68 ))
69 })?;
70 let staged_path = staged_path.ok_or_else(|| {
71 crate::error::Error::Other(anyhow::anyhow!(
72 "source build {bid} ({version_str}) has no staged_path; cannot promote"
73 ))
74 })?;
75 if let Some(req) = &body.version
76 && req != &version_str
77 {
78 return Err(crate::error::Error::GateBlocked(format!(
79 "tier {} is running build {bid} ({version_str}); refusing to promote an \
80 explicit version {req} that is not the build the tier vouched for",
81 source.name
82 )));
83 }
84 let platform = platform
85 .as_deref()
86 .map(crate::domain::Platform::parse)
87 .transpose()
88 .map_err(|e| crate::error::Error::Other(anyhow::anyhow!(e)))?;
89 (
90 Some(bid),
91 version_str,
92 std::path::PathBuf::from(staged_path),
93 platform,
94 )
95 }
96 // Legacy fallback: NULL current_build_id (a pre-identity tier). Resolve by
97 // version string through the versions table, exactly as before. One clean
98 // build-and-promote cycle re-anchors the tier onto the identity path.
99 _ => {
100 let version_str = match body.version.clone() {
101 Some(v) => v,
102 None => sqlx::query_scalar::<_, Option<String>>(
103 "SELECT current_version FROM tier_state WHERE app = ? AND tier = ?",
104 )
105 .bind(&s.cfg.id)
106 .bind(&source.name)
107 .fetch_optional(&s.pool)
108 .await
109 .map_err(crate::error::Error::Db)?
110 .flatten()
111 .ok_or_else(|| {
112 crate::error::Error::GateBlocked(format!(
113 "no version specified and tier {} has no current_version",
114 source.name
115 ))
116 })?,
117 };
118 let bin: Option<(String,)> =
119 sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
120 .bind(&s.cfg.id)
121 .bind(&version_str)
122 .fetch_optional(&s.pool)
123 .await
124 .map_err(crate::error::Error::Db)?;
125 let Some((bin,)) = bin else {
126 return Err(crate::error::Error::NotFound);
127 };
128 // `artifact_path` is the primary binary; the release dir is its parent.
129 let staged_dir = std::path::PathBuf::from(&bin)
130 .parent()
131 .ok_or_else(|| {
132 crate::error::Error::Other(anyhow::anyhow!("artifact_path has no parent"))
133 })?
134 .to_path_buf();
135 (None, version_str, staged_dir, None)
136 }
137 };
138 let version = crate::domain::Version::parse(&version_str)
139 .map_err(|e| crate::error::Error::Other(anyhow::anyhow!(e)))?;
140
141 // 1. Predecessor must have all of its configured gates satisfied for this
142 // build (with optional hotfix override that skips burn_in). Evaluated
143 // against the topology gate list, so a gate that never ran blocks the
144 // promote instead of being treated as green.
145 //
146 // A migration-bearing promote (`bears_migration`) forces a fresh
147 // `manual_confirm` on the predecessor even if the tier does not configure
148 // one: rollback restores the binary + release_contents only, never the
149 // schema, so the operator must consciously acknowledge the one-way advance
150 // (deploy/README.md "Rollback contract"). `hotfix` skips only `burn_in`, so
151 // it does not suppress this confirm.
152 let mut effective_gates = source.gates.clone();
153 if body.bears_migration
154 && !effective_gates
155 .iter()
156 .any(|g| matches!(g, crate::topology::Gate::ManualConfirm))
157 {
158 effective_gates.push(crate::topology::Gate::ManualConfirm);
159 }
160 // Which bytes each node gets is resolved FIRST, because it is what says
161 // how many builds the gate check has to cover. A two-architecture promote
162 // ships two bundles with two sets of evidence, and checking only the one
163 // the source tier points at would wave the sibling through on gate rows
164 // nobody read. It also means a version missing its other half fails here,
165 // before any gate work, rather than halfway down the rollout.
166 let target_nodes: Vec<&crate::topology::Node> = target.nodes.iter().collect();
167 let bundles = bundles_for_nodes(
168 &s,
169 &version_str,
170 &target_nodes,
171 &staged_dir,
172 source_platform.as_ref(),
173 build_id,
174 )
175 .await?;
176 let mut promoted_builds = distinct_builds(&bundles);
177 if promoted_builds.is_empty() {
178 // A provisioned tier with no nodes ships nothing, so there is nothing to
179 // resolve — but the gate check still has to be keyed on the source build.
180 // Left empty it would fall through to the version-string lookup, which is
181 // the pre-identity path and weaker than what this tier is owed. The
182 // `provisioned` guard above makes this unreachable today; it is written
183 // out because the alternative fails quietly in the direction of less
184 // evidence.
185 promoted_builds.push(PromotedBuild {
186 platform: source_platform.clone(),
187 build_id,
188 });
189 }
190
191 let pending = unsatisfied_gates(
192 &s.pool,
193 &s.cfg.id,
194 &source.name,
195 &effective_gates,
196 &version_str,
197 &promoted_builds,
198 body.hotfix,
199 )
200 .await?;
201 if !pending.is_empty() {
202 return Err(crate::error::Error::GateBlocked(format!(
203 "{} gate(s) not satisfied on tier {}: {}",
204 pending.len(),
205 source.name,
206 pending.join(", "),
207 )));
208 }
209
210 // The version this tier was running before this promote — the rollback
211 // target if a canary node fails partway through a multi-node rollout.
212 let prev_version: Option<String> =
213 sqlx::query_scalar("SELECT current_version FROM tier_state WHERE app = ? AND tier = ?")
214 .bind(&s.cfg.id)
215 .bind(&target.name)
216 .fetch_optional(&s.pool)
217 .await
218 .map_err(crate::error::Error::Db)?
219 .flatten();
220
221 // 3. Deploy to each node. Sequential canary is the only policy
222 // implemented in v0; parallel is a one-line change once we trust the
223 // sequential path. Track the nodes already flipped to the new version so
224 // a mid-rollout failure can roll them back (canary rollback).
225 let mut deployed: Vec<&crate::topology::Node> = Vec::new();
226 // The build id is not read here: it did its work above, keying the gate
227 // check to each bundle's own evidence. What ships is decided by `Placement`.
228 for (node, node_bundle, node_bundle_platform, _) in &bundles {
229 // The proof that these bytes belong on this box. Built before the
230 // deploy row is written, so a mismatch never becomes an `in_progress`
231 // deploy that has to be reconciled.
232 let placement =
233 crate::deploy::Placement::check(node, node_bundle, node_bundle_platform.as_ref())
234 .map_err(|e| crate::error::Error::GateBlocked(e.to_string()))?;
235 let started = chrono::Utc::now().to_rfc3339();
236 crate::events::emit(
237 &s.events,
238 crate::events::Event::DeployStart {
239 tier: target.name.clone(),
240 node: node.name.clone(),
241 version: version.clone(),
242 },
243 );
244 let executor = s
245 .executors
246 .get(&node.name)
247 .cloned()
248 .unwrap_or_else(|| crate::state::build_executor(node));
249 // Record the deploy as `in_progress` BEFORE touching the node, so a crash
250 // between the node's symlink swap and this promote's `advance_tier` leaves
251 // a durable trace. Without the pre-write, a SIGKILL after the swap but
252 // before the row is inserted would land the new binary on the node with no
253 // DB evidence at all, and the startup reconcile — which reads `deploys` —
254 // could not see it. The row is finalized to ok/failed immediately below;
255 // any row still `in_progress` at startup is an orphan the reconcile settles
256 // and flags ([`crate::reconcile`]).
257 let deploy_id: i64 = sqlx::query_scalar(
258 "INSERT INTO deploys (app, version, tier, node, started_at, outcome, hotfix, reset_burn_in, build_id)
259 VALUES (?, ?, ?, ?, ?, 'in_progress', ?, ?, ?) RETURNING id",
260 )
261 .bind(&s.cfg.id)
262 .bind(&version).bind(&target.name).bind(&node.name)
263 .bind(&started)
264 .bind(body.hotfix as i64).bind(body.reset_burn_in as i64)
265 .bind(build_id)
266 .fetch_one(&s.pool).await.map_err(crate::error::Error::Db)?;
267 let result = crate::deploy::deploy_node(
268 executor.as_ref(),
269 placement,
270 &version_str,
271 s.cfg.primary_bin(),
272 )
273 .await;
274 let finished = chrono::Utc::now().to_rfc3339();
275 let (outcome_obj, err_for_propagation) = match result {
276 Ok(_) => (crate::outcome::DeployOutcome::ok(), None),
277 Err(e) => {
278 let msg = format!("{e:#}");
279 let kind = crate::classify::classify_deploy_error(&msg);
280 (crate::outcome::DeployOutcome::failed(kind), Some(e))
281 }
282 };
283 let outcome_json = serde_json::to_string(&outcome_obj)
284 .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
285 sqlx::query(
286 "UPDATE deploys SET finished_at = ?, outcome = ?, outcome_json = ? WHERE id = ?",
287 )
288 .bind(&finished)
289 .bind(outcome_obj.status_str())
290 .bind(&outcome_json)
291 .bind(deploy_id)
292 .execute(&s.pool)
293 .await
294 .map_err(crate::error::Error::Db)?;
295 if let Some(e) = err_for_propagation {
296 let crate::outcome::DeployStatus::Failed { failure } = outcome_obj.status else {
297 unreachable!("err_for_propagation is Some iff status is Failed");
298 };
299 tracing::error!(
300 tier = %target.name, node = %node.name, version = %version,
301 failure = failure.summary(),
302 "deploy failed; current symlink left intact, tier_state not advanced"
303 );
304 crate::events::emit(
305 &s.events,
306 crate::events::Event::DeployFailed {
307 tier: target.name.clone(),
308 node: node.name.clone(),
309 version: version.clone(),
310 failure,
311 },
312 );
313
314 // Canary rollback: restore every node this promote touched — the
315 // ones already flipped to the new version AND this failed node
316 // (whose state is indeterminate: the symlink swap may have landed
317 // before the restart failed) — back to the tier's prior version, so
318 // the fleet is left consistent on `prev` rather than split-brain.
319 // Nodes after this one were never touched and stay on `prev`.
320 deployed.push(node);
321 let touched = deployed.len();
322 match prev_version.as_deref() {
323 Some(prev) => {
324 let report = rollback_deployed_nodes(&s, &target.name, &deployed, prev).await;
325 // Say what is true, and say it differently when nothing is
326 // wrong. `restored=0 of=1` read as total failure when the
327 // accurate reading was "0 needed restoring" — the same
328 // confusion as the per-node message, one level up.
329 if report.is_consistent() {
330 tracing::warn!(
331 tier = %target.name,
332 restored = report.restored,
333 already_on_previous = report.already_on_previous,
334 of = report.touched(),
335 from = %version, to = prev,
336 "canary failed mid-rollout; every touched node is on the previous \
337 version and the tier is consistent",
338 );
339 } else {
340 tracing::error!(
341 tier = %target.name,
342 restored = report.restored,
343 already_on_previous = report.already_on_previous,
344 indeterminate = report.indeterminate,
345 of = report.touched(),
346 from = %version, to = prev,
347 "canary failed mid-rollout and the tier is NOT consistent; some nodes \
348 have an indeterminate version",
349 );
350 }
351 if report.restored > 0
352 && let Ok(prev_v) = crate::domain::Version::parse(prev)
353 {
354 crate::events::emit(
355 &s.events,
356 crate::events::Event::Rollback {
357 tier: target.name.clone(),
358 from: version.clone(),
359 to: prev_v,
360 },
361 );
362 }
363 // The tier is consistent when no node's version is unknown —
364 // which includes the case where a rollback "failed" before
365 // the swap and so left the node on `prev` already. Flagging
366 // that as partial would put a permanent scare on /state for
367 // a fleet that is entirely on one version.
368 if report.is_consistent() {
369 clear_partial(&s, &target.name).await;
370 } else {
371 set_partial(&s, &target.name, &format!(
372 "canary rollback incomplete: {indeterminate} of {touched} node(s) have an \
373 indeterminate version and may be on {version}; {restored} restored to \
374 {prev}, {already} were never swapped — manual check needed",
375 touched = report.touched(),
376 indeterminate = report.indeterminate,
377 restored = report.restored,
378 already = report.already_on_previous,
379 )).await;
380 }
381 }
382 None => {
383 tracing::error!(
384 tier = %target.name, count = touched, version = %version,
385 "canary failed on a first deploy (no previous version to restore to); \
386 touched nodes remain on the new version — manual cleanup needed",
387 );
388 set_partial(
389 &s,
390 &target.name,
391 &format!(
392 "first-deploy canary failed: {touched} node(s) left on {version}, \
393 no prior version to restore — manual cleanup needed",
394 ),
395 )
396 .await;
397 }
398 }
399 return Err(crate::error::Error::Other(e));
400 }
401 deployed.push(node);
402 crate::events::emit(
403 &s.events,
404 crate::events::Event::DeployOk {
405 tier: target.name.clone(),
406 node: node.name.clone(),
407 version: version.clone(),
408 },
409 );
410 }
411
412 // 3b. Run this tier's post-deploy gates (node_health) against the freshly
413 // deployed nodes and record their outcomes. These rows are the evidence
414 // the NEXT promote (this tier -> the following one) checks via
415 // `unsatisfied_gates`. Before CF1, only the host tier ran gates, so
416 // A/B/C had no evidence and promotion waved through; node_health now
417 // proves the deployed nodes are serving (Run-2 SERIOUS-3: boot_smoke
418 // used to re-run the staged binary locally and proved nothing about the
419 // node). burn_in / manual_confirm are not run here — they are evaluated
420 // live / by the operator at the next promote. A failed gate does not
421 // unwind this deploy (the artifact is already live on the tier); it
422 // blocks the next promote, which is the fail-closed behavior we want.
423 //
424 // It does, however, fail *this* promote's response. tier_state still
425 // advances below — the nodes genuinely run this version and a stale
426 // `current_version` would send a later rollback to the wrong artifact —
427 // but the tier is flagged partial and the handler returns the failure,
428 // so the operator learns at the promote instead of discovering it as an
429 // unexplained block on the next one.
430 let post_deploy: Vec<crate::topology::Gate> = target
431 .gates
432 .iter()
433 .filter(|g| g.runs_post_deploy())
434 .cloned()
435 .collect();
436 let mut post_deploy_failure: Option<String> = None;
437 if !post_deploy.is_empty() {
438 // node_health probes each node the deploy just shipped to, over the same
439 // executor the deploy used. Build the probe set from the tier's nodes and
440 // the startup executor map; a node missing an executor (shouldn't happen
441 // — both come from the same topology) is skipped, and an empty set makes
442 // node_health Blocked (fail closed).
443 let nodes: Vec<crate::gates::NodeProbe> = target
444 .nodes
445 .iter()
446 .filter_map(|n| {
447 s.executors
448 .get(&n.name)
449 .map(|exec| crate::gates::NodeProbe {
450 node: n.name.clone(),
451 service: n.service_name.clone(),
452 health_url: n.health_url.clone(),
453 executor: exec.clone(),
454 })
455 })
456 .collect();
457 let ctx = crate::gates::GateCtx {
458 pool: s.pool.clone(),
459 cfg: s.cfg.clone(),
460 tier: target.name.clone(),
461 version: version.clone(),
462 // No worktree at promote time; node_health works over executors, not
463 // a checkout. No bundle either: the gates here are about the nodes,
464 // not about bytes on this host.
465 worktree: None,
466 bundle: None,
467 events: s.events.clone(),
468 nodes,
469 // These post-deploy gate rows are the evidence the NEXT promote (this
470 // tier -> the following one) resolves through, so they must carry the
471 // build they vouch for.
472 build_id,
473 // The hostname the public uses, straight from the tier. page_smoke
474 // has to request the site the way a visitor does; anything derived
475 // from a node would reach the origin and miss the CDN, which is the
476 // layer the gate exists to watch.
477 public_url: target.public_url.clone(),
478 // No checkout at promote time, so nothing to resolve against.
479 aux_dirs: std::collections::HashMap::new(),
480 };
481 post_deploy_failure = match crate::gates::run_all(&ctx, &post_deploy).await {
482 Ok(failed) if failed.is_empty() => None,
483 Ok(failed) => {
484 let names = failed
485 .iter()
486 .map(|k| k.as_str())
487 .collect::<Vec<_>>()
488 .join(", ");
489 tracing::warn!(
490 tier = %target.name, version = %version, gates = %names,
491 "post-deploy gate(s) failed; tier advanced but promotion to the next tier is blocked",
492 );
493 Some(format!(
494 "post-deploy gate(s) failed on {version}: {names}; \
495 the tier is serving {version} but cannot promote onward until they pass"
496 ))
497 }
498 Err(e) => {
499 tracing::error!(
500 tier = %target.name, version = %version, error = %e,
501 "post-deploy gate execution errored; promotion to the next tier is blocked",
502 );
503 Some(format!(
504 "post-deploy gate execution errored on {version}: {e}; \
505 the tier is serving {version} but cannot promote onward until the gates pass"
506 ))
507 }
508 };
509 }
510
511 // 4. Advance tier_state through the single sealed forward-advance op (atomic
512 // self-referential UPDATE; no read-modify-write to lose under concurrency,
513 // CF3). We hold deploy_lock for this whole handler, so the advance is
514 // serialized against rollback and the host build path's advance.
515 // reset_burn_in on the *source* tier nulls its clock only when the operator
516 // explicitly asked.
517 crate::runs::advance_tier(&s.pool, &s.cfg.id, target.name.as_str(), &version, build_id)
518 .await
519 .map_err(crate::error::Error::Db)?;
520
521 if body.reset_burn_in {
522 sqlx::query("UPDATE tier_state SET burn_in_started_at = NULL WHERE app = ? AND tier = ?")
523 .bind(&s.cfg.id)
524 .bind(&source.name)
525 .execute(&s.pool)
526 .await
527 .map_err(crate::error::Error::Db)?;
528 }
529
530 // Red post-deploy gates: the rollout itself reached every node, but the tier
531 // is not fit to promote onward. Flag it so /state and the TUI say so, and
532 // return the failure rather than a 200 the operator would read as "shipped,
533 // all good". tier_state has already advanced above — it describes what the
534 // nodes are running, not whether we're happy about it.
535 if let Some(reason) = post_deploy_failure {
536 set_partial(&s, &target.name, &reason).await;
537 return Err(crate::error::Error::GateBlocked(reason));
538 }
539
540 // A clean full rollout to every node clears any prior partial flag on this tier.
541 clear_partial(&s, &target.name).await;
542
543 crate::events::emit(
544 &s.events,
545 crate::events::Event::PromoteComplete {
546 tier: target.name.clone(),
547 version: version.clone(),
548 },
549 );
550 tracing::info!(
551 version = %version, tier = %target.name,
552 hotfix = body.hotfix, reset_burn_in = body.reset_burn_in,
553 "promote complete",
554 );
555
556 Ok(Json(serde_json::json!({
557 "tier": target.name,
558 "version": version,
559 "nodes_deployed": target.nodes.iter().map(|n| n.name.clone()).collect::<Vec<_>>(),
560 })))
561 }
562
563 /// What a canary rollback actually left behind, per node.
564 ///
565 /// Three outcomes, not two, and conflating the middle one with the last is the
566 /// bug this type exists to prevent: a rollback that fails *before* the symlink
567 /// swap leaves the node exactly where it already was, on the previous version.
568 /// Reporting that as "stranded on the new version, manual intervention needed"
569 /// sends an operator to do surgery on a healthy production box.
570 #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
571 pub(super) struct RollbackReport {
572 /// Put back on the previous version by a successful redeploy.
573 pub(super) restored: usize,
574 /// Rollback failed before the swap, so the node never left the previous
575 /// version. Nothing is stranded and nothing needs doing.
576 pub(super) already_on_previous: usize,
577 /// Rollback failed at or after the swap, or could not be attempted at all.
578 /// The node's version is not knowable from here. This is the only outcome
579 /// that warrants a human.
580 pub(super) indeterminate: usize,
581 }
582
583 impl RollbackReport {
584 /// Every touched node accounted for.
585 pub(super) fn touched(self) -> usize {
586 self.restored + self.already_on_previous + self.indeterminate
587 }
588
589 /// True when no node is in an unknown state, whether or not every rollback
590 /// "succeeded". A tier whose rollbacks all failed before the swap is
591 /// consistent on the previous version and is not an incident.
592 pub(super) fn is_consistent(self) -> bool {
593 self.indeterminate == 0
594 }
595 }
596
597 /// A node, the bundle it is to receive, what that bundle runs on, and the build
598 /// row that bundle came from.
599 ///
600 /// The build id is what makes the gate check per-platform: evidence is keyed on
601 /// the build that produced the bytes, so a promote that ships two architectures
602 /// has to look up two sets of gate rows, and this is where it learns which two.
603 /// `None` is the legacy path (a bundle resolved by version string, from a tier
604 /// with no `current_build_id`), where there is no build to key on.
605 pub(super) type NodeBundle<'a> = (
606 &'a crate::topology::Node,
607 std::path::PathBuf,
608 Option<crate::domain::Platform>,
609 Option<i64>,
610 );
611
612 /// One bundle a promote is about to ship, and the evidence key it is judged by.
613 ///
614 /// A single-platform product has exactly one of these and it is the source
615 /// tier's own build, which is what every promote before two-architecture support
616 /// checked. A product like pom has one per architecture, each standing on its
617 /// own intake and its own gate run.
618 #[derive(Debug, Clone, PartialEq, Eq)]
619 pub(super) struct PromotedBuild {
620 /// What the bundle runs on, when it says. Used only to qualify the name of a
621 /// failing gate, so an operator reading "cargo_test" knows which half.
622 pub platform: Option<crate::domain::Platform>,
623 /// The `build_runs` row the evidence is keyed on; `None` falls back to the
624 /// version-keyed lookup for a pre-identity tier.
625 pub build_id: Option<i64>,
626 }
627
628 /// Which bytes each node gets, resolved before any node is touched.
629 ///
630 /// A single-platform product answers with `fallback` for every node, which is
631 /// the bundle the caller already resolved (the source tier's build on a promote,
632 /// the previous version's on a rollback). A product whose one version is several
633 /// bundles answers per node, because the node is the thing that says which
634 /// architecture it can run.
635 ///
636 /// Resolving the whole tier up front is deliberate: a version missing its
637 /// x86_64 half fails the promote before its aarch64 half has been pushed
638 /// anywhere, instead of halfway down a rollout with nodes already flipped.
639 pub(super) async fn bundles_for_nodes<'a>(
640 s: &AppState,
641 version: &str,
642 nodes: &[&'a crate::topology::Node],
643 fallback: &std::path::Path,
644 fallback_platform: Option<&crate::domain::Platform>,
645 fallback_build_id: Option<i64>,
646 ) -> Result<Vec<NodeBundle<'a>>> {
647 let mut out = Vec::with_capacity(nodes.len());
648 for node in nodes.iter().copied() {
649 let resolved = match &node.platform {
650 // The node states nothing, so there is nothing to resolve against;
651 // it gets the caller's bundle, and `Placement::check` decides
652 // whether that pairing is admissible at all.
653 None => (
654 fallback.to_path_buf(),
655 fallback_platform.cloned(),
656 fallback_build_id,
657 ),
658 // The node states a platform. Give it the bundle recorded for that
659 // platform at this version — the caller's own when they agree, its
660 // sibling when they do not.
661 Some(want) if fallback_platform == Some(want) => (
662 fallback.to_path_buf(),
663 fallback_platform.cloned(),
664 fallback_build_id,
665 ),
666 Some(want) => {
667 let (build_id, path) = bundle_for_platform(s, version, want).await?;
668 (path, Some(want.clone()), build_id)
669 }
670 };
671 out.push((node, resolved.0, resolved.1, resolved.2));
672 }
673 Ok(out)
674 }
675
676 /// The distinct builds `bundles` will ship, in a stable order.
677 ///
678 /// Deduplicated because a tier is usually several nodes on one architecture, and
679 /// evaluating one build's gates once per node would say the same thing three
680 /// times in the error an operator reads. Keyed on the whole entry rather than on
681 /// the build id alone, so the legacy `None` case does not collapse two
682 /// version-resolved bundles into one.
683 pub(super) fn distinct_builds(bundles: &[NodeBundle<'_>]) -> Vec<PromotedBuild> {
684 let mut out: Vec<PromotedBuild> = Vec::new();
685 for (_, _, platform, build_id) in bundles {
686 let entry = PromotedBuild {
687 platform: platform.clone(),
688 build_id: *build_id,
689 };
690 if !out.contains(&entry) {
691 out.push(entry);
692 }
693 }
694 out
695 }
696
697 /// The bundle recorded for `version` on `platform`.
698 ///
699 /// This is what makes a two-architecture product promotable. One pom version is
700 /// two bundles with two digests, each built natively by Bento and accepted
701 /// through its own intake; the tier ladder carries the version forward and this
702 /// answers "which of that version's bundles does this box take".
703 ///
704 /// Only a run that settled green qualifies. A sibling that failed its own host
705 /// gates is not a fallback for the one that passed — the source tier's evidence
706 /// says nothing about bytes it never saw, so each architecture's bundle stands
707 /// on its own intake and its own gate run.
708 async fn bundle_for_platform(
709 s: &AppState,
710 version: &str,
711 platform: &crate::domain::Platform,
712 ) -> Result<(Option<i64>, std::path::PathBuf)> {
713 let row: Option<(i64, Option<String>)> = sqlx::query_as(
714 "SELECT id, staged_path FROM build_runs
715 WHERE app = ? AND version = ? AND platform = ? AND result = 'passed'
716 ORDER BY id DESC LIMIT 1",
717 )
718 .bind(&s.cfg.id)
719 .bind(version)
720 .bind(platform.to_string())
721 .fetch_optional(&s.pool)
722 .await
723 .map_err(crate::error::Error::Db)?;
724
725 match row {
726 // The id comes back alongside the path so the caller can check this
727 // build's own gate evidence rather than the source tier's. A green build
728 // row is not a green gate run: the build says the bytes compiled, the
729 // gate run says the tier vouched for them.
730 Some((id, Some(path))) => Ok((Some(id), std::path::PathBuf::from(path))),
731 Some((_, None)) => Err(crate::error::Error::Other(anyhow::anyhow!(
732 "the {platform} build of {version} has no staged_path; cannot promote it"
733 ))),
734 None => Err(crate::error::Error::GateBlocked(format!(
735 "no green {platform} bundle recorded for {version}. Each architecture is \
736 a separate artifact with its own evidence, so this one has to be built \
737 and accepted before a {platform} node can take this version"
738 ))),
739 }
740 }
741
742 /// After a canary node fails mid-promote, restore the nodes already flipped to
743 /// the new version back to `prev_version`, leaving the tier consistent (all on
744 /// the old version) rather than split-brain. Best-effort: every node is
745 /// attempted; a per-node failure is logged but never propagated (the promote is
746 /// already failing).
747 ///
748 /// Returns a [`RollbackReport`] rather than a bare count, because "the rollback
749 /// failed" is not the same claim as "the node is on the new version" and the
750 /// caller has to be able to tell them apart. When the previous version has no
751 /// recorded artifact, no rollback can be attempted and every touched node is
752 /// reported indeterminate.
753 pub(super) async fn rollback_deployed_nodes(
754 s: &AppState,
755 tier: &crate::domain::TierId,
756 nodes: &[&crate::topology::Node],
757 prev_version: &str,
758 ) -> RollbackReport {
759 let bin: Option<(String,)> =
760 match sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
761 .bind(&s.cfg.id)
762 .bind(prev_version)
763 .fetch_optional(&s.pool)
764 .await
765 {
766 Ok(b) => b,
767 Err(e) => {
768 tracing::error!(tier = %tier, prev = prev_version, error = %e,
769 "canary rollback: looking up the previous artifact failed; no rollback attempted");
770 return RollbackReport {
771 indeterminate: nodes.len(),
772 ..Default::default()
773 };
774 }
775 };
776 let Some((bin,)) = bin else {
777 tracing::error!(tier = %tier, prev = prev_version, nodes = nodes.len(),
778 "canary rollback: previous version has no artifact_path; no rollback attempted");
779 return RollbackReport {
780 indeterminate: nodes.len(),
781 ..Default::default()
782 };
783 };
784 let Some(staged_dir) = std::path::PathBuf::from(&bin)
785 .parent()
786 .map(std::path::Path::to_path_buf)
787 else {
788 tracing::error!(tier = %tier, prev = prev_version,
789 "canary rollback: previous artifact_path has no parent dir; no rollback attempted");
790 return RollbackReport {
791 indeterminate: nodes.len(),
792 ..Default::default()
793 };
794 };
795
796 // A rollback ships bytes too, so it needs the same per-node resolution a
797 // promote does: on a two-architecture product the previous version is also
798 // two bundles, and restoring the wrong one would leave the node no better
799 // off than the failed canary did. A resolution failure here cannot be
800 // propagated (the promote is already failing), so it reports the node as
801 // indeterminate — which is the truth: nothing was attempted and the node's
802 // version is whatever the failed deploy left.
803 // No fallback build id: a rollback resolves the previous version's bundles
804 // by version string and re-runs no gates, so there is no evidence to key.
805 // The bytes are ones this tier already ran.
806 let bundles = match bundles_for_nodes(s, prev_version, nodes, &staged_dir, None, None).await {
807 Ok(b) => b,
808 Err(e) => {
809 tracing::error!(tier = %tier, prev = prev_version, error = %e,
810 "canary rollback: could not resolve a previous bundle per node; no rollback attempted");
811 return RollbackReport {
812 indeterminate: nodes.len(),
813 ..Default::default()
814 };
815 }
816 };
817
818 let mut report = RollbackReport::default();
819 for (node, node_bundle, node_bundle_platform, _) in &bundles {
820 let executor = s
821 .executors
822 .get(&node.name)
823 .cloned()
824 .unwrap_or_else(|| crate::state::build_executor(node));
825 let placement =
826 match crate::deploy::Placement::check(node, node_bundle, node_bundle_platform.as_ref())
827 {
828 Ok(p) => p,
829 Err(e) => {
830 report.indeterminate += 1;
831 tracing::error!(tier = %tier, node = %node.name, error = %e,
832 "canary rollback: refused to place the previous bundle on this node");
833 continue;
834 }
835 };
836 match crate::deploy::deploy_node(
837 executor.as_ref(),
838 placement,
839 prev_version,
840 s.cfg.primary_bin(),
841 )
842 .await
843 {
844 Ok(_) => {
845 report.restored += 1;
846 tracing::warn!(tier = %tier, node = %node.name, version = prev_version,
847 "canary rollback: node restored to the previous version");
848 }
849 // A rollback is itself a deploy, so it fails at a stage too. Failing
850 // before the swap means it never touched `current` — the node is
851 // still on the version it was already running, which is the one we
852 // were rolling back TO. That is the intended end state reached by a
853 // different route, not an incident.
854 Err(e) => match crate::deploy::stage_of(&e) {
855 Some(crate::deploy::FailureStage::BeforeSwap) => {
856 report.already_on_previous += 1;
857 tracing::warn!(
858 tier = %tier, node = %node.name, version = prev_version,
859 error = %format!("{e:#}"),
860 "canary rollback did not run, and did not need to: it failed before the \
861 symlink swap, so the node is already on the previous version",
862 );
863 }
864 // Unannotated errors land here deliberately. Guessing "safe"
865 // would reintroduce the original bug in the worse direction.
866 stage => {
867 report.indeterminate += 1;
868 tracing::error!(
869 tier = %tier, node = %node.name, version = prev_version,
870 error = %format!("{e:#}"),
871 stage = ?stage,
872 "canary rollback FAILED for node at or after the symlink swap; its version \
873 is indeterminate — manual intervention needed",
874 );
875 }
876 },
877 }
878 }
879 report
880 }
881
882 /// Flag a tier as left in a partial / mixed-version state, with a human-readable
883 /// reason surfaced through `/state` and the TUI. Best-effort: a failure to record
884 /// the flag is logged, never propagated — the caller is already on an error path
885 /// and the worse outcome is to mask the original failure with a bookkeeping one.
886 pub(super) async fn set_partial(s: &AppState, tier: &crate::domain::TierId, reason: &str) {
887 if let Err(e) =
888 sqlx::query("UPDATE tier_state SET partial_reason = ? WHERE app = ? AND tier = ?")
889 .bind(reason)
890 .bind(&s.cfg.id)
891 .bind(tier)
892 .execute(&s.pool)
893 .await
894 {
895 tracing::error!(tier = %tier, reason, error = %e,
896 "failed to record tier partial state; the fleet may be inconsistent without a /state flag");
897 }
898 }
899
900 /// Clear a tier's partial flag after a clean full promote or rollback. Errors are
901 /// logged but not propagated: the deploy itself succeeded, and a stale flag is a
902 /// visible nuisance, not a safety regression (the operator sees a partial marker
903 /// on a tier that is actually fine, and re-checks).
904 pub(super) async fn clear_partial(s: &AppState, tier: &crate::domain::TierId) {
905 if let Err(e) =
906 sqlx::query("UPDATE tier_state SET partial_reason = NULL WHERE app = ? AND tier = ?")
907 .bind(&s.cfg.id)
908 .bind(tier)
909 .execute(&s.pool)
910 .await
911 {
912 tracing::warn!(tier = %tier, error = %e, "failed to clear tier partial flag");
913 }
914 }
915
916 /// Returns the kinds of `tier`'s *configured* gates that are not satisfied for
917 /// `version`. `hotfix` suppresses the `burn_in` requirement only.
918 ///
919 /// Fail-closed against the topology gate list (the CF1 fix). The previous
920 /// version inspected only existing `gate_runs` rows, so a configured gate that
921 /// had *never run* produced no row and was invisibly treated as green — letting
922 /// a promote wave through with zero evidence (it shipped 0.9.5 to prod with
923 /// tier A's `boot_smoke` never recorded). Now every configured gate must show
924 /// positive evidence:
925 /// - `burn_in` is evaluated live against the tier's clock (a stored `blocked`
926 /// row would otherwise never flip to passed as time elapses);
927 /// - every other kind requires a `passed` row for (tier, version) — a missing
928 /// or non-passed latest row counts as unsatisfied.
929 ///
930 /// `builds` are the artifacts this promote will actually ship (wiki note
931 /// `release-artifact-identity`). Each carries a `build_id`, and the deterministic
932 /// build-evidence gates are checked against the rows that vouched for *that
933 /// build* rather than against any row that happens to carry the version string —
934 /// this is what stops a `promote --version Y` from riding on gate rows a
935 /// different build left under the same version. A `None` build id is the
936 /// legacy/pre-identity path: fall back to the version-keyed lookup so a
937 /// mid-migration tier (NULL `current_build_id`) still promotes.
938 ///
939 /// **One promote can ship several builds**, and every one of them is checked.
940 /// A pom version is two bundles with two digests, each built natively and
941 /// accepted through its own intake, so the source tier's evidence for one
942 /// architecture says nothing about the other. Checking only the build the tier
943 /// points at would let an x86_64 node take bytes whose `cargo_test` row nobody
944 /// looked at. When there is more than one, a failing gate is reported qualified
945 /// by platform, because "cargo_test not satisfied" is not actionable if the
946 /// operator cannot tell which half it is about.
947 ///
948 /// `burn_in` and `manual_confirm` are evaluated once regardless: both are keyed
949 /// on the tier's own clock (`tier_state.burn_in_started_at`), not on a build, so
950 /// asking per build would ask the same question N times and answer it N times in
951 /// the error.
952 pub(super) async fn unsatisfied_gates(
953 pool: &sqlx::SqlitePool,
954 app: &crate::domain::AppId,
955 tier: &crate::domain::TierId,
956 gates: &[crate::topology::Gate],
957 version: &str,
958 builds: &[PromotedBuild],
959 hotfix: bool,
960 ) -> std::result::Result<Vec<String>, crate::error::Error> {
961 use crate::topology::Gate;
962 let mut bad = Vec::new();
963 for gate in gates {
964 let kind = gate.kind();
965 match gate {
966 Gate::BurnIn { hours } => {
967 if hotfix {
968 continue;
969 }
970 let ok = crate::gates::burn_in_satisfied(pool, app, tier, *hours)
971 .await
972 .map_err(crate::error::Error::Other)?;
973 if !ok {
974 bad.push(kind.as_str().to_string());
975 }
976 }
977 Gate::ManualConfirm => {
978 // A confirmation must be *fresh*: recorded at or after the
979 // version's current landing on this tier (tier_state
980 // .burn_in_started_at, the per-deploy clock). Without this a
981 // confirmation row survives a rollback + rollback-forward and
982 // waves a re-deploy of the same version through with no fresh
983 // operator sign-off — weaker than burn_in, which is clock-based.
984 // No baseline (NULL) => fail closed: require a fresh confirm.
985 let confirmed_at: Option<String> = sqlx::query_scalar(
986 "SELECT finished_at FROM gate_runs
987 WHERE app = ?1 AND tier = ?2 AND version = ?3
988 AND gate_kind = 'manual_confirm' AND status = 'passed'
989 ORDER BY id DESC LIMIT 1",
990 )
991 .bind(app)
992 .bind(tier.as_str())
993 .bind(version)
994 .fetch_optional(pool)
995 .await
996 .map_err(crate::error::Error::Db)?
997 .flatten();
998 let landed_at: Option<String> = sqlx::query_scalar(
999 "SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?",
1000 )
1001 .bind(app)
1002 .bind(tier.as_str())
1003 .fetch_optional(pool)
1004 .await
1005 .map_err(crate::error::Error::Db)?
1006 .flatten();
1007 let fresh = match (confirmed_at, landed_at) {
1008 (Some(c), Some(l)) => {
1009 match (
1010 chrono::DateTime::parse_from_rfc3339(&c),
1011 chrono::DateTime::parse_from_rfc3339(&l),
1012 ) {
1013 (Ok(cd), Ok(ld)) => cd >= ld,
1014 _ => false, // unparseable timestamp -> fail closed
1015 }
1016 }
1017 _ => false,
1018 };
1019 if !fresh {
1020 bad.push(kind.as_str().to_string());
1021 }
1022 }
1023 // Build/post-deploy gates that leave a `gate_runs` row: the latest
1024 // row for this (tier, version, kind) must be `passed`. Listed
1025 // explicitly (no `_` catch-all) so adding a new `Gate` variant is a
1026 // compile error here until its promotion semantics are decided —
1027 // a transient-`blocked` kind silently falling into "needs a passed
1028 // row" would be permanently unsatisfiable.
1029 Gate::CargoTest
1030 | Gate::HardeningTest
1031 | Gate::Clippy
1032 | Gate::Fmt
1033 | Gate::CargoAudit
1034 | Gate::CargoDeny
1035 | Gate::MigrationDryRun
1036 | Gate::CodeSmoke
1037 | Gate::BootSmoke
1038 | Gate::NodeHealth
1039 // Same evidence rule as node_health: a passed row for this tier's
1040 // current build, or the promote out of here is refused.
1041 | Gate::PageSmoke => {
1042 // Latest row for this configured gate kind; NULL/missing/any
1043 // non-'passed' status all count as unsatisfied (fail closed).
1044 // Every build this promote ships has to show its own passed row.
1045 // An empty `builds` is the caller saying "no identities to key
1046 // on"; one version-keyed lookup is the pre-identity behaviour.
1047 let lookups: &[PromotedBuild] = if builds.is_empty() {
1048 &[PromotedBuild {
1049 platform: None,
1050 build_id: None,
1051 }]
1052 } else {
1053 builds
1054 };
1055 let qualify = lookups.len() > 1;
1056 for b in lookups {
1057 // Keyed on build_id when the artifact has an identity (the
1058 // evidence must be for *this* build), else on the version.
1059 let status: Option<String> = match b.build_id {
1060 Some(bid) => sqlx::query_scalar(
1061 "SELECT status FROM gate_runs
1062 WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
1063 ORDER BY id DESC LIMIT 1",
1064 )
1065 .bind(app)
1066 .bind(tier.as_str())
1067 .bind(bid)
1068 .bind(kind.as_str()),
1069 None => sqlx::query_scalar(
1070 "SELECT status FROM gate_runs
1071 WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
1072 ORDER BY id DESC LIMIT 1",
1073 )
1074 .bind(app)
1075 .bind(tier.as_str())
1076 .bind(version)
1077 .bind(kind.as_str()),
1078 }
1079 .fetch_optional(pool)
1080 .await
1081 .map_err(crate::error::Error::Db)?
1082 .flatten();
1083 if status.as_deref() != Some("passed") {
1084 // Qualified only when there is more than one build to
1085 // tell apart, so a single-platform product's message is
1086 // exactly what it always was.
1087 let name = match (&b.platform, qualify) {
1088 (Some(p), true) => format!("{} ({p})", kind.as_str()),
1089 _ => kind.as_str().to_string(),
1090 };
1091 if !bad.contains(&name) {
1092 bad.push(name);
1093 }
1094 }
1095 }
1096 }
1097 }
1098 }
1099 Ok(bad)
1100 }
1101