Skip to main content

max / makenotwork

55.4 KB · 1204 lines History Blame Raw
1 //! Promotion service: the deploy state machine lifted out of the HTTP
2 //! layer — promote orchestration, canary rollback of deployed nodes,
3 //! partial-state flags, and the promote-time gate-satisfaction check.
4 //! routes/mod.rs stays HTTP glue that calls into these.
5
6 use super::{AppState, Json, PromoteBody, Result};
7
8 pub(super) async fn promote_inner(
9 s: AppState,
10 tier: String,
11 body: PromoteBody,
12 ) -> Result<Json<serde_json::Value>> {
13 // Serialize the whole check -> deploy -> advance against any concurrent
14 // promote/rollback (CF3). Held for the entire task.
15 let _deploy_guard = s.deploy_lock.lock().await;
16 let tier = crate::domain::TierId::new(tier);
17 let idx = s
18 .topo
19 .tiers
20 .iter()
21 .position(|t| t.name == tier)
22 .ok_or(crate::error::Error::NotFound)?;
23 if idx == 0 {
24 return Err(crate::error::Error::GateBlocked(
25 "cannot /promote to the first tier; use /rebuild".into(),
26 ));
27 }
28 let target = &s.topo.tiers[idx];
29 let source = &s.topo.tiers[idx - 1];
30 // An unprovisioned tier has no nodes, so every step below is a no-op that
31 // still *looks* like success: the deploy loop iterates nothing, node_health
32 // returns Blocked, and advance_tier writes a current_version for a tier that
33 // has never received a byte. /state then reports tier c running a version it
34 // does not have. Refuse instead.
35 if !target.provisioned {
36 return Err(crate::error::Error::GateBlocked(format!(
37 "tier {} is not provisioned (no nodes); promoting to it would record a version it never received",
38 target.name,
39 )));
40 }
41
42 // Resolve the artifact through the SOURCE tier's evidence, not a version
43 // string (invariant 2, wiki [[release-artifact-identity]]). The build that is
44 // *current on the source tier* is the one that burned in and passed its gates
45 // there; promoting anything else lets a version string ride on another build's
46 // evidence and clock — the "promote --version Y inherits X's 48h" hole. A
47 // given `body.version` must name that same build.
48 let source_build: Option<(Option<i64>, Option<String>, Option<String>, Option<String>)> =
49 sqlx::query_as(
50 "SELECT ts.current_build_id, br.version, br.staged_path, br.platform
51 FROM tier_state ts
52 LEFT JOIN build_runs br ON br.id = ts.current_build_id
53 WHERE ts.app = ? AND ts.tier = ?",
54 )
55 .bind(&s.cfg.id)
56 .bind(&source.name)
57 .fetch_optional(&s.pool)
58 .await
59 .map_err(crate::error::Error::Db)?;
60
61 let (build_id, version_str, staged_dir, source_platform) = match source_build {
62 // Identity path: the source tier points at a recorded build.
63 Some((Some(bid), ver, staged_path, platform)) => {
64 let version_str = ver.ok_or_else(|| {
65 crate::error::Error::Other(anyhow::anyhow!(
66 "source build {bid} on tier {} has no recorded version",
67 source.name
68 ))
69 })?;
70 let staged_path = staged_path.ok_or_else(|| {
71 crate::error::Error::Other(anyhow::anyhow!(
72 "source build {bid} ({version_str}) has no staged_path; cannot promote"
73 ))
74 })?;
75 if let Some(req) = &body.version
76 && req != &version_str
77 {
78 return Err(crate::error::Error::GateBlocked(format!(
79 "tier {} is running build {bid} ({version_str}); refusing to promote an \
80 explicit version {req} that is not the build the tier vouched for",
81 source.name
82 )));
83 }
84 let platform = platform
85 .as_deref()
86 .map(crate::domain::Platform::parse)
87 .transpose()
88 .map_err(|e| crate::error::Error::Other(anyhow::anyhow!(e)))?;
89 (
90 Some(bid),
91 version_str,
92 std::path::PathBuf::from(staged_path),
93 platform,
94 )
95 }
96 // Legacy fallback: NULL current_build_id (a pre-identity tier). Resolve by
97 // version string through the versions table, exactly as before. One clean
98 // build-and-promote cycle re-anchors the tier onto the identity path.
99 _ => {
100 let version_str = match body.version.clone() {
101 Some(v) => v,
102 None => sqlx::query_scalar::<_, Option<String>>(
103 "SELECT current_version FROM tier_state WHERE app = ? AND tier = ?",
104 )
105 .bind(&s.cfg.id)
106 .bind(&source.name)
107 .fetch_optional(&s.pool)
108 .await
109 .map_err(crate::error::Error::Db)?
110 .flatten()
111 .ok_or_else(|| {
112 crate::error::Error::GateBlocked(format!(
113 "no version specified and tier {} has no current_version",
114 source.name
115 ))
116 })?,
117 };
118 let bin: Option<(String,)> =
119 sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
120 .bind(&s.cfg.id)
121 .bind(&version_str)
122 .fetch_optional(&s.pool)
123 .await
124 .map_err(crate::error::Error::Db)?;
125 let Some((bin,)) = bin else {
126 return Err(crate::error::Error::NotFound);
127 };
128 // `artifact_path` is the primary binary; the release dir is its parent.
129 let staged_dir = std::path::PathBuf::from(&bin)
130 .parent()
131 .ok_or_else(|| {
132 crate::error::Error::Other(anyhow::anyhow!("artifact_path has no parent"))
133 })?
134 .to_path_buf();
135 (None, version_str, staged_dir, None)
136 }
137 };
138 let version = crate::domain::Version::parse(&version_str)
139 .map_err(|e| crate::error::Error::Other(anyhow::anyhow!(e)))?;
140
141 // 1. Predecessor must have all of its configured gates satisfied for this
142 // build (with optional hotfix override that skips burn_in). Evaluated
143 // against the topology gate list, so a gate that never ran blocks the
144 // promote instead of being treated as green.
145 //
146 // A migration-bearing promote (`bears_migration`) forces a fresh
147 // `manual_confirm` on the predecessor even if the tier does not configure
148 // one: rollback restores the binary + release_contents only, never the
149 // schema, so the operator must consciously acknowledge the one-way advance
150 // (deploy/README.md "Rollback contract"). `hotfix` skips only `burn_in`, so
151 // it does not suppress this confirm.
152 let mut effective_gates = source.gates.clone();
153 if body.bears_migration
154 && !effective_gates
155 .iter()
156 .any(|g| matches!(g, crate::topology::Gate::ManualConfirm))
157 {
158 effective_gates.push(crate::topology::Gate::ManualConfirm);
159 }
160 // Which bytes each node gets is resolved FIRST, because it is what says
161 // how many builds the gate check has to cover. A two-architecture promote
162 // ships two bundles with two sets of evidence, and checking only the one
163 // the source tier points at would wave the sibling through on gate rows
164 // nobody read. It also means a version missing its other half fails here,
165 // before any gate work, rather than halfway down the rollout.
166 let target_nodes: Vec<&crate::topology::Node> = target.nodes.iter().collect();
167 let bundles = bundles_for_nodes(
168 &s,
169 &version_str,
170 &target_nodes,
171 &staged_dir,
172 source_platform.as_ref(),
173 build_id,
174 )
175 .await?;
176 let mut promoted_builds = distinct_builds(&bundles);
177 if promoted_builds.is_empty() {
178 // A provisioned tier with no nodes ships nothing, so there is nothing to
179 // resolve — but the gate check still has to be keyed on the source build.
180 // Left empty it would fall through to the version-string lookup, which is
181 // the pre-identity path and weaker than what this tier is owed. The
182 // `provisioned` guard above makes this unreachable today; it is written
183 // out because the alternative fails quietly in the direction of less
184 // evidence.
185 promoted_builds.push(PromotedBuild {
186 platform: source_platform.clone(),
187 build_id,
188 });
189 }
190
191 // The build the TARGET tier ends up running. The same as the source's for a
192 // single-platform product, and its cross-architecture sibling when the two
193 // tiers differ — promoting pom astra -> hetzner reads an aarch64 build from
194 // the source and ships the x86_64 one. Every row this promote writes about
195 // the target (its `deploys`, its post-deploy gate runs, its `tier_state`)
196 // keys on this, so the next promote out of that tier resolves through
197 // evidence for bytes that box actually ran.
198 //
199 // A tier whose nodes span platforms has no single current build, and NULL
200 // says so rather than naming one of them arbitrarily. No such tier exists
201 // today: pom's astra and hetzner tiers are one node each.
202 let target_build_id = match promoted_builds.as_slice() {
203 [only] => only.build_id,
204 _ => None,
205 };
206
207 let pending = unsatisfied_gates(
208 &s.pool,
209 &s.cfg.id,
210 &source.name,
211 &effective_gates,
212 &Evidence {
213 version: &version_str,
214 builds: &promoted_builds,
215 // The source tier's own build: what its node_health / page_smoke rows
216 // can possibly be about, which is not the same thing as what this
217 // promote ships when the two tiers are different architectures.
218 tier_build: build_id,
219 },
220 body.hotfix,
221 )
222 .await?;
223 if !pending.is_empty() {
224 return Err(crate::error::Error::GateBlocked(format!(
225 "{} gate(s) not satisfied on tier {}: {}",
226 pending.len(),
227 source.name,
228 pending.join(", "),
229 )));
230 }
231
232 // The version this tier was running before this promote — the rollback
233 // target if a canary node fails partway through a multi-node rollout.
234 let prev_version: Option<String> =
235 sqlx::query_scalar("SELECT current_version FROM tier_state WHERE app = ? AND tier = ?")
236 .bind(&s.cfg.id)
237 .bind(&target.name)
238 .fetch_optional(&s.pool)
239 .await
240 .map_err(crate::error::Error::Db)?
241 .flatten();
242
243 // 3. Deploy to each node. Sequential canary is the only policy
244 // implemented in v0; parallel is a one-line change once we trust the
245 // sequential path. Track the nodes already flipped to the new version so
246 // a mid-rollout failure can roll them back (canary rollback).
247 let mut deployed: Vec<&crate::topology::Node> = Vec::new();
248 for (node, node_bundle, node_bundle_platform, node_build_id) in &bundles {
249 // The proof that these bytes belong on this box. Built before the
250 // deploy row is written, so a mismatch never becomes an `in_progress`
251 // deploy that has to be reconciled.
252 let placement =
253 crate::deploy::Placement::check(node, node_bundle, node_bundle_platform.as_ref())
254 .map_err(|e| crate::error::Error::GateBlocked(e.to_string()))?;
255 let started = chrono::Utc::now().to_rfc3339();
256 crate::events::emit(
257 &s.events,
258 crate::events::Event::DeployStart {
259 tier: target.name.clone(),
260 node: node.name.clone(),
261 version: version.clone(),
262 },
263 );
264 let executor = s
265 .executors
266 .get(&node.name)
267 .cloned()
268 .unwrap_or_else(|| crate::state::build_executor(node));
269 // Record the deploy as `in_progress` BEFORE touching the node, so a crash
270 // between the node's symlink swap and this promote's `advance_tier` leaves
271 // a durable trace. Without the pre-write, a SIGKILL after the swap but
272 // before the row is inserted would land the new binary on the node with no
273 // DB evidence at all, and the startup reconcile — which reads `deploys` —
274 // could not see it. The row is finalized to ok/failed immediately below;
275 // any row still `in_progress` at startup is an orphan the reconcile settles
276 // and flags ([`crate::reconcile`]).
277 let deploy_id: i64 = sqlx::query_scalar(
278 "INSERT INTO deploys (app, version, tier, node, started_at, outcome, hotfix, reset_burn_in, build_id)
279 VALUES (?, ?, ?, ?, ?, 'in_progress', ?, ?, ?) RETURNING id",
280 )
281 .bind(&s.cfg.id)
282 .bind(&version).bind(&target.name).bind(&node.name)
283 .bind(&started)
284 .bind(body.hotfix as i64).bind(body.reset_burn_in as i64)
285 // The build THIS node received, which is not the source tier's when the
286 // two are different architectures: promoting pom astra -> hetzner reads
287 // an aarch64 build from the source and ships the x86_64 sibling. Binding
288 // the source's id here recorded the wrong bytes against the deploy, and
289 // `crate::reconcile` reads these rows.
290 .bind(node_build_id)
291 .fetch_one(&s.pool).await.map_err(crate::error::Error::Db)?;
292 let result = crate::deploy::deploy_node(
293 executor.as_ref(),
294 placement,
295 &version_str,
296 s.cfg.primary_bin(),
297 )
298 .await;
299 let finished = chrono::Utc::now().to_rfc3339();
300 let (outcome_obj, err_for_propagation) = match result {
301 Ok(_) => (crate::outcome::DeployOutcome::ok(), None),
302 Err(e) => {
303 let msg = format!("{e:#}");
304 let kind = crate::classify::classify_deploy_error(&msg);
305 (crate::outcome::DeployOutcome::failed(kind), Some(e))
306 }
307 };
308 let outcome_json = serde_json::to_string(&outcome_obj)
309 .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
310 sqlx::query(
311 "UPDATE deploys SET finished_at = ?, outcome = ?, outcome_json = ? WHERE id = ?",
312 )
313 .bind(&finished)
314 .bind(outcome_obj.status_str())
315 .bind(&outcome_json)
316 .bind(deploy_id)
317 .execute(&s.pool)
318 .await
319 .map_err(crate::error::Error::Db)?;
320 if let Some(e) = err_for_propagation {
321 let crate::outcome::DeployStatus::Failed { failure } = outcome_obj.status else {
322 unreachable!("err_for_propagation is Some iff status is Failed");
323 };
324 tracing::error!(
325 tier = %target.name, node = %node.name, version = %version,
326 failure = failure.summary(),
327 "deploy failed; current symlink left intact, tier_state not advanced"
328 );
329 crate::events::emit(
330 &s.events,
331 crate::events::Event::DeployFailed {
332 tier: target.name.clone(),
333 node: node.name.clone(),
334 version: version.clone(),
335 failure,
336 },
337 );
338
339 // Canary rollback: restore every node this promote touched — the
340 // ones already flipped to the new version AND this failed node
341 // (whose state is indeterminate: the symlink swap may have landed
342 // before the restart failed) — back to the tier's prior version, so
343 // the fleet is left consistent on `prev` rather than split-brain.
344 // Nodes after this one were never touched and stay on `prev`.
345 deployed.push(node);
346 let touched = deployed.len();
347 match prev_version.as_deref() {
348 Some(prev) => {
349 let report = rollback_deployed_nodes(&s, &target.name, &deployed, prev).await;
350 // Say what is true, and say it differently when nothing is
351 // wrong. `restored=0 of=1` read as total failure when the
352 // accurate reading was "0 needed restoring" — the same
353 // confusion as the per-node message, one level up.
354 if report.is_consistent() {
355 tracing::warn!(
356 tier = %target.name,
357 restored = report.restored,
358 already_on_previous = report.already_on_previous,
359 of = report.touched(),
360 from = %version, to = prev,
361 "canary failed mid-rollout; every touched node is on the previous \
362 version and the tier is consistent",
363 );
364 } else {
365 tracing::error!(
366 tier = %target.name,
367 restored = report.restored,
368 already_on_previous = report.already_on_previous,
369 indeterminate = report.indeterminate,
370 of = report.touched(),
371 from = %version, to = prev,
372 "canary failed mid-rollout and the tier is NOT consistent; some nodes \
373 have an indeterminate version",
374 );
375 }
376 if report.restored > 0
377 && let Ok(prev_v) = crate::domain::Version::parse(prev)
378 {
379 crate::events::emit(
380 &s.events,
381 crate::events::Event::Rollback {
382 tier: target.name.clone(),
383 from: version.clone(),
384 to: prev_v,
385 },
386 );
387 }
388 // The tier is consistent when no node's version is unknown —
389 // which includes the case where a rollback "failed" before
390 // the swap and so left the node on `prev` already. Flagging
391 // that as partial would put a permanent scare on /state for
392 // a fleet that is entirely on one version.
393 if report.is_consistent() {
394 clear_partial(&s, &target.name).await;
395 } else {
396 set_partial(&s, &target.name, &format!(
397 "canary rollback incomplete: {indeterminate} of {touched} node(s) have an \
398 indeterminate version and may be on {version}; {restored} restored to \
399 {prev}, {already} were never swapped — manual check needed",
400 touched = report.touched(),
401 indeterminate = report.indeterminate,
402 restored = report.restored,
403 already = report.already_on_previous,
404 )).await;
405 }
406 }
407 None => {
408 tracing::error!(
409 tier = %target.name, count = touched, version = %version,
410 "canary failed on a first deploy (no previous version to restore to); \
411 touched nodes remain on the new version — manual cleanup needed",
412 );
413 set_partial(
414 &s,
415 &target.name,
416 &format!(
417 "first-deploy canary failed: {touched} node(s) left on {version}, \
418 no prior version to restore — manual cleanup needed",
419 ),
420 )
421 .await;
422 }
423 }
424 return Err(crate::error::Error::Other(e));
425 }
426 deployed.push(node);
427 crate::events::emit(
428 &s.events,
429 crate::events::Event::DeployOk {
430 tier: target.name.clone(),
431 node: node.name.clone(),
432 version: version.clone(),
433 },
434 );
435 }
436
437 // 3b. Run this tier's post-deploy gates (node_health) against the freshly
438 // deployed nodes and record their outcomes. These rows are the evidence
439 // the NEXT promote (this tier -> the following one) checks via
440 // `unsatisfied_gates`. Before CF1, only the host tier ran gates, so
441 // A/B/C had no evidence and promotion waved through; node_health now
442 // proves the deployed nodes are serving (Run-2 SERIOUS-3: boot_smoke
443 // used to re-run the staged binary locally and proved nothing about the
444 // node). burn_in / manual_confirm are not run here — they are evaluated
445 // live / by the operator at the next promote. A failed gate does not
446 // unwind this deploy (the artifact is already live on the tier); it
447 // blocks the next promote, which is the fail-closed behavior we want.
448 //
449 // It does, however, fail *this* promote's response. tier_state still
450 // advances below — the nodes genuinely run this version and a stale
451 // `current_version` would send a later rollback to the wrong artifact —
452 // but the tier is flagged partial and the handler returns the failure,
453 // so the operator learns at the promote instead of discovering it as an
454 // unexplained block on the next one.
455 let post_deploy: Vec<crate::topology::Gate> = target
456 .gates
457 .iter()
458 .filter(|g| g.runs_post_deploy())
459 .cloned()
460 .collect();
461 let mut post_deploy_failure: Option<String> = None;
462 if !post_deploy.is_empty() {
463 // node_health probes each node the deploy just shipped to, over the same
464 // executor the deploy used. Build the probe set from the tier's nodes and
465 // the startup executor map; a node missing an executor (shouldn't happen
466 // — both come from the same topology) is skipped, and an empty set makes
467 // node_health Blocked (fail closed).
468 let nodes: Vec<crate::gates::NodeProbe> = target
469 .nodes
470 .iter()
471 .filter_map(|n| {
472 s.executors
473 .get(&n.name)
474 .map(|exec| crate::gates::NodeProbe {
475 node: n.name.clone(),
476 service: n.service_name.clone(),
477 health_url: n.health_url.clone(),
478 executor: exec.clone(),
479 })
480 })
481 .collect();
482 let ctx = crate::gates::GateCtx {
483 pool: s.pool.clone(),
484 cfg: s.cfg.clone(),
485 tier: target.name.clone(),
486 version: version.clone(),
487 // No worktree at promote time; node_health works over executors, not
488 // a checkout. No bundle either: the gates here are about the nodes,
489 // not about bytes on this host.
490 worktree: None,
491 bundle: None,
492 events: s.events.clone(),
493 nodes,
494 // These post-deploy gate rows are the evidence the NEXT promote (this
495 // tier -> the following one) resolves through, so they must carry the
496 // build they vouch for — the one this tier just received, which is
497 // not the source's across an architecture boundary.
498 build_id: target_build_id,
499 // The hostname the public uses, straight from the tier. page_smoke
500 // has to request the site the way a visitor does; anything derived
501 // from a node would reach the origin and miss the CDN, which is the
502 // layer the gate exists to watch.
503 public_url: target.public_url.clone(),
504 // No checkout at promote time, so nothing to resolve against.
505 aux_dirs: std::collections::HashMap::new(),
506 };
507 post_deploy_failure = match crate::gates::run_all(&ctx, &post_deploy).await {
508 Ok(failed) if failed.is_empty() => None,
509 Ok(failed) => {
510 let names = failed
511 .iter()
512 .map(|k| k.as_str())
513 .collect::<Vec<_>>()
514 .join(", ");
515 tracing::warn!(
516 tier = %target.name, version = %version, gates = %names,
517 "post-deploy gate(s) failed; tier advanced but promotion to the next tier is blocked",
518 );
519 Some(format!(
520 "post-deploy gate(s) failed on {version}: {names}; \
521 the tier is serving {version} but cannot promote onward until they pass"
522 ))
523 }
524 Err(e) => {
525 tracing::error!(
526 tier = %target.name, version = %version, error = %e,
527 "post-deploy gate execution errored; promotion to the next tier is blocked",
528 );
529 Some(format!(
530 "post-deploy gate execution errored on {version}: {e}; \
531 the tier is serving {version} but cannot promote onward until the gates pass"
532 ))
533 }
534 };
535 }
536
537 // 4. Advance tier_state through the single sealed forward-advance op (atomic
538 // self-referential UPDATE; no read-modify-write to lose under concurrency,
539 // CF3). We hold deploy_lock for this whole handler, so the advance is
540 // serialized against rollback and the host build path's advance.
541 // reset_burn_in on the *source* tier nulls its clock only when the operator
542 // explicitly asked.
543 // `target_build_id`, not the source's: recording the source's would have
544 // left hetzner's `current_build_id` pointing at astra's aarch64 bundle,
545 // i.e. at bytes that box cannot execute.
546 crate::runs::advance_tier(
547 &s.pool,
548 &s.cfg.id,
549 target.name.as_str(),
550 &version,
551 target_build_id,
552 )
553 .await
554 .map_err(crate::error::Error::Db)?;
555
556 if body.reset_burn_in {
557 sqlx::query("UPDATE tier_state SET burn_in_started_at = NULL WHERE app = ? AND tier = ?")
558 .bind(&s.cfg.id)
559 .bind(&source.name)
560 .execute(&s.pool)
561 .await
562 .map_err(crate::error::Error::Db)?;
563 }
564
565 // Red post-deploy gates: the rollout itself reached every node, but the tier
566 // is not fit to promote onward. Flag it so /state and the TUI say so, and
567 // return the failure rather than a 200 the operator would read as "shipped,
568 // all good". tier_state has already advanced above — it describes what the
569 // nodes are running, not whether we're happy about it.
570 if let Some(reason) = post_deploy_failure {
571 set_partial(&s, &target.name, &reason).await;
572 return Err(crate::error::Error::GateBlocked(reason));
573 }
574
575 // A clean full rollout to every node clears any prior partial flag on this tier.
576 clear_partial(&s, &target.name).await;
577
578 crate::events::emit(
579 &s.events,
580 crate::events::Event::PromoteComplete {
581 tier: target.name.clone(),
582 version: version.clone(),
583 },
584 );
585 tracing::info!(
586 version = %version, tier = %target.name,
587 hotfix = body.hotfix, reset_burn_in = body.reset_burn_in,
588 "promote complete",
589 );
590
591 Ok(Json(serde_json::json!({
592 "tier": target.name,
593 "version": version,
594 "nodes_deployed": target.nodes.iter().map(|n| n.name.clone()).collect::<Vec<_>>(),
595 })))
596 }
597
598 /// What a canary rollback actually left behind, per node.
599 ///
600 /// Three outcomes, not two, and conflating the middle one with the last is the
601 /// bug this type exists to prevent: a rollback that fails *before* the symlink
602 /// swap leaves the node exactly where it already was, on the previous version.
603 /// Reporting that as "stranded on the new version, manual intervention needed"
604 /// sends an operator to do surgery on a healthy production box.
605 #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
606 pub(super) struct RollbackReport {
607 /// Put back on the previous version by a successful redeploy.
608 pub(super) restored: usize,
609 /// Rollback failed before the swap, so the node never left the previous
610 /// version. Nothing is stranded and nothing needs doing.
611 pub(super) already_on_previous: usize,
612 /// Rollback failed at or after the swap, or could not be attempted at all.
613 /// The node's version is not knowable from here. This is the only outcome
614 /// that warrants a human.
615 pub(super) indeterminate: usize,
616 }
617
618 impl RollbackReport {
619 /// Every touched node accounted for.
620 pub(super) fn touched(self) -> usize {
621 self.restored + self.already_on_previous + self.indeterminate
622 }
623
624 /// True when no node is in an unknown state, whether or not every rollback
625 /// "succeeded". A tier whose rollbacks all failed before the swap is
626 /// consistent on the previous version and is not an incident.
627 pub(super) fn is_consistent(self) -> bool {
628 self.indeterminate == 0
629 }
630 }
631
632 /// A node, the bundle it is to receive, what that bundle runs on, and the build
633 /// row that bundle came from.
634 ///
635 /// The build id is what makes the gate check per-platform: evidence is keyed on
636 /// the build that produced the bytes, so a promote that ships two architectures
637 /// has to look up two sets of gate rows, and this is where it learns which two.
638 /// `None` is the legacy path (a bundle resolved by version string, from a tier
639 /// with no `current_build_id`), where there is no build to key on.
640 pub(super) type NodeBundle<'a> = (
641 &'a crate::topology::Node,
642 std::path::PathBuf,
643 Option<crate::domain::Platform>,
644 Option<i64>,
645 );
646
647 /// One bundle a promote is about to ship, and the evidence key it is judged by.
648 ///
649 /// A single-platform product has exactly one of these and it is the source
650 /// tier's own build, which is what every promote before two-architecture support
651 /// checked. A product like pom has one per architecture, each standing on its
652 /// own intake and its own gate run.
653 #[derive(Debug, Clone, PartialEq, Eq)]
654 pub(super) struct PromotedBuild {
655 /// What the bundle runs on, when it says. Used only to qualify the name of a
656 /// failing gate, so an operator reading "cargo_test" knows which half.
657 pub platform: Option<crate::domain::Platform>,
658 /// The `build_runs` row the evidence is keyed on; `None` falls back to the
659 /// version-keyed lookup for a pre-identity tier.
660 pub build_id: Option<i64>,
661 }
662
663 /// What a promote is judged against, in one value: the release it names, the
664 /// artifacts it will ship, and the build the tier it is leaving is running.
665 ///
666 /// The three travel together because the gate check needs all three at once and
667 /// they answer different questions. `version` is the label and the pre-identity
668 /// fallback key. `builds` are the bytes going out, which artifact evidence is
669 /// keyed on. `tier_build` is what the source tier is itself running, which
670 /// post-deploy evidence is keyed on — and across an architecture boundary those
671 /// last two are different builds. See [`unsatisfied_gates`].
672 pub(super) struct Evidence<'a> {
673 pub version: &'a str,
674 pub builds: &'a [PromotedBuild],
675 pub tier_build: Option<i64>,
676 }
677
678 /// Which bytes each node gets, resolved before any node is touched.
679 ///
680 /// A single-platform product answers with `fallback` for every node, which is
681 /// the bundle the caller already resolved (the source tier's build on a promote,
682 /// the previous version's on a rollback). A product whose one version is several
683 /// bundles answers per node, because the node is the thing that says which
684 /// architecture it can run.
685 ///
686 /// Resolving the whole tier up front is deliberate: a version missing its
687 /// x86_64 half fails the promote before its aarch64 half has been pushed
688 /// anywhere, instead of halfway down a rollout with nodes already flipped.
689 pub(super) async fn bundles_for_nodes<'a>(
690 s: &AppState,
691 version: &str,
692 nodes: &[&'a crate::topology::Node],
693 fallback: &std::path::Path,
694 fallback_platform: Option<&crate::domain::Platform>,
695 fallback_build_id: Option<i64>,
696 ) -> Result<Vec<NodeBundle<'a>>> {
697 let mut out = Vec::with_capacity(nodes.len());
698 for node in nodes.iter().copied() {
699 let resolved = match &node.platform {
700 // The node states nothing, so there is nothing to resolve against;
701 // it gets the caller's bundle, and `Placement::check` decides
702 // whether that pairing is admissible at all.
703 None => (
704 fallback.to_path_buf(),
705 fallback_platform.cloned(),
706 fallback_build_id,
707 ),
708 // The node states a platform. Give it the bundle recorded for that
709 // platform at this version — the caller's own when they agree, its
710 // sibling when they do not.
711 Some(want) if fallback_platform == Some(want) => (
712 fallback.to_path_buf(),
713 fallback_platform.cloned(),
714 fallback_build_id,
715 ),
716 Some(want) => {
717 let (build_id, path) = bundle_for_platform(s, version, want).await?;
718 (path, Some(want.clone()), build_id)
719 }
720 };
721 out.push((node, resolved.0, resolved.1, resolved.2));
722 }
723 Ok(out)
724 }
725
726 /// The distinct builds `bundles` will ship, in a stable order.
727 ///
728 /// Deduplicated because a tier is usually several nodes on one architecture, and
729 /// evaluating one build's gates once per node would say the same thing three
730 /// times in the error an operator reads. Keyed on the whole entry rather than on
731 /// the build id alone, so the legacy `None` case does not collapse two
732 /// version-resolved bundles into one.
733 pub(super) fn distinct_builds(bundles: &[NodeBundle<'_>]) -> Vec<PromotedBuild> {
734 let mut out: Vec<PromotedBuild> = Vec::new();
735 for (_, _, platform, build_id) in bundles {
736 let entry = PromotedBuild {
737 platform: platform.clone(),
738 build_id: *build_id,
739 };
740 if !out.contains(&entry) {
741 out.push(entry);
742 }
743 }
744 out
745 }
746
747 /// The bundle recorded for `version` on `platform`.
748 ///
749 /// This is what makes a two-architecture product promotable. One pom version is
750 /// two bundles with two digests, each built natively by Bento and accepted
751 /// through its own intake; the tier ladder carries the version forward and this
752 /// answers "which of that version's bundles does this box take".
753 ///
754 /// Only a run that settled green qualifies. A sibling that failed its own host
755 /// gates is not a fallback for the one that passed — the source tier's evidence
756 /// says nothing about bytes it never saw, so each architecture's bundle stands
757 /// on its own intake and its own gate run.
758 async fn bundle_for_platform(
759 s: &AppState,
760 version: &str,
761 platform: &crate::domain::Platform,
762 ) -> Result<(Option<i64>, std::path::PathBuf)> {
763 let row: Option<(i64, Option<String>)> = sqlx::query_as(
764 "SELECT id, staged_path FROM build_runs
765 WHERE app = ? AND version = ? AND platform = ? AND result = 'passed'
766 ORDER BY id DESC LIMIT 1",
767 )
768 .bind(&s.cfg.id)
769 .bind(version)
770 .bind(platform.to_string())
771 .fetch_optional(&s.pool)
772 .await
773 .map_err(crate::error::Error::Db)?;
774
775 match row {
776 // The id comes back alongside the path so the caller can check this
777 // build's own gate evidence rather than the source tier's. A green build
778 // row is not a green gate run: the build says the bytes compiled, the
779 // gate run says the tier vouched for them.
780 Some((id, Some(path))) => Ok((Some(id), std::path::PathBuf::from(path))),
781 Some((_, None)) => Err(crate::error::Error::Other(anyhow::anyhow!(
782 "the {platform} build of {version} has no staged_path; cannot promote it"
783 ))),
784 None => Err(crate::error::Error::GateBlocked(format!(
785 "no green {platform} bundle recorded for {version}. Each architecture is \
786 a separate artifact with its own evidence, so this one has to be built \
787 and accepted before a {platform} node can take this version"
788 ))),
789 }
790 }
791
792 /// After a canary node fails mid-promote, restore the nodes already flipped to
793 /// the new version back to `prev_version`, leaving the tier consistent (all on
794 /// the old version) rather than split-brain. Best-effort: every node is
795 /// attempted; a per-node failure is logged but never propagated (the promote is
796 /// already failing).
797 ///
798 /// Returns a [`RollbackReport`] rather than a bare count, because "the rollback
799 /// failed" is not the same claim as "the node is on the new version" and the
800 /// caller has to be able to tell them apart. When the previous version has no
801 /// recorded artifact, no rollback can be attempted and every touched node is
802 /// reported indeterminate.
803 pub(super) async fn rollback_deployed_nodes(
804 s: &AppState,
805 tier: &crate::domain::TierId,
806 nodes: &[&crate::topology::Node],
807 prev_version: &str,
808 ) -> RollbackReport {
809 let bin: Option<(String,)> =
810 match sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
811 .bind(&s.cfg.id)
812 .bind(prev_version)
813 .fetch_optional(&s.pool)
814 .await
815 {
816 Ok(b) => b,
817 Err(e) => {
818 tracing::error!(tier = %tier, prev = prev_version, error = %e,
819 "canary rollback: looking up the previous artifact failed; no rollback attempted");
820 return RollbackReport {
821 indeterminate: nodes.len(),
822 ..Default::default()
823 };
824 }
825 };
826 let Some((bin,)) = bin else {
827 tracing::error!(tier = %tier, prev = prev_version, nodes = nodes.len(),
828 "canary rollback: previous version has no artifact_path; no rollback attempted");
829 return RollbackReport {
830 indeterminate: nodes.len(),
831 ..Default::default()
832 };
833 };
834 let Some(staged_dir) = std::path::PathBuf::from(&bin)
835 .parent()
836 .map(std::path::Path::to_path_buf)
837 else {
838 tracing::error!(tier = %tier, prev = prev_version,
839 "canary rollback: previous artifact_path has no parent dir; no rollback attempted");
840 return RollbackReport {
841 indeterminate: nodes.len(),
842 ..Default::default()
843 };
844 };
845
846 // A rollback ships bytes too, so it needs the same per-node resolution a
847 // promote does: on a two-architecture product the previous version is also
848 // two bundles, and restoring the wrong one would leave the node no better
849 // off than the failed canary did. A resolution failure here cannot be
850 // propagated (the promote is already failing), so it reports the node as
851 // indeterminate — which is the truth: nothing was attempted and the node's
852 // version is whatever the failed deploy left.
853 // No fallback build id: a rollback resolves the previous version's bundles
854 // by version string and re-runs no gates, so there is no evidence to key.
855 // The bytes are ones this tier already ran.
856 let bundles = match bundles_for_nodes(s, prev_version, nodes, &staged_dir, None, None).await {
857 Ok(b) => b,
858 Err(e) => {
859 tracing::error!(tier = %tier, prev = prev_version, error = %e,
860 "canary rollback: could not resolve a previous bundle per node; no rollback attempted");
861 return RollbackReport {
862 indeterminate: nodes.len(),
863 ..Default::default()
864 };
865 }
866 };
867
868 let mut report = RollbackReport::default();
869 for (node, node_bundle, node_bundle_platform, _) in &bundles {
870 let executor = s
871 .executors
872 .get(&node.name)
873 .cloned()
874 .unwrap_or_else(|| crate::state::build_executor(node));
875 let placement =
876 match crate::deploy::Placement::check(node, node_bundle, node_bundle_platform.as_ref())
877 {
878 Ok(p) => p,
879 Err(e) => {
880 report.indeterminate += 1;
881 tracing::error!(tier = %tier, node = %node.name, error = %e,
882 "canary rollback: refused to place the previous bundle on this node");
883 continue;
884 }
885 };
886 match crate::deploy::deploy_node(
887 executor.as_ref(),
888 placement,
889 prev_version,
890 s.cfg.primary_bin(),
891 )
892 .await
893 {
894 Ok(_) => {
895 report.restored += 1;
896 tracing::warn!(tier = %tier, node = %node.name, version = prev_version,
897 "canary rollback: node restored to the previous version");
898 }
899 // A rollback is itself a deploy, so it fails at a stage too. Failing
900 // before the swap means it never touched `current` — the node is
901 // still on the version it was already running, which is the one we
902 // were rolling back TO. That is the intended end state reached by a
903 // different route, not an incident.
904 Err(e) => match crate::deploy::stage_of(&e) {
905 Some(crate::deploy::FailureStage::BeforeSwap) => {
906 report.already_on_previous += 1;
907 tracing::warn!(
908 tier = %tier, node = %node.name, version = prev_version,
909 error = %format!("{e:#}"),
910 "canary rollback did not run, and did not need to: it failed before the \
911 symlink swap, so the node is already on the previous version",
912 );
913 }
914 // Unannotated errors land here deliberately. Guessing "safe"
915 // would reintroduce the original bug in the worse direction.
916 stage => {
917 report.indeterminate += 1;
918 tracing::error!(
919 tier = %tier, node = %node.name, version = prev_version,
920 error = %format!("{e:#}"),
921 stage = ?stage,
922 "canary rollback FAILED for node at or after the symlink swap; its version \
923 is indeterminate — manual intervention needed",
924 );
925 }
926 },
927 }
928 }
929 report
930 }
931
932 /// Flag a tier as left in a partial / mixed-version state, with a human-readable
933 /// reason surfaced through `/state` and the TUI. Best-effort: a failure to record
934 /// the flag is logged, never propagated — the caller is already on an error path
935 /// and the worse outcome is to mask the original failure with a bookkeeping one.
936 pub(super) async fn set_partial(s: &AppState, tier: &crate::domain::TierId, reason: &str) {
937 if let Err(e) =
938 sqlx::query("UPDATE tier_state SET partial_reason = ? WHERE app = ? AND tier = ?")
939 .bind(reason)
940 .bind(&s.cfg.id)
941 .bind(tier)
942 .execute(&s.pool)
943 .await
944 {
945 tracing::error!(tier = %tier, reason, error = %e,
946 "failed to record tier partial state; the fleet may be inconsistent without a /state flag");
947 }
948 }
949
950 /// Clear a tier's partial flag after a clean full promote or rollback. Errors are
951 /// logged but not propagated: the deploy itself succeeded, and a stale flag is a
952 /// visible nuisance, not a safety regression (the operator sees a partial marker
953 /// on a tier that is actually fine, and re-checks).
954 pub(super) async fn clear_partial(s: &AppState, tier: &crate::domain::TierId) {
955 if let Err(e) =
956 sqlx::query("UPDATE tier_state SET partial_reason = NULL WHERE app = ? AND tier = ?")
957 .bind(&s.cfg.id)
958 .bind(tier)
959 .execute(&s.pool)
960 .await
961 {
962 tracing::warn!(tier = %tier, error = %e, "failed to clear tier partial flag");
963 }
964 }
965
966 /// Returns the kinds of `tier`'s *configured* gates that are not satisfied for
967 /// `version`. `hotfix` suppresses the `burn_in` requirement only.
968 ///
969 /// Fail-closed against the topology gate list (the CF1 fix). The previous
970 /// version inspected only existing `gate_runs` rows, so a configured gate that
971 /// had *never run* produced no row and was invisibly treated as green — letting
972 /// a promote wave through with zero evidence (it shipped 0.9.5 to prod with
973 /// tier A's `boot_smoke` never recorded). Now every configured gate must show
974 /// positive evidence:
975 /// - `burn_in` is evaluated live against the tier's clock (a stored `blocked`
976 /// row would otherwise never flip to passed as time elapses);
977 /// - every other kind requires a `passed` row for (tier, version) — a missing
978 /// or non-passed latest row counts as unsatisfied.
979 ///
980 /// `builds` are the artifacts this promote will actually ship (wiki note
981 /// `release-artifact-identity`). Each carries a `build_id`, and the deterministic
982 /// build-evidence gates are checked against the rows that vouched for *that
983 /// build* rather than against any row that happens to carry the version string —
984 /// this is what stops a `promote --version Y` from riding on gate rows a
985 /// different build left under the same version. A `None` build id is the
986 /// legacy/pre-identity path: fall back to the version-keyed lookup so a
987 /// mid-migration tier (NULL `current_build_id`) still promotes.
988 ///
989 /// **One promote can ship several builds**, and every one of them is checked.
990 /// A pom version is two bundles with two digests, each built natively and
991 /// accepted through its own intake, so the source tier's evidence for one
992 /// architecture says nothing about the other. Checking only the build the tier
993 /// points at would let an x86_64 node take bytes whose `cargo_test` row nobody
994 /// looked at. When there is more than one, a failing gate is reported qualified
995 /// by platform, because "cargo_test not satisfied" is not actionable if the
996 /// operator cannot tell which half it is about.
997 ///
998 /// `burn_in` and `manual_confirm` are evaluated once regardless: both are keyed
999 /// on the tier's own clock (`tier_state.burn_in_started_at`), not on a build, so
1000 /// asking per build would ask the same question N times and answer it N times in
1001 /// the error.
1002 ///
1003 /// **So are `node_health` and `page_smoke`, and for a sharper reason: they are
1004 /// evidence about this tier, not about the bytes leaving it.** `tier_build` is
1005 /// the build this tier is itself running, and it is the only build those two can
1006 /// ever have a row for. Asking them per shipped build is a category error that
1007 /// made a cross-architecture promote structurally impossible: promoting pom
1008 /// 0.4.3 astra -> hetzner ships the **x86_64** bundle, astra has only ever run
1009 /// the **aarch64** one, and so `node_health` was looked up against a build astra
1010 /// never saw and never could — astra is aarch64. Fail-closed reported it
1011 /// unsatisfied, `/state` reported it passed (astra's own row, correctly), and the
1012 /// promote refused twice with the gate satisfied by every reading the operator
1013 /// had. It could not have been cleared by re-running anything.
1014 ///
1015 /// The artifact gates below stay per build and must: `cargo_test` on the aarch64
1016 /// bundle says nothing about the x86_64 one, and each stands on its own intake.
1017 /// What separates the two lists is whether the evidence travels with the bytes.
1018 /// A single-platform product is unaffected either way — its source tier's build
1019 /// *is* the build being shipped, so both readings ask the same question.
1020 pub(super) async fn unsatisfied_gates(
1021 pool: &sqlx::SqlitePool,
1022 app: &crate::domain::AppId,
1023 tier: &crate::domain::TierId,
1024 gates: &[crate::topology::Gate],
1025 evidence: &Evidence<'_>,
1026 hotfix: bool,
1027 ) -> std::result::Result<Vec<String>, crate::error::Error> {
1028 use crate::topology::Gate;
1029 let Evidence {
1030 version,
1031 builds,
1032 tier_build,
1033 } = *evidence;
1034 let mut bad = Vec::new();
1035 for gate in gates {
1036 let kind = gate.kind();
1037 match gate {
1038 Gate::BurnIn { hours } => {
1039 if hotfix {
1040 continue;
1041 }
1042 let ok = crate::gates::burn_in_satisfied(pool, app, tier, *hours)
1043 .await
1044 .map_err(crate::error::Error::Other)?;
1045 if !ok {
1046 bad.push(kind.as_str().to_string());
1047 }
1048 }
1049 Gate::ManualConfirm => {
1050 // A confirmation must be *fresh*: recorded at or after the
1051 // version's current landing on this tier (tier_state
1052 // .burn_in_started_at, the per-deploy clock). Without this a
1053 // confirmation row survives a rollback + rollback-forward and
1054 // waves a re-deploy of the same version through with no fresh
1055 // operator sign-off — weaker than burn_in, which is clock-based.
1056 // No baseline (NULL) => fail closed: require a fresh confirm.
1057 let confirmed_at: Option<String> = sqlx::query_scalar(
1058 "SELECT finished_at FROM gate_runs
1059 WHERE app = ?1 AND tier = ?2 AND version = ?3
1060 AND gate_kind = 'manual_confirm' AND status = 'passed'
1061 ORDER BY id DESC LIMIT 1",
1062 )
1063 .bind(app)
1064 .bind(tier.as_str())
1065 .bind(version)
1066 .fetch_optional(pool)
1067 .await
1068 .map_err(crate::error::Error::Db)?
1069 .flatten();
1070 let landed_at: Option<String> = sqlx::query_scalar(
1071 "SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?",
1072 )
1073 .bind(app)
1074 .bind(tier.as_str())
1075 .fetch_optional(pool)
1076 .await
1077 .map_err(crate::error::Error::Db)?
1078 .flatten();
1079 let fresh = match (confirmed_at, landed_at) {
1080 (Some(c), Some(l)) => {
1081 match (
1082 chrono::DateTime::parse_from_rfc3339(&c),
1083 chrono::DateTime::parse_from_rfc3339(&l),
1084 ) {
1085 (Ok(cd), Ok(ld)) => cd >= ld,
1086 _ => false, // unparseable timestamp -> fail closed
1087 }
1088 }
1089 _ => false,
1090 };
1091 if !fresh {
1092 bad.push(kind.as_str().to_string());
1093 }
1094 }
1095 // Post-deploy tier gates: evidence that THIS tier is healthy running
1096 // what it is running. Keyed on the tier's own build, once, because
1097 // that is the only build a row here can ever exist for — see the note
1098 // on `tier_build` above. `None` is the pre-identity tier, where the
1099 // version-keyed lookup is the legacy behaviour.
1100 Gate::NodeHealth | Gate::PageSmoke => {
1101 let status: Option<String> = match tier_build {
1102 Some(bid) => sqlx::query_scalar(
1103 "SELECT status FROM gate_runs
1104 WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
1105 ORDER BY id DESC LIMIT 1",
1106 )
1107 .bind(app)
1108 .bind(tier.as_str())
1109 .bind(bid)
1110 .bind(kind.as_str()),
1111 None => sqlx::query_scalar(
1112 "SELECT status FROM gate_runs
1113 WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
1114 ORDER BY id DESC LIMIT 1",
1115 )
1116 .bind(app)
1117 .bind(tier.as_str())
1118 .bind(version)
1119 .bind(kind.as_str()),
1120 }
1121 .fetch_optional(pool)
1122 .await
1123 .map_err(crate::error::Error::Db)?
1124 .flatten();
1125 if status.as_deref() != Some("passed") {
1126 bad.push(kind.as_str().to_string());
1127 }
1128 }
1129 // Artifact gates: evidence about the bytes, which travels with them.
1130 // The latest row for this (build, kind) must be `passed`, per build
1131 // this promote ships. Listed explicitly (no `_` catch-all) so adding
1132 // a new `Gate` variant is a compile error here until its promotion
1133 // semantics are decided — a transient-`blocked` kind silently falling
1134 // into "needs a passed row" would be permanently unsatisfiable, and
1135 // a post-deploy kind landing here is the defect the arm above fixes.
1136 Gate::CargoTest
1137 | Gate::HardeningTest
1138 | Gate::Clippy
1139 | Gate::Fmt
1140 | Gate::CargoAudit
1141 | Gate::CargoDeny
1142 | Gate::MigrationDryRun
1143 | Gate::CodeSmoke
1144 | Gate::BootSmoke => {
1145 // Latest row for this configured gate kind; NULL/missing/any
1146 // non-'passed' status all count as unsatisfied (fail closed).
1147 // Every build this promote ships has to show its own passed row.
1148 // An empty `builds` is the caller saying "no identities to key
1149 // on"; one version-keyed lookup is the pre-identity behaviour.
1150 let lookups: &[PromotedBuild] = if builds.is_empty() {
1151 &[PromotedBuild {
1152 platform: None,
1153 build_id: None,
1154 }]
1155 } else {
1156 builds
1157 };
1158 let qualify = lookups.len() > 1;
1159 for b in lookups {
1160 // Keyed on build_id when the artifact has an identity (the
1161 // evidence must be for *this* build), else on the version.
1162 let status: Option<String> = match b.build_id {
1163 Some(bid) => sqlx::query_scalar(
1164 "SELECT status FROM gate_runs
1165 WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
1166 ORDER BY id DESC LIMIT 1",
1167 )
1168 .bind(app)
1169 .bind(tier.as_str())
1170 .bind(bid)
1171 .bind(kind.as_str()),
1172 None => sqlx::query_scalar(
1173 "SELECT status FROM gate_runs
1174 WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
1175 ORDER BY id DESC LIMIT 1",
1176 )
1177 .bind(app)
1178 .bind(tier.as_str())
1179 .bind(version)
1180 .bind(kind.as_str()),
1181 }
1182 .fetch_optional(pool)
1183 .await
1184 .map_err(crate::error::Error::Db)?
1185 .flatten();
1186 if status.as_deref() != Some("passed") {
1187 // Qualified only when there is more than one build to
1188 // tell apart, so a single-platform product's message is
1189 // exactly what it always was.
1190 let name = match (&b.platform, qualify) {
1191 (Some(p), true) => format!("{} ({p})", kind.as_str()),
1192 _ => kind.as_str().to_string(),
1193 };
1194 if !bad.contains(&name) {
1195 bad.push(name);
1196 }
1197 }
1198 }
1199 }
1200 }
1201 }
1202 Ok(bad)
1203 }
1204