max / makenotwork

49.5 KB · 1093 lines History Blame Raw

1	//! Promotion service: the deploy state machine lifted out of the HTTP
2	//! layer — promote orchestration, canary rollback of deployed nodes,
3	//! partial-state flags, and the promote-time gate-satisfaction check.
4	//! routes/mod.rs stays HTTP glue that calls into these.
5
6	use super::{AppState, Json, PromoteBody, Result};
7
8	pub(super) async fn promote_inner(
9	s: AppState,
10	tier: String,
11	body: PromoteBody,
12	) -> Result<Json<serde_json::Value>> {
13	// Serialize the whole check -> deploy -> advance against any concurrent
14	// promote/rollback (CF3). Held for the entire task.
15	let _deploy_guard = s.deploy_lock.lock().await;
16	let tier = crate::domain::TierId::new(tier);
17	let idx = s
18	.topo
19	.tiers
20	.iter()
21	.position(\|t\| t.name == tier)
22	.ok_or(crate::error::Error::NotFound)?;
23	if idx == 0 {
24	return Err(crate::error::Error::GateBlocked(
25	"cannot /promote to the first tier; use /rebuild".into(),
26	));
27	}
28	let target = &s.topo.tiers[idx];
29	let source = &s.topo.tiers[idx - 1];
30	// An unprovisioned tier has no nodes, so every step below is a no-op that
31	// still looks like success: the deploy loop iterates nothing, node_health
32	// returns Blocked, and advance_tier writes a current_version for a tier that
33	// has never received a byte. /state then reports tier c running a version it
34	// does not have. Refuse instead.
35	if !target.provisioned {
36	return Err(crate::error::Error::GateBlocked(format!(
37	"tier {} is not provisioned (no nodes); promoting to it would record a version it never received",
38	target.name,
39	)));
40	}
41
42	// Resolve the artifact through the SOURCE tier's evidence, not a version
43	// string (invariant 2, wiki [[release-artifact-identity]]). The build that is
44	// current on the source tier is the one that burned in and passed its gates
45	// there; promoting anything else lets a version string ride on another build's
46	// evidence and clock — the "promote --version Y inherits X's 48h" hole. A
47	// given `body.version` must name that same build.
48	let source_build: Option<(Option<i64>, Option<String>, Option<String>, Option<String>)> =
49	sqlx::query_as(
50	"SELECT ts.current_build_id, br.version, br.staged_path, br.platform
51	FROM tier_state ts
52	LEFT JOIN build_runs br ON br.id = ts.current_build_id
53	WHERE ts.app = ? AND ts.tier = ?",
54	)
55	.bind(&s.cfg.id)
56	.bind(&source.name)
57	.fetch_optional(&s.pool)
58	.await
59	.map_err(crate::error::Error::Db)?;
60
61	let (build_id, version_str, staged_dir, source_platform) = match source_build {
62	// Identity path: the source tier points at a recorded build.
63	Some((Some(bid), ver, staged_path, platform)) => {
64	let version_str = ver.ok_or_else(\|\| {
65	crate::error::Error::Other(anyhow::anyhow!(
66	"source build {bid} on tier {} has no recorded version",
67	source.name
68	))
69	})?;
70	let staged_path = staged_path.ok_or_else(\|\| {
71	crate::error::Error::Other(anyhow::anyhow!(
72	"source build {bid} ({version_str}) has no staged_path; cannot promote"
73	))
74	})?;
75	if let Some(req) = &body.version
76	&& req != &version_str
77	{
78	return Err(crate::error::Error::GateBlocked(format!(
79	"tier {} is running build {bid} ({version_str}); refusing to promote an \
80	explicit version {req} that is not the build the tier vouched for",
81	source.name
82	)));
83	}
84	let platform = platform
85	.as_deref()
86	.map(crate::domain::Platform::parse)
87	.transpose()
88	.map_err(\|e\| crate::error::Error::Other(anyhow::anyhow!(e)))?;
89	(
90	Some(bid),
91	version_str,
92	std::path::PathBuf::from(staged_path),
93	platform,
94	)
95	}
96	// Legacy fallback: NULL current_build_id (a pre-identity tier). Resolve by
97	// version string through the versions table, exactly as before. One clean
98	// build-and-promote cycle re-anchors the tier onto the identity path.
99	_ => {
100	let version_str = match body.version.clone() {
101	Some(v) => v,
102	None => sqlx::query_scalar::<_, Option<String>>(
103	"SELECT current_version FROM tier_state WHERE app = ? AND tier = ?",
104	)
105	.bind(&s.cfg.id)
106	.bind(&source.name)
107	.fetch_optional(&s.pool)
108	.await
109	.map_err(crate::error::Error::Db)?
110	.flatten()
111	.ok_or_else(\|\| {
112	crate::error::Error::GateBlocked(format!(
113	"no version specified and tier {} has no current_version",
114	source.name
115	))
116	})?,
117	};
118	let bin: Option<(String,)> =
119	sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
120	.bind(&s.cfg.id)
121	.bind(&version_str)
122	.fetch_optional(&s.pool)
123	.await
124	.map_err(crate::error::Error::Db)?;
125	let Some((bin,)) = bin else {
126	return Err(crate::error::Error::NotFound);
127	};
128	// `artifact_path` is the primary binary; the release dir is its parent.
129	let staged_dir = std::path::PathBuf::from(&bin)
130	.parent()
131	.ok_or_else(\|\| {
132	crate::error::Error::Other(anyhow::anyhow!("artifact_path has no parent"))
133	})?
134	.to_path_buf();
135	(None, version_str, staged_dir, None)
136	}
137	};
138	let version = crate::domain::Version::parse(&version_str)
139	.map_err(\|e\| crate::error::Error::Other(anyhow::anyhow!(e)))?;
140
141	// 1. Predecessor must have all of its configured gates satisfied for this
142	// build (with optional hotfix override that skips burn_in). Evaluated
143	// against the topology gate list, so a gate that never ran blocks the
144	// promote instead of being treated as green.
145	//
146	// A migration-bearing promote (`bears_migration`) forces a fresh
147	// `manual_confirm` on the predecessor even if the tier does not configure
148	// one: rollback restores the binary + release_contents only, never the
149	// schema, so the operator must consciously acknowledge the one-way advance
150	// (deploy/README.md "Rollback contract"). `hotfix` skips only `burn_in`, so
151	// it does not suppress this confirm.
152	let mut effective_gates = source.gates.clone();
153	if body.bears_migration
154	&& !effective_gates
155	.iter()
156	.any(\|g\| matches!(g, crate::topology::Gate::ManualConfirm))
157	{
158	effective_gates.push(crate::topology::Gate::ManualConfirm);
159	}
160	// Which bytes each node gets is resolved FIRST, because it is what says
161	// how many builds the gate check has to cover. A two-architecture promote
162	// ships two bundles with two sets of evidence, and checking only the one
163	// the source tier points at would wave the sibling through on gate rows
164	// nobody read. It also means a version missing its other half fails here,
165	// before any gate work, rather than halfway down the rollout.
166	let target_nodes: Vec<&crate::topology::Node> = target.nodes.iter().collect();
167	let bundles = bundles_for_nodes(
168	&s,
169	&version_str,
170	&target_nodes,
171	&staged_dir,
172	source_platform.as_ref(),
173	build_id,
174	)
175	.await?;
176	let mut promoted_builds = distinct_builds(&bundles);
177	if promoted_builds.is_empty() {
178	// A provisioned tier with no nodes ships nothing, so there is nothing to
179	// resolve — but the gate check still has to be keyed on the source build.
180	// Left empty it would fall through to the version-string lookup, which is
181	// the pre-identity path and weaker than what this tier is owed. The
182	// `provisioned` guard above makes this unreachable today; it is written
183	// out because the alternative fails quietly in the direction of less
184	// evidence.
185	promoted_builds.push(PromotedBuild {
186	platform: source_platform.clone(),
187	build_id,
188	});
189	}
190
191	let pending = unsatisfied_gates(
192	&s.pool,
193	&s.cfg.id,
194	&source.name,
195	&effective_gates,
196	&version_str,
197	&promoted_builds,
198	body.hotfix,
199	)
200	.await?;
201	if !pending.is_empty() {
202	return Err(crate::error::Error::GateBlocked(format!(
203	"{} gate(s) not satisfied on tier {}: {}",
204	pending.len(),
205	source.name,
206	pending.join(", "),
207	)));
208	}
209
210	// The version this tier was running before this promote — the rollback
211	// target if a canary node fails partway through a multi-node rollout.
212	let prev_version: Option<String> =
213	sqlx::query_scalar("SELECT current_version FROM tier_state WHERE app = ? AND tier = ?")
214	.bind(&s.cfg.id)
215	.bind(&target.name)
216	.fetch_optional(&s.pool)
217	.await
218	.map_err(crate::error::Error::Db)?
219	.flatten();
220
221	// 3. Deploy to each node. Sequential canary is the only policy
222	// implemented in v0; parallel is a one-line change once we trust the
223	// sequential path. Track the nodes already flipped to the new version so
224	// a mid-rollout failure can roll them back (canary rollback).
225	let mut deployed: Vec<&crate::topology::Node> = Vec::new();
226	// The build id is not read here: it did its work above, keying the gate
227	// check to each bundle's own evidence. What ships is decided by `Placement`.
228	for (node, node_bundle, node_bundle_platform, _) in &bundles {
229	// The proof that these bytes belong on this box. Built before the
230	// deploy row is written, so a mismatch never becomes an `in_progress`
231	// deploy that has to be reconciled.
232	let placement =
233	crate::deploy::Placement::check(node, node_bundle, node_bundle_platform.as_ref())
234	.map_err(\|e\| crate::error::Error::GateBlocked(e.to_string()))?;
235	let started = chrono::Utc::now().to_rfc3339();
236	crate::events::emit(
237	&s.events,
238	crate::events::Event::DeployStart {
239	tier: target.name.clone(),
240	node: node.name.clone(),
241	version: version.clone(),
242	},
243	);
244	let executor = s
245	.executors
246	.get(&node.name)
247	.cloned()
248	.unwrap_or_else(\|\| crate::state::build_executor(node));
249	// Record the deploy as `in_progress` BEFORE touching the node, so a crash
250	// between the node's symlink swap and this promote's `advance_tier` leaves
251	// a durable trace. Without the pre-write, a SIGKILL after the swap but
252	// before the row is inserted would land the new binary on the node with no
253	// DB evidence at all, and the startup reconcile — which reads `deploys` —
254	// could not see it. The row is finalized to ok/failed immediately below;
255	// any row still `in_progress` at startup is an orphan the reconcile settles
256	// and flags ([`crate::reconcile`]).
257	let deploy_id: i64 = sqlx::query_scalar(
258	"INSERT INTO deploys (app, version, tier, node, started_at, outcome, hotfix, reset_burn_in, build_id)
259	VALUES (?, ?, ?, ?, ?, 'in_progress', ?, ?, ?) RETURNING id",
260	)
261	.bind(&s.cfg.id)
262	.bind(&version).bind(&target.name).bind(&node.name)
263	.bind(&started)
264	.bind(body.hotfix as i64).bind(body.reset_burn_in as i64)
265	.bind(build_id)
266	.fetch_one(&s.pool).await.map_err(crate::error::Error::Db)?;
267	let result = crate::deploy::deploy_node(
268	executor.as_ref(),
269	placement,
270	&version_str,
271	s.cfg.primary_bin(),
272	)
273	.await;
274	let finished = chrono::Utc::now().to_rfc3339();
275	let (outcome_obj, err_for_propagation) = match result {
276	Ok(_) => (crate::outcome::DeployOutcome::ok(), None),
277	Err(e) => {
278	let msg = format!("{e:#}");
279	let kind = crate::classify::classify_deploy_error(&msg);
280	(crate::outcome::DeployOutcome::failed(kind), Some(e))
281	}
282	};
283	let outcome_json = serde_json::to_string(&outcome_obj)
284	.unwrap_or_else(\|e\| format!("{{\"_serialize_error\":{e:?}}}"));
285	sqlx::query(
286	"UPDATE deploys SET finished_at = ?, outcome = ?, outcome_json = ? WHERE id = ?",
287	)
288	.bind(&finished)
289	.bind(outcome_obj.status_str())
290	.bind(&outcome_json)
291	.bind(deploy_id)
292	.execute(&s.pool)
293	.await
294	.map_err(crate::error::Error::Db)?;
295	if let Some(e) = err_for_propagation {
296	let crate::outcome::DeployStatus::Failed { failure } = outcome_obj.status else {
297	unreachable!("err_for_propagation is Some iff status is Failed");
298	};
299	tracing::error!(
300	tier = %target.name, node = %node.name, version = %version,
301	failure = failure.summary(),
302	"deploy failed; current symlink left intact, tier_state not advanced"
303	);
304	crate::events::emit(
305	&s.events,
306	crate::events::Event::DeployFailed {
307	tier: target.name.clone(),
308	node: node.name.clone(),
309	version: version.clone(),
310	failure,
311	},
312	);
313
314	// Canary rollback: restore every node this promote touched — the
315	// ones already flipped to the new version AND this failed node
316	// (whose state is indeterminate: the symlink swap may have landed
317	// before the restart failed) — back to the tier's prior version, so
318	// the fleet is left consistent on `prev` rather than split-brain.
319	// Nodes after this one were never touched and stay on `prev`.
320	deployed.push(node);
321	let touched = deployed.len();
322	match prev_version.as_deref() {
323	Some(prev) => {
324	let report = rollback_deployed_nodes(&s, &target.name, &deployed, prev).await;
325	// Say what is true, and say it differently when nothing is
326	// wrong. `restored=0 of=1` read as total failure when the
327	// accurate reading was "0 needed restoring" — the same
328	// confusion as the per-node message, one level up.
329	if report.is_consistent() {
330	tracing::warn!(
331	tier = %target.name,
332	restored = report.restored,
333	already_on_previous = report.already_on_previous,
334	of = report.touched(),
335	from = %version, to = prev,
336	"canary failed mid-rollout; every touched node is on the previous \
337	version and the tier is consistent",
338	);
339	} else {
340	tracing::error!(
341	tier = %target.name,
342	restored = report.restored,
343	already_on_previous = report.already_on_previous,
344	indeterminate = report.indeterminate,
345	of = report.touched(),
346	from = %version, to = prev,
347	"canary failed mid-rollout and the tier is NOT consistent; some nodes \
348	have an indeterminate version",
349	);
350	}
351	if report.restored > 0
352	&& let Ok(prev_v) = crate::domain::Version::parse(prev)
353	{
354	crate::events::emit(
355	&s.events,
356	crate::events::Event::Rollback {
357	tier: target.name.clone(),
358	from: version.clone(),
359	to: prev_v,
360	},
361	);
362	}
363	// The tier is consistent when no node's version is unknown —
364	// which includes the case where a rollback "failed" before
365	// the swap and so left the node on `prev` already. Flagging
366	// that as partial would put a permanent scare on /state for
367	// a fleet that is entirely on one version.
368	if report.is_consistent() {
369	clear_partial(&s, &target.name).await;
370	} else {
371	set_partial(&s, &target.name, &format!(
372	"canary rollback incomplete: {indeterminate} of {touched} node(s) have an \
373	indeterminate version and may be on {version}; {restored} restored to \
374	{prev}, {already} were never swapped — manual check needed",
375	touched = report.touched(),
376	indeterminate = report.indeterminate,
377	restored = report.restored,
378	already = report.already_on_previous,
379	)).await;
380	}
381	}
382	None => {
383	tracing::error!(
384	tier = %target.name, count = touched, version = %version,
385	"canary failed on a first deploy (no previous version to restore to); \
386	touched nodes remain on the new version — manual cleanup needed",
387	);
388	set_partial(
389	&s,
390	&target.name,
391	&format!(
392	"first-deploy canary failed: {touched} node(s) left on {version}, \
393	no prior version to restore — manual cleanup needed",
394	),
395	)
396	.await;
397	}
398	}
399	return Err(crate::error::Error::Other(e));
400	}
401	deployed.push(node);
402	crate::events::emit(
403	&s.events,
404	crate::events::Event::DeployOk {
405	tier: target.name.clone(),
406	node: node.name.clone(),
407	version: version.clone(),
408	},
409	);
410	}
411
412	// 3b. Run this tier's post-deploy gates (node_health) against the freshly
413	// deployed nodes and record their outcomes. These rows are the evidence
414	// the NEXT promote (this tier -> the following one) checks via
415	// `unsatisfied_gates`. Before CF1, only the host tier ran gates, so
416	// A/B/C had no evidence and promotion waved through; node_health now
417	// proves the deployed nodes are serving (Run-2 SERIOUS-3: boot_smoke
418	// used to re-run the staged binary locally and proved nothing about the
419	// node). burn_in / manual_confirm are not run here — they are evaluated
420	// live / by the operator at the next promote. A failed gate does not
421	// unwind this deploy (the artifact is already live on the tier); it
422	// blocks the next promote, which is the fail-closed behavior we want.
423	//
424	// It does, however, fail this promote's response. tier_state still
425	// advances below — the nodes genuinely run this version and a stale
426	// `current_version` would send a later rollback to the wrong artifact —
427	// but the tier is flagged partial and the handler returns the failure,
428	// so the operator learns at the promote instead of discovering it as an
429	// unexplained block on the next one.
430	let post_deploy: Vec<crate::topology::Gate> = target
431	.gates
432	.iter()
433	.filter(\|g\| g.runs_post_deploy())
434	.cloned()
435	.collect();
436	let mut post_deploy_failure: Option<String> = None;
437	if !post_deploy.is_empty() {
438	// node_health probes each node the deploy just shipped to, over the same
439	// executor the deploy used. Build the probe set from the tier's nodes and
440	// the startup executor map; a node missing an executor (shouldn't happen
441	// — both come from the same topology) is skipped, and an empty set makes
442	// node_health Blocked (fail closed).
443	let nodes: Vec<crate::gates::NodeProbe> = target
444	.nodes
445	.iter()
446	.filter_map(\|n\| {
447	s.executors
448	.get(&n.name)
449	.map(\|exec\| crate::gates::NodeProbe {
450	node: n.name.clone(),
451	service: n.service_name.clone(),
452	health_url: n.health_url.clone(),
453	executor: exec.clone(),
454	})
455	})
456	.collect();
457	let ctx = crate::gates::GateCtx {
458	pool: s.pool.clone(),
459	cfg: s.cfg.clone(),
460	tier: target.name.clone(),
461	version: version.clone(),
462	// No worktree at promote time; node_health works over executors, not
463	// a checkout. No bundle either: the gates here are about the nodes,
464	// not about bytes on this host.
465	worktree: None,
466	bundle: None,
467	events: s.events.clone(),
468	nodes,
469	// These post-deploy gate rows are the evidence the NEXT promote (this
470	// tier -> the following one) resolves through, so they must carry the
471	// build they vouch for.
472	build_id,
473	// No checkout at promote time, so nothing to resolve against.
474	aux_dirs: std::collections::HashMap::new(),
475	};
476	post_deploy_failure = match crate::gates::run_all(&ctx, &post_deploy).await {
477	Ok(failed) if failed.is_empty() => None,
478	Ok(failed) => {
479	let names = failed
480	.iter()
481	.map(\|k\| k.as_str())
482	.collect::<Vec<_>>()
483	.join(", ");
484	tracing::warn!(
485	tier = %target.name, version = %version, gates = %names,
486	"post-deploy gate(s) failed; tier advanced but promotion to the next tier is blocked",
487	);
488	Some(format!(
489	"post-deploy gate(s) failed on {version}: {names}; \
490	the tier is serving {version} but cannot promote onward until they pass"
491	))
492	}
493	Err(e) => {
494	tracing::error!(
495	tier = %target.name, version = %version, error = %e,
496	"post-deploy gate execution errored; promotion to the next tier is blocked",
497	);
498	Some(format!(
499	"post-deploy gate execution errored on {version}: {e}; \
500	the tier is serving {version} but cannot promote onward until the gates pass"
501	))
502	}
503	};
504	}
505
506	// 4. Advance tier_state through the single sealed forward-advance op (atomic
507	// self-referential UPDATE; no read-modify-write to lose under concurrency,
508	// CF3). We hold deploy_lock for this whole handler, so the advance is
509	// serialized against rollback and the host build path's advance.
510	// reset_burn_in on the source tier nulls its clock only when the operator
511	// explicitly asked.
512	crate::runs::advance_tier(&s.pool, &s.cfg.id, target.name.as_str(), &version, build_id)
513	.await
514	.map_err(crate::error::Error::Db)?;
515
516	if body.reset_burn_in {
517	sqlx::query("UPDATE tier_state SET burn_in_started_at = NULL WHERE app = ? AND tier = ?")
518	.bind(&s.cfg.id)
519	.bind(&source.name)
520	.execute(&s.pool)
521	.await
522	.map_err(crate::error::Error::Db)?;
523	}
524
525	// Red post-deploy gates: the rollout itself reached every node, but the tier
526	// is not fit to promote onward. Flag it so /state and the TUI say so, and
527	// return the failure rather than a 200 the operator would read as "shipped,
528	// all good". tier_state has already advanced above — it describes what the
529	// nodes are running, not whether we're happy about it.
530	if let Some(reason) = post_deploy_failure {
531	set_partial(&s, &target.name, &reason).await;
532	return Err(crate::error::Error::GateBlocked(reason));
533	}
534
535	// A clean full rollout to every node clears any prior partial flag on this tier.
536	clear_partial(&s, &target.name).await;
537
538	crate::events::emit(
539	&s.events,
540	crate::events::Event::PromoteComplete {
541	tier: target.name.clone(),
542	version: version.clone(),
543	},
544	);
545	tracing::info!(
546	version = %version, tier = %target.name,
547	hotfix = body.hotfix, reset_burn_in = body.reset_burn_in,
548	"promote complete",
549	);
550
551	Ok(Json(serde_json::json!({
552	"tier": target.name,
553	"version": version,
554	"nodes_deployed": target.nodes.iter().map(\|n\| n.name.clone()).collect::<Vec<_>>(),
555	})))
556	}
557
558	/// What a canary rollback actually left behind, per node.
559	///
560	/// Three outcomes, not two, and conflating the middle one with the last is the
561	/// bug this type exists to prevent: a rollback that fails before the symlink
562	/// swap leaves the node exactly where it already was, on the previous version.
563	/// Reporting that as "stranded on the new version, manual intervention needed"
564	/// sends an operator to do surgery on a healthy production box.
565	#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
566	pub(super) struct RollbackReport {
567	/// Put back on the previous version by a successful redeploy.
568	pub(super) restored: usize,
569	/// Rollback failed before the swap, so the node never left the previous
570	/// version. Nothing is stranded and nothing needs doing.
571	pub(super) already_on_previous: usize,
572	/// Rollback failed at or after the swap, or could not be attempted at all.
573	/// The node's version is not knowable from here. This is the only outcome
574	/// that warrants a human.
575	pub(super) indeterminate: usize,
576	}
577
578	impl RollbackReport {
579	/// Every touched node accounted for.
580	pub(super) fn touched(self) -> usize {
581	self.restored + self.already_on_previous + self.indeterminate
582	}
583
584	/// True when no node is in an unknown state, whether or not every rollback
585	/// "succeeded". A tier whose rollbacks all failed before the swap is
586	/// consistent on the previous version and is not an incident.
587	pub(super) fn is_consistent(self) -> bool {
588	self.indeterminate == 0
589	}
590	}
591
592	/// A node, the bundle it is to receive, what that bundle runs on, and the build
593	/// row that bundle came from.
594	///
595	/// The build id is what makes the gate check per-platform: evidence is keyed on
596	/// the build that produced the bytes, so a promote that ships two architectures
597	/// has to look up two sets of gate rows, and this is where it learns which two.
598	/// `None` is the legacy path (a bundle resolved by version string, from a tier
599	/// with no `current_build_id`), where there is no build to key on.
600	pub(super) type NodeBundle<'a> = (
601	&'a crate::topology::Node,
602	std::path::PathBuf,
603	Option<crate::domain::Platform>,
604	Option<i64>,
605	);
606
607	/// One bundle a promote is about to ship, and the evidence key it is judged by.
608	///
609	/// A single-platform product has exactly one of these and it is the source
610	/// tier's own build, which is what every promote before two-architecture support
611	/// checked. A product like pom has one per architecture, each standing on its
612	/// own intake and its own gate run.
613	#[derive(Debug, Clone, PartialEq, Eq)]
614	pub(super) struct PromotedBuild {
615	/// What the bundle runs on, when it says. Used only to qualify the name of a
616	/// failing gate, so an operator reading "cargo_test" knows which half.
617	pub platform: Option<crate::domain::Platform>,
618	/// The `build_runs` row the evidence is keyed on; `None` falls back to the
619	/// version-keyed lookup for a pre-identity tier.
620	pub build_id: Option<i64>,
621	}
622
623	/// Which bytes each node gets, resolved before any node is touched.
624	///
625	/// A single-platform product answers with `fallback` for every node, which is
626	/// the bundle the caller already resolved (the source tier's build on a promote,
627	/// the previous version's on a rollback). A product whose one version is several
628	/// bundles answers per node, because the node is the thing that says which
629	/// architecture it can run.
630	///
631	/// Resolving the whole tier up front is deliberate: a version missing its
632	/// x86_64 half fails the promote before its aarch64 half has been pushed
633	/// anywhere, instead of halfway down a rollout with nodes already flipped.
634	pub(super) async fn bundles_for_nodes<'a>(
635	s: &AppState,
636	version: &str,
637	nodes: &[&'a crate::topology::Node],
638	fallback: &std::path::Path,
639	fallback_platform: Option<&crate::domain::Platform>,
640	fallback_build_id: Option<i64>,
641	) -> Result<Vec<NodeBundle<'a>>> {
642	let mut out = Vec::with_capacity(nodes.len());
643	for node in nodes.iter().copied() {
644	let resolved = match &node.platform {
645	// The node states nothing, so there is nothing to resolve against;
646	// it gets the caller's bundle, and `Placement::check` decides
647	// whether that pairing is admissible at all.
648	None => (
649	fallback.to_path_buf(),
650	fallback_platform.cloned(),
651	fallback_build_id,
652	),
653	// The node states a platform. Give it the bundle recorded for that
654	// platform at this version — the caller's own when they agree, its
655	// sibling when they do not.
656	Some(want) if fallback_platform == Some(want) => (
657	fallback.to_path_buf(),
658	fallback_platform.cloned(),
659	fallback_build_id,
660	),
661	Some(want) => {
662	let (build_id, path) = bundle_for_platform(s, version, want).await?;
663	(path, Some(want.clone()), build_id)
664	}
665	};
666	out.push((node, resolved.0, resolved.1, resolved.2));
667	}
668	Ok(out)
669	}
670
671	/// The distinct builds `bundles` will ship, in a stable order.
672	///
673	/// Deduplicated because a tier is usually several nodes on one architecture, and
674	/// evaluating one build's gates once per node would say the same thing three
675	/// times in the error an operator reads. Keyed on the whole entry rather than on
676	/// the build id alone, so the legacy `None` case does not collapse two
677	/// version-resolved bundles into one.
678	pub(super) fn distinct_builds(bundles: &[NodeBundle<'_>]) -> Vec<PromotedBuild> {
679	let mut out: Vec<PromotedBuild> = Vec::new();
680	for (_, _, platform, build_id) in bundles {
681	let entry = PromotedBuild {
682	platform: platform.clone(),
683	build_id: *build_id,
684	};
685	if !out.contains(&entry) {
686	out.push(entry);
687	}
688	}
689	out
690	}
691
692	/// The bundle recorded for `version` on `platform`.
693	///
694	/// This is what makes a two-architecture product promotable. One pom version is
695	/// two bundles with two digests, each built natively by Bento and accepted
696	/// through its own intake; the tier ladder carries the version forward and this
697	/// answers "which of that version's bundles does this box take".
698	///
699	/// Only a run that settled green qualifies. A sibling that failed its own host
700	/// gates is not a fallback for the one that passed — the source tier's evidence
701	/// says nothing about bytes it never saw, so each architecture's bundle stands
702	/// on its own intake and its own gate run.
703	async fn bundle_for_platform(
704	s: &AppState,
705	version: &str,
706	platform: &crate::domain::Platform,
707	) -> Result<(Option<i64>, std::path::PathBuf)> {
708	let row: Option<(i64, Option<String>)> = sqlx::query_as(
709	"SELECT id, staged_path FROM build_runs
710	WHERE app = ? AND version = ? AND platform = ? AND result = 'passed'
711	ORDER BY id DESC LIMIT 1",
712	)
713	.bind(&s.cfg.id)
714	.bind(version)
715	.bind(platform.to_string())
716	.fetch_optional(&s.pool)
717	.await
718	.map_err(crate::error::Error::Db)?;
719
720	match row {
721	// The id comes back alongside the path so the caller can check this
722	// build's own gate evidence rather than the source tier's. A green build
723	// row is not a green gate run: the build says the bytes compiled, the
724	// gate run says the tier vouched for them.
725	Some((id, Some(path))) => Ok((Some(id), std::path::PathBuf::from(path))),
726	Some((_, None)) => Err(crate::error::Error::Other(anyhow::anyhow!(
727	"the {platform} build of {version} has no staged_path; cannot promote it"
728	))),
729	None => Err(crate::error::Error::GateBlocked(format!(
730	"no green {platform} bundle recorded for {version}. Each architecture is \
731	a separate artifact with its own evidence, so this one has to be built \
732	and accepted before a {platform} node can take this version"
733	))),
734	}
735	}
736
737	/// After a canary node fails mid-promote, restore the nodes already flipped to
738	/// the new version back to `prev_version`, leaving the tier consistent (all on
739	/// the old version) rather than split-brain. Best-effort: every node is
740	/// attempted; a per-node failure is logged but never propagated (the promote is
741	/// already failing).
742	///
743	/// Returns a [`RollbackReport`] rather than a bare count, because "the rollback
744	/// failed" is not the same claim as "the node is on the new version" and the
745	/// caller has to be able to tell them apart. When the previous version has no
746	/// recorded artifact, no rollback can be attempted and every touched node is
747	/// reported indeterminate.
748	pub(super) async fn rollback_deployed_nodes(
749	s: &AppState,
750	tier: &crate::domain::TierId,
751	nodes: &[&crate::topology::Node],
752	prev_version: &str,
753	) -> RollbackReport {
754	let bin: Option<(String,)> =
755	match sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
756	.bind(&s.cfg.id)
757	.bind(prev_version)
758	.fetch_optional(&s.pool)
759	.await
760	{
761	Ok(b) => b,
762	Err(e) => {
763	tracing::error!(tier = %tier, prev = prev_version, error = %e,
764	"canary rollback: looking up the previous artifact failed; no rollback attempted");
765	return RollbackReport {
766	indeterminate: nodes.len(),
767	..Default::default()
768	};
769	}
770	};
771	let Some((bin,)) = bin else {
772	tracing::error!(tier = %tier, prev = prev_version, nodes = nodes.len(),
773	"canary rollback: previous version has no artifact_path; no rollback attempted");
774	return RollbackReport {
775	indeterminate: nodes.len(),
776	..Default::default()
777	};
778	};
779	let Some(staged_dir) = std::path::PathBuf::from(&bin)
780	.parent()
781	.map(std::path::Path::to_path_buf)
782	else {
783	tracing::error!(tier = %tier, prev = prev_version,
784	"canary rollback: previous artifact_path has no parent dir; no rollback attempted");
785	return RollbackReport {
786	indeterminate: nodes.len(),
787	..Default::default()
788	};
789	};
790
791	// A rollback ships bytes too, so it needs the same per-node resolution a
792	// promote does: on a two-architecture product the previous version is also
793	// two bundles, and restoring the wrong one would leave the node no better
794	// off than the failed canary did. A resolution failure here cannot be
795	// propagated (the promote is already failing), so it reports the node as
796	// indeterminate — which is the truth: nothing was attempted and the node's
797	// version is whatever the failed deploy left.
798	// No fallback build id: a rollback resolves the previous version's bundles
799	// by version string and re-runs no gates, so there is no evidence to key.
800	// The bytes are ones this tier already ran.
801	let bundles = match bundles_for_nodes(s, prev_version, nodes, &staged_dir, None, None).await {
802	Ok(b) => b,
803	Err(e) => {
804	tracing::error!(tier = %tier, prev = prev_version, error = %e,
805	"canary rollback: could not resolve a previous bundle per node; no rollback attempted");
806	return RollbackReport {
807	indeterminate: nodes.len(),
808	..Default::default()
809	};
810	}
811	};
812
813	let mut report = RollbackReport::default();
814	for (node, node_bundle, node_bundle_platform, _) in &bundles {
815	let executor = s
816	.executors
817	.get(&node.name)
818	.cloned()
819	.unwrap_or_else(\|\| crate::state::build_executor(node));
820	let placement =
821	match crate::deploy::Placement::check(node, node_bundle, node_bundle_platform.as_ref())
822	{
823	Ok(p) => p,
824	Err(e) => {
825	report.indeterminate += 1;
826	tracing::error!(tier = %tier, node = %node.name, error = %e,
827	"canary rollback: refused to place the previous bundle on this node");
828	continue;
829	}
830	};
831	match crate::deploy::deploy_node(
832	executor.as_ref(),
833	placement,
834	prev_version,
835	s.cfg.primary_bin(),
836	)
837	.await
838	{
839	Ok(_) => {
840	report.restored += 1;
841	tracing::warn!(tier = %tier, node = %node.name, version = prev_version,
842	"canary rollback: node restored to the previous version");
843	}
844	// A rollback is itself a deploy, so it fails at a stage too. Failing
845	// before the swap means it never touched `current` — the node is
846	// still on the version it was already running, which is the one we
847	// were rolling back TO. That is the intended end state reached by a
848	// different route, not an incident.
849	Err(e) => match crate::deploy::stage_of(&e) {
850	Some(crate::deploy::FailureStage::BeforeSwap) => {
851	report.already_on_previous += 1;
852	tracing::warn!(
853	tier = %tier, node = %node.name, version = prev_version,
854	error = %format!("{e:#}"),
855	"canary rollback did not run, and did not need to: it failed before the \
856	symlink swap, so the node is already on the previous version",
857	);
858	}
859	// Unannotated errors land here deliberately. Guessing "safe"
860	// would reintroduce the original bug in the worse direction.
861	stage => {
862	report.indeterminate += 1;
863	tracing::error!(
864	tier = %tier, node = %node.name, version = prev_version,
865	error = %format!("{e:#}"),
866	stage = ?stage,
867	"canary rollback FAILED for node at or after the symlink swap; its version \
868	is indeterminate — manual intervention needed",
869	);
870	}
871	},
872	}
873	}
874	report
875	}
876
877	/// Flag a tier as left in a partial / mixed-version state, with a human-readable
878	/// reason surfaced through `/state` and the TUI. Best-effort: a failure to record
879	/// the flag is logged, never propagated — the caller is already on an error path
880	/// and the worse outcome is to mask the original failure with a bookkeeping one.
881	pub(super) async fn set_partial(s: &AppState, tier: &crate::domain::TierId, reason: &str) {
882	if let Err(e) =
883	sqlx::query("UPDATE tier_state SET partial_reason = ? WHERE app = ? AND tier = ?")
884	.bind(reason)
885	.bind(&s.cfg.id)
886	.bind(tier)
887	.execute(&s.pool)
888	.await
889	{
890	tracing::error!(tier = %tier, reason, error = %e,
891	"failed to record tier partial state; the fleet may be inconsistent without a /state flag");
892	}
893	}
894
895	/// Clear a tier's partial flag after a clean full promote or rollback. Errors are
896	/// logged but not propagated: the deploy itself succeeded, and a stale flag is a
897	/// visible nuisance, not a safety regression (the operator sees a partial marker
898	/// on a tier that is actually fine, and re-checks).
899	pub(super) async fn clear_partial(s: &AppState, tier: &crate::domain::TierId) {
900	if let Err(e) =
901	sqlx::query("UPDATE tier_state SET partial_reason = NULL WHERE app = ? AND tier = ?")
902	.bind(&s.cfg.id)
903	.bind(tier)
904	.execute(&s.pool)
905	.await
906	{
907	tracing::warn!(tier = %tier, error = %e, "failed to clear tier partial flag");
908	}
909	}
910
911	/// Returns the kinds of `tier`'s configured gates that are not satisfied for
912	/// `version`. `hotfix` suppresses the `burn_in` requirement only.
913	///
914	/// Fail-closed against the topology gate list (the CF1 fix). The previous
915	/// version inspected only existing `gate_runs` rows, so a configured gate that
916	/// had never run produced no row and was invisibly treated as green — letting
917	/// a promote wave through with zero evidence (it shipped 0.9.5 to prod with
918	/// tier A's `boot_smoke` never recorded). Now every configured gate must show
919	/// positive evidence:
920	/// - `burn_in` is evaluated live against the tier's clock (a stored `blocked`
921	/// row would otherwise never flip to passed as time elapses);
922	/// - every other kind requires a `passed` row for (tier, version) — a missing
923	/// or non-passed latest row counts as unsatisfied.
924	///
925	/// `builds` are the artifacts this promote will actually ship (wiki note
926	/// `release-artifact-identity`). Each carries a `build_id`, and the deterministic
927	/// build-evidence gates are checked against the rows that vouched for *that
928	/// build* rather than against any row that happens to carry the version string —
929	/// this is what stops a `promote --version Y` from riding on gate rows a
930	/// different build left under the same version. A `None` build id is the
931	/// legacy/pre-identity path: fall back to the version-keyed lookup so a
932	/// mid-migration tier (NULL `current_build_id`) still promotes.
933	///
934	/// One promote can ship several builds, and every one of them is checked.
935	/// A pom version is two bundles with two digests, each built natively and
936	/// accepted through its own intake, so the source tier's evidence for one
937	/// architecture says nothing about the other. Checking only the build the tier
938	/// points at would let an x86_64 node take bytes whose `cargo_test` row nobody
939	/// looked at. When there is more than one, a failing gate is reported qualified
940	/// by platform, because "cargo_test not satisfied" is not actionable if the
941	/// operator cannot tell which half it is about.
942	///
943	/// `burn_in` and `manual_confirm` are evaluated once regardless: both are keyed
944	/// on the tier's own clock (`tier_state.burn_in_started_at`), not on a build, so
945	/// asking per build would ask the same question N times and answer it N times in
946	/// the error.
947	pub(super) async fn unsatisfied_gates(
948	pool: &sqlx::SqlitePool,
949	app: &crate::domain::AppId,
950	tier: &crate::domain::TierId,
951	gates: &[crate::topology::Gate],
952	version: &str,
953	builds: &[PromotedBuild],
954	hotfix: bool,
955	) -> std::result::Result<Vec<String>, crate::error::Error> {
956	use crate::topology::Gate;
957	let mut bad = Vec::new();
958	for gate in gates {
959	let kind = gate.kind();
960	match gate {
961	Gate::BurnIn { hours } => {
962	if hotfix {
963	continue;
964	}
965	let ok = crate::gates::burn_in_satisfied(pool, app, tier, *hours)
966	.await
967	.map_err(crate::error::Error::Other)?;
968	if !ok {
969	bad.push(kind.as_str().to_string());
970	}
971	}
972	Gate::ManualConfirm => {
973	// A confirmation must be fresh: recorded at or after the
974	// version's current landing on this tier (tier_state
975	// .burn_in_started_at, the per-deploy clock). Without this a
976	// confirmation row survives a rollback + rollback-forward and
977	// waves a re-deploy of the same version through with no fresh
978	// operator sign-off — weaker than burn_in, which is clock-based.
979	// No baseline (NULL) => fail closed: require a fresh confirm.
980	let confirmed_at: Option<String> = sqlx::query_scalar(
981	"SELECT finished_at FROM gate_runs
982	WHERE app = ?1 AND tier = ?2 AND version = ?3
983	AND gate_kind = 'manual_confirm' AND status = 'passed'
984	ORDER BY id DESC LIMIT 1",
985	)
986	.bind(app)
987	.bind(tier.as_str())
988	.bind(version)
989	.fetch_optional(pool)
990	.await
991	.map_err(crate::error::Error::Db)?
992	.flatten();
993	let landed_at: Option<String> = sqlx::query_scalar(
994	"SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?",
995	)
996	.bind(app)
997	.bind(tier.as_str())
998	.fetch_optional(pool)
999	.await
1000	.map_err(crate::error::Error::Db)?
1001	.flatten();
1002	let fresh = match (confirmed_at, landed_at) {
1003	(Some(c), Some(l)) => {
1004	match (
1005	chrono::DateTime::parse_from_rfc3339(&c),
1006	chrono::DateTime::parse_from_rfc3339(&l),
1007	) {
1008	(Ok(cd), Ok(ld)) => cd >= ld,
1009	_ => false, // unparseable timestamp -> fail closed
1010	}
1011	}
1012	_ => false,
1013	};
1014	if !fresh {
1015	bad.push(kind.as_str().to_string());
1016	}
1017	}
1018	// Build/post-deploy gates that leave a `gate_runs` row: the latest
1019	// row for this (tier, version, kind) must be `passed`. Listed
1020	// explicitly (no `_` catch-all) so adding a new `Gate` variant is a
1021	// compile error here until its promotion semantics are decided —
1022	// a transient-`blocked` kind silently falling into "needs a passed
1023	// row" would be permanently unsatisfiable.
1024	Gate::CargoTest
1025	\| Gate::HardeningTest
1026	\| Gate::Clippy
1027	\| Gate::Fmt
1028	\| Gate::CargoAudit
1029	\| Gate::CargoDeny
1030	\| Gate::MigrationDryRun
1031	\| Gate::CodeSmoke
1032	\| Gate::BootSmoke
1033	\| Gate::NodeHealth => {
1034	// Latest row for this configured gate kind; NULL/missing/any
1035	// non-'passed' status all count as unsatisfied (fail closed).
1036	// Every build this promote ships has to show its own passed row.
1037	// An empty `builds` is the caller saying "no identities to key
1038	// on"; one version-keyed lookup is the pre-identity behaviour.
1039	let lookups: &[PromotedBuild] = if builds.is_empty() {
1040	&[PromotedBuild {
1041	platform: None,
1042	build_id: None,
1043	}]
1044	} else {
1045	builds
1046	};
1047	let qualify = lookups.len() > 1;
1048	for b in lookups {
1049	// Keyed on build_id when the artifact has an identity (the
1050	// evidence must be for this build), else on the version.
1051	let status: Option<String> = match b.build_id {
1052	Some(bid) => sqlx::query_scalar(
1053	"SELECT status FROM gate_runs
1054	WHERE app = ?1 AND tier = ?2 AND build_id = ?3 AND gate_kind = ?4
1055	ORDER BY id DESC LIMIT 1",
1056	)
1057	.bind(app)
1058	.bind(tier.as_str())
1059	.bind(bid)
1060	.bind(kind.as_str()),
1061	None => sqlx::query_scalar(
1062	"SELECT status FROM gate_runs
1063	WHERE app = ?1 AND tier = ?2 AND version = ?3 AND gate_kind = ?4
1064	ORDER BY id DESC LIMIT 1",
1065	)
1066	.bind(app)
1067	.bind(tier.as_str())
1068	.bind(version)
1069	.bind(kind.as_str()),
1070	}
1071	.fetch_optional(pool)
1072	.await
1073	.map_err(crate::error::Error::Db)?
1074	.flatten();
1075	if status.as_deref() != Some("passed") {
1076	// Qualified only when there is more than one build to
1077	// tell apart, so a single-platform product's message is
1078	// exactly what it always was.
1079	let name = match (&b.platform, qualify) {
1080	(Some(p), true) => format!("{} ({p})", kind.as_str()),
1081	_ => kind.as_str().to_string(),
1082	};
1083	if !bad.contains(&name) {
1084	bad.push(name);
1085	}
1086	}
1087	}
1088	}
1089	}
1090	}
1091	Ok(bad)
1092	}
1093