max / makenotwork

31.4 KB · 715 lines History Blame Raw

1	//! Promotion service: the deploy state machine lifted out of the HTTP
2	//! layer — promote orchestration, canary rollback of deployed nodes,
3	//! partial-state flags, and the promote-time gate-satisfaction check.
4	//! routes/mod.rs stays HTTP glue that calls into these.
5
6	use super::{AppState, Json, PromoteBody, Result};
7
8	pub(super) async fn promote_inner(
9	s: AppState,
10	tier: String,
11	body: PromoteBody,
12	) -> Result<Json<serde_json::Value>> {
13	// Serialize the whole check -> deploy -> advance against any concurrent
14	// promote/rollback (CF3). Held for the entire task.
15	let _deploy_guard = s.deploy_lock.lock().await;
16	let tier = crate::domain::TierId::new(tier);
17	let idx = s
18	.topo
19	.tiers
20	.iter()
21	.position(\|t\| t.name == tier)
22	.ok_or(crate::error::Error::NotFound)?;
23	if idx == 0 {
24	return Err(crate::error::Error::GateBlocked(
25	"cannot /promote to the first tier; use /rebuild".into(),
26	));
27	}
28	let target = &s.topo.tiers[idx];
29	let source = &s.topo.tiers[idx - 1];
30	// An unprovisioned tier has no nodes, so every step below is a no-op that
31	// still looks like success: the deploy loop iterates nothing, node_health
32	// returns Blocked, and advance_tier writes a current_version for a tier that
33	// has never received a byte. /state then reports tier c running a version it
34	// does not have. Refuse instead.
35	if !target.provisioned {
36	return Err(crate::error::Error::GateBlocked(format!(
37	"tier {} is not provisioned (no nodes); promoting to it would record a version it never received",
38	target.name,
39	)));
40	}
41
42	// Resolve the artifact through the SOURCE tier's evidence, not a version
43	// string (invariant 2, wiki [[release-artifact-identity]]). The build that is
44	// current on the source tier is the one that burned in and passed its gates
45	// there; promoting anything else lets a version string ride on another build's
46	// evidence and clock — the "promote --version Y inherits X's 48h" hole. A
47	// given `body.version` must name that same build.
48	let source_build: Option<(Option<i64>, Option<String>, Option<String>)> = sqlx::query_as(
49	"SELECT ts.current_build_id, br.version, br.staged_path
50	FROM tier_state ts
51	LEFT JOIN build_runs br ON br.id = ts.current_build_id
52	WHERE ts.tier = ?",
53	)
54	.bind(&source.name)
55	.fetch_optional(&s.pool)
56	.await
57	.map_err(crate::error::Error::Db)?;
58
59	let (build_id, version_str, staged_dir) = match source_build {
60	// Identity path: the source tier points at a recorded build.
61	Some((Some(bid), ver, staged_path)) => {
62	let version_str = ver.ok_or_else(\|\| {
63	crate::error::Error::Other(anyhow::anyhow!(
64	"source build {bid} on tier {} has no recorded version",
65	source.name
66	))
67	})?;
68	let staged_path = staged_path.ok_or_else(\|\| {
69	crate::error::Error::Other(anyhow::anyhow!(
70	"source build {bid} ({version_str}) has no staged_path; cannot promote"
71	))
72	})?;
73	if let Some(req) = &body.version
74	&& req != &version_str
75	{
76	return Err(crate::error::Error::GateBlocked(format!(
77	"tier {} is running build {bid} ({version_str}); refusing to promote an \
78	explicit version {req} that is not the build the tier vouched for",
79	source.name
80	)));
81	}
82	(
83	Some(bid),
84	version_str,
85	std::path::PathBuf::from(staged_path),
86	)
87	}
88	// Legacy fallback: NULL current_build_id (a pre-identity tier). Resolve by
89	// version string through the versions table, exactly as before. One clean
90	// build-and-promote cycle re-anchors the tier onto the identity path.
91	_ => {
92	let version_str = match body.version.clone() {
93	Some(v) => v,
94	None => sqlx::query_scalar::<_, Option<String>>(
95	"SELECT current_version FROM tier_state WHERE tier = ?",
96	)
97	.bind(&source.name)
98	.fetch_optional(&s.pool)
99	.await
100	.map_err(crate::error::Error::Db)?
101	.flatten()
102	.ok_or_else(\|\| {
103	crate::error::Error::GateBlocked(format!(
104	"no version specified and tier {} has no current_version",
105	source.name
106	))
107	})?,
108	};
109	let bin: Option<(String,)> =
110	sqlx::query_as("SELECT artifact_path FROM versions WHERE version = ?")
111	.bind(&version_str)
112	.fetch_optional(&s.pool)
113	.await
114	.map_err(crate::error::Error::Db)?;
115	let Some((bin,)) = bin else {
116	return Err(crate::error::Error::NotFound);
117	};
118	// `artifact_path` is the primary binary; the release dir is its parent.
119	let staged_dir = std::path::PathBuf::from(&bin)
120	.parent()
121	.ok_or_else(\|\| {
122	crate::error::Error::Other(anyhow::anyhow!("artifact_path has no parent"))
123	})?
124	.to_path_buf();
125	(None, version_str, staged_dir)
126	}
127	};
128	let version = crate::domain::Version::parse(&version_str)
129	.map_err(\|e\| crate::error::Error::Other(anyhow::anyhow!(e)))?;
130
131	// 1. Predecessor must have all of its configured gates satisfied for this
132	// build (with optional hotfix override that skips burn_in). Evaluated
133	// against the topology gate list, so a gate that never ran blocks the
134	// promote instead of being treated as green.
135	//
136	// A migration-bearing promote (`bears_migration`) forces a fresh
137	// `manual_confirm` on the predecessor even if the tier does not configure
138	// one: rollback restores the binary + release_contents only, never the
139	// schema, so the operator must consciously acknowledge the one-way advance
140	// (deploy/README.md "Rollback contract"). `hotfix` skips only `burn_in`, so
141	// it does not suppress this confirm.
142	let mut effective_gates = source.gates.clone();
143	if body.bears_migration
144	&& !effective_gates
145	.iter()
146	.any(\|g\| matches!(g, crate::topology::Gate::ManualConfirm))
147	{
148	effective_gates.push(crate::topology::Gate::ManualConfirm);
149	}
150	let pending = unsatisfied_gates(
151	&s.pool,
152	&source.name,
153	&effective_gates,
154	&version_str,
155	build_id,
156	body.hotfix,
157	)
158	.await?;
159	if !pending.is_empty() {
160	return Err(crate::error::Error::GateBlocked(format!(
161	"{} gate(s) not satisfied on tier {}: {}",
162	pending.len(),
163	source.name,
164	pending.join(", "),
165	)));
166	}
167
168	// The version this tier was running before this promote — the rollback
169	// target if a canary node fails partway through a multi-node rollout.
170	let prev_version: Option<String> =
171	sqlx::query_scalar("SELECT current_version FROM tier_state WHERE tier = ?")
172	.bind(&target.name)
173	.fetch_optional(&s.pool)
174	.await
175	.map_err(crate::error::Error::Db)?
176	.flatten();
177
178	// 3. Deploy to each node. Sequential canary is the only policy
179	// implemented in v0; parallel is a one-line change once we trust the
180	// sequential path. Track the nodes already flipped to the new version so
181	// a mid-rollout failure can roll them back (canary rollback).
182	let mut deployed: Vec<&crate::topology::Node> = Vec::new();
183	for node in &target.nodes {
184	let started = chrono::Utc::now().to_rfc3339();
185	crate::events::emit(
186	&s.events,
187	crate::events::Event::DeployStart {
188	tier: target.name.clone(),
189	node: node.name.clone(),
190	version: version.clone(),
191	},
192	);
193	let executor = s
194	.executors
195	.get(&node.name)
196	.cloned()
197	.unwrap_or_else(\|\| crate::state::build_executor(node));
198	// Record the deploy as `in_progress` BEFORE touching the node, so a crash
199	// between the node's symlink swap and this promote's `advance_tier` leaves
200	// a durable trace. Without the pre-write, a SIGKILL after the swap but
201	// before the row is inserted would land the new binary on the node with no
202	// DB evidence at all, and the startup reconcile — which reads `deploys` —
203	// could not see it. The row is finalized to ok/failed immediately below;
204	// any row still `in_progress` at startup is an orphan the reconcile settles
205	// and flags ([`crate::reconcile`]).
206	let deploy_id: i64 = sqlx::query_scalar(
207	"INSERT INTO deploys (version, tier, node, started_at, outcome, hotfix, reset_burn_in, build_id)
208	VALUES (?, ?, ?, ?, 'in_progress', ?, ?, ?) RETURNING id",
209	)
210	.bind(&version).bind(&target.name).bind(&node.name)
211	.bind(&started)
212	.bind(body.hotfix as i64).bind(body.reset_burn_in as i64)
213	.bind(build_id)
214	.fetch_one(&s.pool).await.map_err(crate::error::Error::Db)?;
215	let result = crate::deploy::deploy_node(
216	executor.as_ref(),
217	node,
218	&version_str,
219	&staged_dir,
220	s.cfg.primary_bin(),
221	)
222	.await;
223	let finished = chrono::Utc::now().to_rfc3339();
224	let (outcome_obj, err_for_propagation) = match result {
225	Ok(_) => (crate::outcome::DeployOutcome::ok(), None),
226	Err(e) => {
227	let msg = format!("{e:#}");
228	let kind = crate::classify::classify_deploy_error(&msg);
229	(crate::outcome::DeployOutcome::failed(kind), Some(e))
230	}
231	};
232	let outcome_json = serde_json::to_string(&outcome_obj)
233	.unwrap_or_else(\|e\| format!("{{\"_serialize_error\":{e:?}}}"));
234	sqlx::query(
235	"UPDATE deploys SET finished_at = ?, outcome = ?, outcome_json = ? WHERE id = ?",
236	)
237	.bind(&finished)
238	.bind(outcome_obj.status_str())
239	.bind(&outcome_json)
240	.bind(deploy_id)
241	.execute(&s.pool)
242	.await
243	.map_err(crate::error::Error::Db)?;
244	if let Some(e) = err_for_propagation {
245	let crate::outcome::DeployStatus::Failed { failure } = outcome_obj.status else {
246	unreachable!("err_for_propagation is Some iff status is Failed");
247	};
248	tracing::error!(
249	tier = %target.name, node = %node.name, version = %version,
250	failure = failure.summary(),
251	"deploy failed; current symlink left intact, tier_state not advanced"
252	);
253	crate::events::emit(
254	&s.events,
255	crate::events::Event::DeployFailed {
256	tier: target.name.clone(),
257	node: node.name.clone(),
258	version: version.clone(),
259	failure,
260	},
261	);
262
263	// Canary rollback: restore every node this promote touched — the
264	// ones already flipped to the new version AND this failed node
265	// (whose state is indeterminate: the symlink swap may have landed
266	// before the restart failed) — back to the tier's prior version, so
267	// the fleet is left consistent on `prev` rather than split-brain.
268	// Nodes after this one were never touched and stay on `prev`.
269	deployed.push(node);
270	let touched = deployed.len();
271	match prev_version.as_deref() {
272	Some(prev) => {
273	let restored = rollback_deployed_nodes(&s, &target.name, &deployed, prev).await;
274	tracing::warn!(
275	tier = %target.name, restored, of = touched,
276	from = %version, to = prev,
277	"canary failed mid-rollout; rolled touched nodes back to the previous version",
278	);
279	if restored > 0
280	&& let Ok(prev_v) = crate::domain::Version::parse(prev)
281	{
282	crate::events::emit(
283	&s.events,
284	crate::events::Event::Rollback {
285	tier: target.name.clone(),
286	from: version.clone(),
287	to: prev_v,
288	},
289	);
290	}
291	// If every touched node was restored, the tier is consistent on
292	// `prev` — clear any stale flag. Otherwise it is genuinely
293	// split-brain: record exactly how, for /state.
294	if restored == touched {
295	clear_partial(&s, &target.name).await;
296	} else {
297	set_partial(&s, &target.name, &format!(
298	"canary rollback incomplete: {restored}/{touched} nodes restored to {prev}; \
299	{} may still be on {version} — manual check needed",
300	touched - restored,
301	)).await;
302	}
303	}
304	None => {
305	tracing::error!(
306	tier = %target.name, count = touched, version = %version,
307	"canary failed on a first deploy (no previous version to restore to); \
308	touched nodes remain on the new version — manual cleanup needed",
309	);
310	set_partial(
311	&s,
312	&target.name,
313	&format!(
314	"first-deploy canary failed: {touched} node(s) left on {version}, \
315	no prior version to restore — manual cleanup needed",
316	),
317	)
318	.await;
319	}
320	}
321	return Err(crate::error::Error::Other(e));
322	}
323	deployed.push(node);
324	crate::events::emit(
325	&s.events,
326	crate::events::Event::DeployOk {
327	tier: target.name.clone(),
328	node: node.name.clone(),
329	version: version.clone(),
330	},
331	);
332	}
333
334	// 3b. Run this tier's post-deploy gates (node_health) against the freshly
335	// deployed nodes and record their outcomes. These rows are the evidence
336	// the NEXT promote (this tier -> the following one) checks via
337	// `unsatisfied_gates`. Before CF1, only the host tier ran gates, so
338	// A/B/C had no evidence and promotion waved through; node_health now
339	// proves the deployed nodes are serving (Run-2 SERIOUS-3: boot_smoke
340	// used to re-run the staged binary locally and proved nothing about the
341	// node). burn_in / manual_confirm are not run here — they are evaluated
342	// live / by the operator at the next promote. A failed gate does not
343	// unwind this deploy (the artifact is already live on the tier); it
344	// blocks the next promote, which is the fail-closed behavior we want.
345	//
346	// It does, however, fail this promote's response. tier_state still
347	// advances below — the nodes genuinely run this version and a stale
348	// `current_version` would send a later rollback to the wrong artifact —
349	// but the tier is flagged partial and the handler returns the failure,
350	// so the operator learns at the promote instead of discovering it as an
351	// unexplained block on the next one.
352	let post_deploy: Vec<crate::topology::Gate> = target
353	.gates
354	.iter()
355	.filter(\|g\| g.runs_post_deploy())
356	.cloned()
357	.collect();
358	let mut post_deploy_failure: Option<String> = None;
359	if !post_deploy.is_empty() {
360	// node_health probes each node the deploy just shipped to, over the same
361	// executor the deploy used. Build the probe set from the tier's nodes and
362	// the startup executor map; a node missing an executor (shouldn't happen
363	// — both come from the same topology) is skipped, and an empty set makes
364	// node_health Blocked (fail closed).
365	let nodes: Vec<crate::gates::NodeProbe> = target
366	.nodes
367	.iter()
368	.filter_map(\|n\| {
369	s.executors
370	.get(&n.name)
371	.map(\|exec\| crate::gates::NodeProbe {
372	node: n.name.clone(),
373	service: n.service_name.clone(),
374	health_url: n.health_url.clone(),
375	executor: exec.clone(),
376	})
377	})
378	.collect();
379	let ctx = crate::gates::GateCtx {
380	pool: s.pool.clone(),
381	cfg: s.cfg.clone(),
382	tier: target.name.clone(),
383	version: version.clone(),
384	// No worktree at promote time; node_health works over executors, not
385	// a checkout.
386	worktree: std::path::PathBuf::new(),
387	events: s.events.clone(),
388	nodes,
389	// These post-deploy gate rows are the evidence the NEXT promote (this
390	// tier -> the following one) resolves through, so they must carry the
391	// build they vouch for.
392	build_id,
393	};
394	post_deploy_failure = match crate::gates::run_all(&ctx, &post_deploy).await {
395	Ok(failed) if failed.is_empty() => None,
396	Ok(failed) => {
397	let names = failed
398	.iter()
399	.map(\|k\| k.as_str())
400	.collect::<Vec<_>>()
401	.join(", ");
402	tracing::warn!(
403	tier = %target.name, version = %version, gates = %names,
404	"post-deploy gate(s) failed; tier advanced but promotion to the next tier is blocked",
405	);
406	Some(format!(
407	"post-deploy gate(s) failed on {version}: {names}; \
408	the tier is serving {version} but cannot promote onward until they pass"
409	))
410	}
411	Err(e) => {
412	tracing::error!(
413	tier = %target.name, version = %version, error = %e,
414	"post-deploy gate execution errored; promotion to the next tier is blocked",
415	);
416	Some(format!(
417	"post-deploy gate execution errored on {version}: {e}; \
418	the tier is serving {version} but cannot promote onward until the gates pass"
419	))
420	}
421	};
422	}
423
424	// 4. Advance tier_state through the single sealed forward-advance op (atomic
425	// self-referential UPDATE; no read-modify-write to lose under concurrency,
426	// CF3). We hold deploy_lock for this whole handler, so the advance is
427	// serialized against rollback and the host build path's advance.
428	// reset_burn_in on the source tier nulls its clock only when the operator
429	// explicitly asked.
430	crate::runs::advance_tier(&s.pool, target.name.as_str(), &version, build_id)
431	.await
432	.map_err(crate::error::Error::Db)?;
433
434	if body.reset_burn_in {
435	sqlx::query("UPDATE tier_state SET burn_in_started_at = NULL WHERE tier = ?")
436	.bind(&source.name)
437	.execute(&s.pool)
438	.await
439	.map_err(crate::error::Error::Db)?;
440	}
441
442	// Red post-deploy gates: the rollout itself reached every node, but the tier
443	// is not fit to promote onward. Flag it so /state and the TUI say so, and
444	// return the failure rather than a 200 the operator would read as "shipped,
445	// all good". tier_state has already advanced above — it describes what the
446	// nodes are running, not whether we're happy about it.
447	if let Some(reason) = post_deploy_failure {
448	set_partial(&s, &target.name, &reason).await;
449	return Err(crate::error::Error::GateBlocked(reason));
450	}
451
452	// A clean full rollout to every node clears any prior partial flag on this tier.
453	clear_partial(&s, &target.name).await;
454
455	crate::events::emit(
456	&s.events,
457	crate::events::Event::PromoteComplete {
458	tier: target.name.clone(),
459	version: version.clone(),
460	},
461	);
462	tracing::info!(
463	version = %version, tier = %target.name,
464	hotfix = body.hotfix, reset_burn_in = body.reset_burn_in,
465	"promote complete",
466	);
467
468	Ok(Json(serde_json::json!({
469	"tier": target.name,
470	"version": version,
471	"nodes_deployed": target.nodes.iter().map(\|n\| n.name.clone()).collect::<Vec<_>>(),
472	})))
473	}
474
475	/// After a canary node fails mid-promote, restore the nodes already flipped to
476	/// the new version back to `prev_version`, leaving the tier consistent (all on
477	/// the old version) rather than split-brain. Best-effort: every node is
478	/// attempted; a per-node failure is logged but never propagated (the promote is
479	/// already failing). Returns how many nodes were successfully restored. Returns
480	/// 0 (with an error log) when the previous version has no recorded artifact to
481	/// roll back to.
482	pub(super) async fn rollback_deployed_nodes(
483	s: &AppState,
484	tier: &crate::domain::TierId,
485	nodes: &[&crate::topology::Node],
486	prev_version: &str,
487	) -> usize {
488	let bin: Option<(String,)> = match sqlx::query_as(
489	"SELECT artifact_path FROM versions WHERE version = ?",
490	)
491	.bind(prev_version)
492	.fetch_optional(&s.pool)
493	.await
494	{
495	Ok(b) => b,
496	Err(e) => {
497	tracing::error!(tier = %tier, prev = prev_version, error = %e,
498	"canary rollback: looking up the previous artifact failed; nodes left on the new version");
499	return 0;
500	}
501	};
502	let Some((bin,)) = bin else {
503	tracing::error!(tier = %tier, prev = prev_version, nodes = nodes.len(),
504	"canary rollback: previous version has no artifact_path; nodes left on the new version");
505	return 0;
506	};
507	let Some(staged_dir) = std::path::PathBuf::from(&bin)
508	.parent()
509	.map(std::path::Path::to_path_buf)
510	else {
511	tracing::error!(tier = %tier, prev = prev_version,
512	"canary rollback: previous artifact_path has no parent dir; nodes left on the new version");
513	return 0;
514	};
515
516	let mut restored = 0usize;
517	for node in nodes {
518	let executor = s
519	.executors
520	.get(&node.name)
521	.cloned()
522	.unwrap_or_else(\|\| crate::state::build_executor(node));
523	match crate::deploy::deploy_node(
524	executor.as_ref(),
525	node,
526	prev_version,
527	&staged_dir,
528	s.cfg.primary_bin(),
529	)
530	.await
531	{
532	Ok(_) => {
533	restored += 1;
534	tracing::warn!(tier = %tier, node = %node.name, version = prev_version,
535	"canary rollback: node restored to the previous version");
536	}
537	Err(e) => tracing::error!(
538	tier = %tier, node = %node.name, version = prev_version, error = %format!("{e:#}"),
539	"canary rollback FAILED for node; it remains on the new version — manual intervention needed",
540	),
541	}
542	}
543	restored
544	}
545
546	/// Flag a tier as left in a partial / mixed-version state, with a human-readable
547	/// reason surfaced through `/state` and the TUI. Best-effort: a failure to record
548	/// the flag is logged, never propagated — the caller is already on an error path
549	/// and the worse outcome is to mask the original failure with a bookkeeping one.
550	pub(super) async fn set_partial(s: &AppState, tier: &crate::domain::TierId, reason: &str) {
551	if let Err(e) = sqlx::query("UPDATE tier_state SET partial_reason = ? WHERE tier = ?")
552	.bind(reason)
553	.bind(tier)
554	.execute(&s.pool)
555	.await
556	{
557	tracing::error!(tier = %tier, reason, error = %e,
558	"failed to record tier partial state; the fleet may be inconsistent without a /state flag");
559	}
560	}
561
562	/// Clear a tier's partial flag after a clean full promote or rollback. Errors are
563	/// logged but not propagated: the deploy itself succeeded, and a stale flag is a
564	/// visible nuisance, not a safety regression (the operator sees a partial marker
565	/// on a tier that is actually fine, and re-checks).
566	pub(super) async fn clear_partial(s: &AppState, tier: &crate::domain::TierId) {
567	if let Err(e) = sqlx::query("UPDATE tier_state SET partial_reason = NULL WHERE tier = ?")
568	.bind(tier)
569	.execute(&s.pool)
570	.await
571	{
572	tracing::warn!(tier = %tier, error = %e, "failed to clear tier partial flag");
573	}
574	}
575
576	/// Returns the kinds of `tier`'s configured gates that are not satisfied for
577	/// `version`. `hotfix` suppresses the `burn_in` requirement only.
578	///
579	/// Fail-closed against the topology gate list (the CF1 fix). The previous
580	/// version inspected only existing `gate_runs` rows, so a configured gate that
581	/// had never run produced no row and was invisibly treated as green — letting
582	/// a promote wave through with zero evidence (it shipped 0.9.5 to prod with
583	/// tier A's `boot_smoke` never recorded). Now every configured gate must show
584	/// positive evidence:
585	/// - `burn_in` is evaluated live against the tier's clock (a stored `blocked`
586	/// row would otherwise never flip to passed as time elapses);
587	/// - every other kind requires a `passed` row for (tier, version) — a missing
588	/// or non-passed latest row counts as unsatisfied.
589	///
590	/// `build_id` is the identity of the artifact being promoted (wiki note
591	/// `release-artifact-identity`). When `Some`, the deterministic build-evidence
592	/// gates are checked against the row that vouched for that build rather than
593	/// against any row that happens to carry the version string — this is what stops
594	/// a `promote --version Y` from riding on gate rows a different build left under
595	/// the same version. `None` is the legacy/pre-identity path: fall back to the
596	/// version-keyed lookup so a mid-migration tier (NULL `current_build_id`) still
597	/// promotes. `burn_in` is clock-based either way — the clock is reset on every
598	/// advance, so it belongs to the build now current on the tier.
599	pub(super) async fn unsatisfied_gates(
600	pool: &sqlx::SqlitePool,
601	tier: &crate::domain::TierId,
602	gates: &[crate::topology::Gate],
603	version: &str,
604	build_id: Option<i64>,
605	hotfix: bool,
606	) -> std::result::Result<Vec<String>, crate::error::Error> {
607	use crate::topology::Gate;
608	let mut bad = Vec::new();
609	for gate in gates {
610	let kind = gate.kind();
611	match gate {
612	Gate::BurnIn { hours } => {
613	if hotfix {
614	continue;
615	}
616	let ok = crate::gates::burn_in_satisfied(pool, tier, *hours)
617	.await
618	.map_err(crate::error::Error::Other)?;
619	if !ok {
620	bad.push(kind.as_str().to_string());
621	}
622	}
623	Gate::ManualConfirm => {
624	// A confirmation must be fresh: recorded at or after the
625	// version's current landing on this tier (tier_state
626	// .burn_in_started_at, the per-deploy clock). Without this a
627	// confirmation row survives a rollback + rollback-forward and
628	// waves a re-deploy of the same version through with no fresh
629	// operator sign-off — weaker than burn_in, which is clock-based.
630	// No baseline (NULL) => fail closed: require a fresh confirm.
631	let confirmed_at: Option<String> = sqlx::query_scalar(
632	"SELECT finished_at FROM gate_runs
633	WHERE tier = ?1 AND version = ?2 AND gate_kind = 'manual_confirm' AND status = 'passed'
634	ORDER BY id DESC LIMIT 1",
635	)
636	.bind(tier.as_str())
637	.bind(version)
638	.fetch_optional(pool)
639	.await
640	.map_err(crate::error::Error::Db)?
641	.flatten();
642	let landed_at: Option<String> =
643	sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE tier = ?")
644	.bind(tier.as_str())
645	.fetch_optional(pool)
646	.await
647	.map_err(crate::error::Error::Db)?
648	.flatten();
649	let fresh = match (confirmed_at, landed_at) {
650	(Some(c), Some(l)) => {
651	match (
652	chrono::DateTime::parse_from_rfc3339(&c),
653	chrono::DateTime::parse_from_rfc3339(&l),
654	) {
655	(Ok(cd), Ok(ld)) => cd >= ld,
656	_ => false, // unparseable timestamp -> fail closed
657	}
658	}
659	_ => false,
660	};
661	if !fresh {
662	bad.push(kind.as_str().to_string());
663	}
664	}
665	// Build/post-deploy gates that leave a `gate_runs` row: the latest
666	// row for this (tier, version, kind) must be `passed`. Listed
667	// explicitly (no `_` catch-all) so adding a new `Gate` variant is a
668	// compile error here until its promotion semantics are decided —
669	// a transient-`blocked` kind silently falling into "needs a passed
670	// row" would be permanently unsatisfiable.
671	Gate::CargoTest
672	\| Gate::HardeningTest
673	\| Gate::Clippy
674	\| Gate::Fmt
675	\| Gate::CargoAudit
676	\| Gate::CargoDeny
677	\| Gate::MigrationDryRun
678	\| Gate::CodeSmoke
679	\| Gate::BootSmoke
680	\| Gate::NodeHealth => {
681	// Latest row for this configured gate kind; NULL/missing/any
682	// non-'passed' status all count as unsatisfied (fail closed).
683	// Keyed on build_id when the artifact has an identity (the
684	// evidence must be for this build), else on the version string.
685	let status: Option<String> = match build_id {
686	Some(bid) => sqlx::query_scalar(
687	"SELECT status FROM gate_runs
688	WHERE tier = ?1 AND build_id = ?2 AND gate_kind = ?3
689	ORDER BY id DESC LIMIT 1",
690	)
691	.bind(tier.as_str())
692	.bind(bid)
693	.bind(kind.as_str()),
694	None => sqlx::query_scalar(
695	"SELECT status FROM gate_runs
696	WHERE tier = ?1 AND version = ?2 AND gate_kind = ?3
697	ORDER BY id DESC LIMIT 1",
698	)
699	.bind(tier.as_str())
700	.bind(version)
701	.bind(kind.as_str()),
702	}
703	.fetch_optional(pool)
704	.await
705	.map_err(crate::error::Error::Db)?
706	.flatten();
707	if status.as_deref() != Some("passed") {
708	bad.push(kind.as_str().to_string());
709	}
710	}
711	}
712	}
713	Ok(bad)
714	}
715