//! Promotion service: the deploy state machine lifted out of the HTTP //! layer — promote orchestration, canary rollback of deployed nodes, //! partial-state flags, and the promote-time gate-satisfaction check. //! routes/mod.rs stays HTTP glue that calls into these. use super::{AppState, Json, PromoteBody, Result}; pub(super) async fn promote_inner( s: AppState, tier: String, body: PromoteBody, ) -> Result> { // Serialize the whole check -> deploy -> advance against any concurrent // promote/rollback (CF3). Held for the entire task. let _deploy_guard = s.deploy_lock.lock().await; let tier = crate::domain::TierId::new(tier); let idx = s .topo .tiers .iter() .position(|t| t.name == tier) .ok_or(crate::error::Error::NotFound)?; if idx == 0 { return Err(crate::error::Error::GateBlocked( "cannot /promote to the first tier; use /rebuild".into(), )); } let target = &s.topo.tiers[idx]; let source = &s.topo.tiers[idx - 1]; // An unprovisioned tier has no nodes, so every step below is a no-op that // still *looks* like success: the deploy loop iterates nothing, node_health // returns Blocked, and advance_tier writes a current_version for a tier that // has never received a byte. /state then reports tier c running a version it // does not have. Refuse instead. if !target.provisioned { return Err(crate::error::Error::GateBlocked(format!( "tier {} is not provisioned (no nodes); promoting to it would record a version it never received", target.name, ))); } // Resolve the artifact through the SOURCE tier's evidence, not a version // string (invariant 2, wiki [[release-artifact-identity]]). The build that is // *current on the source tier* is the one that burned in and passed its gates // there; promoting anything else lets a version string ride on another build's // evidence and clock — the "promote --version Y inherits X's 48h" hole. A // given `body.version` must name that same build. let source_build: Option<(Option, Option, Option)> = sqlx::query_as( "SELECT ts.current_build_id, br.version, br.staged_path FROM tier_state ts LEFT JOIN build_runs br ON br.id = ts.current_build_id WHERE ts.tier = ?", ) .bind(&source.name) .fetch_optional(&s.pool) .await .map_err(crate::error::Error::Db)?; let (build_id, version_str, staged_dir) = match source_build { // Identity path: the source tier points at a recorded build. Some((Some(bid), ver, staged_path)) => { let version_str = ver.ok_or_else(|| { crate::error::Error::Other(anyhow::anyhow!( "source build {bid} on tier {} has no recorded version", source.name )) })?; let staged_path = staged_path.ok_or_else(|| { crate::error::Error::Other(anyhow::anyhow!( "source build {bid} ({version_str}) has no staged_path; cannot promote" )) })?; if let Some(req) = &body.version && req != &version_str { return Err(crate::error::Error::GateBlocked(format!( "tier {} is running build {bid} ({version_str}); refusing to promote an \ explicit version {req} that is not the build the tier vouched for", source.name ))); } ( Some(bid), version_str, std::path::PathBuf::from(staged_path), ) } // Legacy fallback: NULL current_build_id (a pre-identity tier). Resolve by // version string through the versions table, exactly as before. One clean // build-and-promote cycle re-anchors the tier onto the identity path. _ => { let version_str = match body.version.clone() { Some(v) => v, None => sqlx::query_scalar::<_, Option>( "SELECT current_version FROM tier_state WHERE tier = ?", ) .bind(&source.name) .fetch_optional(&s.pool) .await .map_err(crate::error::Error::Db)? .flatten() .ok_or_else(|| { crate::error::Error::GateBlocked(format!( "no version specified and tier {} has no current_version", source.name )) })?, }; let bin: Option<(String,)> = sqlx::query_as("SELECT artifact_path FROM versions WHERE version = ?") .bind(&version_str) .fetch_optional(&s.pool) .await .map_err(crate::error::Error::Db)?; let Some((bin,)) = bin else { return Err(crate::error::Error::NotFound); }; // `artifact_path` is the primary binary; the release dir is its parent. let staged_dir = std::path::PathBuf::from(&bin) .parent() .ok_or_else(|| { crate::error::Error::Other(anyhow::anyhow!("artifact_path has no parent")) })? .to_path_buf(); (None, version_str, staged_dir) } }; let version = crate::domain::Version::parse(&version_str) .map_err(|e| crate::error::Error::Other(anyhow::anyhow!(e)))?; // 1. Predecessor must have all of its configured gates satisfied for this // build (with optional hotfix override that skips burn_in). Evaluated // against the topology gate list, so a gate that never ran blocks the // promote instead of being treated as green. // // A migration-bearing promote (`bears_migration`) forces a fresh // `manual_confirm` on the predecessor even if the tier does not configure // one: rollback restores the binary + release_contents only, never the // schema, so the operator must consciously acknowledge the one-way advance // (deploy/README.md "Rollback contract"). `hotfix` skips only `burn_in`, so // it does not suppress this confirm. let mut effective_gates = source.gates.clone(); if body.bears_migration && !effective_gates .iter() .any(|g| matches!(g, crate::topology::Gate::ManualConfirm)) { effective_gates.push(crate::topology::Gate::ManualConfirm); } let pending = unsatisfied_gates( &s.pool, &source.name, &effective_gates, &version_str, build_id, body.hotfix, ) .await?; if !pending.is_empty() { return Err(crate::error::Error::GateBlocked(format!( "{} gate(s) not satisfied on tier {}: {}", pending.len(), source.name, pending.join(", "), ))); } // The version this tier was running before this promote — the rollback // target if a canary node fails partway through a multi-node rollout. let prev_version: Option = sqlx::query_scalar("SELECT current_version FROM tier_state WHERE tier = ?") .bind(&target.name) .fetch_optional(&s.pool) .await .map_err(crate::error::Error::Db)? .flatten(); // 3. Deploy to each node. Sequential canary is the only policy // implemented in v0; parallel is a one-line change once we trust the // sequential path. Track the nodes already flipped to the new version so // a mid-rollout failure can roll them back (canary rollback). let mut deployed: Vec<&crate::topology::Node> = Vec::new(); for node in &target.nodes { let started = chrono::Utc::now().to_rfc3339(); crate::events::emit( &s.events, crate::events::Event::DeployStart { tier: target.name.clone(), node: node.name.clone(), version: version.clone(), }, ); let executor = s .executors .get(&node.name) .cloned() .unwrap_or_else(|| crate::state::build_executor(node)); // Record the deploy as `in_progress` BEFORE touching the node, so a crash // between the node's symlink swap and this promote's `advance_tier` leaves // a durable trace. Without the pre-write, a SIGKILL after the swap but // before the row is inserted would land the new binary on the node with no // DB evidence at all, and the startup reconcile — which reads `deploys` — // could not see it. The row is finalized to ok/failed immediately below; // any row still `in_progress` at startup is an orphan the reconcile settles // and flags ([`crate::reconcile`]). let deploy_id: i64 = sqlx::query_scalar( "INSERT INTO deploys (version, tier, node, started_at, outcome, hotfix, reset_burn_in, build_id) VALUES (?, ?, ?, ?, 'in_progress', ?, ?, ?) RETURNING id", ) .bind(&version).bind(&target.name).bind(&node.name) .bind(&started) .bind(body.hotfix as i64).bind(body.reset_burn_in as i64) .bind(build_id) .fetch_one(&s.pool).await.map_err(crate::error::Error::Db)?; let result = crate::deploy::deploy_node( executor.as_ref(), node, &version_str, &staged_dir, s.cfg.primary_bin(), ) .await; let finished = chrono::Utc::now().to_rfc3339(); let (outcome_obj, err_for_propagation) = match result { Ok(_) => (crate::outcome::DeployOutcome::ok(), None), Err(e) => { let msg = format!("{e:#}"); let kind = crate::classify::classify_deploy_error(&msg); (crate::outcome::DeployOutcome::failed(kind), Some(e)) } }; let outcome_json = serde_json::to_string(&outcome_obj) .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}")); sqlx::query( "UPDATE deploys SET finished_at = ?, outcome = ?, outcome_json = ? WHERE id = ?", ) .bind(&finished) .bind(outcome_obj.status_str()) .bind(&outcome_json) .bind(deploy_id) .execute(&s.pool) .await .map_err(crate::error::Error::Db)?; if let Some(e) = err_for_propagation { let crate::outcome::DeployStatus::Failed { failure } = outcome_obj.status else { unreachable!("err_for_propagation is Some iff status is Failed"); }; tracing::error!( tier = %target.name, node = %node.name, version = %version, failure = failure.summary(), "deploy failed; current symlink left intact, tier_state not advanced" ); crate::events::emit( &s.events, crate::events::Event::DeployFailed { tier: target.name.clone(), node: node.name.clone(), version: version.clone(), failure, }, ); // Canary rollback: restore every node this promote touched — the // ones already flipped to the new version AND this failed node // (whose state is indeterminate: the symlink swap may have landed // before the restart failed) — back to the tier's prior version, so // the fleet is left consistent on `prev` rather than split-brain. // Nodes after this one were never touched and stay on `prev`. deployed.push(node); let touched = deployed.len(); match prev_version.as_deref() { Some(prev) => { let restored = rollback_deployed_nodes(&s, &target.name, &deployed, prev).await; tracing::warn!( tier = %target.name, restored, of = touched, from = %version, to = prev, "canary failed mid-rollout; rolled touched nodes back to the previous version", ); if restored > 0 && let Ok(prev_v) = crate::domain::Version::parse(prev) { crate::events::emit( &s.events, crate::events::Event::Rollback { tier: target.name.clone(), from: version.clone(), to: prev_v, }, ); } // If every touched node was restored, the tier is consistent on // `prev` — clear any stale flag. Otherwise it is genuinely // split-brain: record exactly how, for /state. if restored == touched { clear_partial(&s, &target.name).await; } else { set_partial(&s, &target.name, &format!( "canary rollback incomplete: {restored}/{touched} nodes restored to {prev}; \ {} may still be on {version} — manual check needed", touched - restored, )).await; } } None => { tracing::error!( tier = %target.name, count = touched, version = %version, "canary failed on a first deploy (no previous version to restore to); \ touched nodes remain on the new version — manual cleanup needed", ); set_partial( &s, &target.name, &format!( "first-deploy canary failed: {touched} node(s) left on {version}, \ no prior version to restore — manual cleanup needed", ), ) .await; } } return Err(crate::error::Error::Other(e)); } deployed.push(node); crate::events::emit( &s.events, crate::events::Event::DeployOk { tier: target.name.clone(), node: node.name.clone(), version: version.clone(), }, ); } // 3b. Run this tier's post-deploy gates (node_health) against the freshly // deployed nodes and record their outcomes. These rows are the evidence // the NEXT promote (this tier -> the following one) checks via // `unsatisfied_gates`. Before CF1, only the host tier ran gates, so // A/B/C had no evidence and promotion waved through; node_health now // proves the deployed nodes are serving (Run-2 SERIOUS-3: boot_smoke // used to re-run the staged binary locally and proved nothing about the // node). burn_in / manual_confirm are not run here — they are evaluated // live / by the operator at the next promote. A failed gate does not // unwind this deploy (the artifact is already live on the tier); it // blocks the next promote, which is the fail-closed behavior we want. // // It does, however, fail *this* promote's response. tier_state still // advances below — the nodes genuinely run this version and a stale // `current_version` would send a later rollback to the wrong artifact — // but the tier is flagged partial and the handler returns the failure, // so the operator learns at the promote instead of discovering it as an // unexplained block on the next one. let post_deploy: Vec = target .gates .iter() .filter(|g| g.runs_post_deploy()) .cloned() .collect(); let mut post_deploy_failure: Option = None; if !post_deploy.is_empty() { // node_health probes each node the deploy just shipped to, over the same // executor the deploy used. Build the probe set from the tier's nodes and // the startup executor map; a node missing an executor (shouldn't happen // — both come from the same topology) is skipped, and an empty set makes // node_health Blocked (fail closed). let nodes: Vec = target .nodes .iter() .filter_map(|n| { s.executors .get(&n.name) .map(|exec| crate::gates::NodeProbe { node: n.name.clone(), service: n.service_name.clone(), health_url: n.health_url.clone(), executor: exec.clone(), }) }) .collect(); let ctx = crate::gates::GateCtx { pool: s.pool.clone(), cfg: s.cfg.clone(), tier: target.name.clone(), version: version.clone(), // No worktree at promote time; node_health works over executors, not // a checkout. worktree: std::path::PathBuf::new(), events: s.events.clone(), nodes, // These post-deploy gate rows are the evidence the NEXT promote (this // tier -> the following one) resolves through, so they must carry the // build they vouch for. build_id, }; post_deploy_failure = match crate::gates::run_all(&ctx, &post_deploy).await { Ok(failed) if failed.is_empty() => None, Ok(failed) => { let names = failed .iter() .map(|k| k.as_str()) .collect::>() .join(", "); tracing::warn!( tier = %target.name, version = %version, gates = %names, "post-deploy gate(s) failed; tier advanced but promotion to the next tier is blocked", ); Some(format!( "post-deploy gate(s) failed on {version}: {names}; \ the tier is serving {version} but cannot promote onward until they pass" )) } Err(e) => { tracing::error!( tier = %target.name, version = %version, error = %e, "post-deploy gate execution errored; promotion to the next tier is blocked", ); Some(format!( "post-deploy gate execution errored on {version}: {e}; \ the tier is serving {version} but cannot promote onward until the gates pass" )) } }; } // 4. Advance tier_state through the single sealed forward-advance op (atomic // self-referential UPDATE; no read-modify-write to lose under concurrency, // CF3). We hold deploy_lock for this whole handler, so the advance is // serialized against rollback and the host build path's advance. // reset_burn_in on the *source* tier nulls its clock only when the operator // explicitly asked. crate::runs::advance_tier(&s.pool, target.name.as_str(), &version, build_id) .await .map_err(crate::error::Error::Db)?; if body.reset_burn_in { sqlx::query("UPDATE tier_state SET burn_in_started_at = NULL WHERE tier = ?") .bind(&source.name) .execute(&s.pool) .await .map_err(crate::error::Error::Db)?; } // Red post-deploy gates: the rollout itself reached every node, but the tier // is not fit to promote onward. Flag it so /state and the TUI say so, and // return the failure rather than a 200 the operator would read as "shipped, // all good". tier_state has already advanced above — it describes what the // nodes are running, not whether we're happy about it. if let Some(reason) = post_deploy_failure { set_partial(&s, &target.name, &reason).await; return Err(crate::error::Error::GateBlocked(reason)); } // A clean full rollout to every node clears any prior partial flag on this tier. clear_partial(&s, &target.name).await; crate::events::emit( &s.events, crate::events::Event::PromoteComplete { tier: target.name.clone(), version: version.clone(), }, ); tracing::info!( version = %version, tier = %target.name, hotfix = body.hotfix, reset_burn_in = body.reset_burn_in, "promote complete", ); Ok(Json(serde_json::json!({ "tier": target.name, "version": version, "nodes_deployed": target.nodes.iter().map(|n| n.name.clone()).collect::>(), }))) } /// After a canary node fails mid-promote, restore the nodes already flipped to /// the new version back to `prev_version`, leaving the tier consistent (all on /// the old version) rather than split-brain. Best-effort: every node is /// attempted; a per-node failure is logged but never propagated (the promote is /// already failing). Returns how many nodes were successfully restored. Returns /// 0 (with an error log) when the previous version has no recorded artifact to /// roll back to. pub(super) async fn rollback_deployed_nodes( s: &AppState, tier: &crate::domain::TierId, nodes: &[&crate::topology::Node], prev_version: &str, ) -> usize { let bin: Option<(String,)> = match sqlx::query_as( "SELECT artifact_path FROM versions WHERE version = ?", ) .bind(prev_version) .fetch_optional(&s.pool) .await { Ok(b) => b, Err(e) => { tracing::error!(tier = %tier, prev = prev_version, error = %e, "canary rollback: looking up the previous artifact failed; nodes left on the new version"); return 0; } }; let Some((bin,)) = bin else { tracing::error!(tier = %tier, prev = prev_version, nodes = nodes.len(), "canary rollback: previous version has no artifact_path; nodes left on the new version"); return 0; }; let Some(staged_dir) = std::path::PathBuf::from(&bin) .parent() .map(std::path::Path::to_path_buf) else { tracing::error!(tier = %tier, prev = prev_version, "canary rollback: previous artifact_path has no parent dir; nodes left on the new version"); return 0; }; let mut restored = 0usize; for node in nodes { let executor = s .executors .get(&node.name) .cloned() .unwrap_or_else(|| crate::state::build_executor(node)); match crate::deploy::deploy_node( executor.as_ref(), node, prev_version, &staged_dir, s.cfg.primary_bin(), ) .await { Ok(_) => { restored += 1; tracing::warn!(tier = %tier, node = %node.name, version = prev_version, "canary rollback: node restored to the previous version"); } Err(e) => tracing::error!( tier = %tier, node = %node.name, version = prev_version, error = %format!("{e:#}"), "canary rollback FAILED for node; it remains on the new version — manual intervention needed", ), } } restored } /// Flag a tier as left in a partial / mixed-version state, with a human-readable /// reason surfaced through `/state` and the TUI. Best-effort: a failure to record /// the flag is logged, never propagated — the caller is already on an error path /// and the worse outcome is to mask the original failure with a bookkeeping one. pub(super) async fn set_partial(s: &AppState, tier: &crate::domain::TierId, reason: &str) { if let Err(e) = sqlx::query("UPDATE tier_state SET partial_reason = ? WHERE tier = ?") .bind(reason) .bind(tier) .execute(&s.pool) .await { tracing::error!(tier = %tier, reason, error = %e, "failed to record tier partial state; the fleet may be inconsistent without a /state flag"); } } /// Clear a tier's partial flag after a clean full promote or rollback. Errors are /// logged but not propagated: the deploy itself succeeded, and a stale flag is a /// visible nuisance, not a safety regression (the operator sees a partial marker /// on a tier that is actually fine, and re-checks). pub(super) async fn clear_partial(s: &AppState, tier: &crate::domain::TierId) { if let Err(e) = sqlx::query("UPDATE tier_state SET partial_reason = NULL WHERE tier = ?") .bind(tier) .execute(&s.pool) .await { tracing::warn!(tier = %tier, error = %e, "failed to clear tier partial flag"); } } /// Returns the kinds of `tier`'s *configured* gates that are not satisfied for /// `version`. `hotfix` suppresses the `burn_in` requirement only. /// /// Fail-closed against the topology gate list (the CF1 fix). The previous /// version inspected only existing `gate_runs` rows, so a configured gate that /// had *never run* produced no row and was invisibly treated as green — letting /// a promote wave through with zero evidence (it shipped 0.9.5 to prod with /// tier A's `boot_smoke` never recorded). Now every configured gate must show /// positive evidence: /// - `burn_in` is evaluated live against the tier's clock (a stored `blocked` /// row would otherwise never flip to passed as time elapses); /// - every other kind requires a `passed` row for (tier, version) — a missing /// or non-passed latest row counts as unsatisfied. /// /// `build_id` is the identity of the artifact being promoted (wiki note /// `release-artifact-identity`). When `Some`, the deterministic build-evidence /// gates are checked against the row that vouched for *that build* rather than /// against any row that happens to carry the version string — this is what stops /// a `promote --version Y` from riding on gate rows a different build left under /// the same version. `None` is the legacy/pre-identity path: fall back to the /// version-keyed lookup so a mid-migration tier (NULL `current_build_id`) still /// promotes. `burn_in` is clock-based either way — the clock is reset on every /// advance, so it belongs to the build now current on the tier. pub(super) async fn unsatisfied_gates( pool: &sqlx::SqlitePool, tier: &crate::domain::TierId, gates: &[crate::topology::Gate], version: &str, build_id: Option, hotfix: bool, ) -> std::result::Result, crate::error::Error> { use crate::topology::Gate; let mut bad = Vec::new(); for gate in gates { let kind = gate.kind(); match gate { Gate::BurnIn { hours } => { if hotfix { continue; } let ok = crate::gates::burn_in_satisfied(pool, tier, *hours) .await .map_err(crate::error::Error::Other)?; if !ok { bad.push(kind.as_str().to_string()); } } Gate::ManualConfirm => { // A confirmation must be *fresh*: recorded at or after the // version's current landing on this tier (tier_state // .burn_in_started_at, the per-deploy clock). Without this a // confirmation row survives a rollback + rollback-forward and // waves a re-deploy of the same version through with no fresh // operator sign-off — weaker than burn_in, which is clock-based. // No baseline (NULL) => fail closed: require a fresh confirm. let confirmed_at: Option = sqlx::query_scalar( "SELECT finished_at FROM gate_runs WHERE tier = ?1 AND version = ?2 AND gate_kind = 'manual_confirm' AND status = 'passed' ORDER BY id DESC LIMIT 1", ) .bind(tier.as_str()) .bind(version) .fetch_optional(pool) .await .map_err(crate::error::Error::Db)? .flatten(); let landed_at: Option = sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE tier = ?") .bind(tier.as_str()) .fetch_optional(pool) .await .map_err(crate::error::Error::Db)? .flatten(); let fresh = match (confirmed_at, landed_at) { (Some(c), Some(l)) => { match ( chrono::DateTime::parse_from_rfc3339(&c), chrono::DateTime::parse_from_rfc3339(&l), ) { (Ok(cd), Ok(ld)) => cd >= ld, _ => false, // unparseable timestamp -> fail closed } } _ => false, }; if !fresh { bad.push(kind.as_str().to_string()); } } // Build/post-deploy gates that leave a `gate_runs` row: the latest // row for this (tier, version, kind) must be `passed`. Listed // explicitly (no `_` catch-all) so adding a new `Gate` variant is a // compile error here until its promotion semantics are decided — // a transient-`blocked` kind silently falling into "needs a passed // row" would be permanently unsatisfiable. Gate::CargoTest | Gate::HardeningTest | Gate::Clippy | Gate::Fmt | Gate::CargoAudit | Gate::CargoDeny | Gate::MigrationDryRun | Gate::CodeSmoke | Gate::BootSmoke | Gate::NodeHealth => { // Latest row for this configured gate kind; NULL/missing/any // non-'passed' status all count as unsatisfied (fail closed). // Keyed on build_id when the artifact has an identity (the // evidence must be for *this* build), else on the version string. let status: Option = match build_id { Some(bid) => sqlx::query_scalar( "SELECT status FROM gate_runs WHERE tier = ?1 AND build_id = ?2 AND gate_kind = ?3 ORDER BY id DESC LIMIT 1", ) .bind(tier.as_str()) .bind(bid) .bind(kind.as_str()), None => sqlx::query_scalar( "SELECT status FROM gate_runs WHERE tier = ?1 AND version = ?2 AND gate_kind = ?3 ORDER BY id DESC LIMIT 1", ) .bind(tier.as_str()) .bind(version) .bind(kind.as_str()), } .fetch_optional(pool) .await .map_err(crate::error::Error::Db)? .flatten(); if status.as_deref() != Some("passed") { bad.push(kind.as_str().to_string()); } } } } Ok(bad) }