//! Gate execution. Each gate kind has a runner that produces a pass/fail //! outcome plus an optional detail string (typically a stderr tail or a //! human-readable reason). Outcomes are persisted to `gate_runs` so /state //! and the TUI can show them. use crate::config::Config; use crate::topology::Gate; use anyhow::Result; use chrono::Utc; use sqlx::SqlitePool; use std::path::PathBuf; use std::sync::Arc; use tokio::process::Command; pub struct GateCtx { pub pool: SqlitePool, pub cfg: Arc, pub tier: String, pub version: String, pub worktree: PathBuf, } #[derive(Debug, Clone)] pub struct GateOutcome { pub passed: bool, pub detail: Option, } /// Run a single gate end-to-end: insert the in-flight row, execute the gate, /// update the row with the outcome. Returns the outcome for the caller. pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result { let kind = kind_str(gate); let started_at = Utc::now().to_rfc3339(); let id: i64 = sqlx::query_scalar( "INSERT INTO gate_runs (version, tier, gate_kind, started_at) VALUES (?, ?, ?, ?) RETURNING id", ) .bind(&ctx.version) .bind(&ctx.tier) .bind(kind) .bind(&started_at) .fetch_one(&ctx.pool) .await?; tracing::info!(tier = %ctx.tier, version = %ctx.version, gate = kind, "gate start"); let outcome = match gate { Gate::CargoTest => cargo_test(ctx).await, Gate::MigrationDryRun => migration_dry_run(ctx).await, Gate::BootSmoke => boot_smoke(ctx).await, Gate::BurnIn { hours } => burn_in(ctx, *hours).await, Gate::ManualConfirm => manual_confirm(ctx).await, }; let outcome = outcome.unwrap_or_else(|e| GateOutcome { passed: false, detail: Some(format!("gate runner errored: {e}")), }); sqlx::query( "UPDATE gate_runs SET finished_at = ?, passed = ?, detail = ? WHERE id = ?", ) .bind(Utc::now().to_rfc3339()) .bind(outcome.passed as i64) .bind(outcome.detail.as_deref()) .bind(id) .execute(&ctx.pool) .await?; tracing::info!( tier = %ctx.tier, version = %ctx.version, gate = kind, passed = outcome.passed, "gate done", ); Ok(outcome) } /// Run a sequence of gates; stops on the first failure (no point running the /// rest if a prerequisite failed). Returns true iff every gate passed. pub async fn run_all(ctx: &GateCtx, gates: &[Gate]) -> Result { for g in gates { let o = run(ctx, g).await?; if !o.passed { return Ok(false); } } Ok(true) } fn kind_str(g: &Gate) -> &'static str { match g { Gate::CargoTest => "cargo_test", Gate::MigrationDryRun => "migration_dry_run", Gate::BootSmoke => "boot_smoke", Gate::BurnIn { .. } => "burn_in", Gate::ManualConfirm => "manual_confirm", } } // ---- individual gate runners ---- async fn cargo_test(ctx: &GateCtx) -> Result { let server_dir = ctx.worktree.join("server"); let out = Command::new("cargo") .args(["test", "--release"]) .current_dir(&server_dir) .output() .await?; Ok(GateOutcome { passed: out.status.success(), detail: Some(tail(&out.stderr, 4_000)), }) } async fn migration_dry_run(ctx: &GateCtx) -> Result { let Some(db_url) = ctx.cfg.scratch_db_url.as_deref() else { return Ok(GateOutcome { passed: false, detail: Some("scratch_db_url unset in daemon config".into()), }); }; let backup: Option<(String,)> = sqlx::query_as( "SELECT local_path FROM backups ORDER BY id DESC LIMIT 1", ) .fetch_optional(&ctx.pool) .await?; let Some((backup_path,)) = backup else { return Ok(GateOutcome { passed: false, detail: Some("no backup fetched; call /backup/fetch first".into()), }); }; // Reset the scratch DB: drop schema public, restore dump, run migrations. if let Err(e) = reset_scratch(db_url).await { return Ok(GateOutcome { passed: false, detail: Some(format!("scratch reset: {e}")) }); } if let Err(e) = restore_dump(db_url, &backup_path).await { return Ok(GateOutcome { passed: false, detail: Some(format!("restore: {e}")) }); } let migrations_dir = ctx.worktree.join("server").join("migrations"); match run_migrator(db_url, &migrations_dir).await { Ok(()) => Ok(GateOutcome { passed: true, detail: Some(format!("restored {backup_path} + migrated")) }), Err(e) => Ok(GateOutcome { passed: false, detail: Some(tail(e.to_string().as_bytes(), 4_000)) }), } } async fn reset_scratch(db_url: &str) -> Result<()> { use sqlx::postgres::PgPoolOptions; use sqlx::Executor; let pool = PgPoolOptions::new().max_connections(1).connect(db_url).await?; pool.execute("DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;") .await?; pool.close().await; Ok(()) } async fn restore_dump(db_url: &str, dump: &str) -> Result<()> { // Two pipelines we accept: // *.sql -> psql $url < dump // *.sql.gz -> gunzip -c dump | psql $url let is_gz = dump.ends_with(".gz"); let shell = if is_gz { format!("gunzip -c {q} | psql {url}", q = shell_escape(dump), url = shell_escape(db_url)) } else { format!("psql {url} < {q}", q = shell_escape(dump), url = shell_escape(db_url)) }; let out = Command::new("sh").arg("-c").arg(&shell).output().await?; anyhow::ensure!( out.status.success(), "restore failed: {}", String::from_utf8_lossy(&out.stderr), ); Ok(()) } async fn run_migrator(db_url: &str, dir: &std::path::Path) -> Result<()> { use sqlx::postgres::PgPoolOptions; let pool = PgPoolOptions::new().max_connections(1).connect(db_url).await?; let migrator = sqlx::migrate::Migrator::new(dir).await?; migrator.run(&pool).await?; pool.close().await; Ok(()) } fn shell_escape(s: &str) -> String { format!("'{}'", s.replace('\'', "'\\''")) } async fn boot_smoke(ctx: &GateCtx) -> Result { let bin: Option<(String,)> = sqlx::query_as( "SELECT artifact_path FROM versions WHERE version = ?", ) .bind(&ctx.version) .fetch_optional(&ctx.pool) .await?; let Some((bin,)) = bin else { return Ok(GateOutcome { passed: false, detail: Some("no artifact for version".into()) }); }; // Lowest-bar smoke: start the binary and verify it stays up for a few // seconds without exiting. Panics in main, missing config, port-bind // failures show up here. Anything more ambitious (probing /healthz on a // real port) needs server config we don't generically know. let mut child = match tokio::process::Command::new(&bin) .env("SANDO_BOOT_SMOKE", "1") .kill_on_drop(true) .spawn() { Ok(c) => c, Err(e) => return Ok(GateOutcome { passed: false, detail: Some(format!("spawn: {e}")) }), }; tokio::time::sleep(std::time::Duration::from_secs(3)).await; match child.try_wait()? { Some(status) => Ok(GateOutcome { passed: false, detail: Some(format!("binary exited early: {status}")), }), None => { let _ = child.kill().await; Ok(GateOutcome { passed: true, detail: Some("stayed up for 3s".into()) }) } } } async fn burn_in(ctx: &GateCtx, hours: u32) -> Result { // Check tier_state.burn_in_started_at on this tier; pass if enough time // has elapsed. The clock is started by /promote when a version lands on // the burn-in tier. let started: Option = sqlx::query_scalar( "SELECT burn_in_started_at FROM tier_state WHERE tier = ?", ) .bind(&ctx.tier) .fetch_optional(&ctx.pool) .await? .flatten(); let Some(started) = started else { return Ok(GateOutcome { passed: false, detail: Some("burn-in clock not started".into()) }); }; let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc); let elapsed = Utc::now() - started; let needed = chrono::Duration::hours(hours as i64); if elapsed >= needed { Ok(GateOutcome { passed: true, detail: Some(format!("{} hours elapsed", elapsed.num_hours())) }) } else { let remaining = needed - elapsed; Ok(GateOutcome { passed: false, detail: Some(format!("{} hours remaining of {hours}", remaining.num_hours())), }) } } async fn manual_confirm(ctx: &GateCtx) -> Result { // Pass iff a row in gate_runs exists with passed=1 for this (tier, version, manual_confirm) // that was inserted out-of-band by an operator action. Since the harness inserts the // in-flight row itself, look for a prior confirmation row. let prior: Option = sqlx::query_scalar( "SELECT COUNT(*) FROM gate_runs WHERE tier = ? AND version = ? AND gate_kind = 'manual_confirm' AND passed = 1", ) .bind(&ctx.tier) .bind(&ctx.version) .fetch_optional(&ctx.pool) .await?; let passed = prior.unwrap_or(0) > 0; Ok(GateOutcome { passed, detail: if passed { None } else { Some("waiting on operator confirmation".into()) }, }) } fn tail(buf: &[u8], max: usize) -> String { let s = String::from_utf8_lossy(buf); if s.len() <= max { s.into_owned() } else { format!("...{}", &s[s.len() - max..]) } }