Skip to main content

max / makenotwork

22.1 KB · 546 lines History Blame Raw
1 //! Gate execution. Each gate kind has a runner that produces a pass/fail
2 //! outcome plus an optional detail string (typically a stderr tail or a
3 //! human-readable reason). Outcomes are persisted to `gate_runs` so /state
4 //! and the TUI can show them.
5 //!
6 //! This module holds the context every runner reads ([`GateCtx`]), the
7 //! dispatcher ([`run`]) and the two gates that are pure table reads (burn-in
8 //! and manual confirmation). The runners themselves sit in siblings, one per
9 //! tool family: `cargo`, `pg`, `migration`, `code_smoke`, `probes`, with `log`
10 //! under all of them.
11 //!
12 //! Nothing here shares mutable state. `GateCtx` is a plain struct every runner
13 //! reads and none writes, which is what lets the families separate.
14 //!
15 //! # Design
16 //!
17 //! The tier/gate/deploy architecture, the typed-observability redesign, and the
18 //! deploy.sh-parity and account-permission notes live in the maintainer wiki.
19 //! <!-- wiki: sando-overview -->
20
21 use self::cargo::{cargo_test, clippy, fmt_check, hardening_test, supply_chain};
22 use self::code_smoke::code_smoke;
23 use self::migration::migration_dry_run;
24 use self::probes::{boot_smoke, node_health, page_smoke};
25 use crate::config::AppConfig;
26 use crate::domain::{AppId, GateKind, GateRunId, TierId, Version};
27 use crate::events::{self, Event, EventTx};
28 use crate::outcome::{GateBlocker, GateFailure, GateOutcome, LogRef, PassNote};
29 use crate::topology::Gate;
30 use anyhow::Result;
31 use chrono::Utc;
32 use sqlx::SqlitePool;
33 use std::collections::HashMap;
34 use std::path::Path;
35 use std::path::PathBuf;
36 use std::sync::Arc;
37
38 mod cargo;
39 mod code_smoke;
40 mod log;
41 mod migration;
42 mod pg;
43 mod probes;
44 #[cfg(test)]
45 mod testkit;
46
47 pub use pg::preflight_scratch_privileges;
48 pub(crate) use pg::{reset_scratch, run_migrator};
49
50 pub struct GateCtx {
51 pub pool: SqlitePool,
52 pub cfg: Arc<AppConfig>,
53 pub tier: TierId,
54 pub version: Version,
55 /// The checkout this run's artifact was built from, when there is one.
56 ///
57 /// `None` for an accepted artifact: it was built elsewhere and Sando has no
58 /// source tree for it. That is the boundary made visible (wiki
59 /// [[sando-bento-boundary]]) — artifact-scoped gates belong to the builder,
60 /// so a gate that reads source is one Sando should refuse to run here rather
61 /// than resolve against a path that does not exist.
62 pub worktree: Option<PathBuf>,
63 /// The published, content-addressed bundle this run is about, when it has
64 /// been published yet. `migration_dry_run` prefers it over the worktree, so
65 /// what it proves is inside the digest rather than beside it.
66 pub bundle: Option<PathBuf>,
67 pub events: EventTx,
68 /// Nodes the `node_health` post-deploy gate probes. Empty for build-time
69 /// gate runs on the host (where `node_health` never appears); filled at
70 /// promote time with each freshly-deployed node and its executor.
71 pub nodes: Vec<NodeProbe>,
72 /// The `build_runs.id` this gate run vouches for — the artifact identity
73 /// (wiki [[release-artifact-identity]]). Recorded on every `gate_runs` row so
74 /// promote can resolve the artifact through the evidence for a specific build,
75 /// not through a version string that a later rebuild can silently reuse.
76 /// `None` for legacy/pre-identity runs and gate unit tests.
77 pub build_id: Option<i64>,
78 /// Where each `[[aux_repo]]` is checked out, by topology name. Aux repos sit
79 /// beside the per-sha worktree rather than under it, so a `test_target` that
80 /// names one cannot be resolved against `worktree` alone. Filled from the
81 /// topology at build time; empty at promote time, where the only gate that
82 /// runs is `node_health` and there is no checkout at all.
83 pub aux_dirs: HashMap<String, PathBuf>,
84 /// The tier's public URL, for [`Gate::PageSmoke`]. `None` on every
85 /// build-time run and on any tier that declares none.
86 ///
87 /// [`Gate::PageSmoke`]: crate::topology::Gate::PageSmoke
88 pub public_url: Option<String>,
89 }
90
91 impl GateCtx {
92 /// The `logs_root` sub-directory this run's gate logs land in.
93 ///
94 /// The build id, so two runs of one version keep two sets of logs. Keying on
95 /// the version would have a rebuild append to the previous attempt's file,
96 /// with every run pointing at the same mixed log.
97 ///
98 /// Falls back to the version when there is no build identity (a
99 /// pre-migration-008 run, or a gate unit test), which also keeps logs written
100 /// under that scheme reachable: their rows record the version path, and
101 /// nothing rewrites them.
102 pub fn log_scope(&self) -> String {
103 self.build_id
104 .map_or_else(|| self.version.to_string(), |id| id.to_string())
105 }
106
107 /// This run's log pointer for `gate`. Always paired with
108 /// [`Self::log_path`], which resolves the same ref to an absolute path.
109 pub fn log_ref(&self, gate: GateKind) -> LogRef {
110 LogRef::new(&self.log_scope(), gate)
111 }
112
113 /// Where `gate`'s log is written on this host: `logs_root` joined to
114 /// [`Self::log_ref`]. The two are derived from one scope so a row's
115 /// `log_ref` can never name a file the gate did not write.
116 pub fn log_path(&self, gate: GateKind) -> PathBuf {
117 self.cfg
118 .logs_root
119 .join(self.log_scope())
120 .join(format!("{}.log", gate.as_str()))
121 }
122
123 /// Absolute directory a `test_target` runs in: under the worktree, or under
124 /// the named aux repo's checkout.
125 ///
126 /// An `aux_repo` naming nothing this run knows about resolves to `None`
127 /// rather than to a wrong path. Callers treat that as "not present in this
128 /// run" and skip, the same as a target missing from an older sha —
129 /// `--check-config` is what stops a genuine typo from reaching here
130 /// (`Topology::ensure_test_target_aux_repos_exist`).
131 pub fn target_dir(&self, target: &crate::config::TestTarget) -> Option<PathBuf> {
132 match target.aux_repo.as_deref() {
133 None => Some(self.worktree.as_ref()?.join(&target.dir)),
134 Some(name) => Some(self.aux_dirs.get(name)?.join(&target.dir)),
135 }
136 }
137
138 /// The checkout, or a typed refusal for a gate that cannot work without one.
139 ///
140 /// Every caller of this is a gate whose evidence is about the *artifact*
141 /// rather than about the artifact in an environment, which the boundary
142 /// assigns to the builder. Reaching this arm means a tier asked Sando to
143 /// re-run a builder's gate against a bundle it was handed, and the honest
144 /// answer is to say so rather than to pass on having run nothing.
145 pub fn worktree_for(&self, gate: GateKind) -> std::result::Result<&Path, GateOutcome> {
146 self.worktree.as_deref().ok_or_else(|| {
147 GateOutcome::failed(GateFailure::NeedsSource {
148 gate,
149 artifact: self.bundle.as_ref().map_or_else(
150 || "an artifact built elsewhere".into(),
151 |b| b.display().to_string(),
152 ),
153 })
154 })
155 }
156
157 /// Where a `migration_check` finds its migrations.
158 ///
159 /// The bundle wins when it carries them. That is the point of shipping
160 /// migrations as a `release_contents` entry: it puts them inside the digest,
161 /// so the dry run proves something about the bytes that ship rather than
162 /// about a checkout that happens to sit next to them. The worktree is the
163 /// fallback for a build whose config has not opted in yet, and for an
164 /// accepted artifact there is no fallback at all — if the builder did not
165 /// bundle its migrations, Sando cannot dry-run them and says so.
166 pub fn migrations_dir(&self, dir: &Path) -> Option<PathBuf> {
167 if let Some(bundle) = &self.bundle {
168 let in_bundle = bundle.join(dir);
169 if in_bundle.is_dir() {
170 return Some(in_bundle);
171 }
172 }
173 let in_worktree = self.worktree.as_ref()?.join(dir);
174 in_worktree.is_dir().then_some(in_worktree)
175 }
176 }
177
178 /// One node the `node_health` gate verifies: its id, the systemd unit to
179 /// confirm active after the restart, an optional HTTP readiness URL, and the
180 /// executor that reaches it (the same transport the deploy used).
181 pub struct NodeProbe {
182 pub node: crate::domain::NodeId,
183 pub service: String,
184 pub health_url: Option<String>,
185 pub executor: Arc<dyn ops_exec::Executor>,
186 }
187
188 /// Run a single gate end-to-end: insert the in-flight row, execute the gate,
189 /// update the row with the outcome. Returns the outcome for the caller.
190 pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> {
191 let kind = gate.kind();
192 let started_at = Utc::now().to_rfc3339();
193
194 let id: i64 = sqlx::query_scalar(
195 "INSERT INTO gate_runs (app, version, tier, gate_kind, started_at, build_id)
196 VALUES (?, ?, ?, ?, ?, ?)
197 RETURNING id",
198 )
199 .bind(&ctx.cfg.id)
200 .bind(&ctx.version)
201 .bind(&ctx.tier)
202 .bind(kind)
203 .bind(&started_at)
204 .bind(ctx.build_id)
205 .fetch_one(&ctx.pool)
206 .await?;
207 let run_id = GateRunId(id);
208
209 tracing::info!(
210 run_id = %run_id, tier = %ctx.tier, version = %ctx.version, gate = %kind,
211 "gate start",
212 );
213 events::emit(
214 &ctx.events,
215 Event::GateStart {
216 run_id,
217 tier: ctx.tier.clone(),
218 version: ctx.version.clone(),
219 gate: kind,
220 },
221 );
222
223 let outcome = match gate {
224 // cargo_test bounds its own run internally (it kills the specific child).
225 Gate::CargoTest => cargo_test(ctx, run_id).await,
226 // hardening_test bounds its own run internally, same as cargo_test.
227 Gate::HardeningTest => hardening_test(ctx, run_id).await,
228 // Each bounds itself the same way cargo_test does: one deadline across
229 // every target, so N crates cannot multiply the ceiling by N.
230 Gate::Clippy => clippy(ctx, run_id).await,
231 Gate::Fmt => fmt_check(ctx, run_id).await,
232 Gate::CargoAudit => supply_chain(ctx, run_id, GateKind::CargoAudit).await,
233 Gate::CargoDeny => supply_chain(ctx, run_id, GateKind::CargoDeny).await,
234 // migration_dry_run's psql restore + sqlx migrate could wedge; bound the
235 // whole gate here. Its bash restore sets kill_on_drop, so a timeout-drop
236 // doesn't orphan it.
237 Gate::MigrationDryRun => {
238 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
239 match tokio::time::timeout(ceiling, migration_dry_run(ctx, run_id)).await {
240 Ok(res) => res,
241 Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout {
242 gate: GateKind::MigrationDryRun,
243 after_s: ctx.cfg.gate_timeout_secs as u32,
244 })
245 .with_log_ref(ctx.log_ref(GateKind::MigrationDryRun))),
246 }
247 }
248 // code_smoke boots the real binary (migrate-from-scratch + seed + serve),
249 // any step of which could wedge; bound the whole gate here. Both child
250 // processes set kill_on_drop, so a timeout-drop can't orphan them. A
251 // timeout leaves the throwaway DB behind; the next run's createdb drops
252 // it first (DROP IF EXISTS), same as migration_dry_run's scratch reset.
253 Gate::CodeSmoke => {
254 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
255 match tokio::time::timeout(ceiling, code_smoke(ctx, run_id)).await {
256 Ok(res) => res,
257 Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout {
258 gate: GateKind::CodeSmoke,
259 after_s: ctx.cfg.gate_timeout_secs as u32,
260 })
261 .with_log_ref(ctx.log_ref(GateKind::CodeSmoke))),
262 }
263 }
264 Gate::BootSmoke => boot_smoke(ctx, run_id).await,
265 Gate::NodeHealth => node_health(ctx).await,
266 Gate::PageSmoke => page_smoke(ctx).await,
267 Gate::BurnIn { hours } => burn_in(ctx, *hours).await,
268 Gate::ManualConfirm => manual_confirm(ctx).await,
269 };
270
271 let outcome = outcome.unwrap_or_else(|e| {
272 GateOutcome::failed(GateFailure::Unclassified {
273 legacy_detail: Some(format!("gate runner errored: {e}")),
274 })
275 });
276
277 let outcome_json = serde_json::to_string(&outcome)
278 .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
279 sqlx::query(
280 "UPDATE gate_runs
281 SET finished_at = ?, status = ?, outcome_json = ?, log_ref = ?
282 WHERE id = ?",
283 )
284 .bind(Utc::now().to_rfc3339())
285 .bind(outcome.status_str())
286 .bind(&outcome_json)
287 .bind(outcome.log_ref.as_ref().map(super::outcome::LogRef::as_str))
288 .bind(id)
289 .execute(&ctx.pool)
290 .await?;
291
292 tracing::info!(
293 tier = %ctx.tier, version = %ctx.version, gate = %kind,
294 status = outcome.status_str(), "gate done",
295 );
296 events::emit(
297 &ctx.events,
298 Event::GateDone {
299 run_id,
300 tier: ctx.tier.clone(),
301 version: ctx.version.clone(),
302 gate: kind,
303 outcome: outcome.clone(),
304 },
305 );
306
307 Ok(outcome)
308 }
309
310 /// Run every gate in order and return the kinds that did not pass (empty means
311 /// green). We deliberately do NOT short-circuit on first failure — every gate's
312 /// outcome is recorded in `gate_runs`, which is the operator's only visibility
313 /// into pipeline health. Hiding later gates because an earlier one failed makes
314 /// diagnosis worse.
315 ///
316 /// Returning the failing kinds rather than a bare bool is what lets the promote
317 /// path name them in the tier's `partial_reason` and in the error it returns to
318 /// the operator, instead of a generic "something was red".
319 pub async fn run_all(ctx: &GateCtx, gates: &[Gate]) -> Result<Vec<GateKind>> {
320 let mut failed = Vec::new();
321 for g in gates {
322 let o = run(ctx, g).await?;
323 if !o.is_passed() {
324 failed.push(g.kind());
325 }
326 }
327 Ok(failed)
328 }
329
330 /// Live check: has `tier`'s burn-in window of `hours` elapsed since its clock
331 /// (`tier_state.burn_in_started_at`, started by a promote onto the tier)? Used
332 /// by the promote-time gate check (`unsatisfied_gates`) so a stale `blocked`
333 /// row never masks an elapsed — or not-yet-elapsed — window. The `burn_in` gate
334 /// runner below wraps the same state with a richer outcome for `/state`.
335 pub async fn burn_in_satisfied(
336 pool: &SqlitePool,
337 app: &AppId,
338 tier: &TierId,
339 hours: u32,
340 ) -> Result<bool> {
341 let started: Option<String> =
342 sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?")
343 .bind(app)
344 .bind(tier)
345 .fetch_optional(pool)
346 .await?
347 .flatten();
348 let Some(started) = started else {
349 return Ok(false);
350 };
351 let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc);
352 Ok(Utc::now() - started >= chrono::Duration::hours(hours as i64))
353 }
354
355 async fn burn_in(ctx: &GateCtx, hours: u32) -> Result<GateOutcome> {
356 // Check tier_state.burn_in_started_at on this tier; pass if enough time
357 // has elapsed. The clock is started by /promote when a version lands on
358 // the burn-in tier.
359 let started: Option<String> =
360 sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?")
361 .bind(&ctx.cfg.id)
362 .bind(&ctx.tier)
363 .fetch_optional(&ctx.pool)
364 .await?
365 .flatten();
366 let Some(started) = started else {
367 return Ok(GateOutcome::blocked(GateBlocker::BurnInClockNotStarted));
368 };
369 let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc);
370 let elapsed = Utc::now() - started;
371 let needed = chrono::Duration::hours(hours as i64);
372 if elapsed >= needed {
373 Ok(GateOutcome::passed(PassNote::BurnInElapsed {
374 hours: elapsed.num_hours() as u32,
375 }))
376 } else {
377 let remaining = (needed - elapsed).num_hours().max(0) as u32;
378 Ok(GateOutcome::blocked(GateBlocker::BurnInRemaining {
379 hours_remaining: remaining,
380 hours_total: hours,
381 }))
382 }
383 }
384
385 async fn manual_confirm(ctx: &GateCtx) -> Result<GateOutcome> {
386 // Pass iff a row in gate_runs exists with status='passed' for this
387 // (tier, version, manual_confirm) that was inserted out-of-band by an
388 // operator action. Since the harness inserts the in-flight row itself,
389 // look for a prior confirmation row.
390 let prior_at: Option<String> = sqlx::query_scalar(
391 "SELECT finished_at FROM gate_runs
392 WHERE app = ? AND tier = ? AND version = ? AND gate_kind = 'manual_confirm'
393 AND status = 'passed'
394 ORDER BY id DESC LIMIT 1",
395 )
396 .bind(&ctx.cfg.id)
397 .bind(&ctx.tier)
398 .bind(&ctx.version)
399 .fetch_optional(&ctx.pool)
400 .await?;
401 match prior_at {
402 Some(at_str) => {
403 let at = chrono::DateTime::parse_from_rfc3339(&at_str)
404 .map_or_else(|_| Utc::now(), |d| d.with_timezone(&Utc));
405 Ok(GateOutcome::passed(PassNote::OperatorConfirmed { at }))
406 }
407 None => Ok(GateOutcome::blocked(
408 GateBlocker::AwaitingOperatorConfirmation,
409 )),
410 }
411 }
412
413 #[cfg(test)]
414 mod tests {
415 use super::*;
416 use crate::gates::testkit::{aux_target, resolving_ctx, target};
417 use sqlx::sqlite::SqlitePoolOptions;
418
419 #[tokio::test]
420 async fn a_plain_target_resolves_under_the_worktree() {
421 let ctx = resolving_ctx("/w/abc123", &[]);
422 assert_eq!(
423 ctx.target_dir(&target("shared/tagtree")),
424 Some(PathBuf::from("/w/abc123/shared/tagtree")),
425 );
426 }
427
428 #[tokio::test]
429 async fn an_aux_target_resolves_beside_the_worktree_not_under_it() {
430 // The whole point: `Libraries/docengine` is a sibling of the per-sha
431 // worktree, so a worktree-relative path can never reach it.
432 let ctx = resolving_ctx("/w/abc123", &[("docengine", "/w/Libraries/docengine")]);
433 assert_eq!(
434 ctx.target_dir(&aux_target("", "docengine")),
435 Some(PathBuf::from("/w/Libraries/docengine")),
436 );
437 // A subdirectory of an aux repo resolves under its checkout.
438 assert_eq!(
439 ctx.target_dir(&aux_target("crates/inner", "docengine")),
440 Some(PathBuf::from("/w/Libraries/docengine/crates/inner")),
441 );
442 }
443
444 #[tokio::test]
445 async fn an_aux_target_with_no_checkout_this_run_resolves_to_nothing() {
446 // Promote-time gates carry no aux dirs. Resolving to a wrong path (say,
447 // the worktree) would run the gate against whatever happened to sit
448 // there; `None` makes the caller skip, and --check-config is what
449 // catches a real typo.
450 let ctx = resolving_ctx("/w/abc123", &[]);
451 assert_eq!(ctx.target_dir(&aux_target("", "docengine")), None);
452 }
453
454 #[test]
455 fn labels_name_the_repo_an_aux_target_lives_in() {
456 assert_eq!(target("server").label(), "server");
457 assert_eq!(aux_target("", "docengine").label(), "docengine (aux)");
458 assert_eq!(
459 aux_target("crates/inner", "docengine").label(),
460 "docengine/crates/inner (aux)",
461 );
462 }
463
464 #[test]
465 fn every_gate_kind_round_trips_through_its_wire_string() {
466 // gate_kind is a TEXT column and a WS event field; as_str and FromStr
467 // disagreeing would make a gate's evidence unreadable by
468 // unsatisfied_gates, which fails the promote closed with no explanation.
469 for k in [
470 GateKind::CargoTest,
471 GateKind::HardeningTest,
472 GateKind::Clippy,
473 GateKind::Fmt,
474 GateKind::CargoAudit,
475 GateKind::CargoDeny,
476 GateKind::MigrationDryRun,
477 GateKind::CodeSmoke,
478 GateKind::BootSmoke,
479 GateKind::NodeHealth,
480 GateKind::BurnIn,
481 GateKind::ManualConfirm,
482 ] {
483 assert_eq!(
484 k.as_str().parse::<GateKind>().unwrap(),
485 k,
486 "round trip for {k:?}"
487 );
488 }
489 }
490
491 /// burn_in returns a typed Blocked when the clock isn't started; the
492 /// runner persists status='blocked' + outcome_json (the json carries
493 /// blocker.kind = 'burn_in_clock_not_started').
494 #[tokio::test]
495 async fn burn_in_blocked_persists_typed_outcome() {
496 let pool = SqlitePoolOptions::new()
497 .max_connections(1)
498 .connect("sqlite::memory:")
499 .await
500 .unwrap();
501 crate::db::migrate(&pool).await.unwrap();
502 // Topology sync expects a tier row before gate_runs can reference it.
503 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 0, 'sequential')")
504 .execute(&pool).await.unwrap();
505 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
506 .execute(&pool)
507 .await
508 .unwrap();
509 // versions FK target.
510 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
511 .execute(&pool).await.unwrap();
512
513 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests());
514 let ctx = GateCtx {
515 public_url: None,
516 pool: pool.clone(),
517 cfg,
518 tier: TierId::new("host"),
519 version: "0.1.0".parse().unwrap(),
520 worktree: Some(std::path::PathBuf::from("/tmp/unused")),
521 bundle: None,
522 events: events::channel(),
523 nodes: Vec::new(),
524 build_id: None,
525 aux_dirs: HashMap::new(),
526 };
527 let out = run(&ctx, &Gate::BurnIn { hours: 24 }).await.unwrap();
528 assert_eq!(out.status_str(), "blocked");
529 assert!(!out.is_passed());
530
531 // Read the persisted row.
532 let row: (Option<String>, Option<String>) =
533 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
534 .fetch_one(&pool)
535 .await
536 .unwrap();
537 assert_eq!(row.0.as_deref(), Some("blocked"), "typed status");
538 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
539 assert_eq!(json["status"]["kind"], "blocked");
540 assert_eq!(
541 json["status"]["blocker"]["kind"],
542 "burn_in_clock_not_started"
543 );
544 }
545 }
546