Skip to main content

max / makenotwork

161.0 KB · 3984 lines History Blame Raw
1 //! Gate execution. Each gate kind has a runner that produces a pass/fail
2 //! outcome plus an optional detail string (typically a stderr tail or a
3 //! human-readable reason). Outcomes are persisted to `gate_runs` so /state
4 //! and the TUI can show them.
5
6 use crate::classify;
7 use crate::config::AppConfig;
8 use crate::domain::{AppId, GateKind, GateRunId, TierId, Version};
9 use crate::events::{self, Event, EventTx};
10 use crate::outcome::{GateBlocker, GateFailure, GateOutcome, LogRef, PassNote};
11 use crate::topology::Gate;
12 use anyhow::{Context, Result};
13 use chrono::Utc;
14 use ops_core::live_log::LiveLog;
15 use ops_core::remote::LogSink; // brings `LiveLog::write_chunk` (the sink trait) into scope
16 use sqlx::SqlitePool;
17 use std::collections::HashMap;
18 use std::path::Path;
19 use std::path::PathBuf;
20 use std::sync::Arc;
21 use tokio::io::AsyncReadExt;
22 use tokio::process::Command;
23
24 /// The gate live-log callback: emit each chunk as a `GateLogChunk` event so the
25 /// TUI sees the tail stream in real time. `ops_core::live_log::LiveLog` owns the
26 /// disk append and the per-run sequence counter; this closure is the one
27 /// tool-specific bit.
28 fn gate_chunk_cb(events: EventTx, run_id: GateRunId) -> ops_core::live_log::ChunkCallback {
29 Box::new(move |seq, text| {
30 events::emit(
31 &events,
32 Event::GateLogChunk {
33 run_id,
34 seq,
35 text: text.to_owned(),
36 },
37 );
38 })
39 }
40
41 pub struct GateCtx {
42 pub pool: SqlitePool,
43 pub cfg: Arc<AppConfig>,
44 pub tier: TierId,
45 pub version: Version,
46 /// The checkout this run's artifact was built from, when there is one.
47 ///
48 /// `None` for an accepted artifact: it was built elsewhere and Sando has no
49 /// source tree for it. That is the boundary made visible (wiki
50 /// [[sando-bento-boundary]]) — artifact-scoped gates belong to the builder,
51 /// so a gate that reads source is one Sando should refuse to run here rather
52 /// than resolve against a path that does not exist.
53 pub worktree: Option<PathBuf>,
54 /// The published, content-addressed bundle this run is about, when it has
55 /// been published yet. `migration_dry_run` prefers it over the worktree, so
56 /// what it proves is inside the digest rather than beside it.
57 pub bundle: Option<PathBuf>,
58 pub events: EventTx,
59 /// Nodes the `node_health` post-deploy gate probes. Empty for build-time
60 /// gate runs on the host (where `node_health` never appears); filled at
61 /// promote time with each freshly-deployed node and its executor.
62 pub nodes: Vec<NodeProbe>,
63 /// The `build_runs.id` this gate run vouches for — the artifact identity
64 /// (wiki [[release-artifact-identity]]). Recorded on every `gate_runs` row so
65 /// promote can resolve the artifact through the evidence for a specific build,
66 /// not through a version string that a later rebuild can silently reuse.
67 /// `None` for legacy/pre-identity runs and gate unit tests.
68 pub build_id: Option<i64>,
69 /// Where each `[[aux_repo]]` is checked out, by topology name. Aux repos sit
70 /// beside the per-sha worktree rather than under it, so a `test_target` that
71 /// names one cannot be resolved against `worktree` alone. Filled from the
72 /// topology at build time; empty at promote time, where the only gate that
73 /// runs is `node_health` and there is no checkout at all.
74 pub aux_dirs: HashMap<String, PathBuf>,
75 /// The tier's public URL, for [`Gate::PageSmoke`]. `None` on every
76 /// build-time run and on any tier that declares none.
77 ///
78 /// [`Gate::PageSmoke`]: crate::topology::Gate::PageSmoke
79 pub public_url: Option<String>,
80 }
81
82 impl GateCtx {
83 /// The `logs_root` sub-directory this run's gate logs land in.
84 ///
85 /// The build id, so two runs of one version keep two sets of logs. Keying on
86 /// the version would have a rebuild append to the previous attempt's file,
87 /// with every run pointing at the same mixed log.
88 ///
89 /// Falls back to the version when there is no build identity (a
90 /// pre-migration-008 run, or a gate unit test), which also keeps logs written
91 /// under that scheme reachable: their rows record the version path, and
92 /// nothing rewrites them.
93 pub fn log_scope(&self) -> String {
94 self.build_id
95 .map_or_else(|| self.version.to_string(), |id| id.to_string())
96 }
97
98 /// This run's log pointer for `gate`. Always paired with
99 /// [`Self::log_path`], which resolves the same ref to an absolute path.
100 pub fn log_ref(&self, gate: GateKind) -> LogRef {
101 LogRef::new(&self.log_scope(), gate)
102 }
103
104 /// Where `gate`'s log is written on this host: `logs_root` joined to
105 /// [`Self::log_ref`]. The two are derived from one scope so a row's
106 /// `log_ref` can never name a file the gate did not write.
107 pub fn log_path(&self, gate: GateKind) -> PathBuf {
108 self.cfg
109 .logs_root
110 .join(self.log_scope())
111 .join(format!("{}.log", gate.as_str()))
112 }
113
114 /// Absolute directory a `test_target` runs in: under the worktree, or under
115 /// the named aux repo's checkout.
116 ///
117 /// An `aux_repo` naming nothing this run knows about resolves to `None`
118 /// rather than to a wrong path. Callers treat that as "not present in this
119 /// run" and skip, the same as a target missing from an older sha —
120 /// `--check-config` is what stops a genuine typo from reaching here
121 /// (`Topology::ensure_test_target_aux_repos_exist`).
122 pub fn target_dir(&self, target: &crate::config::TestTarget) -> Option<PathBuf> {
123 match target.aux_repo.as_deref() {
124 None => Some(self.worktree.as_ref()?.join(&target.dir)),
125 Some(name) => Some(self.aux_dirs.get(name)?.join(&target.dir)),
126 }
127 }
128
129 /// The checkout, or a typed refusal for a gate that cannot work without one.
130 ///
131 /// Every caller of this is a gate whose evidence is about the *artifact*
132 /// rather than about the artifact in an environment, which the boundary
133 /// assigns to the builder. Reaching this arm means a tier asked Sando to
134 /// re-run a builder's gate against a bundle it was handed, and the honest
135 /// answer is to say so rather than to pass on having run nothing.
136 pub fn worktree_for(&self, gate: GateKind) -> std::result::Result<&Path, GateOutcome> {
137 self.worktree.as_deref().ok_or_else(|| {
138 GateOutcome::failed(GateFailure::NeedsSource {
139 gate,
140 artifact: self.bundle.as_ref().map_or_else(
141 || "an artifact built elsewhere".into(),
142 |b| b.display().to_string(),
143 ),
144 })
145 })
146 }
147
148 /// Where a `migration_check` finds its migrations.
149 ///
150 /// The bundle wins when it carries them. That is the point of shipping
151 /// migrations as a `release_contents` entry: it puts them inside the digest,
152 /// so the dry run proves something about the bytes that ship rather than
153 /// about a checkout that happens to sit next to them. The worktree is the
154 /// fallback for a build whose config has not opted in yet, and for an
155 /// accepted artifact there is no fallback at all — if the builder did not
156 /// bundle its migrations, Sando cannot dry-run them and says so.
157 pub fn migrations_dir(&self, dir: &Path) -> Option<PathBuf> {
158 if let Some(bundle) = &self.bundle {
159 let in_bundle = bundle.join(dir);
160 if in_bundle.is_dir() {
161 return Some(in_bundle);
162 }
163 }
164 let in_worktree = self.worktree.as_ref()?.join(dir);
165 in_worktree.is_dir().then_some(in_worktree)
166 }
167 }
168
169 /// One node the `node_health` gate verifies: its id, the systemd unit to
170 /// confirm active after the restart, an optional HTTP readiness URL, and the
171 /// executor that reaches it (the same transport the deploy used).
172 pub struct NodeProbe {
173 pub node: crate::domain::NodeId,
174 pub service: String,
175 pub health_url: Option<String>,
176 pub executor: Arc<dyn ops_exec::Executor>,
177 }
178
179 /// Run a single gate end-to-end: insert the in-flight row, execute the gate,
180 /// update the row with the outcome. Returns the outcome for the caller.
181 pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> {
182 let kind = gate.kind();
183 let started_at = Utc::now().to_rfc3339();
184
185 let id: i64 = sqlx::query_scalar(
186 "INSERT INTO gate_runs (app, version, tier, gate_kind, started_at, build_id)
187 VALUES (?, ?, ?, ?, ?, ?)
188 RETURNING id",
189 )
190 .bind(&ctx.cfg.id)
191 .bind(&ctx.version)
192 .bind(&ctx.tier)
193 .bind(kind)
194 .bind(&started_at)
195 .bind(ctx.build_id)
196 .fetch_one(&ctx.pool)
197 .await?;
198 let run_id = GateRunId(id);
199
200 tracing::info!(
201 run_id = %run_id, tier = %ctx.tier, version = %ctx.version, gate = %kind,
202 "gate start",
203 );
204 events::emit(
205 &ctx.events,
206 Event::GateStart {
207 run_id,
208 tier: ctx.tier.clone(),
209 version: ctx.version.clone(),
210 gate: kind,
211 },
212 );
213
214 let outcome = match gate {
215 // cargo_test bounds its own run internally (it kills the specific child).
216 Gate::CargoTest => cargo_test(ctx, run_id).await,
217 // hardening_test bounds its own run internally, same as cargo_test.
218 Gate::HardeningTest => hardening_test(ctx, run_id).await,
219 // Each bounds itself the same way cargo_test does: one deadline across
220 // every target, so N crates cannot multiply the ceiling by N.
221 Gate::Clippy => clippy(ctx, run_id).await,
222 Gate::Fmt => fmt_check(ctx, run_id).await,
223 Gate::CargoAudit => supply_chain(ctx, run_id, GateKind::CargoAudit).await,
224 Gate::CargoDeny => supply_chain(ctx, run_id, GateKind::CargoDeny).await,
225 // migration_dry_run's psql restore + sqlx migrate could wedge; bound the
226 // whole gate here. Its bash restore sets kill_on_drop, so a timeout-drop
227 // doesn't orphan it.
228 Gate::MigrationDryRun => {
229 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
230 match tokio::time::timeout(ceiling, migration_dry_run(ctx, run_id)).await {
231 Ok(res) => res,
232 Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout {
233 gate: GateKind::MigrationDryRun,
234 after_s: ctx.cfg.gate_timeout_secs as u32,
235 })
236 .with_log_ref(ctx.log_ref(GateKind::MigrationDryRun))),
237 }
238 }
239 // code_smoke boots the real binary (migrate-from-scratch + seed + serve),
240 // any step of which could wedge; bound the whole gate here. Both child
241 // processes set kill_on_drop, so a timeout-drop can't orphan them. A
242 // timeout leaves the throwaway DB behind; the next run's createdb drops
243 // it first (DROP IF EXISTS), same as migration_dry_run's scratch reset.
244 Gate::CodeSmoke => {
245 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
246 match tokio::time::timeout(ceiling, code_smoke(ctx, run_id)).await {
247 Ok(res) => res,
248 Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout {
249 gate: GateKind::CodeSmoke,
250 after_s: ctx.cfg.gate_timeout_secs as u32,
251 })
252 .with_log_ref(ctx.log_ref(GateKind::CodeSmoke))),
253 }
254 }
255 Gate::BootSmoke => boot_smoke(ctx, run_id).await,
256 Gate::NodeHealth => node_health(ctx).await,
257 Gate::PageSmoke => page_smoke(ctx).await,
258 Gate::BurnIn { hours } => burn_in(ctx, *hours).await,
259 Gate::ManualConfirm => manual_confirm(ctx).await,
260 };
261
262 let outcome = outcome.unwrap_or_else(|e| {
263 GateOutcome::failed(GateFailure::Unclassified {
264 legacy_detail: Some(format!("gate runner errored: {e}")),
265 })
266 });
267
268 let outcome_json = serde_json::to_string(&outcome)
269 .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
270 sqlx::query(
271 "UPDATE gate_runs
272 SET finished_at = ?, status = ?, outcome_json = ?, log_ref = ?
273 WHERE id = ?",
274 )
275 .bind(Utc::now().to_rfc3339())
276 .bind(outcome.status_str())
277 .bind(&outcome_json)
278 .bind(outcome.log_ref.as_ref().map(super::outcome::LogRef::as_str))
279 .bind(id)
280 .execute(&ctx.pool)
281 .await?;
282
283 tracing::info!(
284 tier = %ctx.tier, version = %ctx.version, gate = %kind,
285 status = outcome.status_str(), "gate done",
286 );
287 events::emit(
288 &ctx.events,
289 Event::GateDone {
290 run_id,
291 tier: ctx.tier.clone(),
292 version: ctx.version.clone(),
293 gate: kind,
294 outcome: outcome.clone(),
295 },
296 );
297
298 Ok(outcome)
299 }
300
301 /// Run every gate in order and return the kinds that did not pass (empty means
302 /// green). We deliberately do NOT short-circuit on first failure — every gate's
303 /// outcome is recorded in `gate_runs`, which is the operator's only visibility
304 /// into pipeline health. Hiding later gates because an earlier one failed makes
305 /// diagnosis worse.
306 ///
307 /// Returning the failing kinds rather than a bare bool is what lets the promote
308 /// path name them in the tier's `partial_reason` and in the error it returns to
309 /// the operator, instead of a generic "something was red".
310 pub async fn run_all(ctx: &GateCtx, gates: &[Gate]) -> Result<Vec<GateKind>> {
311 let mut failed = Vec::new();
312 for g in gates {
313 let o = run(ctx, g).await?;
314 if !o.is_passed() {
315 failed.push(g.kind());
316 }
317 }
318 Ok(failed)
319 }
320
321 // ---- individual gate runners ----
322
323 /// Run every configured `test_target`'s suite, in order, under one gate.
324 ///
325 /// Targets are configured (`[[test_target]]` in the daemon config), defaulting
326 /// to a single `server` entry. A crate with no target ships ungated, `mnw-cli`
327 /// included, which is built as a companion and installed onto prod-1 in the same
328 /// promote.
329 ///
330 /// The whole set shares one `gate_runs` row and one log file: from the
331 /// pipeline's point of view "the tests" either pass or don't. The first failing
332 /// target ends the gate, since a red suite blocks the promote regardless of what
333 /// the remaining crates would have said, and running them would only delay the
334 /// operator's answer. Its name is carried in the failure so the summary points
335 /// at the crate, not just the test.
336 async fn cargo_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
337 let log_path = ctx.log_path(GateKind::CargoTest);
338 let log_ref = ctx.log_ref(GateKind::CargoTest);
339
340 // Best-effort: drop our own role's stale `mnw_test_*` databases (the
341 // template + any per-test clones orphaned by a previously-killed run)
342 // before the suite, so they can't accumulate or collide. Foreign-owned
343 // leftovers are left alone — the harness now namespaces its template per
344 // role, so they no longer wedge the gate.
345 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
346 clean_stale_test_dbs(scratch_url).await;
347 }
348
349 let started = std::time::Instant::now();
350 // One ceiling for the whole gate, not per target: the point is to bound how
351 // long a hung suite can block the pipeline, and N targets each allowed the
352 // full timeout would multiply that by N.
353 let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
354 let mut ran = 0usize;
355
356 for target in &ctx.cfg.test_targets {
357 let label = target.label();
358 // A target absent from this sha is skipped, not fatal: sando has to be
359 // able to build older shas (bisect, rollback rebuild) from a config that
360 // describes the tip. The zero-targets-ran check below is what stops this
361 // from quietly turning the gate into a no-op.
362 let Some(dir) = ctx
363 .target_dir(target)
364 .filter(|d| d.join("Cargo.toml").is_file())
365 else {
366 tracing::warn!(
367 target = %label, version = %ctx.version,
368 "test_target has no Cargo.toml in this run; skipping",
369 );
370 continue;
371 };
372 let features: Vec<&str> = target.features.iter().map(String::as_str).collect();
373
374 // Fast pre-gate: compile the test targets WITHOUT running them. This
375 // builds the exact artifacts the full run needs (so the subsequent run
376 // reuses the cache — no wasted work), but fails in ~minutes with the
377 // real `error[Ennnn]: ...` on a test-only-target compile break. That
378 // class (a field missing in a `#[cfg(test)]`-only binary like `load`)
379 // otherwise compiles fine under the build step and only blows up here,
380 // after a full build, as an opaque mass test failure.
381 let banner = format!("\n==== test_target: {label} ====\n");
382 append_to_log(&log_path, banner.as_bytes()).await;
383 let mut pre = match cargo_test_command(ctx, &dir, target, &features, &["--no-run"]).spawn()
384 {
385 Ok(c) => c,
386 Err(e) => {
387 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
388 message: format!("{label}: {e}"),
389 })
390 .with_log_ref(log_ref));
391 }
392 };
393 let (pre_out, pre_err, pre_status) = match run_to_deadline_for(
394 &mut pre,
395 ctx,
396 run_id,
397 log_path.clone(),
398 deadline,
399 started,
400 GateKind::CargoTest,
401 )
402 .await?
403 {
404 Ok(v) => v,
405 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
406 };
407 if !pre_status.success() {
408 let failure = classify::classify_compile_error(&pre_out, &pre_err);
409 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
410 }
411
412 // Full run: the test binaries are already built above, so cargo's
413 // up-to-date check skips compilation and this just runs the tests.
414 //
415 // That claim is only true when the crate's build script is up to date
416 // too, and for a long time it was not. server and multithreaded both
417 // watched `.git/HEAD`, a path that does not exist at either package
418 // root, and cargo reads a missing watch as changed: the build script
419 // re-ran and the crate recompiled here, every time. It cost 347s a
420 // pipeline, 35% of this gate, while this comment said it cost nothing.
421 // Fixed 2026-08-20 in both build scripts; see `git_hash` in either.
422 //
423 // If this gate's duration ever climbs back toward the pre-pass's, look
424 // for a new phantom watch before looking anywhere else.
425 let mut child = match cargo_test_command(ctx, &dir, target, &features, &[]).spawn() {
426 Ok(c) => c,
427 Err(e) => {
428 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
429 message: format!("{label}: {e}"),
430 })
431 .with_log_ref(log_ref));
432 }
433 };
434 let (stdout_buf, stderr_buf, status) = match run_to_deadline_for(
435 &mut child,
436 ctx,
437 run_id,
438 log_path.clone(),
439 deadline,
440 started,
441 GateKind::CargoTest,
442 )
443 .await?
444 {
445 Ok(v) => v,
446 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
447 };
448 if !status.success() {
449 let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf);
450 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
451 }
452 ran += 1;
453 }
454
455 // Every configured target was missing from the worktree. Exiting green here
456 // would report "tests passed" having run none of them.
457 if ran == 0 {
458 return Ok(GateOutcome::failed(GateFailure::Unclassified {
459 legacy_detail: Some(format!(
460 "cargo_test ran no targets: none of the {} configured test_target dir(s) \
461 exist in this worktree",
462 ctx.cfg.test_targets.len(),
463 )),
464 })
465 .with_log_ref(log_ref));
466 }
467
468 let duration_s = started.elapsed().as_secs() as u32;
469 Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref))
470 }
471
472 /// Stream a child to the live log, bounded by the gate-wide `deadline`. `Ok(Err(_))`
473 /// is the timeout outcome (child killed); `Err(_)` is an IO error on the stream.
474 #[allow(clippy::type_complexity)]
475 async fn run_to_deadline_for(
476 child: &mut tokio::process::Child,
477 ctx: &GateCtx,
478 run_id: GateRunId,
479 log_path: PathBuf,
480 deadline: std::time::Instant,
481 started: std::time::Instant,
482 kind: GateKind,
483 ) -> Result<std::result::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus), GateOutcome>> {
484 let remaining = deadline.saturating_duration_since(std::time::Instant::now());
485 let stream = stream_child_to_live_log(child, ctx.events.clone(), run_id, log_path);
486 match tokio::time::timeout(remaining, stream).await {
487 Ok(res) => Ok(Ok(res?)),
488 Err(_elapsed) => {
489 child.start_kill().ok();
490 let _ = child.wait().await;
491 Ok(Err(GateOutcome::failed(GateFailure::Timeout {
492 gate: kind,
493 after_s: started.elapsed().as_secs() as u32,
494 })))
495 }
496 }
497 }
498
499 /// Prefix a test/compile failure's headline with the crate it came from, so a
500 /// red gate across many targets says *which* crate broke. Other failure kinds
501 /// are single-target by construction and pass through untouched.
502 fn name_target(failure: GateFailure, dir: &std::path::Path) -> GateFailure {
503 let at = dir.display();
504 match failure {
505 GateFailure::CargoTest {
506 failed_count,
507 first_failed,
508 first_panic,
509 } => GateFailure::CargoTest {
510 failed_count,
511 first_failed: Some(match first_failed {
512 Some(name) => format!("{at}: {name}"),
513 None => at.to_string(),
514 }),
515 first_panic,
516 },
517 GateFailure::CompileError {
518 error_count,
519 first_error,
520 } => GateFailure::CompileError {
521 error_count,
522 first_error: Some(match first_error {
523 Some(e) => format!("{at}: {e}"),
524 None => at.to_string(),
525 }),
526 },
527 other => other,
528 }
529 }
530
531 /// Append raw bytes to a gate log outside the child-streaming path (target
532 /// banners). Best-effort, same as `LiveLog`: a broken log dir never turns a
533 /// passing gate red.
534 async fn append_to_log(path: &std::path::Path, bytes: &[u8]) {
535 use tokio::io::AsyncWriteExt;
536 if let Some(parent) = path.parent()
537 && tokio::fs::create_dir_all(parent).await.is_err()
538 {
539 return;
540 }
541 if let Ok(mut f) = tokio::fs::OpenOptions::new()
542 .create(true)
543 .append(true)
544 .open(path)
545 .await
546 {
547 let _ = f.write_all(bytes).await;
548 }
549 }
550
551 /// `cargo clippy --all-targets -- -D warnings` over every configured
552 /// `test_target`.
553 ///
554 /// The only thing standing between lint drift and prod.
555 async fn clippy(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
556 lint_over_targets(ctx, run_id, GateKind::Clippy, |target, features| {
557 let mut args = vec!["clippy".to_string(), "--all-targets".to_string()];
558 if target.all_features {
559 args.push("--all-features".to_string());
560 } else if !features.is_empty() {
561 args.push("--features".to_string());
562 args.push(features.join(","));
563 }
564 // Everything after `--` goes to rustc, which is where -D lives.
565 args.push("--".to_string());
566 args.push("-D".to_string());
567 args.push("warnings".to_string());
568 args
569 })
570 .await
571 }
572
573 /// `cargo fmt --check` over every configured `test_target`.
574 ///
575 /// No `rustfmt.toml` exists anywhere in the tree, so this is plain rustfmt
576 /// defaults. Cheap: no compilation, just a parse.
577 async fn fmt_check(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
578 lint_over_targets(ctx, run_id, GateKind::Fmt, |_target, _features| {
579 vec!["fmt".to_string(), "--check".to_string()]
580 })
581 .await
582 }
583
584 /// `cargo audit` / `cargo deny check`, over the crates that carry the matching
585 /// config file.
586 ///
587 /// Config-gated on purpose. Both tools are only meaningful against a triaged
588 /// posture: four crates in this repo fail `cargo audit` today purely because
589 /// they have no `.cargo/audit.toml` recording which transitive advisories have
590 /// been reviewed and accepted. Running them everywhere would make the gate
591 /// permanently and uninformatively red. Dropping the config file into a crate
592 /// is what opts it in.
593 async fn supply_chain(ctx: &GateCtx, run_id: GateRunId, kind: GateKind) -> Result<GateOutcome> {
594 let (config_rel, args): (&str, Vec<String>) = match kind {
595 GateKind::CargoAudit => (".cargo/audit.toml", vec!["audit".into()]),
596 GateKind::CargoDeny => ("deny.toml", vec!["deny".into(), "check".into()]),
597 other => unreachable!("supply_chain called for {other:?}"),
598 };
599 run_over_targets(ctx, run_id, kind, |_target, target_dir| {
600 target_dir.join(config_rel).is_file().then(|| args.clone())
601 })
602 .await
603 }
604
605 /// Shared driver for the lint gates: run `cargo <args>` in every configured
606 /// `test_target` that exists in this worktree.
607 async fn lint_over_targets(
608 ctx: &GateCtx,
609 run_id: GateRunId,
610 kind: GateKind,
611 build_args: impl Fn(&crate::config::TestTarget, &[String]) -> Vec<String>,
612 ) -> Result<GateOutcome> {
613 // The target is handed in directly. This used to reverse-look-it-up by
614 // comparing `worktree.join(dir)` against the resolved path, which silently
615 // stopped matching for anything resolved anywhere else — an aux repo, say.
616 run_over_targets(ctx, run_id, kind, move |t, _dir| {
617 Some(build_args(t, &t.features))
618 })
619 .await
620 }
621
622 /// Run one cargo invocation per configured `test_target`, under a single
623 /// gate-wide deadline. `args_for` returns `None` to skip a target (used by the
624 /// supply-chain gates, which only apply where their config file lives).
625 ///
626 /// Shares `cargo_test`'s conventions: per-target log banners, stop at the first
627 /// failure with the crate named, skip targets absent from the worktree, and fail
628 /// closed if that leaves nothing to run.
629 async fn run_over_targets(
630 ctx: &GateCtx,
631 run_id: GateRunId,
632 kind: GateKind,
633 args_for: impl Fn(&crate::config::TestTarget, &std::path::Path) -> Option<Vec<String>>,
634 ) -> Result<GateOutcome> {
635 let log_path = ctx.log_path(kind);
636 let log_ref = ctx.log_ref(kind);
637 let started = std::time::Instant::now();
638 let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
639 let mut ran = 0usize;
640
641 for target in &ctx.cfg.test_targets {
642 let label = target.label();
643 let Some(dir) = ctx
644 .target_dir(target)
645 .filter(|d| d.join("Cargo.toml").is_file())
646 else {
647 tracing::warn!(
648 gate = kind.as_str(), target = %label,
649 "target has no Cargo.toml in this run; skipping",
650 );
651 continue;
652 };
653 let Some(args) = args_for(target, &dir) else {
654 continue;
655 };
656
657 append_to_log(
658 &log_path,
659 format!("\n==== {}: {label} ====\n", kind.as_str()).as_bytes(),
660 )
661 .await;
662
663 let mut cmd = Command::new("cargo");
664 cmd.args(&args)
665 .current_dir(&dir)
666 .stdout(std::process::Stdio::piped())
667 .stderr(std::process::Stdio::piped())
668 .kill_on_drop(true);
669 if let Some(t) = ctx.cfg.cargo_target_dir.as_deref() {
670 cmd.env("CARGO_TARGET_DIR", t);
671 }
672 // clippy type-checks, so it needs the same sqlx online-mode env the
673 // build and cargo_test steps get. fmt/audit/deny never touch the DB.
674 if kind == GateKind::Clippy
675 && let Some(url) = ctx
676 .cfg
677 .scratch_db_url
678 .as_deref()
679 .filter(|_| target.scratch_db)
680 {
681 cmd.env("DATABASE_URL", url);
682 cmd.env(
683 "TEST_DATABASE_URL",
684 url.split_once('?').map_or(url, |(b, _)| b),
685 );
686 }
687
688 let mut child = match cmd.spawn() {
689 Ok(c) => c,
690 Err(e) => {
691 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
692 message: format!("{label}: {e}"),
693 })
694 .with_log_ref(log_ref));
695 }
696 };
697 let (stdout_buf, stderr_buf, status) = match run_to_deadline_for(
698 &mut child,
699 ctx,
700 run_id,
701 log_path.clone(),
702 deadline,
703 started,
704 kind,
705 )
706 .await?
707 {
708 Ok(v) => v,
709 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
710 };
711 if !status.success() {
712 let failure = match kind {
713 // clippy speaks rustc diagnostics, so the compile-error
714 // classifier extracts the real `error: ...` headline.
715 GateKind::Clippy => classify::classify_compile_error(&stdout_buf, &stderr_buf),
716 _ => GateFailure::Unclassified {
717 legacy_detail: Some(first_meaningful_line(&stdout_buf, &stderr_buf)),
718 },
719 };
720 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
721 }
722 ran += 1;
723 }
724
725 if ran == 0 {
726 return Ok(GateOutcome::failed(GateFailure::Unclassified {
727 legacy_detail: Some(format!(
728 "{} ran nothing: no configured target in this worktree qualified",
729 kind.as_str(),
730 )),
731 })
732 .with_log_ref(log_ref));
733 }
734 Ok(GateOutcome::passed(PassNote::TestsPassed {
735 duration_s: started.elapsed().as_secs() as u32,
736 })
737 .with_log_ref(log_ref))
738 }
739
740 /// First line that looks like a diagnostic, for gates whose tools have no
741 /// dedicated classifier (`cargo audit`, `cargo deny`). Falls back to a generic
742 /// note rather than an empty string.
743 fn first_meaningful_line(stdout: &[u8], stderr: &[u8]) -> String {
744 for buf in [stderr, stdout] {
745 let text = String::from_utf8_lossy(buf);
746 if let Some(line) = text
747 .lines()
748 .map(str::trim)
749 .find(|l| l.starts_with("error") || l.contains("vulnerabilit") || l.contains("FAILED"))
750 {
751 return line.chars().take(200).collect();
752 }
753 }
754 "tool reported failure; see the gate log".into()
755 }
756
757 /// The tests `cargo_test` cannot reach, run against production constants.
758 ///
759 /// `cargo_test` builds with `--features fast-tests`, which relaxes
760 /// `AUTH_RATE_LIMIT_BURST` 5 → 20, `SANDBOX_RATE_LIMIT_MS` 30s → 10ms, and
761 /// argon2 from 46 MiB/t=2 to 8 MiB/t=1 — and the rate-limiting suite is
762 /// `#[cfg_attr(feature = "fast-tests", ignore)]`d on top of that, because a
763 /// bucket refilling at 100/sec never depletes under parallel test threads. The
764 /// net effect was that Sando's only code gate silently skipped every test of
765 /// the auth hardening it most needs to protect.
766 ///
767 /// So: no features, `--test-threads=1` (these tests key on a shared per-IP
768 /// bucket and must not interleave), and a name filter rather than the whole
769 /// suite — the rest of the suite is tuned for `fast-tests` and would only go
770 /// slow and flaky here. The filter is a substring match, so it catches
771 /// `..._rate_limit_...` and `..._rate_limited` alike.
772 ///
773 /// This costs a second compile of the lib + integration binary (a different
774 /// feature set is a different cfg, so no artifact sharing with `cargo_test`).
775 /// That is the price of the coverage; the filter keeps the *run* to seconds.
776 async fn hardening_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
777 let server_dir = match ctx.worktree_for(GateKind::HardeningTest) {
778 Ok(w) => w.join("server"),
779 Err(outcome) => return Ok(outcome),
780 };
781 // No features is the whole point; the scratch DB is needed because the
782 // server's sqlx macros type-check against it. Unlike cargo_test, this gate
783 // is deliberately not driven by `test_targets`: it targets one specific
784 // suite in one specific crate, not "the repo's tests".
785 let target = crate::config::TestTarget {
786 dir: std::path::PathBuf::from("server"),
787 aux_repo: None,
788 features: Vec::new(),
789 all_features: false,
790 scratch_db: true,
791 };
792 let log_path = ctx.log_path(GateKind::HardeningTest);
793 let log_ref = ctx.log_ref(GateKind::HardeningTest);
794
795 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
796 clean_stale_test_dbs(scratch_url).await;
797 }
798
799 let started = std::time::Instant::now();
800
801 // Same two-step shape as cargo_test: compile first so a test-target break
802 // reports as a compile error rather than an opaque mass test failure.
803 let mut pre = match cargo_test_command(
804 ctx,
805 &server_dir,
806 &target,
807 &[],
808 &["--no-run", "--test", "integration"],
809 )
810 .spawn()
811 {
812 Ok(c) => c,
813 Err(e) => {
814 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
815 message: e.to_string(),
816 })
817 .with_log_ref(log_ref));
818 }
819 };
820 let (pre_out, pre_err, pre_status) =
821 stream_child_to_live_log(&mut pre, ctx.events.clone(), run_id, log_path.clone()).await?;
822 if !pre_status.success() {
823 let failure = classify::classify_compile_error(&pre_out, &pre_err);
824 return Ok(GateOutcome::failed(failure).with_log_ref(log_ref));
825 }
826
827 let mut child = match cargo_test_command(
828 ctx,
829 &server_dir,
830 &target,
831 &[],
832 &[
833 "--test",
834 "integration",
835 "--",
836 "--test-threads=1",
837 "rate_limit",
838 ],
839 )
840 .spawn()
841 {
842 Ok(c) => c,
843 Err(e) => {
844 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
845 message: e.to_string(),
846 })
847 .with_log_ref(log_ref));
848 }
849 };
850 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
851 let stream = stream_child_to_live_log(&mut child, ctx.events.clone(), run_id, log_path);
852 let (stdout_buf, stderr_buf, status) = match tokio::time::timeout(ceiling, stream).await {
853 Ok(res) => res?,
854 Err(_elapsed) => {
855 child.start_kill().ok();
856 let _ = child.wait().await;
857 return Ok(GateOutcome::failed(GateFailure::Timeout {
858 gate: GateKind::HardeningTest,
859 after_s: started.elapsed().as_secs() as u32,
860 })
861 .with_log_ref(log_ref));
862 }
863 };
864 let duration_s = started.elapsed().as_secs() as u32;
865 if status.success() {
866 // A filter that matches nothing exits 0, which would make this gate a
867 // green no-op the day someone renames the tests. Fail closed instead.
868 if tests_run(&stdout_buf) == 0 {
869 return Ok(GateOutcome::failed(GateFailure::Unclassified {
870 legacy_detail: Some(
871 "hardening_test ran 0 tests: the `rate_limit` filter matched nothing. \
872 The suite was renamed or moved — this gate is proving nothing."
873 .into(),
874 ),
875 })
876 .with_log_ref(log_ref));
877 }
878 Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref))
879 } else {
880 let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf);
881 Ok(GateOutcome::failed(failure).with_log_ref(log_ref))
882 }
883 }
884
885 /// Count the tests libtest reports as run, from its `test result: ok. N passed`
886 /// summary line. Returns 0 when no summary is present, which is itself the
887 /// "nothing ran" case the caller fails on.
888 fn tests_run(stdout: &[u8]) -> u32 {
889 String::from_utf8_lossy(stdout)
890 .lines()
891 .filter_map(|l| l.trim().strip_prefix("test result:"))
892 .filter_map(|rest| rest.split_once(" passed"))
893 .filter_map(|(head, _)| {
894 head.rsplit(' ')
895 .find(|t| !t.is_empty())?
896 .parse::<u32>()
897 .ok()
898 })
899 .sum()
900 }
901
902 /// Configure (but don't spawn) `cargo test --release [--features <features>]
903 /// <extra>` in `dir`, wired to the scratch DB. Shared by the `--no-run`
904 /// pre-gate compile and the full test run so both go through one env setup,
905 /// and by both test gates so they can't drift apart on env.
906 ///
907 /// `cargo_test` passes `fast-tests`, matching what the retired astra CI did:
908 /// it relaxes the auth rate-limit burst (5 → 20) and argon2 cost so
909 /// signup-heavy + lockout workflow tests complete without hitting Governor
910 /// before the hand-rolled lockout check (documented at
911 /// `server/src/constants.rs:87`). `hardening_test` passes no features, which is
912 /// the whole point of that gate — see `GateKind::HardeningTest`.
913 fn cargo_test_command(
914 ctx: &GateCtx,
915 dir: &std::path::Path,
916 target: &crate::config::TestTarget,
917 features: &[&str],
918 extra: &[&str],
919 ) -> Command {
920 let mut cmd = Command::new("cargo");
921 cmd.args(["test", "--release"]);
922 if target.all_features {
923 cmd.arg("--all-features");
924 } else if !features.is_empty() {
925 cmd.args(["--features", &features.join(",")]);
926 }
927 cmd.args(extra)
928 .current_dir(dir)
929 .stdout(std::process::Stdio::piped())
930 .stderr(std::process::Stdio::piped())
931 .kill_on_drop(true);
932 // Share the build step's target dir so the test compile reuses its
933 // artifacts (and the `--no-run` precompile reuses them again). Must match
934 // `build.rs` or the gate would clean-compile the whole tree a second time.
935 if let Some(target) = ctx.cfg.cargo_target_dir.as_deref() {
936 cmd.env("CARGO_TARGET_DIR", target);
937 }
938 // Same online-mode rationale as the build step: sqlx query macros need a
939 // live DB to type-check against. The scratch DB is left in migrated state
940 // by the preceding build, so we can reuse it here.
941 //
942 // Opt-in per target: setting DATABASE_URL switches sqlx OUT of offline mode,
943 // so a crate that ships `.sqlx` query data would stop using it and try to
944 // type-check against a database that has none of its tables.
945 if let Some(scratch_url) = ctx
946 .cfg
947 .scratch_db_url
948 .as_deref()
949 .filter(|_| target.scratch_db)
950 {
951 cmd.env("DATABASE_URL", scratch_url);
952 // The server test harness (tests/harness/db.rs) parses TEST_DATABASE_URL
953 // with rfind('/'), which mangles URLs whose query string contains '/'
954 // (e.g. `?host=/var/run/postgresql`). Strip the query — libpq defaults
955 // to /var/run/postgresql on Debian/Ubuntu when host is unspecified.
956 let test_url = scratch_url
957 .split_once('?')
958 .map_or(scratch_url, |(base, _)| base);
959 cmd.env("TEST_DATABASE_URL", test_url);
960 }
961 cmd
962 }
963
964 async fn migration_dry_run(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
965 let log = GateLog::open(ctx, run_id, GateKind::MigrationDryRun).await;
966 let outcome = migration_dry_run_inner(ctx, &log).await;
967 log.close().await;
968 outcome.map(|o| o.with_log_ref(ctx.log_ref(GateKind::MigrationDryRun)))
969 }
970
971 /// The staged interior of [`migration_dry_run`], writing every step through the
972 /// gate's live log. The caller owns the sink so it can flush it on every exit
973 /// path, and attaches the `log_ref` once instead of at each return.
974 /// Runs one configured check per database, in config order, and stops at the
975 /// first that does not pass — a red gate is a red gate, and continuing would
976 /// bury it under a second restore's output.
977 ///
978 /// The server's check runs against `scratch_db_url` itself and is deliberately
979 /// last-writer for it: `cargo_test` reuses that database in migrated state, so
980 /// every other check must name its own `scratch_db` (enforced at config load).
981 async fn migration_dry_run_inner(ctx: &GateCtx, log: &GateLog) -> Result<GateOutcome> {
982 let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() else {
983 log.line("scratch_db_url unset in daemon config\n").await;
984 return Ok(GateOutcome::blocked(GateBlocker::ScratchDbUrlUnset));
985 };
986
987 let mut checked = Vec::new();
988 let mut primary_backup_path = String::new();
989 for check in &ctx.cfg.migration_checks {
990 let label = check.dir.display().to_string();
991 log.line(&format!("==== migration_check: {label} ====\n"))
992 .await;
993 match run_migration_check(ctx, log, scratch_url, check).await? {
994 CheckResult::Passed { backup_path } => {
995 if primary_backup_path.is_empty() {
996 primary_backup_path = backup_path;
997 }
998 checked.push(label);
999 }
1000 CheckResult::Stopped(outcome) => return Ok(outcome),
1001 }
1002 }
1003
1004 log.line(&format!(
1005 "all {} migration check(s) passed: {}",
1006 checked.len(),
1007 checked.join(", ")
1008 ))
1009 .await;
1010 Ok(GateOutcome::passed(PassNote::Migrated {
1011 backup_path: primary_backup_path,
1012 checks: checked,
1013 }))
1014 }
1015
1016 /// One check's verdict: it passed (against `backup_path`), or it produced the
1017 /// outcome the whole gate reports.
1018 enum CheckResult {
1019 Passed { backup_path: String },
1020 Stopped(GateOutcome),
1021 }
1022
1023 /// Restore one database's dump into its scratch DB and run its migrations on top.
1024 async fn run_migration_check(
1025 ctx: &GateCtx,
1026 log: &GateLog,
1027 scratch_url: &str,
1028 check: &crate::config::MigrationCheck,
1029 ) -> Result<CheckResult> {
1030 let label = check.dir.display().to_string();
1031
1032 let backup: Option<(String, String)> = sqlx::query_as(
1033 "SELECT local_path, fetched_at FROM backups
1034 WHERE app = ? AND name = ? ORDER BY id DESC LIMIT 1",
1035 )
1036 .bind(&ctx.cfg.id)
1037 .bind(&check.backup)
1038 .fetch_optional(&ctx.pool)
1039 .await?;
1040 let Some((backup_path, fetched_at)) = backup else {
1041 log.line(&format!(
1042 "no {} backup fetched; call /backup/fetch first\n",
1043 check.backup
1044 ))
1045 .await;
1046 return Ok(CheckResult::Stopped(GateOutcome::blocked(
1047 GateBlocker::NoBackupAvailable {
1048 check: label,
1049 backup: check.backup.clone(),
1050 },
1051 )));
1052 };
1053
1054 // Presence is not freshness. A fetch that quietly stopped working leaves this
1055 // row in place, and restoring it dry-runs the migrations against a schema prod
1056 // has moved past — green, and worthless. Block on age instead. An unparsable
1057 // timestamp is treated as stale: this row is daemon-written RFC 3339, so a
1058 // value that will not parse means something is wrong, and failing closed on a
1059 // freshness check is the whole point.
1060 let age_hours = chrono::DateTime::parse_from_rfc3339(&fetched_at).map_or(i64::MAX, |t| {
1061 (Utc::now() - t.with_timezone(&Utc)).num_hours()
1062 });
1063 let max_age_hours = ctx.cfg.backup_max_age_hours;
1064 if age_hours > i64::from(max_age_hours) {
1065 let msg = format!(
1066 "backup {backup_path} was fetched {fetched_at} ({age_hours}h ago, max \
1067 {max_age_hours}h); re-run /backup/fetch\n"
1068 );
1069 log.line(&msg).await;
1070 return Ok(CheckResult::Stopped(GateOutcome::blocked(
1071 GateBlocker::BackupStale {
1072 age_hours,
1073 max_age_hours,
1074 check: label,
1075 },
1076 )));
1077 }
1078
1079 // A check with its own `scratch_db` gets that database created here rather
1080 // than by a host bootstrap step: adding a `[[migration_check]]` should not
1081 // silently depend on someone having remembered to `createdb` on the Sando
1082 // host, which is exactly the class of footgun this gate exists to remove.
1083 // DROP + CREATE also makes the database sando-owned, so the PG15+ public
1084 // schema grants `reset_scratch` applies next are the owner's to give.
1085 let db_url = match check.scratch_db.as_deref() {
1086 None => scratch_url.to_string(),
1087 Some(dbname) => {
1088 let maintenance_url = pg_url_with_dbname(scratch_url, "postgres");
1089 log.line(&format!("---- create scratch db {dbname} ----\n"))
1090 .await;
1091 if let Err(e) = pg_create_db(&maintenance_url, dbname).await {
1092 let msg = format!("{label}: creating scratch db {dbname}: {e}");
1093 log.line(&msg).await;
1094 return Ok(CheckResult::Stopped(GateOutcome::failed(
1095 GateFailure::RestoreFailed { reason: msg },
1096 )));
1097 }
1098 pg_url_with_dbname(scratch_url, dbname)
1099 }
1100 };
1101
1102 let owner_role = check
1103 .owner_role
1104 .as_deref()
1105 .unwrap_or(&ctx.cfg.scratch_owner_role);
1106 log.line("---- reset_scratch ----\n").await;
1107 if let Err(e) = reset_scratch(&db_url, owner_role).await {
1108 let msg = format!("{label}: scratch reset: {e}");
1109 log.line(&msg).await;
1110 return Ok(CheckResult::Stopped(GateOutcome::failed(
1111 GateFailure::RestoreFailed { reason: msg },
1112 )));
1113 }
1114 log.line(&format!("---- restore_dump ({backup_path}) ----\n"))
1115 .await;
1116 if let Err(e) = restore_dump(&db_url, &backup_path, log).await {
1117 let msg = format!("{label}: restore: {e}");
1118 log.line(&msg).await;
1119 return Ok(CheckResult::Stopped(GateOutcome::failed(
1120 GateFailure::RestoreFailed { reason: msg },
1121 )));
1122 }
1123
1124 let Some(migrations_dir) = ctx.migrations_dir(&check.dir) else {
1125 // Neither the bundle nor a checkout holds them. For an accepted
1126 // artifact that means the builder did not ship its migrations, and a
1127 // dry run over nothing would report green having proved nothing.
1128 let msg = format!(
1129 "{label}: no migrations at {} in the bundle or a checkout",
1130 check.dir.display()
1131 );
1132 log.line(&msg).await;
1133 return Ok(CheckResult::Stopped(GateOutcome::failed(
1134 GateFailure::RestoreFailed { reason: msg },
1135 )));
1136 };
1137 log.line("---- run_migrator ----\n").await;
1138 match run_migrator(&db_url, &migrations_dir).await {
1139 Ok(()) => {
1140 log.line(&format!("{label}: restored {backup_path} + migrated\n"))
1141 .await;
1142 Ok(CheckResult::Passed { backup_path })
1143 }
1144 Err(e) => {
1145 let err_s = format!("{label}: {e}");
1146 log.line(&err_s).await;
1147 Ok(CheckResult::Stopped(GateOutcome::failed(
1148 classify::classify_migration_error(&err_s, None),
1149 )))
1150 }
1151 }
1152 }
1153
1154 pub(crate) async fn reset_scratch(db_url: &str, owner_role: &str) -> Result<()> {
1155 use sqlx::Executor;
1156 use sqlx::postgres::PgPoolOptions;
1157 let pool = PgPoolOptions::new()
1158 .max_connections(1)
1159 .connect(db_url)
1160 .await?;
1161 // `owner_role` is validated `[A-Za-z0-9_]+` at config load, so interpolating
1162 // it into DDL is sound. It still goes through `format('%I')` inside the DO
1163 // block for the quoting Postgres expects on an identifier.
1164 let sql = format!(
1165 r#"
1166 DO $$
1167 DECLARE s text;
1168 BEGIN
1169 -- The dump restores objects owned by the prod role and re-grants to
1170 -- it (`ALTER ... OWNER TO {owner_role}`), which errors if the role
1171 -- is absent — superuser does not imply the role exists. Create it
1172 -- NOLOGIN: the scratch DB needs the role as an *owner* only, never
1173 -- as a connecting identity. Idempotent, so a re-reset is a no-op.
1174 IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{owner_role}') THEN
1175 EXECUTE format('CREATE ROLE %I NOLOGIN', '{owner_role}');
1176 END IF;
1177
1178 -- Drop every non-system schema, not just public — migrations create
1179 -- custom schemas (e.g. tower_sessions) that survive `DROP SCHEMA
1180 -- public CASCADE` and then collide on the next migration run.
1181 FOR s IN
1182 SELECT nspname FROM pg_namespace
1183 WHERE nspname NOT LIKE 'pg_%'
1184 AND nspname NOT IN ('information_schema')
1185 LOOP
1186 EXECUTE format('DROP SCHEMA IF EXISTS %I CASCADE', s);
1187 END LOOP;
1188 EXECUTE 'CREATE SCHEMA public';
1189 -- Restore the pre-PG15 public-schema default on the throwaway
1190 -- scratch DB. Without this, the freshly-created public is owned by
1191 -- the connecting role (sando) with no grant to anyone else, so a
1192 -- migration's FK/trigger check that Postgres runs as a *restored*
1193 -- prod-owned table's owner ({owner_role} from the backup dump)
1194 -- fails with "permission denied for schema public". Granting to
1195 -- PUBLIC is role-agnostic and safe here — this DB is disposable and
1196 -- exists only to dry-run migrations.
1197 EXECUTE 'GRANT USAGE, CREATE ON SCHEMA public TO PUBLIC';
1198 -- PG15+: the new owner needs CREATE on public in its own right, not
1199 -- only via PUBLIC, for the restore's owner-scoped DDL.
1200 EXECUTE format('GRANT USAGE, CREATE ON SCHEMA public TO %I', '{owner_role}');
1201 END $$;
1202 "#
1203 );
1204 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(sql)))
1205 .await?;
1206 pool.close().await;
1207 Ok(())
1208 }
1209
1210 /// Startup assertion for the scratch cluster: the gates reset it, seed an owner
1211 /// role into it, and drop leftover test databases in it, none of which a plain
1212 /// unprivileged role can do. Satisfied by hand on the build host
1213 /// (`ALTER ROLE sando SUPERUSER`, a created `makenotwork` role); unasserted, a
1214 /// rebuild elsewhere fails one gate at a time with an opaque permissions error.
1215 /// Assert once, at boot, loudly.
1216 ///
1217 /// Not part of `--check-config`: that path is pure by design (no DB, no
1218 /// network), and a green there must mean "this build understands its config",
1219 /// not "the cluster is reachable".
1220 pub async fn preflight_scratch_privileges(db_url: &str) -> Result<()> {
1221 use sqlx::postgres::PgPoolOptions;
1222 let pool = PgPoolOptions::new()
1223 .max_connections(1)
1224 .connect(db_url)
1225 .await
1226 .context("connecting to scratch_db_url for the startup privilege check")?;
1227 let (is_super, can_signal): (bool, bool) = sqlx::query_as(
1228 "SELECT rolsuper, pg_catalog.pg_has_role(current_user, 'pg_signal_backend', 'USAGE')
1229 FROM pg_roles WHERE rolname = current_user",
1230 )
1231 .fetch_one(&pool)
1232 .await?;
1233 pool.close().await;
1234 anyhow::ensure!(
1235 is_super || can_signal,
1236 "the scratch_db_url role has neither SUPERUSER nor pg_signal_backend; migration_dry_run \
1237 and cargo_test cannot reset the scratch DB or clear stale test databases. Grant one:\n \
1238 ALTER ROLE <role> SUPERUSER; -- what fw13 uses\n \
1239 GRANT pg_signal_backend TO <role>; -- narrower: terminate only, cannot drop \
1240 foreign-owned databases",
1241 );
1242 if !is_super {
1243 tracing::warn!(
1244 "scratch role has pg_signal_backend but not SUPERUSER: stale test databases owned by \
1245 another role cannot be dropped, and the scratch owner role cannot be created if absent"
1246 );
1247 }
1248 Ok(())
1249 }
1250
1251 /// Best-effort cleanup of stale per-test database clones (`mnw_test_<uuid>`)
1252 /// left behind by a killed `cargo_test` run.
1253 ///
1254 /// Drops **foreign-owned leftovers too**, which is why the daemon asserts
1255 /// SUPERUSER at startup (`preflight_scratch_privileges`): `DROP DATABASE`
1256 /// requires ownership or superuser, and the `WITH (FORCE)` terminate requires
1257 /// superuser or `pg_signal_backend`. Without both, orphans from a run under a
1258 /// different role accumulate and degrade the gate — the failure this cleanup
1259 /// exists to prevent.
1260 ///
1261 /// OPERATIONAL HAZARD: fw13 runs one Postgres cluster shared with local `cargo
1262 /// test` as `max`, so a gate firing mid-local-test will force-drop that run's
1263 /// databases out from under it. That collision is known and tracked separately
1264 /// (give the gate its own cluster); until then, do not run local tests on fw13
1265 /// while a Sando gate is live.
1266 ///
1267 /// Deliberately **excludes the template** (`mnw_test_template_*`): the harness
1268 /// reuses it across runs when it's migration-current (skipping a full
1269 /// drop+migrate), so dropping it here would force a needless rebuild every
1270 /// gate run. Templates are bounded (one per role) and never accumulate, so
1271 /// leaving them is free. Never returns an error: a cleanup miss must not turn a
1272 /// deploy red.
1273 async fn clean_stale_test_dbs(db_url: &str) {
1274 use sqlx::Executor;
1275 use sqlx::postgres::PgPoolOptions;
1276 let pool = match PgPoolOptions::new()
1277 .max_connections(1)
1278 .connect(db_url)
1279 .await
1280 {
1281 Ok(p) => p,
1282 Err(e) => {
1283 tracing::warn!(error = %e, "stale test-db cleanup: could not connect; skipping");
1284 return;
1285 }
1286 };
1287 // Every per-test clone, whoever owns it. The ownership filter this used to
1288 // carry is what let foreign-owned orphans pile up; superuser (asserted at
1289 // startup) makes them droppable.
1290 let names: Vec<(String,)> = sqlx::query_as(
1291 "SELECT datname FROM pg_database
1292 WHERE datname LIKE 'mnw_test_%'
1293 AND datname NOT LIKE '%template%'",
1294 )
1295 .fetch_all(&pool)
1296 .await
1297 .unwrap_or_default();
1298 let count = names.len();
1299 for (name,) in names {
1300 // `name` comes straight from pg_database; quoting it is sufficient.
1301 if let Err(e) = pool
1302 .execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1303 "DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)"
1304 ))))
1305 .await
1306 {
1307 tracing::warn!(error = %e, db = %name, "stale test-db cleanup: drop failed");
1308 }
1309 }
1310 if count > 0 {
1311 tracing::info!(
1312 count,
1313 "stale test-db cleanup: dropped leftover mnw_test_* databases"
1314 );
1315 }
1316 pool.close().await;
1317 }
1318
1319 /// Build the restore shell line. Two pipelines we accept:
1320 /// *.sql -> psql -v ON_ERROR_STOP=1 $url < dump
1321 /// *.sql.gz -> set -o pipefail; gunzip -c dump | psql -v ON_ERROR_STOP=1 $url
1322 ///
1323 /// Two safety flags are load-bearing (CF4):
1324 /// - `ON_ERROR_STOP=1`: without it, psql exits 0 even when individual statements
1325 /// error, so a partial/corrupt restore would *pass* the gate.
1326 /// - `set -o pipefail`: without it a shell pipeline reports only the last
1327 /// command's status, so a `gunzip` failure on a truncated archive is masked by
1328 /// psql's exit. pipefail is a bash builtin (not POSIX sh), so the runner uses
1329 /// `bash -c`.
1330 fn restore_shell(db_url: &str, dump: &str) -> String {
1331 if std::path::Path::new(dump)
1332 .extension()
1333 .is_some_and(|ext| ext.eq_ignore_ascii_case("gz"))
1334 {
1335 format!(
1336 "set -o pipefail; gunzip -c {q} | psql -v ON_ERROR_STOP=1 {url}",
1337 q = shell_escape(dump),
1338 url = shell_escape(db_url),
1339 )
1340 } else {
1341 format!(
1342 "psql -v ON_ERROR_STOP=1 {url} < {q}",
1343 url = shell_escape(db_url),
1344 q = shell_escape(dump),
1345 )
1346 }
1347 }
1348
1349 async fn restore_dump(db_url: &str, dump: &str, log: &GateLog) -> Result<()> {
1350 // Split the password out of the URL and hand it to psql via PGPASSWORD, so it
1351 // never lands in argv (visible in /proc/<pid>/cmdline to any local user).
1352 // The sanitized URL — user/host/db, no secret — goes on the command line.
1353 let (sanitized, password) = split_pg_password(db_url);
1354 let shell = restore_shell(&sanitized, dump);
1355 // `bash` (not `sh`): `set -o pipefail` is a bash builtin. The restore runs
1356 // locally on the Sando host (fw13), which has bash.
1357 let mut cmd = Command::new("bash");
1358 cmd.arg("-c").arg(&shell);
1359 // kill_on_drop so the gate's wall-clock ceiling (dispatcher-level timeout on
1360 // migration_dry_run) can't orphan a wedged psql restore.
1361 cmd.kill_on_drop(true);
1362 if let Some(pw) = password {
1363 cmd.env("PGPASSWORD", pw);
1364 }
1365 // Streamed, not `.output()`: a prod-sized restore runs for minutes, and
1366 // psql's progress is the only thing an operator has to watch during it.
1367 let (_stdout, stderr, status) = log.run(&mut cmd).await?;
1368 anyhow::ensure!(
1369 status.success(),
1370 "restore failed: {}",
1371 String::from_utf8_lossy(&stderr),
1372 );
1373 Ok(())
1374 }
1375
1376 /// Split a `postgres://user:password@host/db` URL into its password-free form and
1377 /// the (percent-decoded) password. Returns the URL unchanged with `None` when
1378 /// there is no userinfo password. psql reads the password from `PGPASSWORD`, so
1379 /// keeping it off the command line removes the /proc exposure.
1380 fn split_pg_password(db_url: &str) -> (String, Option<String>) {
1381 let Some(after) = db_url.find("://").map(|i| i + 3) else {
1382 return (db_url.to_string(), None);
1383 };
1384 // The authority ends at the first '/', '?' or '#'; the password (if any) is
1385 // between the first ':' and the '@' within the userinfo of that authority.
1386 let authority_end = db_url[after..]
1387 .find(['/', '?', '#'])
1388 .map_or(db_url.len(), |i| after + i);
1389 let Some(at) = db_url[after..authority_end].find('@').map(|i| after + i) else {
1390 return (db_url.to_string(), None);
1391 };
1392 let userinfo = &db_url[after..at];
1393 let Some(colon) = userinfo.find(':') else {
1394 return (db_url.to_string(), None);
1395 };
1396 let password = percent_decode(&userinfo[colon + 1..]);
1397 let sanitized = format!(
1398 "{}{}{}",
1399 &db_url[..after],
1400 &userinfo[..colon],
1401 &db_url[at..]
1402 );
1403 (sanitized, Some(password))
1404 }
1405
1406 /// Minimal `%XX` percent-decode for a URL userinfo component. Non-escape bytes
1407 /// pass through; a malformed escape is left literal.
1408 fn percent_decode(s: &str) -> String {
1409 let b = s.as_bytes();
1410 let mut out = Vec::with_capacity(b.len());
1411 let mut i = 0;
1412 while i < b.len() {
1413 if b[i] == b'%'
1414 && i + 2 < b.len()
1415 && let (Some(h), Some(l)) = (hex_val(b[i + 1]), hex_val(b[i + 2]))
1416 {
1417 out.push((h << 4) | l);
1418 i += 3;
1419 } else {
1420 out.push(b[i]);
1421 i += 1;
1422 }
1423 }
1424 String::from_utf8_lossy(&out).into_owned()
1425 }
1426
1427 fn hex_val(c: u8) -> Option<u8> {
1428 match c {
1429 b'0'..=b'9' => Some(c - b'0'),
1430 b'a'..=b'f' => Some(c - b'a' + 10),
1431 b'A'..=b'F' => Some(c - b'A' + 10),
1432 _ => None,
1433 }
1434 }
1435
1436 pub(crate) async fn run_migrator(db_url: &str, dir: &std::path::Path) -> Result<()> {
1437 use sqlx::postgres::PgPoolOptions;
1438 let pool = PgPoolOptions::new()
1439 .max_connections(1)
1440 .connect(db_url)
1441 .await?;
1442 let migrator = sqlx::migrate::Migrator::new(dir).await?;
1443 migrator.run(&pool).await?;
1444 pool.close().await;
1445 Ok(())
1446 }
1447
1448 fn shell_escape(s: &str) -> String {
1449 format!("'{}'", s.replace('\'', "'\\''"))
1450 }
1451
1452 // ---- code_smoke ----
1453
1454 /// A 32+ char throwaway signing secret for the `code_smoke` boot. The server
1455 /// only enforces length (>= 32) outside production, and code_smoke's loopback
1456 /// `HOST_URL` keeps it in dev mode, so the value is irrelevant beyond that.
1457 const CODE_SMOKE_SIGNING_SECRET: &str = "sando-code-smoke-dummy-signing-secret-0000000000";
1458
1459 /// Seconds to wait for the real server to come up and serve `GET /health`
1460 /// during `code_smoke`. Longer than `boot_smoke`'s 3s: this boots the *full*
1461 /// server (config, pool, session store, webauthn, doc load, app build), not the
1462 /// minimal no-DB smoke server. The whole gate is also bounded by
1463 /// `gate_timeout_secs` at the dispatcher.
1464 const CODE_SMOKE_READY_SECS: u64 = 30;
1465
1466 /// `code_smoke` — the first host gate. Boots the freshly-built binary against a
1467 /// throwaway *empty* DB it migrates from scratch and seeds the example catalog
1468 /// into, then proves the real server serves `GET /health` against that
1469 /// nonempty DB. Fast and infra-light (one local Postgres, no prod-dump restore,
1470 /// no scratch-role reset, no external services), so a green here proves the
1471 /// code is sound and isolates a later `cargo_test`/`migration_dry_run` red as an
1472 /// environment problem rather than a code one.
1473 ///
1474 /// Reuses existing binary entrypoints, so the server needs no smoke-specific
1475 /// mode: `<bin> --seed-examples` loads config, connects, migrates from scratch,
1476 /// seeds, and exits; a plain `<bin>` then serves the real app. Both run with CWD
1477 /// at the server crate root so `docs/business/assumptions.toml` + `site-docs/`
1478 /// resolve (a missing assumptions file panics real startup), and with a loopback
1479 /// `HOST_URL` so config stays in dev mode (no CDN/S3/signing-secret prod
1480 /// enforcement). The seed's host allowlist already admits `127.0.0.1`, and the
1481 /// fresh DB trivially satisfies its no-real-users guard.
1482 async fn code_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
1483 let log = GateLog::open(ctx, run_id, GateKind::CodeSmoke).await;
1484 let outcome = code_smoke_inner(ctx, &log).await;
1485 log.close().await;
1486 outcome.map(|o| o.with_log_ref(ctx.log_ref(GateKind::CodeSmoke)))
1487 }
1488
1489 /// The staged interior of [`code_smoke`], writing every step through the gate's
1490 /// live log. Same split as [`migration_dry_run_inner`]: the caller owns the sink
1491 /// and attaches the `log_ref`.
1492 async fn code_smoke_inner(ctx: &GateCtx, log: &GateLog) -> Result<GateOutcome> {
1493 let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() else {
1494 log.line("scratch_db_url unset in daemon config\n").await;
1495 return Ok(GateOutcome::blocked(GateBlocker::ScratchDbUrlUnset));
1496 };
1497
1498 // The staged binary (set by build_and_run_host before gating). code_smoke
1499 // runs first among the host gates, but staging precedes all gating, so the
1500 // artifact path is already recorded.
1501 let bin: Option<(String,)> =
1502 sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
1503 .bind(&ctx.cfg.id)
1504 .bind(&ctx.version)
1505 .fetch_optional(&ctx.pool)
1506 .await?;
1507 let Some((bin,)) = bin else {
1508 return Ok(GateOutcome::blocked(GateBlocker::ArtifactMissing {
1509 version: ctx.version.clone(),
1510 }));
1511 };
1512
1513 // Frontend builds, before anything else: they need no DB and no staged
1514 // binary, and a `tsc` error is the one failure the Rust build deliberately
1515 // swallows (both MNW build scripts emit `cargo::warning` and succeed against
1516 // a stale `static/dist/`). Failing here is what stops the deploy rsyncing
1517 // the previous build's bundle.
1518 if let Some(outcome) = code_smoke_frontends(ctx, log).await {
1519 return Ok(outcome);
1520 }
1521
1522 // Docs integrity, first and cheapest: run the staged binary's DB-free
1523 // `MNW_CHECK_DOCS` mode before creating the throwaway DB. A broken internal
1524 // docs link (a `[..](x.md)` resolving to a slug no page serves) fails here
1525 // in well under a second instead of after a full migrate+seed+boot, and a
1526 // rotted link never reaches prod as a live 404. Collisions are reported by
1527 // the check but do not fail it; only broken links do.
1528 if let Some(outcome) = code_smoke_docs_check(ctx, &bin, log).await {
1529 return Ok(outcome);
1530 }
1531
1532 let dbname = code_smoke_db_name(&ctx.version);
1533 let maintenance_url = pg_url_with_dbname(scratch_url, "postgres");
1534 let throwaway_url = pg_url_with_dbname(scratch_url, &dbname);
1535
1536 // Create the throwaway DB (dropping any stale one from a killed prior run).
1537 log.line(&format!("---- createdb {dbname} ----\n")).await;
1538 if let Err(e) = pg_create_db(&maintenance_url, &dbname).await {
1539 let reason = format!("createdb {dbname}: {e}");
1540 log.line(&reason).await;
1541 return Ok(GateOutcome::failed(GateFailure::CodeSmokeSetup { reason }));
1542 }
1543
1544 // Everything past createdb must drop the DB on the way out, pass or fail.
1545 let outcome = code_smoke_body(ctx, &bin, &throwaway_url, log).await;
1546
1547 log.line(&format!("\n---- dropdb {dbname} ----\n")).await;
1548 if let Err(e) = pg_drop_db(&maintenance_url, &dbname).await {
1549 // A teardown miss must not turn a passing gate red — log it and move on.
1550 // The next run's createdb drops it first anyway.
1551 tracing::warn!(error = %e, db = %dbname, "code_smoke: dropdb failed; next run will reclaim it");
1552 log.line(&format!("dropdb warning (non-fatal): {e}")).await;
1553 }
1554
1555 Ok(outcome)
1556 }
1557
1558 /// Compile every configured `frontend_build` in the worktree.
1559 ///
1560 /// Returns `Some(failed)` on the first project that does not build; `None` when
1561 /// all of them do (or none are configured). Output streams to `log` either way.
1562 ///
1563 /// `npm ci` runs only when `node_modules` is absent. Usually it is not: the app
1564 /// build script installed it during the `cargo build` that produced the artifact
1565 /// this gate is about to smoke, so the common path here is just `npm run build`
1566 /// against a warm install — seconds. The install branch covers the gate running
1567 /// against a worktree whose build script was skipped or failed at `npm ci`, and
1568 /// it is as fatal as a compile failure, because the alternative is compiling
1569 /// against whatever some earlier sha installed.
1570 ///
1571 /// Unlike the app build scripts, nothing here is best-effort. That asymmetry is
1572 /// the point of the gate.
1573 async fn code_smoke_frontends(ctx: &GateCtx, log: &GateLog) -> Option<GateOutcome> {
1574 if ctx.cfg.frontend_builds.is_empty() {
1575 return None;
1576 }
1577 let worktree = match ctx.worktree_for(GateKind::CodeSmoke) {
1578 Ok(w) => w.to_path_buf(),
1579 Err(outcome) => return Some(outcome),
1580 };
1581 for fe in &ctx.cfg.frontend_builds {
1582 let dir = worktree.join(&fe.dir);
1583 let label = fe.dir.display().to_string();
1584 log.line(&format!("---- frontend build ({label}) ----\n"))
1585 .await;
1586
1587 if !dir.is_dir() {
1588 // An older sha predating the frontend, mid-bisect. Skipping keeps
1589 // sando able to rebuild history; the log says so out loud.
1590 log.line(&format!("{label} absent from this worktree; skipping\n"))
1591 .await;
1592 continue;
1593 }
1594
1595 if !dir.join("node_modules").is_dir()
1596 && let Some(outcome) = run_npm(&dir, &label, &["ci"], "npm ci", ctx, log).await
1597 {
1598 return Some(outcome);
1599 }
1600
1601 if let Some(outcome) = run_npm(
1602 &dir,
1603 &label,
1604 &["run", &fe.script],
1605 &format!("npm run {}", fe.script),
1606 ctx,
1607 log,
1608 )
1609 .await
1610 {
1611 return Some(outcome);
1612 }
1613 }
1614 None
1615 }
1616
1617 /// One `npm` invocation for [`code_smoke_frontends`], bounded by the gate
1618 /// timeout so a wedged install cannot hold the whole pipeline (the enclosing
1619 /// `code_smoke` ceiling would catch it eventually, but this attributes the
1620 /// failure to the project that hung).
1621 async fn run_npm(
1622 dir: &std::path::Path,
1623 label: &str,
1624 args: &[&str],
1625 what: &str,
1626 ctx: &GateCtx,
1627 log: &GateLog,
1628 ) -> Option<GateOutcome> {
1629 log.line(&format!("$ {what}\n")).await;
1630 let mut cmd = tokio::process::Command::new("npm");
1631 cmd.args(args).current_dir(dir).kill_on_drop(true);
1632 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
1633 // On the timeout branch the whole `run` future is dropped, which drops the
1634 // child; `kill_on_drop` is what turns that into an actual kill.
1635 let status = match tokio::time::timeout(ceiling, log.run(&mut cmd)).await {
1636 Ok(Ok((_stdout, _stderr, status))) => status,
1637 Ok(Err(e)) => {
1638 // A missing `npm` lands here. Fatal, not skipped: a build host
1639 // without Node cannot produce the bundle the release serves, and
1640 // silently passing is how the stale bundle shipped in the first place.
1641 log.line(&format!("{what} could not be spawned: {e}\n"))
1642 .await;
1643 return Some(GateOutcome::failed(GateFailure::SpawnFailed {
1644 message: format!("{what} in {label}: {e}"),
1645 }));
1646 }
1647 Err(_elapsed) => {
1648 log.line(&format!(
1649 "{what} timed out after {}s\n",
1650 ctx.cfg.gate_timeout_secs
1651 ))
1652 .await;
1653 return Some(GateOutcome::failed(GateFailure::CodeSmokeFrontend {
1654 dir: label.to_string(),
1655 exit_code: None,
1656 }));
1657 }
1658 };
1659 if status.success() {
1660 return None;
1661 }
1662 Some(GateOutcome::failed(GateFailure::CodeSmokeFrontend {
1663 dir: label.to_string(),
1664 exit_code: status.code(),
1665 }))
1666 }
1667
1668 /// Run the staged binary's DB-free docs integrity check (`MNW_CHECK_DOCS=1`).
1669 ///
1670 /// Returns `Some(failed)` if the check reports broken links, cannot be spawned,
1671 /// or overruns its ceiling; `None` when the docs are clean. Output streams to
1672 /// `log` either way. The 60s ceiling backstops the case where the staged binary
1673 /// predates the flag and would fall through to a normal (DB-needing) boot and
1674 /// hang.
1675 async fn code_smoke_docs_check(ctx: &GateCtx, bin: &str, log: &GateLog) -> Option<GateOutcome> {
1676 let server_dir = match ctx.worktree_for(GateKind::CodeSmoke) {
1677 Ok(w) => w.join("server"),
1678 Err(outcome) => return Some(outcome),
1679 };
1680 log.line("---- docs check (MNW_CHECK_DOCS) ----\n").await;
1681 let mut cmd = tokio::process::Command::new(bin);
1682 cmd.env("MNW_CHECK_DOCS", "1")
1683 .current_dir(&server_dir)
1684 .kill_on_drop(true);
1685 let (stdout, _stderr, status) =
1686 match tokio::time::timeout(std::time::Duration::from_mins(1), log.run(&mut cmd)).await {
1687 Ok(Ok(out)) => out,
1688 Ok(Err(e)) => {
1689 log.line(&format!("docs check spawn failed: {e}\n")).await;
1690 return Some(GateOutcome::failed(GateFailure::SpawnFailed {
1691 message: e.to_string(),
1692 }));
1693 }
1694 Err(_elapsed) => {
1695 let reason =
1696 "docs check timed out after 60s (staged binary may predate MNW_CHECK_DOCS)"
1697 .to_string();
1698 log.line(&format!("{reason}\n")).await;
1699 return Some(GateOutcome::failed(GateFailure::CodeSmokeSetup { reason }));
1700 }
1701 };
1702 if status.success() {
1703 return None;
1704 }
1705 Some(GateOutcome::failed(GateFailure::CodeSmokeDocs {
1706 broken: parse_check_docs_broken_count(&stdout),
1707 }))
1708 }
1709
1710 /// Best-effort parse of the broken-link count from the `MNW_CHECK_DOCS` sentinel
1711 /// line (`MNW_CHECK_DOCS: N broken link(s)`). Returns 0 if absent — the failure
1712 /// still stands, only the summary count is unknown.
1713 fn parse_check_docs_broken_count(stdout: &[u8]) -> u32 {
1714 let text = String::from_utf8_lossy(stdout);
1715 for line in text.lines() {
1716 if let Some(rest) = line.strip_prefix("MNW_CHECK_DOCS:") {
1717 for tok in rest.split_whitespace() {
1718 if let Ok(n) = tok.parse::<u32>() {
1719 return n;
1720 }
1721 }
1722 }
1723 }
1724 0
1725 }
1726
1727 /// The createdb-to-dropdb interior of `code_smoke`: migrate+seed, then boot and
1728 /// probe. Returns the outcome without a `log_ref` (the caller attaches it after
1729 /// teardown). Never returns `Err` — spawn/child failures map to typed outcomes.
1730 async fn code_smoke_body(ctx: &GateCtx, bin: &str, db_url: &str, log: &GateLog) -> GateOutcome {
1731 let server_dir = match ctx.worktree_for(GateKind::CodeSmoke) {
1732 Ok(w) => w.join("server"),
1733 Err(outcome) => return outcome,
1734 };
1735
1736 // Phase 1: migrate-from-scratch + seed. `--seed-examples` loads config,
1737 // connects, runs migrations against the empty DB, seeds the catalog, exits.
1738 // A non-zero exit here is the "code is unsound" signal (broken migration,
1739 // seed error, or config-load failure).
1740 log.line("---- migrate + seed (--seed-examples) ----\n")
1741 .await;
1742 let mut seed_cmd = tokio::process::Command::new(bin);
1743 seed_cmd.arg("--seed-examples").current_dir(&server_dir);
1744 code_smoke_env(&mut seed_cmd, ctx, db_url);
1745 seed_cmd.env("ALLOW_EXAMPLE_SEED", "1").kill_on_drop(true);
1746 let seed_status = match log.run(&mut seed_cmd).await {
1747 Ok((_stdout, _stderr, status)) => status,
1748 Err(e) => {
1749 return GateOutcome::failed(GateFailure::SpawnFailed {
1750 message: e.to_string(),
1751 });
1752 }
1753 };
1754 if !seed_status.success() {
1755 return GateOutcome::failed(GateFailure::CodeSmokeSeed {
1756 exit_code: seed_status.code(),
1757 });
1758 }
1759
1760 // Phase 2: boot the real server against the now-migrated + seeded DB and
1761 // assert both startup signals: it logs `listening` (emitted just before the
1762 // socket bind) AND serves GET /health with a 200. The full stdout/stderr is
1763 // persisted for the operator either way.
1764 log.line("\n---- boot + probe /health ----\n").await;
1765 let mut serve_cmd = tokio::process::Command::new(bin);
1766 serve_cmd.current_dir(&server_dir);
1767 code_smoke_env(&mut serve_cmd, ctx, db_url);
1768 serve_cmd
1769 .stdout(std::process::Stdio::piped())
1770 .stderr(std::process::Stdio::piped())
1771 .kill_on_drop(true);
1772 let mut child = match serve_cmd.spawn() {
1773 Ok(c) => c,
1774 Err(e) => {
1775 return GateOutcome::failed(GateFailure::SpawnFailed {
1776 message: e.to_string(),
1777 });
1778 }
1779 };
1780
1781 // Stream stdout/stderr into the gate log (and out as chunk events) while
1782 // the probe loop runs below; the tasks finish when the pipes close (child
1783 // exits or is killed). The buffers they return are what the `listening`
1784 // assertion reads.
1785 let (stdout_task, stderr_task) = log.drain_pipes(&mut child);
1786
1787 let probe_timeout = std::time::Duration::from_millis(500);
1788 let started = std::time::Instant::now();
1789 let window = std::time::Duration::from_secs(CODE_SMOKE_READY_SECS);
1790 let mut probe_ok_after: Option<u32> = None;
1791 let mut last_probe_err = "never responded".to_string();
1792 let mut early_exit = None;
1793 while started.elapsed() < window {
1794 if let Ok(Some(status)) = child.try_wait() {
1795 early_exit = Some(status);
1796 break;
1797 }
1798 match tokio::time::timeout(probe_timeout, probe_health(ctx.cfg.code_smoke_port)).await {
1799 Ok(Ok(())) => {
1800 probe_ok_after = Some(started.elapsed().as_millis() as u32);
1801 break;
1802 }
1803 Ok(Err(e)) => last_probe_err = e,
1804 Err(_) => last_probe_err = "probe timed out".to_string(),
1805 }
1806 tokio::time::sleep(std::time::Duration::from_millis(250)).await;
1807 }
1808
1809 let exit = match early_exit {
1810 Some(status) => Some(status),
1811 None => {
1812 let e = child.try_wait().ok().flatten();
1813 if e.is_none() {
1814 let _ = child.kill().await;
1815 }
1816 e
1817 }
1818 };
1819 // Keep the serve run's own output so the `listening`-log assertion checks it
1820 // (not the seed run's, which exits before binding). The bytes are already in
1821 // the gate log; these buffers exist only for the assertion.
1822 let serve_stdout = stdout_task.await.unwrap_or_default();
1823 let serve_stderr = stderr_task.await.unwrap_or_default();
1824 let logged_listening =
1825 bytes_contain(&serve_stdout, b"listening") || bytes_contain(&serve_stderr, b"listening");
1826
1827 match (exit, probe_ok_after) {
1828 // Exited on its own within the window — panic / config error / bind fail.
1829 (Some(status), _) => GateOutcome::failed(classify::classify_boot_smoke(status.code())),
1830 // Stayed up and served /health. Assert both required startup signals:
1831 // the `listening` bind log AND the /health 200.
1832 (None, Some(after_ms)) if logged_listening => {
1833 GateOutcome::passed(PassNote::HealthyProbe { after_ms })
1834 }
1835 // Served /health but the `listening` log never appeared.
1836 (None, Some(_)) => GateOutcome::failed(GateFailure::CodeSmokeNoListeningLog),
1837 // Stayed up but never served /health — started, not ready.
1838 (None, None) => GateOutcome::failed(GateFailure::BootHealthProbeFailed {
1839 last_error: last_probe_err,
1840 }),
1841 }
1842 }
1843
1844 /// Substring search over raw bytes (the server's log output), for the
1845 /// `code_smoke` startup-log assertion. Avoids a lossy UTF-8 conversion of the
1846 /// whole buffer just to run `str::contains`.
1847 fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool {
1848 if needle.is_empty() || haystack.len() < needle.len() {
1849 return needle.is_empty();
1850 }
1851 haystack.windows(needle.len()).any(|w| w == needle)
1852 }
1853
1854 /// Apply the minimal env every `code_smoke` invocation shares: point the binary
1855 /// at the throwaway DB, force loopback (dev-mode config, no prod enforcement),
1856 /// hand it a dummy signing secret, and disable file scanning (no AV/YARA on the
1857 /// build host). The worktree is a clean git checkout, so no stray `.env` shadows
1858 /// these (and dotenvy never overrides already-set vars).
1859 ///
1860 /// This list has to carry EVERY var the server's `Config::from_env` treats as
1861 /// mandatory, because `code_smoke` is the only gate that reaches that function
1862 /// at all: `boot_smoke` and the docs check both short-circuit in `main` before
1863 /// config is loaded. So when the server makes a new var required, this is where
1864 /// it has to be answered, and nothing connects the two lists automatically.
1865 ///
1866 /// The values are deliberately throwaway. The gate asks whether this code can
1867 /// migrate, seed, boot and serve; whether a given deployment's env is complete
1868 /// is the `config_check_env_file` guard's job, on the node, against that node's
1869 /// real env file.
1870 ///
1871 /// `app.code_smoke_env` is prepended to all of this, for vars a product needs
1872 /// that sando has no business knowing about. The fixed set overwrites it on a
1873 /// collision, so nothing in a config file can redirect the gate off its own
1874 /// throwaway DB.
1875 fn code_smoke_env(cmd: &mut tokio::process::Command, ctx: &GateCtx, db_url: &str) {
1876 // `localhost`, not `127.0.0.1`, and the distinction is load-bearing. The
1877 // server derives its WebAuthn relying-party id from HOST_URL's host, and
1878 // `WebauthnBuilder::new` validates that id against `Url::domain()` — which
1879 // is `None` for an IP literal, so an origin of `http://127.0.0.1:<port>`
1880 // fails with WebauthnError::Configuration before the server ever binds. It
1881 // is a real ceiling on the gate, not a preference: no IP-literal origin can
1882 // boot this binary. `localhost` is a domain, and matches the derived rp_id.
1883 //
1884 // HOST stays 127.0.0.1: that is the bind address, and the gate probes the
1885 // loopback address directly, so only the advertised origin changes.
1886 let origin = format!("http://localhost:{}", ctx.cfg.code_smoke_port);
1887 // Project-supplied extras go on first so the fixed set below overwrites any
1888 // key they collide on: what points this run at its throwaway DB and its
1889 // loopback port is not negotiable from a config file.
1890 cmd.envs(&ctx.cfg.code_smoke_env);
1891 cmd.env("DATABASE_URL", db_url)
1892 .env("HOST", "127.0.0.1")
1893 .env("PORT", ctx.cfg.code_smoke_port.to_string())
1894 .env("HOST_URL", &origin)
1895 // Required unconditionally by Config::from_env. Pointing it at the smoke
1896 // server's own origin keeps every rendered media URL resolvable within
1897 // the gate; no request is ever made to it.
1898 .env("CDN_BASE_URL", &origin)
1899 .env("SIGNING_SECRET", CODE_SMOKE_SIGNING_SECRET)
1900 .env("SCAN_ENABLED", "false")
1901 .env("INSECURE_COOKIES", "1");
1902 }
1903
1904 /// The throwaway smoke DB name for a version: `sando_code_smoke_<version>` with
1905 /// every non-alphanumeric char folded to `_` and lowercased, capped at Postgres'
1906 /// 63-byte identifier limit. Sanitized to `[a-z0-9_]` so it's safe to quote into
1907 /// DDL. Deterministic per version, so a stale DB from a killed run is reclaimed
1908 /// by the next run's `DROP DATABASE IF EXISTS` rather than accumulating.
1909 fn code_smoke_db_name(version: &Version) -> String {
1910 let mut name = String::from("sando_code_smoke_");
1911 for c in version.to_string().chars() {
1912 name.push(if c.is_ascii_alphanumeric() {
1913 c.to_ascii_lowercase()
1914 } else {
1915 '_'
1916 });
1917 }
1918 name.truncate(63);
1919 name
1920 }
1921
1922 /// Rewrite a `postgres://` URL to point at database `dbname`, preserving scheme,
1923 /// userinfo, host/port, and any query (e.g. the socket `?host=/var/run/postgresql`
1924 /// form) + fragment. Used to derive the maintenance connection (`postgres`) and
1925 /// the throwaway smoke DB URL from the configured `scratch_db_url`.
1926 fn pg_url_with_dbname(url: &str, dbname: &str) -> String {
1927 let Some(after_scheme) = url.find("://").map(|i| i + 3) else {
1928 return url.to_string();
1929 };
1930 let rest = &url[after_scheme..];
1931 // Authority ends at the first '/', '?' or '#'; whatever follows is the
1932 // path (the old dbname) plus an optional query/fragment we must keep.
1933 let auth_end = rest.find(['/', '?', '#']).unwrap_or(rest.len());
1934 let authority = &rest[..auth_end];
1935 let tail = &rest[auth_end..];
1936 let query_and_frag = match tail.find(['?', '#']) {
1937 Some(i) => &tail[i..],
1938 None => "",
1939 };
1940 format!(
1941 "{}{}/{}{}",
1942 &url[..after_scheme],
1943 authority,
1944 dbname,
1945 query_and_frag
1946 )
1947 }
1948
1949 /// Create the throwaway smoke DB on the cluster `maintenance_url` points at,
1950 /// dropping any stale one first. `dbname` is sanitized to `[a-z0-9_]` by
1951 /// `code_smoke_db_name`, so quoting it is sufficient. `CREATE DATABASE` cannot
1952 /// run inside a transaction, so these go through the simple-query protocol (a
1953 /// raw `&str` execute), matching `reset_scratch`.
1954 async fn pg_create_db(maintenance_url: &str, dbname: &str) -> Result<()> {
1955 use sqlx::Executor;
1956 use sqlx::postgres::PgPoolOptions;
1957 let pool = PgPoolOptions::new()
1958 .max_connections(1)
1959 .connect(maintenance_url)
1960 .await?;
1961 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1962 "DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE)"
1963 ))))
1964 .await?;
1965 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1966 "CREATE DATABASE \"{dbname}\""
1967 ))))
1968 .await?;
1969 pool.close().await;
1970 Ok(())
1971 }
1972
1973 /// Drop the throwaway smoke DB, forcing off any lingering connection (the killed
1974 /// server's pool). Best-effort at the call site — a failure is logged, not fatal.
1975 async fn pg_drop_db(maintenance_url: &str, dbname: &str) -> Result<()> {
1976 use sqlx::Executor;
1977 use sqlx::postgres::PgPoolOptions;
1978 let pool = PgPoolOptions::new()
1979 .max_connections(1)
1980 .connect(maintenance_url)
1981 .await?;
1982 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1983 "DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE)"
1984 ))))
1985 .await?;
1986 pool.close().await;
1987 Ok(())
1988 }
1989
1990 async fn boot_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
1991 let bin: Option<(String,)> =
1992 sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
1993 .bind(&ctx.cfg.id)
1994 .bind(&ctx.version)
1995 .fetch_optional(&ctx.pool)
1996 .await?;
1997 let Some((bin,)) = bin else {
1998 return Ok(GateOutcome::blocked(GateBlocker::ArtifactMissing {
1999 version: ctx.version.clone(),
2000 }));
2001 };
2002
2003 // Readiness smoke: start the binary and confirm it serves `GET /health`
2004 // within the window, not merely that the process stays up. Panics in main,
2005 // missing config, and port-bind failures still surface as an early exit; a
2006 // process that comes up but never serves /health is now its own failure.
2007 //
2008 // The server requires DATABASE_URL or it panics on config load before
2009 // we can observe anything. We point it at the scratch DB (already
2010 // migrated by the build step and refreshed by migration_dry_run if
2011 // that gate ran first). SCAN_ENABLED=false skips loading YARA rules
2012 // from /opt/makenotwork/yara-rules which doesn't exist on the build
2013 // host. SANDO_BOOT_SMOKE_PORT tells the smoke server which loopback port
2014 // to bind so we know where to probe. Other config has sane optional defaults.
2015 let mut cmd = tokio::process::Command::new(&bin);
2016 cmd.env("SANDO_BOOT_SMOKE", "1")
2017 .env("SANDO_BOOT_SMOKE_PORT", ctx.cfg.boot_smoke_port.to_string())
2018 .env("SCAN_ENABLED", "false")
2019 .stdout(std::process::Stdio::piped())
2020 .stderr(std::process::Stdio::piped())
2021 .kill_on_drop(true);
2022 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
2023 cmd.env("DATABASE_URL", scratch_url);
2024 }
2025 let log_path = ctx.log_path(GateKind::BootSmoke);
2026 let log_ref = ctx.log_ref(GateKind::BootSmoke);
2027 let mut child = match cmd.spawn() {
2028 Ok(c) => c,
2029 Err(e) => {
2030 // Spawn failures get a one-off log line via LiveLog so the
2031 // on-disk file still exists for `GET /logs/...`.
2032 let mut log = LiveLog::open(log_path, gate_chunk_cb(ctx.events.clone(), run_id)).await;
2033 log.write_chunk(format!("spawn: {e}\n").as_bytes()).await;
2034 log.close().await;
2035 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
2036 message: e.to_string(),
2037 })
2038 .with_log_ref(log_ref));
2039 }
2040 };
2041
2042 // The boot smoke window is 3s. Drain stdout/stderr concurrently through
2043 // a shared LiveLog sink so the operator sees panics/log lines stream in
2044 // real time before the kill, AND the on-disk log gets the full byte
2045 // stream for post-mortem reads. The drainers exit when their pipe
2046 // closes — which happens when the child exits naturally or after kill.
2047 let log = std::sync::Arc::new(tokio::sync::Mutex::new(
2048 LiveLog::open(log_path, gate_chunk_cb(ctx.events.clone(), run_id)).await,
2049 ));
2050 let stdout_task = tokio::spawn(stream_into_log(child.stdout.take(), log.clone()));
2051 let stderr_task = tokio::spawn(stream_into_log(child.stderr.take(), log.clone()));
2052
2053 // Poll readiness across the 3s window instead of a flat sleep: GET /health
2054 // must return 2xx. A crash mid-window short-circuits to the exit-code
2055 // failure path (try_wait below); a process that stays up but never serves
2056 // /health is a distinct readiness failure.
2057 let probe_timeout = std::time::Duration::from_millis(500);
2058 let started = std::time::Instant::now();
2059 let window = std::time::Duration::from_secs(3);
2060 let mut probe_ok_after: Option<u32> = None;
2061 let mut last_probe_err = "never responded".to_string();
2062 let mut early_exit = None;
2063 while started.elapsed() < window {
2064 if let Some(status) = child.try_wait()? {
2065 early_exit = Some(status);
2066 break;
2067 }
2068 match tokio::time::timeout(probe_timeout, probe_health(ctx.cfg.boot_smoke_port)).await {
2069 Ok(Ok(())) => {
2070 probe_ok_after = Some(started.elapsed().as_millis() as u32);
2071 break;
2072 }
2073 Ok(Err(e)) => last_probe_err = e,
2074 Err(_) => last_probe_err = "probe timed out".to_string(),
2075 }
2076 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
2077 }
2078
2079 // Stop the child unless it already exited, then drain the log tasks.
2080 let exit = match early_exit {
2081 Some(status) => Some(status),
2082 None => {
2083 let e = child.try_wait()?;
2084 if e.is_none() {
2085 let _ = child.kill().await;
2086 }
2087 e
2088 }
2089 };
2090 // The streamed bytes already landed in the live log and the on-disk file for
2091 // the post-mortem reader. Drain the join handles to avoid hangs.
2092 let _ = stdout_task.await;
2093 let _ = stderr_task.await;
2094 // Unique owner of the Arc at this point (both tasks dropped their clones).
2095 if let Ok(mutex) = std::sync::Arc::try_unwrap(log) {
2096 mutex.into_inner().close().await;
2097 }
2098
2099 match (exit, probe_ok_after) {
2100 // Exited on its own within the window — a crash/panic/bind failure.
2101 (Some(status), _) => {
2102 let failure = classify::classify_boot_smoke(status.code());
2103 Ok(GateOutcome::failed(failure).with_log_ref(log_ref))
2104 }
2105 // Stayed up and served /health — readiness proven.
2106 (None, Some(after_ms)) => {
2107 Ok(GateOutcome::passed(PassNote::HealthyProbe { after_ms }).with_log_ref(log_ref))
2108 }
2109 // Stayed up but never served /health — started, not ready.
2110 (None, None) => Ok(GateOutcome::failed(GateFailure::BootHealthProbeFailed {
2111 last_error: last_probe_err,
2112 })
2113 .with_log_ref(log_ref)),
2114 }
2115 }
2116
2117 /// One readiness probe of the boot-smoke server: connect to `127.0.0.1:port`
2118 /// and `GET /health`, returning `Ok(())` only on a `200`. A hand-rolled HTTP/1.0
2119 /// request over a raw `TcpStream` keeps the outbound probe dependency-free
2120 /// (reqwest is dev-only); the smoke server serves the one route over axum, which
2121 /// speaks 1.0. `Err` carries a short reason for the operator's failure note. The
2122 /// caller wraps each call in a timeout.
2123 async fn probe_health(port: u16) -> std::result::Result<(), String> {
2124 use tokio::io::{AsyncReadExt, AsyncWriteExt};
2125 let mut stream = tokio::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, port))
2126 .await
2127 .map_err(|e| format!("connect: {e}"))?;
2128 stream
2129 .write_all(b"GET /health HTTP/1.0\r\nHost: localhost\r\nConnection: close\r\n\r\n")
2130 .await
2131 .map_err(|e| format!("write: {e}"))?;
2132 let mut buf = Vec::new();
2133 stream
2134 .read_to_end(&mut buf)
2135 .await
2136 .map_err(|e| format!("read: {e}"))?;
2137 let text = String::from_utf8_lossy(&buf);
2138 let status_line = text.lines().next().unwrap_or("");
2139 if status_line.contains(" 200 ") {
2140 Ok(())
2141 } else {
2142 Err(format!("unexpected status line: {status_line:?}"))
2143 }
2144 }
2145
2146 /// Sink that drops streamed output. `node_health` keeps the probe's stderr from
2147 /// its `RunOutput` for the failure note, so the live byte stream isn't needed.
2148 struct DiscardSink;
2149 #[async_trait::async_trait]
2150 impl ops_exec::LogSink for DiscardSink {
2151 async fn write_chunk(&mut self, _bytes: &[u8]) {}
2152 }
2153
2154 /// `node_health` — the post-deploy gate that proves the *deployed nodes* are
2155 /// serving, recording one outcome per (tier, version) that the next promote
2156 /// checks. Distinct from `boot_smoke`, which boots the staged artifact on the
2157 /// build host: this probes each node over the same executor the deploy used, so
2158 /// a node that took a corrupt artifact, wrong-arch binary, or failed restart is
2159 /// caught here rather than waved through. Fails closed: any
2160 /// unhealthy node fails the gate, and an empty node set is `Blocked`.
2161 /// Load the tier's public pages in a real browser and fail if the JavaScript
2162 /// did not run.
2163 ///
2164 /// The only gate here that crosses the CDN. `boot_smoke` runs on the build host
2165 /// and `node_health` reaches a node over its executor, so between them nothing
2166 /// requests the site the way a visitor does. A CDN holding a module from an
2167 /// earlier deploy can fail to link against the fresh one beside it, and since
2168 /// the bundle's entry point side-effect-imports every island, one bad link takes
2169 /// all of them down together while every artifact is individually correct: right
2170 /// markup, right stylesheets, every module answering 200 with current bytes.
2171 /// Composition is observable in a browser and nowhere else.
2172 ///
2173 /// Runs on the daemon host rather than on a node, because it is a *client*: it
2174 /// should reach the site through whatever the public reaches it through, and a
2175 /// probe that ran on the origin would inherit the blind spot this exists to
2176 /// close.
2177 async fn page_smoke(ctx: &GateCtx) -> Result<GateOutcome> {
2178 let Some(cmd) = ctx.cfg.page_smoke_cmd.as_deref() else {
2179 // A service with no pages says so by configuring no command. Blocked
2180 // rather than passed: a gate that proves nothing must not read green.
2181 return Ok(GateOutcome::blocked(GateBlocker::NotConfigured {
2182 what: "page_smoke_cmd".into(),
2183 }));
2184 };
2185 let Some(base) = ctx.public_url.as_deref() else {
2186 return Ok(GateOutcome::blocked(GateBlocker::NotConfigured {
2187 what: "public_url".into(),
2188 }));
2189 };
2190
2191 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
2192 let child = tokio::process::Command::new("sh")
2193 .arg("-c")
2194 .arg(cmd)
2195 .env("BASE", base)
2196 .stdout(std::process::Stdio::piped())
2197 .stderr(std::process::Stdio::piped())
2198 .kill_on_drop(true)
2199 .spawn()?;
2200
2201 let out = match tokio::time::timeout(ceiling, child.wait_with_output()).await {
2202 Ok(res) => res?,
2203 Err(_elapsed) => {
2204 return Ok(GateOutcome::failed(GateFailure::Timeout {
2205 gate: GateKind::PageSmoke,
2206 after_s: ctx.cfg.gate_timeout_secs as u32,
2207 })
2208 .with_log_ref(ctx.log_ref(GateKind::PageSmoke)));
2209 }
2210 };
2211
2212 let log = format!(
2213 "{}{}",
2214 String::from_utf8_lossy(&out.stdout),
2215 String::from_utf8_lossy(&out.stderr)
2216 );
2217 append_to_log(&ctx.log_path(GateKind::PageSmoke), log.as_bytes()).await;
2218
2219 if out.status.success() {
2220 return Ok(
2221 GateOutcome::passed(PassNote::PagesClean { base: base.into() })
2222 .with_log_ref(ctx.log_ref(GateKind::PageSmoke)),
2223 );
2224 }
2225
2226 // The script prints one `FAIL <url>` line per bad page and indents the
2227 // reasons under it. Lift the first reason into the summary so a red gate
2228 // says what broke without anyone opening the log.
2229 let first = log
2230 .lines()
2231 .skip_while(|l| !l.starts_with("FAIL"))
2232 .nth(1)
2233 .map(str::trim)
2234 .filter(|l| !l.is_empty())
2235 .unwrap_or("see log");
2236 Ok(GateOutcome::failed(GateFailure::PagesBroken {
2237 base: base.into(),
2238 detail: first.to_string(),
2239 })
2240 .with_log_ref(ctx.log_ref(GateKind::PageSmoke)))
2241 }
2242
2243 async fn node_health(ctx: &GateCtx) -> Result<GateOutcome> {
2244 if ctx.nodes.is_empty() {
2245 return Ok(GateOutcome::blocked(GateBlocker::NoNodesToProbe));
2246 }
2247 for probe in &ctx.nodes {
2248 if let Err(detail) = probe_node(probe).await {
2249 return Ok(GateOutcome::failed(GateFailure::NodeUnhealthy {
2250 node: probe.node.to_string(),
2251 detail,
2252 }));
2253 }
2254 }
2255 Ok(GateOutcome::passed(PassNote::NodesHealthy {
2256 nodes: ctx.nodes.len() as u32,
2257 }))
2258 }
2259
2260 /// Probe one node over its executor: confirm the unit is active post-restart
2261 /// and, when a `health_url` is configured, that it serves a 2xx. Retries across
2262 /// ~10s because the service may still be restarting / warming. Runs under the
2263 /// read-only `Observe(Health)` capability (every Sando node grants it), so the
2264 /// probe needs no deploy authority. `Ok(())` = healthy; `Err(detail)` carries a
2265 /// short reason for the gate's failure note.
2266 async fn probe_node(probe: &NodeProbe) -> std::result::Result<(), String> {
2267 use ops_exec::{Action, ObserveKind, Step, sh_quote};
2268 let svc = sh_quote(&probe.service);
2269 let url = probe
2270 .health_url
2271 .as_deref()
2272 .map_or_else(|| "''".to_string(), sh_quote);
2273 // One executor round-trip with the retry loop on the node: is-active, then
2274 // (if a url is set) curl it for a 2xx. Exit 0 only when both hold.
2275 let script = format!(
2276 "svc={svc}; url={url}; \
2277 for _ in $(seq 1 10); do \
2278 if systemctl is-active --quiet \"$svc\"; then \
2279 if [ -z \"$url\" ] || curl -fsS --max-time 5 \"$url\" >/dev/null 2>&1; then exit 0; fi; \
2280 fi; \
2281 sleep 1; \
2282 done; \
2283 echo 'service not active or health url not 2xx after retries' >&2; exit 1"
2284 );
2285 let step = Step::shell(Action::Observe(ObserveKind::Health), script);
2286 let mut sink = DiscardSink;
2287 let out = probe
2288 .executor
2289 .run_streaming(&step, &mut sink)
2290 .await
2291 .map_err(|e| format!("probe spawn: {e}"))?;
2292 if out.status.success() {
2293 Ok(())
2294 } else {
2295 let code = out
2296 .status
2297 .code()
2298 .map_or_else(|| "signal".to_string(), |c| c.to_string());
2299 let stderr: String = String::from_utf8_lossy(&out.stderr)
2300 .chars()
2301 .take(200)
2302 .collect();
2303 Err(format!("exit {code}: {stderr}"))
2304 }
2305 }
2306
2307 /// Drain `stream` into the shared `LiveLog` (which forwards each chunk to
2308 /// the on-disk log file AND broadcasts a `GateLogChunk` event), and return
2309 /// the concatenated bytes so the classifier can still operate on the full
2310 /// output post-hoc.
2311 async fn stream_into_log<R>(
2312 stream: Option<R>,
2313 log: std::sync::Arc<tokio::sync::Mutex<LiveLog>>,
2314 ) -> Vec<u8>
2315 where
2316 R: tokio::io::AsyncRead + Unpin + Send + 'static,
2317 {
2318 let mut total = Vec::new();
2319 let Some(mut s) = stream else { return total };
2320 let mut buf = [0u8; 4096];
2321 loop {
2322 match s.read(&mut buf).await {
2323 Ok(0) => break,
2324 Err(_) => break,
2325 Ok(n) => {
2326 total.extend_from_slice(&buf[..n]);
2327 log.lock().await.write_chunk(&buf[..n]).await;
2328 }
2329 }
2330 }
2331 total
2332 }
2333
2334 /// Spawn a child, drain its stdout/stderr through a `LiveLog`, return the
2335 /// combined buffers and exit status. Shared by `cargo_test` (no deadline)
2336 /// and ad-hoc callers — `boot_smoke` rolls its own variant because of its
2337 /// 3s kill window.
2338 async fn stream_child_to_live_log(
2339 child: &mut tokio::process::Child,
2340 events: EventTx,
2341 run_id: GateRunId,
2342 log_path: PathBuf,
2343 ) -> Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> {
2344 let log = GateLog::new(LiveLog::open(log_path, gate_chunk_cb(events, run_id)).await);
2345 let out = log.stream_child(child).await;
2346 log.close().await;
2347 out
2348 }
2349
2350 /// One gate's live log, held across every step of a multi-step gate.
2351 ///
2352 /// Single-child gates call [`stream_child_to_live_log`] and are done. The
2353 /// staged gates (`code_smoke`, `migration_dry_run`) instead run several
2354 /// children plus banner lines between them, and they hold one `GateLog` across
2355 /// the lot: a single sink means chunk sequence numbers stay monotonic for the
2356 /// whole gate, and the on-disk log reads in the order things actually happened
2357 /// rather than as stdout-then-stderr assembled at the end.
2358 ///
2359 /// Every write is best-effort in the same way `LiveLog` is: a log directory
2360 /// that cannot be written degrades to callback-only and never turns a passing
2361 /// gate red.
2362 struct GateLog {
2363 sink: Arc<tokio::sync::Mutex<LiveLog>>,
2364 }
2365
2366 impl GateLog {
2367 fn new(sink: LiveLog) -> Self {
2368 Self {
2369 sink: Arc::new(tokio::sync::Mutex::new(sink)),
2370 }
2371 }
2372
2373 /// Open the sink for `gate`'s log file, streaming to the TUI under `run_id`.
2374 async fn open(ctx: &GateCtx, run_id: GateRunId, gate: GateKind) -> Self {
2375 Self::new(
2376 LiveLog::open(
2377 ctx.log_path(gate),
2378 gate_chunk_cb(ctx.events.clone(), run_id),
2379 )
2380 .await,
2381 )
2382 }
2383
2384 /// Emit a banner (or any line the gate itself produces) through the same
2385 /// sink the children stream to, so it lands in sequence with their output.
2386 async fn write(&self, bytes: &[u8]) {
2387 self.sink.lock().await.write_chunk(bytes).await;
2388 }
2389
2390 /// Same, for the common `format!`-a-line case.
2391 async fn line(&self, s: &str) {
2392 self.write(s.as_bytes()).await;
2393 }
2394
2395 /// Spawn `cmd` with both pipes captured, stream them into the sink as they
2396 /// arrive, and return the buffers plus the exit status. The buffers are
2397 /// what the classifiers still operate on post-hoc.
2398 async fn run(
2399 &self,
2400 cmd: &mut Command,
2401 ) -> std::io::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> {
2402 cmd.stdout(std::process::Stdio::piped())
2403 .stderr(std::process::Stdio::piped());
2404 let mut child = cmd.spawn()?;
2405 self.stream_child(&mut child)
2406 .await
2407 .map_err(std::io::Error::other)
2408 }
2409
2410 /// Drain an already-spawned child's pipes into the sink and wait for it.
2411 async fn stream_child(
2412 &self,
2413 child: &mut tokio::process::Child,
2414 ) -> Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> {
2415 let (stdout_task, stderr_task) = self.drain_pipes(child);
2416 let status = child.wait().await?;
2417 let stdout_buf = stdout_task.await.unwrap_or_default();
2418 let stderr_buf = stderr_task.await.unwrap_or_default();
2419 Ok((stdout_buf, stderr_buf, status))
2420 }
2421
2422 /// Start draining a child's pipes into the sink *without* waiting on the
2423 /// child. For `code_smoke`'s serve phase, which probes `/health` while the
2424 /// server is still up. The caller must await both handles for the buffers
2425 /// (and before [`Self::close`], so the flush isn't skipped).
2426 #[allow(clippy::type_complexity)]
2427 fn drain_pipes(
2428 &self,
2429 child: &mut tokio::process::Child,
2430 ) -> (
2431 tokio::task::JoinHandle<Vec<u8>>,
2432 tokio::task::JoinHandle<Vec<u8>>,
2433 ) {
2434 (
2435 tokio::spawn(stream_into_log(child.stdout.take(), self.sink.clone())),
2436 tokio::spawn(stream_into_log(child.stderr.take(), self.sink.clone())),
2437 )
2438 }
2439
2440 /// Flush the file. A still-outstanding streaming task (only possible if the
2441 /// gate returned without awaiting it) leaves the `Arc` shared, in which case
2442 /// the flush is skipped — `LiveLog` writes unbuffered to the OS either way,
2443 /// so nothing already written is lost.
2444 async fn close(self) {
2445 if let Ok(mutex) = Arc::try_unwrap(self.sink) {
2446 mutex.into_inner().close().await;
2447 }
2448 }
2449 }
2450
2451 /// Live check: has `tier`'s burn-in window of `hours` elapsed since its clock
2452 /// (`tier_state.burn_in_started_at`, started by a promote onto the tier)? Used
2453 /// by the promote-time gate check (`unsatisfied_gates`) so a stale `blocked`
2454 /// row never masks an elapsed — or not-yet-elapsed — window. The `burn_in` gate
2455 /// runner below wraps the same state with a richer outcome for `/state`.
2456 pub async fn burn_in_satisfied(
2457 pool: &SqlitePool,
2458 app: &AppId,
2459 tier: &TierId,
2460 hours: u32,
2461 ) -> Result<bool> {
2462 let started: Option<String> =
2463 sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?")
2464 .bind(app)
2465 .bind(tier)
2466 .fetch_optional(pool)
2467 .await?
2468 .flatten();
2469 let Some(started) = started else {
2470 return Ok(false);
2471 };
2472 let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc);
2473 Ok(Utc::now() - started >= chrono::Duration::hours(hours as i64))
2474 }
2475
2476 async fn burn_in(ctx: &GateCtx, hours: u32) -> Result<GateOutcome> {
2477 // Check tier_state.burn_in_started_at on this tier; pass if enough time
2478 // has elapsed. The clock is started by /promote when a version lands on
2479 // the burn-in tier.
2480 let started: Option<String> =
2481 sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?")
2482 .bind(&ctx.cfg.id)
2483 .bind(&ctx.tier)
2484 .fetch_optional(&ctx.pool)
2485 .await?
2486 .flatten();
2487 let Some(started) = started else {
2488 return Ok(GateOutcome::blocked(GateBlocker::BurnInClockNotStarted));
2489 };
2490 let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc);
2491 let elapsed = Utc::now() - started;
2492 let needed = chrono::Duration::hours(hours as i64);
2493 if elapsed >= needed {
2494 Ok(GateOutcome::passed(PassNote::BurnInElapsed {
2495 hours: elapsed.num_hours() as u32,
2496 }))
2497 } else {
2498 let remaining = (needed - elapsed).num_hours().max(0) as u32;
2499 Ok(GateOutcome::blocked(GateBlocker::BurnInRemaining {
2500 hours_remaining: remaining,
2501 hours_total: hours,
2502 }))
2503 }
2504 }
2505
2506 async fn manual_confirm(ctx: &GateCtx) -> Result<GateOutcome> {
2507 // Pass iff a row in gate_runs exists with status='passed' for this
2508 // (tier, version, manual_confirm) that was inserted out-of-band by an
2509 // operator action. Since the harness inserts the in-flight row itself,
2510 // look for a prior confirmation row.
2511 let prior_at: Option<String> = sqlx::query_scalar(
2512 "SELECT finished_at FROM gate_runs
2513 WHERE app = ? AND tier = ? AND version = ? AND gate_kind = 'manual_confirm'
2514 AND status = 'passed'
2515 ORDER BY id DESC LIMIT 1",
2516 )
2517 .bind(&ctx.cfg.id)
2518 .bind(&ctx.tier)
2519 .bind(&ctx.version)
2520 .fetch_optional(&ctx.pool)
2521 .await?;
2522 match prior_at {
2523 Some(at_str) => {
2524 let at = chrono::DateTime::parse_from_rfc3339(&at_str)
2525 .map_or_else(|_| Utc::now(), |d| d.with_timezone(&Utc));
2526 Ok(GateOutcome::passed(PassNote::OperatorConfirmed { at }))
2527 }
2528 None => Ok(GateOutcome::blocked(
2529 GateBlocker::AwaitingOperatorConfirmation,
2530 )),
2531 }
2532 }
2533
2534 #[cfg(test)]
2535 mod tests {
2536 use super::*;
2537 use crate::events;
2538 use sqlx::sqlite::SqlitePoolOptions;
2539
2540 fn target(dir: &str) -> crate::config::TestTarget {
2541 crate::config::TestTarget {
2542 dir: std::path::PathBuf::from(dir),
2543 aux_repo: None,
2544 features: Vec::new(),
2545 all_features: false,
2546 scratch_db: false,
2547 }
2548 }
2549
2550 #[test]
2551 fn parse_check_docs_broken_count_reads_the_sentinel() {
2552 // The failing sentinel carries the count.
2553 assert_eq!(
2554 parse_check_docs_broken_count(b"some log\nMNW_CHECK_DOCS: 3 broken link(s)\n"),
2555 3
2556 );
2557 // Count with a preceding log line still parses (first bare int wins).
2558 assert_eq!(
2559 parse_check_docs_broken_count(
2560 b" broken link: a -> b\nMNW_CHECK_DOCS: 1 broken link(s)\n"
2561 ),
2562 1
2563 );
2564 // The parser only runs on failure; the "ok" sentinel is never fed to it,
2565 // and its "(2" token is not a bare int, so it yields 0 harmlessly.
2566 assert_eq!(
2567 parse_check_docs_broken_count(b"MNW_CHECK_DOCS: ok (2 collision(s) reported)\n"),
2568 0
2569 );
2570 // Absent sentinel -> 0; the failure still stands, only the count is lost.
2571 assert_eq!(parse_check_docs_broken_count(b"unrelated output"), 0);
2572 }
2573
2574 #[test]
2575 fn name_target_points_a_test_failure_at_its_crate() {
2576 // With one target the crate was implicit; with fifteen the operator
2577 // needs the summary to say which one broke.
2578 let f = name_target(
2579 GateFailure::CargoTest {
2580 failed_count: 3,
2581 first_failed: Some("workflows::sync::round_trip".into()),
2582 first_panic: None,
2583 },
2584 std::path::Path::new("shared/synckit-client"),
2585 );
2586 assert_eq!(
2587 f.summary(),
2588 "3 test(s) failed; first: shared/synckit-client: workflows::sync::round_trip",
2589 );
2590 }
2591
2592 #[test]
2593 fn name_target_points_a_compile_failure_at_its_crate() {
2594 let f = name_target(
2595 GateFailure::CompileError {
2596 error_count: 1,
2597 first_error: Some("error[E0063]".into()),
2598 },
2599 std::path::Path::new("mnw-cli"),
2600 );
2601 assert_eq!(
2602 f.summary(),
2603 "compile failed (1 error(s)); first: mnw-cli: error[E0063]"
2604 );
2605 }
2606
2607 #[test]
2608 fn name_target_names_the_crate_even_without_a_test_name() {
2609 let f = name_target(
2610 GateFailure::CargoTest {
2611 failed_count: 2,
2612 first_failed: None,
2613 first_panic: None,
2614 },
2615 std::path::Path::new("pom"),
2616 );
2617 assert_eq!(f.summary(), "2 test(s) failed; first: pom");
2618 }
2619
2620 #[test]
2621 fn name_target_leaves_unrelated_failures_alone() {
2622 let f = name_target(
2623 GateFailure::SpawnFailed {
2624 message: "no cargo".into(),
2625 },
2626 std::path::Path::new("pom"),
2627 );
2628 assert!(matches!(f, GateFailure::SpawnFailed { .. }));
2629 }
2630
2631 /// `target()` above, but resolved against an aux repo's checkout.
2632 fn aux_target(dir: &str, repo: &str) -> crate::config::TestTarget {
2633 crate::config::TestTarget {
2634 aux_repo: Some(repo.to_string()),
2635 ..target(dir)
2636 }
2637 }
2638
2639 /// True when the URL's host parses as a domain rather than an IP literal,
2640 /// which is the distinction `Url::domain()` draws and WebAuthn depends on.
2641 fn url_host_is_a_domain(url: &str) -> bool {
2642 let after = url.split("://").nth(1).unwrap_or("");
2643 let host = after.split(['/', '?', '#']).next().unwrap_or("");
2644 let host = host.rsplit('@').next().unwrap_or(host);
2645 let host = if let Some(rest) = host.strip_prefix('[') {
2646 rest.split(']').next().unwrap_or("")
2647 } else {
2648 host.split(':').next().unwrap_or("")
2649 };
2650 !host.is_empty() && host.parse::<std::net::IpAddr>().is_err()
2651 }
2652
2653 fn resolving_ctx(worktree: &str, aux: &[(&str, &str)]) -> GateCtx {
2654 GateCtx {
2655 public_url: None,
2656 pool: SqlitePool::connect_lazy("sqlite::memory:").unwrap(),
2657 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
2658 tier: TierId::new("host"),
2659 version: "0.1.0".parse().unwrap(),
2660 worktree: Some(PathBuf::from(worktree)),
2661 bundle: None,
2662 events: events::channel(),
2663 nodes: Vec::new(),
2664 build_id: None,
2665 aux_dirs: aux
2666 .iter()
2667 .map(|(n, d)| ((*n).to_string(), PathBuf::from(d)))
2668 .collect(),
2669 }
2670 }
2671
2672 #[tokio::test]
2673 async fn a_plain_target_resolves_under_the_worktree() {
2674 let ctx = resolving_ctx("/w/abc123", &[]);
2675 assert_eq!(
2676 ctx.target_dir(&target("shared/tagtree")),
2677 Some(PathBuf::from("/w/abc123/shared/tagtree")),
2678 );
2679 }
2680
2681 #[tokio::test]
2682 async fn an_aux_target_resolves_beside_the_worktree_not_under_it() {
2683 // The whole point: `Libraries/docengine` is a sibling of the per-sha
2684 // worktree, so a worktree-relative path can never reach it.
2685 let ctx = resolving_ctx("/w/abc123", &[("docengine", "/w/Libraries/docengine")]);
2686 assert_eq!(
2687 ctx.target_dir(&aux_target("", "docengine")),
2688 Some(PathBuf::from("/w/Libraries/docengine")),
2689 );
2690 // A subdirectory of an aux repo resolves under its checkout.
2691 assert_eq!(
2692 ctx.target_dir(&aux_target("crates/inner", "docengine")),
2693 Some(PathBuf::from("/w/Libraries/docengine/crates/inner")),
2694 );
2695 }
2696
2697 #[tokio::test]
2698 async fn an_aux_target_with_no_checkout_this_run_resolves_to_nothing() {
2699 // Promote-time gates carry no aux dirs. Resolving to a wrong path (say,
2700 // the worktree) would run the gate against whatever happened to sit
2701 // there; `None` makes the caller skip, and --check-config is what
2702 // catches a real typo.
2703 let ctx = resolving_ctx("/w/abc123", &[]);
2704 assert_eq!(ctx.target_dir(&aux_target("", "docengine")), None);
2705 }
2706
2707 #[test]
2708 fn labels_name_the_repo_an_aux_target_lives_in() {
2709 assert_eq!(target("server").label(), "server");
2710 assert_eq!(aux_target("", "docengine").label(), "docengine (aux)");
2711 assert_eq!(
2712 aux_target("crates/inner", "docengine").label(),
2713 "docengine/crates/inner (aux)",
2714 );
2715 }
2716
2717 /// A `GateCtx` over `worktree` with the given frontend projects configured.
2718 /// No DB, no artifact — `code_smoke_frontends` touches neither.
2719 async fn frontend_ctx(worktree: &std::path::Path, dirs: &[&str]) -> GateCtx {
2720 let mut cfg = crate::config::AppConfig::for_tests();
2721 cfg.frontend_builds = dirs
2722 .iter()
2723 .map(|d| crate::config::FrontendBuild {
2724 dir: PathBuf::from(d),
2725 script: "build".into(),
2726 })
2727 .collect();
2728 cfg.logs_root = worktree.join("logs");
2729 GateCtx {
2730 public_url: None,
2731 pool: SqlitePoolOptions::new()
2732 .max_connections(1)
2733 .connect("sqlite::memory:")
2734 .await
2735 .unwrap(),
2736 cfg: std::sync::Arc::new(cfg),
2737 tier: TierId::new("host"),
2738 version: "0.1.0".parse().unwrap(),
2739 worktree: Some(worktree.to_path_buf()),
2740 bundle: None,
2741 events: events::channel(),
2742 nodes: Vec::new(),
2743 build_id: None,
2744 aux_dirs: HashMap::new(),
2745 }
2746 }
2747
2748 /// A `GateCtx` for the `migration_dry_run` freshness checks: a migrated
2749 /// in-memory pool (so `backups` exists) and a scratch URL set, so the gate
2750 /// reaches the backup lookup instead of bailing on config. Nothing here
2751 /// touches postgres — every assertion below blocks before `reset_scratch`.
2752 async fn dry_run_ctx(worktree: &std::path::Path, max_age_hours: u32) -> GateCtx {
2753 let mut cfg = crate::config::AppConfig::for_tests();
2754 cfg.scratch_db_url = Some("postgres:///sando_scratch".into());
2755 cfg.backup_max_age_hours = max_age_hours;
2756 cfg.logs_root = worktree.join("logs");
2757 let pool = SqlitePoolOptions::new()
2758 .max_connections(1)
2759 .connect("sqlite::memory:")
2760 .await
2761 .unwrap();
2762 crate::db::migrate(&pool).await.unwrap();
2763 GateCtx {
2764 public_url: None,
2765 pool,
2766 cfg: std::sync::Arc::new(cfg),
2767 tier: TierId::new("host"),
2768 version: "0.1.0".parse().unwrap(),
2769 worktree: Some(worktree.to_path_buf()),
2770 bundle: None,
2771 events: events::channel(),
2772 nodes: Vec::new(),
2773 build_id: None,
2774 aux_dirs: HashMap::new(),
2775 }
2776 }
2777
2778 /// Record a `server` backup row fetched `hours_ago`, as `/backup/fetch` would.
2779 async fn seed_backup(ctx: &GateCtx, hours_ago: i64) {
2780 seed_named_backup(ctx, "server", hours_ago).await;
2781 }
2782
2783 /// Record a backup row for one named dump.
2784 async fn seed_named_backup(ctx: &GateCtx, name: &str, hours_ago: i64) {
2785 let at = (Utc::now() - chrono::Duration::hours(hours_ago)).to_rfc3339();
2786 sqlx::query(
2787 "INSERT INTO backups (name, fetched_at, source, local_path, byte_size)
2788 VALUES (?, ?, 'file:///x.sql.gz', '/tmp/sando-test-backup.sql.gz', 1000000)",
2789 )
2790 .bind(name)
2791 .bind(at)
2792 .execute(&ctx.pool)
2793 .await
2794 .unwrap();
2795 }
2796
2797 /// The multithreaded check, as `sando-daemon.toml` configures it.
2798 fn mt_check() -> crate::config::MigrationCheck {
2799 crate::config::MigrationCheck {
2800 dir: std::path::PathBuf::from("multithreaded/migrations"),
2801 backup: "multithreaded".into(),
2802 scratch_db: Some("sando_scratch_mt".into()),
2803 owner_role: Some("multithreaded".into()),
2804 }
2805 }
2806
2807 /// Re-point a `dry_run_ctx` at one check, keeping its pool and scratch URL.
2808 fn with_check(ctx: &mut GateCtx, check: crate::config::MigrationCheck) {
2809 let mut cfg = crate::config::AppConfig::for_tests();
2810 cfg.scratch_db_url = ctx.cfg.scratch_db_url.clone();
2811 cfg.backup_max_age_hours = ctx.cfg.backup_max_age_hours;
2812 cfg.logs_root = ctx.cfg.logs_root.clone();
2813 cfg.migration_checks = vec![check];
2814 ctx.cfg = std::sync::Arc::new(cfg);
2815 }
2816
2817 #[tokio::test]
2818 async fn migration_dry_run_blocks_when_a_checks_own_dump_was_never_fetched() {
2819 // The hazard the check list exists for: multithreaded applies its own
2820 // migrations at boot against its own database, so the server's dump must
2821 // never stand in for it. A fetched `server` row with no `multithreaded`
2822 // row is exactly that substitution, and it has to block.
2823 let tmp = tempfile::tempdir().unwrap();
2824 let mut ctx = dry_run_ctx(tmp.path(), 48).await;
2825 with_check(&mut ctx, mt_check());
2826 seed_named_backup(&ctx, "server", 1).await;
2827 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2828
2829 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2830 log.close().await;
2831
2832 let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else {
2833 panic!("a missing multithreaded dump must block");
2834 };
2835 let GateBlocker::NoBackupAvailable { check, backup } = blocker else {
2836 panic!("expected NoBackupAvailable, got {blocker:?}");
2837 };
2838 assert_eq!(backup, "multithreaded", "names the dump that is missing");
2839 assert!(
2840 check.contains("multithreaded/migrations"),
2841 "names the check that wanted it, got {check}"
2842 );
2843 }
2844
2845 #[tokio::test]
2846 async fn migration_dry_run_freshness_is_per_dump() {
2847 // A fresh server dump must not make a 45-day-old multithreaded dump look
2848 // current: the clock is per-database, or the second check inherits the
2849 // first's freshness and the gate is theatre.
2850 let tmp = tempfile::tempdir().unwrap();
2851 let mut ctx = dry_run_ctx(tmp.path(), 48).await;
2852 with_check(&mut ctx, mt_check());
2853 seed_named_backup(&ctx, "server", 1).await;
2854 seed_named_backup(&ctx, "multithreaded", 24 * 45).await;
2855 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2856
2857 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2858 log.close().await;
2859
2860 let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else {
2861 panic!("a 45-day-old multithreaded dump must block");
2862 };
2863 let GateBlocker::BackupStale { check, .. } = blocker else {
2864 panic!("expected BackupStale, got {blocker:?}");
2865 };
2866 assert!(
2867 check.contains("multithreaded/migrations"),
2868 "names the check whose dump is stale, got {check}"
2869 );
2870 }
2871
2872 #[tokio::test]
2873 async fn migration_dry_run_blocks_on_a_stale_backup() {
2874 // The failure this closes: the gate used to check only that a backups row
2875 // existed, so a fetch that silently stopped working left it green against
2876 // an ever-older schema. Sando ran 45 days that way.
2877 let tmp = tempfile::tempdir().unwrap();
2878 let ctx = dry_run_ctx(tmp.path(), 48).await;
2879 seed_backup(&ctx, 24 * 45).await;
2880 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2881
2882 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2883 log.close().await;
2884
2885 let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else {
2886 panic!("a 45-day-old backup must block");
2887 };
2888 let GateBlocker::BackupStale {
2889 age_hours,
2890 max_age_hours,
2891 ..
2892 } = blocker
2893 else {
2894 panic!("expected BackupStale, got {blocker:?}");
2895 };
2896 assert_eq!(max_age_hours, 48);
2897 assert!(
2898 age_hours >= 24 * 45,
2899 "reports the real age, got {age_hours}"
2900 );
2901 }
2902
2903 #[tokio::test]
2904 async fn migration_dry_run_accepts_a_fresh_backup() {
2905 // The other side of the boundary: a backup inside the window must not be
2906 // blocked on freshness. It fails later (there is no such dump on disk),
2907 // which is exactly the proof the age check let it through.
2908 let tmp = tempfile::tempdir().unwrap();
2909 let ctx = dry_run_ctx(tmp.path(), 48).await;
2910 seed_backup(&ctx, 6).await;
2911 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2912
2913 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2914 log.close().await;
2915
2916 assert!(
2917 !matches!(
2918 outcome.status,
2919 crate::outcome::GateStatus::Blocked {
2920 blocker: GateBlocker::BackupStale { .. }
2921 }
2922 ),
2923 "a 6h-old backup is fresh, got {:?}",
2924 outcome.status,
2925 );
2926 }
2927
2928 #[tokio::test]
2929 async fn migration_dry_run_treats_an_unparsable_fetched_at_as_stale() {
2930 // Fail closed: `fetched_at` is daemon-written RFC 3339, so a value that
2931 // will not parse means the row is untrustworthy — and a freshness check
2932 // that shrugs at a timestamp it cannot read is not a freshness check.
2933 let tmp = tempfile::tempdir().unwrap();
2934 let ctx = dry_run_ctx(tmp.path(), 48).await;
2935 sqlx::query(
2936 "INSERT INTO backups (fetched_at, source, local_path, byte_size)
2937 VALUES ('not-a-timestamp', 'file:///x.sql.gz', '/tmp/x.sql.gz', 1000000)",
2938 )
2939 .execute(&ctx.pool)
2940 .await
2941 .unwrap();
2942 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2943
2944 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2945 log.close().await;
2946
2947 assert!(
2948 matches!(
2949 outcome.status,
2950 crate::outcome::GateStatus::Blocked {
2951 blocker: GateBlocker::BackupStale { .. }
2952 }
2953 ),
2954 "an unreadable fetched_at must block, got {:?}",
2955 outcome.status,
2956 );
2957 }
2958
2959 /// A `code_smoke` live log over `ctx.cfg.logs_root`, for the helpers that
2960 /// take one. `GateRunId(0)` never matches a real row; nothing reads the
2961 /// chunk events in these tests.
2962 async fn test_gate_log(ctx: &GateCtx) -> GateLog {
2963 GateLog::open(ctx, GateRunId(0), GateKind::CodeSmoke).await
2964 }
2965
2966 /// Close `log` (flushing it) and read back what it wrote on disk.
2967 async fn read_gate_log(ctx: &GateCtx, log: GateLog) -> String {
2968 log.close().await;
2969 tokio::fs::read_to_string(ctx.log_path(GateKind::CodeSmoke))
2970 .await
2971 .expect("the gate log must exist on disk")
2972 }
2973
2974 /// Write a minimal npm project at `worktree/<dir>` whose `build` script
2975 /// exits with `exit_code`. Pre-creates `node_modules` so the gate skips
2976 /// `npm ci` — these tests are about the build step, not the network.
2977 fn fake_npm_project(worktree: &std::path::Path, dir: &str, exit_code: u8) {
2978 let root = worktree.join(dir);
2979 std::fs::create_dir_all(root.join("node_modules")).unwrap();
2980 std::fs::write(
2981 root.join("package.json"),
2982 format!(
2983 r#"{{"name":"fake","version":"0.0.0","private":true,
2984 "scripts":{{"build":"exit {exit_code}"}}}}"#
2985 ),
2986 )
2987 .unwrap();
2988 }
2989
2990 #[tokio::test]
2991 async fn frontend_gate_fails_on_a_build_error_and_names_the_project() {
2992 // The whole point of the gate: the app build scripts downgrade this to a
2993 // cargo::warning, so if it passes here nothing stops a stale bundle.
2994 let tmp = tempfile::tempdir().unwrap();
2995 fake_npm_project(tmp.path(), "server/frontend", 0);
2996 fake_npm_project(tmp.path(), "multithreaded/frontend", 2);
2997 let ctx = frontend_ctx(tmp.path(), &["server/frontend", "multithreaded/frontend"]).await;
2998
2999 let log = test_gate_log(&ctx).await;
3000 let outcome = code_smoke_frontends(&ctx, &log)
3001 .await
3002 .expect("a failing tsc must fail the gate");
3003 let crate::outcome::GateStatus::Failed { failure } = &outcome.status else {
3004 panic!("expected a failure, got {:?}", outcome.status)
3005 };
3006 assert!(
3007 matches!(
3008 failure,
3009 GateFailure::CodeSmokeFrontend { dir, exit_code: Some(2) }
3010 if dir == "multithreaded/frontend"
3011 ),
3012 "got: {failure:?}"
3013 );
3014 // The passing project ran first; its output belongs in the log too.
3015 let text = read_gate_log(&ctx, log).await;
3016 assert!(text.contains("server/frontend"), "log: {text}");
3017 }
3018
3019 #[tokio::test]
3020 async fn frontend_gate_passes_when_every_project_builds() {
3021 let tmp = tempfile::tempdir().unwrap();
3022 fake_npm_project(tmp.path(), "server/frontend", 0);
3023 let ctx = frontend_ctx(tmp.path(), &["server/frontend"]).await;
3024 let log = test_gate_log(&ctx).await;
3025 assert!(
3026 code_smoke_frontends(&ctx, &log).await.is_none(),
3027 "a clean build must not fail the gate"
3028 );
3029 }
3030
3031 /// The point of routing `code_smoke` through `LiveLog`: its output reaches
3032 /// the operator *while* the gate runs, as `GateLogChunk` events, instead of
3033 /// appearing all at once when the gate finishes.
3034 #[tokio::test]
3035 async fn code_smoke_streams_chunks_as_it_runs() {
3036 let tmp = tempfile::tempdir().unwrap();
3037 fake_npm_project(tmp.path(), "server/frontend", 0);
3038 let ctx = frontend_ctx(tmp.path(), &["server/frontend"]).await;
3039 let mut rx = ctx.events.subscribe_logs();
3040
3041 let log = GateLog::open(&ctx, GateRunId(7), GateKind::CodeSmoke).await;
3042 assert!(code_smoke_frontends(&ctx, &log).await.is_none());
3043 log.close().await;
3044
3045 let mut chunks = Vec::new();
3046 while let Ok(envelope) = rx.try_recv() {
3047 if let Event::GateLogChunk { run_id, seq, text } = envelope.event {
3048 assert_eq!(run_id, GateRunId(7));
3049 chunks.push((seq, text));
3050 }
3051 }
3052 assert!(!chunks.is_empty(), "no chunk ever reached the bus");
3053 // Sequence numbers are per-run and monotonic across every step of the
3054 // gate, which is why the whole gate shares one sink.
3055 let seqs: Vec<u32> = chunks.iter().map(|(seq, _)| *seq).collect();
3056 assert!(
3057 seqs.windows(2).all(|w| w[0] < w[1]),
3058 "chunk seq must be monotonic, got {seqs:?}"
3059 );
3060 let joined: String = chunks.into_iter().map(|(_, text)| text).collect();
3061 assert!(joined.contains("server/frontend"), "chunks: {joined}");
3062 }
3063
3064 #[tokio::test]
3065 async fn frontend_gate_skips_a_project_absent_from_the_worktree() {
3066 // Rebuilding an older sha that predates the frontend must stay possible.
3067 let tmp = tempfile::tempdir().unwrap();
3068 let ctx = frontend_ctx(tmp.path(), &["multithreaded/frontend"]).await;
3069 let log = test_gate_log(&ctx).await;
3070 assert!(code_smoke_frontends(&ctx, &log).await.is_none());
3071 assert!(
3072 read_gate_log(&ctx, log).await.contains("skipping"),
3073 "the skip must be visible in the log"
3074 );
3075 }
3076
3077 #[tokio::test]
3078 async fn cargo_test_fails_closed_when_no_target_exists_in_the_worktree() {
3079 // A worktree missing every configured crate must not report "tests
3080 // passed" having run none. Uses an empty tempdir as the worktree, so
3081 // no cargo process is ever spawned.
3082 let tmp = tempfile::tempdir().unwrap();
3083 let mut cfg = crate::config::AppConfig::for_tests();
3084 cfg.test_targets = vec![target("server"), target("mnw-cli")];
3085 cfg.logs_root = tmp.path().join("logs");
3086 let pool = SqlitePoolOptions::new()
3087 .max_connections(1)
3088 .connect("sqlite::memory:")
3089 .await
3090 .unwrap();
3091 let ctx = GateCtx {
3092 public_url: None,
3093 pool,
3094 cfg: std::sync::Arc::new(cfg),
3095 tier: TierId::new("host"),
3096 version: "0.1.0".parse().unwrap(),
3097 worktree: Some(tmp.path().to_path_buf()),
3098 bundle: None,
3099 events: events::channel(),
3100 nodes: Vec::new(),
3101 build_id: None,
3102 aux_dirs: HashMap::new(),
3103 };
3104 let out = cargo_test(&ctx, GateRunId(1)).await.unwrap();
3105 assert_eq!(
3106 out.status_str(),
3107 "failed",
3108 "green here would be a silent no-op gate"
3109 );
3110 let crate::outcome::GateStatus::Failed { failure } = &out.status else {
3111 panic!("expected a failure")
3112 };
3113 assert!(
3114 failure.summary().contains("ran no targets"),
3115 "got: {}",
3116 failure.summary()
3117 );
3118 }
3119
3120 #[tokio::test]
3121 async fn scratch_db_env_is_opt_in_per_target() {
3122 // Exporting DATABASE_URL knocks sqlx out of offline mode, so a crate
3123 // shipping .sqlx data must not see it.
3124 let mut cfg = crate::config::AppConfig::for_tests();
3125 cfg.scratch_db_url = Some("postgres://sando@127.0.0.1/sando_scratch".into());
3126 let ctx = GateCtx {
3127 public_url: None,
3128 pool: SqlitePoolOptions::new()
3129 .max_connections(1)
3130 .connect_lazy("sqlite::memory:")
3131 .unwrap(),
3132 cfg: std::sync::Arc::new(cfg),
3133 tier: TierId::new("host"),
3134 version: "0.1.0".parse().unwrap(),
3135 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
3136 bundle: None,
3137 events: events::channel(),
3138 nodes: Vec::new(),
3139 build_id: None,
3140 aux_dirs: HashMap::new(),
3141 };
3142 let dir = std::path::Path::new("/tmp/wt/x");
3143
3144 let off = cargo_test_command(&ctx, dir, &target("x"), &[], &[]);
3145 let has_db = |c: &Command| {
3146 c.as_std()
3147 .get_envs()
3148 .any(|(k, v)| k == "DATABASE_URL" && v.is_some())
3149 };
3150 assert!(!has_db(&off), "scratch_db defaults off");
3151
3152 let mut on_target = target("x");
3153 on_target.scratch_db = true;
3154 assert!(
3155 has_db(&cargo_test_command(&ctx, dir, &on_target, &[], &[])),
3156 "opt-in exports it"
3157 );
3158 }
3159
3160 #[tokio::test]
3161 async fn all_features_replaces_the_feature_list() {
3162 let ctx = GateCtx {
3163 public_url: None,
3164 pool: SqlitePoolOptions::new()
3165 .max_connections(1)
3166 .connect_lazy("sqlite::memory:")
3167 .unwrap(),
3168 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
3169 tier: TierId::new("host"),
3170 version: "0.1.0".parse().unwrap(),
3171 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
3172 bundle: None,
3173 events: events::channel(),
3174 nodes: Vec::new(),
3175 build_id: None,
3176 aux_dirs: HashMap::new(),
3177 };
3178 let mut t = target("shared/ops-exec");
3179 t.all_features = true;
3180 let cmd = cargo_test_command(&ctx, std::path::Path::new("/tmp/wt"), &t, &[], &[]);
3181 let args: Vec<_> = cmd
3182 .as_std()
3183 .get_args()
3184 .map(|a| a.to_string_lossy().into_owned())
3185 .collect();
3186 assert!(args.iter().any(|a| a == "--all-features"), "got: {args:?}");
3187 assert!(!args.iter().any(|a| a == "--features"), "got: {args:?}");
3188 }
3189
3190 #[test]
3191 fn every_gate_kind_round_trips_through_its_wire_string() {
3192 // gate_kind is a TEXT column and a WS event field; as_str and FromStr
3193 // disagreeing would make a gate's evidence unreadable by
3194 // unsatisfied_gates, which fails the promote closed with no explanation.
3195 for k in [
3196 GateKind::CargoTest,
3197 GateKind::HardeningTest,
3198 GateKind::Clippy,
3199 GateKind::Fmt,
3200 GateKind::CargoAudit,
3201 GateKind::CargoDeny,
3202 GateKind::MigrationDryRun,
3203 GateKind::CodeSmoke,
3204 GateKind::BootSmoke,
3205 GateKind::NodeHealth,
3206 GateKind::BurnIn,
3207 GateKind::ManualConfirm,
3208 ] {
3209 assert_eq!(
3210 k.as_str().parse::<GateKind>().unwrap(),
3211 k,
3212 "round trip for {k:?}"
3213 );
3214 }
3215 }
3216
3217 #[test]
3218 fn first_meaningful_line_prefers_the_diagnostic() {
3219 let stderr = b" Updating crates.io index\nerror: 1 vulnerability found!\n";
3220 assert_eq!(
3221 first_meaningful_line(b"", stderr),
3222 "error: 1 vulnerability found!"
3223 );
3224 }
3225
3226 #[test]
3227 fn first_meaningful_line_finds_a_deny_verdict() {
3228 let out = b"advisories FAILED, bans ok, licenses FAILED, sources ok\n";
3229 assert!(first_meaningful_line(out, b"").contains("FAILED"));
3230 }
3231
3232 #[test]
3233 fn first_meaningful_line_falls_back_rather_than_returning_empty() {
3234 assert!(first_meaningful_line(b"", b"").contains("see the gate log"));
3235 }
3236
3237 #[tokio::test]
3238 async fn supply_chain_gates_skip_crates_without_their_config() {
3239 // Four crates in this repo fail `cargo audit` purely for want of a
3240 // triaged .cargo/audit.toml. Running it there would make the gate
3241 // permanently red, so a target only qualifies once it carries the file.
3242 // The worktree here has a Cargo.toml but no audit config, so nothing
3243 // qualifies and the gate fails closed rather than passing over zero work.
3244 let tmp = tempfile::tempdir().unwrap();
3245 let crate_dir = tmp.path().join("server");
3246 std::fs::create_dir_all(&crate_dir).unwrap();
3247 std::fs::write(crate_dir.join("Cargo.toml"), "[package]\nname = \"x\"\n").unwrap();
3248
3249 let mut cfg = crate::config::AppConfig::for_tests();
3250 cfg.test_targets = vec![target("server")];
3251 cfg.logs_root = tmp.path().join("logs");
3252 let ctx = GateCtx {
3253 public_url: None,
3254 pool: SqlitePoolOptions::new()
3255 .max_connections(1)
3256 .connect("sqlite::memory:")
3257 .await
3258 .unwrap(),
3259 cfg: std::sync::Arc::new(cfg),
3260 tier: TierId::new("host"),
3261 version: "0.1.0".parse().unwrap(),
3262 worktree: Some(tmp.path().to_path_buf()),
3263 bundle: None,
3264 events: events::channel(),
3265 nodes: Vec::new(),
3266 build_id: None,
3267 aux_dirs: HashMap::new(),
3268 };
3269 let out = supply_chain(&ctx, GateRunId(1), GateKind::CargoAudit)
3270 .await
3271 .unwrap();
3272 assert_eq!(out.status_str(), "failed");
3273 let crate::outcome::GateStatus::Failed { failure } = &out.status else {
3274 panic!("expected a failure")
3275 };
3276 assert!(
3277 failure.summary().contains("ran nothing"),
3278 "got: {}",
3279 failure.summary()
3280 );
3281
3282 // Drop the config in and the same target now qualifies.
3283 std::fs::create_dir_all(crate_dir.join(".cargo")).unwrap();
3284 std::fs::write(crate_dir.join(".cargo/audit.toml"), "[advisories]\n").unwrap();
3285 // The fixture crate is not a real cargo project, so the tool itself
3286 // still errors — but on its own terms, not with "ran nothing". That
3287 // distinction is the thing under test: the target was attempted.
3288 let out = supply_chain(&ctx, GateRunId(2), GateKind::CargoAudit)
3289 .await
3290 .unwrap();
3291 if let crate::outcome::GateStatus::Failed { failure } = &out.status {
3292 assert!(
3293 !failure.summary().contains("ran nothing"),
3294 "a target carrying the config must be attempted, not skipped; got: {}",
3295 failure.summary(),
3296 );
3297 }
3298 }
3299
3300 #[test]
3301 fn tests_run_reads_the_libtest_summary() {
3302 let out = b"running 6 tests\ntest auth_rate_limit_triggers_on_burst ... ok\n\n\
3303 test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 118 filtered out\n";
3304 assert_eq!(tests_run(out), 6);
3305 }
3306
3307 #[test]
3308 fn tests_run_sums_across_test_binaries() {
3309 let out = b"test result: ok. 6 passed; 0 failed\ntest result: ok. 2 passed; 0 failed\n";
3310 assert_eq!(tests_run(out), 8);
3311 }
3312
3313 #[test]
3314 fn tests_run_is_zero_when_the_filter_matched_nothing() {
3315 // The case hardening_test fails closed on: a filter matching no tests
3316 // exits 0, so a rename would otherwise make the gate a green no-op.
3317 let out = b"running 0 tests\n\n\
3318 test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 124 filtered out\n";
3319 assert_eq!(tests_run(out), 0);
3320 }
3321
3322 #[test]
3323 fn tests_run_is_zero_without_a_summary_line() {
3324 assert_eq!(tests_run(b"error: could not compile `makenotwork`\n"), 0);
3325 }
3326
3327 #[tokio::test]
3328 async fn hardening_test_command_carries_no_features() {
3329 // The entire point of the gate: production constants, which means no
3330 // `fast-tests`. A stray feature here silently restores the blind spot.
3331 let ctx = GateCtx {
3332 public_url: None,
3333 pool: SqlitePoolOptions::new()
3334 .max_connections(1)
3335 .connect_lazy("sqlite::memory:")
3336 .unwrap(),
3337 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
3338 tier: TierId::new("host"),
3339 version: "0.1.0".parse().unwrap(),
3340 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
3341 bundle: None,
3342 events: events::channel(),
3343 nodes: Vec::new(),
3344 build_id: None,
3345 aux_dirs: HashMap::new(),
3346 };
3347 let plain = crate::config::TestTarget {
3348 dir: std::path::PathBuf::from("server"),
3349 aux_repo: None,
3350 features: Vec::new(),
3351 all_features: false,
3352 scratch_db: true,
3353 };
3354 let cmd = cargo_test_command(
3355 &ctx,
3356 std::path::Path::new("/tmp/wt/server"),
3357 &plain,
3358 &[],
3359 &["--test", "integration"],
3360 );
3361 let args: Vec<_> = cmd
3362 .as_std()
3363 .get_args()
3364 .map(|a| a.to_string_lossy().into_owned())
3365 .collect();
3366 assert!(
3367 !args.iter().any(|a| a == "--features"),
3368 "hardening_test must pass no features: {args:?}"
3369 );
3370 assert!(
3371 !args.iter().any(|a| a.contains("fast-tests")),
3372 "got: {args:?}"
3373 );
3374
3375 let fast_target = crate::config::TestTarget {
3376 dir: std::path::PathBuf::from("server"),
3377 aux_repo: None,
3378 features: vec!["fast-tests".into()],
3379 all_features: false,
3380 scratch_db: true,
3381 };
3382 let fast = cargo_test_command(
3383 &ctx,
3384 std::path::Path::new("/tmp/wt/server"),
3385 &fast_target,
3386 &["fast-tests"],
3387 &[],
3388 );
3389 let fast_args: Vec<_> = fast
3390 .as_std()
3391 .get_args()
3392 .map(|a| a.to_string_lossy().into_owned())
3393 .collect();
3394 assert!(
3395 fast_args
3396 .windows(2)
3397 .any(|w| w == ["--features", "fast-tests"]),
3398 "got: {fast_args:?}"
3399 );
3400 }
3401
3402 /// Spawn a one-shot loopback server that answers the first connection with
3403 /// `status_line` + a tiny body, then closes. Returns the bound port.
3404 async fn oneshot_http(status_line: &'static str) -> u16 {
3405 use tokio::io::{AsyncReadExt, AsyncWriteExt};
3406 let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
3407 .await
3408 .unwrap();
3409 let port = listener.local_addr().unwrap().port();
3410 tokio::spawn(async move {
3411 if let Ok((mut sock, _)) = listener.accept().await {
3412 let mut scratch = [0u8; 1024];
3413 let _ = sock.read(&mut scratch).await; // drain the request line
3414 let resp =
3415 format!("{status_line}\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok");
3416 let _ = sock.write_all(resp.as_bytes()).await;
3417 }
3418 });
3419 port
3420 }
3421
3422 #[tokio::test]
3423 async fn probe_health_ok_on_200() {
3424 let port = oneshot_http("HTTP/1.1 200 OK").await;
3425 assert!(probe_health(port).await.is_ok());
3426 }
3427
3428 #[tokio::test]
3429 async fn probe_health_err_on_non_200() {
3430 let port = oneshot_http("HTTP/1.1 503 Service Unavailable").await;
3431 let err = probe_health(port).await.unwrap_err();
3432 assert!(err.contains("status line"), "{err}");
3433 }
3434
3435 #[tokio::test]
3436 async fn probe_health_err_on_connection_refused() {
3437 // Bind then drop to get an almost-certainly-free port nothing listens on.
3438 let port = {
3439 let l = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
3440 .await
3441 .unwrap();
3442 l.local_addr().unwrap().port()
3443 };
3444 let err = probe_health(port).await.unwrap_err();
3445 // Under load the kernel does not always refuse the connect: a socket
3446 // left in TIME_WAIT on that port completes the handshake and then
3447 // resets, so the failure surfaces on the write or the read instead.
3448 // Refused-on-connect and reset-on-write/read are the same fact, that
3449 // nothing is serving the port, and no other outcome counts as a pass.
3450 let refused = err.starts_with("connect: ") && err.contains("refused");
3451 let reset =
3452 (err.starts_with("write: ") || err.starts_with("read: ")) && err.contains("reset");
3453 assert!(refused || reset, "{err}");
3454 }
3455
3456 /// burn_in returns a typed Blocked when the clock isn't started; the
3457 /// runner persists status='blocked' + outcome_json (the json carries
3458 /// blocker.kind = 'burn_in_clock_not_started').
3459 #[tokio::test]
3460 async fn burn_in_blocked_persists_typed_outcome() {
3461 let pool = SqlitePoolOptions::new()
3462 .max_connections(1)
3463 .connect("sqlite::memory:")
3464 .await
3465 .unwrap();
3466 crate::db::migrate(&pool).await.unwrap();
3467 // Topology sync expects a tier row before gate_runs can reference it.
3468 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 0, 'sequential')")
3469 .execute(&pool).await.unwrap();
3470 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
3471 .execute(&pool)
3472 .await
3473 .unwrap();
3474 // versions FK target.
3475 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
3476 .execute(&pool).await.unwrap();
3477
3478 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests());
3479 let ctx = GateCtx {
3480 public_url: None,
3481 pool: pool.clone(),
3482 cfg,
3483 tier: TierId::new("host"),
3484 version: "0.1.0".parse().unwrap(),
3485 worktree: Some(std::path::PathBuf::from("/tmp/unused")),
3486 bundle: None,
3487 events: events::channel(),
3488 nodes: Vec::new(),
3489 build_id: None,
3490 aux_dirs: HashMap::new(),
3491 };
3492 let out = run(&ctx, &Gate::BurnIn { hours: 24 }).await.unwrap();
3493 assert_eq!(out.status_str(), "blocked");
3494 assert!(!out.is_passed());
3495
3496 // Read the persisted row.
3497 let row: (Option<String>, Option<String>) =
3498 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
3499 .fetch_one(&pool)
3500 .await
3501 .unwrap();
3502 assert_eq!(row.0.as_deref(), Some("blocked"), "typed status");
3503 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
3504 assert_eq!(json["status"]["kind"], "blocked");
3505 assert_eq!(
3506 json["status"]["blocker"]["kind"],
3507 "burn_in_clock_not_started"
3508 );
3509 }
3510
3511 /// node_health fails closed when there are no nodes to probe: a serving tier
3512 /// should always carry nodes, so an empty set is a misconfiguration that must
3513 /// block promotion, not pass it.
3514 #[tokio::test]
3515 async fn node_health_blocks_with_no_nodes() {
3516 let pool = SqlitePoolOptions::new()
3517 .max_connections(1)
3518 .connect("sqlite::memory:")
3519 .await
3520 .unwrap();
3521 crate::db::migrate(&pool).await.unwrap();
3522 sqlx::query(
3523 "INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('b', 2, 1, 'sequential')",
3524 )
3525 .execute(&pool)
3526 .await
3527 .unwrap();
3528 sqlx::query("INSERT INTO tier_state (tier) VALUES ('b')")
3529 .execute(&pool)
3530 .await
3531 .unwrap();
3532 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
3533 .execute(&pool).await.unwrap();
3534
3535 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests());
3536 let ctx = GateCtx {
3537 public_url: None,
3538 pool: pool.clone(),
3539 cfg,
3540 tier: TierId::new("b"),
3541 version: "0.1.0".parse().unwrap(),
3542 worktree: None,
3543 bundle: None,
3544 events: events::channel(),
3545 nodes: Vec::new(), // no nodes -> fail closed
3546 build_id: None,
3547 aux_dirs: HashMap::new(),
3548 };
3549 let out = run(&ctx, &Gate::NodeHealth).await.unwrap();
3550 assert_eq!(out.status_str(), "blocked");
3551 assert!(!out.is_passed());
3552 let row: (Option<String>, Option<String>) =
3553 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
3554 .fetch_one(&pool)
3555 .await
3556 .unwrap();
3557 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
3558 assert_eq!(json["status"]["blocker"]["kind"], "no_nodes_to_probe");
3559 }
3560
3561 /// reset_scratch must drop every non-system schema, not just `public` —
3562 /// otherwise migrations that create custom schemas (e.g. tower_sessions)
3563 /// collide on the next run. This regressed once (Phase 0) and the fix is
3564 /// load-bearing for migration_dry_run.
3565 ///
3566 /// Gated on `SANDO_TEST_PG_URL` so it only runs where postgres is
3567 /// available. Set `SANDO_TEST_PG_URL=postgres:///sando_scratch?host=/var/run/postgresql`
3568 /// (or similar) before `cargo test`.
3569 #[tokio::test]
3570 async fn reset_scratch_drops_all_non_system_schemas() {
3571 let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else {
3572 eprintln!("skipping: SANDO_TEST_PG_URL not set");
3573 return;
3574 };
3575 use sqlx::Executor;
3576 use sqlx::postgres::PgPoolOptions;
3577
3578 let pool = PgPoolOptions::new()
3579 .max_connections(1)
3580 .connect(&url)
3581 .await
3582 .unwrap();
3583 // Plant two non-system schemas + a table in each.
3584 pool.execute(
3585 "DROP SCHEMA IF EXISTS foo CASCADE; CREATE SCHEMA foo; CREATE TABLE foo.t (i int);",
3586 )
3587 .await
3588 .unwrap();
3589 pool.execute("DROP SCHEMA IF EXISTS tower_sessions CASCADE; CREATE SCHEMA tower_sessions; CREATE TABLE tower_sessions.session (id text);")
3590 .await.unwrap();
3591 pool.close().await;
3592
3593 reset_scratch(&url, "makenotwork")
3594 .await
3595 .expect("reset_scratch");
3596
3597 let pool = PgPoolOptions::new()
3598 .max_connections(1)
3599 .connect(&url)
3600 .await
3601 .unwrap();
3602 let rows: Vec<(String,)> = sqlx::query_as(
3603 "SELECT nspname FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname <> 'information_schema'",
3604 )
3605 .fetch_all(&pool)
3606 .await
3607 .unwrap();
3608 let names: Vec<String> = rows.into_iter().map(|(s,)| s).collect();
3609 // After reset, only `public` should remain among non-system schemas.
3610 assert_eq!(names, vec!["public".to_string()], "got: {names:?}");
3611 pool.close().await;
3612 }
3613
3614 /// reset_scratch must leave the dump's owner role existing and able to
3615 /// create in `public`, because a prod `pg_dump` carries `ALTER ... OWNER TO
3616 /// <role>` for every object. This was satisfied by a hand-created NOLOGIN
3617 /// role on fw13; nothing recorded it, so any other box failed
3618 /// migration_dry_run at the restore with "role does not exist".
3619 ///
3620 /// Uses a throwaway role name so it can prove the *creation* path rather
3621 /// than passing on fw13's pre-existing `makenotwork`. Same
3622 /// `SANDO_TEST_PG_URL` gate as above; needs a superuser connection.
3623 #[tokio::test]
3624 async fn reset_scratch_seeds_the_dump_owner_role_when_absent() {
3625 let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else {
3626 eprintln!("skipping: SANDO_TEST_PG_URL not set");
3627 return;
3628 };
3629 use sqlx::Executor;
3630 use sqlx::postgres::PgPoolOptions;
3631
3632 let role = "sando_test_owner_probe";
3633 // `DROP ROLE` refuses while the role still holds the grants reset_scratch
3634 // gave it, so drop what it owns first. Idempotent, and a no-op when the
3635 // role is absent (the usual case on a first run).
3636 let drop_role = format!(
3637 "DO $$ BEGIN
3638 IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{role}') THEN
3639 EXECUTE 'DROP OWNED BY {role}';
3640 EXECUTE 'DROP ROLE {role}';
3641 END IF;
3642 END $$;"
3643 );
3644
3645 let pool = PgPoolOptions::new()
3646 .max_connections(1)
3647 .connect(&url)
3648 .await
3649 .unwrap();
3650 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(drop_role.clone())))
3651 .await
3652 .unwrap();
3653 pool.close().await;
3654
3655 reset_scratch(&url, role)
3656 .await
3657 .expect("reset_scratch creates the owner role");
3658
3659 let pool = PgPoolOptions::new()
3660 .max_connections(1)
3661 .connect(&url)
3662 .await
3663 .unwrap();
3664 let (exists, can_login): (bool, bool) =
3665 sqlx::query_as("SELECT true, rolcanlogin FROM pg_roles WHERE rolname = $1")
3666 .bind(role)
3667 .fetch_one(&pool)
3668 .await
3669 .expect("owner role exists after reset");
3670 assert!(exists);
3671 assert!(
3672 !can_login,
3673 "the owner role is an owner only, never a login identity"
3674 );
3675
3676 // The restore's owner-scoped DDL needs CREATE on public in the role's
3677 // own right (PG15+ dropped the implicit grant).
3678 let (has_create,): (bool,) =
3679 sqlx::query_as("SELECT pg_catalog.has_schema_privilege($1, 'public', 'CREATE')")
3680 .bind(role)
3681 .fetch_one(&pool)
3682 .await
3683 .unwrap();
3684 assert!(has_create, "owner role must be able to create in public");
3685
3686 // Idempotent: a second reset must not error on the now-existing role.
3687 pool.close().await;
3688 reset_scratch(&url, role)
3689 .await
3690 .expect("reset_scratch is idempotent");
3691
3692 let pool = PgPoolOptions::new()
3693 .max_connections(1)
3694 .connect(&url)
3695 .await
3696 .unwrap();
3697 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(drop_role)))
3698 .await
3699 .unwrap();
3700 pool.close().await;
3701 }
3702
3703 /// The preflight must pass against a privileged scratch connection. Guards
3704 /// the catalog query itself: a wrong column or a `current_user` that matches
3705 /// no `pg_roles` row would make `fetch_one` error (or, worse, a silently
3706 /// swapped pair would invert the check) and brick startup for everyone.
3707 /// `SANDO_TEST_PG_URL` is expected to be a superuser connection, as the
3708 /// gates require.
3709 #[tokio::test]
3710 async fn preflight_passes_on_a_privileged_scratch_connection() {
3711 let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else {
3712 eprintln!("skipping: SANDO_TEST_PG_URL not set");
3713 return;
3714 };
3715 preflight_scratch_privileges(&url)
3716 .await
3717 .expect("a superuser scratch connection must satisfy the preflight");
3718 }
3719
3720 /// CF4: the restore pipeline must carry `ON_ERROR_STOP=1` (so psql fails on
3721 /// a bad statement instead of exiting 0 on a partial restore) and, for a
3722 /// gzip source, `set -o pipefail` (so a `gunzip` failure on a truncated
3723 /// archive isn't masked by psql's exit). Pure string check — no postgres.
3724 #[test]
3725 fn restore_shell_has_error_stop_and_pipefail() {
3726 let gz = restore_shell("postgres:///scratch", "/srv/sando/backups/latest.sql.gz");
3727 assert!(gz.contains("ON_ERROR_STOP=1"), "gz: {gz}");
3728 assert!(gz.contains("set -o pipefail"), "gz: {gz}");
3729 assert!(gz.contains("gunzip -c"), "gz: {gz}");
3730
3731 let plain = restore_shell("postgres:///scratch", "/srv/sando/backups/dump.sql");
3732 assert!(plain.contains("ON_ERROR_STOP=1"), "plain: {plain}");
3733 // No pipeline for a plain .sql, so pipefail is unnecessary there.
3734 assert!(!plain.contains("gunzip"), "plain: {plain}");
3735 // The db url is single-quote escaped in both forms.
3736 assert!(plain.contains("'postgres:///scratch'"), "plain: {plain}");
3737 }
3738
3739 #[test]
3740 fn split_pg_password_extracts_and_sanitizes() {
3741 // Password lifted out of the URL; the sanitized form keeps user/host/db.
3742 let (url, pw) = split_pg_password("postgres://sando:s3cret@db.host:5432/scratch");
3743 assert_eq!(url, "postgres://sando@db.host:5432/scratch");
3744 assert_eq!(pw.as_deref(), Some("s3cret"));
3745 // Percent-encoded password is decoded for PGPASSWORD.
3746 let (url, pw) = split_pg_password("postgresql://u:p%40ss%2Fword@h/d");
3747 assert_eq!(url, "postgresql://u@h/d");
3748 assert_eq!(pw.as_deref(), Some("p@ss/word"));
3749 }
3750
3751 #[test]
3752 fn split_pg_password_noop_without_password() {
3753 // No userinfo password -> unchanged, None. (A ':' after the '@', e.g. a
3754 // port, must not be mistaken for the password delimiter.)
3755 assert_eq!(
3756 split_pg_password("postgres:///scratch"),
3757 ("postgres:///scratch".to_string(), None),
3758 );
3759 assert_eq!(
3760 split_pg_password("postgres://sando@db.host:5432/scratch"),
3761 ("postgres://sando@db.host:5432/scratch".to_string(), None),
3762 );
3763 }
3764
3765 #[test]
3766 fn percent_decode_handles_escapes_and_malformed() {
3767 assert_eq!(percent_decode("plain"), "plain");
3768 assert_eq!(percent_decode("a%2Fb"), "a/b");
3769 // A malformed trailing escape is left literal, not dropped.
3770 assert_eq!(percent_decode("ab%2"), "ab%2");
3771 assert_eq!(percent_decode("ab%zz"), "ab%zz");
3772 }
3773
3774 #[test]
3775 fn pg_url_with_dbname_rewrites_the_database() {
3776 // user:pass@host:port/db?query — swap db, keep everything else.
3777 assert_eq!(
3778 pg_url_with_dbname(
3779 "postgres://sando:pw@db.host:5432/sando_scratch?sslmode=require",
3780 "postgres"
3781 ),
3782 "postgres://sando:pw@db.host:5432/postgres?sslmode=require",
3783 );
3784 // Socket form: the query carries `host=/var/run/postgresql` and must survive.
3785 assert_eq!(
3786 pg_url_with_dbname(
3787 "postgres:///sando_scratch?host=/var/run/postgresql",
3788 "sando_code_smoke_0_9_6"
3789 ),
3790 "postgres:///sando_code_smoke_0_9_6?host=/var/run/postgresql",
3791 );
3792 // Plain host/db, no query.
3793 assert_eq!(
3794 pg_url_with_dbname("postgres://localhost/scratch", "postgres"),
3795 "postgres://localhost/scratch".replace("scratch", "postgres"),
3796 );
3797 // No authority, no query (loopback socket, default db path).
3798 assert_eq!(
3799 pg_url_with_dbname("postgres:///scratch", "postgres"),
3800 "postgres:///postgres",
3801 );
3802 }
3803
3804 #[test]
3805 fn bytes_contain_matches_listening_in_log_output() {
3806 // JSON release log carries the message field verbatim.
3807 assert!(bytes_contain(
3808 br#"{"timestamp":"...","level":"INFO","fields":{"message":"listening","addr":"127.0.0.1:18182"}}"#,
3809 b"listening",
3810 ));
3811 // Human-format dev log.
3812 assert!(bytes_contain(
3813 b"2026-07-17 INFO makenotwork: listening addr=127.0.0.1:18182",
3814 b"listening"
3815 ));
3816 assert!(!bytes_contain(
3817 b"migrations complete; seeding catalog",
3818 b"listening"
3819 ));
3820 assert!(!bytes_contain(b"", b"listening"));
3821 }
3822
3823 #[tokio::test]
3824 async fn code_smoke_env_supplies_every_mandatory_server_var() {
3825 // code_smoke is the only gate that reaches the server's Config::from_env
3826 // (boot_smoke and the docs check short-circuit before it), so anything
3827 // that function requires has to be answered here. CDN_BASE_URL became
3828 // mandatory 13 days after this env was written and went unnoticed until
3829 // the gate first ran on the host; this test is what makes the next one
3830 // fail here instead of in a promote.
3831 let ctx = resolving_ctx("/w/abc", &[]);
3832 let mut cmd = tokio::process::Command::new("true");
3833 code_smoke_env(&mut cmd, &ctx, "postgres:///throwaway");
3834 let set: std::collections::HashMap<String, String> = cmd
3835 .as_std()
3836 .get_envs()
3837 .filter_map(|(k, v)| Some((k.to_str()?.to_string(), v?.to_str()?.to_string())))
3838 .collect();
3839 for key in [
3840 "DATABASE_URL",
3841 "HOST",
3842 "PORT",
3843 "HOST_URL",
3844 "CDN_BASE_URL",
3845 "SIGNING_SECRET",
3846 ] {
3847 assert!(set.contains_key(key), "code_smoke_env must set {key}");
3848 assert!(!set[key].is_empty(), "{key} must not be empty");
3849 }
3850 // Loopback, so Config::from_env's is_production branch stays false and
3851 // the gate never trips MissingPublicBucket for want of an S3 bucket.
3852 assert!(set["HOST_URL"].starts_with("http://localhost"));
3853 // Not an IP literal: the server derives its WebAuthn rp_id from this
3854 // host, and WebauthnBuilder rejects an origin whose Url::domain() is
3855 // None, which is every IP address. An IP here cannot boot the server.
3856 assert!(
3857 url_host_is_a_domain(&set["HOST_URL"]),
3858 "HOST_URL host must be a domain, not an IP literal: {}",
3859 set["HOST_URL"],
3860 );
3861 assert_eq!(set["HOST"], "127.0.0.1");
3862 // The signing secret has to clear the server's 32-char floor, or the
3863 // gate fails with WeakSigningSecret instead of testing anything.
3864 assert!(set["SIGNING_SECRET"].len() >= 32);
3865 }
3866
3867 #[tokio::test]
3868 async fn code_smoke_env_passes_extras_through_but_never_lets_them_win() {
3869 // The pass-through exists so a product can hand its own binary a var
3870 // sando has no business knowing about (MNW points SEED_MEDIA_CACHE at a
3871 // persistent dir, because PrivateTmp=true made the seed's media cache
3872 // cold on every build). What it must never become is a way to aim a
3873 // smoke run at a real database.
3874 let mut cfg = crate::config::AppConfig::for_tests();
3875 cfg.code_smoke_env = [
3876 (
3877 "SEED_MEDIA_CACHE".to_string(),
3878 "/srv/sando/seed".to_string(),
3879 ),
3880 (
3881 "DATABASE_URL".to_string(),
3882 "postgres://prod-1/makenotwork".to_string(),
3883 ),
3884 ]
3885 .into_iter()
3886 .collect();
3887 let mut ctx = resolving_ctx("/w/abc", &[]);
3888 ctx.cfg = std::sync::Arc::new(cfg);
3889
3890 let mut cmd = tokio::process::Command::new("true");
3891 code_smoke_env(&mut cmd, &ctx, "postgres:///throwaway");
3892 let set: std::collections::HashMap<String, String> = cmd
3893 .as_std()
3894 .get_envs()
3895 .filter_map(|(k, v)| Some((k.to_str()?.to_string(), v?.to_str()?.to_string())))
3896 .collect();
3897
3898 assert_eq!(set["SEED_MEDIA_CACHE"], "/srv/sando/seed");
3899 assert_eq!(
3900 set["DATABASE_URL"], "postgres:///throwaway",
3901 "the fixed set must overwrite a colliding extra, or a config typo \
3902 could point code_smoke at a real database",
3903 );
3904 }
3905
3906 #[test]
3907 fn code_smoke_db_name_sanitizes_and_caps() {
3908 assert_eq!(
3909 code_smoke_db_name(&"0.9.6".parse().unwrap()),
3910 "sando_code_smoke_0_9_6"
3911 );
3912 // Pre-release/build metadata folds to underscores; result stays [a-z0-9_].
3913 let n = code_smoke_db_name(&"1.0.0-rc.1+build".parse().unwrap());
3914 assert_eq!(n, "sando_code_smoke_1_0_0_rc_1_build");
3915 assert!(
3916 n.bytes()
3917 .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
3918 );
3919 assert!(n.len() <= 63);
3920 }
3921
3922 /// code_smoke is Blocked (not Failed) when the daemon has no scratch_db_url:
3923 /// there's no cluster to create the throwaway DB in, and that's an operator
3924 /// precondition, rendered yellow — the same shape as migration_dry_run.
3925 #[tokio::test]
3926 async fn code_smoke_blocks_without_scratch_db_url() {
3927 let pool = SqlitePoolOptions::new()
3928 .max_connections(1)
3929 .connect("sqlite::memory:")
3930 .await
3931 .unwrap();
3932 crate::db::migrate(&pool).await.unwrap();
3933 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 1, 'sequential')")
3934 .execute(&pool).await.unwrap();
3935 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
3936 .execute(&pool)
3937 .await
3938 .unwrap();
3939 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
3940 .execute(&pool).await.unwrap();
3941
3942 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests()); // scratch_db_url: None
3943 let ctx = GateCtx {
3944 public_url: None,
3945 pool: pool.clone(),
3946 cfg,
3947 tier: TierId::new("host"),
3948 version: "0.1.0".parse().unwrap(),
3949 worktree: Some(std::path::PathBuf::from("/tmp/unused")),
3950 bundle: None,
3951 events: events::channel(),
3952 nodes: Vec::new(),
3953 build_id: None,
3954 aux_dirs: HashMap::new(),
3955 };
3956 let out = run(&ctx, &Gate::CodeSmoke).await.unwrap();
3957 assert_eq!(out.status_str(), "blocked");
3958 assert!(!out.is_passed());
3959 let row: (Option<String>, Option<String>) =
3960 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
3961 .fetch_one(&pool)
3962 .await
3963 .unwrap();
3964 assert_eq!(row.0.as_deref(), Some("blocked"));
3965 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
3966 assert_eq!(json["status"]["blocker"]["kind"], "scratch_db_url_unset");
3967 }
3968
3969 /// Sanity: applying MNW migrations from a *non-existent* dir errors,
3970 /// rather than silently no-op'ing. Cheap pure check, no postgres needed
3971 /// (the sqlx::Migrator::new constructor itself reads the dir).
3972 #[tokio::test]
3973 async fn run_migrator_errors_on_missing_dir() {
3974 // The first thing run_migrator does is `Migrator::new(dir)`, which
3975 // needs a real dir to read migration files from.
3976 let res = run_migrator(
3977 "postgres:///does-not-matter",
3978 std::path::Path::new("/nonexistent/sando-test-migrations"),
3979 )
3980 .await;
3981 assert!(res.is_err());
3982 }
3983 }
3984