Skip to main content

max / makenotwork

157.8 KB · 3909 lines History Blame Raw
1 //! Gate execution. Each gate kind has a runner that produces a pass/fail
2 //! outcome plus an optional detail string (typically a stderr tail or a
3 //! human-readable reason). Outcomes are persisted to `gate_runs` so /state
4 //! and the TUI can show them.
5
6 use crate::classify;
7 use crate::config::AppConfig;
8 use crate::domain::{AppId, GateKind, GateRunId, TierId, Version};
9 use crate::events::{self, Event, EventTx};
10 use crate::outcome::{GateBlocker, GateFailure, GateOutcome, LogRef, PassNote};
11 use crate::topology::Gate;
12 use anyhow::{Context, Result};
13 use chrono::Utc;
14 use ops_core::live_log::LiveLog;
15 use ops_core::remote::LogSink; // brings `LiveLog::write_chunk` (the sink trait) into scope
16 use sqlx::SqlitePool;
17 use std::collections::HashMap;
18 use std::path::Path;
19 use std::path::PathBuf;
20 use std::sync::Arc;
21 use tokio::io::AsyncReadExt;
22 use tokio::process::Command;
23
24 /// The gate live-log callback: emit each chunk as a `GateLogChunk` event so the
25 /// TUI sees the tail stream in real time. `ops_core::live_log::LiveLog` owns the
26 /// disk append and the per-run sequence counter; this closure is the one
27 /// tool-specific bit (Sando previously carried a whole `live_log.rs` copy that
28 /// hardcoded exactly this emit).
29 fn gate_chunk_cb(events: EventTx, run_id: GateRunId) -> ops_core::live_log::ChunkCallback {
30 Box::new(move |seq, text| {
31 events::emit(
32 &events,
33 Event::GateLogChunk {
34 run_id,
35 seq,
36 text: text.to_owned(),
37 },
38 );
39 })
40 }
41
42 pub struct GateCtx {
43 pub pool: SqlitePool,
44 pub cfg: Arc<AppConfig>,
45 pub tier: TierId,
46 pub version: Version,
47 /// The checkout this run's artifact was built from, when there is one.
48 ///
49 /// `None` for an accepted artifact: it was built elsewhere and Sando has no
50 /// source tree for it. That is the boundary made visible (wiki
51 /// [[sando-bento-boundary]]) — artifact-scoped gates belong to the builder,
52 /// so a gate that reads source is one Sando should refuse to run here rather
53 /// than resolve against a path that does not exist.
54 pub worktree: Option<PathBuf>,
55 /// The published, content-addressed bundle this run is about, when it has
56 /// been published yet. `migration_dry_run` prefers it over the worktree, so
57 /// what it proves is inside the digest rather than beside it.
58 pub bundle: Option<PathBuf>,
59 pub events: EventTx,
60 /// Nodes the `node_health` post-deploy gate probes. Empty for build-time
61 /// gate runs on the host (where `node_health` never appears); filled at
62 /// promote time with each freshly-deployed node and its executor.
63 pub nodes: Vec<NodeProbe>,
64 /// The `build_runs.id` this gate run vouches for — the artifact identity
65 /// (wiki [[release-artifact-identity]]). Recorded on every `gate_runs` row so
66 /// promote can resolve the artifact through the evidence for a specific build,
67 /// not through a version string that a later rebuild can silently reuse.
68 /// `None` for legacy/pre-identity runs and gate unit tests.
69 pub build_id: Option<i64>,
70 /// Where each `[[aux_repo]]` is checked out, by topology name. Aux repos sit
71 /// beside the per-sha worktree rather than under it, so a `test_target` that
72 /// names one cannot be resolved against `worktree` alone. Filled from the
73 /// topology at build time; empty at promote time, where the only gate that
74 /// runs is `node_health` and there is no checkout at all.
75 pub aux_dirs: HashMap<String, PathBuf>,
76 /// The tier's public URL, for [`Gate::PageSmoke`]. `None` on every
77 /// build-time run and on any tier that declares none.
78 ///
79 /// [`Gate::PageSmoke`]: crate::topology::Gate::PageSmoke
80 pub public_url: Option<String>,
81 }
82
83 impl GateCtx {
84 /// Absolute directory a `test_target` runs in: under the worktree, or under
85 /// the named aux repo's checkout.
86 ///
87 /// An `aux_repo` naming nothing this run knows about resolves to `None`
88 /// rather than to a wrong path. Callers treat that as "not present in this
89 /// run" and skip, the same as a target missing from an older sha —
90 /// `--check-config` is what stops a genuine typo from reaching here
91 /// (`Topology::ensure_test_target_aux_repos_exist`).
92 pub fn target_dir(&self, target: &crate::config::TestTarget) -> Option<PathBuf> {
93 match target.aux_repo.as_deref() {
94 None => Some(self.worktree.as_ref()?.join(&target.dir)),
95 Some(name) => Some(self.aux_dirs.get(name)?.join(&target.dir)),
96 }
97 }
98
99 /// The checkout, or a typed refusal for a gate that cannot work without one.
100 ///
101 /// Every caller of this is a gate whose evidence is about the *artifact*
102 /// rather than about the artifact in an environment, which the boundary
103 /// assigns to the builder. Reaching this arm means a tier asked Sando to
104 /// re-run a builder's gate against a bundle it was handed, and the honest
105 /// answer is to say so rather than to pass on having run nothing.
106 pub fn worktree_for(&self, gate: GateKind) -> std::result::Result<&Path, GateOutcome> {
107 self.worktree.as_deref().ok_or_else(|| {
108 GateOutcome::failed(GateFailure::NeedsSource {
109 gate,
110 artifact: self.bundle.as_ref().map_or_else(
111 || "an artifact built elsewhere".into(),
112 |b| b.display().to_string(),
113 ),
114 })
115 })
116 }
117
118 /// Where a `migration_check` finds its migrations.
119 ///
120 /// The bundle wins when it carries them. That is the point of shipping
121 /// migrations as a `release_contents` entry: it puts them inside the digest,
122 /// so the dry run proves something about the bytes that ship rather than
123 /// about a checkout that happens to sit next to them. The worktree is the
124 /// fallback for a build whose config has not opted in yet, and for an
125 /// accepted artifact there is no fallback at all — if the builder did not
126 /// bundle its migrations, Sando cannot dry-run them and says so.
127 pub fn migrations_dir(&self, dir: &Path) -> Option<PathBuf> {
128 if let Some(bundle) = &self.bundle {
129 let in_bundle = bundle.join(dir);
130 if in_bundle.is_dir() {
131 return Some(in_bundle);
132 }
133 }
134 let in_worktree = self.worktree.as_ref()?.join(dir);
135 in_worktree.is_dir().then_some(in_worktree)
136 }
137 }
138
139 /// One node the `node_health` gate verifies: its id, the systemd unit to
140 /// confirm active after the restart, an optional HTTP readiness URL, and the
141 /// executor that reaches it (the same transport the deploy used).
142 pub struct NodeProbe {
143 pub node: crate::domain::NodeId,
144 pub service: String,
145 pub health_url: Option<String>,
146 pub executor: Arc<dyn ops_exec::Executor>,
147 }
148
149 /// Run a single gate end-to-end: insert the in-flight row, execute the gate,
150 /// update the row with the outcome. Returns the outcome for the caller.
151 pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> {
152 let kind = gate.kind();
153 let started_at = Utc::now().to_rfc3339();
154
155 let id: i64 = sqlx::query_scalar(
156 "INSERT INTO gate_runs (app, version, tier, gate_kind, started_at, build_id)
157 VALUES (?, ?, ?, ?, ?, ?)
158 RETURNING id",
159 )
160 .bind(&ctx.cfg.id)
161 .bind(&ctx.version)
162 .bind(&ctx.tier)
163 .bind(kind)
164 .bind(&started_at)
165 .bind(ctx.build_id)
166 .fetch_one(&ctx.pool)
167 .await?;
168 let run_id = GateRunId(id);
169
170 tracing::info!(
171 run_id = %run_id, tier = %ctx.tier, version = %ctx.version, gate = %kind,
172 "gate start",
173 );
174 events::emit(
175 &ctx.events,
176 Event::GateStart {
177 run_id,
178 tier: ctx.tier.clone(),
179 version: ctx.version.clone(),
180 gate: kind,
181 },
182 );
183
184 let outcome = match gate {
185 // cargo_test bounds its own run internally (it kills the specific child).
186 Gate::CargoTest => cargo_test(ctx, run_id).await,
187 // hardening_test bounds its own run internally, same as cargo_test.
188 Gate::HardeningTest => hardening_test(ctx, run_id).await,
189 // Each bounds itself the same way cargo_test does: one deadline across
190 // every target, so N crates cannot multiply the ceiling by N.
191 Gate::Clippy => clippy(ctx, run_id).await,
192 Gate::Fmt => fmt_check(ctx, run_id).await,
193 Gate::CargoAudit => supply_chain(ctx, run_id, GateKind::CargoAudit).await,
194 Gate::CargoDeny => supply_chain(ctx, run_id, GateKind::CargoDeny).await,
195 // migration_dry_run's psql restore + sqlx migrate could wedge; bound the
196 // whole gate here. Its bash restore sets kill_on_drop, so a timeout-drop
197 // doesn't orphan it.
198 Gate::MigrationDryRun => {
199 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
200 match tokio::time::timeout(ceiling, migration_dry_run(ctx, run_id)).await {
201 Ok(res) => res,
202 Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout {
203 gate: GateKind::MigrationDryRun,
204 after_s: ctx.cfg.gate_timeout_secs as u32,
205 })
206 .with_log_ref(LogRef::new(&ctx.version, GateKind::MigrationDryRun))),
207 }
208 }
209 // code_smoke boots the real binary (migrate-from-scratch + seed + serve),
210 // any step of which could wedge; bound the whole gate here. Both child
211 // processes set kill_on_drop, so a timeout-drop can't orphan them. A
212 // timeout leaves the throwaway DB behind; the next run's createdb drops
213 // it first (DROP IF EXISTS), same as migration_dry_run's scratch reset.
214 Gate::CodeSmoke => {
215 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
216 match tokio::time::timeout(ceiling, code_smoke(ctx, run_id)).await {
217 Ok(res) => res,
218 Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout {
219 gate: GateKind::CodeSmoke,
220 after_s: ctx.cfg.gate_timeout_secs as u32,
221 })
222 .with_log_ref(LogRef::new(&ctx.version, GateKind::CodeSmoke))),
223 }
224 }
225 Gate::BootSmoke => boot_smoke(ctx, run_id).await,
226 Gate::NodeHealth => node_health(ctx).await,
227 Gate::PageSmoke => page_smoke(ctx).await,
228 Gate::BurnIn { hours } => burn_in(ctx, *hours).await,
229 Gate::ManualConfirm => manual_confirm(ctx).await,
230 };
231
232 let outcome = outcome.unwrap_or_else(|e| {
233 GateOutcome::failed(GateFailure::Unclassified {
234 legacy_detail: Some(format!("gate runner errored: {e}")),
235 })
236 });
237
238 let outcome_json = serde_json::to_string(&outcome)
239 .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
240 sqlx::query(
241 "UPDATE gate_runs
242 SET finished_at = ?, status = ?, outcome_json = ?, log_ref = ?
243 WHERE id = ?",
244 )
245 .bind(Utc::now().to_rfc3339())
246 .bind(outcome.status_str())
247 .bind(&outcome_json)
248 .bind(outcome.log_ref.as_ref().map(super::outcome::LogRef::as_str))
249 .bind(id)
250 .execute(&ctx.pool)
251 .await?;
252
253 tracing::info!(
254 tier = %ctx.tier, version = %ctx.version, gate = %kind,
255 status = outcome.status_str(), "gate done",
256 );
257 events::emit(
258 &ctx.events,
259 Event::GateDone {
260 run_id,
261 tier: ctx.tier.clone(),
262 version: ctx.version.clone(),
263 gate: kind,
264 outcome: outcome.clone(),
265 },
266 );
267
268 Ok(outcome)
269 }
270
271 /// Run every gate in order and return the kinds that did not pass (empty means
272 /// green). We deliberately do NOT short-circuit on first failure — every gate's
273 /// outcome is recorded in `gate_runs`, which is the operator's only visibility
274 /// into pipeline health. Hiding later gates because an earlier one failed makes
275 /// diagnosis worse.
276 ///
277 /// Returning the failing kinds rather than a bare bool is what lets the promote
278 /// path name them in the tier's `partial_reason` and in the error it returns to
279 /// the operator, instead of a generic "something was red".
280 pub async fn run_all(ctx: &GateCtx, gates: &[Gate]) -> Result<Vec<GateKind>> {
281 let mut failed = Vec::new();
282 for g in gates {
283 let o = run(ctx, g).await?;
284 if !o.is_passed() {
285 failed.push(g.kind());
286 }
287 }
288 Ok(failed)
289 }
290
291 // ---- individual gate runners ----
292
293 /// Run every configured `test_target`'s suite, in order, under one gate.
294 ///
295 /// This used to be hardcoded to `worktree/server`, which meant every other
296 /// crate in the repo shipped ungated — including `mnw-cli`, which is built as a
297 /// companion and installed onto prod-1 in the same promote. The targets are now
298 /// configured (`[[test_target]]` in the daemon config), defaulting to the
299 /// historical single `server` entry.
300 ///
301 /// The whole set shares one `gate_runs` row and one log file: from the
302 /// pipeline's point of view "the tests" either pass or don't. The first failing
303 /// target ends the gate, since a red suite blocks the promote regardless of what
304 /// the remaining crates would have said, and running them would only delay the
305 /// operator's answer. Its name is carried in the failure so the summary points
306 /// at the crate, not just the test.
307 async fn cargo_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
308 let log_path = gate_log_path(ctx, GateKind::CargoTest);
309 let log_ref = LogRef::new(&ctx.version, GateKind::CargoTest);
310
311 // Best-effort: drop our own role's stale `mnw_test_*` databases (the
312 // template + any per-test clones orphaned by a previously-killed run)
313 // before the suite, so they can't accumulate or collide. Foreign-owned
314 // leftovers are left alone — the harness now namespaces its template per
315 // role, so they no longer wedge the gate.
316 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
317 clean_stale_test_dbs(scratch_url).await;
318 }
319
320 let started = std::time::Instant::now();
321 // One ceiling for the whole gate, not per target: the point is to bound how
322 // long a hung suite can block the pipeline, and N targets each allowed the
323 // full timeout would multiply that by N.
324 let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
325 let mut ran = 0usize;
326
327 for target in &ctx.cfg.test_targets {
328 let label = target.label();
329 // A target absent from this sha is skipped, not fatal: sando has to be
330 // able to build older shas (bisect, rollback rebuild) from a config that
331 // describes the tip. The zero-targets-ran check below is what stops this
332 // from quietly turning the gate into a no-op.
333 let Some(dir) = ctx
334 .target_dir(target)
335 .filter(|d| d.join("Cargo.toml").is_file())
336 else {
337 tracing::warn!(
338 target = %label, version = %ctx.version,
339 "test_target has no Cargo.toml in this run; skipping",
340 );
341 continue;
342 };
343 let features: Vec<&str> = target.features.iter().map(String::as_str).collect();
344
345 // Fast pre-gate: compile the test targets WITHOUT running them. This
346 // builds the exact artifacts the full run needs (so the subsequent run
347 // reuses the cache — no wasted work), but fails in ~minutes with the
348 // real `error[Ennnn]: ...` on a test-only-target compile break. That
349 // class (a field missing in a `#[cfg(test)]`-only binary like `load`)
350 // otherwise compiles fine under the build step and only blows up here,
351 // after a full build, as an opaque mass test failure.
352 let banner = format!("\n==== test_target: {label} ====\n");
353 append_to_log(&log_path, banner.as_bytes()).await;
354 let mut pre = match cargo_test_command(ctx, &dir, target, &features, &["--no-run"]).spawn()
355 {
356 Ok(c) => c,
357 Err(e) => {
358 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
359 message: format!("{label}: {e}"),
360 })
361 .with_log_ref(log_ref));
362 }
363 };
364 let (pre_out, pre_err, pre_status) = match run_to_deadline_for(
365 &mut pre,
366 ctx,
367 run_id,
368 log_path.clone(),
369 deadline,
370 started,
371 GateKind::CargoTest,
372 )
373 .await?
374 {
375 Ok(v) => v,
376 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
377 };
378 if !pre_status.success() {
379 let failure = classify::classify_compile_error(&pre_out, &pre_err);
380 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
381 }
382
383 // Full run: the test binaries are already built above, so cargo's
384 // up-to-date check skips compilation and this just runs the tests.
385 let mut child = match cargo_test_command(ctx, &dir, target, &features, &[]).spawn() {
386 Ok(c) => c,
387 Err(e) => {
388 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
389 message: format!("{label}: {e}"),
390 })
391 .with_log_ref(log_ref));
392 }
393 };
394 let (stdout_buf, stderr_buf, status) = match run_to_deadline_for(
395 &mut child,
396 ctx,
397 run_id,
398 log_path.clone(),
399 deadline,
400 started,
401 GateKind::CargoTest,
402 )
403 .await?
404 {
405 Ok(v) => v,
406 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
407 };
408 if !status.success() {
409 let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf);
410 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
411 }
412 ran += 1;
413 }
414
415 // Every configured target was missing from the worktree. Exiting green here
416 // would report "tests passed" having run none of them.
417 if ran == 0 {
418 return Ok(GateOutcome::failed(GateFailure::Unclassified {
419 legacy_detail: Some(format!(
420 "cargo_test ran no targets: none of the {} configured test_target dir(s) \
421 exist in this worktree",
422 ctx.cfg.test_targets.len(),
423 )),
424 })
425 .with_log_ref(log_ref));
426 }
427
428 let duration_s = started.elapsed().as_secs() as u32;
429 Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref))
430 }
431
432 /// Stream a child to the live log, bounded by the gate-wide `deadline`. `Ok(Err(_))`
433 /// is the timeout outcome (child killed); `Err(_)` is an IO error on the stream.
434 #[allow(clippy::type_complexity)]
435 async fn run_to_deadline_for(
436 child: &mut tokio::process::Child,
437 ctx: &GateCtx,
438 run_id: GateRunId,
439 log_path: PathBuf,
440 deadline: std::time::Instant,
441 started: std::time::Instant,
442 kind: GateKind,
443 ) -> Result<std::result::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus), GateOutcome>> {
444 let remaining = deadline.saturating_duration_since(std::time::Instant::now());
445 let stream = stream_child_to_live_log(child, ctx.events.clone(), run_id, log_path);
446 match tokio::time::timeout(remaining, stream).await {
447 Ok(res) => Ok(Ok(res?)),
448 Err(_elapsed) => {
449 child.start_kill().ok();
450 let _ = child.wait().await;
451 Ok(Err(GateOutcome::failed(GateFailure::Timeout {
452 gate: kind,
453 after_s: started.elapsed().as_secs() as u32,
454 })))
455 }
456 }
457 }
458
459 /// Prefix a test/compile failure's headline with the crate it came from, so a
460 /// red gate across many targets says *which* crate broke. Other failure kinds
461 /// are single-target by construction and pass through untouched.
462 fn name_target(failure: GateFailure, dir: &std::path::Path) -> GateFailure {
463 let at = dir.display();
464 match failure {
465 GateFailure::CargoTest {
466 failed_count,
467 first_failed,
468 first_panic,
469 } => GateFailure::CargoTest {
470 failed_count,
471 first_failed: Some(match first_failed {
472 Some(name) => format!("{at}: {name}"),
473 None => at.to_string(),
474 }),
475 first_panic,
476 },
477 GateFailure::CompileError {
478 error_count,
479 first_error,
480 } => GateFailure::CompileError {
481 error_count,
482 first_error: Some(match first_error {
483 Some(e) => format!("{at}: {e}"),
484 None => at.to_string(),
485 }),
486 },
487 other => other,
488 }
489 }
490
491 /// Append raw bytes to a gate log outside the child-streaming path (target
492 /// banners). Best-effort, same as `LiveLog`: a broken log dir never turns a
493 /// passing gate red.
494 async fn append_to_log(path: &std::path::Path, bytes: &[u8]) {
495 use tokio::io::AsyncWriteExt;
496 if let Some(parent) = path.parent()
497 && tokio::fs::create_dir_all(parent).await.is_err()
498 {
499 return;
500 }
501 if let Ok(mut f) = tokio::fs::OpenOptions::new()
502 .create(true)
503 .append(true)
504 .open(path)
505 .await
506 {
507 let _ = f.write_all(bytes).await;
508 }
509 }
510
511 /// `cargo clippy --all-targets -- -D warnings` over every configured
512 /// `test_target`.
513 ///
514 /// Before this gate, `-D warnings` was enforced in exactly one place —
515 /// `server/deploy/run-ci.sh`, which died with the astra pipeline — so lint drift
516 /// reached prod ungated. Two crates even carried comments referring to "the
517 /// -D warnings CI gate" for a gate that did not exist.
518 async fn clippy(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
519 lint_over_targets(ctx, run_id, GateKind::Clippy, |target, features| {
520 let mut args = vec!["clippy".to_string(), "--all-targets".to_string()];
521 if target.all_features {
522 args.push("--all-features".to_string());
523 } else if !features.is_empty() {
524 args.push("--features".to_string());
525 args.push(features.join(","));
526 }
527 // Everything after `--` goes to rustc, which is where -D lives.
528 args.push("--".to_string());
529 args.push("-D".to_string());
530 args.push("warnings".to_string());
531 args
532 })
533 .await
534 }
535
536 /// `cargo fmt --check` over every configured `test_target`.
537 ///
538 /// No `rustfmt.toml` exists anywhere in the tree, so this is plain rustfmt
539 /// defaults. Cheap: no compilation, just a parse.
540 async fn fmt_check(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
541 lint_over_targets(ctx, run_id, GateKind::Fmt, |_target, _features| {
542 vec!["fmt".to_string(), "--check".to_string()]
543 })
544 .await
545 }
546
547 /// `cargo audit` / `cargo deny check`, over the crates that carry the matching
548 /// config file.
549 ///
550 /// Config-gated on purpose. Both tools are only meaningful against a triaged
551 /// posture: four crates in this repo fail `cargo audit` today purely because
552 /// they have no `.cargo/audit.toml` recording which transitive advisories have
553 /// been reviewed and accepted. Running them everywhere would make the gate
554 /// permanently and uninformatively red. Dropping the config file into a crate
555 /// is what opts it in.
556 async fn supply_chain(ctx: &GateCtx, run_id: GateRunId, kind: GateKind) -> Result<GateOutcome> {
557 let (config_rel, args): (&str, Vec<String>) = match kind {
558 GateKind::CargoAudit => (".cargo/audit.toml", vec!["audit".into()]),
559 GateKind::CargoDeny => ("deny.toml", vec!["deny".into(), "check".into()]),
560 other => unreachable!("supply_chain called for {other:?}"),
561 };
562 run_over_targets(ctx, run_id, kind, |_target, target_dir| {
563 target_dir.join(config_rel).is_file().then(|| args.clone())
564 })
565 .await
566 }
567
568 /// Shared driver for the lint gates: run `cargo <args>` in every configured
569 /// `test_target` that exists in this worktree.
570 async fn lint_over_targets(
571 ctx: &GateCtx,
572 run_id: GateRunId,
573 kind: GateKind,
574 build_args: impl Fn(&crate::config::TestTarget, &[String]) -> Vec<String>,
575 ) -> Result<GateOutcome> {
576 // The target is handed in directly. This used to reverse-look-it-up by
577 // comparing `worktree.join(dir)` against the resolved path, which silently
578 // stopped matching for anything resolved anywhere else — an aux repo, say.
579 run_over_targets(ctx, run_id, kind, move |t, _dir| {
580 Some(build_args(t, &t.features))
581 })
582 .await
583 }
584
585 /// Run one cargo invocation per configured `test_target`, under a single
586 /// gate-wide deadline. `args_for` returns `None` to skip a target (used by the
587 /// supply-chain gates, which only apply where their config file lives).
588 ///
589 /// Shares `cargo_test`'s conventions: per-target log banners, stop at the first
590 /// failure with the crate named, skip targets absent from the worktree, and fail
591 /// closed if that leaves nothing to run.
592 async fn run_over_targets(
593 ctx: &GateCtx,
594 run_id: GateRunId,
595 kind: GateKind,
596 args_for: impl Fn(&crate::config::TestTarget, &std::path::Path) -> Option<Vec<String>>,
597 ) -> Result<GateOutcome> {
598 let log_path = gate_log_path(ctx, kind);
599 let log_ref = LogRef::new(&ctx.version, kind);
600 let started = std::time::Instant::now();
601 let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
602 let mut ran = 0usize;
603
604 for target in &ctx.cfg.test_targets {
605 let label = target.label();
606 let Some(dir) = ctx
607 .target_dir(target)
608 .filter(|d| d.join("Cargo.toml").is_file())
609 else {
610 tracing::warn!(
611 gate = kind.as_str(), target = %label,
612 "target has no Cargo.toml in this run; skipping",
613 );
614 continue;
615 };
616 let Some(args) = args_for(target, &dir) else {
617 continue;
618 };
619
620 append_to_log(
621 &log_path,
622 format!("\n==== {}: {label} ====\n", kind.as_str()).as_bytes(),
623 )
624 .await;
625
626 let mut cmd = Command::new("cargo");
627 cmd.args(&args)
628 .current_dir(&dir)
629 .stdout(std::process::Stdio::piped())
630 .stderr(std::process::Stdio::piped())
631 .kill_on_drop(true);
632 if let Some(t) = ctx.cfg.cargo_target_dir.as_deref() {
633 cmd.env("CARGO_TARGET_DIR", t);
634 }
635 // clippy type-checks, so it needs the same sqlx online-mode env the
636 // build and cargo_test steps get. fmt/audit/deny never touch the DB.
637 if kind == GateKind::Clippy
638 && let Some(url) = ctx
639 .cfg
640 .scratch_db_url
641 .as_deref()
642 .filter(|_| target.scratch_db)
643 {
644 cmd.env("DATABASE_URL", url);
645 cmd.env(
646 "TEST_DATABASE_URL",
647 url.split_once('?').map_or(url, |(b, _)| b),
648 );
649 }
650
651 let mut child = match cmd.spawn() {
652 Ok(c) => c,
653 Err(e) => {
654 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
655 message: format!("{label}: {e}"),
656 })
657 .with_log_ref(log_ref));
658 }
659 };
660 let (stdout_buf, stderr_buf, status) = match run_to_deadline_for(
661 &mut child,
662 ctx,
663 run_id,
664 log_path.clone(),
665 deadline,
666 started,
667 kind,
668 )
669 .await?
670 {
671 Ok(v) => v,
672 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
673 };
674 if !status.success() {
675 let failure = match kind {
676 // clippy speaks rustc diagnostics, so the compile-error
677 // classifier extracts the real `error: ...` headline.
678 GateKind::Clippy => classify::classify_compile_error(&stdout_buf, &stderr_buf),
679 _ => GateFailure::Unclassified {
680 legacy_detail: Some(first_meaningful_line(&stdout_buf, &stderr_buf)),
681 },
682 };
683 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
684 }
685 ran += 1;
686 }
687
688 if ran == 0 {
689 return Ok(GateOutcome::failed(GateFailure::Unclassified {
690 legacy_detail: Some(format!(
691 "{} ran nothing: no configured target in this worktree qualified",
692 kind.as_str(),
693 )),
694 })
695 .with_log_ref(log_ref));
696 }
697 Ok(GateOutcome::passed(PassNote::TestsPassed {
698 duration_s: started.elapsed().as_secs() as u32,
699 })
700 .with_log_ref(log_ref))
701 }
702
703 /// First line that looks like a diagnostic, for gates whose tools have no
704 /// dedicated classifier (`cargo audit`, `cargo deny`). Falls back to a generic
705 /// note rather than an empty string.
706 fn first_meaningful_line(stdout: &[u8], stderr: &[u8]) -> String {
707 for buf in [stderr, stdout] {
708 let text = String::from_utf8_lossy(buf);
709 if let Some(line) = text
710 .lines()
711 .map(str::trim)
712 .find(|l| l.starts_with("error") || l.contains("vulnerabilit") || l.contains("FAILED"))
713 {
714 return line.chars().take(200).collect();
715 }
716 }
717 "tool reported failure; see the gate log".into()
718 }
719
720 /// The tests `cargo_test` cannot reach, run against production constants.
721 ///
722 /// `cargo_test` builds with `--features fast-tests`, which relaxes
723 /// `AUTH_RATE_LIMIT_BURST` 5 → 20, `SANDBOX_RATE_LIMIT_MS` 30s → 10ms, and
724 /// argon2 from 46 MiB/t=2 to 8 MiB/t=1 — and the rate-limiting suite is
725 /// `#[cfg_attr(feature = "fast-tests", ignore)]`d on top of that, because a
726 /// bucket refilling at 100/sec never depletes under parallel test threads. The
727 /// net effect was that Sando's only code gate silently skipped every test of
728 /// the auth hardening it most needs to protect.
729 ///
730 /// So: no features, `--test-threads=1` (these tests key on a shared per-IP
731 /// bucket and must not interleave), and a name filter rather than the whole
732 /// suite — the rest of the suite is tuned for `fast-tests` and would only go
733 /// slow and flaky here. The filter is a substring match, so it catches
734 /// `..._rate_limit_...` and `..._rate_limited` alike.
735 ///
736 /// This costs a second compile of the lib + integration binary (a different
737 /// feature set is a different cfg, so no artifact sharing with `cargo_test`).
738 /// That is the price of the coverage; the filter keeps the *run* to seconds.
739 async fn hardening_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
740 let server_dir = match ctx.worktree_for(GateKind::HardeningTest) {
741 Ok(w) => w.join("server"),
742 Err(outcome) => return Ok(outcome),
743 };
744 // No features is the whole point; the scratch DB is needed because the
745 // server's sqlx macros type-check against it. Unlike cargo_test, this gate
746 // is deliberately not driven by `test_targets`: it targets one specific
747 // suite in one specific crate, not "the repo's tests".
748 let target = crate::config::TestTarget {
749 dir: std::path::PathBuf::from("server"),
750 aux_repo: None,
751 features: Vec::new(),
752 all_features: false,
753 scratch_db: true,
754 };
755 let log_path = gate_log_path(ctx, GateKind::HardeningTest);
756 let log_ref = LogRef::new(&ctx.version, GateKind::HardeningTest);
757
758 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
759 clean_stale_test_dbs(scratch_url).await;
760 }
761
762 let started = std::time::Instant::now();
763
764 // Same two-step shape as cargo_test: compile first so a test-target break
765 // reports as a compile error rather than an opaque mass test failure.
766 let mut pre = match cargo_test_command(
767 ctx,
768 &server_dir,
769 &target,
770 &[],
771 &["--no-run", "--test", "integration"],
772 )
773 .spawn()
774 {
775 Ok(c) => c,
776 Err(e) => {
777 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
778 message: e.to_string(),
779 })
780 .with_log_ref(log_ref));
781 }
782 };
783 let (pre_out, pre_err, pre_status) =
784 stream_child_to_live_log(&mut pre, ctx.events.clone(), run_id, log_path.clone()).await?;
785 if !pre_status.success() {
786 let failure = classify::classify_compile_error(&pre_out, &pre_err);
787 return Ok(GateOutcome::failed(failure).with_log_ref(log_ref));
788 }
789
790 let mut child = match cargo_test_command(
791 ctx,
792 &server_dir,
793 &target,
794 &[],
795 &[
796 "--test",
797 "integration",
798 "--",
799 "--test-threads=1",
800 "rate_limit",
801 ],
802 )
803 .spawn()
804 {
805 Ok(c) => c,
806 Err(e) => {
807 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
808 message: e.to_string(),
809 })
810 .with_log_ref(log_ref));
811 }
812 };
813 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
814 let stream = stream_child_to_live_log(&mut child, ctx.events.clone(), run_id, log_path);
815 let (stdout_buf, stderr_buf, status) = match tokio::time::timeout(ceiling, stream).await {
816 Ok(res) => res?,
817 Err(_elapsed) => {
818 child.start_kill().ok();
819 let _ = child.wait().await;
820 return Ok(GateOutcome::failed(GateFailure::Timeout {
821 gate: GateKind::HardeningTest,
822 after_s: started.elapsed().as_secs() as u32,
823 })
824 .with_log_ref(log_ref));
825 }
826 };
827 let duration_s = started.elapsed().as_secs() as u32;
828 if status.success() {
829 // A filter that matches nothing exits 0, which would make this gate a
830 // green no-op the day someone renames the tests. Fail closed instead.
831 if tests_run(&stdout_buf) == 0 {
832 return Ok(GateOutcome::failed(GateFailure::Unclassified {
833 legacy_detail: Some(
834 "hardening_test ran 0 tests: the `rate_limit` filter matched nothing. \
835 The suite was renamed or moved — this gate is proving nothing."
836 .into(),
837 ),
838 })
839 .with_log_ref(log_ref));
840 }
841 Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref))
842 } else {
843 let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf);
844 Ok(GateOutcome::failed(failure).with_log_ref(log_ref))
845 }
846 }
847
848 /// Count the tests libtest reports as run, from its `test result: ok. N passed`
849 /// summary line. Returns 0 when no summary is present, which is itself the
850 /// "nothing ran" case the caller fails on.
851 fn tests_run(stdout: &[u8]) -> u32 {
852 String::from_utf8_lossy(stdout)
853 .lines()
854 .filter_map(|l| l.trim().strip_prefix("test result:"))
855 .filter_map(|rest| rest.split_once(" passed"))
856 .filter_map(|(head, _)| {
857 head.rsplit(' ')
858 .find(|t| !t.is_empty())?
859 .parse::<u32>()
860 .ok()
861 })
862 .sum()
863 }
864
865 /// Configure (but don't spawn) `cargo test --release [--features <features>]
866 /// <extra>` in `dir`, wired to the scratch DB. Shared by the `--no-run`
867 /// pre-gate compile and the full test run so both go through one env setup,
868 /// and by both test gates so they can't drift apart on env.
869 ///
870 /// `cargo_test` passes `fast-tests`, matching what the retired astra CI did:
871 /// it relaxes the auth rate-limit burst (5 → 20) and argon2 cost so
872 /// signup-heavy + lockout workflow tests complete without hitting Governor
873 /// before the hand-rolled lockout check (documented at
874 /// `server/src/constants.rs:87`). `hardening_test` passes no features, which is
875 /// the whole point of that gate — see `GateKind::HardeningTest`.
876 fn cargo_test_command(
877 ctx: &GateCtx,
878 dir: &std::path::Path,
879 target: &crate::config::TestTarget,
880 features: &[&str],
881 extra: &[&str],
882 ) -> Command {
883 let mut cmd = Command::new("cargo");
884 cmd.args(["test", "--release"]);
885 if target.all_features {
886 cmd.arg("--all-features");
887 } else if !features.is_empty() {
888 cmd.args(["--features", &features.join(",")]);
889 }
890 cmd.args(extra)
891 .current_dir(dir)
892 .stdout(std::process::Stdio::piped())
893 .stderr(std::process::Stdio::piped())
894 .kill_on_drop(true);
895 // Share the build step's target dir so the test compile reuses its
896 // artifacts (and the `--no-run` precompile reuses them again). Must match
897 // `build.rs` or the gate would clean-compile the whole tree a second time.
898 if let Some(target) = ctx.cfg.cargo_target_dir.as_deref() {
899 cmd.env("CARGO_TARGET_DIR", target);
900 }
901 // Same online-mode rationale as the build step: sqlx query macros need a
902 // live DB to type-check against. The scratch DB is left in migrated state
903 // by the preceding build, so we can reuse it here.
904 //
905 // Opt-in per target: setting DATABASE_URL switches sqlx OUT of offline mode,
906 // so a crate that ships `.sqlx` query data would stop using it and try to
907 // type-check against a database that has none of its tables.
908 if let Some(scratch_url) = ctx
909 .cfg
910 .scratch_db_url
911 .as_deref()
912 .filter(|_| target.scratch_db)
913 {
914 cmd.env("DATABASE_URL", scratch_url);
915 // The server test harness (tests/harness/db.rs) parses TEST_DATABASE_URL
916 // with rfind('/'), which mangles URLs whose query string contains '/'
917 // (e.g. `?host=/var/run/postgresql`). Strip the query — libpq defaults
918 // to /var/run/postgresql on Debian/Ubuntu when host is unspecified.
919 let test_url = scratch_url
920 .split_once('?')
921 .map_or(scratch_url, |(base, _)| base);
922 cmd.env("TEST_DATABASE_URL", test_url);
923 }
924 cmd
925 }
926
927 async fn migration_dry_run(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
928 let log = GateLog::open(ctx, run_id, GateKind::MigrationDryRun).await;
929 let outcome = migration_dry_run_inner(ctx, &log).await;
930 log.close().await;
931 outcome.map(|o| o.with_log_ref(LogRef::new(&ctx.version, GateKind::MigrationDryRun)))
932 }
933
934 /// The staged interior of [`migration_dry_run`], writing every step through the
935 /// gate's live log. The caller owns the sink so it can flush it on every exit
936 /// path, and attaches the `log_ref` once instead of at each return.
937 /// Runs one configured check per database, in config order, and stops at the
938 /// first that does not pass — a red gate is a red gate, and continuing would
939 /// bury it under a second restore's output.
940 ///
941 /// The server's check runs against `scratch_db_url` itself and is deliberately
942 /// last-writer for it: `cargo_test` reuses that database in migrated state, so
943 /// every other check must name its own `scratch_db` (enforced at config load).
944 async fn migration_dry_run_inner(ctx: &GateCtx, log: &GateLog) -> Result<GateOutcome> {
945 let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() else {
946 log.line("scratch_db_url unset in daemon config\n").await;
947 return Ok(GateOutcome::blocked(GateBlocker::ScratchDbUrlUnset));
948 };
949
950 let mut checked = Vec::new();
951 let mut primary_backup_path = String::new();
952 for check in &ctx.cfg.migration_checks {
953 let label = check.dir.display().to_string();
954 log.line(&format!("==== migration_check: {label} ====\n"))
955 .await;
956 match run_migration_check(ctx, log, scratch_url, check).await? {
957 CheckResult::Passed { backup_path } => {
958 if primary_backup_path.is_empty() {
959 primary_backup_path = backup_path;
960 }
961 checked.push(label);
962 }
963 CheckResult::Stopped(outcome) => return Ok(outcome),
964 }
965 }
966
967 log.line(&format!(
968 "all {} migration check(s) passed: {}",
969 checked.len(),
970 checked.join(", ")
971 ))
972 .await;
973 Ok(GateOutcome::passed(PassNote::Migrated {
974 backup_path: primary_backup_path,
975 checks: checked,
976 }))
977 }
978
979 /// One check's verdict: it passed (against `backup_path`), or it produced the
980 /// outcome the whole gate reports.
981 enum CheckResult {
982 Passed { backup_path: String },
983 Stopped(GateOutcome),
984 }
985
986 /// Restore one database's dump into its scratch DB and run its migrations on top.
987 async fn run_migration_check(
988 ctx: &GateCtx,
989 log: &GateLog,
990 scratch_url: &str,
991 check: &crate::config::MigrationCheck,
992 ) -> Result<CheckResult> {
993 let label = check.dir.display().to_string();
994
995 let backup: Option<(String, String)> = sqlx::query_as(
996 "SELECT local_path, fetched_at FROM backups
997 WHERE app = ? AND name = ? ORDER BY id DESC LIMIT 1",
998 )
999 .bind(&ctx.cfg.id)
1000 .bind(&check.backup)
1001 .fetch_optional(&ctx.pool)
1002 .await?;
1003 let Some((backup_path, fetched_at)) = backup else {
1004 log.line(&format!(
1005 "no {} backup fetched; call /backup/fetch first\n",
1006 check.backup
1007 ))
1008 .await;
1009 return Ok(CheckResult::Stopped(GateOutcome::blocked(
1010 GateBlocker::NoBackupAvailable {
1011 check: label,
1012 backup: check.backup.clone(),
1013 },
1014 )));
1015 };
1016
1017 // Presence is not freshness. A fetch that quietly stopped working leaves this
1018 // row in place, and restoring it dry-runs the migrations against a schema prod
1019 // has moved past — green, and worthless. Block on age instead. An unparsable
1020 // timestamp is treated as stale: this row is daemon-written RFC 3339, so a
1021 // value that will not parse means something is wrong, and failing closed on a
1022 // freshness check is the whole point.
1023 let age_hours = chrono::DateTime::parse_from_rfc3339(&fetched_at).map_or(i64::MAX, |t| {
1024 (Utc::now() - t.with_timezone(&Utc)).num_hours()
1025 });
1026 let max_age_hours = ctx.cfg.backup_max_age_hours;
1027 if age_hours > i64::from(max_age_hours) {
1028 let msg = format!(
1029 "backup {backup_path} was fetched {fetched_at} ({age_hours}h ago, max \
1030 {max_age_hours}h); re-run /backup/fetch\n"
1031 );
1032 log.line(&msg).await;
1033 return Ok(CheckResult::Stopped(GateOutcome::blocked(
1034 GateBlocker::BackupStale {
1035 age_hours,
1036 max_age_hours,
1037 check: label,
1038 },
1039 )));
1040 }
1041
1042 // A check with its own `scratch_db` gets that database created here rather
1043 // than by a host bootstrap step: adding a `[[migration_check]]` should not
1044 // silently depend on someone having remembered to `createdb` on the Sando
1045 // host, which is exactly the class of footgun this gate exists to remove.
1046 // DROP + CREATE also makes the database sando-owned, so the PG15+ public
1047 // schema grants `reset_scratch` applies next are the owner's to give.
1048 let db_url = match check.scratch_db.as_deref() {
1049 None => scratch_url.to_string(),
1050 Some(dbname) => {
1051 let maintenance_url = pg_url_with_dbname(scratch_url, "postgres");
1052 log.line(&format!("---- create scratch db {dbname} ----\n"))
1053 .await;
1054 if let Err(e) = pg_create_db(&maintenance_url, dbname).await {
1055 let msg = format!("{label}: creating scratch db {dbname}: {e}");
1056 log.line(&msg).await;
1057 return Ok(CheckResult::Stopped(GateOutcome::failed(
1058 GateFailure::RestoreFailed { reason: msg },
1059 )));
1060 }
1061 pg_url_with_dbname(scratch_url, dbname)
1062 }
1063 };
1064
1065 let owner_role = check
1066 .owner_role
1067 .as_deref()
1068 .unwrap_or(&ctx.cfg.scratch_owner_role);
1069 log.line("---- reset_scratch ----\n").await;
1070 if let Err(e) = reset_scratch(&db_url, owner_role).await {
1071 let msg = format!("{label}: scratch reset: {e}");
1072 log.line(&msg).await;
1073 return Ok(CheckResult::Stopped(GateOutcome::failed(
1074 GateFailure::RestoreFailed { reason: msg },
1075 )));
1076 }
1077 log.line(&format!("---- restore_dump ({backup_path}) ----\n"))
1078 .await;
1079 if let Err(e) = restore_dump(&db_url, &backup_path, log).await {
1080 let msg = format!("{label}: restore: {e}");
1081 log.line(&msg).await;
1082 return Ok(CheckResult::Stopped(GateOutcome::failed(
1083 GateFailure::RestoreFailed { reason: msg },
1084 )));
1085 }
1086
1087 let Some(migrations_dir) = ctx.migrations_dir(&check.dir) else {
1088 // Neither the bundle nor a checkout holds them. For an accepted
1089 // artifact that means the builder did not ship its migrations, and a
1090 // dry run over nothing would report green having proved nothing.
1091 let msg = format!(
1092 "{label}: no migrations at {} in the bundle or a checkout",
1093 check.dir.display()
1094 );
1095 log.line(&msg).await;
1096 return Ok(CheckResult::Stopped(GateOutcome::failed(
1097 GateFailure::RestoreFailed { reason: msg },
1098 )));
1099 };
1100 log.line("---- run_migrator ----\n").await;
1101 match run_migrator(&db_url, &migrations_dir).await {
1102 Ok(()) => {
1103 log.line(&format!("{label}: restored {backup_path} + migrated\n"))
1104 .await;
1105 Ok(CheckResult::Passed { backup_path })
1106 }
1107 Err(e) => {
1108 let err_s = format!("{label}: {e}");
1109 log.line(&err_s).await;
1110 Ok(CheckResult::Stopped(GateOutcome::failed(
1111 classify::classify_migration_error(&err_s, None),
1112 )))
1113 }
1114 }
1115 }
1116
1117 pub(crate) async fn reset_scratch(db_url: &str, owner_role: &str) -> Result<()> {
1118 use sqlx::Executor;
1119 use sqlx::postgres::PgPoolOptions;
1120 let pool = PgPoolOptions::new()
1121 .max_connections(1)
1122 .connect(db_url)
1123 .await?;
1124 // `owner_role` is validated `[A-Za-z0-9_]+` at config load, so interpolating
1125 // it into DDL is sound. It still goes through `format('%I')` inside the DO
1126 // block for the quoting Postgres expects on an identifier.
1127 let sql = format!(
1128 r#"
1129 DO $$
1130 DECLARE s text;
1131 BEGIN
1132 -- The dump restores objects owned by the prod role and re-grants to
1133 -- it (`ALTER ... OWNER TO {owner_role}`), which errors if the role
1134 -- is absent — superuser does not imply the role exists. Create it
1135 -- NOLOGIN: the scratch DB needs the role as an *owner* only, never
1136 -- as a connecting identity. Idempotent, so a re-reset is a no-op.
1137 IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{owner_role}') THEN
1138 EXECUTE format('CREATE ROLE %I NOLOGIN', '{owner_role}');
1139 END IF;
1140
1141 -- Drop every non-system schema, not just public — migrations create
1142 -- custom schemas (e.g. tower_sessions) that survive `DROP SCHEMA
1143 -- public CASCADE` and then collide on the next migration run.
1144 FOR s IN
1145 SELECT nspname FROM pg_namespace
1146 WHERE nspname NOT LIKE 'pg_%'
1147 AND nspname NOT IN ('information_schema')
1148 LOOP
1149 EXECUTE format('DROP SCHEMA IF EXISTS %I CASCADE', s);
1150 END LOOP;
1151 EXECUTE 'CREATE SCHEMA public';
1152 -- Restore the pre-PG15 public-schema default on the throwaway
1153 -- scratch DB. Without this, the freshly-created public is owned by
1154 -- the connecting role (sando) with no grant to anyone else, so a
1155 -- migration's FK/trigger check that Postgres runs as a *restored*
1156 -- prod-owned table's owner ({owner_role} from the backup dump)
1157 -- fails with "permission denied for schema public". Granting to
1158 -- PUBLIC is role-agnostic and safe here — this DB is disposable and
1159 -- exists only to dry-run migrations.
1160 EXECUTE 'GRANT USAGE, CREATE ON SCHEMA public TO PUBLIC';
1161 -- PG15+: the new owner needs CREATE on public in its own right, not
1162 -- only via PUBLIC, for the restore's owner-scoped DDL.
1163 EXECUTE format('GRANT USAGE, CREATE ON SCHEMA public TO %I', '{owner_role}');
1164 END $$;
1165 "#
1166 );
1167 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(sql)))
1168 .await?;
1169 pool.close().await;
1170 Ok(())
1171 }
1172
1173 /// Startup assertion for the scratch cluster: the gates reset it, seed an owner
1174 /// role into it, and drop leftover test databases in it — none of which a plain
1175 /// unprivileged role can do. Historically these preconditions were satisfied by
1176 /// hand on fw13 (`ALTER ROLE sando SUPERUSER`, a hand-created `makenotwork`
1177 /// role) and nothing recorded that, so a rebuild elsewhere would fail one gate
1178 /// at a time with an opaque permissions error. Assert once, at boot, loudly.
1179 ///
1180 /// Not part of `--check-config`: that path is pure by design (no DB, no
1181 /// network), and a green there must mean "this build understands its config",
1182 /// not "the cluster is reachable".
1183 pub async fn preflight_scratch_privileges(db_url: &str) -> Result<()> {
1184 use sqlx::postgres::PgPoolOptions;
1185 let pool = PgPoolOptions::new()
1186 .max_connections(1)
1187 .connect(db_url)
1188 .await
1189 .context("connecting to scratch_db_url for the startup privilege check")?;
1190 let (is_super, can_signal): (bool, bool) = sqlx::query_as(
1191 "SELECT rolsuper, pg_catalog.pg_has_role(current_user, 'pg_signal_backend', 'USAGE')
1192 FROM pg_roles WHERE rolname = current_user",
1193 )
1194 .fetch_one(&pool)
1195 .await?;
1196 pool.close().await;
1197 anyhow::ensure!(
1198 is_super || can_signal,
1199 "the scratch_db_url role has neither SUPERUSER nor pg_signal_backend; migration_dry_run \
1200 and cargo_test cannot reset the scratch DB or clear stale test databases. Grant one:\n \
1201 ALTER ROLE <role> SUPERUSER; -- what fw13 uses\n \
1202 GRANT pg_signal_backend TO <role>; -- narrower: terminate only, cannot drop \
1203 foreign-owned databases",
1204 );
1205 if !is_super {
1206 tracing::warn!(
1207 "scratch role has pg_signal_backend but not SUPERUSER: stale test databases owned by \
1208 another role cannot be dropped, and the scratch owner role cannot be created if absent"
1209 );
1210 }
1211 Ok(())
1212 }
1213
1214 /// Best-effort cleanup of stale per-test database clones (`mnw_test_<uuid>`)
1215 /// left behind by a previously-killed `cargo_test` run.
1216 ///
1217 /// Drops **foreign-owned leftovers too**, which is why the daemon asserts
1218 /// SUPERUSER at startup (`preflight_scratch_privileges`): `DROP DATABASE`
1219 /// requires ownership or superuser, and the `WITH (FORCE)` terminate requires
1220 /// superuser or `pg_signal_backend`. Without both, orphans from a run under a
1221 /// different role accumulate and degrade the gate — the failure this cleanup
1222 /// exists to prevent.
1223 ///
1224 /// OPERATIONAL HAZARD: fw13 runs one Postgres cluster shared with local `cargo
1225 /// test` as `max`, so a gate firing mid-local-test will force-drop that run's
1226 /// databases out from under it. That collision is known and tracked separately
1227 /// (give the gate its own cluster); until then, do not run local tests on fw13
1228 /// while a Sando gate is live.
1229 ///
1230 /// Deliberately **excludes the template** (`mnw_test_template_*`): the harness
1231 /// reuses it across runs when it's migration-current (skipping a full
1232 /// drop+migrate), so dropping it here would force a needless rebuild every
1233 /// gate run. Templates are bounded (one per role) and never accumulate, so
1234 /// leaving them is free. Never returns an error: a cleanup miss must not turn a
1235 /// deploy red.
1236 async fn clean_stale_test_dbs(db_url: &str) {
1237 use sqlx::Executor;
1238 use sqlx::postgres::PgPoolOptions;
1239 let pool = match PgPoolOptions::new()
1240 .max_connections(1)
1241 .connect(db_url)
1242 .await
1243 {
1244 Ok(p) => p,
1245 Err(e) => {
1246 tracing::warn!(error = %e, "stale test-db cleanup: could not connect; skipping");
1247 return;
1248 }
1249 };
1250 // Every per-test clone, whoever owns it. The ownership filter this used to
1251 // carry is what let foreign-owned orphans pile up; superuser (asserted at
1252 // startup) makes them droppable.
1253 let names: Vec<(String,)> = sqlx::query_as(
1254 "SELECT datname FROM pg_database
1255 WHERE datname LIKE 'mnw_test_%'
1256 AND datname NOT LIKE '%template%'",
1257 )
1258 .fetch_all(&pool)
1259 .await
1260 .unwrap_or_default();
1261 let count = names.len();
1262 for (name,) in names {
1263 // `name` comes straight from pg_database; quoting it is sufficient.
1264 if let Err(e) = pool
1265 .execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1266 "DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)"
1267 ))))
1268 .await
1269 {
1270 tracing::warn!(error = %e, db = %name, "stale test-db cleanup: drop failed");
1271 }
1272 }
1273 if count > 0 {
1274 tracing::info!(
1275 count,
1276 "stale test-db cleanup: dropped leftover mnw_test_* databases"
1277 );
1278 }
1279 pool.close().await;
1280 }
1281
1282 /// Build the restore shell line. Two pipelines we accept:
1283 /// *.sql -> psql -v ON_ERROR_STOP=1 $url < dump
1284 /// *.sql.gz -> set -o pipefail; gunzip -c dump | psql -v ON_ERROR_STOP=1 $url
1285 ///
1286 /// Two safety flags are load-bearing (CF4):
1287 /// - `ON_ERROR_STOP=1`: without it, psql exits 0 even when individual statements
1288 /// error, so a partial/corrupt restore would *pass* the gate.
1289 /// - `set -o pipefail`: without it a shell pipeline reports only the last
1290 /// command's status, so a `gunzip` failure on a truncated archive is masked by
1291 /// psql's exit. pipefail is a bash builtin (not POSIX sh), so the runner uses
1292 /// `bash -c`.
1293 fn restore_shell(db_url: &str, dump: &str) -> String {
1294 if std::path::Path::new(dump)
1295 .extension()
1296 .is_some_and(|ext| ext.eq_ignore_ascii_case("gz"))
1297 {
1298 format!(
1299 "set -o pipefail; gunzip -c {q} | psql -v ON_ERROR_STOP=1 {url}",
1300 q = shell_escape(dump),
1301 url = shell_escape(db_url),
1302 )
1303 } else {
1304 format!(
1305 "psql -v ON_ERROR_STOP=1 {url} < {q}",
1306 url = shell_escape(db_url),
1307 q = shell_escape(dump),
1308 )
1309 }
1310 }
1311
1312 async fn restore_dump(db_url: &str, dump: &str, log: &GateLog) -> Result<()> {
1313 // Split the password out of the URL and hand it to psql via PGPASSWORD, so it
1314 // never lands in argv (visible in /proc/<pid>/cmdline to any local user).
1315 // The sanitized URL — user/host/db, no secret — goes on the command line.
1316 let (sanitized, password) = split_pg_password(db_url);
1317 let shell = restore_shell(&sanitized, dump);
1318 // `bash` (not `sh`): `set -o pipefail` is a bash builtin. The restore runs
1319 // locally on the Sando host (fw13), which has bash.
1320 let mut cmd = Command::new("bash");
1321 cmd.arg("-c").arg(&shell);
1322 // kill_on_drop so the gate's wall-clock ceiling (dispatcher-level timeout on
1323 // migration_dry_run) can't orphan a wedged psql restore.
1324 cmd.kill_on_drop(true);
1325 if let Some(pw) = password {
1326 cmd.env("PGPASSWORD", pw);
1327 }
1328 // Streamed, not `.output()`: a prod-sized restore runs for minutes, and
1329 // psql's progress is the only thing an operator has to watch during it.
1330 let (_stdout, stderr, status) = log.run(&mut cmd).await?;
1331 anyhow::ensure!(
1332 status.success(),
1333 "restore failed: {}",
1334 String::from_utf8_lossy(&stderr),
1335 );
1336 Ok(())
1337 }
1338
1339 /// Split a `postgres://user:password@host/db` URL into its password-free form and
1340 /// the (percent-decoded) password. Returns the URL unchanged with `None` when
1341 /// there is no userinfo password. psql reads the password from `PGPASSWORD`, so
1342 /// keeping it off the command line removes the /proc exposure.
1343 fn split_pg_password(db_url: &str) -> (String, Option<String>) {
1344 let Some(after) = db_url.find("://").map(|i| i + 3) else {
1345 return (db_url.to_string(), None);
1346 };
1347 // The authority ends at the first '/', '?' or '#'; the password (if any) is
1348 // between the first ':' and the '@' within the userinfo of that authority.
1349 let authority_end = db_url[after..]
1350 .find(['/', '?', '#'])
1351 .map_or(db_url.len(), |i| after + i);
1352 let Some(at) = db_url[after..authority_end].find('@').map(|i| after + i) else {
1353 return (db_url.to_string(), None);
1354 };
1355 let userinfo = &db_url[after..at];
1356 let Some(colon) = userinfo.find(':') else {
1357 return (db_url.to_string(), None);
1358 };
1359 let password = percent_decode(&userinfo[colon + 1..]);
1360 let sanitized = format!(
1361 "{}{}{}",
1362 &db_url[..after],
1363 &userinfo[..colon],
1364 &db_url[at..]
1365 );
1366 (sanitized, Some(password))
1367 }
1368
1369 /// Minimal `%XX` percent-decode for a URL userinfo component. Non-escape bytes
1370 /// pass through; a malformed escape is left literal.
1371 fn percent_decode(s: &str) -> String {
1372 let b = s.as_bytes();
1373 let mut out = Vec::with_capacity(b.len());
1374 let mut i = 0;
1375 while i < b.len() {
1376 if b[i] == b'%'
1377 && i + 2 < b.len()
1378 && let (Some(h), Some(l)) = (hex_val(b[i + 1]), hex_val(b[i + 2]))
1379 {
1380 out.push((h << 4) | l);
1381 i += 3;
1382 } else {
1383 out.push(b[i]);
1384 i += 1;
1385 }
1386 }
1387 String::from_utf8_lossy(&out).into_owned()
1388 }
1389
1390 fn hex_val(c: u8) -> Option<u8> {
1391 match c {
1392 b'0'..=b'9' => Some(c - b'0'),
1393 b'a'..=b'f' => Some(c - b'a' + 10),
1394 b'A'..=b'F' => Some(c - b'A' + 10),
1395 _ => None,
1396 }
1397 }
1398
1399 pub(crate) async fn run_migrator(db_url: &str, dir: &std::path::Path) -> Result<()> {
1400 use sqlx::postgres::PgPoolOptions;
1401 let pool = PgPoolOptions::new()
1402 .max_connections(1)
1403 .connect(db_url)
1404 .await?;
1405 let migrator = sqlx::migrate::Migrator::new(dir).await?;
1406 migrator.run(&pool).await?;
1407 pool.close().await;
1408 Ok(())
1409 }
1410
1411 fn shell_escape(s: &str) -> String {
1412 format!("'{}'", s.replace('\'', "'\\''"))
1413 }
1414
1415 // ---- code_smoke ----
1416
1417 /// A 32+ char throwaway signing secret for the `code_smoke` boot. The server
1418 /// only enforces length (>= 32) outside production, and code_smoke's loopback
1419 /// `HOST_URL` keeps it in dev mode, so the value is irrelevant beyond that.
1420 const CODE_SMOKE_SIGNING_SECRET: &str = "sando-code-smoke-dummy-signing-secret-0000000000";
1421
1422 /// Seconds to wait for the real server to come up and serve `GET /health`
1423 /// during `code_smoke`. Longer than `boot_smoke`'s 3s: this boots the *full*
1424 /// server (config, pool, session store, webauthn, doc load, app build), not the
1425 /// minimal no-DB smoke server. The whole gate is also bounded by
1426 /// `gate_timeout_secs` at the dispatcher.
1427 const CODE_SMOKE_READY_SECS: u64 = 30;
1428
1429 /// `code_smoke` — the first host gate. Boots the freshly-built binary against a
1430 /// throwaway *empty* DB it migrates from scratch and seeds the example catalog
1431 /// into, then proves the real server serves `GET /health` against that
1432 /// nonempty DB. Fast and infra-light (one local Postgres, no prod-dump restore,
1433 /// no scratch-role reset, no external services), so a green here proves the
1434 /// code is sound and isolates a later `cargo_test`/`migration_dry_run` red as an
1435 /// environment problem rather than a code one.
1436 ///
1437 /// Reuses existing binary entrypoints, so the server needs no smoke-specific
1438 /// mode: `<bin> --seed-examples` loads config, connects, migrates from scratch,
1439 /// seeds, and exits; a plain `<bin>` then serves the real app. Both run with CWD
1440 /// at the server crate root so `docs/business/assumptions.toml` + `site-docs/`
1441 /// resolve (a missing assumptions file panics real startup), and with a loopback
1442 /// `HOST_URL` so config stays in dev mode (no CDN/S3/signing-secret prod
1443 /// enforcement). The seed's host allowlist already admits `127.0.0.1`, and the
1444 /// fresh DB trivially satisfies its no-real-users guard.
1445 async fn code_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
1446 let log = GateLog::open(ctx, run_id, GateKind::CodeSmoke).await;
1447 let outcome = code_smoke_inner(ctx, &log).await;
1448 log.close().await;
1449 outcome.map(|o| o.with_log_ref(LogRef::new(&ctx.version, GateKind::CodeSmoke)))
1450 }
1451
1452 /// The staged interior of [`code_smoke`], writing every step through the gate's
1453 /// live log. Same split as [`migration_dry_run_inner`]: the caller owns the sink
1454 /// and attaches the `log_ref`.
1455 async fn code_smoke_inner(ctx: &GateCtx, log: &GateLog) -> Result<GateOutcome> {
1456 let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() else {
1457 log.line("scratch_db_url unset in daemon config\n").await;
1458 return Ok(GateOutcome::blocked(GateBlocker::ScratchDbUrlUnset));
1459 };
1460
1461 // The staged binary (set by build_and_run_host before gating). code_smoke
1462 // runs first among the host gates, but staging precedes all gating, so the
1463 // artifact path is already recorded.
1464 let bin: Option<(String,)> =
1465 sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
1466 .bind(&ctx.cfg.id)
1467 .bind(&ctx.version)
1468 .fetch_optional(&ctx.pool)
1469 .await?;
1470 let Some((bin,)) = bin else {
1471 return Ok(GateOutcome::blocked(GateBlocker::ArtifactMissing {
1472 version: ctx.version.clone(),
1473 }));
1474 };
1475
1476 // Frontend builds, before anything else: they need no DB and no staged
1477 // binary, and a `tsc` error is the one failure the Rust build deliberately
1478 // swallows (both MNW build scripts emit `cargo::warning` and succeed against
1479 // a stale `static/dist/`). Failing here is what stops the deploy rsyncing
1480 // the previous build's bundle.
1481 if let Some(outcome) = code_smoke_frontends(ctx, log).await {
1482 return Ok(outcome);
1483 }
1484
1485 // Docs integrity, first and cheapest: run the staged binary's DB-free
1486 // `MNW_CHECK_DOCS` mode before creating the throwaway DB. A broken internal
1487 // docs link (a `[..](x.md)` resolving to a slug no page serves) fails here
1488 // in well under a second instead of after a full migrate+seed+boot, and a
1489 // rotted link never reaches prod as a live 404. Collisions are reported by
1490 // the check but do not fail it; only broken links do.
1491 if let Some(outcome) = code_smoke_docs_check(ctx, &bin, log).await {
1492 return Ok(outcome);
1493 }
1494
1495 let dbname = code_smoke_db_name(&ctx.version);
1496 let maintenance_url = pg_url_with_dbname(scratch_url, "postgres");
1497 let throwaway_url = pg_url_with_dbname(scratch_url, &dbname);
1498
1499 // Create the throwaway DB (dropping any stale one from a killed prior run).
1500 log.line(&format!("---- createdb {dbname} ----\n")).await;
1501 if let Err(e) = pg_create_db(&maintenance_url, &dbname).await {
1502 let reason = format!("createdb {dbname}: {e}");
1503 log.line(&reason).await;
1504 return Ok(GateOutcome::failed(GateFailure::CodeSmokeSetup { reason }));
1505 }
1506
1507 // Everything past createdb must drop the DB on the way out, pass or fail.
1508 let outcome = code_smoke_body(ctx, &bin, &throwaway_url, log).await;
1509
1510 log.line(&format!("\n---- dropdb {dbname} ----\n")).await;
1511 if let Err(e) = pg_drop_db(&maintenance_url, &dbname).await {
1512 // A teardown miss must not turn a passing gate red — log it and move on.
1513 // The next run's createdb drops it first anyway.
1514 tracing::warn!(error = %e, db = %dbname, "code_smoke: dropdb failed; next run will reclaim it");
1515 log.line(&format!("dropdb warning (non-fatal): {e}")).await;
1516 }
1517
1518 Ok(outcome)
1519 }
1520
1521 /// Compile every configured `frontend_build` in the worktree.
1522 ///
1523 /// Returns `Some(failed)` on the first project that does not build; `None` when
1524 /// all of them do (or none are configured). Output streams to `log` either way.
1525 ///
1526 /// `npm ci` runs only when `node_modules` is absent. Usually it is not: the app
1527 /// build script installed it during the `cargo build` that produced the artifact
1528 /// this gate is about to smoke, so the common path here is just `npm run build`
1529 /// against a warm install — seconds. The install branch covers the gate running
1530 /// against a worktree whose build script was skipped or failed at `npm ci`, and
1531 /// it is as fatal as a compile failure, because the alternative is compiling
1532 /// against whatever some earlier sha installed.
1533 ///
1534 /// Unlike the app build scripts, nothing here is best-effort. That asymmetry is
1535 /// the point of the gate.
1536 async fn code_smoke_frontends(ctx: &GateCtx, log: &GateLog) -> Option<GateOutcome> {
1537 if ctx.cfg.frontend_builds.is_empty() {
1538 return None;
1539 }
1540 let worktree = match ctx.worktree_for(GateKind::CodeSmoke) {
1541 Ok(w) => w.to_path_buf(),
1542 Err(outcome) => return Some(outcome),
1543 };
1544 for fe in &ctx.cfg.frontend_builds {
1545 let dir = worktree.join(&fe.dir);
1546 let label = fe.dir.display().to_string();
1547 log.line(&format!("---- frontend build ({label}) ----\n"))
1548 .await;
1549
1550 if !dir.is_dir() {
1551 // An older sha predating the frontend, mid-bisect. Skipping keeps
1552 // sando able to rebuild history; the log says so out loud.
1553 log.line(&format!("{label} absent from this worktree; skipping\n"))
1554 .await;
1555 continue;
1556 }
1557
1558 if !dir.join("node_modules").is_dir()
1559 && let Some(outcome) = run_npm(&dir, &label, &["ci"], "npm ci", ctx, log).await
1560 {
1561 return Some(outcome);
1562 }
1563
1564 if let Some(outcome) = run_npm(
1565 &dir,
1566 &label,
1567 &["run", &fe.script],
1568 &format!("npm run {}", fe.script),
1569 ctx,
1570 log,
1571 )
1572 .await
1573 {
1574 return Some(outcome);
1575 }
1576 }
1577 None
1578 }
1579
1580 /// One `npm` invocation for [`code_smoke_frontends`], bounded by the gate
1581 /// timeout so a wedged install cannot hold the whole pipeline (the enclosing
1582 /// `code_smoke` ceiling would catch it eventually, but this attributes the
1583 /// failure to the project that hung).
1584 async fn run_npm(
1585 dir: &std::path::Path,
1586 label: &str,
1587 args: &[&str],
1588 what: &str,
1589 ctx: &GateCtx,
1590 log: &GateLog,
1591 ) -> Option<GateOutcome> {
1592 log.line(&format!("$ {what}\n")).await;
1593 let mut cmd = tokio::process::Command::new("npm");
1594 cmd.args(args).current_dir(dir).kill_on_drop(true);
1595 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
1596 // On the timeout branch the whole `run` future is dropped, which drops the
1597 // child; `kill_on_drop` is what turns that into an actual kill.
1598 let status = match tokio::time::timeout(ceiling, log.run(&mut cmd)).await {
1599 Ok(Ok((_stdout, _stderr, status))) => status,
1600 Ok(Err(e)) => {
1601 // A missing `npm` lands here. Fatal, not skipped: a build host
1602 // without Node cannot produce the bundle the release serves, and
1603 // silently passing is how the stale bundle shipped in the first place.
1604 log.line(&format!("{what} could not be spawned: {e}\n"))
1605 .await;
1606 return Some(GateOutcome::failed(GateFailure::SpawnFailed {
1607 message: format!("{what} in {label}: {e}"),
1608 }));
1609 }
1610 Err(_elapsed) => {
1611 log.line(&format!(
1612 "{what} timed out after {}s\n",
1613 ctx.cfg.gate_timeout_secs
1614 ))
1615 .await;
1616 return Some(GateOutcome::failed(GateFailure::CodeSmokeFrontend {
1617 dir: label.to_string(),
1618 exit_code: None,
1619 }));
1620 }
1621 };
1622 if status.success() {
1623 return None;
1624 }
1625 Some(GateOutcome::failed(GateFailure::CodeSmokeFrontend {
1626 dir: label.to_string(),
1627 exit_code: status.code(),
1628 }))
1629 }
1630
1631 /// Run the staged binary's DB-free docs integrity check (`MNW_CHECK_DOCS=1`).
1632 ///
1633 /// Returns `Some(failed)` if the check reports broken links, cannot be spawned,
1634 /// or overruns its ceiling; `None` when the docs are clean. Output streams to
1635 /// `log` either way. The 60s ceiling backstops the case where the staged binary
1636 /// predates the flag and would fall through to a normal (DB-needing) boot and
1637 /// hang.
1638 async fn code_smoke_docs_check(ctx: &GateCtx, bin: &str, log: &GateLog) -> Option<GateOutcome> {
1639 let server_dir = match ctx.worktree_for(GateKind::CodeSmoke) {
1640 Ok(w) => w.join("server"),
1641 Err(outcome) => return Some(outcome),
1642 };
1643 log.line("---- docs check (MNW_CHECK_DOCS) ----\n").await;
1644 let mut cmd = tokio::process::Command::new(bin);
1645 cmd.env("MNW_CHECK_DOCS", "1")
1646 .current_dir(&server_dir)
1647 .kill_on_drop(true);
1648 let (stdout, _stderr, status) =
1649 match tokio::time::timeout(std::time::Duration::from_mins(1), log.run(&mut cmd)).await {
1650 Ok(Ok(out)) => out,
1651 Ok(Err(e)) => {
1652 log.line(&format!("docs check spawn failed: {e}\n")).await;
1653 return Some(GateOutcome::failed(GateFailure::SpawnFailed {
1654 message: e.to_string(),
1655 }));
1656 }
1657 Err(_elapsed) => {
1658 let reason =
1659 "docs check timed out after 60s (staged binary may predate MNW_CHECK_DOCS)"
1660 .to_string();
1661 log.line(&format!("{reason}\n")).await;
1662 return Some(GateOutcome::failed(GateFailure::CodeSmokeSetup { reason }));
1663 }
1664 };
1665 if status.success() {
1666 return None;
1667 }
1668 Some(GateOutcome::failed(GateFailure::CodeSmokeDocs {
1669 broken: parse_check_docs_broken_count(&stdout),
1670 }))
1671 }
1672
1673 /// Best-effort parse of the broken-link count from the `MNW_CHECK_DOCS` sentinel
1674 /// line (`MNW_CHECK_DOCS: N broken link(s)`). Returns 0 if absent — the failure
1675 /// still stands, only the summary count is unknown.
1676 fn parse_check_docs_broken_count(stdout: &[u8]) -> u32 {
1677 let text = String::from_utf8_lossy(stdout);
1678 for line in text.lines() {
1679 if let Some(rest) = line.strip_prefix("MNW_CHECK_DOCS:") {
1680 for tok in rest.split_whitespace() {
1681 if let Ok(n) = tok.parse::<u32>() {
1682 return n;
1683 }
1684 }
1685 }
1686 }
1687 0
1688 }
1689
1690 /// The createdb-to-dropdb interior of `code_smoke`: migrate+seed, then boot and
1691 /// probe. Returns the outcome without a `log_ref` (the caller attaches it after
1692 /// teardown). Never returns `Err` — spawn/child failures map to typed outcomes.
1693 async fn code_smoke_body(ctx: &GateCtx, bin: &str, db_url: &str, log: &GateLog) -> GateOutcome {
1694 let server_dir = match ctx.worktree_for(GateKind::CodeSmoke) {
1695 Ok(w) => w.join("server"),
1696 Err(outcome) => return outcome,
1697 };
1698
1699 // Phase 1: migrate-from-scratch + seed. `--seed-examples` loads config,
1700 // connects, runs migrations against the empty DB, seeds the catalog, exits.
1701 // A non-zero exit here is the "code is unsound" signal (broken migration,
1702 // seed error, or config-load failure).
1703 log.line("---- migrate + seed (--seed-examples) ----\n")
1704 .await;
1705 let mut seed_cmd = tokio::process::Command::new(bin);
1706 seed_cmd.arg("--seed-examples").current_dir(&server_dir);
1707 code_smoke_env(&mut seed_cmd, ctx, db_url);
1708 seed_cmd.env("ALLOW_EXAMPLE_SEED", "1").kill_on_drop(true);
1709 let seed_status = match log.run(&mut seed_cmd).await {
1710 Ok((_stdout, _stderr, status)) => status,
1711 Err(e) => {
1712 return GateOutcome::failed(GateFailure::SpawnFailed {
1713 message: e.to_string(),
1714 });
1715 }
1716 };
1717 if !seed_status.success() {
1718 return GateOutcome::failed(GateFailure::CodeSmokeSeed {
1719 exit_code: seed_status.code(),
1720 });
1721 }
1722
1723 // Phase 2: boot the real server against the now-migrated + seeded DB and
1724 // assert both startup signals: it logs `listening` (emitted just before the
1725 // socket bind) AND serves GET /health with a 200. The full stdout/stderr is
1726 // persisted for the operator either way.
1727 log.line("\n---- boot + probe /health ----\n").await;
1728 let mut serve_cmd = tokio::process::Command::new(bin);
1729 serve_cmd.current_dir(&server_dir);
1730 code_smoke_env(&mut serve_cmd, ctx, db_url);
1731 serve_cmd
1732 .stdout(std::process::Stdio::piped())
1733 .stderr(std::process::Stdio::piped())
1734 .kill_on_drop(true);
1735 let mut child = match serve_cmd.spawn() {
1736 Ok(c) => c,
1737 Err(e) => {
1738 return GateOutcome::failed(GateFailure::SpawnFailed {
1739 message: e.to_string(),
1740 });
1741 }
1742 };
1743
1744 // Stream stdout/stderr into the gate log (and out as chunk events) while
1745 // the probe loop runs below; the tasks finish when the pipes close (child
1746 // exits or is killed). The buffers they return are what the `listening`
1747 // assertion reads.
1748 let (stdout_task, stderr_task) = log.drain_pipes(&mut child);
1749
1750 let probe_timeout = std::time::Duration::from_millis(500);
1751 let started = std::time::Instant::now();
1752 let window = std::time::Duration::from_secs(CODE_SMOKE_READY_SECS);
1753 let mut probe_ok_after: Option<u32> = None;
1754 let mut last_probe_err = "never responded".to_string();
1755 let mut early_exit = None;
1756 while started.elapsed() < window {
1757 if let Ok(Some(status)) = child.try_wait() {
1758 early_exit = Some(status);
1759 break;
1760 }
1761 match tokio::time::timeout(probe_timeout, probe_health(ctx.cfg.code_smoke_port)).await {
1762 Ok(Ok(())) => {
1763 probe_ok_after = Some(started.elapsed().as_millis() as u32);
1764 break;
1765 }
1766 Ok(Err(e)) => last_probe_err = e,
1767 Err(_) => last_probe_err = "probe timed out".to_string(),
1768 }
1769 tokio::time::sleep(std::time::Duration::from_millis(250)).await;
1770 }
1771
1772 let exit = match early_exit {
1773 Some(status) => Some(status),
1774 None => {
1775 let e = child.try_wait().ok().flatten();
1776 if e.is_none() {
1777 let _ = child.kill().await;
1778 }
1779 e
1780 }
1781 };
1782 // Keep the serve run's own output so the `listening`-log assertion checks it
1783 // (not the seed run's, which exits before binding). The bytes are already in
1784 // the gate log; these buffers exist only for the assertion.
1785 let serve_stdout = stdout_task.await.unwrap_or_default();
1786 let serve_stderr = stderr_task.await.unwrap_or_default();
1787 let logged_listening =
1788 bytes_contain(&serve_stdout, b"listening") || bytes_contain(&serve_stderr, b"listening");
1789
1790 match (exit, probe_ok_after) {
1791 // Exited on its own within the window — panic / config error / bind fail.
1792 (Some(status), _) => GateOutcome::failed(classify::classify_boot_smoke(status.code())),
1793 // Stayed up and served /health. Assert both required startup signals:
1794 // the `listening` bind log AND the /health 200.
1795 (None, Some(after_ms)) if logged_listening => {
1796 GateOutcome::passed(PassNote::HealthyProbe { after_ms })
1797 }
1798 // Served /health but the `listening` log never appeared.
1799 (None, Some(_)) => GateOutcome::failed(GateFailure::CodeSmokeNoListeningLog),
1800 // Stayed up but never served /health — started, not ready.
1801 (None, None) => GateOutcome::failed(GateFailure::BootHealthProbeFailed {
1802 last_error: last_probe_err,
1803 }),
1804 }
1805 }
1806
1807 /// Substring search over raw bytes (the server's log output), for the
1808 /// `code_smoke` startup-log assertion. Avoids a lossy UTF-8 conversion of the
1809 /// whole buffer just to run `str::contains`.
1810 fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool {
1811 if needle.is_empty() || haystack.len() < needle.len() {
1812 return needle.is_empty();
1813 }
1814 haystack.windows(needle.len()).any(|w| w == needle)
1815 }
1816
1817 /// Apply the minimal env every `code_smoke` invocation shares: point the binary
1818 /// at the throwaway DB, force loopback (dev-mode config, no prod enforcement),
1819 /// hand it a dummy signing secret, and disable file scanning (no AV/YARA on the
1820 /// build host). The worktree is a clean git checkout, so no stray `.env` shadows
1821 /// these (and dotenvy never overrides already-set vars).
1822 ///
1823 /// This list has to carry EVERY var the server's `Config::from_env` treats as
1824 /// mandatory, because `code_smoke` is the only gate that reaches that function
1825 /// at all: `boot_smoke` and the docs check both short-circuit in `main` before
1826 /// config is loaded. So when the server makes a new var required, this is where
1827 /// it has to be answered, and nothing connects the two lists automatically.
1828 ///
1829 /// That has already bitten once. `CDN_BASE_URL` became mandatory on 2026-07-30
1830 /// (server 3e3b1d15, closing a cover URL that could expire), 13 days after this
1831 /// function was written, and `code_smoke` failed with `MissingCdnBaseUrl` the
1832 /// first time it ran on the host — which was 2026-08-03, because the gate was
1833 /// configured in the repo and absent from `/etc/sando` in between.
1834 ///
1835 /// The values are deliberately throwaway. The gate asks whether this code can
1836 /// migrate, seed, boot and serve; whether a given deployment's env is complete
1837 /// is the `config_check_env_file` guard's job, on the node, against that node's
1838 /// real env file.
1839 fn code_smoke_env(cmd: &mut tokio::process::Command, ctx: &GateCtx, db_url: &str) {
1840 // `localhost`, not `127.0.0.1`, and the distinction is load-bearing. The
1841 // server derives its WebAuthn relying-party id from HOST_URL's host, and
1842 // `WebauthnBuilder::new` validates that id against `Url::domain()` — which
1843 // is `None` for an IP literal, so an origin of `http://127.0.0.1:<port>`
1844 // fails with WebauthnError::Configuration before the server ever binds. It
1845 // is a real ceiling on the gate, not a preference: no IP-literal origin can
1846 // boot this binary. `localhost` is a domain, and matches the derived rp_id.
1847 //
1848 // HOST stays 127.0.0.1: that is the bind address, and the gate probes the
1849 // loopback address directly, so only the advertised origin changes.
1850 let origin = format!("http://localhost:{}", ctx.cfg.code_smoke_port);
1851 cmd.env("DATABASE_URL", db_url)
1852 .env("HOST", "127.0.0.1")
1853 .env("PORT", ctx.cfg.code_smoke_port.to_string())
1854 .env("HOST_URL", &origin)
1855 // Required unconditionally by Config::from_env. Pointing it at the smoke
1856 // server's own origin keeps every rendered media URL resolvable within
1857 // the gate; no request is ever made to it.
1858 .env("CDN_BASE_URL", &origin)
1859 .env("SIGNING_SECRET", CODE_SMOKE_SIGNING_SECRET)
1860 .env("SCAN_ENABLED", "false")
1861 .env("INSECURE_COOKIES", "1");
1862 }
1863
1864 /// The throwaway smoke DB name for a version: `sando_code_smoke_<version>` with
1865 /// every non-alphanumeric char folded to `_` and lowercased, capped at Postgres'
1866 /// 63-byte identifier limit. Sanitized to `[a-z0-9_]` so it's safe to quote into
1867 /// DDL. Deterministic per version, so a stale DB from a killed run is reclaimed
1868 /// by the next run's `DROP DATABASE IF EXISTS` rather than accumulating.
1869 fn code_smoke_db_name(version: &Version) -> String {
1870 let mut name = String::from("sando_code_smoke_");
1871 for c in version.to_string().chars() {
1872 name.push(if c.is_ascii_alphanumeric() {
1873 c.to_ascii_lowercase()
1874 } else {
1875 '_'
1876 });
1877 }
1878 name.truncate(63);
1879 name
1880 }
1881
1882 /// Rewrite a `postgres://` URL to point at database `dbname`, preserving scheme,
1883 /// userinfo, host/port, and any query (e.g. the socket `?host=/var/run/postgresql`
1884 /// form) + fragment. Used to derive the maintenance connection (`postgres`) and
1885 /// the throwaway smoke DB URL from the configured `scratch_db_url`.
1886 fn pg_url_with_dbname(url: &str, dbname: &str) -> String {
1887 let Some(after_scheme) = url.find("://").map(|i| i + 3) else {
1888 return url.to_string();
1889 };
1890 let rest = &url[after_scheme..];
1891 // Authority ends at the first '/', '?' or '#'; whatever follows is the
1892 // path (the old dbname) plus an optional query/fragment we must keep.
1893 let auth_end = rest.find(['/', '?', '#']).unwrap_or(rest.len());
1894 let authority = &rest[..auth_end];
1895 let tail = &rest[auth_end..];
1896 let query_and_frag = match tail.find(['?', '#']) {
1897 Some(i) => &tail[i..],
1898 None => "",
1899 };
1900 format!(
1901 "{}{}/{}{}",
1902 &url[..after_scheme],
1903 authority,
1904 dbname,
1905 query_and_frag
1906 )
1907 }
1908
1909 /// Create the throwaway smoke DB on the cluster `maintenance_url` points at,
1910 /// dropping any stale one first. `dbname` is sanitized to `[a-z0-9_]` by
1911 /// `code_smoke_db_name`, so quoting it is sufficient. `CREATE DATABASE` cannot
1912 /// run inside a transaction, so these go through the simple-query protocol (a
1913 /// raw `&str` execute), matching `reset_scratch`.
1914 async fn pg_create_db(maintenance_url: &str, dbname: &str) -> Result<()> {
1915 use sqlx::Executor;
1916 use sqlx::postgres::PgPoolOptions;
1917 let pool = PgPoolOptions::new()
1918 .max_connections(1)
1919 .connect(maintenance_url)
1920 .await?;
1921 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1922 "DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE)"
1923 ))))
1924 .await?;
1925 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1926 "CREATE DATABASE \"{dbname}\""
1927 ))))
1928 .await?;
1929 pool.close().await;
1930 Ok(())
1931 }
1932
1933 /// Drop the throwaway smoke DB, forcing off any lingering connection (the killed
1934 /// server's pool). Best-effort at the call site — a failure is logged, not fatal.
1935 async fn pg_drop_db(maintenance_url: &str, dbname: &str) -> Result<()> {
1936 use sqlx::Executor;
1937 use sqlx::postgres::PgPoolOptions;
1938 let pool = PgPoolOptions::new()
1939 .max_connections(1)
1940 .connect(maintenance_url)
1941 .await?;
1942 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1943 "DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE)"
1944 ))))
1945 .await?;
1946 pool.close().await;
1947 Ok(())
1948 }
1949
1950 async fn boot_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
1951 let bin: Option<(String,)> =
1952 sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
1953 .bind(&ctx.cfg.id)
1954 .bind(&ctx.version)
1955 .fetch_optional(&ctx.pool)
1956 .await?;
1957 let Some((bin,)) = bin else {
1958 return Ok(GateOutcome::blocked(GateBlocker::ArtifactMissing {
1959 version: ctx.version.clone(),
1960 }));
1961 };
1962
1963 // Readiness smoke: start the binary and confirm it serves `GET /health`
1964 // within the window, not merely that the process stays up. Panics in main,
1965 // missing config, and port-bind failures still surface as an early exit; a
1966 // process that comes up but never serves /health is now its own failure.
1967 //
1968 // The server requires DATABASE_URL or it panics on config load before
1969 // we can observe anything. We point it at the scratch DB (already
1970 // migrated by the build step and refreshed by migration_dry_run if
1971 // that gate ran first). SCAN_ENABLED=false skips loading YARA rules
1972 // from /opt/makenotwork/yara-rules which doesn't exist on the build
1973 // host. SANDO_BOOT_SMOKE_PORT tells the smoke server which loopback port
1974 // to bind so we know where to probe. Other config has sane optional defaults.
1975 let mut cmd = tokio::process::Command::new(&bin);
1976 cmd.env("SANDO_BOOT_SMOKE", "1")
1977 .env("SANDO_BOOT_SMOKE_PORT", ctx.cfg.boot_smoke_port.to_string())
1978 .env("SCAN_ENABLED", "false")
1979 .stdout(std::process::Stdio::piped())
1980 .stderr(std::process::Stdio::piped())
1981 .kill_on_drop(true);
1982 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
1983 cmd.env("DATABASE_URL", scratch_url);
1984 }
1985 let log_path = gate_log_path(ctx, GateKind::BootSmoke);
1986 let log_ref = LogRef::new(&ctx.version, GateKind::BootSmoke);
1987 let mut child = match cmd.spawn() {
1988 Ok(c) => c,
1989 Err(e) => {
1990 // Spawn failures get a one-off log line via LiveLog so the
1991 // on-disk file still exists for `GET /logs/...`.
1992 let mut log = LiveLog::open(log_path, gate_chunk_cb(ctx.events.clone(), run_id)).await;
1993 log.write_chunk(format!("spawn: {e}\n").as_bytes()).await;
1994 log.close().await;
1995 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
1996 message: e.to_string(),
1997 })
1998 .with_log_ref(log_ref));
1999 }
2000 };
2001
2002 // The boot smoke window is 3s. Drain stdout/stderr concurrently through
2003 // a shared LiveLog sink so the operator sees panics/log lines stream in
2004 // real time before the kill, AND the on-disk log gets the full byte
2005 // stream for post-mortem reads. The drainers exit when their pipe
2006 // closes — which happens when the child exits naturally or after kill.
2007 let log = std::sync::Arc::new(tokio::sync::Mutex::new(
2008 LiveLog::open(log_path, gate_chunk_cb(ctx.events.clone(), run_id)).await,
2009 ));
2010 let stdout_task = tokio::spawn(stream_into_log(child.stdout.take(), log.clone()));
2011 let stderr_task = tokio::spawn(stream_into_log(child.stderr.take(), log.clone()));
2012
2013 // Poll readiness across the 3s window instead of a flat sleep: GET /health
2014 // must return 2xx. A crash mid-window short-circuits to the exit-code
2015 // failure path (try_wait below); a process that stays up but never serves
2016 // /health is a distinct readiness failure.
2017 let probe_timeout = std::time::Duration::from_millis(500);
2018 let started = std::time::Instant::now();
2019 let window = std::time::Duration::from_secs(3);
2020 let mut probe_ok_after: Option<u32> = None;
2021 let mut last_probe_err = "never responded".to_string();
2022 let mut early_exit = None;
2023 while started.elapsed() < window {
2024 if let Some(status) = child.try_wait()? {
2025 early_exit = Some(status);
2026 break;
2027 }
2028 match tokio::time::timeout(probe_timeout, probe_health(ctx.cfg.boot_smoke_port)).await {
2029 Ok(Ok(())) => {
2030 probe_ok_after = Some(started.elapsed().as_millis() as u32);
2031 break;
2032 }
2033 Ok(Err(e)) => last_probe_err = e,
2034 Err(_) => last_probe_err = "probe timed out".to_string(),
2035 }
2036 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
2037 }
2038
2039 // Stop the child unless it already exited, then drain the log tasks.
2040 let exit = match early_exit {
2041 Some(status) => Some(status),
2042 None => {
2043 let e = child.try_wait()?;
2044 if e.is_none() {
2045 let _ = child.kill().await;
2046 }
2047 e
2048 }
2049 };
2050 // The streamed bytes already landed in the live log and the on-disk file for
2051 // the post-mortem reader. Drain the join handles to avoid hangs.
2052 let _ = stdout_task.await;
2053 let _ = stderr_task.await;
2054 // Unique owner of the Arc at this point (both tasks dropped their clones).
2055 if let Ok(mutex) = std::sync::Arc::try_unwrap(log) {
2056 mutex.into_inner().close().await;
2057 }
2058
2059 match (exit, probe_ok_after) {
2060 // Exited on its own within the window — a crash/panic/bind failure.
2061 (Some(status), _) => {
2062 let failure = classify::classify_boot_smoke(status.code());
2063 Ok(GateOutcome::failed(failure).with_log_ref(log_ref))
2064 }
2065 // Stayed up and served /health — readiness proven.
2066 (None, Some(after_ms)) => {
2067 Ok(GateOutcome::passed(PassNote::HealthyProbe { after_ms }).with_log_ref(log_ref))
2068 }
2069 // Stayed up but never served /health — started, not ready.
2070 (None, None) => Ok(GateOutcome::failed(GateFailure::BootHealthProbeFailed {
2071 last_error: last_probe_err,
2072 })
2073 .with_log_ref(log_ref)),
2074 }
2075 }
2076
2077 /// One readiness probe of the boot-smoke server: connect to `127.0.0.1:port`
2078 /// and `GET /health`, returning `Ok(())` only on a `200`. A hand-rolled HTTP/1.0
2079 /// request over a raw `TcpStream` keeps the outbound probe dependency-free
2080 /// (reqwest is dev-only); the smoke server serves the one route over axum, which
2081 /// speaks 1.0. `Err` carries a short reason for the operator's failure note. The
2082 /// caller wraps each call in a timeout.
2083 async fn probe_health(port: u16) -> std::result::Result<(), String> {
2084 use tokio::io::{AsyncReadExt, AsyncWriteExt};
2085 let mut stream = tokio::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, port))
2086 .await
2087 .map_err(|e| format!("connect: {e}"))?;
2088 stream
2089 .write_all(b"GET /health HTTP/1.0\r\nHost: localhost\r\nConnection: close\r\n\r\n")
2090 .await
2091 .map_err(|e| format!("write: {e}"))?;
2092 let mut buf = Vec::new();
2093 stream
2094 .read_to_end(&mut buf)
2095 .await
2096 .map_err(|e| format!("read: {e}"))?;
2097 let text = String::from_utf8_lossy(&buf);
2098 let status_line = text.lines().next().unwrap_or("");
2099 if status_line.contains(" 200 ") {
2100 Ok(())
2101 } else {
2102 Err(format!("unexpected status line: {status_line:?}"))
2103 }
2104 }
2105
2106 /// Sink that drops streamed output. `node_health` keeps the probe's stderr from
2107 /// its `RunOutput` for the failure note, so the live byte stream isn't needed.
2108 struct DiscardSink;
2109 #[async_trait::async_trait]
2110 impl ops_exec::LogSink for DiscardSink {
2111 async fn write_chunk(&mut self, _bytes: &[u8]) {}
2112 }
2113
2114 /// `node_health` — the post-deploy gate that proves the *deployed nodes* are
2115 /// serving, recording one outcome per (tier, version) that the next promote
2116 /// checks. Distinct from `boot_smoke`, which boots the staged artifact on the
2117 /// build host: this probes each node over the same executor the deploy used, so
2118 /// a node that took a corrupt artifact, wrong-arch binary, or failed restart is
2119 /// caught here rather than waved through (Run-2 SERIOUS-3). Fails closed: any
2120 /// unhealthy node fails the gate, and an empty node set is `Blocked`.
2121 /// Load the tier's public pages in a real browser and fail if the JavaScript
2122 /// did not run.
2123 ///
2124 /// The only gate here that crosses the CDN. `boot_smoke` runs on the build host
2125 /// and `node_health` reaches a node over its executor, so between them nothing
2126 /// ever requested the site the way a visitor does. On 2026-08-14 that gap
2127 /// shipped: testnot served a landing page whose JavaScript did not run, for
2128 /// hours, behind nine green gates. Cloudflare was holding a module from an
2129 /// earlier deploy; it failed to link against the fresh one beside it, and since
2130 /// the bundle's entry point side-effect-imports every island, one bad link took
2131 /// all of them down together.
2132 ///
2133 /// It could not have been caught earlier. Each artifact was individually
2134 /// correct -- right markup, right stylesheets, every module answering 200 with
2135 /// current bytes. Only their composition was broken, and composition is
2136 /// observable in a browser and nowhere else.
2137 ///
2138 /// Runs on the daemon host rather than on a node, because it is a *client*: it
2139 /// should reach the site through whatever the public reaches it through, and a
2140 /// probe that ran on the origin would inherit the blind spot this exists to
2141 /// close.
2142 async fn page_smoke(ctx: &GateCtx) -> Result<GateOutcome> {
2143 let Some(cmd) = ctx.cfg.page_smoke_cmd.as_deref() else {
2144 // A service with no pages says so by configuring no command. Blocked
2145 // rather than passed: a gate that proves nothing must not read green.
2146 return Ok(GateOutcome::blocked(GateBlocker::NotConfigured {
2147 what: "page_smoke_cmd".into(),
2148 }));
2149 };
2150 let Some(base) = ctx.public_url.as_deref() else {
2151 return Ok(GateOutcome::blocked(GateBlocker::NotConfigured {
2152 what: "public_url".into(),
2153 }));
2154 };
2155
2156 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
2157 let mut child = tokio::process::Command::new("sh")
2158 .arg("-c")
2159 .arg(cmd)
2160 .env("BASE", base)
2161 .stdout(std::process::Stdio::piped())
2162 .stderr(std::process::Stdio::piped())
2163 .kill_on_drop(true)
2164 .spawn()?;
2165
2166 let out = match tokio::time::timeout(ceiling, child.wait_with_output()).await {
2167 Ok(res) => res?,
2168 Err(_elapsed) => {
2169 return Ok(GateOutcome::failed(GateFailure::Timeout {
2170 gate: GateKind::PageSmoke,
2171 after_s: ctx.cfg.gate_timeout_secs as u32,
2172 })
2173 .with_log_ref(LogRef::new(&ctx.version, GateKind::PageSmoke)));
2174 }
2175 };
2176
2177 let log = format!(
2178 "{}{}",
2179 String::from_utf8_lossy(&out.stdout),
2180 String::from_utf8_lossy(&out.stderr)
2181 );
2182 append_to_log(&gate_log_path(ctx, GateKind::PageSmoke), log.as_bytes()).await;
2183
2184 if out.status.success() {
2185 return Ok(
2186 GateOutcome::passed(PassNote::PagesClean { base: base.into() })
2187 .with_log_ref(LogRef::new(&ctx.version, GateKind::PageSmoke)),
2188 );
2189 }
2190
2191 // The script prints one `FAIL <url>` line per bad page and indents the
2192 // reasons under it. Lift the first reason into the summary so a red gate
2193 // says what broke without anyone opening the log.
2194 let first = log
2195 .lines()
2196 .skip_while(|l| !l.starts_with("FAIL"))
2197 .nth(1)
2198 .map(str::trim)
2199 .filter(|l| !l.is_empty())
2200 .unwrap_or("see log");
2201 Ok(GateOutcome::failed(GateFailure::PagesBroken {
2202 base: base.into(),
2203 detail: first.to_string(),
2204 })
2205 .with_log_ref(LogRef::new(&ctx.version, GateKind::PageSmoke)))
2206 }
2207
2208 async fn node_health(ctx: &GateCtx) -> Result<GateOutcome> {
2209 if ctx.nodes.is_empty() {
2210 return Ok(GateOutcome::blocked(GateBlocker::NoNodesToProbe));
2211 }
2212 for probe in &ctx.nodes {
2213 if let Err(detail) = probe_node(probe).await {
2214 return Ok(GateOutcome::failed(GateFailure::NodeUnhealthy {
2215 node: probe.node.to_string(),
2216 detail,
2217 }));
2218 }
2219 }
2220 Ok(GateOutcome::passed(PassNote::NodesHealthy {
2221 nodes: ctx.nodes.len() as u32,
2222 }))
2223 }
2224
2225 /// Probe one node over its executor: confirm the unit is active post-restart
2226 /// and, when a `health_url` is configured, that it serves a 2xx. Retries across
2227 /// ~10s because the service may still be restarting / warming. Runs under the
2228 /// read-only `Observe(Health)` capability (every Sando node grants it), so the
2229 /// probe needs no deploy authority. `Ok(())` = healthy; `Err(detail)` carries a
2230 /// short reason for the gate's failure note.
2231 async fn probe_node(probe: &NodeProbe) -> std::result::Result<(), String> {
2232 use ops_exec::{Action, ObserveKind, Step, sh_quote};
2233 let svc = sh_quote(&probe.service);
2234 let url = probe
2235 .health_url
2236 .as_deref()
2237 .map_or_else(|| "''".to_string(), sh_quote);
2238 // One executor round-trip with the retry loop on the node: is-active, then
2239 // (if a url is set) curl it for a 2xx. Exit 0 only when both hold.
2240 let script = format!(
2241 "svc={svc}; url={url}; \
2242 for _ in $(seq 1 10); do \
2243 if systemctl is-active --quiet \"$svc\"; then \
2244 if [ -z \"$url\" ] || curl -fsS --max-time 5 \"$url\" >/dev/null 2>&1; then exit 0; fi; \
2245 fi; \
2246 sleep 1; \
2247 done; \
2248 echo 'service not active or health url not 2xx after retries' >&2; exit 1"
2249 );
2250 let step = Step::shell(Action::Observe(ObserveKind::Health), script);
2251 let mut sink = DiscardSink;
2252 let out = probe
2253 .executor
2254 .run_streaming(&step, &mut sink)
2255 .await
2256 .map_err(|e| format!("probe spawn: {e}"))?;
2257 if out.status.success() {
2258 Ok(())
2259 } else {
2260 let code = out
2261 .status
2262 .code()
2263 .map_or_else(|| "signal".to_string(), |c| c.to_string());
2264 let stderr: String = String::from_utf8_lossy(&out.stderr)
2265 .chars()
2266 .take(200)
2267 .collect();
2268 Err(format!("exit {code}: {stderr}"))
2269 }
2270 }
2271
2272 /// Drain `stream` into the shared `LiveLog` (which forwards each chunk to
2273 /// the on-disk log file AND broadcasts a `GateLogChunk` event), and return
2274 /// the concatenated bytes so the classifier can still operate on the full
2275 /// output post-hoc.
2276 async fn stream_into_log<R>(
2277 stream: Option<R>,
2278 log: std::sync::Arc<tokio::sync::Mutex<LiveLog>>,
2279 ) -> Vec<u8>
2280 where
2281 R: tokio::io::AsyncRead + Unpin + Send + 'static,
2282 {
2283 let mut total = Vec::new();
2284 let Some(mut s) = stream else { return total };
2285 let mut buf = [0u8; 4096];
2286 loop {
2287 match s.read(&mut buf).await {
2288 Ok(0) => break,
2289 Err(_) => break,
2290 Ok(n) => {
2291 total.extend_from_slice(&buf[..n]);
2292 log.lock().await.write_chunk(&buf[..n]).await;
2293 }
2294 }
2295 }
2296 total
2297 }
2298
2299 /// Spawn a child, drain its stdout/stderr through a `LiveLog`, return the
2300 /// combined buffers and exit status. Shared by `cargo_test` (no deadline)
2301 /// and ad-hoc callers — `boot_smoke` rolls its own variant because of its
2302 /// 3s kill window.
2303 async fn stream_child_to_live_log(
2304 child: &mut tokio::process::Child,
2305 events: EventTx,
2306 run_id: GateRunId,
2307 log_path: PathBuf,
2308 ) -> Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> {
2309 let log = GateLog::new(LiveLog::open(log_path, gate_chunk_cb(events, run_id)).await);
2310 let out = log.stream_child(child).await;
2311 log.close().await;
2312 out
2313 }
2314
2315 /// One gate's live log, held across every step of a multi-step gate.
2316 ///
2317 /// Single-child gates call [`stream_child_to_live_log`] and are done. The
2318 /// staged gates (`code_smoke`, `migration_dry_run`) instead run several
2319 /// children plus banner lines between them, and they hold one `GateLog` across
2320 /// the lot: a single sink means chunk sequence numbers stay monotonic for the
2321 /// whole gate, and the on-disk log reads in the order things actually happened
2322 /// rather than as stdout-then-stderr assembled at the end.
2323 ///
2324 /// Every write is best-effort in the same way `LiveLog` is: a log directory
2325 /// that cannot be written degrades to callback-only and never turns a passing
2326 /// gate red.
2327 struct GateLog {
2328 sink: Arc<tokio::sync::Mutex<LiveLog>>,
2329 }
2330
2331 impl GateLog {
2332 fn new(sink: LiveLog) -> Self {
2333 Self {
2334 sink: Arc::new(tokio::sync::Mutex::new(sink)),
2335 }
2336 }
2337
2338 /// Open the sink for `gate`'s log file, streaming to the TUI under `run_id`.
2339 async fn open(ctx: &GateCtx, run_id: GateRunId, gate: GateKind) -> Self {
2340 Self::new(
2341 LiveLog::open(
2342 gate_log_path(ctx, gate),
2343 gate_chunk_cb(ctx.events.clone(), run_id),
2344 )
2345 .await,
2346 )
2347 }
2348
2349 /// Emit a banner (or any line the gate itself produces) through the same
2350 /// sink the children stream to, so it lands in sequence with their output.
2351 async fn write(&self, bytes: &[u8]) {
2352 self.sink.lock().await.write_chunk(bytes).await;
2353 }
2354
2355 /// Same, for the common `format!`-a-line case.
2356 async fn line(&self, s: &str) {
2357 self.write(s.as_bytes()).await;
2358 }
2359
2360 /// Spawn `cmd` with both pipes captured, stream them into the sink as they
2361 /// arrive, and return the buffers plus the exit status. The buffers are
2362 /// what the classifiers still operate on post-hoc.
2363 async fn run(
2364 &self,
2365 cmd: &mut Command,
2366 ) -> std::io::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> {
2367 cmd.stdout(std::process::Stdio::piped())
2368 .stderr(std::process::Stdio::piped());
2369 let mut child = cmd.spawn()?;
2370 self.stream_child(&mut child)
2371 .await
2372 .map_err(std::io::Error::other)
2373 }
2374
2375 /// Drain an already-spawned child's pipes into the sink and wait for it.
2376 async fn stream_child(
2377 &self,
2378 child: &mut tokio::process::Child,
2379 ) -> Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> {
2380 let (stdout_task, stderr_task) = self.drain_pipes(child);
2381 let status = child.wait().await?;
2382 let stdout_buf = stdout_task.await.unwrap_or_default();
2383 let stderr_buf = stderr_task.await.unwrap_or_default();
2384 Ok((stdout_buf, stderr_buf, status))
2385 }
2386
2387 /// Start draining a child's pipes into the sink *without* waiting on the
2388 /// child. For `code_smoke`'s serve phase, which probes `/health` while the
2389 /// server is still up. The caller must await both handles for the buffers
2390 /// (and before [`Self::close`], so the flush isn't skipped).
2391 #[allow(clippy::type_complexity)]
2392 fn drain_pipes(
2393 &self,
2394 child: &mut tokio::process::Child,
2395 ) -> (
2396 tokio::task::JoinHandle<Vec<u8>>,
2397 tokio::task::JoinHandle<Vec<u8>>,
2398 ) {
2399 (
2400 tokio::spawn(stream_into_log(child.stdout.take(), self.sink.clone())),
2401 tokio::spawn(stream_into_log(child.stderr.take(), self.sink.clone())),
2402 )
2403 }
2404
2405 /// Flush the file. A still-outstanding streaming task (only possible if the
2406 /// gate returned without awaiting it) leaves the `Arc` shared, in which case
2407 /// the flush is skipped — `LiveLog` writes unbuffered to the OS either way,
2408 /// so nothing already written is lost.
2409 async fn close(self) {
2410 if let Ok(mutex) = Arc::try_unwrap(self.sink) {
2411 mutex.into_inner().close().await;
2412 }
2413 }
2414 }
2415
2416 fn gate_log_path(ctx: &GateCtx, gate: GateKind) -> PathBuf {
2417 ctx.cfg
2418 .logs_root
2419 .join(ctx.version.to_string())
2420 .join(format!("{}.log", gate.as_str()))
2421 }
2422
2423 /// Live check: has `tier`'s burn-in window of `hours` elapsed since its clock
2424 /// (`tier_state.burn_in_started_at`, started by a promote onto the tier)? Used
2425 /// by the promote-time gate check (`unsatisfied_gates`) so a stale `blocked`
2426 /// row never masks an elapsed — or not-yet-elapsed — window. The `burn_in` gate
2427 /// runner below wraps the same state with a richer outcome for `/state`.
2428 pub async fn burn_in_satisfied(
2429 pool: &SqlitePool,
2430 app: &AppId,
2431 tier: &TierId,
2432 hours: u32,
2433 ) -> Result<bool> {
2434 let started: Option<String> =
2435 sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?")
2436 .bind(app)
2437 .bind(tier)
2438 .fetch_optional(pool)
2439 .await?
2440 .flatten();
2441 let Some(started) = started else {
2442 return Ok(false);
2443 };
2444 let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc);
2445 Ok(Utc::now() - started >= chrono::Duration::hours(hours as i64))
2446 }
2447
2448 async fn burn_in(ctx: &GateCtx, hours: u32) -> Result<GateOutcome> {
2449 // Check tier_state.burn_in_started_at on this tier; pass if enough time
2450 // has elapsed. The clock is started by /promote when a version lands on
2451 // the burn-in tier.
2452 let started: Option<String> =
2453 sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?")
2454 .bind(&ctx.cfg.id)
2455 .bind(&ctx.tier)
2456 .fetch_optional(&ctx.pool)
2457 .await?
2458 .flatten();
2459 let Some(started) = started else {
2460 return Ok(GateOutcome::blocked(GateBlocker::BurnInClockNotStarted));
2461 };
2462 let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc);
2463 let elapsed = Utc::now() - started;
2464 let needed = chrono::Duration::hours(hours as i64);
2465 if elapsed >= needed {
2466 Ok(GateOutcome::passed(PassNote::BurnInElapsed {
2467 hours: elapsed.num_hours() as u32,
2468 }))
2469 } else {
2470 let remaining = (needed - elapsed).num_hours().max(0) as u32;
2471 Ok(GateOutcome::blocked(GateBlocker::BurnInRemaining {
2472 hours_remaining: remaining,
2473 hours_total: hours,
2474 }))
2475 }
2476 }
2477
2478 async fn manual_confirm(ctx: &GateCtx) -> Result<GateOutcome> {
2479 // Pass iff a row in gate_runs exists with status='passed' for this
2480 // (tier, version, manual_confirm) that was inserted out-of-band by an
2481 // operator action. Since the harness inserts the in-flight row itself,
2482 // look for a prior confirmation row.
2483 let prior_at: Option<String> = sqlx::query_scalar(
2484 "SELECT finished_at FROM gate_runs
2485 WHERE app = ? AND tier = ? AND version = ? AND gate_kind = 'manual_confirm'
2486 AND status = 'passed'
2487 ORDER BY id DESC LIMIT 1",
2488 )
2489 .bind(&ctx.cfg.id)
2490 .bind(&ctx.tier)
2491 .bind(&ctx.version)
2492 .fetch_optional(&ctx.pool)
2493 .await?;
2494 match prior_at {
2495 Some(at_str) => {
2496 let at = chrono::DateTime::parse_from_rfc3339(&at_str)
2497 .map_or_else(|_| Utc::now(), |d| d.with_timezone(&Utc));
2498 Ok(GateOutcome::passed(PassNote::OperatorConfirmed { at }))
2499 }
2500 None => Ok(GateOutcome::blocked(
2501 GateBlocker::AwaitingOperatorConfirmation,
2502 )),
2503 }
2504 }
2505
2506 #[cfg(test)]
2507 mod tests {
2508 use super::*;
2509 use crate::events;
2510 use sqlx::sqlite::SqlitePoolOptions;
2511
2512 fn target(dir: &str) -> crate::config::TestTarget {
2513 crate::config::TestTarget {
2514 dir: std::path::PathBuf::from(dir),
2515 aux_repo: None,
2516 features: Vec::new(),
2517 all_features: false,
2518 scratch_db: false,
2519 }
2520 }
2521
2522 #[test]
2523 fn parse_check_docs_broken_count_reads_the_sentinel() {
2524 // The failing sentinel carries the count.
2525 assert_eq!(
2526 parse_check_docs_broken_count(b"some log\nMNW_CHECK_DOCS: 3 broken link(s)\n"),
2527 3
2528 );
2529 // Count with a preceding log line still parses (first bare int wins).
2530 assert_eq!(
2531 parse_check_docs_broken_count(
2532 b" broken link: a -> b\nMNW_CHECK_DOCS: 1 broken link(s)\n"
2533 ),
2534 1
2535 );
2536 // The parser only runs on failure; the "ok" sentinel is never fed to it,
2537 // and its "(2" token is not a bare int, so it yields 0 harmlessly.
2538 assert_eq!(
2539 parse_check_docs_broken_count(b"MNW_CHECK_DOCS: ok (2 collision(s) reported)\n"),
2540 0
2541 );
2542 // Absent sentinel -> 0; the failure still stands, only the count is lost.
2543 assert_eq!(parse_check_docs_broken_count(b"unrelated output"), 0);
2544 }
2545
2546 #[test]
2547 fn name_target_points_a_test_failure_at_its_crate() {
2548 // With one target the crate was implicit; with fifteen the operator
2549 // needs the summary to say which one broke.
2550 let f = name_target(
2551 GateFailure::CargoTest {
2552 failed_count: 3,
2553 first_failed: Some("workflows::sync::round_trip".into()),
2554 first_panic: None,
2555 },
2556 std::path::Path::new("shared/synckit-client"),
2557 );
2558 assert_eq!(
2559 f.summary(),
2560 "3 test(s) failed; first: shared/synckit-client: workflows::sync::round_trip",
2561 );
2562 }
2563
2564 #[test]
2565 fn name_target_points_a_compile_failure_at_its_crate() {
2566 let f = name_target(
2567 GateFailure::CompileError {
2568 error_count: 1,
2569 first_error: Some("error[E0063]".into()),
2570 },
2571 std::path::Path::new("mnw-cli"),
2572 );
2573 assert_eq!(
2574 f.summary(),
2575 "compile failed (1 error(s)); first: mnw-cli: error[E0063]"
2576 );
2577 }
2578
2579 #[test]
2580 fn name_target_names_the_crate_even_without_a_test_name() {
2581 let f = name_target(
2582 GateFailure::CargoTest {
2583 failed_count: 2,
2584 first_failed: None,
2585 first_panic: None,
2586 },
2587 std::path::Path::new("pom"),
2588 );
2589 assert_eq!(f.summary(), "2 test(s) failed; first: pom");
2590 }
2591
2592 #[test]
2593 fn name_target_leaves_unrelated_failures_alone() {
2594 let f = name_target(
2595 GateFailure::SpawnFailed {
2596 message: "no cargo".into(),
2597 },
2598 std::path::Path::new("pom"),
2599 );
2600 assert!(matches!(f, GateFailure::SpawnFailed { .. }));
2601 }
2602
2603 /// `target()` above, but resolved against an aux repo's checkout.
2604 fn aux_target(dir: &str, repo: &str) -> crate::config::TestTarget {
2605 crate::config::TestTarget {
2606 aux_repo: Some(repo.to_string()),
2607 ..target(dir)
2608 }
2609 }
2610
2611 /// True when the URL's host parses as a domain rather than an IP literal,
2612 /// which is the distinction `Url::domain()` draws and WebAuthn depends on.
2613 fn url_host_is_a_domain(url: &str) -> bool {
2614 let after = url.split("://").nth(1).unwrap_or("");
2615 let host = after.split(['/', '?', '#']).next().unwrap_or("");
2616 let host = host.rsplit('@').next().unwrap_or(host);
2617 let host = if let Some(rest) = host.strip_prefix('[') {
2618 rest.split(']').next().unwrap_or("")
2619 } else {
2620 host.split(':').next().unwrap_or("")
2621 };
2622 !host.is_empty() && host.parse::<std::net::IpAddr>().is_err()
2623 }
2624
2625 fn resolving_ctx(worktree: &str, aux: &[(&str, &str)]) -> GateCtx {
2626 GateCtx {
2627 public_url: None,
2628 pool: SqlitePool::connect_lazy("sqlite::memory:").unwrap(),
2629 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
2630 tier: TierId::new("host"),
2631 version: "0.1.0".parse().unwrap(),
2632 worktree: Some(PathBuf::from(worktree)),
2633 bundle: None,
2634 events: events::channel(),
2635 nodes: Vec::new(),
2636 build_id: None,
2637 aux_dirs: aux
2638 .iter()
2639 .map(|(n, d)| ((*n).to_string(), PathBuf::from(d)))
2640 .collect(),
2641 }
2642 }
2643
2644 #[tokio::test]
2645 async fn a_plain_target_resolves_under_the_worktree() {
2646 let ctx = resolving_ctx("/w/abc123", &[]);
2647 assert_eq!(
2648 ctx.target_dir(&target("shared/tagtree")),
2649 Some(PathBuf::from("/w/abc123/shared/tagtree")),
2650 );
2651 }
2652
2653 #[tokio::test]
2654 async fn an_aux_target_resolves_beside_the_worktree_not_under_it() {
2655 // The whole point: `Libraries/docengine` is a sibling of the per-sha
2656 // worktree, so a worktree-relative path can never reach it.
2657 let ctx = resolving_ctx("/w/abc123", &[("docengine", "/w/Libraries/docengine")]);
2658 assert_eq!(
2659 ctx.target_dir(&aux_target("", "docengine")),
2660 Some(PathBuf::from("/w/Libraries/docengine")),
2661 );
2662 // A subdirectory of an aux repo resolves under its checkout.
2663 assert_eq!(
2664 ctx.target_dir(&aux_target("crates/inner", "docengine")),
2665 Some(PathBuf::from("/w/Libraries/docengine/crates/inner")),
2666 );
2667 }
2668
2669 #[tokio::test]
2670 async fn an_aux_target_with_no_checkout_this_run_resolves_to_nothing() {
2671 // Promote-time gates carry no aux dirs. Resolving to a wrong path (say,
2672 // the worktree) would run the gate against whatever happened to sit
2673 // there; `None` makes the caller skip, and --check-config is what
2674 // catches a real typo.
2675 let ctx = resolving_ctx("/w/abc123", &[]);
2676 assert_eq!(ctx.target_dir(&aux_target("", "docengine")), None);
2677 }
2678
2679 #[test]
2680 fn labels_name_the_repo_an_aux_target_lives_in() {
2681 assert_eq!(target("server").label(), "server");
2682 assert_eq!(aux_target("", "docengine").label(), "docengine (aux)");
2683 assert_eq!(
2684 aux_target("crates/inner", "docengine").label(),
2685 "docengine/crates/inner (aux)",
2686 );
2687 }
2688
2689 /// A `GateCtx` over `worktree` with the given frontend projects configured.
2690 /// No DB, no artifact — `code_smoke_frontends` touches neither.
2691 async fn frontend_ctx(worktree: &std::path::Path, dirs: &[&str]) -> GateCtx {
2692 let mut cfg = crate::config::AppConfig::for_tests();
2693 cfg.frontend_builds = dirs
2694 .iter()
2695 .map(|d| crate::config::FrontendBuild {
2696 dir: PathBuf::from(d),
2697 script: "build".into(),
2698 })
2699 .collect();
2700 cfg.logs_root = worktree.join("logs");
2701 GateCtx {
2702 public_url: None,
2703 pool: SqlitePoolOptions::new()
2704 .max_connections(1)
2705 .connect("sqlite::memory:")
2706 .await
2707 .unwrap(),
2708 cfg: std::sync::Arc::new(cfg),
2709 tier: TierId::new("host"),
2710 version: "0.1.0".parse().unwrap(),
2711 worktree: Some(worktree.to_path_buf()),
2712 bundle: None,
2713 events: events::channel(),
2714 nodes: Vec::new(),
2715 build_id: None,
2716 aux_dirs: HashMap::new(),
2717 }
2718 }
2719
2720 /// A `GateCtx` for the `migration_dry_run` freshness checks: a migrated
2721 /// in-memory pool (so `backups` exists) and a scratch URL set, so the gate
2722 /// reaches the backup lookup instead of bailing on config. Nothing here
2723 /// touches postgres — every assertion below blocks before `reset_scratch`.
2724 async fn dry_run_ctx(worktree: &std::path::Path, max_age_hours: u32) -> GateCtx {
2725 let mut cfg = crate::config::AppConfig::for_tests();
2726 cfg.scratch_db_url = Some("postgres:///sando_scratch".into());
2727 cfg.backup_max_age_hours = max_age_hours;
2728 cfg.logs_root = worktree.join("logs");
2729 let pool = SqlitePoolOptions::new()
2730 .max_connections(1)
2731 .connect("sqlite::memory:")
2732 .await
2733 .unwrap();
2734 crate::db::migrate(&pool).await.unwrap();
2735 GateCtx {
2736 public_url: None,
2737 pool,
2738 cfg: std::sync::Arc::new(cfg),
2739 tier: TierId::new("host"),
2740 version: "0.1.0".parse().unwrap(),
2741 worktree: Some(worktree.to_path_buf()),
2742 bundle: None,
2743 events: events::channel(),
2744 nodes: Vec::new(),
2745 build_id: None,
2746 aux_dirs: HashMap::new(),
2747 }
2748 }
2749
2750 /// Record a `server` backup row fetched `hours_ago`, as `/backup/fetch` would.
2751 async fn seed_backup(ctx: &GateCtx, hours_ago: i64) {
2752 seed_named_backup(ctx, "server", hours_ago).await;
2753 }
2754
2755 /// Record a backup row for one named dump.
2756 async fn seed_named_backup(ctx: &GateCtx, name: &str, hours_ago: i64) {
2757 let at = (Utc::now() - chrono::Duration::hours(hours_ago)).to_rfc3339();
2758 sqlx::query(
2759 "INSERT INTO backups (name, fetched_at, source, local_path, byte_size)
2760 VALUES (?, ?, 'file:///x.sql.gz', '/tmp/sando-test-backup.sql.gz', 1000000)",
2761 )
2762 .bind(name)
2763 .bind(at)
2764 .execute(&ctx.pool)
2765 .await
2766 .unwrap();
2767 }
2768
2769 /// The multithreaded check, as `sando-daemon.toml` configures it.
2770 fn mt_check() -> crate::config::MigrationCheck {
2771 crate::config::MigrationCheck {
2772 dir: std::path::PathBuf::from("multithreaded/migrations"),
2773 backup: "multithreaded".into(),
2774 scratch_db: Some("sando_scratch_mt".into()),
2775 owner_role: Some("multithreaded".into()),
2776 }
2777 }
2778
2779 /// Re-point a `dry_run_ctx` at one check, keeping its pool and scratch URL.
2780 fn with_check(ctx: &mut GateCtx, check: crate::config::MigrationCheck) {
2781 let mut cfg = crate::config::AppConfig::for_tests();
2782 cfg.scratch_db_url = ctx.cfg.scratch_db_url.clone();
2783 cfg.backup_max_age_hours = ctx.cfg.backup_max_age_hours;
2784 cfg.logs_root = ctx.cfg.logs_root.clone();
2785 cfg.migration_checks = vec![check];
2786 ctx.cfg = std::sync::Arc::new(cfg);
2787 }
2788
2789 #[tokio::test]
2790 async fn migration_dry_run_blocks_when_a_checks_own_dump_was_never_fetched() {
2791 // The hazard the check list exists for: multithreaded applies its own
2792 // migrations at boot against its own database, so the server's dump must
2793 // never stand in for it. A fetched `server` row with no `multithreaded`
2794 // row is exactly that substitution, and it has to block.
2795 let tmp = tempfile::tempdir().unwrap();
2796 let mut ctx = dry_run_ctx(tmp.path(), 48).await;
2797 with_check(&mut ctx, mt_check());
2798 seed_named_backup(&ctx, "server", 1).await;
2799 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2800
2801 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2802 log.close().await;
2803
2804 let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else {
2805 panic!("a missing multithreaded dump must block");
2806 };
2807 let GateBlocker::NoBackupAvailable { check, backup } = blocker else {
2808 panic!("expected NoBackupAvailable, got {blocker:?}");
2809 };
2810 assert_eq!(backup, "multithreaded", "names the dump that is missing");
2811 assert!(
2812 check.contains("multithreaded/migrations"),
2813 "names the check that wanted it, got {check}"
2814 );
2815 }
2816
2817 #[tokio::test]
2818 async fn migration_dry_run_freshness_is_per_dump() {
2819 // A fresh server dump must not make a 45-day-old multithreaded dump look
2820 // current: the clock is per-database, or the second check inherits the
2821 // first's freshness and the gate is theatre.
2822 let tmp = tempfile::tempdir().unwrap();
2823 let mut ctx = dry_run_ctx(tmp.path(), 48).await;
2824 with_check(&mut ctx, mt_check());
2825 seed_named_backup(&ctx, "server", 1).await;
2826 seed_named_backup(&ctx, "multithreaded", 24 * 45).await;
2827 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2828
2829 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2830 log.close().await;
2831
2832 let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else {
2833 panic!("a 45-day-old multithreaded dump must block");
2834 };
2835 let GateBlocker::BackupStale { check, .. } = blocker else {
2836 panic!("expected BackupStale, got {blocker:?}");
2837 };
2838 assert!(
2839 check.contains("multithreaded/migrations"),
2840 "names the check whose dump is stale, got {check}"
2841 );
2842 }
2843
2844 #[tokio::test]
2845 async fn migration_dry_run_blocks_on_a_stale_backup() {
2846 // The failure this closes: the gate used to check only that a backups row
2847 // existed, so a fetch that silently stopped working left it green against
2848 // an ever-older schema. Sando ran 45 days that way.
2849 let tmp = tempfile::tempdir().unwrap();
2850 let ctx = dry_run_ctx(tmp.path(), 48).await;
2851 seed_backup(&ctx, 24 * 45).await;
2852 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2853
2854 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2855 log.close().await;
2856
2857 let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else {
2858 panic!("a 45-day-old backup must block");
2859 };
2860 let GateBlocker::BackupStale {
2861 age_hours,
2862 max_age_hours,
2863 ..
2864 } = blocker
2865 else {
2866 panic!("expected BackupStale, got {blocker:?}");
2867 };
2868 assert_eq!(max_age_hours, 48);
2869 assert!(
2870 age_hours >= 24 * 45,
2871 "reports the real age, got {age_hours}"
2872 );
2873 }
2874
2875 #[tokio::test]
2876 async fn migration_dry_run_accepts_a_fresh_backup() {
2877 // The other side of the boundary: a backup inside the window must not be
2878 // blocked on freshness. It fails later (there is no such dump on disk),
2879 // which is exactly the proof the age check let it through.
2880 let tmp = tempfile::tempdir().unwrap();
2881 let ctx = dry_run_ctx(tmp.path(), 48).await;
2882 seed_backup(&ctx, 6).await;
2883 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2884
2885 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2886 log.close().await;
2887
2888 assert!(
2889 !matches!(
2890 outcome.status,
2891 crate::outcome::GateStatus::Blocked {
2892 blocker: GateBlocker::BackupStale { .. }
2893 }
2894 ),
2895 "a 6h-old backup is fresh, got {:?}",
2896 outcome.status,
2897 );
2898 }
2899
2900 #[tokio::test]
2901 async fn migration_dry_run_treats_an_unparsable_fetched_at_as_stale() {
2902 // Fail closed: `fetched_at` is daemon-written RFC 3339, so a value that
2903 // will not parse means the row is untrustworthy — and a freshness check
2904 // that shrugs at a timestamp it cannot read is not a freshness check.
2905 let tmp = tempfile::tempdir().unwrap();
2906 let ctx = dry_run_ctx(tmp.path(), 48).await;
2907 sqlx::query(
2908 "INSERT INTO backups (fetched_at, source, local_path, byte_size)
2909 VALUES ('not-a-timestamp', 'file:///x.sql.gz', '/tmp/x.sql.gz', 1000000)",
2910 )
2911 .execute(&ctx.pool)
2912 .await
2913 .unwrap();
2914 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2915
2916 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2917 log.close().await;
2918
2919 assert!(
2920 matches!(
2921 outcome.status,
2922 crate::outcome::GateStatus::Blocked {
2923 blocker: GateBlocker::BackupStale { .. }
2924 }
2925 ),
2926 "an unreadable fetched_at must block, got {:?}",
2927 outcome.status,
2928 );
2929 }
2930
2931 /// A `code_smoke` live log over `ctx.cfg.logs_root`, for the helpers that
2932 /// take one. `GateRunId(0)` never matches a real row; nothing reads the
2933 /// chunk events in these tests.
2934 async fn test_gate_log(ctx: &GateCtx) -> GateLog {
2935 GateLog::open(ctx, GateRunId(0), GateKind::CodeSmoke).await
2936 }
2937
2938 /// Close `log` (flushing it) and read back what it wrote on disk.
2939 async fn read_gate_log(ctx: &GateCtx, log: GateLog) -> String {
2940 log.close().await;
2941 tokio::fs::read_to_string(gate_log_path(ctx, GateKind::CodeSmoke))
2942 .await
2943 .expect("the gate log must exist on disk")
2944 }
2945
2946 /// Write a minimal npm project at `worktree/<dir>` whose `build` script
2947 /// exits with `exit_code`. Pre-creates `node_modules` so the gate skips
2948 /// `npm ci` — these tests are about the build step, not the network.
2949 fn fake_npm_project(worktree: &std::path::Path, dir: &str, exit_code: u8) {
2950 let root = worktree.join(dir);
2951 std::fs::create_dir_all(root.join("node_modules")).unwrap();
2952 std::fs::write(
2953 root.join("package.json"),
2954 format!(
2955 r#"{{"name":"fake","version":"0.0.0","private":true,
2956 "scripts":{{"build":"exit {exit_code}"}}}}"#
2957 ),
2958 )
2959 .unwrap();
2960 }
2961
2962 #[tokio::test]
2963 async fn frontend_gate_fails_on_a_build_error_and_names_the_project() {
2964 // The whole point of the gate: the app build scripts downgrade this to a
2965 // cargo::warning, so if it passes here nothing stops a stale bundle.
2966 let tmp = tempfile::tempdir().unwrap();
2967 fake_npm_project(tmp.path(), "server/frontend", 0);
2968 fake_npm_project(tmp.path(), "multithreaded/frontend", 2);
2969 let ctx = frontend_ctx(tmp.path(), &["server/frontend", "multithreaded/frontend"]).await;
2970
2971 let log = test_gate_log(&ctx).await;
2972 let outcome = code_smoke_frontends(&ctx, &log)
2973 .await
2974 .expect("a failing tsc must fail the gate");
2975 let crate::outcome::GateStatus::Failed { failure } = &outcome.status else {
2976 panic!("expected a failure, got {:?}", outcome.status)
2977 };
2978 assert!(
2979 matches!(
2980 failure,
2981 GateFailure::CodeSmokeFrontend { dir, exit_code: Some(2) }
2982 if dir == "multithreaded/frontend"
2983 ),
2984 "got: {failure:?}"
2985 );
2986 // The passing project ran first; its output belongs in the log too.
2987 let text = read_gate_log(&ctx, log).await;
2988 assert!(text.contains("server/frontend"), "log: {text}");
2989 }
2990
2991 #[tokio::test]
2992 async fn frontend_gate_passes_when_every_project_builds() {
2993 let tmp = tempfile::tempdir().unwrap();
2994 fake_npm_project(tmp.path(), "server/frontend", 0);
2995 let ctx = frontend_ctx(tmp.path(), &["server/frontend"]).await;
2996 let log = test_gate_log(&ctx).await;
2997 assert!(
2998 code_smoke_frontends(&ctx, &log).await.is_none(),
2999 "a clean build must not fail the gate"
3000 );
3001 }
3002
3003 /// The point of routing `code_smoke` through `LiveLog`: its output reaches
3004 /// the operator *while* the gate runs, as `GateLogChunk` events, instead of
3005 /// appearing all at once when the gate finishes.
3006 #[tokio::test]
3007 async fn code_smoke_streams_chunks_as_it_runs() {
3008 let tmp = tempfile::tempdir().unwrap();
3009 fake_npm_project(tmp.path(), "server/frontend", 0);
3010 let ctx = frontend_ctx(tmp.path(), &["server/frontend"]).await;
3011 let mut rx = ctx.events.subscribe_logs();
3012
3013 let log = GateLog::open(&ctx, GateRunId(7), GateKind::CodeSmoke).await;
3014 assert!(code_smoke_frontends(&ctx, &log).await.is_none());
3015 log.close().await;
3016
3017 let mut chunks = Vec::new();
3018 while let Ok(envelope) = rx.try_recv() {
3019 if let Event::GateLogChunk { run_id, seq, text } = envelope.event {
3020 assert_eq!(run_id, GateRunId(7));
3021 chunks.push((seq, text));
3022 }
3023 }
3024 assert!(!chunks.is_empty(), "no chunk ever reached the bus");
3025 // Sequence numbers are per-run and monotonic across every step of the
3026 // gate, which is why the whole gate shares one sink.
3027 let seqs: Vec<u32> = chunks.iter().map(|(seq, _)| *seq).collect();
3028 assert!(
3029 seqs.windows(2).all(|w| w[0] < w[1]),
3030 "chunk seq must be monotonic, got {seqs:?}"
3031 );
3032 let joined: String = chunks.into_iter().map(|(_, text)| text).collect();
3033 assert!(joined.contains("server/frontend"), "chunks: {joined}");
3034 }
3035
3036 #[tokio::test]
3037 async fn frontend_gate_skips_a_project_absent_from_the_worktree() {
3038 // Rebuilding an older sha that predates the frontend must stay possible.
3039 let tmp = tempfile::tempdir().unwrap();
3040 let ctx = frontend_ctx(tmp.path(), &["multithreaded/frontend"]).await;
3041 let log = test_gate_log(&ctx).await;
3042 assert!(code_smoke_frontends(&ctx, &log).await.is_none());
3043 assert!(
3044 read_gate_log(&ctx, log).await.contains("skipping"),
3045 "the skip must be visible in the log"
3046 );
3047 }
3048
3049 #[tokio::test]
3050 async fn cargo_test_fails_closed_when_no_target_exists_in_the_worktree() {
3051 // A worktree missing every configured crate must not report "tests
3052 // passed" having run none. Uses an empty tempdir as the worktree, so
3053 // no cargo process is ever spawned.
3054 let tmp = tempfile::tempdir().unwrap();
3055 let mut cfg = crate::config::AppConfig::for_tests();
3056 cfg.test_targets = vec![target("server"), target("mnw-cli")];
3057 cfg.logs_root = tmp.path().join("logs");
3058 let pool = SqlitePoolOptions::new()
3059 .max_connections(1)
3060 .connect("sqlite::memory:")
3061 .await
3062 .unwrap();
3063 let ctx = GateCtx {
3064 public_url: None,
3065 pool,
3066 cfg: std::sync::Arc::new(cfg),
3067 tier: TierId::new("host"),
3068 version: "0.1.0".parse().unwrap(),
3069 worktree: Some(tmp.path().to_path_buf()),
3070 bundle: None,
3071 events: events::channel(),
3072 nodes: Vec::new(),
3073 build_id: None,
3074 aux_dirs: HashMap::new(),
3075 };
3076 let out = cargo_test(&ctx, GateRunId(1)).await.unwrap();
3077 assert_eq!(
3078 out.status_str(),
3079 "failed",
3080 "green here would be a silent no-op gate"
3081 );
3082 let crate::outcome::GateStatus::Failed { failure } = &out.status else {
3083 panic!("expected a failure")
3084 };
3085 assert!(
3086 failure.summary().contains("ran no targets"),
3087 "got: {}",
3088 failure.summary()
3089 );
3090 }
3091
3092 #[tokio::test]
3093 async fn scratch_db_env_is_opt_in_per_target() {
3094 // Exporting DATABASE_URL knocks sqlx out of offline mode, so a crate
3095 // shipping .sqlx data must not see it.
3096 let mut cfg = crate::config::AppConfig::for_tests();
3097 cfg.scratch_db_url = Some("postgres://sando@127.0.0.1/sando_scratch".into());
3098 let ctx = GateCtx {
3099 public_url: None,
3100 pool: SqlitePoolOptions::new()
3101 .max_connections(1)
3102 .connect_lazy("sqlite::memory:")
3103 .unwrap(),
3104 cfg: std::sync::Arc::new(cfg),
3105 tier: TierId::new("host"),
3106 version: "0.1.0".parse().unwrap(),
3107 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
3108 bundle: None,
3109 events: events::channel(),
3110 nodes: Vec::new(),
3111 build_id: None,
3112 aux_dirs: HashMap::new(),
3113 };
3114 let dir = std::path::Path::new("/tmp/wt/x");
3115
3116 let off = cargo_test_command(&ctx, dir, &target("x"), &[], &[]);
3117 let has_db = |c: &Command| {
3118 c.as_std()
3119 .get_envs()
3120 .any(|(k, v)| k == "DATABASE_URL" && v.is_some())
3121 };
3122 assert!(!has_db(&off), "scratch_db defaults off");
3123
3124 let mut on_target = target("x");
3125 on_target.scratch_db = true;
3126 assert!(
3127 has_db(&cargo_test_command(&ctx, dir, &on_target, &[], &[])),
3128 "opt-in exports it"
3129 );
3130 }
3131
3132 #[tokio::test]
3133 async fn all_features_replaces_the_feature_list() {
3134 let ctx = GateCtx {
3135 public_url: None,
3136 pool: SqlitePoolOptions::new()
3137 .max_connections(1)
3138 .connect_lazy("sqlite::memory:")
3139 .unwrap(),
3140 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
3141 tier: TierId::new("host"),
3142 version: "0.1.0".parse().unwrap(),
3143 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
3144 bundle: None,
3145 events: events::channel(),
3146 nodes: Vec::new(),
3147 build_id: None,
3148 aux_dirs: HashMap::new(),
3149 };
3150 let mut t = target("shared/ops-exec");
3151 t.all_features = true;
3152 let cmd = cargo_test_command(&ctx, std::path::Path::new("/tmp/wt"), &t, &[], &[]);
3153 let args: Vec<_> = cmd
3154 .as_std()
3155 .get_args()
3156 .map(|a| a.to_string_lossy().into_owned())
3157 .collect();
3158 assert!(args.iter().any(|a| a == "--all-features"), "got: {args:?}");
3159 assert!(!args.iter().any(|a| a == "--features"), "got: {args:?}");
3160 }
3161
3162 #[test]
3163 fn every_gate_kind_round_trips_through_its_wire_string() {
3164 // gate_kind is a TEXT column and a WS event field; as_str and FromStr
3165 // disagreeing would make a gate's evidence unreadable by
3166 // unsatisfied_gates, which fails the promote closed with no explanation.
3167 for k in [
3168 GateKind::CargoTest,
3169 GateKind::HardeningTest,
3170 GateKind::Clippy,
3171 GateKind::Fmt,
3172 GateKind::CargoAudit,
3173 GateKind::CargoDeny,
3174 GateKind::MigrationDryRun,
3175 GateKind::CodeSmoke,
3176 GateKind::BootSmoke,
3177 GateKind::NodeHealth,
3178 GateKind::BurnIn,
3179 GateKind::ManualConfirm,
3180 ] {
3181 assert_eq!(
3182 k.as_str().parse::<GateKind>().unwrap(),
3183 k,
3184 "round trip for {k:?}"
3185 );
3186 }
3187 }
3188
3189 #[test]
3190 fn first_meaningful_line_prefers_the_diagnostic() {
3191 let stderr = b" Updating crates.io index\nerror: 1 vulnerability found!\n";
3192 assert_eq!(
3193 first_meaningful_line(b"", stderr),
3194 "error: 1 vulnerability found!"
3195 );
3196 }
3197
3198 #[test]
3199 fn first_meaningful_line_finds_a_deny_verdict() {
3200 let out = b"advisories FAILED, bans ok, licenses FAILED, sources ok\n";
3201 assert!(first_meaningful_line(out, b"").contains("FAILED"));
3202 }
3203
3204 #[test]
3205 fn first_meaningful_line_falls_back_rather_than_returning_empty() {
3206 assert!(first_meaningful_line(b"", b"").contains("see the gate log"));
3207 }
3208
3209 #[tokio::test]
3210 async fn supply_chain_gates_skip_crates_without_their_config() {
3211 // Four crates in this repo fail `cargo audit` purely for want of a
3212 // triaged .cargo/audit.toml. Running it there would make the gate
3213 // permanently red, so a target only qualifies once it carries the file.
3214 // The worktree here has a Cargo.toml but no audit config, so nothing
3215 // qualifies and the gate fails closed rather than passing over zero work.
3216 let tmp = tempfile::tempdir().unwrap();
3217 let crate_dir = tmp.path().join("server");
3218 std::fs::create_dir_all(&crate_dir).unwrap();
3219 std::fs::write(crate_dir.join("Cargo.toml"), "[package]\nname = \"x\"\n").unwrap();
3220
3221 let mut cfg = crate::config::AppConfig::for_tests();
3222 cfg.test_targets = vec![target("server")];
3223 cfg.logs_root = tmp.path().join("logs");
3224 let ctx = GateCtx {
3225 public_url: None,
3226 pool: SqlitePoolOptions::new()
3227 .max_connections(1)
3228 .connect("sqlite::memory:")
3229 .await
3230 .unwrap(),
3231 cfg: std::sync::Arc::new(cfg),
3232 tier: TierId::new("host"),
3233 version: "0.1.0".parse().unwrap(),
3234 worktree: Some(tmp.path().to_path_buf()),
3235 bundle: None,
3236 events: events::channel(),
3237 nodes: Vec::new(),
3238 build_id: None,
3239 aux_dirs: HashMap::new(),
3240 };
3241 let out = supply_chain(&ctx, GateRunId(1), GateKind::CargoAudit)
3242 .await
3243 .unwrap();
3244 assert_eq!(out.status_str(), "failed");
3245 let crate::outcome::GateStatus::Failed { failure } = &out.status else {
3246 panic!("expected a failure")
3247 };
3248 assert!(
3249 failure.summary().contains("ran nothing"),
3250 "got: {}",
3251 failure.summary()
3252 );
3253
3254 // Drop the config in and the same target now qualifies.
3255 std::fs::create_dir_all(crate_dir.join(".cargo")).unwrap();
3256 std::fs::write(crate_dir.join(".cargo/audit.toml"), "[advisories]\n").unwrap();
3257 // The fixture crate is not a real cargo project, so the tool itself
3258 // still errors — but on its own terms, not with "ran nothing". That
3259 // distinction is the thing under test: the target was attempted.
3260 let out = supply_chain(&ctx, GateRunId(2), GateKind::CargoAudit)
3261 .await
3262 .unwrap();
3263 if let crate::outcome::GateStatus::Failed { failure } = &out.status {
3264 assert!(
3265 !failure.summary().contains("ran nothing"),
3266 "a target carrying the config must be attempted, not skipped; got: {}",
3267 failure.summary(),
3268 );
3269 }
3270 }
3271
3272 #[test]
3273 fn tests_run_reads_the_libtest_summary() {
3274 let out = b"running 6 tests\ntest auth_rate_limit_triggers_on_burst ... ok\n\n\
3275 test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 118 filtered out\n";
3276 assert_eq!(tests_run(out), 6);
3277 }
3278
3279 #[test]
3280 fn tests_run_sums_across_test_binaries() {
3281 let out = b"test result: ok. 6 passed; 0 failed\ntest result: ok. 2 passed; 0 failed\n";
3282 assert_eq!(tests_run(out), 8);
3283 }
3284
3285 #[test]
3286 fn tests_run_is_zero_when_the_filter_matched_nothing() {
3287 // The case hardening_test fails closed on: a filter matching no tests
3288 // exits 0, so a rename would otherwise make the gate a green no-op.
3289 let out = b"running 0 tests\n\n\
3290 test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 124 filtered out\n";
3291 assert_eq!(tests_run(out), 0);
3292 }
3293
3294 #[test]
3295 fn tests_run_is_zero_without_a_summary_line() {
3296 assert_eq!(tests_run(b"error: could not compile `makenotwork`\n"), 0);
3297 }
3298
3299 #[tokio::test]
3300 async fn hardening_test_command_carries_no_features() {
3301 // The entire point of the gate: production constants, which means no
3302 // `fast-tests`. A stray feature here silently restores the blind spot.
3303 let ctx = GateCtx {
3304 public_url: None,
3305 pool: SqlitePoolOptions::new()
3306 .max_connections(1)
3307 .connect_lazy("sqlite::memory:")
3308 .unwrap(),
3309 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
3310 tier: TierId::new("host"),
3311 version: "0.1.0".parse().unwrap(),
3312 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
3313 bundle: None,
3314 events: events::channel(),
3315 nodes: Vec::new(),
3316 build_id: None,
3317 aux_dirs: HashMap::new(),
3318 };
3319 let plain = crate::config::TestTarget {
3320 dir: std::path::PathBuf::from("server"),
3321 aux_repo: None,
3322 features: Vec::new(),
3323 all_features: false,
3324 scratch_db: true,
3325 };
3326 let cmd = cargo_test_command(
3327 &ctx,
3328 std::path::Path::new("/tmp/wt/server"),
3329 &plain,
3330 &[],
3331 &["--test", "integration"],
3332 );
3333 let args: Vec<_> = cmd
3334 .as_std()
3335 .get_args()
3336 .map(|a| a.to_string_lossy().into_owned())
3337 .collect();
3338 assert!(
3339 !args.iter().any(|a| a == "--features"),
3340 "hardening_test must pass no features: {args:?}"
3341 );
3342 assert!(
3343 !args.iter().any(|a| a.contains("fast-tests")),
3344 "got: {args:?}"
3345 );
3346
3347 let fast_target = crate::config::TestTarget {
3348 dir: std::path::PathBuf::from("server"),
3349 aux_repo: None,
3350 features: vec!["fast-tests".into()],
3351 all_features: false,
3352 scratch_db: true,
3353 };
3354 let fast = cargo_test_command(
3355 &ctx,
3356 std::path::Path::new("/tmp/wt/server"),
3357 &fast_target,
3358 &["fast-tests"],
3359 &[],
3360 );
3361 let fast_args: Vec<_> = fast
3362 .as_std()
3363 .get_args()
3364 .map(|a| a.to_string_lossy().into_owned())
3365 .collect();
3366 assert!(
3367 fast_args
3368 .windows(2)
3369 .any(|w| w == ["--features", "fast-tests"]),
3370 "got: {fast_args:?}"
3371 );
3372 }
3373
3374 /// Spawn a one-shot loopback server that answers the first connection with
3375 /// `status_line` + a tiny body, then closes. Returns the bound port.
3376 async fn oneshot_http(status_line: &'static str) -> u16 {
3377 use tokio::io::{AsyncReadExt, AsyncWriteExt};
3378 let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
3379 .await
3380 .unwrap();
3381 let port = listener.local_addr().unwrap().port();
3382 tokio::spawn(async move {
3383 if let Ok((mut sock, _)) = listener.accept().await {
3384 let mut scratch = [0u8; 1024];
3385 let _ = sock.read(&mut scratch).await; // drain the request line
3386 let resp =
3387 format!("{status_line}\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok");
3388 let _ = sock.write_all(resp.as_bytes()).await;
3389 }
3390 });
3391 port
3392 }
3393
3394 #[tokio::test]
3395 async fn probe_health_ok_on_200() {
3396 let port = oneshot_http("HTTP/1.1 200 OK").await;
3397 assert!(probe_health(port).await.is_ok());
3398 }
3399
3400 #[tokio::test]
3401 async fn probe_health_err_on_non_200() {
3402 let port = oneshot_http("HTTP/1.1 503 Service Unavailable").await;
3403 let err = probe_health(port).await.unwrap_err();
3404 assert!(err.contains("status line"), "{err}");
3405 }
3406
3407 #[tokio::test]
3408 async fn probe_health_err_on_connection_refused() {
3409 // Bind then drop to get an almost-certainly-free port nothing listens on.
3410 let port = {
3411 let l = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
3412 .await
3413 .unwrap();
3414 l.local_addr().unwrap().port()
3415 };
3416 let err = probe_health(port).await.unwrap_err();
3417 assert!(err.contains("connect"), "{err}");
3418 }
3419
3420 /// burn_in returns a typed Blocked when the clock isn't started; the
3421 /// runner persists status='blocked' + outcome_json (the json carries
3422 /// blocker.kind = 'burn_in_clock_not_started').
3423 #[tokio::test]
3424 async fn burn_in_blocked_persists_typed_outcome() {
3425 let pool = SqlitePoolOptions::new()
3426 .max_connections(1)
3427 .connect("sqlite::memory:")
3428 .await
3429 .unwrap();
3430 crate::db::migrate(&pool).await.unwrap();
3431 // Topology sync expects a tier row before gate_runs can reference it.
3432 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 0, 'sequential')")
3433 .execute(&pool).await.unwrap();
3434 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
3435 .execute(&pool)
3436 .await
3437 .unwrap();
3438 // versions FK target.
3439 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
3440 .execute(&pool).await.unwrap();
3441
3442 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests());
3443 let ctx = GateCtx {
3444 public_url: None,
3445 pool: pool.clone(),
3446 cfg,
3447 tier: TierId::new("host"),
3448 version: "0.1.0".parse().unwrap(),
3449 worktree: Some(std::path::PathBuf::from("/tmp/unused")),
3450 bundle: None,
3451 events: events::channel(),
3452 nodes: Vec::new(),
3453 build_id: None,
3454 aux_dirs: HashMap::new(),
3455 };
3456 let out = run(&ctx, &Gate::BurnIn { hours: 24 }).await.unwrap();
3457 assert_eq!(out.status_str(), "blocked");
3458 assert!(!out.is_passed());
3459
3460 // Read the persisted row.
3461 let row: (Option<String>, Option<String>) =
3462 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
3463 .fetch_one(&pool)
3464 .await
3465 .unwrap();
3466 assert_eq!(row.0.as_deref(), Some("blocked"), "typed status");
3467 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
3468 assert_eq!(json["status"]["kind"], "blocked");
3469 assert_eq!(
3470 json["status"]["blocker"]["kind"],
3471 "burn_in_clock_not_started"
3472 );
3473 }
3474
3475 /// node_health fails closed when there are no nodes to probe: a serving tier
3476 /// should always carry nodes, so an empty set is a misconfiguration that must
3477 /// block promotion, not pass it.
3478 #[tokio::test]
3479 async fn node_health_blocks_with_no_nodes() {
3480 let pool = SqlitePoolOptions::new()
3481 .max_connections(1)
3482 .connect("sqlite::memory:")
3483 .await
3484 .unwrap();
3485 crate::db::migrate(&pool).await.unwrap();
3486 sqlx::query(
3487 "INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('b', 2, 1, 'sequential')",
3488 )
3489 .execute(&pool)
3490 .await
3491 .unwrap();
3492 sqlx::query("INSERT INTO tier_state (tier) VALUES ('b')")
3493 .execute(&pool)
3494 .await
3495 .unwrap();
3496 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
3497 .execute(&pool).await.unwrap();
3498
3499 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests());
3500 let ctx = GateCtx {
3501 public_url: None,
3502 pool: pool.clone(),
3503 cfg,
3504 tier: TierId::new("b"),
3505 version: "0.1.0".parse().unwrap(),
3506 worktree: None,
3507 bundle: None,
3508 events: events::channel(),
3509 nodes: Vec::new(), // no nodes -> fail closed
3510 build_id: None,
3511 aux_dirs: HashMap::new(),
3512 };
3513 let out = run(&ctx, &Gate::NodeHealth).await.unwrap();
3514 assert_eq!(out.status_str(), "blocked");
3515 assert!(!out.is_passed());
3516 let row: (Option<String>, Option<String>) =
3517 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
3518 .fetch_one(&pool)
3519 .await
3520 .unwrap();
3521 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
3522 assert_eq!(json["status"]["blocker"]["kind"], "no_nodes_to_probe");
3523 }
3524
3525 /// reset_scratch must drop every non-system schema, not just `public` —
3526 /// otherwise migrations that create custom schemas (e.g. tower_sessions)
3527 /// collide on the next run. This regressed once (Phase 0) and the fix is
3528 /// load-bearing for migration_dry_run.
3529 ///
3530 /// Gated on `SANDO_TEST_PG_URL` so it only runs where postgres is
3531 /// available. Set `SANDO_TEST_PG_URL=postgres:///sando_scratch?host=/var/run/postgresql`
3532 /// (or similar) before `cargo test`.
3533 #[tokio::test]
3534 async fn reset_scratch_drops_all_non_system_schemas() {
3535 let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else {
3536 eprintln!("skipping: SANDO_TEST_PG_URL not set");
3537 return;
3538 };
3539 use sqlx::Executor;
3540 use sqlx::postgres::PgPoolOptions;
3541
3542 let pool = PgPoolOptions::new()
3543 .max_connections(1)
3544 .connect(&url)
3545 .await
3546 .unwrap();
3547 // Plant two non-system schemas + a table in each.
3548 pool.execute(
3549 "DROP SCHEMA IF EXISTS foo CASCADE; CREATE SCHEMA foo; CREATE TABLE foo.t (i int);",
3550 )
3551 .await
3552 .unwrap();
3553 pool.execute("DROP SCHEMA IF EXISTS tower_sessions CASCADE; CREATE SCHEMA tower_sessions; CREATE TABLE tower_sessions.session (id text);")
3554 .await.unwrap();
3555 pool.close().await;
3556
3557 reset_scratch(&url, "makenotwork")
3558 .await
3559 .expect("reset_scratch");
3560
3561 let pool = PgPoolOptions::new()
3562 .max_connections(1)
3563 .connect(&url)
3564 .await
3565 .unwrap();
3566 let rows: Vec<(String,)> = sqlx::query_as(
3567 "SELECT nspname FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname <> 'information_schema'",
3568 )
3569 .fetch_all(&pool)
3570 .await
3571 .unwrap();
3572 let names: Vec<String> = rows.into_iter().map(|(s,)| s).collect();
3573 // After reset, only `public` should remain among non-system schemas.
3574 assert_eq!(names, vec!["public".to_string()], "got: {names:?}");
3575 pool.close().await;
3576 }
3577
3578 /// reset_scratch must leave the dump's owner role existing and able to
3579 /// create in `public`, because a prod `pg_dump` carries `ALTER ... OWNER TO
3580 /// <role>` for every object. This was satisfied by a hand-created NOLOGIN
3581 /// role on fw13; nothing recorded it, so any other box failed
3582 /// migration_dry_run at the restore with "role does not exist".
3583 ///
3584 /// Uses a throwaway role name so it can prove the *creation* path rather
3585 /// than passing on fw13's pre-existing `makenotwork`. Same
3586 /// `SANDO_TEST_PG_URL` gate as above; needs a superuser connection.
3587 #[tokio::test]
3588 async fn reset_scratch_seeds_the_dump_owner_role_when_absent() {
3589 let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else {
3590 eprintln!("skipping: SANDO_TEST_PG_URL not set");
3591 return;
3592 };
3593 use sqlx::Executor;
3594 use sqlx::postgres::PgPoolOptions;
3595
3596 let role = "sando_test_owner_probe";
3597 // `DROP ROLE` refuses while the role still holds the grants reset_scratch
3598 // gave it, so drop what it owns first. Idempotent, and a no-op when the
3599 // role is absent (the usual case on a first run).
3600 let drop_role = format!(
3601 "DO $$ BEGIN
3602 IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{role}') THEN
3603 EXECUTE 'DROP OWNED BY {role}';
3604 EXECUTE 'DROP ROLE {role}';
3605 END IF;
3606 END $$;"
3607 );
3608
3609 let pool = PgPoolOptions::new()
3610 .max_connections(1)
3611 .connect(&url)
3612 .await
3613 .unwrap();
3614 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(drop_role.clone())))
3615 .await
3616 .unwrap();
3617 pool.close().await;
3618
3619 reset_scratch(&url, role)
3620 .await
3621 .expect("reset_scratch creates the owner role");
3622
3623 let pool = PgPoolOptions::new()
3624 .max_connections(1)
3625 .connect(&url)
3626 .await
3627 .unwrap();
3628 let (exists, can_login): (bool, bool) =
3629 sqlx::query_as("SELECT true, rolcanlogin FROM pg_roles WHERE rolname = $1")
3630 .bind(role)
3631 .fetch_one(&pool)
3632 .await
3633 .expect("owner role exists after reset");
3634 assert!(exists);
3635 assert!(
3636 !can_login,
3637 "the owner role is an owner only, never a login identity"
3638 );
3639
3640 // The restore's owner-scoped DDL needs CREATE on public in the role's
3641 // own right (PG15+ dropped the implicit grant).
3642 let (has_create,): (bool,) =
3643 sqlx::query_as("SELECT pg_catalog.has_schema_privilege($1, 'public', 'CREATE')")
3644 .bind(role)
3645 .fetch_one(&pool)
3646 .await
3647 .unwrap();
3648 assert!(has_create, "owner role must be able to create in public");
3649
3650 // Idempotent: a second reset must not error on the now-existing role.
3651 pool.close().await;
3652 reset_scratch(&url, role)
3653 .await
3654 .expect("reset_scratch is idempotent");
3655
3656 let pool = PgPoolOptions::new()
3657 .max_connections(1)
3658 .connect(&url)
3659 .await
3660 .unwrap();
3661 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(drop_role)))
3662 .await
3663 .unwrap();
3664 pool.close().await;
3665 }
3666
3667 /// The preflight must pass against a privileged scratch connection. Guards
3668 /// the catalog query itself: a wrong column or a `current_user` that matches
3669 /// no `pg_roles` row would make `fetch_one` error (or, worse, a silently
3670 /// swapped pair would invert the check) and brick startup for everyone.
3671 /// `SANDO_TEST_PG_URL` is expected to be a superuser connection, as the
3672 /// gates require.
3673 #[tokio::test]
3674 async fn preflight_passes_on_a_privileged_scratch_connection() {
3675 let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else {
3676 eprintln!("skipping: SANDO_TEST_PG_URL not set");
3677 return;
3678 };
3679 preflight_scratch_privileges(&url)
3680 .await
3681 .expect("a superuser scratch connection must satisfy the preflight");
3682 }
3683
3684 /// CF4: the restore pipeline must carry `ON_ERROR_STOP=1` (so psql fails on
3685 /// a bad statement instead of exiting 0 on a partial restore) and, for a
3686 /// gzip source, `set -o pipefail` (so a `gunzip` failure on a truncated
3687 /// archive isn't masked by psql's exit). Pure string check — no postgres.
3688 #[test]
3689 fn restore_shell_has_error_stop_and_pipefail() {
3690 let gz = restore_shell("postgres:///scratch", "/srv/sando/backups/latest.sql.gz");
3691 assert!(gz.contains("ON_ERROR_STOP=1"), "gz: {gz}");
3692 assert!(gz.contains("set -o pipefail"), "gz: {gz}");
3693 assert!(gz.contains("gunzip -c"), "gz: {gz}");
3694
3695 let plain = restore_shell("postgres:///scratch", "/srv/sando/backups/dump.sql");
3696 assert!(plain.contains("ON_ERROR_STOP=1"), "plain: {plain}");
3697 // No pipeline for a plain .sql, so pipefail is unnecessary there.
3698 assert!(!plain.contains("gunzip"), "plain: {plain}");
3699 // The db url is single-quote escaped in both forms.
3700 assert!(plain.contains("'postgres:///scratch'"), "plain: {plain}");
3701 }
3702
3703 #[test]
3704 fn split_pg_password_extracts_and_sanitizes() {
3705 // Password lifted out of the URL; the sanitized form keeps user/host/db.
3706 let (url, pw) = split_pg_password("postgres://sando:s3cret@db.host:5432/scratch");
3707 assert_eq!(url, "postgres://sando@db.host:5432/scratch");
3708 assert_eq!(pw.as_deref(), Some("s3cret"));
3709 // Percent-encoded password is decoded for PGPASSWORD.
3710 let (url, pw) = split_pg_password("postgresql://u:p%40ss%2Fword@h/d");
3711 assert_eq!(url, "postgresql://u@h/d");
3712 assert_eq!(pw.as_deref(), Some("p@ss/word"));
3713 }
3714
3715 #[test]
3716 fn split_pg_password_noop_without_password() {
3717 // No userinfo password -> unchanged, None. (A ':' after the '@', e.g. a
3718 // port, must not be mistaken for the password delimiter.)
3719 assert_eq!(
3720 split_pg_password("postgres:///scratch"),
3721 ("postgres:///scratch".to_string(), None),
3722 );
3723 assert_eq!(
3724 split_pg_password("postgres://sando@db.host:5432/scratch"),
3725 ("postgres://sando@db.host:5432/scratch".to_string(), None),
3726 );
3727 }
3728
3729 #[test]
3730 fn percent_decode_handles_escapes_and_malformed() {
3731 assert_eq!(percent_decode("plain"), "plain");
3732 assert_eq!(percent_decode("a%2Fb"), "a/b");
3733 // A malformed trailing escape is left literal, not dropped.
3734 assert_eq!(percent_decode("ab%2"), "ab%2");
3735 assert_eq!(percent_decode("ab%zz"), "ab%zz");
3736 }
3737
3738 #[test]
3739 fn pg_url_with_dbname_rewrites_the_database() {
3740 // user:pass@host:port/db?query — swap db, keep everything else.
3741 assert_eq!(
3742 pg_url_with_dbname(
3743 "postgres://sando:pw@db.host:5432/sando_scratch?sslmode=require",
3744 "postgres"
3745 ),
3746 "postgres://sando:pw@db.host:5432/postgres?sslmode=require",
3747 );
3748 // Socket form: the query carries `host=/var/run/postgresql` and must survive.
3749 assert_eq!(
3750 pg_url_with_dbname(
3751 "postgres:///sando_scratch?host=/var/run/postgresql",
3752 "sando_code_smoke_0_9_6"
3753 ),
3754 "postgres:///sando_code_smoke_0_9_6?host=/var/run/postgresql",
3755 );
3756 // Plain host/db, no query.
3757 assert_eq!(
3758 pg_url_with_dbname("postgres://localhost/scratch", "postgres"),
3759 "postgres://localhost/scratch".replace("scratch", "postgres"),
3760 );
3761 // No authority, no query (loopback socket, default db path).
3762 assert_eq!(
3763 pg_url_with_dbname("postgres:///scratch", "postgres"),
3764 "postgres:///postgres",
3765 );
3766 }
3767
3768 #[test]
3769 fn bytes_contain_matches_listening_in_log_output() {
3770 // JSON release log carries the message field verbatim.
3771 assert!(bytes_contain(
3772 br#"{"timestamp":"...","level":"INFO","fields":{"message":"listening","addr":"127.0.0.1:18182"}}"#,
3773 b"listening",
3774 ));
3775 // Human-format dev log.
3776 assert!(bytes_contain(
3777 b"2026-07-17 INFO makenotwork: listening addr=127.0.0.1:18182",
3778 b"listening"
3779 ));
3780 assert!(!bytes_contain(
3781 b"migrations complete; seeding catalog",
3782 b"listening"
3783 ));
3784 assert!(!bytes_contain(b"", b"listening"));
3785 }
3786
3787 #[tokio::test]
3788 async fn code_smoke_env_supplies_every_mandatory_server_var() {
3789 // code_smoke is the only gate that reaches the server's Config::from_env
3790 // (boot_smoke and the docs check short-circuit before it), so anything
3791 // that function requires has to be answered here. CDN_BASE_URL became
3792 // mandatory 13 days after this env was written and went unnoticed until
3793 // the gate first ran on the host; this test is what makes the next one
3794 // fail here instead of in a promote.
3795 let ctx = resolving_ctx("/w/abc", &[]);
3796 let mut cmd = tokio::process::Command::new("true");
3797 code_smoke_env(&mut cmd, &ctx, "postgres:///throwaway");
3798 let set: std::collections::HashMap<String, String> = cmd
3799 .as_std()
3800 .get_envs()
3801 .filter_map(|(k, v)| Some((k.to_str()?.to_string(), v?.to_str()?.to_string())))
3802 .collect();
3803 for key in [
3804 "DATABASE_URL",
3805 "HOST",
3806 "PORT",
3807 "HOST_URL",
3808 "CDN_BASE_URL",
3809 "SIGNING_SECRET",
3810 ] {
3811 assert!(set.contains_key(key), "code_smoke_env must set {key}");
3812 assert!(!set[key].is_empty(), "{key} must not be empty");
3813 }
3814 // Loopback, so Config::from_env's is_production branch stays false and
3815 // the gate never trips MissingPublicBucket for want of an S3 bucket.
3816 assert!(set["HOST_URL"].starts_with("http://localhost"));
3817 // Not an IP literal: the server derives its WebAuthn rp_id from this
3818 // host, and WebauthnBuilder rejects an origin whose Url::domain() is
3819 // None, which is every IP address. An IP here cannot boot the server.
3820 assert!(
3821 url_host_is_a_domain(&set["HOST_URL"]),
3822 "HOST_URL host must be a domain, not an IP literal: {}",
3823 set["HOST_URL"],
3824 );
3825 assert_eq!(set["HOST"], "127.0.0.1");
3826 // The signing secret has to clear the server's 32-char floor, or the
3827 // gate fails with WeakSigningSecret instead of testing anything.
3828 assert!(set["SIGNING_SECRET"].len() >= 32);
3829 }
3830
3831 #[test]
3832 fn code_smoke_db_name_sanitizes_and_caps() {
3833 assert_eq!(
3834 code_smoke_db_name(&"0.9.6".parse().unwrap()),
3835 "sando_code_smoke_0_9_6"
3836 );
3837 // Pre-release/build metadata folds to underscores; result stays [a-z0-9_].
3838 let n = code_smoke_db_name(&"1.0.0-rc.1+build".parse().unwrap());
3839 assert_eq!(n, "sando_code_smoke_1_0_0_rc_1_build");
3840 assert!(
3841 n.bytes()
3842 .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
3843 );
3844 assert!(n.len() <= 63);
3845 }
3846
3847 /// code_smoke is Blocked (not Failed) when the daemon has no scratch_db_url:
3848 /// there's no cluster to create the throwaway DB in, and that's an operator
3849 /// precondition, rendered yellow — the same shape as migration_dry_run.
3850 #[tokio::test]
3851 async fn code_smoke_blocks_without_scratch_db_url() {
3852 let pool = SqlitePoolOptions::new()
3853 .max_connections(1)
3854 .connect("sqlite::memory:")
3855 .await
3856 .unwrap();
3857 crate::db::migrate(&pool).await.unwrap();
3858 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 1, 'sequential')")
3859 .execute(&pool).await.unwrap();
3860 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
3861 .execute(&pool)
3862 .await
3863 .unwrap();
3864 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
3865 .execute(&pool).await.unwrap();
3866
3867 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests()); // scratch_db_url: None
3868 let ctx = GateCtx {
3869 public_url: None,
3870 pool: pool.clone(),
3871 cfg,
3872 tier: TierId::new("host"),
3873 version: "0.1.0".parse().unwrap(),
3874 worktree: Some(std::path::PathBuf::from("/tmp/unused")),
3875 bundle: None,
3876 events: events::channel(),
3877 nodes: Vec::new(),
3878 build_id: None,
3879 aux_dirs: HashMap::new(),
3880 };
3881 let out = run(&ctx, &Gate::CodeSmoke).await.unwrap();
3882 assert_eq!(out.status_str(), "blocked");
3883 assert!(!out.is_passed());
3884 let row: (Option<String>, Option<String>) =
3885 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
3886 .fetch_one(&pool)
3887 .await
3888 .unwrap();
3889 assert_eq!(row.0.as_deref(), Some("blocked"));
3890 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
3891 assert_eq!(json["status"]["blocker"]["kind"], "scratch_db_url_unset");
3892 }
3893
3894 /// Sanity: applying MNW migrations from a *non-existent* dir errors,
3895 /// rather than silently no-op'ing. Cheap pure check, no postgres needed
3896 /// (the sqlx::Migrator::new constructor itself reads the dir).
3897 #[tokio::test]
3898 async fn run_migrator_errors_on_missing_dir() {
3899 // The first thing run_migrator does is `Migrator::new(dir)`, which
3900 // needs a real dir to read migration files from.
3901 let res = run_migrator(
3902 "postgres:///does-not-matter",
3903 std::path::Path::new("/nonexistent/sando-test-migrations"),
3904 )
3905 .await;
3906 assert!(res.is_err());
3907 }
3908 }
3909