Skip to main content

max / makenotwork

162.1 KB · 4000 lines History Blame Raw
1 //! Gate execution. Each gate kind has a runner that produces a pass/fail
2 //! outcome plus an optional detail string (typically a stderr tail or a
3 //! human-readable reason). Outcomes are persisted to `gate_runs` so /state
4 //! and the TUI can show them.
5
6 use crate::classify;
7 use crate::config::AppConfig;
8 use crate::domain::{AppId, GateKind, GateRunId, TierId, Version};
9 use crate::events::{self, Event, EventTx};
10 use crate::outcome::{GateBlocker, GateFailure, GateOutcome, LogRef, PassNote};
11 use crate::topology::Gate;
12 use anyhow::{Context, Result};
13 use chrono::Utc;
14 use ops_core::live_log::LiveLog;
15 use ops_core::remote::LogSink; // brings `LiveLog::write_chunk` (the sink trait) into scope
16 use sqlx::SqlitePool;
17 use std::collections::HashMap;
18 use std::path::Path;
19 use std::path::PathBuf;
20 use std::sync::Arc;
21 use tokio::io::AsyncReadExt;
22 use tokio::process::Command;
23
24 /// The gate live-log callback: emit each chunk as a `GateLogChunk` event so the
25 /// TUI sees the tail stream in real time. `ops_core::live_log::LiveLog` owns the
26 /// disk append and the per-run sequence counter; this closure is the one
27 /// tool-specific bit (Sando previously carried a whole `live_log.rs` copy that
28 /// hardcoded exactly this emit).
29 fn gate_chunk_cb(events: EventTx, run_id: GateRunId) -> ops_core::live_log::ChunkCallback {
30 Box::new(move |seq, text| {
31 events::emit(
32 &events,
33 Event::GateLogChunk {
34 run_id,
35 seq,
36 text: text.to_owned(),
37 },
38 );
39 })
40 }
41
42 pub struct GateCtx {
43 pub pool: SqlitePool,
44 pub cfg: Arc<AppConfig>,
45 pub tier: TierId,
46 pub version: Version,
47 /// The checkout this run's artifact was built from, when there is one.
48 ///
49 /// `None` for an accepted artifact: it was built elsewhere and Sando has no
50 /// source tree for it. That is the boundary made visible (wiki
51 /// [[sando-bento-boundary]]) — artifact-scoped gates belong to the builder,
52 /// so a gate that reads source is one Sando should refuse to run here rather
53 /// than resolve against a path that does not exist.
54 pub worktree: Option<PathBuf>,
55 /// The published, content-addressed bundle this run is about, when it has
56 /// been published yet. `migration_dry_run` prefers it over the worktree, so
57 /// what it proves is inside the digest rather than beside it.
58 pub bundle: Option<PathBuf>,
59 pub events: EventTx,
60 /// Nodes the `node_health` post-deploy gate probes. Empty for build-time
61 /// gate runs on the host (where `node_health` never appears); filled at
62 /// promote time with each freshly-deployed node and its executor.
63 pub nodes: Vec<NodeProbe>,
64 /// The `build_runs.id` this gate run vouches for — the artifact identity
65 /// (wiki [[release-artifact-identity]]). Recorded on every `gate_runs` row so
66 /// promote can resolve the artifact through the evidence for a specific build,
67 /// not through a version string that a later rebuild can silently reuse.
68 /// `None` for legacy/pre-identity runs and gate unit tests.
69 pub build_id: Option<i64>,
70 /// Where each `[[aux_repo]]` is checked out, by topology name. Aux repos sit
71 /// beside the per-sha worktree rather than under it, so a `test_target` that
72 /// names one cannot be resolved against `worktree` alone. Filled from the
73 /// topology at build time; empty at promote time, where the only gate that
74 /// runs is `node_health` and there is no checkout at all.
75 pub aux_dirs: HashMap<String, PathBuf>,
76 /// The tier's public URL, for [`Gate::PageSmoke`]. `None` on every
77 /// build-time run and on any tier that declares none.
78 ///
79 /// [`Gate::PageSmoke`]: crate::topology::Gate::PageSmoke
80 pub public_url: Option<String>,
81 }
82
83 impl GateCtx {
84 /// The `logs_root` sub-directory this run's gate logs land in.
85 ///
86 /// The build id, so two runs of one version keep two sets of logs. Before
87 /// 2026-08-20 this was the version, which meant a rebuild appended to the
88 /// previous attempt's file and every run pointed at the same mixed log.
89 ///
90 /// Falls back to the version when there is no build identity — a
91 /// pre-migration-008 run, or a gate unit test. That is also what keeps logs
92 /// written under the old scheme reachable: their rows still record the
93 /// version path, and nothing rewrites them.
94 pub fn log_scope(&self) -> String {
95 self.build_id
96 .map_or_else(|| self.version.to_string(), |id| id.to_string())
97 }
98
99 /// This run's log pointer for `gate`. Always paired with
100 /// [`Self::log_path`], which resolves the same ref to an absolute path.
101 pub fn log_ref(&self, gate: GateKind) -> LogRef {
102 LogRef::new(&self.log_scope(), gate)
103 }
104
105 /// Where `gate`'s log is written on this host: `logs_root` joined to
106 /// [`Self::log_ref`]. The two are derived from one scope so a row's
107 /// `log_ref` can never name a file the gate did not write.
108 pub fn log_path(&self, gate: GateKind) -> PathBuf {
109 self.cfg
110 .logs_root
111 .join(self.log_scope())
112 .join(format!("{}.log", gate.as_str()))
113 }
114
115 /// Absolute directory a `test_target` runs in: under the worktree, or under
116 /// the named aux repo's checkout.
117 ///
118 /// An `aux_repo` naming nothing this run knows about resolves to `None`
119 /// rather than to a wrong path. Callers treat that as "not present in this
120 /// run" and skip, the same as a target missing from an older sha —
121 /// `--check-config` is what stops a genuine typo from reaching here
122 /// (`Topology::ensure_test_target_aux_repos_exist`).
123 pub fn target_dir(&self, target: &crate::config::TestTarget) -> Option<PathBuf> {
124 match target.aux_repo.as_deref() {
125 None => Some(self.worktree.as_ref()?.join(&target.dir)),
126 Some(name) => Some(self.aux_dirs.get(name)?.join(&target.dir)),
127 }
128 }
129
130 /// The checkout, or a typed refusal for a gate that cannot work without one.
131 ///
132 /// Every caller of this is a gate whose evidence is about the *artifact*
133 /// rather than about the artifact in an environment, which the boundary
134 /// assigns to the builder. Reaching this arm means a tier asked Sando to
135 /// re-run a builder's gate against a bundle it was handed, and the honest
136 /// answer is to say so rather than to pass on having run nothing.
137 pub fn worktree_for(&self, gate: GateKind) -> std::result::Result<&Path, GateOutcome> {
138 self.worktree.as_deref().ok_or_else(|| {
139 GateOutcome::failed(GateFailure::NeedsSource {
140 gate,
141 artifact: self.bundle.as_ref().map_or_else(
142 || "an artifact built elsewhere".into(),
143 |b| b.display().to_string(),
144 ),
145 })
146 })
147 }
148
149 /// Where a `migration_check` finds its migrations.
150 ///
151 /// The bundle wins when it carries them. That is the point of shipping
152 /// migrations as a `release_contents` entry: it puts them inside the digest,
153 /// so the dry run proves something about the bytes that ship rather than
154 /// about a checkout that happens to sit next to them. The worktree is the
155 /// fallback for a build whose config has not opted in yet, and for an
156 /// accepted artifact there is no fallback at all — if the builder did not
157 /// bundle its migrations, Sando cannot dry-run them and says so.
158 pub fn migrations_dir(&self, dir: &Path) -> Option<PathBuf> {
159 if let Some(bundle) = &self.bundle {
160 let in_bundle = bundle.join(dir);
161 if in_bundle.is_dir() {
162 return Some(in_bundle);
163 }
164 }
165 let in_worktree = self.worktree.as_ref()?.join(dir);
166 in_worktree.is_dir().then_some(in_worktree)
167 }
168 }
169
170 /// One node the `node_health` gate verifies: its id, the systemd unit to
171 /// confirm active after the restart, an optional HTTP readiness URL, and the
172 /// executor that reaches it (the same transport the deploy used).
173 pub struct NodeProbe {
174 pub node: crate::domain::NodeId,
175 pub service: String,
176 pub health_url: Option<String>,
177 pub executor: Arc<dyn ops_exec::Executor>,
178 }
179
180 /// Run a single gate end-to-end: insert the in-flight row, execute the gate,
181 /// update the row with the outcome. Returns the outcome for the caller.
182 pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> {
183 let kind = gate.kind();
184 let started_at = Utc::now().to_rfc3339();
185
186 let id: i64 = sqlx::query_scalar(
187 "INSERT INTO gate_runs (app, version, tier, gate_kind, started_at, build_id)
188 VALUES (?, ?, ?, ?, ?, ?)
189 RETURNING id",
190 )
191 .bind(&ctx.cfg.id)
192 .bind(&ctx.version)
193 .bind(&ctx.tier)
194 .bind(kind)
195 .bind(&started_at)
196 .bind(ctx.build_id)
197 .fetch_one(&ctx.pool)
198 .await?;
199 let run_id = GateRunId(id);
200
201 tracing::info!(
202 run_id = %run_id, tier = %ctx.tier, version = %ctx.version, gate = %kind,
203 "gate start",
204 );
205 events::emit(
206 &ctx.events,
207 Event::GateStart {
208 run_id,
209 tier: ctx.tier.clone(),
210 version: ctx.version.clone(),
211 gate: kind,
212 },
213 );
214
215 let outcome = match gate {
216 // cargo_test bounds its own run internally (it kills the specific child).
217 Gate::CargoTest => cargo_test(ctx, run_id).await,
218 // hardening_test bounds its own run internally, same as cargo_test.
219 Gate::HardeningTest => hardening_test(ctx, run_id).await,
220 // Each bounds itself the same way cargo_test does: one deadline across
221 // every target, so N crates cannot multiply the ceiling by N.
222 Gate::Clippy => clippy(ctx, run_id).await,
223 Gate::Fmt => fmt_check(ctx, run_id).await,
224 Gate::CargoAudit => supply_chain(ctx, run_id, GateKind::CargoAudit).await,
225 Gate::CargoDeny => supply_chain(ctx, run_id, GateKind::CargoDeny).await,
226 // migration_dry_run's psql restore + sqlx migrate could wedge; bound the
227 // whole gate here. Its bash restore sets kill_on_drop, so a timeout-drop
228 // doesn't orphan it.
229 Gate::MigrationDryRun => {
230 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
231 match tokio::time::timeout(ceiling, migration_dry_run(ctx, run_id)).await {
232 Ok(res) => res,
233 Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout {
234 gate: GateKind::MigrationDryRun,
235 after_s: ctx.cfg.gate_timeout_secs as u32,
236 })
237 .with_log_ref(ctx.log_ref(GateKind::MigrationDryRun))),
238 }
239 }
240 // code_smoke boots the real binary (migrate-from-scratch + seed + serve),
241 // any step of which could wedge; bound the whole gate here. Both child
242 // processes set kill_on_drop, so a timeout-drop can't orphan them. A
243 // timeout leaves the throwaway DB behind; the next run's createdb drops
244 // it first (DROP IF EXISTS), same as migration_dry_run's scratch reset.
245 Gate::CodeSmoke => {
246 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
247 match tokio::time::timeout(ceiling, code_smoke(ctx, run_id)).await {
248 Ok(res) => res,
249 Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout {
250 gate: GateKind::CodeSmoke,
251 after_s: ctx.cfg.gate_timeout_secs as u32,
252 })
253 .with_log_ref(ctx.log_ref(GateKind::CodeSmoke))),
254 }
255 }
256 Gate::BootSmoke => boot_smoke(ctx, run_id).await,
257 Gate::NodeHealth => node_health(ctx).await,
258 Gate::PageSmoke => page_smoke(ctx).await,
259 Gate::BurnIn { hours } => burn_in(ctx, *hours).await,
260 Gate::ManualConfirm => manual_confirm(ctx).await,
261 };
262
263 let outcome = outcome.unwrap_or_else(|e| {
264 GateOutcome::failed(GateFailure::Unclassified {
265 legacy_detail: Some(format!("gate runner errored: {e}")),
266 })
267 });
268
269 let outcome_json = serde_json::to_string(&outcome)
270 .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
271 sqlx::query(
272 "UPDATE gate_runs
273 SET finished_at = ?, status = ?, outcome_json = ?, log_ref = ?
274 WHERE id = ?",
275 )
276 .bind(Utc::now().to_rfc3339())
277 .bind(outcome.status_str())
278 .bind(&outcome_json)
279 .bind(outcome.log_ref.as_ref().map(super::outcome::LogRef::as_str))
280 .bind(id)
281 .execute(&ctx.pool)
282 .await?;
283
284 tracing::info!(
285 tier = %ctx.tier, version = %ctx.version, gate = %kind,
286 status = outcome.status_str(), "gate done",
287 );
288 events::emit(
289 &ctx.events,
290 Event::GateDone {
291 run_id,
292 tier: ctx.tier.clone(),
293 version: ctx.version.clone(),
294 gate: kind,
295 outcome: outcome.clone(),
296 },
297 );
298
299 Ok(outcome)
300 }
301
302 /// Run every gate in order and return the kinds that did not pass (empty means
303 /// green). We deliberately do NOT short-circuit on first failure — every gate's
304 /// outcome is recorded in `gate_runs`, which is the operator's only visibility
305 /// into pipeline health. Hiding later gates because an earlier one failed makes
306 /// diagnosis worse.
307 ///
308 /// Returning the failing kinds rather than a bare bool is what lets the promote
309 /// path name them in the tier's `partial_reason` and in the error it returns to
310 /// the operator, instead of a generic "something was red".
311 pub async fn run_all(ctx: &GateCtx, gates: &[Gate]) -> Result<Vec<GateKind>> {
312 let mut failed = Vec::new();
313 for g in gates {
314 let o = run(ctx, g).await?;
315 if !o.is_passed() {
316 failed.push(g.kind());
317 }
318 }
319 Ok(failed)
320 }
321
322 // ---- individual gate runners ----
323
324 /// Run every configured `test_target`'s suite, in order, under one gate.
325 ///
326 /// This used to be hardcoded to `worktree/server`, which meant every other
327 /// crate in the repo shipped ungated — including `mnw-cli`, which is built as a
328 /// companion and installed onto prod-1 in the same promote. The targets are now
329 /// configured (`[[test_target]]` in the daemon config), defaulting to the
330 /// historical single `server` entry.
331 ///
332 /// The whole set shares one `gate_runs` row and one log file: from the
333 /// pipeline's point of view "the tests" either pass or don't. The first failing
334 /// target ends the gate, since a red suite blocks the promote regardless of what
335 /// the remaining crates would have said, and running them would only delay the
336 /// operator's answer. Its name is carried in the failure so the summary points
337 /// at the crate, not just the test.
338 async fn cargo_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
339 let log_path = ctx.log_path(GateKind::CargoTest);
340 let log_ref = ctx.log_ref(GateKind::CargoTest);
341
342 // Best-effort: drop our own role's stale `mnw_test_*` databases (the
343 // template + any per-test clones orphaned by a previously-killed run)
344 // before the suite, so they can't accumulate or collide. Foreign-owned
345 // leftovers are left alone — the harness now namespaces its template per
346 // role, so they no longer wedge the gate.
347 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
348 clean_stale_test_dbs(scratch_url).await;
349 }
350
351 let started = std::time::Instant::now();
352 // One ceiling for the whole gate, not per target: the point is to bound how
353 // long a hung suite can block the pipeline, and N targets each allowed the
354 // full timeout would multiply that by N.
355 let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
356 let mut ran = 0usize;
357
358 for target in &ctx.cfg.test_targets {
359 let label = target.label();
360 // A target absent from this sha is skipped, not fatal: sando has to be
361 // able to build older shas (bisect, rollback rebuild) from a config that
362 // describes the tip. The zero-targets-ran check below is what stops this
363 // from quietly turning the gate into a no-op.
364 let Some(dir) = ctx
365 .target_dir(target)
366 .filter(|d| d.join("Cargo.toml").is_file())
367 else {
368 tracing::warn!(
369 target = %label, version = %ctx.version,
370 "test_target has no Cargo.toml in this run; skipping",
371 );
372 continue;
373 };
374 let features: Vec<&str> = target.features.iter().map(String::as_str).collect();
375
376 // Fast pre-gate: compile the test targets WITHOUT running them. This
377 // builds the exact artifacts the full run needs (so the subsequent run
378 // reuses the cache — no wasted work), but fails in ~minutes with the
379 // real `error[Ennnn]: ...` on a test-only-target compile break. That
380 // class (a field missing in a `#[cfg(test)]`-only binary like `load`)
381 // otherwise compiles fine under the build step and only blows up here,
382 // after a full build, as an opaque mass test failure.
383 let banner = format!("\n==== test_target: {label} ====\n");
384 append_to_log(&log_path, banner.as_bytes()).await;
385 let mut pre = match cargo_test_command(ctx, &dir, target, &features, &["--no-run"]).spawn()
386 {
387 Ok(c) => c,
388 Err(e) => {
389 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
390 message: format!("{label}: {e}"),
391 })
392 .with_log_ref(log_ref));
393 }
394 };
395 let (pre_out, pre_err, pre_status) = match run_to_deadline_for(
396 &mut pre,
397 ctx,
398 run_id,
399 log_path.clone(),
400 deadline,
401 started,
402 GateKind::CargoTest,
403 )
404 .await?
405 {
406 Ok(v) => v,
407 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
408 };
409 if !pre_status.success() {
410 let failure = classify::classify_compile_error(&pre_out, &pre_err);
411 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
412 }
413
414 // Full run: the test binaries are already built above, so cargo's
415 // up-to-date check skips compilation and this just runs the tests.
416 //
417 // That claim is only true when the crate's build script is up to date
418 // too, and for a long time it was not. server and multithreaded both
419 // watched `.git/HEAD`, a path that does not exist at either package
420 // root, and cargo reads a missing watch as changed: the build script
421 // re-ran and the crate recompiled here, every time. It cost 347s a
422 // pipeline, 35% of this gate, while this comment said it cost nothing.
423 // Fixed 2026-08-20 in both build scripts; see `git_hash` in either.
424 //
425 // If this gate's duration ever climbs back toward the pre-pass's, look
426 // for a new phantom watch before looking anywhere else.
427 let mut child = match cargo_test_command(ctx, &dir, target, &features, &[]).spawn() {
428 Ok(c) => c,
429 Err(e) => {
430 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
431 message: format!("{label}: {e}"),
432 })
433 .with_log_ref(log_ref));
434 }
435 };
436 let (stdout_buf, stderr_buf, status) = match run_to_deadline_for(
437 &mut child,
438 ctx,
439 run_id,
440 log_path.clone(),
441 deadline,
442 started,
443 GateKind::CargoTest,
444 )
445 .await?
446 {
447 Ok(v) => v,
448 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
449 };
450 if !status.success() {
451 let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf);
452 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
453 }
454 ran += 1;
455 }
456
457 // Every configured target was missing from the worktree. Exiting green here
458 // would report "tests passed" having run none of them.
459 if ran == 0 {
460 return Ok(GateOutcome::failed(GateFailure::Unclassified {
461 legacy_detail: Some(format!(
462 "cargo_test ran no targets: none of the {} configured test_target dir(s) \
463 exist in this worktree",
464 ctx.cfg.test_targets.len(),
465 )),
466 })
467 .with_log_ref(log_ref));
468 }
469
470 let duration_s = started.elapsed().as_secs() as u32;
471 Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref))
472 }
473
474 /// Stream a child to the live log, bounded by the gate-wide `deadline`. `Ok(Err(_))`
475 /// is the timeout outcome (child killed); `Err(_)` is an IO error on the stream.
476 #[allow(clippy::type_complexity)]
477 async fn run_to_deadline_for(
478 child: &mut tokio::process::Child,
479 ctx: &GateCtx,
480 run_id: GateRunId,
481 log_path: PathBuf,
482 deadline: std::time::Instant,
483 started: std::time::Instant,
484 kind: GateKind,
485 ) -> Result<std::result::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus), GateOutcome>> {
486 let remaining = deadline.saturating_duration_since(std::time::Instant::now());
487 let stream = stream_child_to_live_log(child, ctx.events.clone(), run_id, log_path);
488 match tokio::time::timeout(remaining, stream).await {
489 Ok(res) => Ok(Ok(res?)),
490 Err(_elapsed) => {
491 child.start_kill().ok();
492 let _ = child.wait().await;
493 Ok(Err(GateOutcome::failed(GateFailure::Timeout {
494 gate: kind,
495 after_s: started.elapsed().as_secs() as u32,
496 })))
497 }
498 }
499 }
500
501 /// Prefix a test/compile failure's headline with the crate it came from, so a
502 /// red gate across many targets says *which* crate broke. Other failure kinds
503 /// are single-target by construction and pass through untouched.
504 fn name_target(failure: GateFailure, dir: &std::path::Path) -> GateFailure {
505 let at = dir.display();
506 match failure {
507 GateFailure::CargoTest {
508 failed_count,
509 first_failed,
510 first_panic,
511 } => GateFailure::CargoTest {
512 failed_count,
513 first_failed: Some(match first_failed {
514 Some(name) => format!("{at}: {name}"),
515 None => at.to_string(),
516 }),
517 first_panic,
518 },
519 GateFailure::CompileError {
520 error_count,
521 first_error,
522 } => GateFailure::CompileError {
523 error_count,
524 first_error: Some(match first_error {
525 Some(e) => format!("{at}: {e}"),
526 None => at.to_string(),
527 }),
528 },
529 other => other,
530 }
531 }
532
533 /// Append raw bytes to a gate log outside the child-streaming path (target
534 /// banners). Best-effort, same as `LiveLog`: a broken log dir never turns a
535 /// passing gate red.
536 async fn append_to_log(path: &std::path::Path, bytes: &[u8]) {
537 use tokio::io::AsyncWriteExt;
538 if let Some(parent) = path.parent()
539 && tokio::fs::create_dir_all(parent).await.is_err()
540 {
541 return;
542 }
543 if let Ok(mut f) = tokio::fs::OpenOptions::new()
544 .create(true)
545 .append(true)
546 .open(path)
547 .await
548 {
549 let _ = f.write_all(bytes).await;
550 }
551 }
552
553 /// `cargo clippy --all-targets -- -D warnings` over every configured
554 /// `test_target`.
555 ///
556 /// Before this gate, `-D warnings` was enforced in exactly one place —
557 /// `server/deploy/run-ci.sh`, which died with the astra pipeline — so lint drift
558 /// reached prod ungated. Two crates even carried comments referring to "the
559 /// -D warnings CI gate" for a gate that did not exist.
560 async fn clippy(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
561 lint_over_targets(ctx, run_id, GateKind::Clippy, |target, features| {
562 let mut args = vec!["clippy".to_string(), "--all-targets".to_string()];
563 if target.all_features {
564 args.push("--all-features".to_string());
565 } else if !features.is_empty() {
566 args.push("--features".to_string());
567 args.push(features.join(","));
568 }
569 // Everything after `--` goes to rustc, which is where -D lives.
570 args.push("--".to_string());
571 args.push("-D".to_string());
572 args.push("warnings".to_string());
573 args
574 })
575 .await
576 }
577
578 /// `cargo fmt --check` over every configured `test_target`.
579 ///
580 /// No `rustfmt.toml` exists anywhere in the tree, so this is plain rustfmt
581 /// defaults. Cheap: no compilation, just a parse.
582 async fn fmt_check(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
583 lint_over_targets(ctx, run_id, GateKind::Fmt, |_target, _features| {
584 vec!["fmt".to_string(), "--check".to_string()]
585 })
586 .await
587 }
588
589 /// `cargo audit` / `cargo deny check`, over the crates that carry the matching
590 /// config file.
591 ///
592 /// Config-gated on purpose. Both tools are only meaningful against a triaged
593 /// posture: four crates in this repo fail `cargo audit` today purely because
594 /// they have no `.cargo/audit.toml` recording which transitive advisories have
595 /// been reviewed and accepted. Running them everywhere would make the gate
596 /// permanently and uninformatively red. Dropping the config file into a crate
597 /// is what opts it in.
598 async fn supply_chain(ctx: &GateCtx, run_id: GateRunId, kind: GateKind) -> Result<GateOutcome> {
599 let (config_rel, args): (&str, Vec<String>) = match kind {
600 GateKind::CargoAudit => (".cargo/audit.toml", vec!["audit".into()]),
601 GateKind::CargoDeny => ("deny.toml", vec!["deny".into(), "check".into()]),
602 other => unreachable!("supply_chain called for {other:?}"),
603 };
604 run_over_targets(ctx, run_id, kind, |_target, target_dir| {
605 target_dir.join(config_rel).is_file().then(|| args.clone())
606 })
607 .await
608 }
609
610 /// Shared driver for the lint gates: run `cargo <args>` in every configured
611 /// `test_target` that exists in this worktree.
612 async fn lint_over_targets(
613 ctx: &GateCtx,
614 run_id: GateRunId,
615 kind: GateKind,
616 build_args: impl Fn(&crate::config::TestTarget, &[String]) -> Vec<String>,
617 ) -> Result<GateOutcome> {
618 // The target is handed in directly. This used to reverse-look-it-up by
619 // comparing `worktree.join(dir)` against the resolved path, which silently
620 // stopped matching for anything resolved anywhere else — an aux repo, say.
621 run_over_targets(ctx, run_id, kind, move |t, _dir| {
622 Some(build_args(t, &t.features))
623 })
624 .await
625 }
626
627 /// Run one cargo invocation per configured `test_target`, under a single
628 /// gate-wide deadline. `args_for` returns `None` to skip a target (used by the
629 /// supply-chain gates, which only apply where their config file lives).
630 ///
631 /// Shares `cargo_test`'s conventions: per-target log banners, stop at the first
632 /// failure with the crate named, skip targets absent from the worktree, and fail
633 /// closed if that leaves nothing to run.
634 async fn run_over_targets(
635 ctx: &GateCtx,
636 run_id: GateRunId,
637 kind: GateKind,
638 args_for: impl Fn(&crate::config::TestTarget, &std::path::Path) -> Option<Vec<String>>,
639 ) -> Result<GateOutcome> {
640 let log_path = ctx.log_path(kind);
641 let log_ref = ctx.log_ref(kind);
642 let started = std::time::Instant::now();
643 let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
644 let mut ran = 0usize;
645
646 for target in &ctx.cfg.test_targets {
647 let label = target.label();
648 let Some(dir) = ctx
649 .target_dir(target)
650 .filter(|d| d.join("Cargo.toml").is_file())
651 else {
652 tracing::warn!(
653 gate = kind.as_str(), target = %label,
654 "target has no Cargo.toml in this run; skipping",
655 );
656 continue;
657 };
658 let Some(args) = args_for(target, &dir) else {
659 continue;
660 };
661
662 append_to_log(
663 &log_path,
664 format!("\n==== {}: {label} ====\n", kind.as_str()).as_bytes(),
665 )
666 .await;
667
668 let mut cmd = Command::new("cargo");
669 cmd.args(&args)
670 .current_dir(&dir)
671 .stdout(std::process::Stdio::piped())
672 .stderr(std::process::Stdio::piped())
673 .kill_on_drop(true);
674 if let Some(t) = ctx.cfg.cargo_target_dir.as_deref() {
675 cmd.env("CARGO_TARGET_DIR", t);
676 }
677 // clippy type-checks, so it needs the same sqlx online-mode env the
678 // build and cargo_test steps get. fmt/audit/deny never touch the DB.
679 if kind == GateKind::Clippy
680 && let Some(url) = ctx
681 .cfg
682 .scratch_db_url
683 .as_deref()
684 .filter(|_| target.scratch_db)
685 {
686 cmd.env("DATABASE_URL", url);
687 cmd.env(
688 "TEST_DATABASE_URL",
689 url.split_once('?').map_or(url, |(b, _)| b),
690 );
691 }
692
693 let mut child = match cmd.spawn() {
694 Ok(c) => c,
695 Err(e) => {
696 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
697 message: format!("{label}: {e}"),
698 })
699 .with_log_ref(log_ref));
700 }
701 };
702 let (stdout_buf, stderr_buf, status) = match run_to_deadline_for(
703 &mut child,
704 ctx,
705 run_id,
706 log_path.clone(),
707 deadline,
708 started,
709 kind,
710 )
711 .await?
712 {
713 Ok(v) => v,
714 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
715 };
716 if !status.success() {
717 let failure = match kind {
718 // clippy speaks rustc diagnostics, so the compile-error
719 // classifier extracts the real `error: ...` headline.
720 GateKind::Clippy => classify::classify_compile_error(&stdout_buf, &stderr_buf),
721 _ => GateFailure::Unclassified {
722 legacy_detail: Some(first_meaningful_line(&stdout_buf, &stderr_buf)),
723 },
724 };
725 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
726 }
727 ran += 1;
728 }
729
730 if ran == 0 {
731 return Ok(GateOutcome::failed(GateFailure::Unclassified {
732 legacy_detail: Some(format!(
733 "{} ran nothing: no configured target in this worktree qualified",
734 kind.as_str(),
735 )),
736 })
737 .with_log_ref(log_ref));
738 }
739 Ok(GateOutcome::passed(PassNote::TestsPassed {
740 duration_s: started.elapsed().as_secs() as u32,
741 })
742 .with_log_ref(log_ref))
743 }
744
745 /// First line that looks like a diagnostic, for gates whose tools have no
746 /// dedicated classifier (`cargo audit`, `cargo deny`). Falls back to a generic
747 /// note rather than an empty string.
748 fn first_meaningful_line(stdout: &[u8], stderr: &[u8]) -> String {
749 for buf in [stderr, stdout] {
750 let text = String::from_utf8_lossy(buf);
751 if let Some(line) = text
752 .lines()
753 .map(str::trim)
754 .find(|l| l.starts_with("error") || l.contains("vulnerabilit") || l.contains("FAILED"))
755 {
756 return line.chars().take(200).collect();
757 }
758 }
759 "tool reported failure; see the gate log".into()
760 }
761
762 /// The tests `cargo_test` cannot reach, run against production constants.
763 ///
764 /// `cargo_test` builds with `--features fast-tests`, which relaxes
765 /// `AUTH_RATE_LIMIT_BURST` 5 → 20, `SANDBOX_RATE_LIMIT_MS` 30s → 10ms, and
766 /// argon2 from 46 MiB/t=2 to 8 MiB/t=1 — and the rate-limiting suite is
767 /// `#[cfg_attr(feature = "fast-tests", ignore)]`d on top of that, because a
768 /// bucket refilling at 100/sec never depletes under parallel test threads. The
769 /// net effect was that Sando's only code gate silently skipped every test of
770 /// the auth hardening it most needs to protect.
771 ///
772 /// So: no features, `--test-threads=1` (these tests key on a shared per-IP
773 /// bucket and must not interleave), and a name filter rather than the whole
774 /// suite — the rest of the suite is tuned for `fast-tests` and would only go
775 /// slow and flaky here. The filter is a substring match, so it catches
776 /// `..._rate_limit_...` and `..._rate_limited` alike.
777 ///
778 /// This costs a second compile of the lib + integration binary (a different
779 /// feature set is a different cfg, so no artifact sharing with `cargo_test`).
780 /// That is the price of the coverage; the filter keeps the *run* to seconds.
781 async fn hardening_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
782 let server_dir = match ctx.worktree_for(GateKind::HardeningTest) {
783 Ok(w) => w.join("server"),
784 Err(outcome) => return Ok(outcome),
785 };
786 // No features is the whole point; the scratch DB is needed because the
787 // server's sqlx macros type-check against it. Unlike cargo_test, this gate
788 // is deliberately not driven by `test_targets`: it targets one specific
789 // suite in one specific crate, not "the repo's tests".
790 let target = crate::config::TestTarget {
791 dir: std::path::PathBuf::from("server"),
792 aux_repo: None,
793 features: Vec::new(),
794 all_features: false,
795 scratch_db: true,
796 };
797 let log_path = ctx.log_path(GateKind::HardeningTest);
798 let log_ref = ctx.log_ref(GateKind::HardeningTest);
799
800 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
801 clean_stale_test_dbs(scratch_url).await;
802 }
803
804 let started = std::time::Instant::now();
805
806 // Same two-step shape as cargo_test: compile first so a test-target break
807 // reports as a compile error rather than an opaque mass test failure.
808 let mut pre = match cargo_test_command(
809 ctx,
810 &server_dir,
811 &target,
812 &[],
813 &["--no-run", "--test", "integration"],
814 )
815 .spawn()
816 {
817 Ok(c) => c,
818 Err(e) => {
819 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
820 message: e.to_string(),
821 })
822 .with_log_ref(log_ref));
823 }
824 };
825 let (pre_out, pre_err, pre_status) =
826 stream_child_to_live_log(&mut pre, ctx.events.clone(), run_id, log_path.clone()).await?;
827 if !pre_status.success() {
828 let failure = classify::classify_compile_error(&pre_out, &pre_err);
829 return Ok(GateOutcome::failed(failure).with_log_ref(log_ref));
830 }
831
832 let mut child = match cargo_test_command(
833 ctx,
834 &server_dir,
835 &target,
836 &[],
837 &[
838 "--test",
839 "integration",
840 "--",
841 "--test-threads=1",
842 "rate_limit",
843 ],
844 )
845 .spawn()
846 {
847 Ok(c) => c,
848 Err(e) => {
849 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
850 message: e.to_string(),
851 })
852 .with_log_ref(log_ref));
853 }
854 };
855 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
856 let stream = stream_child_to_live_log(&mut child, ctx.events.clone(), run_id, log_path);
857 let (stdout_buf, stderr_buf, status) = match tokio::time::timeout(ceiling, stream).await {
858 Ok(res) => res?,
859 Err(_elapsed) => {
860 child.start_kill().ok();
861 let _ = child.wait().await;
862 return Ok(GateOutcome::failed(GateFailure::Timeout {
863 gate: GateKind::HardeningTest,
864 after_s: started.elapsed().as_secs() as u32,
865 })
866 .with_log_ref(log_ref));
867 }
868 };
869 let duration_s = started.elapsed().as_secs() as u32;
870 if status.success() {
871 // A filter that matches nothing exits 0, which would make this gate a
872 // green no-op the day someone renames the tests. Fail closed instead.
873 if tests_run(&stdout_buf) == 0 {
874 return Ok(GateOutcome::failed(GateFailure::Unclassified {
875 legacy_detail: Some(
876 "hardening_test ran 0 tests: the `rate_limit` filter matched nothing. \
877 The suite was renamed or moved — this gate is proving nothing."
878 .into(),
879 ),
880 })
881 .with_log_ref(log_ref));
882 }
883 Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref))
884 } else {
885 let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf);
886 Ok(GateOutcome::failed(failure).with_log_ref(log_ref))
887 }
888 }
889
890 /// Count the tests libtest reports as run, from its `test result: ok. N passed`
891 /// summary line. Returns 0 when no summary is present, which is itself the
892 /// "nothing ran" case the caller fails on.
893 fn tests_run(stdout: &[u8]) -> u32 {
894 String::from_utf8_lossy(stdout)
895 .lines()
896 .filter_map(|l| l.trim().strip_prefix("test result:"))
897 .filter_map(|rest| rest.split_once(" passed"))
898 .filter_map(|(head, _)| {
899 head.rsplit(' ')
900 .find(|t| !t.is_empty())?
901 .parse::<u32>()
902 .ok()
903 })
904 .sum()
905 }
906
907 /// Configure (but don't spawn) `cargo test --release [--features <features>]
908 /// <extra>` in `dir`, wired to the scratch DB. Shared by the `--no-run`
909 /// pre-gate compile and the full test run so both go through one env setup,
910 /// and by both test gates so they can't drift apart on env.
911 ///
912 /// `cargo_test` passes `fast-tests`, matching what the retired astra CI did:
913 /// it relaxes the auth rate-limit burst (5 → 20) and argon2 cost so
914 /// signup-heavy + lockout workflow tests complete without hitting Governor
915 /// before the hand-rolled lockout check (documented at
916 /// `server/src/constants.rs:87`). `hardening_test` passes no features, which is
917 /// the whole point of that gate — see `GateKind::HardeningTest`.
918 fn cargo_test_command(
919 ctx: &GateCtx,
920 dir: &std::path::Path,
921 target: &crate::config::TestTarget,
922 features: &[&str],
923 extra: &[&str],
924 ) -> Command {
925 let mut cmd = Command::new("cargo");
926 cmd.args(["test", "--release"]);
927 if target.all_features {
928 cmd.arg("--all-features");
929 } else if !features.is_empty() {
930 cmd.args(["--features", &features.join(",")]);
931 }
932 cmd.args(extra)
933 .current_dir(dir)
934 .stdout(std::process::Stdio::piped())
935 .stderr(std::process::Stdio::piped())
936 .kill_on_drop(true);
937 // Share the build step's target dir so the test compile reuses its
938 // artifacts (and the `--no-run` precompile reuses them again). Must match
939 // `build.rs` or the gate would clean-compile the whole tree a second time.
940 if let Some(target) = ctx.cfg.cargo_target_dir.as_deref() {
941 cmd.env("CARGO_TARGET_DIR", target);
942 }
943 // Same online-mode rationale as the build step: sqlx query macros need a
944 // live DB to type-check against. The scratch DB is left in migrated state
945 // by the preceding build, so we can reuse it here.
946 //
947 // Opt-in per target: setting DATABASE_URL switches sqlx OUT of offline mode,
948 // so a crate that ships `.sqlx` query data would stop using it and try to
949 // type-check against a database that has none of its tables.
950 if let Some(scratch_url) = ctx
951 .cfg
952 .scratch_db_url
953 .as_deref()
954 .filter(|_| target.scratch_db)
955 {
956 cmd.env("DATABASE_URL", scratch_url);
957 // The server test harness (tests/harness/db.rs) parses TEST_DATABASE_URL
958 // with rfind('/'), which mangles URLs whose query string contains '/'
959 // (e.g. `?host=/var/run/postgresql`). Strip the query — libpq defaults
960 // to /var/run/postgresql on Debian/Ubuntu when host is unspecified.
961 let test_url = scratch_url
962 .split_once('?')
963 .map_or(scratch_url, |(base, _)| base);
964 cmd.env("TEST_DATABASE_URL", test_url);
965 }
966 cmd
967 }
968
969 async fn migration_dry_run(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
970 let log = GateLog::open(ctx, run_id, GateKind::MigrationDryRun).await;
971 let outcome = migration_dry_run_inner(ctx, &log).await;
972 log.close().await;
973 outcome.map(|o| o.with_log_ref(ctx.log_ref(GateKind::MigrationDryRun)))
974 }
975
976 /// The staged interior of [`migration_dry_run`], writing every step through the
977 /// gate's live log. The caller owns the sink so it can flush it on every exit
978 /// path, and attaches the `log_ref` once instead of at each return.
979 /// Runs one configured check per database, in config order, and stops at the
980 /// first that does not pass — a red gate is a red gate, and continuing would
981 /// bury it under a second restore's output.
982 ///
983 /// The server's check runs against `scratch_db_url` itself and is deliberately
984 /// last-writer for it: `cargo_test` reuses that database in migrated state, so
985 /// every other check must name its own `scratch_db` (enforced at config load).
986 async fn migration_dry_run_inner(ctx: &GateCtx, log: &GateLog) -> Result<GateOutcome> {
987 let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() else {
988 log.line("scratch_db_url unset in daemon config\n").await;
989 return Ok(GateOutcome::blocked(GateBlocker::ScratchDbUrlUnset));
990 };
991
992 let mut checked = Vec::new();
993 let mut primary_backup_path = String::new();
994 for check in &ctx.cfg.migration_checks {
995 let label = check.dir.display().to_string();
996 log.line(&format!("==== migration_check: {label} ====\n"))
997 .await;
998 match run_migration_check(ctx, log, scratch_url, check).await? {
999 CheckResult::Passed { backup_path } => {
1000 if primary_backup_path.is_empty() {
1001 primary_backup_path = backup_path;
1002 }
1003 checked.push(label);
1004 }
1005 CheckResult::Stopped(outcome) => return Ok(outcome),
1006 }
1007 }
1008
1009 log.line(&format!(
1010 "all {} migration check(s) passed: {}",
1011 checked.len(),
1012 checked.join(", ")
1013 ))
1014 .await;
1015 Ok(GateOutcome::passed(PassNote::Migrated {
1016 backup_path: primary_backup_path,
1017 checks: checked,
1018 }))
1019 }
1020
1021 /// One check's verdict: it passed (against `backup_path`), or it produced the
1022 /// outcome the whole gate reports.
1023 enum CheckResult {
1024 Passed { backup_path: String },
1025 Stopped(GateOutcome),
1026 }
1027
1028 /// Restore one database's dump into its scratch DB and run its migrations on top.
1029 async fn run_migration_check(
1030 ctx: &GateCtx,
1031 log: &GateLog,
1032 scratch_url: &str,
1033 check: &crate::config::MigrationCheck,
1034 ) -> Result<CheckResult> {
1035 let label = check.dir.display().to_string();
1036
1037 let backup: Option<(String, String)> = sqlx::query_as(
1038 "SELECT local_path, fetched_at FROM backups
1039 WHERE app = ? AND name = ? ORDER BY id DESC LIMIT 1",
1040 )
1041 .bind(&ctx.cfg.id)
1042 .bind(&check.backup)
1043 .fetch_optional(&ctx.pool)
1044 .await?;
1045 let Some((backup_path, fetched_at)) = backup else {
1046 log.line(&format!(
1047 "no {} backup fetched; call /backup/fetch first\n",
1048 check.backup
1049 ))
1050 .await;
1051 return Ok(CheckResult::Stopped(GateOutcome::blocked(
1052 GateBlocker::NoBackupAvailable {
1053 check: label,
1054 backup: check.backup.clone(),
1055 },
1056 )));
1057 };
1058
1059 // Presence is not freshness. A fetch that quietly stopped working leaves this
1060 // row in place, and restoring it dry-runs the migrations against a schema prod
1061 // has moved past — green, and worthless. Block on age instead. An unparsable
1062 // timestamp is treated as stale: this row is daemon-written RFC 3339, so a
1063 // value that will not parse means something is wrong, and failing closed on a
1064 // freshness check is the whole point.
1065 let age_hours = chrono::DateTime::parse_from_rfc3339(&fetched_at).map_or(i64::MAX, |t| {
1066 (Utc::now() - t.with_timezone(&Utc)).num_hours()
1067 });
1068 let max_age_hours = ctx.cfg.backup_max_age_hours;
1069 if age_hours > i64::from(max_age_hours) {
1070 let msg = format!(
1071 "backup {backup_path} was fetched {fetched_at} ({age_hours}h ago, max \
1072 {max_age_hours}h); re-run /backup/fetch\n"
1073 );
1074 log.line(&msg).await;
1075 return Ok(CheckResult::Stopped(GateOutcome::blocked(
1076 GateBlocker::BackupStale {
1077 age_hours,
1078 max_age_hours,
1079 check: label,
1080 },
1081 )));
1082 }
1083
1084 // A check with its own `scratch_db` gets that database created here rather
1085 // than by a host bootstrap step: adding a `[[migration_check]]` should not
1086 // silently depend on someone having remembered to `createdb` on the Sando
1087 // host, which is exactly the class of footgun this gate exists to remove.
1088 // DROP + CREATE also makes the database sando-owned, so the PG15+ public
1089 // schema grants `reset_scratch` applies next are the owner's to give.
1090 let db_url = match check.scratch_db.as_deref() {
1091 None => scratch_url.to_string(),
1092 Some(dbname) => {
1093 let maintenance_url = pg_url_with_dbname(scratch_url, "postgres");
1094 log.line(&format!("---- create scratch db {dbname} ----\n"))
1095 .await;
1096 if let Err(e) = pg_create_db(&maintenance_url, dbname).await {
1097 let msg = format!("{label}: creating scratch db {dbname}: {e}");
1098 log.line(&msg).await;
1099 return Ok(CheckResult::Stopped(GateOutcome::failed(
1100 GateFailure::RestoreFailed { reason: msg },
1101 )));
1102 }
1103 pg_url_with_dbname(scratch_url, dbname)
1104 }
1105 };
1106
1107 let owner_role = check
1108 .owner_role
1109 .as_deref()
1110 .unwrap_or(&ctx.cfg.scratch_owner_role);
1111 log.line("---- reset_scratch ----\n").await;
1112 if let Err(e) = reset_scratch(&db_url, owner_role).await {
1113 let msg = format!("{label}: scratch reset: {e}");
1114 log.line(&msg).await;
1115 return Ok(CheckResult::Stopped(GateOutcome::failed(
1116 GateFailure::RestoreFailed { reason: msg },
1117 )));
1118 }
1119 log.line(&format!("---- restore_dump ({backup_path}) ----\n"))
1120 .await;
1121 if let Err(e) = restore_dump(&db_url, &backup_path, log).await {
1122 let msg = format!("{label}: restore: {e}");
1123 log.line(&msg).await;
1124 return Ok(CheckResult::Stopped(GateOutcome::failed(
1125 GateFailure::RestoreFailed { reason: msg },
1126 )));
1127 }
1128
1129 let Some(migrations_dir) = ctx.migrations_dir(&check.dir) else {
1130 // Neither the bundle nor a checkout holds them. For an accepted
1131 // artifact that means the builder did not ship its migrations, and a
1132 // dry run over nothing would report green having proved nothing.
1133 let msg = format!(
1134 "{label}: no migrations at {} in the bundle or a checkout",
1135 check.dir.display()
1136 );
1137 log.line(&msg).await;
1138 return Ok(CheckResult::Stopped(GateOutcome::failed(
1139 GateFailure::RestoreFailed { reason: msg },
1140 )));
1141 };
1142 log.line("---- run_migrator ----\n").await;
1143 match run_migrator(&db_url, &migrations_dir).await {
1144 Ok(()) => {
1145 log.line(&format!("{label}: restored {backup_path} + migrated\n"))
1146 .await;
1147 Ok(CheckResult::Passed { backup_path })
1148 }
1149 Err(e) => {
1150 let err_s = format!("{label}: {e}");
1151 log.line(&err_s).await;
1152 Ok(CheckResult::Stopped(GateOutcome::failed(
1153 classify::classify_migration_error(&err_s, None),
1154 )))
1155 }
1156 }
1157 }
1158
1159 pub(crate) async fn reset_scratch(db_url: &str, owner_role: &str) -> Result<()> {
1160 use sqlx::Executor;
1161 use sqlx::postgres::PgPoolOptions;
1162 let pool = PgPoolOptions::new()
1163 .max_connections(1)
1164 .connect(db_url)
1165 .await?;
1166 // `owner_role` is validated `[A-Za-z0-9_]+` at config load, so interpolating
1167 // it into DDL is sound. It still goes through `format('%I')` inside the DO
1168 // block for the quoting Postgres expects on an identifier.
1169 let sql = format!(
1170 r#"
1171 DO $$
1172 DECLARE s text;
1173 BEGIN
1174 -- The dump restores objects owned by the prod role and re-grants to
1175 -- it (`ALTER ... OWNER TO {owner_role}`), which errors if the role
1176 -- is absent — superuser does not imply the role exists. Create it
1177 -- NOLOGIN: the scratch DB needs the role as an *owner* only, never
1178 -- as a connecting identity. Idempotent, so a re-reset is a no-op.
1179 IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{owner_role}') THEN
1180 EXECUTE format('CREATE ROLE %I NOLOGIN', '{owner_role}');
1181 END IF;
1182
1183 -- Drop every non-system schema, not just public — migrations create
1184 -- custom schemas (e.g. tower_sessions) that survive `DROP SCHEMA
1185 -- public CASCADE` and then collide on the next migration run.
1186 FOR s IN
1187 SELECT nspname FROM pg_namespace
1188 WHERE nspname NOT LIKE 'pg_%'
1189 AND nspname NOT IN ('information_schema')
1190 LOOP
1191 EXECUTE format('DROP SCHEMA IF EXISTS %I CASCADE', s);
1192 END LOOP;
1193 EXECUTE 'CREATE SCHEMA public';
1194 -- Restore the pre-PG15 public-schema default on the throwaway
1195 -- scratch DB. Without this, the freshly-created public is owned by
1196 -- the connecting role (sando) with no grant to anyone else, so a
1197 -- migration's FK/trigger check that Postgres runs as a *restored*
1198 -- prod-owned table's owner ({owner_role} from the backup dump)
1199 -- fails with "permission denied for schema public". Granting to
1200 -- PUBLIC is role-agnostic and safe here — this DB is disposable and
1201 -- exists only to dry-run migrations.
1202 EXECUTE 'GRANT USAGE, CREATE ON SCHEMA public TO PUBLIC';
1203 -- PG15+: the new owner needs CREATE on public in its own right, not
1204 -- only via PUBLIC, for the restore's owner-scoped DDL.
1205 EXECUTE format('GRANT USAGE, CREATE ON SCHEMA public TO %I', '{owner_role}');
1206 END $$;
1207 "#
1208 );
1209 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(sql)))
1210 .await?;
1211 pool.close().await;
1212 Ok(())
1213 }
1214
1215 /// Startup assertion for the scratch cluster: the gates reset it, seed an owner
1216 /// role into it, and drop leftover test databases in it — none of which a plain
1217 /// unprivileged role can do. Historically these preconditions were satisfied by
1218 /// hand on fw13 (`ALTER ROLE sando SUPERUSER`, a hand-created `makenotwork`
1219 /// role) and nothing recorded that, so a rebuild elsewhere would fail one gate
1220 /// at a time with an opaque permissions error. Assert once, at boot, loudly.
1221 ///
1222 /// Not part of `--check-config`: that path is pure by design (no DB, no
1223 /// network), and a green there must mean "this build understands its config",
1224 /// not "the cluster is reachable".
1225 pub async fn preflight_scratch_privileges(db_url: &str) -> Result<()> {
1226 use sqlx::postgres::PgPoolOptions;
1227 let pool = PgPoolOptions::new()
1228 .max_connections(1)
1229 .connect(db_url)
1230 .await
1231 .context("connecting to scratch_db_url for the startup privilege check")?;
1232 let (is_super, can_signal): (bool, bool) = sqlx::query_as(
1233 "SELECT rolsuper, pg_catalog.pg_has_role(current_user, 'pg_signal_backend', 'USAGE')
1234 FROM pg_roles WHERE rolname = current_user",
1235 )
1236 .fetch_one(&pool)
1237 .await?;
1238 pool.close().await;
1239 anyhow::ensure!(
1240 is_super || can_signal,
1241 "the scratch_db_url role has neither SUPERUSER nor pg_signal_backend; migration_dry_run \
1242 and cargo_test cannot reset the scratch DB or clear stale test databases. Grant one:\n \
1243 ALTER ROLE <role> SUPERUSER; -- what fw13 uses\n \
1244 GRANT pg_signal_backend TO <role>; -- narrower: terminate only, cannot drop \
1245 foreign-owned databases",
1246 );
1247 if !is_super {
1248 tracing::warn!(
1249 "scratch role has pg_signal_backend but not SUPERUSER: stale test databases owned by \
1250 another role cannot be dropped, and the scratch owner role cannot be created if absent"
1251 );
1252 }
1253 Ok(())
1254 }
1255
1256 /// Best-effort cleanup of stale per-test database clones (`mnw_test_<uuid>`)
1257 /// left behind by a previously-killed `cargo_test` run.
1258 ///
1259 /// Drops **foreign-owned leftovers too**, which is why the daemon asserts
1260 /// SUPERUSER at startup (`preflight_scratch_privileges`): `DROP DATABASE`
1261 /// requires ownership or superuser, and the `WITH (FORCE)` terminate requires
1262 /// superuser or `pg_signal_backend`. Without both, orphans from a run under a
1263 /// different role accumulate and degrade the gate — the failure this cleanup
1264 /// exists to prevent.
1265 ///
1266 /// OPERATIONAL HAZARD: fw13 runs one Postgres cluster shared with local `cargo
1267 /// test` as `max`, so a gate firing mid-local-test will force-drop that run's
1268 /// databases out from under it. That collision is known and tracked separately
1269 /// (give the gate its own cluster); until then, do not run local tests on fw13
1270 /// while a Sando gate is live.
1271 ///
1272 /// Deliberately **excludes the template** (`mnw_test_template_*`): the harness
1273 /// reuses it across runs when it's migration-current (skipping a full
1274 /// drop+migrate), so dropping it here would force a needless rebuild every
1275 /// gate run. Templates are bounded (one per role) and never accumulate, so
1276 /// leaving them is free. Never returns an error: a cleanup miss must not turn a
1277 /// deploy red.
1278 async fn clean_stale_test_dbs(db_url: &str) {
1279 use sqlx::Executor;
1280 use sqlx::postgres::PgPoolOptions;
1281 let pool = match PgPoolOptions::new()
1282 .max_connections(1)
1283 .connect(db_url)
1284 .await
1285 {
1286 Ok(p) => p,
1287 Err(e) => {
1288 tracing::warn!(error = %e, "stale test-db cleanup: could not connect; skipping");
1289 return;
1290 }
1291 };
1292 // Every per-test clone, whoever owns it. The ownership filter this used to
1293 // carry is what let foreign-owned orphans pile up; superuser (asserted at
1294 // startup) makes them droppable.
1295 let names: Vec<(String,)> = sqlx::query_as(
1296 "SELECT datname FROM pg_database
1297 WHERE datname LIKE 'mnw_test_%'
1298 AND datname NOT LIKE '%template%'",
1299 )
1300 .fetch_all(&pool)
1301 .await
1302 .unwrap_or_default();
1303 let count = names.len();
1304 for (name,) in names {
1305 // `name` comes straight from pg_database; quoting it is sufficient.
1306 if let Err(e) = pool
1307 .execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1308 "DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)"
1309 ))))
1310 .await
1311 {
1312 tracing::warn!(error = %e, db = %name, "stale test-db cleanup: drop failed");
1313 }
1314 }
1315 if count > 0 {
1316 tracing::info!(
1317 count,
1318 "stale test-db cleanup: dropped leftover mnw_test_* databases"
1319 );
1320 }
1321 pool.close().await;
1322 }
1323
1324 /// Build the restore shell line. Two pipelines we accept:
1325 /// *.sql -> psql -v ON_ERROR_STOP=1 $url < dump
1326 /// *.sql.gz -> set -o pipefail; gunzip -c dump | psql -v ON_ERROR_STOP=1 $url
1327 ///
1328 /// Two safety flags are load-bearing (CF4):
1329 /// - `ON_ERROR_STOP=1`: without it, psql exits 0 even when individual statements
1330 /// error, so a partial/corrupt restore would *pass* the gate.
1331 /// - `set -o pipefail`: without it a shell pipeline reports only the last
1332 /// command's status, so a `gunzip` failure on a truncated archive is masked by
1333 /// psql's exit. pipefail is a bash builtin (not POSIX sh), so the runner uses
1334 /// `bash -c`.
1335 fn restore_shell(db_url: &str, dump: &str) -> String {
1336 if std::path::Path::new(dump)
1337 .extension()
1338 .is_some_and(|ext| ext.eq_ignore_ascii_case("gz"))
1339 {
1340 format!(
1341 "set -o pipefail; gunzip -c {q} | psql -v ON_ERROR_STOP=1 {url}",
1342 q = shell_escape(dump),
1343 url = shell_escape(db_url),
1344 )
1345 } else {
1346 format!(
1347 "psql -v ON_ERROR_STOP=1 {url} < {q}",
1348 url = shell_escape(db_url),
1349 q = shell_escape(dump),
1350 )
1351 }
1352 }
1353
1354 async fn restore_dump(db_url: &str, dump: &str, log: &GateLog) -> Result<()> {
1355 // Split the password out of the URL and hand it to psql via PGPASSWORD, so it
1356 // never lands in argv (visible in /proc/<pid>/cmdline to any local user).
1357 // The sanitized URL — user/host/db, no secret — goes on the command line.
1358 let (sanitized, password) = split_pg_password(db_url);
1359 let shell = restore_shell(&sanitized, dump);
1360 // `bash` (not `sh`): `set -o pipefail` is a bash builtin. The restore runs
1361 // locally on the Sando host (fw13), which has bash.
1362 let mut cmd = Command::new("bash");
1363 cmd.arg("-c").arg(&shell);
1364 // kill_on_drop so the gate's wall-clock ceiling (dispatcher-level timeout on
1365 // migration_dry_run) can't orphan a wedged psql restore.
1366 cmd.kill_on_drop(true);
1367 if let Some(pw) = password {
1368 cmd.env("PGPASSWORD", pw);
1369 }
1370 // Streamed, not `.output()`: a prod-sized restore runs for minutes, and
1371 // psql's progress is the only thing an operator has to watch during it.
1372 let (_stdout, stderr, status) = log.run(&mut cmd).await?;
1373 anyhow::ensure!(
1374 status.success(),
1375 "restore failed: {}",
1376 String::from_utf8_lossy(&stderr),
1377 );
1378 Ok(())
1379 }
1380
1381 /// Split a `postgres://user:password@host/db` URL into its password-free form and
1382 /// the (percent-decoded) password. Returns the URL unchanged with `None` when
1383 /// there is no userinfo password. psql reads the password from `PGPASSWORD`, so
1384 /// keeping it off the command line removes the /proc exposure.
1385 fn split_pg_password(db_url: &str) -> (String, Option<String>) {
1386 let Some(after) = db_url.find("://").map(|i| i + 3) else {
1387 return (db_url.to_string(), None);
1388 };
1389 // The authority ends at the first '/', '?' or '#'; the password (if any) is
1390 // between the first ':' and the '@' within the userinfo of that authority.
1391 let authority_end = db_url[after..]
1392 .find(['/', '?', '#'])
1393 .map_or(db_url.len(), |i| after + i);
1394 let Some(at) = db_url[after..authority_end].find('@').map(|i| after + i) else {
1395 return (db_url.to_string(), None);
1396 };
1397 let userinfo = &db_url[after..at];
1398 let Some(colon) = userinfo.find(':') else {
1399 return (db_url.to_string(), None);
1400 };
1401 let password = percent_decode(&userinfo[colon + 1..]);
1402 let sanitized = format!(
1403 "{}{}{}",
1404 &db_url[..after],
1405 &userinfo[..colon],
1406 &db_url[at..]
1407 );
1408 (sanitized, Some(password))
1409 }
1410
1411 /// Minimal `%XX` percent-decode for a URL userinfo component. Non-escape bytes
1412 /// pass through; a malformed escape is left literal.
1413 fn percent_decode(s: &str) -> String {
1414 let b = s.as_bytes();
1415 let mut out = Vec::with_capacity(b.len());
1416 let mut i = 0;
1417 while i < b.len() {
1418 if b[i] == b'%'
1419 && i + 2 < b.len()
1420 && let (Some(h), Some(l)) = (hex_val(b[i + 1]), hex_val(b[i + 2]))
1421 {
1422 out.push((h << 4) | l);
1423 i += 3;
1424 } else {
1425 out.push(b[i]);
1426 i += 1;
1427 }
1428 }
1429 String::from_utf8_lossy(&out).into_owned()
1430 }
1431
1432 fn hex_val(c: u8) -> Option<u8> {
1433 match c {
1434 b'0'..=b'9' => Some(c - b'0'),
1435 b'a'..=b'f' => Some(c - b'a' + 10),
1436 b'A'..=b'F' => Some(c - b'A' + 10),
1437 _ => None,
1438 }
1439 }
1440
1441 pub(crate) async fn run_migrator(db_url: &str, dir: &std::path::Path) -> Result<()> {
1442 use sqlx::postgres::PgPoolOptions;
1443 let pool = PgPoolOptions::new()
1444 .max_connections(1)
1445 .connect(db_url)
1446 .await?;
1447 let migrator = sqlx::migrate::Migrator::new(dir).await?;
1448 migrator.run(&pool).await?;
1449 pool.close().await;
1450 Ok(())
1451 }
1452
1453 fn shell_escape(s: &str) -> String {
1454 format!("'{}'", s.replace('\'', "'\\''"))
1455 }
1456
1457 // ---- code_smoke ----
1458
1459 /// A 32+ char throwaway signing secret for the `code_smoke` boot. The server
1460 /// only enforces length (>= 32) outside production, and code_smoke's loopback
1461 /// `HOST_URL` keeps it in dev mode, so the value is irrelevant beyond that.
1462 const CODE_SMOKE_SIGNING_SECRET: &str = "sando-code-smoke-dummy-signing-secret-0000000000";
1463
1464 /// Seconds to wait for the real server to come up and serve `GET /health`
1465 /// during `code_smoke`. Longer than `boot_smoke`'s 3s: this boots the *full*
1466 /// server (config, pool, session store, webauthn, doc load, app build), not the
1467 /// minimal no-DB smoke server. The whole gate is also bounded by
1468 /// `gate_timeout_secs` at the dispatcher.
1469 const CODE_SMOKE_READY_SECS: u64 = 30;
1470
1471 /// `code_smoke` — the first host gate. Boots the freshly-built binary against a
1472 /// throwaway *empty* DB it migrates from scratch and seeds the example catalog
1473 /// into, then proves the real server serves `GET /health` against that
1474 /// nonempty DB. Fast and infra-light (one local Postgres, no prod-dump restore,
1475 /// no scratch-role reset, no external services), so a green here proves the
1476 /// code is sound and isolates a later `cargo_test`/`migration_dry_run` red as an
1477 /// environment problem rather than a code one.
1478 ///
1479 /// Reuses existing binary entrypoints, so the server needs no smoke-specific
1480 /// mode: `<bin> --seed-examples` loads config, connects, migrates from scratch,
1481 /// seeds, and exits; a plain `<bin>` then serves the real app. Both run with CWD
1482 /// at the server crate root so `docs/business/assumptions.toml` + `site-docs/`
1483 /// resolve (a missing assumptions file panics real startup), and with a loopback
1484 /// `HOST_URL` so config stays in dev mode (no CDN/S3/signing-secret prod
1485 /// enforcement). The seed's host allowlist already admits `127.0.0.1`, and the
1486 /// fresh DB trivially satisfies its no-real-users guard.
1487 async fn code_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
1488 let log = GateLog::open(ctx, run_id, GateKind::CodeSmoke).await;
1489 let outcome = code_smoke_inner(ctx, &log).await;
1490 log.close().await;
1491 outcome.map(|o| o.with_log_ref(ctx.log_ref(GateKind::CodeSmoke)))
1492 }
1493
1494 /// The staged interior of [`code_smoke`], writing every step through the gate's
1495 /// live log. Same split as [`migration_dry_run_inner`]: the caller owns the sink
1496 /// and attaches the `log_ref`.
1497 async fn code_smoke_inner(ctx: &GateCtx, log: &GateLog) -> Result<GateOutcome> {
1498 let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() else {
1499 log.line("scratch_db_url unset in daemon config\n").await;
1500 return Ok(GateOutcome::blocked(GateBlocker::ScratchDbUrlUnset));
1501 };
1502
1503 // The staged binary (set by build_and_run_host before gating). code_smoke
1504 // runs first among the host gates, but staging precedes all gating, so the
1505 // artifact path is already recorded.
1506 let bin: Option<(String,)> =
1507 sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
1508 .bind(&ctx.cfg.id)
1509 .bind(&ctx.version)
1510 .fetch_optional(&ctx.pool)
1511 .await?;
1512 let Some((bin,)) = bin else {
1513 return Ok(GateOutcome::blocked(GateBlocker::ArtifactMissing {
1514 version: ctx.version.clone(),
1515 }));
1516 };
1517
1518 // Frontend builds, before anything else: they need no DB and no staged
1519 // binary, and a `tsc` error is the one failure the Rust build deliberately
1520 // swallows (both MNW build scripts emit `cargo::warning` and succeed against
1521 // a stale `static/dist/`). Failing here is what stops the deploy rsyncing
1522 // the previous build's bundle.
1523 if let Some(outcome) = code_smoke_frontends(ctx, log).await {
1524 return Ok(outcome);
1525 }
1526
1527 // Docs integrity, first and cheapest: run the staged binary's DB-free
1528 // `MNW_CHECK_DOCS` mode before creating the throwaway DB. A broken internal
1529 // docs link (a `[..](x.md)` resolving to a slug no page serves) fails here
1530 // in well under a second instead of after a full migrate+seed+boot, and a
1531 // rotted link never reaches prod as a live 404. Collisions are reported by
1532 // the check but do not fail it; only broken links do.
1533 if let Some(outcome) = code_smoke_docs_check(ctx, &bin, log).await {
1534 return Ok(outcome);
1535 }
1536
1537 let dbname = code_smoke_db_name(&ctx.version);
1538 let maintenance_url = pg_url_with_dbname(scratch_url, "postgres");
1539 let throwaway_url = pg_url_with_dbname(scratch_url, &dbname);
1540
1541 // Create the throwaway DB (dropping any stale one from a killed prior run).
1542 log.line(&format!("---- createdb {dbname} ----\n")).await;
1543 if let Err(e) = pg_create_db(&maintenance_url, &dbname).await {
1544 let reason = format!("createdb {dbname}: {e}");
1545 log.line(&reason).await;
1546 return Ok(GateOutcome::failed(GateFailure::CodeSmokeSetup { reason }));
1547 }
1548
1549 // Everything past createdb must drop the DB on the way out, pass or fail.
1550 let outcome = code_smoke_body(ctx, &bin, &throwaway_url, log).await;
1551
1552 log.line(&format!("\n---- dropdb {dbname} ----\n")).await;
1553 if let Err(e) = pg_drop_db(&maintenance_url, &dbname).await {
1554 // A teardown miss must not turn a passing gate red — log it and move on.
1555 // The next run's createdb drops it first anyway.
1556 tracing::warn!(error = %e, db = %dbname, "code_smoke: dropdb failed; next run will reclaim it");
1557 log.line(&format!("dropdb warning (non-fatal): {e}")).await;
1558 }
1559
1560 Ok(outcome)
1561 }
1562
1563 /// Compile every configured `frontend_build` in the worktree.
1564 ///
1565 /// Returns `Some(failed)` on the first project that does not build; `None` when
1566 /// all of them do (or none are configured). Output streams to `log` either way.
1567 ///
1568 /// `npm ci` runs only when `node_modules` is absent. Usually it is not: the app
1569 /// build script installed it during the `cargo build` that produced the artifact
1570 /// this gate is about to smoke, so the common path here is just `npm run build`
1571 /// against a warm install — seconds. The install branch covers the gate running
1572 /// against a worktree whose build script was skipped or failed at `npm ci`, and
1573 /// it is as fatal as a compile failure, because the alternative is compiling
1574 /// against whatever some earlier sha installed.
1575 ///
1576 /// Unlike the app build scripts, nothing here is best-effort. That asymmetry is
1577 /// the point of the gate.
1578 async fn code_smoke_frontends(ctx: &GateCtx, log: &GateLog) -> Option<GateOutcome> {
1579 if ctx.cfg.frontend_builds.is_empty() {
1580 return None;
1581 }
1582 let worktree = match ctx.worktree_for(GateKind::CodeSmoke) {
1583 Ok(w) => w.to_path_buf(),
1584 Err(outcome) => return Some(outcome),
1585 };
1586 for fe in &ctx.cfg.frontend_builds {
1587 let dir = worktree.join(&fe.dir);
1588 let label = fe.dir.display().to_string();
1589 log.line(&format!("---- frontend build ({label}) ----\n"))
1590 .await;
1591
1592 if !dir.is_dir() {
1593 // An older sha predating the frontend, mid-bisect. Skipping keeps
1594 // sando able to rebuild history; the log says so out loud.
1595 log.line(&format!("{label} absent from this worktree; skipping\n"))
1596 .await;
1597 continue;
1598 }
1599
1600 if !dir.join("node_modules").is_dir()
1601 && let Some(outcome) = run_npm(&dir, &label, &["ci"], "npm ci", ctx, log).await
1602 {
1603 return Some(outcome);
1604 }
1605
1606 if let Some(outcome) = run_npm(
1607 &dir,
1608 &label,
1609 &["run", &fe.script],
1610 &format!("npm run {}", fe.script),
1611 ctx,
1612 log,
1613 )
1614 .await
1615 {
1616 return Some(outcome);
1617 }
1618 }
1619 None
1620 }
1621
1622 /// One `npm` invocation for [`code_smoke_frontends`], bounded by the gate
1623 /// timeout so a wedged install cannot hold the whole pipeline (the enclosing
1624 /// `code_smoke` ceiling would catch it eventually, but this attributes the
1625 /// failure to the project that hung).
1626 async fn run_npm(
1627 dir: &std::path::Path,
1628 label: &str,
1629 args: &[&str],
1630 what: &str,
1631 ctx: &GateCtx,
1632 log: &GateLog,
1633 ) -> Option<GateOutcome> {
1634 log.line(&format!("$ {what}\n")).await;
1635 let mut cmd = tokio::process::Command::new("npm");
1636 cmd.args(args).current_dir(dir).kill_on_drop(true);
1637 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
1638 // On the timeout branch the whole `run` future is dropped, which drops the
1639 // child; `kill_on_drop` is what turns that into an actual kill.
1640 let status = match tokio::time::timeout(ceiling, log.run(&mut cmd)).await {
1641 Ok(Ok((_stdout, _stderr, status))) => status,
1642 Ok(Err(e)) => {
1643 // A missing `npm` lands here. Fatal, not skipped: a build host
1644 // without Node cannot produce the bundle the release serves, and
1645 // silently passing is how the stale bundle shipped in the first place.
1646 log.line(&format!("{what} could not be spawned: {e}\n"))
1647 .await;
1648 return Some(GateOutcome::failed(GateFailure::SpawnFailed {
1649 message: format!("{what} in {label}: {e}"),
1650 }));
1651 }
1652 Err(_elapsed) => {
1653 log.line(&format!(
1654 "{what} timed out after {}s\n",
1655 ctx.cfg.gate_timeout_secs
1656 ))
1657 .await;
1658 return Some(GateOutcome::failed(GateFailure::CodeSmokeFrontend {
1659 dir: label.to_string(),
1660 exit_code: None,
1661 }));
1662 }
1663 };
1664 if status.success() {
1665 return None;
1666 }
1667 Some(GateOutcome::failed(GateFailure::CodeSmokeFrontend {
1668 dir: label.to_string(),
1669 exit_code: status.code(),
1670 }))
1671 }
1672
1673 /// Run the staged binary's DB-free docs integrity check (`MNW_CHECK_DOCS=1`).
1674 ///
1675 /// Returns `Some(failed)` if the check reports broken links, cannot be spawned,
1676 /// or overruns its ceiling; `None` when the docs are clean. Output streams to
1677 /// `log` either way. The 60s ceiling backstops the case where the staged binary
1678 /// predates the flag and would fall through to a normal (DB-needing) boot and
1679 /// hang.
1680 async fn code_smoke_docs_check(ctx: &GateCtx, bin: &str, log: &GateLog) -> Option<GateOutcome> {
1681 let server_dir = match ctx.worktree_for(GateKind::CodeSmoke) {
1682 Ok(w) => w.join("server"),
1683 Err(outcome) => return Some(outcome),
1684 };
1685 log.line("---- docs check (MNW_CHECK_DOCS) ----\n").await;
1686 let mut cmd = tokio::process::Command::new(bin);
1687 cmd.env("MNW_CHECK_DOCS", "1")
1688 .current_dir(&server_dir)
1689 .kill_on_drop(true);
1690 let (stdout, _stderr, status) =
1691 match tokio::time::timeout(std::time::Duration::from_mins(1), log.run(&mut cmd)).await {
1692 Ok(Ok(out)) => out,
1693 Ok(Err(e)) => {
1694 log.line(&format!("docs check spawn failed: {e}\n")).await;
1695 return Some(GateOutcome::failed(GateFailure::SpawnFailed {
1696 message: e.to_string(),
1697 }));
1698 }
1699 Err(_elapsed) => {
1700 let reason =
1701 "docs check timed out after 60s (staged binary may predate MNW_CHECK_DOCS)"
1702 .to_string();
1703 log.line(&format!("{reason}\n")).await;
1704 return Some(GateOutcome::failed(GateFailure::CodeSmokeSetup { reason }));
1705 }
1706 };
1707 if status.success() {
1708 return None;
1709 }
1710 Some(GateOutcome::failed(GateFailure::CodeSmokeDocs {
1711 broken: parse_check_docs_broken_count(&stdout),
1712 }))
1713 }
1714
1715 /// Best-effort parse of the broken-link count from the `MNW_CHECK_DOCS` sentinel
1716 /// line (`MNW_CHECK_DOCS: N broken link(s)`). Returns 0 if absent — the failure
1717 /// still stands, only the summary count is unknown.
1718 fn parse_check_docs_broken_count(stdout: &[u8]) -> u32 {
1719 let text = String::from_utf8_lossy(stdout);
1720 for line in text.lines() {
1721 if let Some(rest) = line.strip_prefix("MNW_CHECK_DOCS:") {
1722 for tok in rest.split_whitespace() {
1723 if let Ok(n) = tok.parse::<u32>() {
1724 return n;
1725 }
1726 }
1727 }
1728 }
1729 0
1730 }
1731
1732 /// The createdb-to-dropdb interior of `code_smoke`: migrate+seed, then boot and
1733 /// probe. Returns the outcome without a `log_ref` (the caller attaches it after
1734 /// teardown). Never returns `Err` — spawn/child failures map to typed outcomes.
1735 async fn code_smoke_body(ctx: &GateCtx, bin: &str, db_url: &str, log: &GateLog) -> GateOutcome {
1736 let server_dir = match ctx.worktree_for(GateKind::CodeSmoke) {
1737 Ok(w) => w.join("server"),
1738 Err(outcome) => return outcome,
1739 };
1740
1741 // Phase 1: migrate-from-scratch + seed. `--seed-examples` loads config,
1742 // connects, runs migrations against the empty DB, seeds the catalog, exits.
1743 // A non-zero exit here is the "code is unsound" signal (broken migration,
1744 // seed error, or config-load failure).
1745 log.line("---- migrate + seed (--seed-examples) ----\n")
1746 .await;
1747 let mut seed_cmd = tokio::process::Command::new(bin);
1748 seed_cmd.arg("--seed-examples").current_dir(&server_dir);
1749 code_smoke_env(&mut seed_cmd, ctx, db_url);
1750 seed_cmd.env("ALLOW_EXAMPLE_SEED", "1").kill_on_drop(true);
1751 let seed_status = match log.run(&mut seed_cmd).await {
1752 Ok((_stdout, _stderr, status)) => status,
1753 Err(e) => {
1754 return GateOutcome::failed(GateFailure::SpawnFailed {
1755 message: e.to_string(),
1756 });
1757 }
1758 };
1759 if !seed_status.success() {
1760 return GateOutcome::failed(GateFailure::CodeSmokeSeed {
1761 exit_code: seed_status.code(),
1762 });
1763 }
1764
1765 // Phase 2: boot the real server against the now-migrated + seeded DB and
1766 // assert both startup signals: it logs `listening` (emitted just before the
1767 // socket bind) AND serves GET /health with a 200. The full stdout/stderr is
1768 // persisted for the operator either way.
1769 log.line("\n---- boot + probe /health ----\n").await;
1770 let mut serve_cmd = tokio::process::Command::new(bin);
1771 serve_cmd.current_dir(&server_dir);
1772 code_smoke_env(&mut serve_cmd, ctx, db_url);
1773 serve_cmd
1774 .stdout(std::process::Stdio::piped())
1775 .stderr(std::process::Stdio::piped())
1776 .kill_on_drop(true);
1777 let mut child = match serve_cmd.spawn() {
1778 Ok(c) => c,
1779 Err(e) => {
1780 return GateOutcome::failed(GateFailure::SpawnFailed {
1781 message: e.to_string(),
1782 });
1783 }
1784 };
1785
1786 // Stream stdout/stderr into the gate log (and out as chunk events) while
1787 // the probe loop runs below; the tasks finish when the pipes close (child
1788 // exits or is killed). The buffers they return are what the `listening`
1789 // assertion reads.
1790 let (stdout_task, stderr_task) = log.drain_pipes(&mut child);
1791
1792 let probe_timeout = std::time::Duration::from_millis(500);
1793 let started = std::time::Instant::now();
1794 let window = std::time::Duration::from_secs(CODE_SMOKE_READY_SECS);
1795 let mut probe_ok_after: Option<u32> = None;
1796 let mut last_probe_err = "never responded".to_string();
1797 let mut early_exit = None;
1798 while started.elapsed() < window {
1799 if let Ok(Some(status)) = child.try_wait() {
1800 early_exit = Some(status);
1801 break;
1802 }
1803 match tokio::time::timeout(probe_timeout, probe_health(ctx.cfg.code_smoke_port)).await {
1804 Ok(Ok(())) => {
1805 probe_ok_after = Some(started.elapsed().as_millis() as u32);
1806 break;
1807 }
1808 Ok(Err(e)) => last_probe_err = e,
1809 Err(_) => last_probe_err = "probe timed out".to_string(),
1810 }
1811 tokio::time::sleep(std::time::Duration::from_millis(250)).await;
1812 }
1813
1814 let exit = match early_exit {
1815 Some(status) => Some(status),
1816 None => {
1817 let e = child.try_wait().ok().flatten();
1818 if e.is_none() {
1819 let _ = child.kill().await;
1820 }
1821 e
1822 }
1823 };
1824 // Keep the serve run's own output so the `listening`-log assertion checks it
1825 // (not the seed run's, which exits before binding). The bytes are already in
1826 // the gate log; these buffers exist only for the assertion.
1827 let serve_stdout = stdout_task.await.unwrap_or_default();
1828 let serve_stderr = stderr_task.await.unwrap_or_default();
1829 let logged_listening =
1830 bytes_contain(&serve_stdout, b"listening") || bytes_contain(&serve_stderr, b"listening");
1831
1832 match (exit, probe_ok_after) {
1833 // Exited on its own within the window — panic / config error / bind fail.
1834 (Some(status), _) => GateOutcome::failed(classify::classify_boot_smoke(status.code())),
1835 // Stayed up and served /health. Assert both required startup signals:
1836 // the `listening` bind log AND the /health 200.
1837 (None, Some(after_ms)) if logged_listening => {
1838 GateOutcome::passed(PassNote::HealthyProbe { after_ms })
1839 }
1840 // Served /health but the `listening` log never appeared.
1841 (None, Some(_)) => GateOutcome::failed(GateFailure::CodeSmokeNoListeningLog),
1842 // Stayed up but never served /health — started, not ready.
1843 (None, None) => GateOutcome::failed(GateFailure::BootHealthProbeFailed {
1844 last_error: last_probe_err,
1845 }),
1846 }
1847 }
1848
1849 /// Substring search over raw bytes (the server's log output), for the
1850 /// `code_smoke` startup-log assertion. Avoids a lossy UTF-8 conversion of the
1851 /// whole buffer just to run `str::contains`.
1852 fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool {
1853 if needle.is_empty() || haystack.len() < needle.len() {
1854 return needle.is_empty();
1855 }
1856 haystack.windows(needle.len()).any(|w| w == needle)
1857 }
1858
1859 /// Apply the minimal env every `code_smoke` invocation shares: point the binary
1860 /// at the throwaway DB, force loopback (dev-mode config, no prod enforcement),
1861 /// hand it a dummy signing secret, and disable file scanning (no AV/YARA on the
1862 /// build host). The worktree is a clean git checkout, so no stray `.env` shadows
1863 /// these (and dotenvy never overrides already-set vars).
1864 ///
1865 /// This list has to carry EVERY var the server's `Config::from_env` treats as
1866 /// mandatory, because `code_smoke` is the only gate that reaches that function
1867 /// at all: `boot_smoke` and the docs check both short-circuit in `main` before
1868 /// config is loaded. So when the server makes a new var required, this is where
1869 /// it has to be answered, and nothing connects the two lists automatically.
1870 ///
1871 /// That has already bitten once. `CDN_BASE_URL` became mandatory on 2026-07-30
1872 /// (server 3e3b1d15, closing a cover URL that could expire), 13 days after this
1873 /// function was written, and `code_smoke` failed with `MissingCdnBaseUrl` the
1874 /// first time it ran on the host — which was 2026-08-03, because the gate was
1875 /// configured in the repo and absent from `/etc/sando` in between.
1876 ///
1877 /// The values are deliberately throwaway. The gate asks whether this code can
1878 /// migrate, seed, boot and serve; whether a given deployment's env is complete
1879 /// is the `config_check_env_file` guard's job, on the node, against that node's
1880 /// real env file.
1881 ///
1882 /// `app.code_smoke_env` is prepended to all of this, for vars a product needs
1883 /// that sando has no business knowing about. The fixed set overwrites it on a
1884 /// collision, so nothing in a config file can redirect the gate off its own
1885 /// throwaway DB.
1886 fn code_smoke_env(cmd: &mut tokio::process::Command, ctx: &GateCtx, db_url: &str) {
1887 // `localhost`, not `127.0.0.1`, and the distinction is load-bearing. The
1888 // server derives its WebAuthn relying-party id from HOST_URL's host, and
1889 // `WebauthnBuilder::new` validates that id against `Url::domain()` — which
1890 // is `None` for an IP literal, so an origin of `http://127.0.0.1:<port>`
1891 // fails with WebauthnError::Configuration before the server ever binds. It
1892 // is a real ceiling on the gate, not a preference: no IP-literal origin can
1893 // boot this binary. `localhost` is a domain, and matches the derived rp_id.
1894 //
1895 // HOST stays 127.0.0.1: that is the bind address, and the gate probes the
1896 // loopback address directly, so only the advertised origin changes.
1897 let origin = format!("http://localhost:{}", ctx.cfg.code_smoke_port);
1898 // Project-supplied extras go on first so the fixed set below overwrites any
1899 // key they collide on: what points this run at its throwaway DB and its
1900 // loopback port is not negotiable from a config file.
1901 cmd.envs(&ctx.cfg.code_smoke_env);
1902 cmd.env("DATABASE_URL", db_url)
1903 .env("HOST", "127.0.0.1")
1904 .env("PORT", ctx.cfg.code_smoke_port.to_string())
1905 .env("HOST_URL", &origin)
1906 // Required unconditionally by Config::from_env. Pointing it at the smoke
1907 // server's own origin keeps every rendered media URL resolvable within
1908 // the gate; no request is ever made to it.
1909 .env("CDN_BASE_URL", &origin)
1910 .env("SIGNING_SECRET", CODE_SMOKE_SIGNING_SECRET)
1911 .env("SCAN_ENABLED", "false")
1912 .env("INSECURE_COOKIES", "1");
1913 }
1914
1915 /// The throwaway smoke DB name for a version: `sando_code_smoke_<version>` with
1916 /// every non-alphanumeric char folded to `_` and lowercased, capped at Postgres'
1917 /// 63-byte identifier limit. Sanitized to `[a-z0-9_]` so it's safe to quote into
1918 /// DDL. Deterministic per version, so a stale DB from a killed run is reclaimed
1919 /// by the next run's `DROP DATABASE IF EXISTS` rather than accumulating.
1920 fn code_smoke_db_name(version: &Version) -> String {
1921 let mut name = String::from("sando_code_smoke_");
1922 for c in version.to_string().chars() {
1923 name.push(if c.is_ascii_alphanumeric() {
1924 c.to_ascii_lowercase()
1925 } else {
1926 '_'
1927 });
1928 }
1929 name.truncate(63);
1930 name
1931 }
1932
1933 /// Rewrite a `postgres://` URL to point at database `dbname`, preserving scheme,
1934 /// userinfo, host/port, and any query (e.g. the socket `?host=/var/run/postgresql`
1935 /// form) + fragment. Used to derive the maintenance connection (`postgres`) and
1936 /// the throwaway smoke DB URL from the configured `scratch_db_url`.
1937 fn pg_url_with_dbname(url: &str, dbname: &str) -> String {
1938 let Some(after_scheme) = url.find("://").map(|i| i + 3) else {
1939 return url.to_string();
1940 };
1941 let rest = &url[after_scheme..];
1942 // Authority ends at the first '/', '?' or '#'; whatever follows is the
1943 // path (the old dbname) plus an optional query/fragment we must keep.
1944 let auth_end = rest.find(['/', '?', '#']).unwrap_or(rest.len());
1945 let authority = &rest[..auth_end];
1946 let tail = &rest[auth_end..];
1947 let query_and_frag = match tail.find(['?', '#']) {
1948 Some(i) => &tail[i..],
1949 None => "",
1950 };
1951 format!(
1952 "{}{}/{}{}",
1953 &url[..after_scheme],
1954 authority,
1955 dbname,
1956 query_and_frag
1957 )
1958 }
1959
1960 /// Create the throwaway smoke DB on the cluster `maintenance_url` points at,
1961 /// dropping any stale one first. `dbname` is sanitized to `[a-z0-9_]` by
1962 /// `code_smoke_db_name`, so quoting it is sufficient. `CREATE DATABASE` cannot
1963 /// run inside a transaction, so these go through the simple-query protocol (a
1964 /// raw `&str` execute), matching `reset_scratch`.
1965 async fn pg_create_db(maintenance_url: &str, dbname: &str) -> Result<()> {
1966 use sqlx::Executor;
1967 use sqlx::postgres::PgPoolOptions;
1968 let pool = PgPoolOptions::new()
1969 .max_connections(1)
1970 .connect(maintenance_url)
1971 .await?;
1972 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1973 "DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE)"
1974 ))))
1975 .await?;
1976 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1977 "CREATE DATABASE \"{dbname}\""
1978 ))))
1979 .await?;
1980 pool.close().await;
1981 Ok(())
1982 }
1983
1984 /// Drop the throwaway smoke DB, forcing off any lingering connection (the killed
1985 /// server's pool). Best-effort at the call site — a failure is logged, not fatal.
1986 async fn pg_drop_db(maintenance_url: &str, dbname: &str) -> Result<()> {
1987 use sqlx::Executor;
1988 use sqlx::postgres::PgPoolOptions;
1989 let pool = PgPoolOptions::new()
1990 .max_connections(1)
1991 .connect(maintenance_url)
1992 .await?;
1993 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
1994 "DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE)"
1995 ))))
1996 .await?;
1997 pool.close().await;
1998 Ok(())
1999 }
2000
2001 async fn boot_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
2002 let bin: Option<(String,)> =
2003 sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?")
2004 .bind(&ctx.cfg.id)
2005 .bind(&ctx.version)
2006 .fetch_optional(&ctx.pool)
2007 .await?;
2008 let Some((bin,)) = bin else {
2009 return Ok(GateOutcome::blocked(GateBlocker::ArtifactMissing {
2010 version: ctx.version.clone(),
2011 }));
2012 };
2013
2014 // Readiness smoke: start the binary and confirm it serves `GET /health`
2015 // within the window, not merely that the process stays up. Panics in main,
2016 // missing config, and port-bind failures still surface as an early exit; a
2017 // process that comes up but never serves /health is now its own failure.
2018 //
2019 // The server requires DATABASE_URL or it panics on config load before
2020 // we can observe anything. We point it at the scratch DB (already
2021 // migrated by the build step and refreshed by migration_dry_run if
2022 // that gate ran first). SCAN_ENABLED=false skips loading YARA rules
2023 // from /opt/makenotwork/yara-rules which doesn't exist on the build
2024 // host. SANDO_BOOT_SMOKE_PORT tells the smoke server which loopback port
2025 // to bind so we know where to probe. Other config has sane optional defaults.
2026 let mut cmd = tokio::process::Command::new(&bin);
2027 cmd.env("SANDO_BOOT_SMOKE", "1")
2028 .env("SANDO_BOOT_SMOKE_PORT", ctx.cfg.boot_smoke_port.to_string())
2029 .env("SCAN_ENABLED", "false")
2030 .stdout(std::process::Stdio::piped())
2031 .stderr(std::process::Stdio::piped())
2032 .kill_on_drop(true);
2033 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
2034 cmd.env("DATABASE_URL", scratch_url);
2035 }
2036 let log_path = ctx.log_path(GateKind::BootSmoke);
2037 let log_ref = ctx.log_ref(GateKind::BootSmoke);
2038 let mut child = match cmd.spawn() {
2039 Ok(c) => c,
2040 Err(e) => {
2041 // Spawn failures get a one-off log line via LiveLog so the
2042 // on-disk file still exists for `GET /logs/...`.
2043 let mut log = LiveLog::open(log_path, gate_chunk_cb(ctx.events.clone(), run_id)).await;
2044 log.write_chunk(format!("spawn: {e}\n").as_bytes()).await;
2045 log.close().await;
2046 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
2047 message: e.to_string(),
2048 })
2049 .with_log_ref(log_ref));
2050 }
2051 };
2052
2053 // The boot smoke window is 3s. Drain stdout/stderr concurrently through
2054 // a shared LiveLog sink so the operator sees panics/log lines stream in
2055 // real time before the kill, AND the on-disk log gets the full byte
2056 // stream for post-mortem reads. The drainers exit when their pipe
2057 // closes — which happens when the child exits naturally or after kill.
2058 let log = std::sync::Arc::new(tokio::sync::Mutex::new(
2059 LiveLog::open(log_path, gate_chunk_cb(ctx.events.clone(), run_id)).await,
2060 ));
2061 let stdout_task = tokio::spawn(stream_into_log(child.stdout.take(), log.clone()));
2062 let stderr_task = tokio::spawn(stream_into_log(child.stderr.take(), log.clone()));
2063
2064 // Poll readiness across the 3s window instead of a flat sleep: GET /health
2065 // must return 2xx. A crash mid-window short-circuits to the exit-code
2066 // failure path (try_wait below); a process that stays up but never serves
2067 // /health is a distinct readiness failure.
2068 let probe_timeout = std::time::Duration::from_millis(500);
2069 let started = std::time::Instant::now();
2070 let window = std::time::Duration::from_secs(3);
2071 let mut probe_ok_after: Option<u32> = None;
2072 let mut last_probe_err = "never responded".to_string();
2073 let mut early_exit = None;
2074 while started.elapsed() < window {
2075 if let Some(status) = child.try_wait()? {
2076 early_exit = Some(status);
2077 break;
2078 }
2079 match tokio::time::timeout(probe_timeout, probe_health(ctx.cfg.boot_smoke_port)).await {
2080 Ok(Ok(())) => {
2081 probe_ok_after = Some(started.elapsed().as_millis() as u32);
2082 break;
2083 }
2084 Ok(Err(e)) => last_probe_err = e,
2085 Err(_) => last_probe_err = "probe timed out".to_string(),
2086 }
2087 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
2088 }
2089
2090 // Stop the child unless it already exited, then drain the log tasks.
2091 let exit = match early_exit {
2092 Some(status) => Some(status),
2093 None => {
2094 let e = child.try_wait()?;
2095 if e.is_none() {
2096 let _ = child.kill().await;
2097 }
2098 e
2099 }
2100 };
2101 // The streamed bytes already landed in the live log and the on-disk file for
2102 // the post-mortem reader. Drain the join handles to avoid hangs.
2103 let _ = stdout_task.await;
2104 let _ = stderr_task.await;
2105 // Unique owner of the Arc at this point (both tasks dropped their clones).
2106 if let Ok(mutex) = std::sync::Arc::try_unwrap(log) {
2107 mutex.into_inner().close().await;
2108 }
2109
2110 match (exit, probe_ok_after) {
2111 // Exited on its own within the window — a crash/panic/bind failure.
2112 (Some(status), _) => {
2113 let failure = classify::classify_boot_smoke(status.code());
2114 Ok(GateOutcome::failed(failure).with_log_ref(log_ref))
2115 }
2116 // Stayed up and served /health — readiness proven.
2117 (None, Some(after_ms)) => {
2118 Ok(GateOutcome::passed(PassNote::HealthyProbe { after_ms }).with_log_ref(log_ref))
2119 }
2120 // Stayed up but never served /health — started, not ready.
2121 (None, None) => Ok(GateOutcome::failed(GateFailure::BootHealthProbeFailed {
2122 last_error: last_probe_err,
2123 })
2124 .with_log_ref(log_ref)),
2125 }
2126 }
2127
2128 /// One readiness probe of the boot-smoke server: connect to `127.0.0.1:port`
2129 /// and `GET /health`, returning `Ok(())` only on a `200`. A hand-rolled HTTP/1.0
2130 /// request over a raw `TcpStream` keeps the outbound probe dependency-free
2131 /// (reqwest is dev-only); the smoke server serves the one route over axum, which
2132 /// speaks 1.0. `Err` carries a short reason for the operator's failure note. The
2133 /// caller wraps each call in a timeout.
2134 async fn probe_health(port: u16) -> std::result::Result<(), String> {
2135 use tokio::io::{AsyncReadExt, AsyncWriteExt};
2136 let mut stream = tokio::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, port))
2137 .await
2138 .map_err(|e| format!("connect: {e}"))?;
2139 stream
2140 .write_all(b"GET /health HTTP/1.0\r\nHost: localhost\r\nConnection: close\r\n\r\n")
2141 .await
2142 .map_err(|e| format!("write: {e}"))?;
2143 let mut buf = Vec::new();
2144 stream
2145 .read_to_end(&mut buf)
2146 .await
2147 .map_err(|e| format!("read: {e}"))?;
2148 let text = String::from_utf8_lossy(&buf);
2149 let status_line = text.lines().next().unwrap_or("");
2150 if status_line.contains(" 200 ") {
2151 Ok(())
2152 } else {
2153 Err(format!("unexpected status line: {status_line:?}"))
2154 }
2155 }
2156
2157 /// Sink that drops streamed output. `node_health` keeps the probe's stderr from
2158 /// its `RunOutput` for the failure note, so the live byte stream isn't needed.
2159 struct DiscardSink;
2160 #[async_trait::async_trait]
2161 impl ops_exec::LogSink for DiscardSink {
2162 async fn write_chunk(&mut self, _bytes: &[u8]) {}
2163 }
2164
2165 /// `node_health` — the post-deploy gate that proves the *deployed nodes* are
2166 /// serving, recording one outcome per (tier, version) that the next promote
2167 /// checks. Distinct from `boot_smoke`, which boots the staged artifact on the
2168 /// build host: this probes each node over the same executor the deploy used, so
2169 /// a node that took a corrupt artifact, wrong-arch binary, or failed restart is
2170 /// caught here rather than waved through (Run-2 SERIOUS-3). Fails closed: any
2171 /// unhealthy node fails the gate, and an empty node set is `Blocked`.
2172 /// Load the tier's public pages in a real browser and fail if the JavaScript
2173 /// did not run.
2174 ///
2175 /// The only gate here that crosses the CDN. `boot_smoke` runs on the build host
2176 /// and `node_health` reaches a node over its executor, so between them nothing
2177 /// ever requested the site the way a visitor does. On 2026-08-14 that gap
2178 /// shipped: testnot served a landing page whose JavaScript did not run, for
2179 /// hours, behind nine green gates. Cloudflare was holding a module from an
2180 /// earlier deploy; it failed to link against the fresh one beside it, and since
2181 /// the bundle's entry point side-effect-imports every island, one bad link took
2182 /// all of them down together.
2183 ///
2184 /// It could not have been caught earlier. Each artifact was individually
2185 /// correct -- right markup, right stylesheets, every module answering 200 with
2186 /// current bytes. Only their composition was broken, and composition is
2187 /// observable in a browser and nowhere else.
2188 ///
2189 /// Runs on the daemon host rather than on a node, because it is a *client*: it
2190 /// should reach the site through whatever the public reaches it through, and a
2191 /// probe that ran on the origin would inherit the blind spot this exists to
2192 /// close.
2193 async fn page_smoke(ctx: &GateCtx) -> Result<GateOutcome> {
2194 let Some(cmd) = ctx.cfg.page_smoke_cmd.as_deref() else {
2195 // A service with no pages says so by configuring no command. Blocked
2196 // rather than passed: a gate that proves nothing must not read green.
2197 return Ok(GateOutcome::blocked(GateBlocker::NotConfigured {
2198 what: "page_smoke_cmd".into(),
2199 }));
2200 };
2201 let Some(base) = ctx.public_url.as_deref() else {
2202 return Ok(GateOutcome::blocked(GateBlocker::NotConfigured {
2203 what: "public_url".into(),
2204 }));
2205 };
2206
2207 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
2208 let child = tokio::process::Command::new("sh")
2209 .arg("-c")
2210 .arg(cmd)
2211 .env("BASE", base)
2212 .stdout(std::process::Stdio::piped())
2213 .stderr(std::process::Stdio::piped())
2214 .kill_on_drop(true)
2215 .spawn()?;
2216
2217 let out = match tokio::time::timeout(ceiling, child.wait_with_output()).await {
2218 Ok(res) => res?,
2219 Err(_elapsed) => {
2220 return Ok(GateOutcome::failed(GateFailure::Timeout {
2221 gate: GateKind::PageSmoke,
2222 after_s: ctx.cfg.gate_timeout_secs as u32,
2223 })
2224 .with_log_ref(ctx.log_ref(GateKind::PageSmoke)));
2225 }
2226 };
2227
2228 let log = format!(
2229 "{}{}",
2230 String::from_utf8_lossy(&out.stdout),
2231 String::from_utf8_lossy(&out.stderr)
2232 );
2233 append_to_log(&ctx.log_path(GateKind::PageSmoke), log.as_bytes()).await;
2234
2235 if out.status.success() {
2236 return Ok(
2237 GateOutcome::passed(PassNote::PagesClean { base: base.into() })
2238 .with_log_ref(ctx.log_ref(GateKind::PageSmoke)),
2239 );
2240 }
2241
2242 // The script prints one `FAIL <url>` line per bad page and indents the
2243 // reasons under it. Lift the first reason into the summary so a red gate
2244 // says what broke without anyone opening the log.
2245 let first = log
2246 .lines()
2247 .skip_while(|l| !l.starts_with("FAIL"))
2248 .nth(1)
2249 .map(str::trim)
2250 .filter(|l| !l.is_empty())
2251 .unwrap_or("see log");
2252 Ok(GateOutcome::failed(GateFailure::PagesBroken {
2253 base: base.into(),
2254 detail: first.to_string(),
2255 })
2256 .with_log_ref(ctx.log_ref(GateKind::PageSmoke)))
2257 }
2258
2259 async fn node_health(ctx: &GateCtx) -> Result<GateOutcome> {
2260 if ctx.nodes.is_empty() {
2261 return Ok(GateOutcome::blocked(GateBlocker::NoNodesToProbe));
2262 }
2263 for probe in &ctx.nodes {
2264 if let Err(detail) = probe_node(probe).await {
2265 return Ok(GateOutcome::failed(GateFailure::NodeUnhealthy {
2266 node: probe.node.to_string(),
2267 detail,
2268 }));
2269 }
2270 }
2271 Ok(GateOutcome::passed(PassNote::NodesHealthy {
2272 nodes: ctx.nodes.len() as u32,
2273 }))
2274 }
2275
2276 /// Probe one node over its executor: confirm the unit is active post-restart
2277 /// and, when a `health_url` is configured, that it serves a 2xx. Retries across
2278 /// ~10s because the service may still be restarting / warming. Runs under the
2279 /// read-only `Observe(Health)` capability (every Sando node grants it), so the
2280 /// probe needs no deploy authority. `Ok(())` = healthy; `Err(detail)` carries a
2281 /// short reason for the gate's failure note.
2282 async fn probe_node(probe: &NodeProbe) -> std::result::Result<(), String> {
2283 use ops_exec::{Action, ObserveKind, Step, sh_quote};
2284 let svc = sh_quote(&probe.service);
2285 let url = probe
2286 .health_url
2287 .as_deref()
2288 .map_or_else(|| "''".to_string(), sh_quote);
2289 // One executor round-trip with the retry loop on the node: is-active, then
2290 // (if a url is set) curl it for a 2xx. Exit 0 only when both hold.
2291 let script = format!(
2292 "svc={svc}; url={url}; \
2293 for _ in $(seq 1 10); do \
2294 if systemctl is-active --quiet \"$svc\"; then \
2295 if [ -z \"$url\" ] || curl -fsS --max-time 5 \"$url\" >/dev/null 2>&1; then exit 0; fi; \
2296 fi; \
2297 sleep 1; \
2298 done; \
2299 echo 'service not active or health url not 2xx after retries' >&2; exit 1"
2300 );
2301 let step = Step::shell(Action::Observe(ObserveKind::Health), script);
2302 let mut sink = DiscardSink;
2303 let out = probe
2304 .executor
2305 .run_streaming(&step, &mut sink)
2306 .await
2307 .map_err(|e| format!("probe spawn: {e}"))?;
2308 if out.status.success() {
2309 Ok(())
2310 } else {
2311 let code = out
2312 .status
2313 .code()
2314 .map_or_else(|| "signal".to_string(), |c| c.to_string());
2315 let stderr: String = String::from_utf8_lossy(&out.stderr)
2316 .chars()
2317 .take(200)
2318 .collect();
2319 Err(format!("exit {code}: {stderr}"))
2320 }
2321 }
2322
2323 /// Drain `stream` into the shared `LiveLog` (which forwards each chunk to
2324 /// the on-disk log file AND broadcasts a `GateLogChunk` event), and return
2325 /// the concatenated bytes so the classifier can still operate on the full
2326 /// output post-hoc.
2327 async fn stream_into_log<R>(
2328 stream: Option<R>,
2329 log: std::sync::Arc<tokio::sync::Mutex<LiveLog>>,
2330 ) -> Vec<u8>
2331 where
2332 R: tokio::io::AsyncRead + Unpin + Send + 'static,
2333 {
2334 let mut total = Vec::new();
2335 let Some(mut s) = stream else { return total };
2336 let mut buf = [0u8; 4096];
2337 loop {
2338 match s.read(&mut buf).await {
2339 Ok(0) => break,
2340 Err(_) => break,
2341 Ok(n) => {
2342 total.extend_from_slice(&buf[..n]);
2343 log.lock().await.write_chunk(&buf[..n]).await;
2344 }
2345 }
2346 }
2347 total
2348 }
2349
2350 /// Spawn a child, drain its stdout/stderr through a `LiveLog`, return the
2351 /// combined buffers and exit status. Shared by `cargo_test` (no deadline)
2352 /// and ad-hoc callers — `boot_smoke` rolls its own variant because of its
2353 /// 3s kill window.
2354 async fn stream_child_to_live_log(
2355 child: &mut tokio::process::Child,
2356 events: EventTx,
2357 run_id: GateRunId,
2358 log_path: PathBuf,
2359 ) -> Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> {
2360 let log = GateLog::new(LiveLog::open(log_path, gate_chunk_cb(events, run_id)).await);
2361 let out = log.stream_child(child).await;
2362 log.close().await;
2363 out
2364 }
2365
2366 /// One gate's live log, held across every step of a multi-step gate.
2367 ///
2368 /// Single-child gates call [`stream_child_to_live_log`] and are done. The
2369 /// staged gates (`code_smoke`, `migration_dry_run`) instead run several
2370 /// children plus banner lines between them, and they hold one `GateLog` across
2371 /// the lot: a single sink means chunk sequence numbers stay monotonic for the
2372 /// whole gate, and the on-disk log reads in the order things actually happened
2373 /// rather than as stdout-then-stderr assembled at the end.
2374 ///
2375 /// Every write is best-effort in the same way `LiveLog` is: a log directory
2376 /// that cannot be written degrades to callback-only and never turns a passing
2377 /// gate red.
2378 struct GateLog {
2379 sink: Arc<tokio::sync::Mutex<LiveLog>>,
2380 }
2381
2382 impl GateLog {
2383 fn new(sink: LiveLog) -> Self {
2384 Self {
2385 sink: Arc::new(tokio::sync::Mutex::new(sink)),
2386 }
2387 }
2388
2389 /// Open the sink for `gate`'s log file, streaming to the TUI under `run_id`.
2390 async fn open(ctx: &GateCtx, run_id: GateRunId, gate: GateKind) -> Self {
2391 Self::new(
2392 LiveLog::open(
2393 ctx.log_path(gate),
2394 gate_chunk_cb(ctx.events.clone(), run_id),
2395 )
2396 .await,
2397 )
2398 }
2399
2400 /// Emit a banner (or any line the gate itself produces) through the same
2401 /// sink the children stream to, so it lands in sequence with their output.
2402 async fn write(&self, bytes: &[u8]) {
2403 self.sink.lock().await.write_chunk(bytes).await;
2404 }
2405
2406 /// Same, for the common `format!`-a-line case.
2407 async fn line(&self, s: &str) {
2408 self.write(s.as_bytes()).await;
2409 }
2410
2411 /// Spawn `cmd` with both pipes captured, stream them into the sink as they
2412 /// arrive, and return the buffers plus the exit status. The buffers are
2413 /// what the classifiers still operate on post-hoc.
2414 async fn run(
2415 &self,
2416 cmd: &mut Command,
2417 ) -> std::io::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> {
2418 cmd.stdout(std::process::Stdio::piped())
2419 .stderr(std::process::Stdio::piped());
2420 let mut child = cmd.spawn()?;
2421 self.stream_child(&mut child)
2422 .await
2423 .map_err(std::io::Error::other)
2424 }
2425
2426 /// Drain an already-spawned child's pipes into the sink and wait for it.
2427 async fn stream_child(
2428 &self,
2429 child: &mut tokio::process::Child,
2430 ) -> Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> {
2431 let (stdout_task, stderr_task) = self.drain_pipes(child);
2432 let status = child.wait().await?;
2433 let stdout_buf = stdout_task.await.unwrap_or_default();
2434 let stderr_buf = stderr_task.await.unwrap_or_default();
2435 Ok((stdout_buf, stderr_buf, status))
2436 }
2437
2438 /// Start draining a child's pipes into the sink *without* waiting on the
2439 /// child. For `code_smoke`'s serve phase, which probes `/health` while the
2440 /// server is still up. The caller must await both handles for the buffers
2441 /// (and before [`Self::close`], so the flush isn't skipped).
2442 #[allow(clippy::type_complexity)]
2443 fn drain_pipes(
2444 &self,
2445 child: &mut tokio::process::Child,
2446 ) -> (
2447 tokio::task::JoinHandle<Vec<u8>>,
2448 tokio::task::JoinHandle<Vec<u8>>,
2449 ) {
2450 (
2451 tokio::spawn(stream_into_log(child.stdout.take(), self.sink.clone())),
2452 tokio::spawn(stream_into_log(child.stderr.take(), self.sink.clone())),
2453 )
2454 }
2455
2456 /// Flush the file. A still-outstanding streaming task (only possible if the
2457 /// gate returned without awaiting it) leaves the `Arc` shared, in which case
2458 /// the flush is skipped — `LiveLog` writes unbuffered to the OS either way,
2459 /// so nothing already written is lost.
2460 async fn close(self) {
2461 if let Ok(mutex) = Arc::try_unwrap(self.sink) {
2462 mutex.into_inner().close().await;
2463 }
2464 }
2465 }
2466
2467 /// Live check: has `tier`'s burn-in window of `hours` elapsed since its clock
2468 /// (`tier_state.burn_in_started_at`, started by a promote onto the tier)? Used
2469 /// by the promote-time gate check (`unsatisfied_gates`) so a stale `blocked`
2470 /// row never masks an elapsed — or not-yet-elapsed — window. The `burn_in` gate
2471 /// runner below wraps the same state with a richer outcome for `/state`.
2472 pub async fn burn_in_satisfied(
2473 pool: &SqlitePool,
2474 app: &AppId,
2475 tier: &TierId,
2476 hours: u32,
2477 ) -> Result<bool> {
2478 let started: Option<String> =
2479 sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?")
2480 .bind(app)
2481 .bind(tier)
2482 .fetch_optional(pool)
2483 .await?
2484 .flatten();
2485 let Some(started) = started else {
2486 return Ok(false);
2487 };
2488 let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc);
2489 Ok(Utc::now() - started >= chrono::Duration::hours(hours as i64))
2490 }
2491
2492 async fn burn_in(ctx: &GateCtx, hours: u32) -> Result<GateOutcome> {
2493 // Check tier_state.burn_in_started_at on this tier; pass if enough time
2494 // has elapsed. The clock is started by /promote when a version lands on
2495 // the burn-in tier.
2496 let started: Option<String> =
2497 sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?")
2498 .bind(&ctx.cfg.id)
2499 .bind(&ctx.tier)
2500 .fetch_optional(&ctx.pool)
2501 .await?
2502 .flatten();
2503 let Some(started) = started else {
2504 return Ok(GateOutcome::blocked(GateBlocker::BurnInClockNotStarted));
2505 };
2506 let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc);
2507 let elapsed = Utc::now() - started;
2508 let needed = chrono::Duration::hours(hours as i64);
2509 if elapsed >= needed {
2510 Ok(GateOutcome::passed(PassNote::BurnInElapsed {
2511 hours: elapsed.num_hours() as u32,
2512 }))
2513 } else {
2514 let remaining = (needed - elapsed).num_hours().max(0) as u32;
2515 Ok(GateOutcome::blocked(GateBlocker::BurnInRemaining {
2516 hours_remaining: remaining,
2517 hours_total: hours,
2518 }))
2519 }
2520 }
2521
2522 async fn manual_confirm(ctx: &GateCtx) -> Result<GateOutcome> {
2523 // Pass iff a row in gate_runs exists with status='passed' for this
2524 // (tier, version, manual_confirm) that was inserted out-of-band by an
2525 // operator action. Since the harness inserts the in-flight row itself,
2526 // look for a prior confirmation row.
2527 let prior_at: Option<String> = sqlx::query_scalar(
2528 "SELECT finished_at FROM gate_runs
2529 WHERE app = ? AND tier = ? AND version = ? AND gate_kind = 'manual_confirm'
2530 AND status = 'passed'
2531 ORDER BY id DESC LIMIT 1",
2532 )
2533 .bind(&ctx.cfg.id)
2534 .bind(&ctx.tier)
2535 .bind(&ctx.version)
2536 .fetch_optional(&ctx.pool)
2537 .await?;
2538 match prior_at {
2539 Some(at_str) => {
2540 let at = chrono::DateTime::parse_from_rfc3339(&at_str)
2541 .map_or_else(|_| Utc::now(), |d| d.with_timezone(&Utc));
2542 Ok(GateOutcome::passed(PassNote::OperatorConfirmed { at }))
2543 }
2544 None => Ok(GateOutcome::blocked(
2545 GateBlocker::AwaitingOperatorConfirmation,
2546 )),
2547 }
2548 }
2549
2550 #[cfg(test)]
2551 mod tests {
2552 use super::*;
2553 use crate::events;
2554 use sqlx::sqlite::SqlitePoolOptions;
2555
2556 fn target(dir: &str) -> crate::config::TestTarget {
2557 crate::config::TestTarget {
2558 dir: std::path::PathBuf::from(dir),
2559 aux_repo: None,
2560 features: Vec::new(),
2561 all_features: false,
2562 scratch_db: false,
2563 }
2564 }
2565
2566 #[test]
2567 fn parse_check_docs_broken_count_reads_the_sentinel() {
2568 // The failing sentinel carries the count.
2569 assert_eq!(
2570 parse_check_docs_broken_count(b"some log\nMNW_CHECK_DOCS: 3 broken link(s)\n"),
2571 3
2572 );
2573 // Count with a preceding log line still parses (first bare int wins).
2574 assert_eq!(
2575 parse_check_docs_broken_count(
2576 b" broken link: a -> b\nMNW_CHECK_DOCS: 1 broken link(s)\n"
2577 ),
2578 1
2579 );
2580 // The parser only runs on failure; the "ok" sentinel is never fed to it,
2581 // and its "(2" token is not a bare int, so it yields 0 harmlessly.
2582 assert_eq!(
2583 parse_check_docs_broken_count(b"MNW_CHECK_DOCS: ok (2 collision(s) reported)\n"),
2584 0
2585 );
2586 // Absent sentinel -> 0; the failure still stands, only the count is lost.
2587 assert_eq!(parse_check_docs_broken_count(b"unrelated output"), 0);
2588 }
2589
2590 #[test]
2591 fn name_target_points_a_test_failure_at_its_crate() {
2592 // With one target the crate was implicit; with fifteen the operator
2593 // needs the summary to say which one broke.
2594 let f = name_target(
2595 GateFailure::CargoTest {
2596 failed_count: 3,
2597 first_failed: Some("workflows::sync::round_trip".into()),
2598 first_panic: None,
2599 },
2600 std::path::Path::new("shared/synckit-client"),
2601 );
2602 assert_eq!(
2603 f.summary(),
2604 "3 test(s) failed; first: shared/synckit-client: workflows::sync::round_trip",
2605 );
2606 }
2607
2608 #[test]
2609 fn name_target_points_a_compile_failure_at_its_crate() {
2610 let f = name_target(
2611 GateFailure::CompileError {
2612 error_count: 1,
2613 first_error: Some("error[E0063]".into()),
2614 },
2615 std::path::Path::new("mnw-cli"),
2616 );
2617 assert_eq!(
2618 f.summary(),
2619 "compile failed (1 error(s)); first: mnw-cli: error[E0063]"
2620 );
2621 }
2622
2623 #[test]
2624 fn name_target_names_the_crate_even_without_a_test_name() {
2625 let f = name_target(
2626 GateFailure::CargoTest {
2627 failed_count: 2,
2628 first_failed: None,
2629 first_panic: None,
2630 },
2631 std::path::Path::new("pom"),
2632 );
2633 assert_eq!(f.summary(), "2 test(s) failed; first: pom");
2634 }
2635
2636 #[test]
2637 fn name_target_leaves_unrelated_failures_alone() {
2638 let f = name_target(
2639 GateFailure::SpawnFailed {
2640 message: "no cargo".into(),
2641 },
2642 std::path::Path::new("pom"),
2643 );
2644 assert!(matches!(f, GateFailure::SpawnFailed { .. }));
2645 }
2646
2647 /// `target()` above, but resolved against an aux repo's checkout.
2648 fn aux_target(dir: &str, repo: &str) -> crate::config::TestTarget {
2649 crate::config::TestTarget {
2650 aux_repo: Some(repo.to_string()),
2651 ..target(dir)
2652 }
2653 }
2654
2655 /// True when the URL's host parses as a domain rather than an IP literal,
2656 /// which is the distinction `Url::domain()` draws and WebAuthn depends on.
2657 fn url_host_is_a_domain(url: &str) -> bool {
2658 let after = url.split("://").nth(1).unwrap_or("");
2659 let host = after.split(['/', '?', '#']).next().unwrap_or("");
2660 let host = host.rsplit('@').next().unwrap_or(host);
2661 let host = if let Some(rest) = host.strip_prefix('[') {
2662 rest.split(']').next().unwrap_or("")
2663 } else {
2664 host.split(':').next().unwrap_or("")
2665 };
2666 !host.is_empty() && host.parse::<std::net::IpAddr>().is_err()
2667 }
2668
2669 fn resolving_ctx(worktree: &str, aux: &[(&str, &str)]) -> GateCtx {
2670 GateCtx {
2671 public_url: None,
2672 pool: SqlitePool::connect_lazy("sqlite::memory:").unwrap(),
2673 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
2674 tier: TierId::new("host"),
2675 version: "0.1.0".parse().unwrap(),
2676 worktree: Some(PathBuf::from(worktree)),
2677 bundle: None,
2678 events: events::channel(),
2679 nodes: Vec::new(),
2680 build_id: None,
2681 aux_dirs: aux
2682 .iter()
2683 .map(|(n, d)| ((*n).to_string(), PathBuf::from(d)))
2684 .collect(),
2685 }
2686 }
2687
2688 #[tokio::test]
2689 async fn a_plain_target_resolves_under_the_worktree() {
2690 let ctx = resolving_ctx("/w/abc123", &[]);
2691 assert_eq!(
2692 ctx.target_dir(&target("shared/tagtree")),
2693 Some(PathBuf::from("/w/abc123/shared/tagtree")),
2694 );
2695 }
2696
2697 #[tokio::test]
2698 async fn an_aux_target_resolves_beside_the_worktree_not_under_it() {
2699 // The whole point: `Libraries/docengine` is a sibling of the per-sha
2700 // worktree, so a worktree-relative path can never reach it.
2701 let ctx = resolving_ctx("/w/abc123", &[("docengine", "/w/Libraries/docengine")]);
2702 assert_eq!(
2703 ctx.target_dir(&aux_target("", "docengine")),
2704 Some(PathBuf::from("/w/Libraries/docengine")),
2705 );
2706 // A subdirectory of an aux repo resolves under its checkout.
2707 assert_eq!(
2708 ctx.target_dir(&aux_target("crates/inner", "docengine")),
2709 Some(PathBuf::from("/w/Libraries/docengine/crates/inner")),
2710 );
2711 }
2712
2713 #[tokio::test]
2714 async fn an_aux_target_with_no_checkout_this_run_resolves_to_nothing() {
2715 // Promote-time gates carry no aux dirs. Resolving to a wrong path (say,
2716 // the worktree) would run the gate against whatever happened to sit
2717 // there; `None` makes the caller skip, and --check-config is what
2718 // catches a real typo.
2719 let ctx = resolving_ctx("/w/abc123", &[]);
2720 assert_eq!(ctx.target_dir(&aux_target("", "docengine")), None);
2721 }
2722
2723 #[test]
2724 fn labels_name_the_repo_an_aux_target_lives_in() {
2725 assert_eq!(target("server").label(), "server");
2726 assert_eq!(aux_target("", "docengine").label(), "docengine (aux)");
2727 assert_eq!(
2728 aux_target("crates/inner", "docengine").label(),
2729 "docengine/crates/inner (aux)",
2730 );
2731 }
2732
2733 /// A `GateCtx` over `worktree` with the given frontend projects configured.
2734 /// No DB, no artifact — `code_smoke_frontends` touches neither.
2735 async fn frontend_ctx(worktree: &std::path::Path, dirs: &[&str]) -> GateCtx {
2736 let mut cfg = crate::config::AppConfig::for_tests();
2737 cfg.frontend_builds = dirs
2738 .iter()
2739 .map(|d| crate::config::FrontendBuild {
2740 dir: PathBuf::from(d),
2741 script: "build".into(),
2742 })
2743 .collect();
2744 cfg.logs_root = worktree.join("logs");
2745 GateCtx {
2746 public_url: None,
2747 pool: SqlitePoolOptions::new()
2748 .max_connections(1)
2749 .connect("sqlite::memory:")
2750 .await
2751 .unwrap(),
2752 cfg: std::sync::Arc::new(cfg),
2753 tier: TierId::new("host"),
2754 version: "0.1.0".parse().unwrap(),
2755 worktree: Some(worktree.to_path_buf()),
2756 bundle: None,
2757 events: events::channel(),
2758 nodes: Vec::new(),
2759 build_id: None,
2760 aux_dirs: HashMap::new(),
2761 }
2762 }
2763
2764 /// A `GateCtx` for the `migration_dry_run` freshness checks: a migrated
2765 /// in-memory pool (so `backups` exists) and a scratch URL set, so the gate
2766 /// reaches the backup lookup instead of bailing on config. Nothing here
2767 /// touches postgres — every assertion below blocks before `reset_scratch`.
2768 async fn dry_run_ctx(worktree: &std::path::Path, max_age_hours: u32) -> GateCtx {
2769 let mut cfg = crate::config::AppConfig::for_tests();
2770 cfg.scratch_db_url = Some("postgres:///sando_scratch".into());
2771 cfg.backup_max_age_hours = max_age_hours;
2772 cfg.logs_root = worktree.join("logs");
2773 let pool = SqlitePoolOptions::new()
2774 .max_connections(1)
2775 .connect("sqlite::memory:")
2776 .await
2777 .unwrap();
2778 crate::db::migrate(&pool).await.unwrap();
2779 GateCtx {
2780 public_url: None,
2781 pool,
2782 cfg: std::sync::Arc::new(cfg),
2783 tier: TierId::new("host"),
2784 version: "0.1.0".parse().unwrap(),
2785 worktree: Some(worktree.to_path_buf()),
2786 bundle: None,
2787 events: events::channel(),
2788 nodes: Vec::new(),
2789 build_id: None,
2790 aux_dirs: HashMap::new(),
2791 }
2792 }
2793
2794 /// Record a `server` backup row fetched `hours_ago`, as `/backup/fetch` would.
2795 async fn seed_backup(ctx: &GateCtx, hours_ago: i64) {
2796 seed_named_backup(ctx, "server", hours_ago).await;
2797 }
2798
2799 /// Record a backup row for one named dump.
2800 async fn seed_named_backup(ctx: &GateCtx, name: &str, hours_ago: i64) {
2801 let at = (Utc::now() - chrono::Duration::hours(hours_ago)).to_rfc3339();
2802 sqlx::query(
2803 "INSERT INTO backups (name, fetched_at, source, local_path, byte_size)
2804 VALUES (?, ?, 'file:///x.sql.gz', '/tmp/sando-test-backup.sql.gz', 1000000)",
2805 )
2806 .bind(name)
2807 .bind(at)
2808 .execute(&ctx.pool)
2809 .await
2810 .unwrap();
2811 }
2812
2813 /// The multithreaded check, as `sando-daemon.toml` configures it.
2814 fn mt_check() -> crate::config::MigrationCheck {
2815 crate::config::MigrationCheck {
2816 dir: std::path::PathBuf::from("multithreaded/migrations"),
2817 backup: "multithreaded".into(),
2818 scratch_db: Some("sando_scratch_mt".into()),
2819 owner_role: Some("multithreaded".into()),
2820 }
2821 }
2822
2823 /// Re-point a `dry_run_ctx` at one check, keeping its pool and scratch URL.
2824 fn with_check(ctx: &mut GateCtx, check: crate::config::MigrationCheck) {
2825 let mut cfg = crate::config::AppConfig::for_tests();
2826 cfg.scratch_db_url = ctx.cfg.scratch_db_url.clone();
2827 cfg.backup_max_age_hours = ctx.cfg.backup_max_age_hours;
2828 cfg.logs_root = ctx.cfg.logs_root.clone();
2829 cfg.migration_checks = vec![check];
2830 ctx.cfg = std::sync::Arc::new(cfg);
2831 }
2832
2833 #[tokio::test]
2834 async fn migration_dry_run_blocks_when_a_checks_own_dump_was_never_fetched() {
2835 // The hazard the check list exists for: multithreaded applies its own
2836 // migrations at boot against its own database, so the server's dump must
2837 // never stand in for it. A fetched `server` row with no `multithreaded`
2838 // row is exactly that substitution, and it has to block.
2839 let tmp = tempfile::tempdir().unwrap();
2840 let mut ctx = dry_run_ctx(tmp.path(), 48).await;
2841 with_check(&mut ctx, mt_check());
2842 seed_named_backup(&ctx, "server", 1).await;
2843 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2844
2845 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2846 log.close().await;
2847
2848 let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else {
2849 panic!("a missing multithreaded dump must block");
2850 };
2851 let GateBlocker::NoBackupAvailable { check, backup } = blocker else {
2852 panic!("expected NoBackupAvailable, got {blocker:?}");
2853 };
2854 assert_eq!(backup, "multithreaded", "names the dump that is missing");
2855 assert!(
2856 check.contains("multithreaded/migrations"),
2857 "names the check that wanted it, got {check}"
2858 );
2859 }
2860
2861 #[tokio::test]
2862 async fn migration_dry_run_freshness_is_per_dump() {
2863 // A fresh server dump must not make a 45-day-old multithreaded dump look
2864 // current: the clock is per-database, or the second check inherits the
2865 // first's freshness and the gate is theatre.
2866 let tmp = tempfile::tempdir().unwrap();
2867 let mut ctx = dry_run_ctx(tmp.path(), 48).await;
2868 with_check(&mut ctx, mt_check());
2869 seed_named_backup(&ctx, "server", 1).await;
2870 seed_named_backup(&ctx, "multithreaded", 24 * 45).await;
2871 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2872
2873 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2874 log.close().await;
2875
2876 let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else {
2877 panic!("a 45-day-old multithreaded dump must block");
2878 };
2879 let GateBlocker::BackupStale { check, .. } = blocker else {
2880 panic!("expected BackupStale, got {blocker:?}");
2881 };
2882 assert!(
2883 check.contains("multithreaded/migrations"),
2884 "names the check whose dump is stale, got {check}"
2885 );
2886 }
2887
2888 #[tokio::test]
2889 async fn migration_dry_run_blocks_on_a_stale_backup() {
2890 // The failure this closes: the gate used to check only that a backups row
2891 // existed, so a fetch that silently stopped working left it green against
2892 // an ever-older schema. Sando ran 45 days that way.
2893 let tmp = tempfile::tempdir().unwrap();
2894 let ctx = dry_run_ctx(tmp.path(), 48).await;
2895 seed_backup(&ctx, 24 * 45).await;
2896 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2897
2898 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2899 log.close().await;
2900
2901 let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else {
2902 panic!("a 45-day-old backup must block");
2903 };
2904 let GateBlocker::BackupStale {
2905 age_hours,
2906 max_age_hours,
2907 ..
2908 } = blocker
2909 else {
2910 panic!("expected BackupStale, got {blocker:?}");
2911 };
2912 assert_eq!(max_age_hours, 48);
2913 assert!(
2914 age_hours >= 24 * 45,
2915 "reports the real age, got {age_hours}"
2916 );
2917 }
2918
2919 #[tokio::test]
2920 async fn migration_dry_run_accepts_a_fresh_backup() {
2921 // The other side of the boundary: a backup inside the window must not be
2922 // blocked on freshness. It fails later (there is no such dump on disk),
2923 // which is exactly the proof the age check let it through.
2924 let tmp = tempfile::tempdir().unwrap();
2925 let ctx = dry_run_ctx(tmp.path(), 48).await;
2926 seed_backup(&ctx, 6).await;
2927 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2928
2929 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2930 log.close().await;
2931
2932 assert!(
2933 !matches!(
2934 outcome.status,
2935 crate::outcome::GateStatus::Blocked {
2936 blocker: GateBlocker::BackupStale { .. }
2937 }
2938 ),
2939 "a 6h-old backup is fresh, got {:?}",
2940 outcome.status,
2941 );
2942 }
2943
2944 #[tokio::test]
2945 async fn migration_dry_run_treats_an_unparsable_fetched_at_as_stale() {
2946 // Fail closed: `fetched_at` is daemon-written RFC 3339, so a value that
2947 // will not parse means the row is untrustworthy — and a freshness check
2948 // that shrugs at a timestamp it cannot read is not a freshness check.
2949 let tmp = tempfile::tempdir().unwrap();
2950 let ctx = dry_run_ctx(tmp.path(), 48).await;
2951 sqlx::query(
2952 "INSERT INTO backups (fetched_at, source, local_path, byte_size)
2953 VALUES ('not-a-timestamp', 'file:///x.sql.gz', '/tmp/x.sql.gz', 1000000)",
2954 )
2955 .execute(&ctx.pool)
2956 .await
2957 .unwrap();
2958 let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await;
2959
2960 let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap();
2961 log.close().await;
2962
2963 assert!(
2964 matches!(
2965 outcome.status,
2966 crate::outcome::GateStatus::Blocked {
2967 blocker: GateBlocker::BackupStale { .. }
2968 }
2969 ),
2970 "an unreadable fetched_at must block, got {:?}",
2971 outcome.status,
2972 );
2973 }
2974
2975 /// A `code_smoke` live log over `ctx.cfg.logs_root`, for the helpers that
2976 /// take one. `GateRunId(0)` never matches a real row; nothing reads the
2977 /// chunk events in these tests.
2978 async fn test_gate_log(ctx: &GateCtx) -> GateLog {
2979 GateLog::open(ctx, GateRunId(0), GateKind::CodeSmoke).await
2980 }
2981
2982 /// Close `log` (flushing it) and read back what it wrote on disk.
2983 async fn read_gate_log(ctx: &GateCtx, log: GateLog) -> String {
2984 log.close().await;
2985 tokio::fs::read_to_string(ctx.log_path(GateKind::CodeSmoke))
2986 .await
2987 .expect("the gate log must exist on disk")
2988 }
2989
2990 /// Write a minimal npm project at `worktree/<dir>` whose `build` script
2991 /// exits with `exit_code`. Pre-creates `node_modules` so the gate skips
2992 /// `npm ci` — these tests are about the build step, not the network.
2993 fn fake_npm_project(worktree: &std::path::Path, dir: &str, exit_code: u8) {
2994 let root = worktree.join(dir);
2995 std::fs::create_dir_all(root.join("node_modules")).unwrap();
2996 std::fs::write(
2997 root.join("package.json"),
2998 format!(
2999 r#"{{"name":"fake","version":"0.0.0","private":true,
3000 "scripts":{{"build":"exit {exit_code}"}}}}"#
3001 ),
3002 )
3003 .unwrap();
3004 }
3005
3006 #[tokio::test]
3007 async fn frontend_gate_fails_on_a_build_error_and_names_the_project() {
3008 // The whole point of the gate: the app build scripts downgrade this to a
3009 // cargo::warning, so if it passes here nothing stops a stale bundle.
3010 let tmp = tempfile::tempdir().unwrap();
3011 fake_npm_project(tmp.path(), "server/frontend", 0);
3012 fake_npm_project(tmp.path(), "multithreaded/frontend", 2);
3013 let ctx = frontend_ctx(tmp.path(), &["server/frontend", "multithreaded/frontend"]).await;
3014
3015 let log = test_gate_log(&ctx).await;
3016 let outcome = code_smoke_frontends(&ctx, &log)
3017 .await
3018 .expect("a failing tsc must fail the gate");
3019 let crate::outcome::GateStatus::Failed { failure } = &outcome.status else {
3020 panic!("expected a failure, got {:?}", outcome.status)
3021 };
3022 assert!(
3023 matches!(
3024 failure,
3025 GateFailure::CodeSmokeFrontend { dir, exit_code: Some(2) }
3026 if dir == "multithreaded/frontend"
3027 ),
3028 "got: {failure:?}"
3029 );
3030 // The passing project ran first; its output belongs in the log too.
3031 let text = read_gate_log(&ctx, log).await;
3032 assert!(text.contains("server/frontend"), "log: {text}");
3033 }
3034
3035 #[tokio::test]
3036 async fn frontend_gate_passes_when_every_project_builds() {
3037 let tmp = tempfile::tempdir().unwrap();
3038 fake_npm_project(tmp.path(), "server/frontend", 0);
3039 let ctx = frontend_ctx(tmp.path(), &["server/frontend"]).await;
3040 let log = test_gate_log(&ctx).await;
3041 assert!(
3042 code_smoke_frontends(&ctx, &log).await.is_none(),
3043 "a clean build must not fail the gate"
3044 );
3045 }
3046
3047 /// The point of routing `code_smoke` through `LiveLog`: its output reaches
3048 /// the operator *while* the gate runs, as `GateLogChunk` events, instead of
3049 /// appearing all at once when the gate finishes.
3050 #[tokio::test]
3051 async fn code_smoke_streams_chunks_as_it_runs() {
3052 let tmp = tempfile::tempdir().unwrap();
3053 fake_npm_project(tmp.path(), "server/frontend", 0);
3054 let ctx = frontend_ctx(tmp.path(), &["server/frontend"]).await;
3055 let mut rx = ctx.events.subscribe_logs();
3056
3057 let log = GateLog::open(&ctx, GateRunId(7), GateKind::CodeSmoke).await;
3058 assert!(code_smoke_frontends(&ctx, &log).await.is_none());
3059 log.close().await;
3060
3061 let mut chunks = Vec::new();
3062 while let Ok(envelope) = rx.try_recv() {
3063 if let Event::GateLogChunk { run_id, seq, text } = envelope.event {
3064 assert_eq!(run_id, GateRunId(7));
3065 chunks.push((seq, text));
3066 }
3067 }
3068 assert!(!chunks.is_empty(), "no chunk ever reached the bus");
3069 // Sequence numbers are per-run and monotonic across every step of the
3070 // gate, which is why the whole gate shares one sink.
3071 let seqs: Vec<u32> = chunks.iter().map(|(seq, _)| *seq).collect();
3072 assert!(
3073 seqs.windows(2).all(|w| w[0] < w[1]),
3074 "chunk seq must be monotonic, got {seqs:?}"
3075 );
3076 let joined: String = chunks.into_iter().map(|(_, text)| text).collect();
3077 assert!(joined.contains("server/frontend"), "chunks: {joined}");
3078 }
3079
3080 #[tokio::test]
3081 async fn frontend_gate_skips_a_project_absent_from_the_worktree() {
3082 // Rebuilding an older sha that predates the frontend must stay possible.
3083 let tmp = tempfile::tempdir().unwrap();
3084 let ctx = frontend_ctx(tmp.path(), &["multithreaded/frontend"]).await;
3085 let log = test_gate_log(&ctx).await;
3086 assert!(code_smoke_frontends(&ctx, &log).await.is_none());
3087 assert!(
3088 read_gate_log(&ctx, log).await.contains("skipping"),
3089 "the skip must be visible in the log"
3090 );
3091 }
3092
3093 #[tokio::test]
3094 async fn cargo_test_fails_closed_when_no_target_exists_in_the_worktree() {
3095 // A worktree missing every configured crate must not report "tests
3096 // passed" having run none. Uses an empty tempdir as the worktree, so
3097 // no cargo process is ever spawned.
3098 let tmp = tempfile::tempdir().unwrap();
3099 let mut cfg = crate::config::AppConfig::for_tests();
3100 cfg.test_targets = vec![target("server"), target("mnw-cli")];
3101 cfg.logs_root = tmp.path().join("logs");
3102 let pool = SqlitePoolOptions::new()
3103 .max_connections(1)
3104 .connect("sqlite::memory:")
3105 .await
3106 .unwrap();
3107 let ctx = GateCtx {
3108 public_url: None,
3109 pool,
3110 cfg: std::sync::Arc::new(cfg),
3111 tier: TierId::new("host"),
3112 version: "0.1.0".parse().unwrap(),
3113 worktree: Some(tmp.path().to_path_buf()),
3114 bundle: None,
3115 events: events::channel(),
3116 nodes: Vec::new(),
3117 build_id: None,
3118 aux_dirs: HashMap::new(),
3119 };
3120 let out = cargo_test(&ctx, GateRunId(1)).await.unwrap();
3121 assert_eq!(
3122 out.status_str(),
3123 "failed",
3124 "green here would be a silent no-op gate"
3125 );
3126 let crate::outcome::GateStatus::Failed { failure } = &out.status else {
3127 panic!("expected a failure")
3128 };
3129 assert!(
3130 failure.summary().contains("ran no targets"),
3131 "got: {}",
3132 failure.summary()
3133 );
3134 }
3135
3136 #[tokio::test]
3137 async fn scratch_db_env_is_opt_in_per_target() {
3138 // Exporting DATABASE_URL knocks sqlx out of offline mode, so a crate
3139 // shipping .sqlx data must not see it.
3140 let mut cfg = crate::config::AppConfig::for_tests();
3141 cfg.scratch_db_url = Some("postgres://sando@127.0.0.1/sando_scratch".into());
3142 let ctx = GateCtx {
3143 public_url: None,
3144 pool: SqlitePoolOptions::new()
3145 .max_connections(1)
3146 .connect_lazy("sqlite::memory:")
3147 .unwrap(),
3148 cfg: std::sync::Arc::new(cfg),
3149 tier: TierId::new("host"),
3150 version: "0.1.0".parse().unwrap(),
3151 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
3152 bundle: None,
3153 events: events::channel(),
3154 nodes: Vec::new(),
3155 build_id: None,
3156 aux_dirs: HashMap::new(),
3157 };
3158 let dir = std::path::Path::new("/tmp/wt/x");
3159
3160 let off = cargo_test_command(&ctx, dir, &target("x"), &[], &[]);
3161 let has_db = |c: &Command| {
3162 c.as_std()
3163 .get_envs()
3164 .any(|(k, v)| k == "DATABASE_URL" && v.is_some())
3165 };
3166 assert!(!has_db(&off), "scratch_db defaults off");
3167
3168 let mut on_target = target("x");
3169 on_target.scratch_db = true;
3170 assert!(
3171 has_db(&cargo_test_command(&ctx, dir, &on_target, &[], &[])),
3172 "opt-in exports it"
3173 );
3174 }
3175
3176 #[tokio::test]
3177 async fn all_features_replaces_the_feature_list() {
3178 let ctx = GateCtx {
3179 public_url: None,
3180 pool: SqlitePoolOptions::new()
3181 .max_connections(1)
3182 .connect_lazy("sqlite::memory:")
3183 .unwrap(),
3184 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
3185 tier: TierId::new("host"),
3186 version: "0.1.0".parse().unwrap(),
3187 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
3188 bundle: None,
3189 events: events::channel(),
3190 nodes: Vec::new(),
3191 build_id: None,
3192 aux_dirs: HashMap::new(),
3193 };
3194 let mut t = target("shared/ops-exec");
3195 t.all_features = true;
3196 let cmd = cargo_test_command(&ctx, std::path::Path::new("/tmp/wt"), &t, &[], &[]);
3197 let args: Vec<_> = cmd
3198 .as_std()
3199 .get_args()
3200 .map(|a| a.to_string_lossy().into_owned())
3201 .collect();
3202 assert!(args.iter().any(|a| a == "--all-features"), "got: {args:?}");
3203 assert!(!args.iter().any(|a| a == "--features"), "got: {args:?}");
3204 }
3205
3206 #[test]
3207 fn every_gate_kind_round_trips_through_its_wire_string() {
3208 // gate_kind is a TEXT column and a WS event field; as_str and FromStr
3209 // disagreeing would make a gate's evidence unreadable by
3210 // unsatisfied_gates, which fails the promote closed with no explanation.
3211 for k in [
3212 GateKind::CargoTest,
3213 GateKind::HardeningTest,
3214 GateKind::Clippy,
3215 GateKind::Fmt,
3216 GateKind::CargoAudit,
3217 GateKind::CargoDeny,
3218 GateKind::MigrationDryRun,
3219 GateKind::CodeSmoke,
3220 GateKind::BootSmoke,
3221 GateKind::NodeHealth,
3222 GateKind::BurnIn,
3223 GateKind::ManualConfirm,
3224 ] {
3225 assert_eq!(
3226 k.as_str().parse::<GateKind>().unwrap(),
3227 k,
3228 "round trip for {k:?}"
3229 );
3230 }
3231 }
3232
3233 #[test]
3234 fn first_meaningful_line_prefers_the_diagnostic() {
3235 let stderr = b" Updating crates.io index\nerror: 1 vulnerability found!\n";
3236 assert_eq!(
3237 first_meaningful_line(b"", stderr),
3238 "error: 1 vulnerability found!"
3239 );
3240 }
3241
3242 #[test]
3243 fn first_meaningful_line_finds_a_deny_verdict() {
3244 let out = b"advisories FAILED, bans ok, licenses FAILED, sources ok\n";
3245 assert!(first_meaningful_line(out, b"").contains("FAILED"));
3246 }
3247
3248 #[test]
3249 fn first_meaningful_line_falls_back_rather_than_returning_empty() {
3250 assert!(first_meaningful_line(b"", b"").contains("see the gate log"));
3251 }
3252
3253 #[tokio::test]
3254 async fn supply_chain_gates_skip_crates_without_their_config() {
3255 // Four crates in this repo fail `cargo audit` purely for want of a
3256 // triaged .cargo/audit.toml. Running it there would make the gate
3257 // permanently red, so a target only qualifies once it carries the file.
3258 // The worktree here has a Cargo.toml but no audit config, so nothing
3259 // qualifies and the gate fails closed rather than passing over zero work.
3260 let tmp = tempfile::tempdir().unwrap();
3261 let crate_dir = tmp.path().join("server");
3262 std::fs::create_dir_all(&crate_dir).unwrap();
3263 std::fs::write(crate_dir.join("Cargo.toml"), "[package]\nname = \"x\"\n").unwrap();
3264
3265 let mut cfg = crate::config::AppConfig::for_tests();
3266 cfg.test_targets = vec![target("server")];
3267 cfg.logs_root = tmp.path().join("logs");
3268 let ctx = GateCtx {
3269 public_url: None,
3270 pool: SqlitePoolOptions::new()
3271 .max_connections(1)
3272 .connect("sqlite::memory:")
3273 .await
3274 .unwrap(),
3275 cfg: std::sync::Arc::new(cfg),
3276 tier: TierId::new("host"),
3277 version: "0.1.0".parse().unwrap(),
3278 worktree: Some(tmp.path().to_path_buf()),
3279 bundle: None,
3280 events: events::channel(),
3281 nodes: Vec::new(),
3282 build_id: None,
3283 aux_dirs: HashMap::new(),
3284 };
3285 let out = supply_chain(&ctx, GateRunId(1), GateKind::CargoAudit)
3286 .await
3287 .unwrap();
3288 assert_eq!(out.status_str(), "failed");
3289 let crate::outcome::GateStatus::Failed { failure } = &out.status else {
3290 panic!("expected a failure")
3291 };
3292 assert!(
3293 failure.summary().contains("ran nothing"),
3294 "got: {}",
3295 failure.summary()
3296 );
3297
3298 // Drop the config in and the same target now qualifies.
3299 std::fs::create_dir_all(crate_dir.join(".cargo")).unwrap();
3300 std::fs::write(crate_dir.join(".cargo/audit.toml"), "[advisories]\n").unwrap();
3301 // The fixture crate is not a real cargo project, so the tool itself
3302 // still errors — but on its own terms, not with "ran nothing". That
3303 // distinction is the thing under test: the target was attempted.
3304 let out = supply_chain(&ctx, GateRunId(2), GateKind::CargoAudit)
3305 .await
3306 .unwrap();
3307 if let crate::outcome::GateStatus::Failed { failure } = &out.status {
3308 assert!(
3309 !failure.summary().contains("ran nothing"),
3310 "a target carrying the config must be attempted, not skipped; got: {}",
3311 failure.summary(),
3312 );
3313 }
3314 }
3315
3316 #[test]
3317 fn tests_run_reads_the_libtest_summary() {
3318 let out = b"running 6 tests\ntest auth_rate_limit_triggers_on_burst ... ok\n\n\
3319 test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 118 filtered out\n";
3320 assert_eq!(tests_run(out), 6);
3321 }
3322
3323 #[test]
3324 fn tests_run_sums_across_test_binaries() {
3325 let out = b"test result: ok. 6 passed; 0 failed\ntest result: ok. 2 passed; 0 failed\n";
3326 assert_eq!(tests_run(out), 8);
3327 }
3328
3329 #[test]
3330 fn tests_run_is_zero_when_the_filter_matched_nothing() {
3331 // The case hardening_test fails closed on: a filter matching no tests
3332 // exits 0, so a rename would otherwise make the gate a green no-op.
3333 let out = b"running 0 tests\n\n\
3334 test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 124 filtered out\n";
3335 assert_eq!(tests_run(out), 0);
3336 }
3337
3338 #[test]
3339 fn tests_run_is_zero_without_a_summary_line() {
3340 assert_eq!(tests_run(b"error: could not compile `makenotwork`\n"), 0);
3341 }
3342
3343 #[tokio::test]
3344 async fn hardening_test_command_carries_no_features() {
3345 // The entire point of the gate: production constants, which means no
3346 // `fast-tests`. A stray feature here silently restores the blind spot.
3347 let ctx = GateCtx {
3348 public_url: None,
3349 pool: SqlitePoolOptions::new()
3350 .max_connections(1)
3351 .connect_lazy("sqlite::memory:")
3352 .unwrap(),
3353 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
3354 tier: TierId::new("host"),
3355 version: "0.1.0".parse().unwrap(),
3356 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
3357 bundle: None,
3358 events: events::channel(),
3359 nodes: Vec::new(),
3360 build_id: None,
3361 aux_dirs: HashMap::new(),
3362 };
3363 let plain = crate::config::TestTarget {
3364 dir: std::path::PathBuf::from("server"),
3365 aux_repo: None,
3366 features: Vec::new(),
3367 all_features: false,
3368 scratch_db: true,
3369 };
3370 let cmd = cargo_test_command(
3371 &ctx,
3372 std::path::Path::new("/tmp/wt/server"),
3373 &plain,
3374 &[],
3375 &["--test", "integration"],
3376 );
3377 let args: Vec<_> = cmd
3378 .as_std()
3379 .get_args()
3380 .map(|a| a.to_string_lossy().into_owned())
3381 .collect();
3382 assert!(
3383 !args.iter().any(|a| a == "--features"),
3384 "hardening_test must pass no features: {args:?}"
3385 );
3386 assert!(
3387 !args.iter().any(|a| a.contains("fast-tests")),
3388 "got: {args:?}"
3389 );
3390
3391 let fast_target = crate::config::TestTarget {
3392 dir: std::path::PathBuf::from("server"),
3393 aux_repo: None,
3394 features: vec!["fast-tests".into()],
3395 all_features: false,
3396 scratch_db: true,
3397 };
3398 let fast = cargo_test_command(
3399 &ctx,
3400 std::path::Path::new("/tmp/wt/server"),
3401 &fast_target,
3402 &["fast-tests"],
3403 &[],
3404 );
3405 let fast_args: Vec<_> = fast
3406 .as_std()
3407 .get_args()
3408 .map(|a| a.to_string_lossy().into_owned())
3409 .collect();
3410 assert!(
3411 fast_args
3412 .windows(2)
3413 .any(|w| w == ["--features", "fast-tests"]),
3414 "got: {fast_args:?}"
3415 );
3416 }
3417
3418 /// Spawn a one-shot loopback server that answers the first connection with
3419 /// `status_line` + a tiny body, then closes. Returns the bound port.
3420 async fn oneshot_http(status_line: &'static str) -> u16 {
3421 use tokio::io::{AsyncReadExt, AsyncWriteExt};
3422 let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
3423 .await
3424 .unwrap();
3425 let port = listener.local_addr().unwrap().port();
3426 tokio::spawn(async move {
3427 if let Ok((mut sock, _)) = listener.accept().await {
3428 let mut scratch = [0u8; 1024];
3429 let _ = sock.read(&mut scratch).await; // drain the request line
3430 let resp =
3431 format!("{status_line}\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok");
3432 let _ = sock.write_all(resp.as_bytes()).await;
3433 }
3434 });
3435 port
3436 }
3437
3438 #[tokio::test]
3439 async fn probe_health_ok_on_200() {
3440 let port = oneshot_http("HTTP/1.1 200 OK").await;
3441 assert!(probe_health(port).await.is_ok());
3442 }
3443
3444 #[tokio::test]
3445 async fn probe_health_err_on_non_200() {
3446 let port = oneshot_http("HTTP/1.1 503 Service Unavailable").await;
3447 let err = probe_health(port).await.unwrap_err();
3448 assert!(err.contains("status line"), "{err}");
3449 }
3450
3451 #[tokio::test]
3452 async fn probe_health_err_on_connection_refused() {
3453 // Bind then drop to get an almost-certainly-free port nothing listens on.
3454 let port = {
3455 let l = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
3456 .await
3457 .unwrap();
3458 l.local_addr().unwrap().port()
3459 };
3460 let err = probe_health(port).await.unwrap_err();
3461 // Under load the kernel does not always refuse the connect: a socket
3462 // left in TIME_WAIT on that port completes the handshake and then
3463 // resets, so the failure surfaces on the write or the read instead.
3464 // Refused-on-connect and reset-on-write/read are the same fact, that
3465 // nothing is serving the port, and no other outcome counts as a pass.
3466 let refused = err.starts_with("connect: ") && err.contains("refused");
3467 let reset =
3468 (err.starts_with("write: ") || err.starts_with("read: ")) && err.contains("reset");
3469 assert!(refused || reset, "{err}");
3470 }
3471
3472 /// burn_in returns a typed Blocked when the clock isn't started; the
3473 /// runner persists status='blocked' + outcome_json (the json carries
3474 /// blocker.kind = 'burn_in_clock_not_started').
3475 #[tokio::test]
3476 async fn burn_in_blocked_persists_typed_outcome() {
3477 let pool = SqlitePoolOptions::new()
3478 .max_connections(1)
3479 .connect("sqlite::memory:")
3480 .await
3481 .unwrap();
3482 crate::db::migrate(&pool).await.unwrap();
3483 // Topology sync expects a tier row before gate_runs can reference it.
3484 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 0, 'sequential')")
3485 .execute(&pool).await.unwrap();
3486 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
3487 .execute(&pool)
3488 .await
3489 .unwrap();
3490 // versions FK target.
3491 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
3492 .execute(&pool).await.unwrap();
3493
3494 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests());
3495 let ctx = GateCtx {
3496 public_url: None,
3497 pool: pool.clone(),
3498 cfg,
3499 tier: TierId::new("host"),
3500 version: "0.1.0".parse().unwrap(),
3501 worktree: Some(std::path::PathBuf::from("/tmp/unused")),
3502 bundle: None,
3503 events: events::channel(),
3504 nodes: Vec::new(),
3505 build_id: None,
3506 aux_dirs: HashMap::new(),
3507 };
3508 let out = run(&ctx, &Gate::BurnIn { hours: 24 }).await.unwrap();
3509 assert_eq!(out.status_str(), "blocked");
3510 assert!(!out.is_passed());
3511
3512 // Read the persisted row.
3513 let row: (Option<String>, Option<String>) =
3514 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
3515 .fetch_one(&pool)
3516 .await
3517 .unwrap();
3518 assert_eq!(row.0.as_deref(), Some("blocked"), "typed status");
3519 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
3520 assert_eq!(json["status"]["kind"], "blocked");
3521 assert_eq!(
3522 json["status"]["blocker"]["kind"],
3523 "burn_in_clock_not_started"
3524 );
3525 }
3526
3527 /// node_health fails closed when there are no nodes to probe: a serving tier
3528 /// should always carry nodes, so an empty set is a misconfiguration that must
3529 /// block promotion, not pass it.
3530 #[tokio::test]
3531 async fn node_health_blocks_with_no_nodes() {
3532 let pool = SqlitePoolOptions::new()
3533 .max_connections(1)
3534 .connect("sqlite::memory:")
3535 .await
3536 .unwrap();
3537 crate::db::migrate(&pool).await.unwrap();
3538 sqlx::query(
3539 "INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('b', 2, 1, 'sequential')",
3540 )
3541 .execute(&pool)
3542 .await
3543 .unwrap();
3544 sqlx::query("INSERT INTO tier_state (tier) VALUES ('b')")
3545 .execute(&pool)
3546 .await
3547 .unwrap();
3548 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
3549 .execute(&pool).await.unwrap();
3550
3551 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests());
3552 let ctx = GateCtx {
3553 public_url: None,
3554 pool: pool.clone(),
3555 cfg,
3556 tier: TierId::new("b"),
3557 version: "0.1.0".parse().unwrap(),
3558 worktree: None,
3559 bundle: None,
3560 events: events::channel(),
3561 nodes: Vec::new(), // no nodes -> fail closed
3562 build_id: None,
3563 aux_dirs: HashMap::new(),
3564 };
3565 let out = run(&ctx, &Gate::NodeHealth).await.unwrap();
3566 assert_eq!(out.status_str(), "blocked");
3567 assert!(!out.is_passed());
3568 let row: (Option<String>, Option<String>) =
3569 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
3570 .fetch_one(&pool)
3571 .await
3572 .unwrap();
3573 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
3574 assert_eq!(json["status"]["blocker"]["kind"], "no_nodes_to_probe");
3575 }
3576
3577 /// reset_scratch must drop every non-system schema, not just `public` —
3578 /// otherwise migrations that create custom schemas (e.g. tower_sessions)
3579 /// collide on the next run. This regressed once (Phase 0) and the fix is
3580 /// load-bearing for migration_dry_run.
3581 ///
3582 /// Gated on `SANDO_TEST_PG_URL` so it only runs where postgres is
3583 /// available. Set `SANDO_TEST_PG_URL=postgres:///sando_scratch?host=/var/run/postgresql`
3584 /// (or similar) before `cargo test`.
3585 #[tokio::test]
3586 async fn reset_scratch_drops_all_non_system_schemas() {
3587 let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else {
3588 eprintln!("skipping: SANDO_TEST_PG_URL not set");
3589 return;
3590 };
3591 use sqlx::Executor;
3592 use sqlx::postgres::PgPoolOptions;
3593
3594 let pool = PgPoolOptions::new()
3595 .max_connections(1)
3596 .connect(&url)
3597 .await
3598 .unwrap();
3599 // Plant two non-system schemas + a table in each.
3600 pool.execute(
3601 "DROP SCHEMA IF EXISTS foo CASCADE; CREATE SCHEMA foo; CREATE TABLE foo.t (i int);",
3602 )
3603 .await
3604 .unwrap();
3605 pool.execute("DROP SCHEMA IF EXISTS tower_sessions CASCADE; CREATE SCHEMA tower_sessions; CREATE TABLE tower_sessions.session (id text);")
3606 .await.unwrap();
3607 pool.close().await;
3608
3609 reset_scratch(&url, "makenotwork")
3610 .await
3611 .expect("reset_scratch");
3612
3613 let pool = PgPoolOptions::new()
3614 .max_connections(1)
3615 .connect(&url)
3616 .await
3617 .unwrap();
3618 let rows: Vec<(String,)> = sqlx::query_as(
3619 "SELECT nspname FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname <> 'information_schema'",
3620 )
3621 .fetch_all(&pool)
3622 .await
3623 .unwrap();
3624 let names: Vec<String> = rows.into_iter().map(|(s,)| s).collect();
3625 // After reset, only `public` should remain among non-system schemas.
3626 assert_eq!(names, vec!["public".to_string()], "got: {names:?}");
3627 pool.close().await;
3628 }
3629
3630 /// reset_scratch must leave the dump's owner role existing and able to
3631 /// create in `public`, because a prod `pg_dump` carries `ALTER ... OWNER TO
3632 /// <role>` for every object. This was satisfied by a hand-created NOLOGIN
3633 /// role on fw13; nothing recorded it, so any other box failed
3634 /// migration_dry_run at the restore with "role does not exist".
3635 ///
3636 /// Uses a throwaway role name so it can prove the *creation* path rather
3637 /// than passing on fw13's pre-existing `makenotwork`. Same
3638 /// `SANDO_TEST_PG_URL` gate as above; needs a superuser connection.
3639 #[tokio::test]
3640 async fn reset_scratch_seeds_the_dump_owner_role_when_absent() {
3641 let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else {
3642 eprintln!("skipping: SANDO_TEST_PG_URL not set");
3643 return;
3644 };
3645 use sqlx::Executor;
3646 use sqlx::postgres::PgPoolOptions;
3647
3648 let role = "sando_test_owner_probe";
3649 // `DROP ROLE` refuses while the role still holds the grants reset_scratch
3650 // gave it, so drop what it owns first. Idempotent, and a no-op when the
3651 // role is absent (the usual case on a first run).
3652 let drop_role = format!(
3653 "DO $$ BEGIN
3654 IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{role}') THEN
3655 EXECUTE 'DROP OWNED BY {role}';
3656 EXECUTE 'DROP ROLE {role}';
3657 END IF;
3658 END $$;"
3659 );
3660
3661 let pool = PgPoolOptions::new()
3662 .max_connections(1)
3663 .connect(&url)
3664 .await
3665 .unwrap();
3666 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(drop_role.clone())))
3667 .await
3668 .unwrap();
3669 pool.close().await;
3670
3671 reset_scratch(&url, role)
3672 .await
3673 .expect("reset_scratch creates the owner role");
3674
3675 let pool = PgPoolOptions::new()
3676 .max_connections(1)
3677 .connect(&url)
3678 .await
3679 .unwrap();
3680 let (exists, can_login): (bool, bool) =
3681 sqlx::query_as("SELECT true, rolcanlogin FROM pg_roles WHERE rolname = $1")
3682 .bind(role)
3683 .fetch_one(&pool)
3684 .await
3685 .expect("owner role exists after reset");
3686 assert!(exists);
3687 assert!(
3688 !can_login,
3689 "the owner role is an owner only, never a login identity"
3690 );
3691
3692 // The restore's owner-scoped DDL needs CREATE on public in the role's
3693 // own right (PG15+ dropped the implicit grant).
3694 let (has_create,): (bool,) =
3695 sqlx::query_as("SELECT pg_catalog.has_schema_privilege($1, 'public', 'CREATE')")
3696 .bind(role)
3697 .fetch_one(&pool)
3698 .await
3699 .unwrap();
3700 assert!(has_create, "owner role must be able to create in public");
3701
3702 // Idempotent: a second reset must not error on the now-existing role.
3703 pool.close().await;
3704 reset_scratch(&url, role)
3705 .await
3706 .expect("reset_scratch is idempotent");
3707
3708 let pool = PgPoolOptions::new()
3709 .max_connections(1)
3710 .connect(&url)
3711 .await
3712 .unwrap();
3713 pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(drop_role)))
3714 .await
3715 .unwrap();
3716 pool.close().await;
3717 }
3718
3719 /// The preflight must pass against a privileged scratch connection. Guards
3720 /// the catalog query itself: a wrong column or a `current_user` that matches
3721 /// no `pg_roles` row would make `fetch_one` error (or, worse, a silently
3722 /// swapped pair would invert the check) and brick startup for everyone.
3723 /// `SANDO_TEST_PG_URL` is expected to be a superuser connection, as the
3724 /// gates require.
3725 #[tokio::test]
3726 async fn preflight_passes_on_a_privileged_scratch_connection() {
3727 let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else {
3728 eprintln!("skipping: SANDO_TEST_PG_URL not set");
3729 return;
3730 };
3731 preflight_scratch_privileges(&url)
3732 .await
3733 .expect("a superuser scratch connection must satisfy the preflight");
3734 }
3735
3736 /// CF4: the restore pipeline must carry `ON_ERROR_STOP=1` (so psql fails on
3737 /// a bad statement instead of exiting 0 on a partial restore) and, for a
3738 /// gzip source, `set -o pipefail` (so a `gunzip` failure on a truncated
3739 /// archive isn't masked by psql's exit). Pure string check — no postgres.
3740 #[test]
3741 fn restore_shell_has_error_stop_and_pipefail() {
3742 let gz = restore_shell("postgres:///scratch", "/srv/sando/backups/latest.sql.gz");
3743 assert!(gz.contains("ON_ERROR_STOP=1"), "gz: {gz}");
3744 assert!(gz.contains("set -o pipefail"), "gz: {gz}");
3745 assert!(gz.contains("gunzip -c"), "gz: {gz}");
3746
3747 let plain = restore_shell("postgres:///scratch", "/srv/sando/backups/dump.sql");
3748 assert!(plain.contains("ON_ERROR_STOP=1"), "plain: {plain}");
3749 // No pipeline for a plain .sql, so pipefail is unnecessary there.
3750 assert!(!plain.contains("gunzip"), "plain: {plain}");
3751 // The db url is single-quote escaped in both forms.
3752 assert!(plain.contains("'postgres:///scratch'"), "plain: {plain}");
3753 }
3754
3755 #[test]
3756 fn split_pg_password_extracts_and_sanitizes() {
3757 // Password lifted out of the URL; the sanitized form keeps user/host/db.
3758 let (url, pw) = split_pg_password("postgres://sando:s3cret@db.host:5432/scratch");
3759 assert_eq!(url, "postgres://sando@db.host:5432/scratch");
3760 assert_eq!(pw.as_deref(), Some("s3cret"));
3761 // Percent-encoded password is decoded for PGPASSWORD.
3762 let (url, pw) = split_pg_password("postgresql://u:p%40ss%2Fword@h/d");
3763 assert_eq!(url, "postgresql://u@h/d");
3764 assert_eq!(pw.as_deref(), Some("p@ss/word"));
3765 }
3766
3767 #[test]
3768 fn split_pg_password_noop_without_password() {
3769 // No userinfo password -> unchanged, None. (A ':' after the '@', e.g. a
3770 // port, must not be mistaken for the password delimiter.)
3771 assert_eq!(
3772 split_pg_password("postgres:///scratch"),
3773 ("postgres:///scratch".to_string(), None),
3774 );
3775 assert_eq!(
3776 split_pg_password("postgres://sando@db.host:5432/scratch"),
3777 ("postgres://sando@db.host:5432/scratch".to_string(), None),
3778 );
3779 }
3780
3781 #[test]
3782 fn percent_decode_handles_escapes_and_malformed() {
3783 assert_eq!(percent_decode("plain"), "plain");
3784 assert_eq!(percent_decode("a%2Fb"), "a/b");
3785 // A malformed trailing escape is left literal, not dropped.
3786 assert_eq!(percent_decode("ab%2"), "ab%2");
3787 assert_eq!(percent_decode("ab%zz"), "ab%zz");
3788 }
3789
3790 #[test]
3791 fn pg_url_with_dbname_rewrites_the_database() {
3792 // user:pass@host:port/db?query — swap db, keep everything else.
3793 assert_eq!(
3794 pg_url_with_dbname(
3795 "postgres://sando:pw@db.host:5432/sando_scratch?sslmode=require",
3796 "postgres"
3797 ),
3798 "postgres://sando:pw@db.host:5432/postgres?sslmode=require",
3799 );
3800 // Socket form: the query carries `host=/var/run/postgresql` and must survive.
3801 assert_eq!(
3802 pg_url_with_dbname(
3803 "postgres:///sando_scratch?host=/var/run/postgresql",
3804 "sando_code_smoke_0_9_6"
3805 ),
3806 "postgres:///sando_code_smoke_0_9_6?host=/var/run/postgresql",
3807 );
3808 // Plain host/db, no query.
3809 assert_eq!(
3810 pg_url_with_dbname("postgres://localhost/scratch", "postgres"),
3811 "postgres://localhost/scratch".replace("scratch", "postgres"),
3812 );
3813 // No authority, no query (loopback socket, default db path).
3814 assert_eq!(
3815 pg_url_with_dbname("postgres:///scratch", "postgres"),
3816 "postgres:///postgres",
3817 );
3818 }
3819
3820 #[test]
3821 fn bytes_contain_matches_listening_in_log_output() {
3822 // JSON release log carries the message field verbatim.
3823 assert!(bytes_contain(
3824 br#"{"timestamp":"...","level":"INFO","fields":{"message":"listening","addr":"127.0.0.1:18182"}}"#,
3825 b"listening",
3826 ));
3827 // Human-format dev log.
3828 assert!(bytes_contain(
3829 b"2026-07-17 INFO makenotwork: listening addr=127.0.0.1:18182",
3830 b"listening"
3831 ));
3832 assert!(!bytes_contain(
3833 b"migrations complete; seeding catalog",
3834 b"listening"
3835 ));
3836 assert!(!bytes_contain(b"", b"listening"));
3837 }
3838
3839 #[tokio::test]
3840 async fn code_smoke_env_supplies_every_mandatory_server_var() {
3841 // code_smoke is the only gate that reaches the server's Config::from_env
3842 // (boot_smoke and the docs check short-circuit before it), so anything
3843 // that function requires has to be answered here. CDN_BASE_URL became
3844 // mandatory 13 days after this env was written and went unnoticed until
3845 // the gate first ran on the host; this test is what makes the next one
3846 // fail here instead of in a promote.
3847 let ctx = resolving_ctx("/w/abc", &[]);
3848 let mut cmd = tokio::process::Command::new("true");
3849 code_smoke_env(&mut cmd, &ctx, "postgres:///throwaway");
3850 let set: std::collections::HashMap<String, String> = cmd
3851 .as_std()
3852 .get_envs()
3853 .filter_map(|(k, v)| Some((k.to_str()?.to_string(), v?.to_str()?.to_string())))
3854 .collect();
3855 for key in [
3856 "DATABASE_URL",
3857 "HOST",
3858 "PORT",
3859 "HOST_URL",
3860 "CDN_BASE_URL",
3861 "SIGNING_SECRET",
3862 ] {
3863 assert!(set.contains_key(key), "code_smoke_env must set {key}");
3864 assert!(!set[key].is_empty(), "{key} must not be empty");
3865 }
3866 // Loopback, so Config::from_env's is_production branch stays false and
3867 // the gate never trips MissingPublicBucket for want of an S3 bucket.
3868 assert!(set["HOST_URL"].starts_with("http://localhost"));
3869 // Not an IP literal: the server derives its WebAuthn rp_id from this
3870 // host, and WebauthnBuilder rejects an origin whose Url::domain() is
3871 // None, which is every IP address. An IP here cannot boot the server.
3872 assert!(
3873 url_host_is_a_domain(&set["HOST_URL"]),
3874 "HOST_URL host must be a domain, not an IP literal: {}",
3875 set["HOST_URL"],
3876 );
3877 assert_eq!(set["HOST"], "127.0.0.1");
3878 // The signing secret has to clear the server's 32-char floor, or the
3879 // gate fails with WeakSigningSecret instead of testing anything.
3880 assert!(set["SIGNING_SECRET"].len() >= 32);
3881 }
3882
3883 #[tokio::test]
3884 async fn code_smoke_env_passes_extras_through_but_never_lets_them_win() {
3885 // The pass-through exists so a product can hand its own binary a var
3886 // sando has no business knowing about (MNW points SEED_MEDIA_CACHE at a
3887 // persistent dir, because PrivateTmp=true made the seed's media cache
3888 // cold on every build). What it must never become is a way to aim a
3889 // smoke run at a real database.
3890 let mut cfg = crate::config::AppConfig::for_tests();
3891 cfg.code_smoke_env = [
3892 (
3893 "SEED_MEDIA_CACHE".to_string(),
3894 "/srv/sando/seed".to_string(),
3895 ),
3896 (
3897 "DATABASE_URL".to_string(),
3898 "postgres://prod-1/makenotwork".to_string(),
3899 ),
3900 ]
3901 .into_iter()
3902 .collect();
3903 let mut ctx = resolving_ctx("/w/abc", &[]);
3904 ctx.cfg = std::sync::Arc::new(cfg);
3905
3906 let mut cmd = tokio::process::Command::new("true");
3907 code_smoke_env(&mut cmd, &ctx, "postgres:///throwaway");
3908 let set: std::collections::HashMap<String, String> = cmd
3909 .as_std()
3910 .get_envs()
3911 .filter_map(|(k, v)| Some((k.to_str()?.to_string(), v?.to_str()?.to_string())))
3912 .collect();
3913
3914 assert_eq!(set["SEED_MEDIA_CACHE"], "/srv/sando/seed");
3915 assert_eq!(
3916 set["DATABASE_URL"], "postgres:///throwaway",
3917 "the fixed set must overwrite a colliding extra, or a config typo \
3918 could point code_smoke at a real database",
3919 );
3920 }
3921
3922 #[test]
3923 fn code_smoke_db_name_sanitizes_and_caps() {
3924 assert_eq!(
3925 code_smoke_db_name(&"0.9.6".parse().unwrap()),
3926 "sando_code_smoke_0_9_6"
3927 );
3928 // Pre-release/build metadata folds to underscores; result stays [a-z0-9_].
3929 let n = code_smoke_db_name(&"1.0.0-rc.1+build".parse().unwrap());
3930 assert_eq!(n, "sando_code_smoke_1_0_0_rc_1_build");
3931 assert!(
3932 n.bytes()
3933 .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
3934 );
3935 assert!(n.len() <= 63);
3936 }
3937
3938 /// code_smoke is Blocked (not Failed) when the daemon has no scratch_db_url:
3939 /// there's no cluster to create the throwaway DB in, and that's an operator
3940 /// precondition, rendered yellow — the same shape as migration_dry_run.
3941 #[tokio::test]
3942 async fn code_smoke_blocks_without_scratch_db_url() {
3943 let pool = SqlitePoolOptions::new()
3944 .max_connections(1)
3945 .connect("sqlite::memory:")
3946 .await
3947 .unwrap();
3948 crate::db::migrate(&pool).await.unwrap();
3949 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 1, 'sequential')")
3950 .execute(&pool).await.unwrap();
3951 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
3952 .execute(&pool)
3953 .await
3954 .unwrap();
3955 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')")
3956 .execute(&pool).await.unwrap();
3957
3958 let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests()); // scratch_db_url: None
3959 let ctx = GateCtx {
3960 public_url: None,
3961 pool: pool.clone(),
3962 cfg,
3963 tier: TierId::new("host"),
3964 version: "0.1.0".parse().unwrap(),
3965 worktree: Some(std::path::PathBuf::from("/tmp/unused")),
3966 bundle: None,
3967 events: events::channel(),
3968 nodes: Vec::new(),
3969 build_id: None,
3970 aux_dirs: HashMap::new(),
3971 };
3972 let out = run(&ctx, &Gate::CodeSmoke).await.unwrap();
3973 assert_eq!(out.status_str(), "blocked");
3974 assert!(!out.is_passed());
3975 let row: (Option<String>, Option<String>) =
3976 sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1")
3977 .fetch_one(&pool)
3978 .await
3979 .unwrap();
3980 assert_eq!(row.0.as_deref(), Some("blocked"));
3981 let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap();
3982 assert_eq!(json["status"]["blocker"]["kind"], "scratch_db_url_unset");
3983 }
3984
3985 /// Sanity: applying MNW migrations from a *non-existent* dir errors,
3986 /// rather than silently no-op'ing. Cheap pure check, no postgres needed
3987 /// (the sqlx::Migrator::new constructor itself reads the dir).
3988 #[tokio::test]
3989 async fn run_migrator_errors_on_missing_dir() {
3990 // The first thing run_migrator does is `Migrator::new(dir)`, which
3991 // needs a real dir to read migration files from.
3992 let res = run_migrator(
3993 "postgres:///does-not-matter",
3994 std::path::Path::new("/nonexistent/sando-test-migrations"),
3995 )
3996 .await;
3997 assert!(res.is_err());
3998 }
3999 }
4000