Skip to main content

max / makenotwork

38.2 KB · 1004 lines History Blame Raw
1 //! The cargo-shaped gates: test, clippy, fmt, supply chain and the hardening
2 //! build, plus the machinery that runs one command over a list of targets and
3 //! turns its output into a failure note.
4
5 use super::GateCtx;
6 use super::log::{append_to_log, stream_child_to_live_log};
7 use super::pg::clean_stale_test_dbs;
8 use crate::classify;
9 use crate::domain::{GateKind, GateRunId};
10 use crate::outcome::{GateFailure, GateOutcome, PassNote};
11 use anyhow::Result;
12 use std::path::PathBuf;
13 use tokio::process::Command;
14
15 /// Run every configured `test_target`'s suite, in order, under one gate.
16 ///
17 /// Targets are configured (`[[test_target]]` in the daemon config), defaulting
18 /// to a single `server` entry. A crate with no target ships ungated, `mnw-cli`
19 /// included, which is built as a companion and installed onto prod-1 in the same
20 /// promote.
21 ///
22 /// The whole set shares one `gate_runs` row and one log file: from the
23 /// pipeline's point of view "the tests" either pass or don't. The first failing
24 /// target ends the gate, since a red suite blocks the promote regardless of what
25 /// the remaining crates would have said, and running them would only delay the
26 /// operator's answer. Its name is carried in the failure so the summary points
27 /// at the crate, not just the test.
28 pub(super) async fn cargo_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
29 let log_path = ctx.log_path(GateKind::CargoTest);
30 let log_ref = ctx.log_ref(GateKind::CargoTest);
31
32 // Best-effort: drop our own role's stale `mnw_test_*` databases (the
33 // template + any per-test clones orphaned by a previously-killed run)
34 // before the suite, so they can't accumulate or collide. Foreign-owned
35 // leftovers are left alone — the harness now namespaces its template per
36 // role, so they no longer wedge the gate.
37 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
38 clean_stale_test_dbs(scratch_url).await;
39 }
40
41 let started = std::time::Instant::now();
42 // One ceiling for the whole gate, not per target: the point is to bound how
43 // long a hung suite can block the pipeline, and N targets each allowed the
44 // full timeout would multiply that by N.
45 let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
46 let mut ran = 0usize;
47
48 for target in &ctx.cfg.test_targets {
49 let label = target.label();
50 // A target absent from this sha is skipped, not fatal: sando has to be
51 // able to build older shas (bisect, rollback rebuild) from a config that
52 // describes the tip. The zero-targets-ran check below is what stops this
53 // from quietly turning the gate into a no-op.
54 let Some(dir) = ctx
55 .target_dir(target)
56 .filter(|d| d.join("Cargo.toml").is_file())
57 else {
58 tracing::warn!(
59 target = %label, version = %ctx.version,
60 "test_target has no Cargo.toml in this run; skipping",
61 );
62 continue;
63 };
64 let features: Vec<&str> = target.features.iter().map(String::as_str).collect();
65
66 // Fast pre-gate: compile the test targets WITHOUT running them. This
67 // builds the exact artifacts the full run needs (so the subsequent run
68 // reuses the cache — no wasted work), but fails in ~minutes with the
69 // real `error[Ennnn]: ...` on a test-only-target compile break. That
70 // class (a field missing in a `#[cfg(test)]`-only binary like `load`)
71 // otherwise compiles fine under the build step and only blows up here,
72 // after a full build, as an opaque mass test failure.
73 let banner = format!("\n==== test_target: {label} ====\n");
74 append_to_log(&log_path, banner.as_bytes()).await;
75 let mut pre = match cargo_test_command(ctx, &dir, target, &features, &["--no-run"]).spawn()
76 {
77 Ok(c) => c,
78 Err(e) => {
79 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
80 message: format!("{label}: {e}"),
81 })
82 .with_log_ref(log_ref));
83 }
84 };
85 let (pre_out, pre_err, pre_status) = match run_to_deadline_for(
86 &mut pre,
87 ctx,
88 run_id,
89 log_path.clone(),
90 deadline,
91 started,
92 GateKind::CargoTest,
93 )
94 .await?
95 {
96 Ok(v) => v,
97 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
98 };
99 if !pre_status.success() {
100 let failure = classify::classify_compile_error(&pre_out, &pre_err);
101 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
102 }
103
104 // Full run: the test binaries are already built above, so cargo's
105 // up-to-date check skips compilation and this just runs the tests.
106 //
107 // That claim is only true when the crate's build script is up to date
108 // too, and for a long time it was not. server and multithreaded both
109 // watched `.git/HEAD`, a path that does not exist at either package
110 // root, and cargo reads a missing watch as changed: the build script
111 // re-ran and the crate recompiled here, every time. It cost 347s a
112 // pipeline, 35% of this gate, while this comment said it cost nothing.
113 // Fixed 2026-08-20 in both build scripts; see `git_hash` in either.
114 //
115 // If this gate's duration ever climbs back toward the pre-pass's, look
116 // for a new phantom watch before looking anywhere else.
117 let mut child = match cargo_test_command(ctx, &dir, target, &features, &[]).spawn() {
118 Ok(c) => c,
119 Err(e) => {
120 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
121 message: format!("{label}: {e}"),
122 })
123 .with_log_ref(log_ref));
124 }
125 };
126 let (stdout_buf, stderr_buf, status) = match run_to_deadline_for(
127 &mut child,
128 ctx,
129 run_id,
130 log_path.clone(),
131 deadline,
132 started,
133 GateKind::CargoTest,
134 )
135 .await?
136 {
137 Ok(v) => v,
138 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
139 };
140 if !status.success() {
141 let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf);
142 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
143 }
144 ran += 1;
145 }
146
147 // Every configured target was missing from the worktree. Exiting green here
148 // would report "tests passed" having run none of them.
149 if ran == 0 {
150 return Ok(GateOutcome::failed(GateFailure::Unclassified {
151 legacy_detail: Some(format!(
152 "cargo_test ran no targets: none of the {} configured test_target dir(s) \
153 exist in this worktree",
154 ctx.cfg.test_targets.len(),
155 )),
156 })
157 .with_log_ref(log_ref));
158 }
159
160 let duration_s = started.elapsed().as_secs() as u32;
161 Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref))
162 }
163
164 /// Stream a child to the live log, bounded by the gate-wide `deadline`. `Ok(Err(_))`
165 /// is the timeout outcome (child killed); `Err(_)` is an IO error on the stream.
166 #[allow(clippy::type_complexity)]
167 async fn run_to_deadline_for(
168 child: &mut tokio::process::Child,
169 ctx: &GateCtx,
170 run_id: GateRunId,
171 log_path: PathBuf,
172 deadline: std::time::Instant,
173 started: std::time::Instant,
174 kind: GateKind,
175 ) -> Result<std::result::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus), GateOutcome>> {
176 let remaining = deadline.saturating_duration_since(std::time::Instant::now());
177 let stream = stream_child_to_live_log(child, ctx.events.clone(), run_id, log_path);
178 match tokio::time::timeout(remaining, stream).await {
179 Ok(res) => Ok(Ok(res?)),
180 Err(_elapsed) => {
181 child.start_kill().ok();
182 let _ = child.wait().await;
183 Ok(Err(GateOutcome::failed(GateFailure::Timeout {
184 gate: kind,
185 after_s: started.elapsed().as_secs() as u32,
186 })))
187 }
188 }
189 }
190
191 /// Prefix a test/compile failure's headline with the crate it came from, so a
192 /// red gate across many targets says *which* crate broke. Other failure kinds
193 /// are single-target by construction and pass through untouched.
194 fn name_target(failure: GateFailure, dir: &std::path::Path) -> GateFailure {
195 let at = dir.display();
196 match failure {
197 GateFailure::CargoTest {
198 failed_count,
199 first_failed,
200 first_panic,
201 } => GateFailure::CargoTest {
202 failed_count,
203 first_failed: Some(match first_failed {
204 Some(name) => format!("{at}: {name}"),
205 None => at.to_string(),
206 }),
207 first_panic,
208 },
209 GateFailure::CompileError {
210 error_count,
211 first_error,
212 } => GateFailure::CompileError {
213 error_count,
214 first_error: Some(match first_error {
215 Some(e) => format!("{at}: {e}"),
216 None => at.to_string(),
217 }),
218 },
219 other => other,
220 }
221 }
222
223 /// `cargo clippy --all-targets -- -D warnings` over every configured
224 /// `test_target`.
225 ///
226 /// The only thing standing between lint drift and prod.
227 pub(super) async fn clippy(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
228 lint_over_targets(ctx, run_id, GateKind::Clippy, |target, features| {
229 let mut args = vec!["clippy".to_string(), "--all-targets".to_string()];
230 if target.all_features {
231 args.push("--all-features".to_string());
232 } else if !features.is_empty() {
233 args.push("--features".to_string());
234 args.push(features.join(","));
235 }
236 // Everything after `--` goes to rustc, which is where -D lives.
237 args.push("--".to_string());
238 args.push("-D".to_string());
239 args.push("warnings".to_string());
240 args
241 })
242 .await
243 }
244
245 /// `cargo fmt --check` over every configured `test_target`.
246 ///
247 /// No `rustfmt.toml` exists anywhere in the tree, so this is plain rustfmt
248 /// defaults. Cheap: no compilation, just a parse.
249 pub(super) async fn fmt_check(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
250 lint_over_targets(ctx, run_id, GateKind::Fmt, |_target, _features| {
251 vec!["fmt".to_string(), "--check".to_string()]
252 })
253 .await
254 }
255
256 /// `cargo audit` / `cargo deny check`, over the crates that carry the matching
257 /// config file.
258 ///
259 /// Config-gated on purpose. Both tools are only meaningful against a triaged
260 /// posture: four crates in this repo fail `cargo audit` today purely because
261 /// they have no `.cargo/audit.toml` recording which transitive advisories have
262 /// been reviewed and accepted. Running them everywhere would make the gate
263 /// permanently and uninformatively red. Dropping the config file into a crate
264 /// is what opts it in.
265 pub(super) async fn supply_chain(
266 ctx: &GateCtx,
267 run_id: GateRunId,
268 kind: GateKind,
269 ) -> Result<GateOutcome> {
270 let (config_rel, args): (&str, Vec<String>) = match kind {
271 GateKind::CargoAudit => (".cargo/audit.toml", vec!["audit".into()]),
272 GateKind::CargoDeny => ("deny.toml", vec!["deny".into(), "check".into()]),
273 other => unreachable!("supply_chain called for {other:?}"),
274 };
275 run_over_targets(ctx, run_id, kind, |_target, target_dir| {
276 target_dir.join(config_rel).is_file().then(|| args.clone())
277 })
278 .await
279 }
280
281 /// Shared driver for the lint gates: run `cargo <args>` in every configured
282 /// `test_target` that exists in this worktree.
283 async fn lint_over_targets(
284 ctx: &GateCtx,
285 run_id: GateRunId,
286 kind: GateKind,
287 build_args: impl Fn(&crate::config::TestTarget, &[String]) -> Vec<String>,
288 ) -> Result<GateOutcome> {
289 // The target is handed in directly. This used to reverse-look-it-up by
290 // comparing `worktree.join(dir)` against the resolved path, which silently
291 // stopped matching for anything resolved anywhere else — an aux repo, say.
292 run_over_targets(ctx, run_id, kind, move |t, _dir| {
293 Some(build_args(t, &t.features))
294 })
295 .await
296 }
297
298 /// Run one cargo invocation per configured `test_target`, under a single
299 /// gate-wide deadline. `args_for` returns `None` to skip a target (used by the
300 /// supply-chain gates, which only apply where their config file lives).
301 ///
302 /// Shares `cargo_test`'s conventions: per-target log banners, stop at the first
303 /// failure with the crate named, skip targets absent from the worktree, and fail
304 /// closed if that leaves nothing to run.
305 async fn run_over_targets(
306 ctx: &GateCtx,
307 run_id: GateRunId,
308 kind: GateKind,
309 args_for: impl Fn(&crate::config::TestTarget, &std::path::Path) -> Option<Vec<String>>,
310 ) -> Result<GateOutcome> {
311 let log_path = ctx.log_path(kind);
312 let log_ref = ctx.log_ref(kind);
313 let started = std::time::Instant::now();
314 let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
315 let mut ran = 0usize;
316
317 for target in &ctx.cfg.test_targets {
318 let label = target.label();
319 let Some(dir) = ctx
320 .target_dir(target)
321 .filter(|d| d.join("Cargo.toml").is_file())
322 else {
323 tracing::warn!(
324 gate = kind.as_str(), target = %label,
325 "target has no Cargo.toml in this run; skipping",
326 );
327 continue;
328 };
329 let Some(args) = args_for(target, &dir) else {
330 continue;
331 };
332
333 append_to_log(
334 &log_path,
335 format!("\n==== {}: {label} ====\n", kind.as_str()).as_bytes(),
336 )
337 .await;
338
339 let mut cmd = Command::new("cargo");
340 cmd.args(&args)
341 .current_dir(&dir)
342 .stdout(std::process::Stdio::piped())
343 .stderr(std::process::Stdio::piped())
344 .kill_on_drop(true);
345 if let Some(t) = ctx.cfg.cargo_target_dir.as_deref() {
346 cmd.env("CARGO_TARGET_DIR", t);
347 }
348 // clippy type-checks, so it needs the same sqlx online-mode env the
349 // build and cargo_test steps get. fmt/audit/deny never touch the DB.
350 if kind == GateKind::Clippy
351 && let Some(url) = ctx
352 .cfg
353 .scratch_db_url
354 .as_deref()
355 .filter(|_| target.scratch_db)
356 {
357 cmd.env("DATABASE_URL", url);
358 cmd.env(
359 "TEST_DATABASE_URL",
360 url.split_once('?').map_or(url, |(b, _)| b),
361 );
362 }
363
364 let mut child = match cmd.spawn() {
365 Ok(c) => c,
366 Err(e) => {
367 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
368 message: format!("{label}: {e}"),
369 })
370 .with_log_ref(log_ref));
371 }
372 };
373 let (stdout_buf, stderr_buf, status) = match run_to_deadline_for(
374 &mut child,
375 ctx,
376 run_id,
377 log_path.clone(),
378 deadline,
379 started,
380 kind,
381 )
382 .await?
383 {
384 Ok(v) => v,
385 Err(timeout) => return Ok(timeout.with_log_ref(log_ref)),
386 };
387 if !status.success() {
388 let failure = match kind {
389 // clippy speaks rustc diagnostics, so the compile-error
390 // classifier extracts the real `error: ...` headline.
391 GateKind::Clippy => classify::classify_compile_error(&stdout_buf, &stderr_buf),
392 _ => GateFailure::Unclassified {
393 legacy_detail: Some(first_meaningful_line(&stdout_buf, &stderr_buf)),
394 },
395 };
396 return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref));
397 }
398 ran += 1;
399 }
400
401 if ran == 0 {
402 return Ok(GateOutcome::failed(GateFailure::Unclassified {
403 legacy_detail: Some(format!(
404 "{} ran nothing: no configured target in this worktree qualified",
405 kind.as_str(),
406 )),
407 })
408 .with_log_ref(log_ref));
409 }
410 Ok(GateOutcome::passed(PassNote::TestsPassed {
411 duration_s: started.elapsed().as_secs() as u32,
412 })
413 .with_log_ref(log_ref))
414 }
415
416 /// First line that looks like a diagnostic, for gates whose tools have no
417 /// dedicated classifier (`cargo audit`, `cargo deny`). Falls back to a generic
418 /// note rather than an empty string.
419 fn first_meaningful_line(stdout: &[u8], stderr: &[u8]) -> String {
420 for buf in [stderr, stdout] {
421 let text = String::from_utf8_lossy(buf);
422 if let Some(line) = text
423 .lines()
424 .map(str::trim)
425 .find(|l| l.starts_with("error") || l.contains("vulnerabilit") || l.contains("FAILED"))
426 {
427 return line.chars().take(200).collect();
428 }
429 }
430 "tool reported failure; see the gate log".into()
431 }
432
433 /// The tests `cargo_test` cannot reach, run against production constants.
434 ///
435 /// `cargo_test` builds with `--features fast-tests`, which relaxes
436 /// `AUTH_RATE_LIMIT_BURST` 5 → 20, `SANDBOX_RATE_LIMIT_MS` 30s → 10ms, and
437 /// argon2 from 46 MiB/t=2 to 8 MiB/t=1 — and the rate-limiting suite is
438 /// `#[cfg_attr(feature = "fast-tests", ignore)]`d on top of that, because a
439 /// bucket refilling at 100/sec never depletes under parallel test threads. The
440 /// net effect was that Sando's only code gate silently skipped every test of
441 /// the auth hardening it most needs to protect.
442 ///
443 /// So: no features, `--test-threads=1` (these tests key on a shared per-IP
444 /// bucket and must not interleave), and a name filter rather than the whole
445 /// suite — the rest of the suite is tuned for `fast-tests` and would only go
446 /// slow and flaky here. The filter is a substring match, so it catches
447 /// `..._rate_limit_...` and `..._rate_limited` alike.
448 ///
449 /// This costs a second compile of the lib + integration binary (a different
450 /// feature set is a different cfg, so no artifact sharing with `cargo_test`).
451 /// That is the price of the coverage; the filter keeps the *run* to seconds.
452 pub(super) async fn hardening_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> {
453 let server_dir = match ctx.worktree_for(GateKind::HardeningTest) {
454 Ok(w) => w.join("server"),
455 Err(outcome) => return Ok(outcome),
456 };
457 // No features is the whole point; the scratch DB is needed because the
458 // server's sqlx macros type-check against it. Unlike cargo_test, this gate
459 // is deliberately not driven by `test_targets`: it targets one specific
460 // suite in one specific crate, not "the repo's tests".
461 let target = crate::config::TestTarget {
462 dir: std::path::PathBuf::from("server"),
463 aux_repo: None,
464 features: Vec::new(),
465 all_features: false,
466 scratch_db: true,
467 };
468 let log_path = ctx.log_path(GateKind::HardeningTest);
469 let log_ref = ctx.log_ref(GateKind::HardeningTest);
470
471 if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() {
472 clean_stale_test_dbs(scratch_url).await;
473 }
474
475 let started = std::time::Instant::now();
476
477 // Same two-step shape as cargo_test: compile first so a test-target break
478 // reports as a compile error rather than an opaque mass test failure.
479 let mut pre = match cargo_test_command(
480 ctx,
481 &server_dir,
482 &target,
483 &[],
484 &["--no-run", "--test", "integration"],
485 )
486 .spawn()
487 {
488 Ok(c) => c,
489 Err(e) => {
490 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
491 message: e.to_string(),
492 })
493 .with_log_ref(log_ref));
494 }
495 };
496 let (pre_out, pre_err, pre_status) =
497 stream_child_to_live_log(&mut pre, ctx.events.clone(), run_id, log_path.clone()).await?;
498 if !pre_status.success() {
499 let failure = classify::classify_compile_error(&pre_out, &pre_err);
500 return Ok(GateOutcome::failed(failure).with_log_ref(log_ref));
501 }
502
503 let mut child = match cargo_test_command(
504 ctx,
505 &server_dir,
506 &target,
507 &[],
508 &[
509 "--test",
510 "integration",
511 "--",
512 "--test-threads=1",
513 "rate_limit",
514 ],
515 )
516 .spawn()
517 {
518 Ok(c) => c,
519 Err(e) => {
520 return Ok(GateOutcome::failed(GateFailure::SpawnFailed {
521 message: e.to_string(),
522 })
523 .with_log_ref(log_ref));
524 }
525 };
526 let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs);
527 let stream = stream_child_to_live_log(&mut child, ctx.events.clone(), run_id, log_path);
528 let (stdout_buf, stderr_buf, status) = match tokio::time::timeout(ceiling, stream).await {
529 Ok(res) => res?,
530 Err(_elapsed) => {
531 child.start_kill().ok();
532 let _ = child.wait().await;
533 return Ok(GateOutcome::failed(GateFailure::Timeout {
534 gate: GateKind::HardeningTest,
535 after_s: started.elapsed().as_secs() as u32,
536 })
537 .with_log_ref(log_ref));
538 }
539 };
540 let duration_s = started.elapsed().as_secs() as u32;
541 if status.success() {
542 // A filter that matches nothing exits 0, which would make this gate a
543 // green no-op the day someone renames the tests. Fail closed instead.
544 if tests_run(&stdout_buf) == 0 {
545 return Ok(GateOutcome::failed(GateFailure::Unclassified {
546 legacy_detail: Some(
547 "hardening_test ran 0 tests: the `rate_limit` filter matched nothing. \
548 The suite was renamed or moved — this gate is proving nothing."
549 .into(),
550 ),
551 })
552 .with_log_ref(log_ref));
553 }
554 Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref))
555 } else {
556 let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf);
557 Ok(GateOutcome::failed(failure).with_log_ref(log_ref))
558 }
559 }
560
561 /// Count the tests libtest reports as run, from its `test result: ok. N passed`
562 /// summary line. Returns 0 when no summary is present, which is itself the
563 /// "nothing ran" case the caller fails on.
564 fn tests_run(stdout: &[u8]) -> u32 {
565 String::from_utf8_lossy(stdout)
566 .lines()
567 .filter_map(|l| l.trim().strip_prefix("test result:"))
568 .filter_map(|rest| rest.split_once(" passed"))
569 .filter_map(|(head, _)| {
570 head.rsplit(' ')
571 .find(|t| !t.is_empty())?
572 .parse::<u32>()
573 .ok()
574 })
575 .sum()
576 }
577
578 /// Configure (but don't spawn) `cargo test --release [--features <features>]
579 /// <extra>` in `dir`, wired to the scratch DB. Shared by the `--no-run`
580 /// pre-gate compile and the full test run so both go through one env setup,
581 /// and by both test gates so they can't drift apart on env.
582 ///
583 /// `cargo_test` passes `fast-tests`, matching what the retired astra CI did:
584 /// it relaxes the auth rate-limit burst (5 → 20) and argon2 cost so
585 /// signup-heavy + lockout workflow tests complete without hitting Governor
586 /// before the hand-rolled lockout check (documented at
587 /// `server/src/constants.rs:87`). `hardening_test` passes no features, which is
588 /// the whole point of that gate — see `GateKind::HardeningTest`.
589 fn cargo_test_command(
590 ctx: &GateCtx,
591 dir: &std::path::Path,
592 target: &crate::config::TestTarget,
593 features: &[&str],
594 extra: &[&str],
595 ) -> Command {
596 let mut cmd = Command::new("cargo");
597 cmd.args(["test", "--release"]);
598 if target.all_features {
599 cmd.arg("--all-features");
600 } else if !features.is_empty() {
601 cmd.args(["--features", &features.join(",")]);
602 }
603 cmd.args(extra)
604 .current_dir(dir)
605 .stdout(std::process::Stdio::piped())
606 .stderr(std::process::Stdio::piped())
607 .kill_on_drop(true);
608 // Share the build step's target dir so the test compile reuses its
609 // artifacts (and the `--no-run` precompile reuses them again). Must match
610 // `build.rs` or the gate would clean-compile the whole tree a second time.
611 if let Some(target) = ctx.cfg.cargo_target_dir.as_deref() {
612 cmd.env("CARGO_TARGET_DIR", target);
613 }
614 // Same online-mode rationale as the build step: sqlx query macros need a
615 // live DB to type-check against. The scratch DB is left in migrated state
616 // by the preceding build, so we can reuse it here.
617 //
618 // Opt-in per target: setting DATABASE_URL switches sqlx OUT of offline mode,
619 // so a crate that ships `.sqlx` query data would stop using it and try to
620 // type-check against a database that has none of its tables.
621 if let Some(scratch_url) = ctx
622 .cfg
623 .scratch_db_url
624 .as_deref()
625 .filter(|_| target.scratch_db)
626 {
627 cmd.env("DATABASE_URL", scratch_url);
628 // The server test harness (tests/harness/db.rs) parses TEST_DATABASE_URL
629 // with rfind('/'), which mangles URLs whose query string contains '/'
630 // (e.g. `?host=/var/run/postgresql`). Strip the query — libpq defaults
631 // to /var/run/postgresql on Debian/Ubuntu when host is unspecified.
632 let test_url = scratch_url
633 .split_once('?')
634 .map_or(scratch_url, |(base, _)| base);
635 cmd.env("TEST_DATABASE_URL", test_url);
636 }
637 cmd
638 }
639
640 #[cfg(test)]
641 mod tests {
642 use super::*;
643 use crate::domain::TierId;
644 use crate::events;
645 use crate::gates::testkit::target;
646 use sqlx::sqlite::SqlitePoolOptions;
647 use std::collections::HashMap;
648
649 #[test]
650 fn name_target_points_a_test_failure_at_its_crate() {
651 // With one target the crate was implicit; with fifteen the operator
652 // needs the summary to say which one broke.
653 let f = name_target(
654 GateFailure::CargoTest {
655 failed_count: 3,
656 first_failed: Some("workflows::sync::round_trip".into()),
657 first_panic: None,
658 },
659 std::path::Path::new("shared/synckit-client"),
660 );
661 assert_eq!(
662 f.summary(),
663 "3 test(s) failed; first: shared/synckit-client: workflows::sync::round_trip",
664 );
665 }
666
667 #[test]
668 fn name_target_points_a_compile_failure_at_its_crate() {
669 let f = name_target(
670 GateFailure::CompileError {
671 error_count: 1,
672 first_error: Some("error[E0063]".into()),
673 },
674 std::path::Path::new("mnw-cli"),
675 );
676 assert_eq!(
677 f.summary(),
678 "compile failed (1 error(s)); first: mnw-cli: error[E0063]"
679 );
680 }
681
682 #[test]
683 fn name_target_names_the_crate_even_without_a_test_name() {
684 let f = name_target(
685 GateFailure::CargoTest {
686 failed_count: 2,
687 first_failed: None,
688 first_panic: None,
689 },
690 std::path::Path::new("pom"),
691 );
692 assert_eq!(f.summary(), "2 test(s) failed; first: pom");
693 }
694
695 #[test]
696 fn name_target_leaves_unrelated_failures_alone() {
697 let f = name_target(
698 GateFailure::SpawnFailed {
699 message: "no cargo".into(),
700 },
701 std::path::Path::new("pom"),
702 );
703 assert!(matches!(f, GateFailure::SpawnFailed { .. }));
704 }
705
706 #[tokio::test]
707 async fn cargo_test_fails_closed_when_no_target_exists_in_the_worktree() {
708 // A worktree missing every configured crate must not report "tests
709 // passed" having run none. Uses an empty tempdir as the worktree, so
710 // no cargo process is ever spawned.
711 let tmp = tempfile::tempdir().unwrap();
712 let mut cfg = crate::config::AppConfig::for_tests();
713 cfg.test_targets = vec![target("server"), target("mnw-cli")];
714 cfg.logs_root = tmp.path().join("logs");
715 let pool = SqlitePoolOptions::new()
716 .max_connections(1)
717 .connect("sqlite::memory:")
718 .await
719 .unwrap();
720 let ctx = GateCtx {
721 public_url: None,
722 pool,
723 cfg: std::sync::Arc::new(cfg),
724 tier: TierId::new("host"),
725 version: "0.1.0".parse().unwrap(),
726 worktree: Some(tmp.path().to_path_buf()),
727 bundle: None,
728 events: events::channel(),
729 nodes: Vec::new(),
730 build_id: None,
731 aux_dirs: HashMap::new(),
732 };
733 let out = cargo_test(&ctx, GateRunId(1)).await.unwrap();
734 assert_eq!(
735 out.status_str(),
736 "failed",
737 "green here would be a silent no-op gate"
738 );
739 let crate::outcome::GateStatus::Failed { failure } = &out.status else {
740 panic!("expected a failure")
741 };
742 assert!(
743 failure.summary().contains("ran no targets"),
744 "got: {}",
745 failure.summary()
746 );
747 }
748
749 #[tokio::test]
750 async fn scratch_db_env_is_opt_in_per_target() {
751 // Exporting DATABASE_URL knocks sqlx out of offline mode, so a crate
752 // shipping .sqlx data must not see it.
753 let mut cfg = crate::config::AppConfig::for_tests();
754 cfg.scratch_db_url = Some("postgres://sando@127.0.0.1/sando_scratch".into());
755 let ctx = GateCtx {
756 public_url: None,
757 pool: SqlitePoolOptions::new()
758 .max_connections(1)
759 .connect_lazy("sqlite::memory:")
760 .unwrap(),
761 cfg: std::sync::Arc::new(cfg),
762 tier: TierId::new("host"),
763 version: "0.1.0".parse().unwrap(),
764 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
765 bundle: None,
766 events: events::channel(),
767 nodes: Vec::new(),
768 build_id: None,
769 aux_dirs: HashMap::new(),
770 };
771 let dir = std::path::Path::new("/tmp/wt/x");
772
773 let off = cargo_test_command(&ctx, dir, &target("x"), &[], &[]);
774 let has_db = |c: &Command| {
775 c.as_std()
776 .get_envs()
777 .any(|(k, v)| k == "DATABASE_URL" && v.is_some())
778 };
779 assert!(!has_db(&off), "scratch_db defaults off");
780
781 let mut on_target = target("x");
782 on_target.scratch_db = true;
783 assert!(
784 has_db(&cargo_test_command(&ctx, dir, &on_target, &[], &[])),
785 "opt-in exports it"
786 );
787 }
788
789 #[tokio::test]
790 async fn all_features_replaces_the_feature_list() {
791 let ctx = GateCtx {
792 public_url: None,
793 pool: SqlitePoolOptions::new()
794 .max_connections(1)
795 .connect_lazy("sqlite::memory:")
796 .unwrap(),
797 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
798 tier: TierId::new("host"),
799 version: "0.1.0".parse().unwrap(),
800 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
801 bundle: None,
802 events: events::channel(),
803 nodes: Vec::new(),
804 build_id: None,
805 aux_dirs: HashMap::new(),
806 };
807 let mut t = target("shared/ops-exec");
808 t.all_features = true;
809 let cmd = cargo_test_command(&ctx, std::path::Path::new("/tmp/wt"), &t, &[], &[]);
810 let args: Vec<_> = cmd
811 .as_std()
812 .get_args()
813 .map(|a| a.to_string_lossy().into_owned())
814 .collect();
815 assert!(args.iter().any(|a| a == "--all-features"), "got: {args:?}");
816 assert!(!args.iter().any(|a| a == "--features"), "got: {args:?}");
817 }
818
819 #[test]
820 fn first_meaningful_line_prefers_the_diagnostic() {
821 let stderr = b" Updating crates.io index\nerror: 1 vulnerability found!\n";
822 assert_eq!(
823 first_meaningful_line(b"", stderr),
824 "error: 1 vulnerability found!"
825 );
826 }
827
828 #[test]
829 fn first_meaningful_line_finds_a_deny_verdict() {
830 let out = b"advisories FAILED, bans ok, licenses FAILED, sources ok\n";
831 assert!(first_meaningful_line(out, b"").contains("FAILED"));
832 }
833
834 #[test]
835 fn first_meaningful_line_falls_back_rather_than_returning_empty() {
836 assert!(first_meaningful_line(b"", b"").contains("see the gate log"));
837 }
838
839 #[tokio::test]
840 async fn supply_chain_gates_skip_crates_without_their_config() {
841 // Four crates in this repo fail `cargo audit` purely for want of a
842 // triaged .cargo/audit.toml. Running it there would make the gate
843 // permanently red, so a target only qualifies once it carries the file.
844 // The worktree here has a Cargo.toml but no audit config, so nothing
845 // qualifies and the gate fails closed rather than passing over zero work.
846 let tmp = tempfile::tempdir().unwrap();
847 let crate_dir = tmp.path().join("server");
848 std::fs::create_dir_all(&crate_dir).unwrap();
849 std::fs::write(crate_dir.join("Cargo.toml"), "[package]\nname = \"x\"\n").unwrap();
850
851 let mut cfg = crate::config::AppConfig::for_tests();
852 cfg.test_targets = vec![target("server")];
853 cfg.logs_root = tmp.path().join("logs");
854 let ctx = GateCtx {
855 public_url: None,
856 pool: SqlitePoolOptions::new()
857 .max_connections(1)
858 .connect("sqlite::memory:")
859 .await
860 .unwrap(),
861 cfg: std::sync::Arc::new(cfg),
862 tier: TierId::new("host"),
863 version: "0.1.0".parse().unwrap(),
864 worktree: Some(tmp.path().to_path_buf()),
865 bundle: None,
866 events: events::channel(),
867 nodes: Vec::new(),
868 build_id: None,
869 aux_dirs: HashMap::new(),
870 };
871 let out = supply_chain(&ctx, GateRunId(1), GateKind::CargoAudit)
872 .await
873 .unwrap();
874 assert_eq!(out.status_str(), "failed");
875 let crate::outcome::GateStatus::Failed { failure } = &out.status else {
876 panic!("expected a failure")
877 };
878 assert!(
879 failure.summary().contains("ran nothing"),
880 "got: {}",
881 failure.summary()
882 );
883
884 // Drop the config in and the same target now qualifies.
885 std::fs::create_dir_all(crate_dir.join(".cargo")).unwrap();
886 std::fs::write(crate_dir.join(".cargo/audit.toml"), "[advisories]\n").unwrap();
887 // The fixture crate is not a real cargo project, so the tool itself
888 // still errors — but on its own terms, not with "ran nothing". That
889 // distinction is the thing under test: the target was attempted.
890 let out = supply_chain(&ctx, GateRunId(2), GateKind::CargoAudit)
891 .await
892 .unwrap();
893 if let crate::outcome::GateStatus::Failed { failure } = &out.status {
894 assert!(
895 !failure.summary().contains("ran nothing"),
896 "a target carrying the config must be attempted, not skipped; got: {}",
897 failure.summary(),
898 );
899 }
900 }
901
902 #[test]
903 fn tests_run_reads_the_libtest_summary() {
904 let out = b"running 6 tests\ntest auth_rate_limit_triggers_on_burst ... ok\n\n\
905 test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 118 filtered out\n";
906 assert_eq!(tests_run(out), 6);
907 }
908
909 #[test]
910 fn tests_run_sums_across_test_binaries() {
911 let out = b"test result: ok. 6 passed; 0 failed\ntest result: ok. 2 passed; 0 failed\n";
912 assert_eq!(tests_run(out), 8);
913 }
914
915 #[test]
916 fn tests_run_is_zero_when_the_filter_matched_nothing() {
917 // The case hardening_test fails closed on: a filter matching no tests
918 // exits 0, so a rename would otherwise make the gate a green no-op.
919 let out = b"running 0 tests\n\n\
920 test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 124 filtered out\n";
921 assert_eq!(tests_run(out), 0);
922 }
923
924 #[test]
925 fn tests_run_is_zero_without_a_summary_line() {
926 assert_eq!(tests_run(b"error: could not compile `makenotwork`\n"), 0);
927 }
928
929 #[tokio::test]
930 async fn hardening_test_command_carries_no_features() {
931 // The entire point of the gate: production constants, which means no
932 // `fast-tests`. A stray feature here silently restores the blind spot.
933 let ctx = GateCtx {
934 public_url: None,
935 pool: SqlitePoolOptions::new()
936 .max_connections(1)
937 .connect_lazy("sqlite::memory:")
938 .unwrap(),
939 cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()),
940 tier: TierId::new("host"),
941 version: "0.1.0".parse().unwrap(),
942 worktree: Some(std::path::PathBuf::from("/tmp/wt")),
943 bundle: None,
944 events: events::channel(),
945 nodes: Vec::new(),
946 build_id: None,
947 aux_dirs: HashMap::new(),
948 };
949 let plain = crate::config::TestTarget {
950 dir: std::path::PathBuf::from("server"),
951 aux_repo: None,
952 features: Vec::new(),
953 all_features: false,
954 scratch_db: true,
955 };
956 let cmd = cargo_test_command(
957 &ctx,
958 std::path::Path::new("/tmp/wt/server"),
959 &plain,
960 &[],
961 &["--test", "integration"],
962 );
963 let args: Vec<_> = cmd
964 .as_std()
965 .get_args()
966 .map(|a| a.to_string_lossy().into_owned())
967 .collect();
968 assert!(
969 !args.iter().any(|a| a == "--features"),
970 "hardening_test must pass no features: {args:?}"
971 );
972 assert!(
973 !args.iter().any(|a| a.contains("fast-tests")),
974 "got: {args:?}"
975 );
976
977 let fast_target = crate::config::TestTarget {
978 dir: std::path::PathBuf::from("server"),
979 aux_repo: None,
980 features: vec!["fast-tests".into()],
981 all_features: false,
982 scratch_db: true,
983 };
984 let fast = cargo_test_command(
985 &ctx,
986 std::path::Path::new("/tmp/wt/server"),
987 &fast_target,
988 &["fast-tests"],
989 &[],
990 );
991 let fast_args: Vec<_> = fast
992 .as_std()
993 .get_args()
994 .map(|a| a.to_string_lossy().into_owned())
995 .collect();
996 assert!(
997 fast_args
998 .windows(2)
999 .any(|w| w == ["--features", "fast-tests"]),
1000 "got: {fast_args:?}"
1001 );
1002 }
1003 }
1004