Skip to main content

max / makenotwork

23.0 KB · 616 lines History Blame Raw
1 //! Gate-output classifiers.
2 //!
3 //! Each `classify_*` function takes the raw signals produced by a gate
4 //! runner (exit status, stdout/stderr tails, sqlx error strings) and
5 //! maps them to a typed `GateFailure`. Anything that doesn't match a
6 //! known pattern returns `GateFailure::Unclassified` with the original
7 //! detail attached — the on-disk gate log is the ultimate fallback.
8 //!
9 //! Classifiers are pure functions: no IO, no async. That makes them
10 //! fixture-testable, and it keeps the `gates.rs` runner code in charge
11 //! of side effects (process spawning, log persistence).
12
13 use crate::outcome::GateFailure;
14
15 /// `cargo_test`: derive a `CargoTest` failure with whatever counts can
16 /// be lifted out of the test runner's output.
17 ///
18 /// libtest emits a `test result: FAILED. P passed; F failed; ...` line
19 /// near the end of stdout. We grab `F` from that. If the output never
20 /// reached that line (compile error, runtime panic in the harness), we
21 /// fall through to `Unclassified`.
22 pub fn classify_cargo_test(stdout: &[u8], stderr: &[u8]) -> GateFailure {
23 let stdout_s = String::from_utf8_lossy(stdout);
24
25 let mut failed_count: u32 = 0;
26 let mut first_failed: Option<String> = None;
27
28 // `test result: FAILED. P passed; F failed; ...` lives near the
29 // end. Walk backwards to find it cheaply on very large outputs.
30 for line in stdout_s.lines().rev().take(50) {
31 if let Some(rest) = line.strip_prefix("test result: FAILED.") {
32 // Expect "P passed; F failed; ..."
33 for piece in rest.split(';') {
34 let p = piece.trim();
35 if let Some(num_str) = p.strip_suffix(" failed")
36 && let Ok(n) = num_str.parse::<u32>()
37 {
38 failed_count = n;
39 }
40 }
41 break;
42 }
43 }
44
45 // libtest prints "failures:\n foo::bar" near the end too. Grab
46 // the first one for the summary line.
47 if let Some(idx) = stdout_s.find("\nfailures:\n") {
48 // The "failures:" block repeats — once with stdout per failure, once as
49 // a plain name list. Either way the first non-empty line is a candidate.
50 if let Some(line) = stdout_s[idx + 11..].lines().next() {
51 let trimmed = line.trim();
52 if !trimmed.is_empty() {
53 first_failed = Some(trimmed.to_string());
54 }
55 }
56 }
57
58 if failed_count == 0 && first_failed.is_none() {
59 // Compile error or harness panic — no usable signal in stdout.
60 return GateFailure::Unclassified {
61 legacy_detail: Some(combined_tail_for_classifier(stdout, stderr)),
62 };
63 }
64
65 let first_panic = extract_first_panic(&stdout_s);
66 GateFailure::CargoTest {
67 failed_count,
68 first_failed,
69 first_panic,
70 }
71 }
72
73 /// Pull the first *root-cause* panic message out of libtest's captured
74 /// output. libtest (Rust 2021+) prints each captured panic as:
75 /// thread '<test>' panicked at <file>:<line>:<col>:
76 /// <message>
77 /// We return the first panic's message — but skip "...poisoned" messages in
78 /// favour of the first non-poison one, because a single real panic in shared
79 /// setup (a `std::sync::Once`) poisons it and makes every *other* test report
80 /// "Once instance has previously been poisoned". The root cause is the one
81 /// panic that isn't a poison report. Falls back to the first panic of any
82 /// kind if every message looks like poison.
83 fn extract_first_panic(stdout: &str) -> Option<String> {
84 let mut first: Option<String> = None;
85 let mut lines = stdout.lines();
86 while let Some(line) = lines.next() {
87 if !line.contains("panicked at ") {
88 continue;
89 }
90 // The message is the first non-empty line after the `panicked at` loc.
91 let msg = lines.by_ref().map(str::trim).find(|l| !l.is_empty());
92 let Some(msg) = msg else { continue };
93 if first.is_none() {
94 first = Some(msg.to_string());
95 }
96 let is_poison = msg.contains("poisoned") || msg.contains("PoisonError");
97 if !is_poison {
98 return Some(msg.to_string());
99 }
100 }
101 first
102 }
103
104 /// `cargo test --no-run` (the fast pre-gate compile): pull the first
105 /// compiler diagnostic out of cargo's stderr so a test-only-target
106 /// compile break (e.g. a missing struct field in a `#[cfg(test)]`-only
107 /// target) surfaces as the actual `error[E0063]: missing field ...`
108 /// line, instead of after a full build + a partial run reported as an
109 /// opaque "N tests failed".
110 ///
111 /// Cargo writes diagnostics to stderr. We prefer the first coded
112 /// `error[Ennnn]: ...` headline over the trailing `error: could not
113 /// compile <crate> ... due to N previous errors` summary, which names
114 /// the crate but not the cause; the summary still gives us the count.
115 pub fn classify_compile_error(stdout: &[u8], stderr: &[u8]) -> GateFailure {
116 let stderr_s = String::from_utf8_lossy(stderr);
117 let mut first_error: Option<String> = None;
118 let mut error_count: u32 = 0;
119
120 for line in stderr_s.lines() {
121 let t = line.trim_start();
122 if first_error.is_none() && t.starts_with("error[") {
123 first_error = Some(t.to_string());
124 }
125 if let Some(rest) = t.strip_prefix("error: could not compile")
126 && let Some(n) = parse_due_to_count(rest)
127 {
128 error_count = n;
129 }
130 }
131
132 // No coded diagnostic (e.g. a macro or resolver error prints a bare
133 // `error: ...`). Take the first such line that isn't cargo's own
134 // summary/abort noise.
135 if first_error.is_none() {
136 for line in stderr_s.lines() {
137 let t = line.trim_start();
138 if t.starts_with("error:")
139 && !t.starts_with("error: could not compile")
140 && !t.starts_with("error: aborting")
141 {
142 first_error = Some(t.to_string());
143 break;
144 }
145 }
146 }
147
148 if first_error.is_none() && error_count == 0 {
149 // Didn't look like a compile failure — don't masquerade as one.
150 return GateFailure::Unclassified {
151 legacy_detail: Some(combined_tail_for_classifier(stdout, stderr)),
152 };
153 }
154 GateFailure::CompileError {
155 error_count,
156 first_error,
157 }
158 }
159
160 /// Parse the count out of `... due to N previous error(s)`.
161 fn parse_due_to_count(s: &str) -> Option<u32> {
162 let idx = s.find("due to ")?;
163 let digits: String = s[idx + 7..]
164 .chars()
165 .take_while(char::is_ascii_digit)
166 .collect();
167 digits.parse().ok()
168 }
169
170 /// `migration_dry_run` is staged: scratch reset → restore dump → run
171 /// migrator. Each stage has its own failure mode. The caller (the gate
172 /// runner) knows which stage tripped; classifiers here turn the stage's
173 /// error string into a typed variant.
174 ///
175 /// Inputs are the migration name (when known) and the error string sqlx
176 /// returned. `migration` defaults to "?" when sqlx couldn't tell us
177 /// which file blew up.
178 pub fn classify_migration_error(err: &str, migration_hint: Option<&str>) -> GateFailure {
179 // sqlx::migrate::MigrateError variants are stringified consistently.
180 // Examples from `plans/migration-dryrun-failures.md`:
181 // "migration 47 was previously applied but is missing in the resolved migrations"
182 // "migration 47 was previously applied but has been modified"
183 // sqlx::Error::Database with sqlstate (e.g. "42P01" relation does not exist)
184
185 if let Some(m) = extract_drift(err) {
186 return GateFailure::MigrationDrift { migration: m };
187 }
188 if let Some(m) = extract_modified(err) {
189 return GateFailure::MigrationModified { migration: m };
190 }
191 let sqlstate = extract_sqlstate(err);
192 let migration = migration_hint.map_or_else(|| "?".to_owned(), str::to_owned);
193 if sqlstate.is_some() {
194 return GateFailure::MigrationSqlError {
195 migration,
196 sqlstate,
197 };
198 }
199 GateFailure::Unclassified {
200 legacy_detail: Some(err.chars().take(4_000).collect()),
201 }
202 }
203
204 fn extract_drift(err: &str) -> Option<String> {
205 // "migration N was previously applied but is missing in the resolved migrations"
206 let idx = err.find(" was previously applied but is missing")?;
207 let prefix = &err[..idx];
208 let mig = prefix.rsplit_once(' ').map_or(prefix, |(_, m)| m);
209 Some(mig.to_string())
210 }
211
212 fn extract_modified(err: &str) -> Option<String> {
213 let idx = err.find(" was previously applied but has been modified")?;
214 let prefix = &err[..idx];
215 let mig = prefix.rsplit_once(' ').map_or(prefix, |(_, m)| m);
216 Some(mig.to_string())
217 }
218
219 fn extract_sqlstate(err: &str) -> Option<String> {
220 // Postgres errors surface as `... code: "42P01" ...` in the Debug
221 // form sqlx produces. Be tolerant of the surrounding quoting.
222 let idx = err.find("code: \"")?;
223 let rest = &err[idx + 7..];
224 let end = rest.find('"')?;
225 Some(rest[..end].to_string())
226 }
227
228 /// `boot_smoke`: process exit info is the dominant signal. If the
229 /// binary exited with a status during the smoke window, we map exit
230 /// code 101 (Rust default for panic) to `BootPanic`, everything else
231 /// to `BootExitedEarly`. If it stayed up and served `/health`, the caller
232 /// constructs `PassNote::HealthyProbe` directly without consulting this.
233 pub fn classify_boot_smoke(exit_code: Option<i32>) -> GateFailure {
234 match exit_code {
235 Some(101) => GateFailure::BootPanic {
236 exit_code: Some(101),
237 },
238 Some(c) if c < 0 => GateFailure::BootPanic { exit_code: Some(c) }, // killed by signal
239 Some(c) => GateFailure::BootExitedEarly { exit_code: Some(c) },
240 None => GateFailure::BootExitedEarly { exit_code: None },
241 }
242 }
243
244 /// `Event::DeployFailed`: classify an anyhow chain produced by
245 /// `deploy::deploy_node` into a typed `DeployFailureKind`.
246 ///
247 /// The anyhow chain is the `format!("{e:#}")` string the caller built,
248 /// which joins each `.context(...)` layer with ": ". We probe for the
249 /// contexts attached by `deploy_remote` (and well-known stderr patterns
250 /// from ssh/rsync) in order of specificity.
251 pub fn classify_deploy_error(err: &str) -> crate::outcome::DeployFailureKind {
252 use crate::outcome::DeployFailureKind as K;
253
254 // SSH-level transport failures bubble up under whatever context
255 // their caller attached. Probe for the canonical OpenSSH stderr
256 // patterns first so a "creating remote release dir: ... Connection
257 // refused" doesn't get filed under NodeUnreachable's prose label.
258 let unreachable_signals = [
259 "Connection refused",
260 "Connection timed out",
261 "Network is unreachable",
262 "No route to host",
263 "Could not resolve hostname",
264 "Host key verification failed",
265 "Permission denied (publickey",
266 ];
267 if unreachable_signals.iter().any(|p| err.contains(p)) {
268 return K::NodeUnreachable {
269 detail: err.chars().take(400).collect(),
270 };
271 }
272
273 // The contexts attached by `deploy_remote` (deploy.rs) are stable
274 // strings; treat them as anchors. Order matters — "symlink swap +
275 // systemctl" appears after a successful rsync, so probe rsync first
276 // to avoid catching it under the swap heading.
277 if err.contains("rsync failed") || err.contains("spawning rsync") {
278 return K::RsyncFailed {
279 detail: err.chars().take(400).collect(),
280 };
281 }
282 if err.contains("creating remote release dir") {
283 return K::NodeUnreachable {
284 detail: err.chars().take(400).collect(),
285 };
286 }
287 if err.contains("symlink swap + systemctl") {
288 // Heuristic split inside the combined step: stderr containing
289 // "systemctl" suggests the swap succeeded and the restart failed.
290 if err.contains("systemctl") && !err.contains("ln:") {
291 return K::ServiceRestartFailed {
292 detail: err.chars().take(400).collect(),
293 };
294 }
295 return K::SymlinkSwapFailed {
296 detail: err.chars().take(400).collect(),
297 };
298 }
299 if err.contains("symlink swap failed") {
300 return K::SymlinkSwapFailed {
301 detail: err.chars().take(400).collect(),
302 };
303 }
304
305 K::Unclassified {
306 detail: err.chars().take(400).collect(),
307 }
308 }
309
310 /// Concatenate stdout + stderr tails the way the legacy runner did, so
311 /// `Unclassified.legacy_detail` looks like what operators are used to
312 /// seeing in `gate_runs.detail` today.
313 fn combined_tail_for_classifier(stdout: &[u8], stderr: &[u8]) -> String {
314 let mut joined = Vec::with_capacity(stdout.len() + stderr.len() + 32);
315 joined.extend_from_slice(b"==== stdout ====\n");
316 joined.extend_from_slice(stdout);
317 if stdout.last().is_none_or(|b| *b != b'\n') {
318 joined.push(b'\n');
319 }
320 joined.extend_from_slice(b"==== stderr ====\n");
321 joined.extend_from_slice(stderr);
322 let s = String::from_utf8_lossy(&joined);
323 if s.len() <= 4_000 {
324 s.into_owned()
325 } else {
326 format!("...{}", &s[s.len() - 4_000..])
327 }
328 }
329
330 #[cfg(test)]
331 mod tests {
332 use super::*;
333
334 #[test]
335 fn cargo_test_extracts_failed_count() {
336 let stdout = b"running 12 tests\n\
337 test foo ... ok\n\
338 test bar ... FAILED\n\
339 test baz ... FAILED\n\
340 \n\
341 failures:\n\
342 foo::bar\n\
343 foo::baz\n\
344 \n\
345 test result: FAILED. 10 passed; 2 failed; 0 ignored\n";
346 let GateFailure::CargoTest {
347 failed_count,
348 first_failed,
349 first_panic,
350 } = classify_cargo_test(stdout, b"")
351 else {
352 panic!("expected CargoTest variant");
353 };
354 assert_eq!(failed_count, 2);
355 assert_eq!(first_failed.as_deref(), Some("foo::bar"));
356 // No `panicked at` lines in this fixture.
357 assert_eq!(first_panic, None);
358 }
359
360 #[test]
361 fn cargo_test_sees_through_poison_cascade_to_root_panic() {
362 // The shape that produced the opaque "856 failed": one real panic in
363 // shared setup poisons a `Once`, and every other test then reports the
364 // poison. The classifier must surface the real cause, not the poison.
365 let stdout = b"failures:\n\n\
366 ---- harness::a stdout ----\n\
367 thread 'harness::a' panicked at tests/harness/db.rs:42:9:\n\
368 Once instance has previously been poisoned\n\
369 \n\
370 ---- harness::root stdout ----\n\
371 thread 'harness::root' panicked at tests/harness/db.rs:30:5:\n\
372 template database \"mnw_test_template\" does not exist\n\
373 \n\
374 failures:\n\
375 harness::a\n\
376 harness::root\n\
377 \n\
378 test result: FAILED. 0 passed; 856 failed; 0 ignored\n";
379 let GateFailure::CargoTest {
380 failed_count,
381 first_panic,
382 ..
383 } = classify_cargo_test(stdout, b"")
384 else {
385 panic!("expected CargoTest variant");
386 };
387 assert_eq!(failed_count, 856);
388 assert_eq!(
389 first_panic.as_deref(),
390 Some("template database \"mnw_test_template\" does not exist"),
391 "must skip the poison message for the root cause",
392 );
393 }
394
395 #[test]
396 fn cargo_test_panic_falls_back_when_all_poison() {
397 // If every panic is a poison report, return the first one rather than
398 // nothing — better than an opaque count.
399 let stdout = b"failures:\n\n\
400 ---- harness::a stdout ----\n\
401 thread 'harness::a' panicked at x.rs:1:1:\n\
402 Once instance has previously been poisoned\n\
403 \n\
404 failures:\n harness::a\n\
405 \n\
406 test result: FAILED. 0 passed; 3 failed; 0 ignored\n";
407 let GateFailure::CargoTest { first_panic, .. } = classify_cargo_test(stdout, b"") else {
408 panic!("expected CargoTest variant");
409 };
410 assert_eq!(
411 first_panic.as_deref(),
412 Some("Once instance has previously been poisoned")
413 );
414 }
415
416 #[test]
417 fn cargo_test_compile_error_is_unclassified() {
418 // No "test result:" line because cargo never got to running.
419 let stderr = b"error[E0382]: borrow of moved value: `x`\n";
420 let f = classify_cargo_test(b"", stderr);
421 match f {
422 GateFailure::Unclassified {
423 legacy_detail: Some(d),
424 } => {
425 assert!(d.contains("borrow of moved value"));
426 }
427 other => panic!("expected Unclassified, got {other:?}"),
428 }
429 }
430
431 #[test]
432 fn compile_error_extracts_first_coded_diagnostic_and_count() {
433 // Real `cargo test --no-run` shape: the headline diagnostic, then
434 // the trailing summary that carries the count.
435 let stderr = b" Compiling makenotwork v0.10.2\n\
436 error[E0063]: missing field `user_pages_host` in initializer of `Config`\n \
437 --> src/config.rs:412:21\n\
438 error: could not compile `makenotwork` (lib test) due to 1 previous error\n";
439 let GateFailure::CompileError {
440 error_count,
441 first_error,
442 } = classify_compile_error(b"", stderr)
443 else {
444 panic!("expected CompileError variant");
445 };
446 assert_eq!(error_count, 1);
447 assert_eq!(
448 first_error.as_deref(),
449 Some("error[E0063]: missing field `user_pages_host` in initializer of `Config`"),
450 );
451 }
452
453 #[test]
454 fn compile_error_falls_back_to_bare_error_line() {
455 // A macro/resolver error has no `error[Ennnn]` code; we still want
456 // the first real `error:` line, not the cargo summary.
457 let stderr = b"error: cannot find macro `foo` in this scope\n\
458 error: could not compile `makenotwork` (lib test) due to 2 previous errors\n";
459 let GateFailure::CompileError {
460 error_count,
461 first_error,
462 } = classify_compile_error(b"", stderr)
463 else {
464 panic!("expected CompileError variant");
465 };
466 assert_eq!(error_count, 2);
467 assert_eq!(
468 first_error.as_deref(),
469 Some("error: cannot find macro `foo` in this scope")
470 );
471 }
472
473 #[test]
474 fn compile_error_unclassified_when_not_a_compile_failure() {
475 // No `error[...]`, no `could not compile` — hand back the tail.
476 let f = classify_compile_error(b"", b"warning: unused import\n");
477 match f {
478 GateFailure::Unclassified {
479 legacy_detail: Some(d),
480 } => {
481 assert!(d.contains("unused import"));
482 }
483 other => panic!("expected Unclassified, got {other:?}"),
484 }
485 }
486
487 #[test]
488 fn migration_drift_extracts_name() {
489 let err = "migration 0047_widgets was previously applied but is missing in the resolved migrations";
490 let f = classify_migration_error(err, None);
491 match f {
492 GateFailure::MigrationDrift { migration } => assert_eq!(migration, "0047_widgets"),
493 other => panic!("expected MigrationDrift, got {other:?}"),
494 }
495 }
496
497 #[test]
498 fn migration_modified_extracts_name() {
499 let err = "migration 0042_seed was previously applied but has been modified";
500 let f = classify_migration_error(err, None);
501 match f {
502 GateFailure::MigrationModified { migration } => assert_eq!(migration, "0042_seed"),
503 other => panic!("expected MigrationModified, got {other:?}"),
504 }
505 }
506
507 #[test]
508 fn migration_sql_error_extracts_sqlstate() {
509 let err = r#"while executing migrations: error returned from database: code: "42P01" message: "relation \"widgets\" does not exist""#;
510 let f = classify_migration_error(err, Some("0050_drop_widgets"));
511 match f {
512 GateFailure::MigrationSqlError {
513 migration,
514 sqlstate,
515 } => {
516 assert_eq!(migration, "0050_drop_widgets");
517 assert_eq!(sqlstate.as_deref(), Some("42P01"));
518 }
519 other => panic!("expected MigrationSqlError, got {other:?}"),
520 }
521 }
522
523 #[test]
524 fn migration_unknown_error_is_unclassified() {
525 let err = "something went wrong with the universe";
526 let f = classify_migration_error(err, None);
527 match f {
528 GateFailure::Unclassified {
529 legacy_detail: Some(d),
530 } => {
531 assert!(d.contains("universe"));
532 }
533 other => panic!("expected Unclassified, got {other:?}"),
534 }
535 }
536
537 #[test]
538 fn boot_smoke_101_is_panic() {
539 match classify_boot_smoke(Some(101)) {
540 GateFailure::BootPanic {
541 exit_code: Some(101),
542 } => {}
543 other => panic!("expected BootPanic(101), got {other:?}"),
544 }
545 }
546
547 #[test]
548 fn boot_smoke_signal_is_panic() {
549 match classify_boot_smoke(Some(-9)) {
550 GateFailure::BootPanic {
551 exit_code: Some(-9),
552 } => {}
553 other => panic!("expected BootPanic(-9), got {other:?}"),
554 }
555 }
556
557 #[test]
558 fn boot_smoke_other_exit_is_exited_early() {
559 match classify_boot_smoke(Some(2)) {
560 GateFailure::BootExitedEarly { exit_code: Some(2) } => {}
561 other => panic!("expected BootExitedEarly(2), got {other:?}"),
562 }
563 }
564
565 #[test]
566 fn deploy_connection_refused_is_node_unreachable() {
567 use crate::outcome::DeployFailureKind as K;
568 let err = "creating remote release dir: ssh testnot-1 failed: ssh: connect to host testnot-1 port 22: Connection refused";
569 match classify_deploy_error(err) {
570 K::NodeUnreachable { .. } => {}
571 other => panic!("expected NodeUnreachable, got {other:?}"),
572 }
573 }
574
575 #[test]
576 fn deploy_rsync_failure_is_rsync_failed() {
577 use crate::outcome::DeployFailureKind as K;
578 let err = "rsync failed (current symlink left intact): rsync: write failed on \"/srv/.../makenotwork\": No space left on device (28)";
579 match classify_deploy_error(err) {
580 K::RsyncFailed { detail } => assert!(detail.contains("No space left")),
581 other => panic!("expected RsyncFailed, got {other:?}"),
582 }
583 }
584
585 #[test]
586 fn deploy_systemctl_failure_is_service_restart_failed() {
587 use crate::outcome::DeployFailureKind as K;
588 // The combined "swap + restart" step where stderr mentions systemctl.
589 let err = "symlink swap + systemctl reload-or-restart: ssh testnot-1 failed: Failed to restart makenotwork.service: Unit makenotwork.service failed to start";
590 match classify_deploy_error(err) {
591 K::ServiceRestartFailed { .. } => {}
592 other => panic!("expected ServiceRestartFailed, got {other:?}"),
593 }
594 }
595
596 #[test]
597 fn deploy_ln_failure_is_symlink_swap_failed() {
598 use crate::outcome::DeployFailureKind as K;
599 let err = "symlink swap + systemctl reload-or-restart: ssh testnot-1 failed: ln: failed to create symbolic link: Permission denied";
600 match classify_deploy_error(err) {
601 K::SymlinkSwapFailed { .. } => {}
602 other => panic!("expected SymlinkSwapFailed, got {other:?}"),
603 }
604 }
605
606 #[test]
607 fn deploy_unknown_is_unclassified() {
608 use crate::outcome::DeployFailureKind as K;
609 let err = "something went wrong in a way we did not anticipate";
610 match classify_deploy_error(err) {
611 K::Unclassified { detail } => assert!(detail.contains("anticipate")),
612 other => panic!("expected Unclassified, got {other:?}"),
613 }
614 }
615 }
616