Skip to main content

max / makenotwork

22.9 KB · 615 lines History Blame Raw
1 //! Gate-output classifiers.
2 //!
3 //! Each `classify_*` function takes the raw signals produced by a gate
4 //! runner (exit status, stdout/stderr tails, sqlx error strings) and
5 //! maps them to a typed `GateFailure`. Anything that doesn't match a
6 //! known pattern returns `GateFailure::Unclassified` with the original
7 //! detail attached — the on-disk gate log is the ultimate fallback.
8 //!
9 //! Classifiers are pure functions: no IO, no async. That makes them
10 //! fixture-testable, and it keeps the `gates.rs` runner code in charge
11 //! of side effects (process spawning, log persistence).
12
13 use crate::outcome::GateFailure;
14
15 /// `cargo_test`: derive a `CargoTest` failure with whatever counts can
16 /// be lifted out of the test runner's output.
17 ///
18 /// libtest emits a `test result: FAILED. P passed; F failed; ...` line
19 /// near the end of stdout. We grab `F` from that. If the output never
20 /// reached that line (compile error, runtime panic in the harness), we
21 /// fall through to `Unclassified`.
22 pub fn classify_cargo_test(stdout: &[u8], stderr: &[u8]) -> GateFailure {
23 let stdout_s = String::from_utf8_lossy(stdout);
24
25 let mut failed_count: u32 = 0;
26 let mut first_failed: Option<String> = None;
27
28 // `test result: FAILED. P passed; F failed; ...` lives near the
29 // end. Walk backwards to find it cheaply on very large outputs.
30 for line in stdout_s.lines().rev().take(50) {
31 if let Some(rest) = line.strip_prefix("test result: FAILED.") {
32 // Expect "P passed; F failed; ..."
33 for piece in rest.split(';') {
34 let p = piece.trim();
35 if let Some(num_str) = p.strip_suffix(" failed")
36 && let Ok(n) = num_str.parse::<u32>()
37 {
38 failed_count = n;
39 }
40 }
41 break;
42 }
43 }
44
45 // libtest prints "failures:\n foo::bar" near the end too. Grab
46 // the first one for the summary line.
47 if let Some(idx) = stdout_s.find("\nfailures:\n") {
48 // The "failures:" block repeats — once with stdout per failure, once as
49 // a plain name list. Either way the first non-empty line is a candidate.
50 if let Some(line) = stdout_s[idx + 11..].lines().next() {
51 let trimmed = line.trim();
52 if !trimmed.is_empty() {
53 first_failed = Some(trimmed.to_string());
54 }
55 }
56 }
57
58 if failed_count == 0 && first_failed.is_none() {
59 // Compile error or harness panic — no usable signal in stdout.
60 return GateFailure::Unclassified {
61 legacy_detail: Some(combined_tail_for_classifier(stdout, stderr)),
62 };
63 }
64
65 let first_panic = extract_first_panic(&stdout_s);
66 GateFailure::CargoTest {
67 failed_count,
68 first_failed,
69 first_panic,
70 }
71 }
72
73 /// Pull the first *root-cause* panic message out of libtest's captured
74 /// output. libtest (Rust 2021+) prints each captured panic as:
75 /// thread '<test>' panicked at <file>:<line>:<col>:
76 /// <message>
77 /// We return the first panic's message, skipping "...poisoned" messages in
78 /// favour of the first non-poison one, because a single real panic in shared
79 /// setup (a `std::sync::Once`) poisons it and makes every *other* test report
80 /// "Once instance has previously been poisoned". The root cause is the one
81 /// panic that isn't a poison report. Falls back to the first panic of any
82 /// kind if every message looks like poison.
83 fn extract_first_panic(stdout: &str) -> Option<String> {
84 let mut first: Option<String> = None;
85 let mut lines = stdout.lines();
86 while let Some(line) = lines.next() {
87 if !line.contains("panicked at ") {
88 continue;
89 }
90 // The message is the first non-empty line after the `panicked at` loc.
91 let msg = lines.by_ref().map(str::trim).find(|l| !l.is_empty());
92 let Some(msg) = msg else { continue };
93 if first.is_none() {
94 first = Some(msg.to_string());
95 }
96 let is_poison = msg.contains("poisoned") || msg.contains("PoisonError");
97 if !is_poison {
98 return Some(msg.to_string());
99 }
100 }
101 first
102 }
103
104 /// `cargo test --no-run` (the fast pre-gate compile): pull the first
105 /// compiler diagnostic out of cargo's stderr so a test-only-target
106 /// compile break (e.g. a missing struct field in a `#[cfg(test)]`-only
107 /// target) surfaces as the actual `error[E0063]: missing field ...`
108 /// line, instead of after a full build + a partial run reported as an
109 /// opaque "N tests failed".
110 ///
111 /// Cargo writes diagnostics to stderr. We prefer the first coded
112 /// `error[Ennnn]: ...` headline over the trailing `error: could not
113 /// compile <crate> ... due to N previous errors` summary, which names
114 /// the crate but not the cause; the summary still gives us the count.
115 pub fn classify_compile_error(stdout: &[u8], stderr: &[u8]) -> GateFailure {
116 let stderr_s = String::from_utf8_lossy(stderr);
117 let mut first_error: Option<String> = None;
118 let mut error_count: u32 = 0;
119
120 for line in stderr_s.lines() {
121 let t = line.trim_start();
122 if first_error.is_none() && t.starts_with("error[") {
123 first_error = Some(t.to_string());
124 }
125 if let Some(rest) = t.strip_prefix("error: could not compile")
126 && let Some(n) = parse_due_to_count(rest)
127 {
128 error_count = n;
129 }
130 }
131
132 // No coded diagnostic (e.g. a macro or resolver error prints a bare
133 // `error: ...`). Take the first such line that isn't cargo's own
134 // summary/abort noise.
135 if first_error.is_none() {
136 for line in stderr_s.lines() {
137 let t = line.trim_start();
138 if t.starts_with("error:")
139 && !t.starts_with("error: could not compile")
140 && !t.starts_with("error: aborting")
141 {
142 first_error = Some(t.to_string());
143 break;
144 }
145 }
146 }
147
148 if first_error.is_none() && error_count == 0 {
149 // Didn't look like a compile failure — don't masquerade as one.
150 return GateFailure::Unclassified {
151 legacy_detail: Some(combined_tail_for_classifier(stdout, stderr)),
152 };
153 }
154 GateFailure::CompileError {
155 error_count,
156 first_error,
157 }
158 }
159
160 /// Parse the count out of `... due to N previous error(s)`.
161 fn parse_due_to_count(s: &str) -> Option<u32> {
162 let idx = s.find("due to ")?;
163 let digits: String = s[idx + 7..]
164 .chars()
165 .take_while(char::is_ascii_digit)
166 .collect();
167 digits.parse().ok()
168 }
169
170 /// `migration_dry_run` is staged: scratch reset → restore dump → run
171 /// migrator. Each stage has its own failure mode. The caller (the gate
172 /// runner) knows which stage tripped; classifiers here turn the stage's
173 /// error string into a typed variant.
174 ///
175 /// Inputs are the migration name (when known) and the error string sqlx
176 /// returned. `migration` defaults to "?" when sqlx couldn't tell us
177 /// which file blew up.
178 pub fn classify_migration_error(err: &str, migration_hint: Option<&str>) -> GateFailure {
179 // sqlx::migrate::MigrateError variants are stringified consistently.
180 // Examples from `plans/migration-dryrun-failures.md`:
181 // "migration 47 was previously applied but is missing in the resolved migrations"
182 // "migration 47 was previously applied but has been modified"
183 // sqlx::Error::Database with sqlstate (e.g. "42P01" relation does not exist)
184
185 if let Some(m) = extract_drift(err) {
186 return GateFailure::MigrationDrift { migration: m };
187 }
188 if let Some(m) = extract_modified(err) {
189 return GateFailure::MigrationModified { migration: m };
190 }
191 let sqlstate = extract_sqlstate(err);
192 let migration = migration_hint.map_or_else(|| "?".to_owned(), str::to_owned);
193 if sqlstate.is_some() {
194 return GateFailure::MigrationSqlError {
195 migration,
196 sqlstate,
197 };
198 }
199 GateFailure::Unclassified {
200 legacy_detail: Some(err.chars().take(4_000).collect()),
201 }
202 }
203
204 fn extract_drift(err: &str) -> Option<String> {
205 // "migration N was previously applied but is missing in the resolved migrations"
206 let idx = err.find(" was previously applied but is missing")?;
207 let prefix = &err[..idx];
208 let mig = prefix.rsplit_once(' ').map_or(prefix, |(_, m)| m);
209 Some(mig.to_string())
210 }
211
212 fn extract_modified(err: &str) -> Option<String> {
213 let idx = err.find(" was previously applied but has been modified")?;
214 let prefix = &err[..idx];
215 let mig = prefix.rsplit_once(' ').map_or(prefix, |(_, m)| m);
216 Some(mig.to_string())
217 }
218
219 fn extract_sqlstate(err: &str) -> Option<String> {
220 // Postgres errors surface as `... code: "42P01" ...` in the Debug
221 // form sqlx produces. Be tolerant of the surrounding quoting.
222 let idx = err.find("code: \"")?;
223 let rest = &err[idx + 7..];
224 let end = rest.find('"')?;
225 Some(rest[..end].to_string())
226 }
227
228 /// `boot_smoke`: process exit info is the dominant signal. If the
229 /// binary exited with a status during the smoke window, we map exit
230 /// code 101 (Rust default for panic) to `BootPanic`, everything else
231 /// to `BootExitedEarly`. If it stayed up and served `/health`, the caller
232 /// constructs `PassNote::HealthyProbe` directly without consulting this.
233 pub fn classify_boot_smoke(exit_code: Option<i32>) -> GateFailure {
234 match exit_code {
235 Some(101) => GateFailure::BootPanic {
236 exit_code: Some(101),
237 },
238 Some(c) if c < 0 => GateFailure::BootPanic { exit_code: Some(c) }, // killed by signal
239 Some(c) => GateFailure::BootExitedEarly { exit_code: Some(c) },
240 None => GateFailure::BootExitedEarly { exit_code: None },
241 }
242 }
243
244 /// `Event::DeployFailed`: classify an anyhow chain produced by
245 /// `deploy::deploy_node` into a typed `DeployFailureKind`.
246 ///
247 /// The anyhow chain is the `format!("{e:#}")` string the caller built,
248 /// which joins each `.context(...)` layer with ": ". We probe for the
249 /// contexts attached by `deploy_remote` (and well-known stderr patterns
250 /// from ssh/rsync) in order of specificity.
251 pub fn classify_deploy_error(err: &str) -> crate::outcome::DeployFailureKind {
252 use crate::outcome::DeployFailureKind as K;
253
254 // SSH-level transport failures bubble up under whatever context
255 // their caller attached. Probe for the canonical OpenSSH stderr
256 // patterns first so a "creating remote release dir: ... Connection
257 // refused" doesn't get filed under NodeUnreachable's prose label.
258 let unreachable_signals = [
259 "Connection refused",
260 "Connection timed out",
261 "Network is unreachable",
262 "No route to host",
263 "Could not resolve hostname",
264 "Host key verification failed",
265 "Permission denied (publickey",
266 ];
267 if unreachable_signals.iter().any(|p| err.contains(p)) {
268 return K::NodeUnreachable {
269 detail: err.chars().take(400).collect(),
270 };
271 }
272
273 // The contexts attached by `deploy_remote` (deploy.rs) are stable
274 // strings; treat them as anchors. Order matters — "symlink swap +
275 // systemctl" appears after a successful rsync, so probe rsync first
276 // to avoid catching it under the swap heading.
277 if err.contains("rsync failed") || err.contains("spawning rsync") {
278 return K::RsyncFailed {
279 detail: err.chars().take(400).collect(),
280 };
281 }
282 if err.contains("creating remote release dir") {
283 return K::NodeUnreachable {
284 detail: err.chars().take(400).collect(),
285 };
286 }
287 if err.contains("symlink swap + systemctl") {
288 // Heuristic split inside the combined step: stderr containing
289 // "systemctl" suggests the swap succeeded and the restart failed.
290 if err.contains("systemctl") && !err.contains("ln:") {
291 return K::ServiceRestartFailed {
292 detail: err.chars().take(400).collect(),
293 };
294 }
295 return K::SymlinkSwapFailed {
296 detail: err.chars().take(400).collect(),
297 };
298 }
299 if err.contains("symlink swap failed") {
300 return K::SymlinkSwapFailed {
301 detail: err.chars().take(400).collect(),
302 };
303 }
304
305 K::Unclassified {
306 detail: err.chars().take(400).collect(),
307 }
308 }
309
310 /// Concatenate stdout + stderr tails into the shape `Unclassified.legacy_detail`
311 /// carries, matching what `gate_runs.detail` shows.
312 fn combined_tail_for_classifier(stdout: &[u8], stderr: &[u8]) -> String {
313 let mut joined = Vec::with_capacity(stdout.len() + stderr.len() + 32);
314 joined.extend_from_slice(b"==== stdout ====\n");
315 joined.extend_from_slice(stdout);
316 if stdout.last().is_none_or(|b| *b != b'\n') {
317 joined.push(b'\n');
318 }
319 joined.extend_from_slice(b"==== stderr ====\n");
320 joined.extend_from_slice(stderr);
321 let s = String::from_utf8_lossy(&joined);
322 if s.len() <= 4_000 {
323 s.into_owned()
324 } else {
325 format!("...{}", &s[s.len() - 4_000..])
326 }
327 }
328
329 #[cfg(test)]
330 mod tests {
331 use super::*;
332
333 #[test]
334 fn cargo_test_extracts_failed_count() {
335 let stdout = b"running 12 tests\n\
336 test foo ... ok\n\
337 test bar ... FAILED\n\
338 test baz ... FAILED\n\
339 \n\
340 failures:\n\
341 foo::bar\n\
342 foo::baz\n\
343 \n\
344 test result: FAILED. 10 passed; 2 failed; 0 ignored\n";
345 let GateFailure::CargoTest {
346 failed_count,
347 first_failed,
348 first_panic,
349 } = classify_cargo_test(stdout, b"")
350 else {
351 panic!("expected CargoTest variant");
352 };
353 assert_eq!(failed_count, 2);
354 assert_eq!(first_failed.as_deref(), Some("foo::bar"));
355 // No `panicked at` lines in this fixture.
356 assert_eq!(first_panic, None);
357 }
358
359 #[test]
360 fn cargo_test_sees_through_poison_cascade_to_root_panic() {
361 // The shape that produced the opaque "856 failed": one real panic in
362 // shared setup poisons a `Once`, and every other test then reports the
363 // poison. The classifier must surface the real cause, not the poison.
364 let stdout = b"failures:\n\n\
365 ---- harness::a stdout ----\n\
366 thread 'harness::a' panicked at tests/harness/db.rs:42:9:\n\
367 Once instance has previously been poisoned\n\
368 \n\
369 ---- harness::root stdout ----\n\
370 thread 'harness::root' panicked at tests/harness/db.rs:30:5:\n\
371 template database \"mnw_test_template\" does not exist\n\
372 \n\
373 failures:\n\
374 harness::a\n\
375 harness::root\n\
376 \n\
377 test result: FAILED. 0 passed; 856 failed; 0 ignored\n";
378 let GateFailure::CargoTest {
379 failed_count,
380 first_panic,
381 ..
382 } = classify_cargo_test(stdout, b"")
383 else {
384 panic!("expected CargoTest variant");
385 };
386 assert_eq!(failed_count, 856);
387 assert_eq!(
388 first_panic.as_deref(),
389 Some("template database \"mnw_test_template\" does not exist"),
390 "must skip the poison message for the root cause",
391 );
392 }
393
394 #[test]
395 fn cargo_test_panic_falls_back_when_all_poison() {
396 // If every panic is a poison report, return the first one rather than
397 // nothing — better than an opaque count.
398 let stdout = b"failures:\n\n\
399 ---- harness::a stdout ----\n\
400 thread 'harness::a' panicked at x.rs:1:1:\n\
401 Once instance has previously been poisoned\n\
402 \n\
403 failures:\n harness::a\n\
404 \n\
405 test result: FAILED. 0 passed; 3 failed; 0 ignored\n";
406 let GateFailure::CargoTest { first_panic, .. } = classify_cargo_test(stdout, b"") else {
407 panic!("expected CargoTest variant");
408 };
409 assert_eq!(
410 first_panic.as_deref(),
411 Some("Once instance has previously been poisoned")
412 );
413 }
414
415 #[test]
416 fn cargo_test_compile_error_is_unclassified() {
417 // No "test result:" line because cargo never got to running.
418 let stderr = b"error[E0382]: borrow of moved value: `x`\n";
419 let f = classify_cargo_test(b"", stderr);
420 match f {
421 GateFailure::Unclassified {
422 legacy_detail: Some(d),
423 } => {
424 assert!(d.contains("borrow of moved value"));
425 }
426 other => panic!("expected Unclassified, got {other:?}"),
427 }
428 }
429
430 #[test]
431 fn compile_error_extracts_first_coded_diagnostic_and_count() {
432 // Real `cargo test --no-run` shape: the headline diagnostic, then
433 // the trailing summary that carries the count.
434 let stderr = b" Compiling makenotwork v0.10.2\n\
435 error[E0063]: missing field `user_pages_host` in initializer of `Config`\n \
436 --> src/config.rs:412:21\n\
437 error: could not compile `makenotwork` (lib test) due to 1 previous error\n";
438 let GateFailure::CompileError {
439 error_count,
440 first_error,
441 } = classify_compile_error(b"", stderr)
442 else {
443 panic!("expected CompileError variant");
444 };
445 assert_eq!(error_count, 1);
446 assert_eq!(
447 first_error.as_deref(),
448 Some("error[E0063]: missing field `user_pages_host` in initializer of `Config`"),
449 );
450 }
451
452 #[test]
453 fn compile_error_falls_back_to_bare_error_line() {
454 // A macro/resolver error has no `error[Ennnn]` code; we still want
455 // the first real `error:` line, not the cargo summary.
456 let stderr = b"error: cannot find macro `foo` in this scope\n\
457 error: could not compile `makenotwork` (lib test) due to 2 previous errors\n";
458 let GateFailure::CompileError {
459 error_count,
460 first_error,
461 } = classify_compile_error(b"", stderr)
462 else {
463 panic!("expected CompileError variant");
464 };
465 assert_eq!(error_count, 2);
466 assert_eq!(
467 first_error.as_deref(),
468 Some("error: cannot find macro `foo` in this scope")
469 );
470 }
471
472 #[test]
473 fn compile_error_unclassified_when_not_a_compile_failure() {
474 // No `error[...]`, no `could not compile` — hand back the tail.
475 let f = classify_compile_error(b"", b"warning: unused import\n");
476 match f {
477 GateFailure::Unclassified {
478 legacy_detail: Some(d),
479 } => {
480 assert!(d.contains("unused import"));
481 }
482 other => panic!("expected Unclassified, got {other:?}"),
483 }
484 }
485
486 #[test]
487 fn migration_drift_extracts_name() {
488 let err = "migration 0047_widgets was previously applied but is missing in the resolved migrations";
489 let f = classify_migration_error(err, None);
490 match f {
491 GateFailure::MigrationDrift { migration } => assert_eq!(migration, "0047_widgets"),
492 other => panic!("expected MigrationDrift, got {other:?}"),
493 }
494 }
495
496 #[test]
497 fn migration_modified_extracts_name() {
498 let err = "migration 0042_seed was previously applied but has been modified";
499 let f = classify_migration_error(err, None);
500 match f {
501 GateFailure::MigrationModified { migration } => assert_eq!(migration, "0042_seed"),
502 other => panic!("expected MigrationModified, got {other:?}"),
503 }
504 }
505
506 #[test]
507 fn migration_sql_error_extracts_sqlstate() {
508 let err = r#"while executing migrations: error returned from database: code: "42P01" message: "relation \"widgets\" does not exist""#;
509 let f = classify_migration_error(err, Some("0050_drop_widgets"));
510 match f {
511 GateFailure::MigrationSqlError {
512 migration,
513 sqlstate,
514 } => {
515 assert_eq!(migration, "0050_drop_widgets");
516 assert_eq!(sqlstate.as_deref(), Some("42P01"));
517 }
518 other => panic!("expected MigrationSqlError, got {other:?}"),
519 }
520 }
521
522 #[test]
523 fn migration_unknown_error_is_unclassified() {
524 let err = "something went wrong with the universe";
525 let f = classify_migration_error(err, None);
526 match f {
527 GateFailure::Unclassified {
528 legacy_detail: Some(d),
529 } => {
530 assert!(d.contains("universe"));
531 }
532 other => panic!("expected Unclassified, got {other:?}"),
533 }
534 }
535
536 #[test]
537 fn boot_smoke_101_is_panic() {
538 match classify_boot_smoke(Some(101)) {
539 GateFailure::BootPanic {
540 exit_code: Some(101),
541 } => {}
542 other => panic!("expected BootPanic(101), got {other:?}"),
543 }
544 }
545
546 #[test]
547 fn boot_smoke_signal_is_panic() {
548 match classify_boot_smoke(Some(-9)) {
549 GateFailure::BootPanic {
550 exit_code: Some(-9),
551 } => {}
552 other => panic!("expected BootPanic(-9), got {other:?}"),
553 }
554 }
555
556 #[test]
557 fn boot_smoke_other_exit_is_exited_early() {
558 match classify_boot_smoke(Some(2)) {
559 GateFailure::BootExitedEarly { exit_code: Some(2) } => {}
560 other => panic!("expected BootExitedEarly(2), got {other:?}"),
561 }
562 }
563
564 #[test]
565 fn deploy_connection_refused_is_node_unreachable() {
566 use crate::outcome::DeployFailureKind as K;
567 let err = "creating remote release dir: ssh testnot-1 failed: ssh: connect to host testnot-1 port 22: Connection refused";
568 match classify_deploy_error(err) {
569 K::NodeUnreachable { .. } => {}
570 other => panic!("expected NodeUnreachable, got {other:?}"),
571 }
572 }
573
574 #[test]
575 fn deploy_rsync_failure_is_rsync_failed() {
576 use crate::outcome::DeployFailureKind as K;
577 let err = "rsync failed (current symlink left intact): rsync: write failed on \"/srv/.../makenotwork\": No space left on device (28)";
578 match classify_deploy_error(err) {
579 K::RsyncFailed { detail } => assert!(detail.contains("No space left")),
580 other => panic!("expected RsyncFailed, got {other:?}"),
581 }
582 }
583
584 #[test]
585 fn deploy_systemctl_failure_is_service_restart_failed() {
586 use crate::outcome::DeployFailureKind as K;
587 // The combined "swap + restart" step where stderr mentions systemctl.
588 let err = "symlink swap + systemctl reload-or-restart: ssh testnot-1 failed: Failed to restart makenotwork.service: Unit makenotwork.service failed to start";
589 match classify_deploy_error(err) {
590 K::ServiceRestartFailed { .. } => {}
591 other => panic!("expected ServiceRestartFailed, got {other:?}"),
592 }
593 }
594
595 #[test]
596 fn deploy_ln_failure_is_symlink_swap_failed() {
597 use crate::outcome::DeployFailureKind as K;
598 let err = "symlink swap + systemctl reload-or-restart: ssh testnot-1 failed: ln: failed to create symbolic link: Permission denied";
599 match classify_deploy_error(err) {
600 K::SymlinkSwapFailed { .. } => {}
601 other => panic!("expected SymlinkSwapFailed, got {other:?}"),
602 }
603 }
604
605 #[test]
606 fn deploy_unknown_is_unclassified() {
607 use crate::outcome::DeployFailureKind as K;
608 let err = "something went wrong in a way we did not anticipate";
609 match classify_deploy_error(err) {
610 K::Unclassified { detail } => assert!(detail.contains("anticipate")),
611 other => panic!("expected Unclassified, got {other:?}"),
612 }
613 }
614 }
615