Skip to main content

max / makenotwork

2.7 KB · 77 lines History Blame Raw
1 //! Replay every committed fuzz input through the oracle, on stable.
2 //!
3 //! The soak tier runs on astra, on nightly, when the box is idle. That is the
4 //! wrong place for the only copy of a property to live: a crash found there is
5 //! fixed here, and nothing on a developer's machine would notice it coming back.
6 //!
7 //! So both corpora replay as an ordinary `cargo test`:
8 //!
9 //! - `fuzz/seeds/command/` is human intent, the shapes worth reaching first.
10 //! - `fuzz/regressions/` is inputs that once found a bug. Anything landing there
11 //! is a permanent test by virtue of the directory, with no test function to
12 //! write and no chance of forgetting one.
13 //!
14 //! The assertion is `git_command::oracle::check_line`, the same function the
15 //! libFuzzer target calls, so the two cannot check different things.
16
17 use std::path::Path;
18
19 /// Replay one directory, returning how many inputs it held.
20 ///
21 /// A missing directory is fine and returns zero: `fuzz/regressions/` does not
22 /// exist until the first crash does.
23 fn replay(dir: &Path) -> usize {
24 let Ok(entries) = std::fs::read_dir(dir) else {
25 return 0;
26 };
27
28 let mut count = 0;
29 for entry in entries {
30 let path = entry.expect("readable dir entry").path();
31 if !path.is_file() {
32 continue;
33 }
34 // README and friends are documentation, not inputs.
35 if path.extension().is_some_and(|e| e == "md") {
36 continue;
37 }
38
39 let bytes = std::fs::read(&path).expect("readable input");
40 // The target takes `&str`, so a non-UTF-8 artifact is not one of ours.
41 let Ok(line) = std::str::from_utf8(&bytes) else {
42 continue;
43 };
44
45 // Panics here name the file, because a bare oracle panic in a loop over
46 // 20 inputs says nothing about which one broke.
47 let outcome = std::panic::catch_unwind(|| git_command::oracle::check_line(line));
48 assert!(
49 outcome.is_ok(),
50 "oracle failed on {}: {line:?}",
51 path.display()
52 );
53 count += 1;
54 }
55 count
56 }
57
58 #[test]
59 fn every_seed_holds_the_oracle() {
60 let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("fuzz/seeds/command");
61 let n = replay(&dir);
62 // A seed directory that silently emptied would make this test pass while
63 // checking nothing, which is the failure mode a corpus-replay test has.
64 assert!(
65 n >= 20,
66 "expected the committed seed corpus, found {n} inputs"
67 );
68 }
69
70 #[test]
71 fn every_regression_holds_the_oracle() {
72 let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("fuzz/regressions");
73 // No lower bound. This is empty until the first crash, and that is the
74 // good state.
75 replay(&dir);
76 }
77