Skip to main content

max / makenotwork

5.3 KB · 139 lines History Blame Raw
1 //! Replay every committed sanitizer input through the oracle, on stable.
2 //!
3 //! The soak tier runs on astra, on nightly, when the box is idle. That is the
4 //! wrong place for the only copy of a property to live: a crash found there is
5 //! fixed here, and nothing on a developer's machine would notice it coming
6 //! back.
7 //!
8 //! So both corpora replay as an ordinary `cargo test`:
9 //!
10 //! - `fuzz/seeds/{html,css}/` is human intent, the shapes worth reaching first.
11 //! - `fuzz/regressions/` is inputs that once found a bug. Anything landing
12 //! there is a permanent test by virtue of the directory, with no test
13 //! function to write and no chance of forgetting one.
14 //!
15 //! The assertions are `custom_pages::oracle::check_html` and `check_css`, the
16 //! same functions the libFuzzer targets will call, so the two cannot check
17 //! different things. (The fuzz crate itself belongs to the soak-target tasks
18 //! this extraction unblocks; the oracle and these corpora do not wait on it.)
19
20 use std::path::Path;
21
22 const OWNER_SCOPE: &str = "11111111-1111-1111-1111-111111111111";
23
24 fn policy() -> custom_pages::UrlPolicy {
25 custom_pages::UrlPolicy::new(
26 "https://u.makenot.work/alice/proj",
27 [
28 "makenot.work".to_string(),
29 "u.makenot.work".to_string(),
30 "cdn.makenot.work".to_string(),
31 ],
32 )
33 .expect("the fixture policy is well-formed")
34 }
35
36 /// Replay one directory, returning how many inputs it held.
37 ///
38 /// A missing directory is fine and returns zero: `fuzz/regressions/` does not
39 /// exist until the first crash does.
40 ///
41 /// **Each input is replayed on its own thread, with a large stack**, for the
42 /// reason the css fuzz target spawns one too: an input can flatten to a
43 /// stylesheet far larger and more deeply nested than itself (214 bytes to
44 /// 219 KB, measured), and the oracle's reparse of that recurses per nesting
45 /// level. A debug build's frames are several times a release build's, so this
46 /// suite aborted on a default 2 MiB test-thread stack while the same inputs
47 /// passed under `--release`. It aborted on both fw13 and astra, and was found
48 /// by a `cargo mutants` BASELINE rather than by a test run, because a stack
49 /// overflow kills the process before libtest can print a result line -- so
50 /// `cargo test | grep "test result"` showed the lib suite passing and said
51 /// nothing at all about this one.
52 ///
53 /// A spawned thread is also the only option: a stack overflow aborts the
54 /// process and `catch_unwind` cannot see it.
55 fn replay(dir: &Path, check: impl Fn(&str) + Copy + Send + 'static) -> usize {
56 let Ok(entries) = std::fs::read_dir(dir) else {
57 return 0;
58 };
59
60 let mut count = 0;
61 for entry in entries {
62 let path = entry.expect("readable dir entry").path();
63 if !path.is_file() {
64 continue;
65 }
66 // READMEs are documentation, not inputs.
67 if path.extension().is_some_and(|e| e == "md") {
68 continue;
69 }
70
71 let bytes = std::fs::read(&path).expect("readable input");
72 // The targets take `&str`, so a non-UTF-8 artifact is not one of ours.
73 let Ok(text) = std::str::from_utf8(&bytes) else {
74 continue;
75 };
76 let owned = text.to_string();
77
78 let handle = std::thread::Builder::new()
79 .stack_size(256 * 1024 * 1024)
80 .spawn(move || check(&owned))
81 .expect("spawning the replay thread");
82 // Naming the file matters: a bare oracle panic in a loop over twenty
83 // inputs says nothing about which one broke.
84 assert!(handle.join().is_ok(), "oracle failed on {}", path.display());
85 count += 1;
86 }
87 count
88 }
89
90 /// A regressions directory holds inputs for both targets, so each is replayed
91 /// through both oracles. An HTML crash file is valid CSS input and vice versa;
92 /// running both costs nothing and removes the question of which subdirectory a
93 /// new artifact belongs in.
94 fn replay_both(dir: &Path) -> usize {
95 let html = replay(dir, |s: &str| {
96 custom_pages::oracle::check_html(s, &policy());
97 });
98 let css = replay(dir, |s: &str| {
99 custom_pages::oracle::check_css(s, OWNER_SCOPE, &policy());
100 });
101 assert_eq!(html, css, "the same directory yielded two different counts");
102 html
103 }
104
105 #[test]
106 fn html_seeds_satisfy_the_oracle() {
107 let n = replay(
108 Path::new(env!("CARGO_MANIFEST_DIR"))
109 .join("fuzz/seeds/html")
110 .as_path(),
111 |s: &str| custom_pages::oracle::check_html(s, &policy()),
112 );
113 assert!(n > 0, "no HTML seeds found; the corpus is the point");
114 }
115
116 #[test]
117 fn css_seeds_satisfy_the_oracle() {
118 let n = replay(
119 Path::new(env!("CARGO_MANIFEST_DIR"))
120 .join("fuzz/seeds/css")
121 .as_path(),
122 |s: &str| custom_pages::oracle::check_css(s, OWNER_SCOPE, &policy()),
123 );
124 assert!(n > 0, "no CSS seeds found; the corpus is the point");
125 }
126
127 #[test]
128 fn every_past_crash_stays_fixed() {
129 // No assertion on the count. An empty regressions directory is the good
130 // state, and a test that demanded a crash to have happened would be a test
131 // that fails on a healthy crate.
132 let n = replay_both(
133 Path::new(env!("CARGO_MANIFEST_DIR"))
134 .join("fuzz/regressions")
135 .as_path(),
136 );
137 println!("replayed {n} regression inputs");
138 }
139