Skip to main content

max / makenotwork

14.6 KB · 407 lines History Blame Raw
1 //! Layer 7: URLhaus URL-reputation lookups against strings extracted from
2 //! the uploaded file.
3 //!
4 //! Pulls printable-ASCII URL substrings out of the binary, dedupes by host,
5 //! and queries abuse.ch's URLhaus `host` endpoint for the first N hosts
6 //! (capped to keep us within free-tier limits and to bound per-upload work).
7 //! Any known-malicious host triggers `Fail`; clean lookups or no extracted
8 //! URLs trigger `Pass`; auth or network errors fail-open by policy.
9 //!
10 //! Like MalwareBazaar, URLhaus requires the abuse.ch `Auth-Key` header.
11 //! Same key powers both services, register at <https://auth.abuse.ch>.
12
13 use std::collections::HashSet;
14 use std::time::Duration;
15
16 use crate::constants;
17
18 use super::{ErrorPolicy, LayerResult, LayerVerdict};
19
20 /// External third-party network layer. Same reasoning as MalwareBazaar: an
21 /// outage at abuse.ch must not block every upload across the platform.
22 pub const ERROR_POLICY: ErrorPolicy = ErrorPolicy::FailOpen;
23
24 const URLHAUS_API_URL: &str = "https://urlhaus-api.abuse.ch/v1/host/";
25 /// Cap per-upload host lookups to keep within free-tier quotas and bound work.
26 pub(crate) const MAX_HOSTS_PER_FILE: usize = 5;
27 /// Cap the byte window we scan for URLs; URLs in the wild fit easily here.
28 const MAX_SCAN_BYTES: usize = 4 * 1024 * 1024;
29 /// Minimum printable-ASCII run length to consider as a potential string.
30 const MIN_RUN_LEN: usize = 6;
31
32 /// URLhaus lookup over already-extracted hosts. The scan pipeline always extracts
33 /// candidate hosts off the async runtime (in `spawn_blocking`, since the byte walk
34 /// can page-fault on an mmap) and passes the result here, leaving only the network
35 /// lookups on the runtime, one path for buffered and streamed scans alike.
36 pub(crate) async fn check_urlhaus_hosts(hosts: Vec<String>, auth_key: Option<&str>) -> LayerResult {
37 let Some(key) = auth_key else {
38 return LayerResult {
39 layer: "urlhaus",
40 verdict: LayerVerdict::Skip,
41 detail: Some("No abuse.ch Auth-Key configured".to_string()),
42 };
43 };
44
45 if hosts.is_empty() {
46 return LayerResult {
47 layer: "urlhaus",
48 verdict: LayerVerdict::Pass,
49 detail: Some("No URLs extracted".to_string()),
50 };
51 }
52
53 let per_host = Duration::from_secs(constants::SCAN_MALWAREBAZAAR_TIMEOUT_SECS);
54 // Aggregate deadline across ALL hosts (Perf-S2): the per-host timeout alone
55 // let up to MAX_HOSTS_PER_FILE sequential lookups stack (5 × per_host ≈ 25s),
56 // holding one of the two scan workers the whole time. Cap the total at the
57 // shared external-lookup budget so two adversarial uploads can't saturate the
58 // pool. On the deadline we return Error (degraded per urlhaus's fail policy),
59 // same verdict shape as a per-host timeout.
60 let aggregate = Duration::from_secs(constants::SCAN_EXTERNAL_LOOKUP_TIMEOUT_SECS);
61 let host_count = hosts.len();
62 let scan = async {
63 for host in &hosts {
64 match tokio::time::timeout(per_host, query_host(host, key)).await {
65 Ok(Ok(verdict)) => match verdict {
66 HostVerdict::Clean => {}
67 HostVerdict::Bad(reason) => {
68 return LayerResult {
69 layer: "urlhaus",
70 verdict: LayerVerdict::Fail,
71 detail: Some(format!("Known-malicious host {host}: {reason}")),
72 };
73 }
74 HostVerdict::AuthError(detail) => {
75 return LayerResult {
76 layer: "urlhaus",
77 verdict: LayerVerdict::Error,
78 detail: Some(detail),
79 };
80 }
81 },
82 Ok(Err(e)) => {
83 return LayerResult {
84 layer: "urlhaus",
85 verdict: LayerVerdict::Error,
86 detail: Some(format!("URLhaus error: {e}")),
87 };
88 }
89 Err(_) => {
90 return LayerResult {
91 layer: "urlhaus",
92 verdict: LayerVerdict::Error,
93 detail: Some("URLhaus lookup timed out".to_string()),
94 };
95 }
96 }
97 }
98
99 LayerResult {
100 layer: "urlhaus",
101 verdict: LayerVerdict::Pass,
102 detail: Some(format!("Checked {host_count} host(s); none known-bad")),
103 }
104 };
105
106 match tokio::time::timeout(aggregate, scan).await {
107 Ok(result) => result,
108 Err(_) => LayerResult {
109 layer: "urlhaus",
110 verdict: LayerVerdict::Error,
111 detail: Some("URLhaus aggregate lookup deadline exceeded".to_string()),
112 },
113 }
114 }
115
116 enum HostVerdict {
117 Clean,
118 Bad(String),
119 AuthError(String),
120 }
121
122 async fn query_host(host: &str, auth_key: &str) -> Result<HostVerdict, String> {
123 // Explicit request + connect timeouts (Perf-S2): a default client has neither,
124 // so a hung host could block past the layer's own per-host timeout. Falls back
125 // to a default client only if the builder somehow fails.
126 static CLIENT: std::sync::LazyLock<reqwest::Client> = std::sync::LazyLock::new(|| {
127 crate::crypto::install_default_crypto_provider();
128 reqwest::Client::builder()
129 .timeout(Duration::from_secs(
130 constants::SCAN_MALWAREBAZAAR_TIMEOUT_SECS,
131 ))
132 .connect_timeout(Duration::from_secs(
133 constants::SCAN_HTTP_CONNECT_TIMEOUT_SECS,
134 ))
135 .build()
136 .unwrap_or_default()
137 });
138 let client = &*CLIENT;
139
140 let params = [("host", host)];
141 let response = client
142 .post(URLHAUS_API_URL)
143 .header("Auth-Key", auth_key)
144 .form(&params)
145 .send()
146 .await
147 .map_err(|e| format!("HTTP request failed: {e}"))?;
148
149 let status = response.status();
150 if status.as_u16() == 401 || status.as_u16() == 403 {
151 return Ok(HostVerdict::AuthError(format!("HTTP {}", status.as_u16())));
152 }
153 if !status.is_success() {
154 return Err(format!("HTTP {} from URLhaus", status.as_u16()));
155 }
156 let body: serde_json::Value = response
157 .json()
158 .await
159 .map_err(|e| format!("Failed to parse response: {e}"))?;
160 Ok(classify_urlhaus_response(&body))
161 }
162
163 fn classify_urlhaus_response(body: &serde_json::Value) -> HostVerdict {
164 let query_status = body
165 .get("query_status")
166 .and_then(|v| v.as_str())
167 .unwrap_or("unknown");
168
169 match query_status {
170 // No record for this host.
171 "no_results" => HostVerdict::Clean,
172 // Host found in the URLhaus index.
173 "ok" => {
174 // Look at the urls array; report the threat label of the first
175 // entry if available. URLhaus puts the actual classification in
176 // `threat` / `tags` per-URL.
177 let threat = body
178 .get("urls")
179 .and_then(|u| u.get(0))
180 .and_then(|entry| entry.get("threat"))
181 .and_then(|t| t.as_str())
182 .unwrap_or("malicious");
183 HostVerdict::Bad(threat.to_string())
184 }
185 "unauthorized" | "key_required" | "key_invalid" => {
186 HostVerdict::AuthError(format!("abuse.ch auth: {query_status}"))
187 }
188 // URLhaus also returns "invalid_host" / "no_host_provided", treat as
189 // clean for this layer's purposes (the host was malformed, not malicious).
190 "invalid_host" | "no_host_provided" => HostVerdict::Clean,
191 _ => HostVerdict::Clean, // unknown status from a known-degraded API: don't fail closed
192 }
193 }
194
195 /// Pull printable-ASCII URL hosts out of the byte buffer. Cap at `max` unique
196 /// hosts to bound per-upload work and free-tier quota use.
197 pub(crate) fn extract_unique_hosts(data: &[u8], max: usize) -> Vec<String> {
198 let scan = if data.len() > MAX_SCAN_BYTES {
199 &data[..MAX_SCAN_BYTES]
200 } else {
201 data
202 };
203
204 let mut hosts: HashSet<String> = HashSet::new();
205 let mut out: Vec<String> = Vec::new();
206
207 let mut start = 0usize;
208 while start < scan.len() {
209 // Find next printable run of length >= MIN_RUN_LEN.
210 while start < scan.len() && !is_url_char(scan[start]) {
211 start += 1;
212 }
213 let mut end = start;
214 while end < scan.len() && is_url_char(scan[end]) {
215 end += 1;
216 }
217 if end - start >= MIN_RUN_LEN {
218 // Cheap heuristic: only attempt URL parse if the run contains "://".
219 let bytes = &scan[start..end];
220 if let Some(idx) = find_scheme(bytes) {
221 let run = &bytes[idx..];
222 if let Ok(s) = std::str::from_utf8(run)
223 && let Some(host) = extract_host(s)
224 {
225 let host = host.to_ascii_lowercase();
226 if !hosts.contains(&host) {
227 hosts.insert(host.clone());
228 out.push(host);
229 if out.len() >= max {
230 return out;
231 }
232 }
233 }
234 }
235 }
236 start = end + 1;
237 }
238 out
239 }
240
241 fn is_url_char(b: u8) -> bool {
242 // Printable ASCII excluding whitespace and common delimiters that would
243 // break a URL run. Permissive enough to catch real URLs, strict enough to
244 // exclude noise.
245 matches!(
246 b,
247 b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' |
248 b'-' | b'.' | b':' | b'/' | b'?' | b'&' | b'=' |
249 b'#' | b'_' | b'%' | b'+' | b',' | b';' | b'~' | b'!' | b'*' | b'$' | b'@'
250 )
251 }
252
253 fn find_scheme(bytes: &[u8]) -> Option<usize> {
254 // Find ":/" position then back up to the scheme start.
255 bytes.windows(3).position(|w| w == b"://").map(|colon_at| {
256 let mut start = colon_at;
257 while start > 0
258 && (bytes[start - 1].is_ascii_alphanumeric()
259 || bytes[start - 1] == b'+'
260 || bytes[start - 1] == b'-'
261 || bytes[start - 1] == b'.')
262 {
263 start -= 1;
264 }
265 start
266 })
267 }
268
269 fn extract_host(url_run: &str) -> Option<&str> {
270 let after_scheme = url_run.split_once("://")?.1;
271 let host = after_scheme.split(['/', '?', '#', ':']).next()?;
272 if host.is_empty() || !host.contains('.') {
273 return None;
274 }
275 Some(host)
276 }
277
278 #[cfg(test)]
279 mod tests {
280 use super::*;
281 use serde_json::json;
282
283 /// Exercise the production scan path: extract hosts, then look them up. The
284 /// scan pipeline always extracts on a blocking thread then calls
285 /// `check_urlhaus_hosts`; there is no separate buffered helper.
286 async fn check(data: &[u8], auth_key: Option<&str>) -> LayerResult {
287 let hosts = extract_unique_hosts(data, MAX_HOSTS_PER_FILE);
288 check_urlhaus_hosts(hosts, auth_key).await
289 }
290
291 #[tokio::test]
292 async fn no_auth_key_returns_skip() {
293 let result = check(b"http://example.com/", None).await;
294 assert_eq!(result.verdict, LayerVerdict::Skip);
295 }
296
297 #[tokio::test]
298 async fn no_urls_in_data_passes() {
299 // With an auth key, an empty buffer should still pass (no URLs to check).
300 let result = check(b"plain binary blob no urls here", Some("test")).await;
301 assert_eq!(result.verdict, LayerVerdict::Pass);
302 assert!(result.detail.unwrap().contains("No URLs"));
303 }
304
305 #[test]
306 fn extracts_http_url_host() {
307 let data = b"\x00\x00\x00 random https://malicious.example.com/payload.bin trailing junk";
308 let hosts = extract_unique_hosts(data, 5);
309 assert_eq!(hosts, vec!["malicious.example.com"]);
310 }
311
312 #[test]
313 fn deduplicates_hosts_across_urls() {
314 let data = b"https://a.example/x https://a.example/y http://b.example/z";
315 let hosts = extract_unique_hosts(data, 5);
316 assert_eq!(hosts.len(), 2);
317 assert!(hosts.iter().any(|h| h == "a.example"));
318 assert!(hosts.iter().any(|h| h == "b.example"));
319 }
320
321 #[test]
322 fn respects_max_cap() {
323 let data = b"https://a.example/ https://b.example/ https://c.example/ https://d.example/ https://e.example/ https://f.example/";
324 let hosts = extract_unique_hosts(data, 3);
325 assert_eq!(hosts.len(), 3);
326 }
327
328 #[test]
329 fn ignores_runs_without_scheme() {
330 let data = b"justastring not a url at all 1234567890";
331 let hosts = extract_unique_hosts(data, 5);
332 assert!(hosts.is_empty());
333 }
334
335 #[test]
336 fn rejects_hosts_without_dot() {
337 let data = b"http://localhost/payload";
338 let hosts = extract_unique_hosts(data, 5);
339 assert!(hosts.is_empty());
340 }
341
342 #[test]
343 fn ipv4_literal_host_extracted() {
344 let data = b"http://192.168.1.1/get";
345 let hosts = extract_unique_hosts(data, 5);
346 assert_eq!(hosts, vec!["192.168.1.1"]);
347 }
348
349 #[test]
350 fn lowercases_host() {
351 let data = b"https://EvIl.ExAMPle.com/x";
352 let hosts = extract_unique_hosts(data, 5);
353 assert_eq!(hosts, vec!["evil.example.com"]);
354 }
355
356 #[test]
357 fn no_results_is_clean() {
358 let body = json!({"query_status": "no_results"});
359 assert!(matches!(
360 classify_urlhaus_response(&body),
361 HostVerdict::Clean
362 ));
363 }
364
365 #[test]
366 fn ok_response_is_bad() {
367 let body = json!({
368 "query_status": "ok",
369 "urls": [{"threat": "malware_download", "url": "http://bad.example/x"}]
370 });
371 match classify_urlhaus_response(&body) {
372 HostVerdict::Bad(reason) => assert!(reason.contains("malware_download")),
373 _ => panic!("expected Bad"),
374 }
375 }
376
377 #[test]
378 fn unauthorized_is_auth_error() {
379 let body = json!({"query_status": "unauthorized"});
380 match classify_urlhaus_response(&body) {
381 HostVerdict::AuthError(d) => assert!(d.contains("unauthorized")),
382 _ => panic!("expected AuthError"),
383 }
384 }
385
386 #[test]
387 fn invalid_host_treated_as_clean() {
388 let body = json!({"query_status": "invalid_host"});
389 assert!(matches!(
390 classify_urlhaus_response(&body),
391 HostVerdict::Clean
392 ));
393 }
394
395 #[test]
396 fn unknown_status_defaults_to_clean() {
397 // URLhaus is fail-open by policy. A response shape we don't recognize
398 // shouldn't fail the upload, the layer aggregator's error counts
399 // will surface this via the dashboard health panel separately.
400 let body = json!({"query_status": "totally_new_thing"});
401 assert!(matches!(
402 classify_urlhaus_response(&body),
403 HostVerdict::Clean
404 ));
405 }
406 }
407