Skip to main content

max / audiofiles

Record the drive behind every benchmark path These numbers are I/O bound, so a run belongs to a drive as much as to the code. Nothing in the output said which drive produced it, so a baseline saved from the external test disk and a run taken against the internal NVMe were indistinguishable after the fact, and the gap between them is larger than any optimisation is likely to buy. Each mode now opens with a conditions block naming the filesystem, device, model, rotational flag, and USB link speed behind each path it uses, and the ingest and analysis modes serialise the same under a "storage" key. Ingest reports corpus and vault separately: reading a corpus from the external disk while writing a vault to the internal one is a third measurement, distinct from either drive alone, and the JSON has to say which. Two conditions are called out because they silently invalidate a run. A USB link below 5 Gb/s means the timing measures the cable, which is what an Apple USB-C cable on this drive produces. A corpus small enough to sit in page cache means a repeat run reads from RAM, so two drives measured back to back can look identical however far apart they are. The analysis mode also gains the AF_BENCH_JSON output it never had, carrying the per-format decode means, p95s, and file counts, so the format comparison is diffable across drives rather than only readable in a scrollback. No schema bump: these are added keys, and a comparison tool treats a missing key as not measured.
Author: Max Johnson <me@maxj.phd> · 2026-07-29 17:08 UTC
Signed with PGP, not checked
Commit: 72708ea32a04c268b130c7481f83a2b2f33b3055
Parent: 2d5234a
5 files changed, +415 insertions, -1 deletion
M crates/audiofiles-bench/src/accuracy.rs +8
@@ -26,6 +26,8 @@
26 26 use audiofiles_core::analysis::{bpm, decode};
27 27 use rayon::prelude::*;
28 28
29 + use crate::storage;
30 +
29 31 /// Ground truth for one sound, after consensus across annotators.
30 32 struct Truth {
31 33 bpm: f64,
@@ -319,6 +321,12 @@
319 321 return;
320 322 };
321 323
324 + // Printed but not serialised: this mode emits no timings, so the drive
325 + // cannot change its results. It is recorded for provenance, so a scorecard
326 + // says which copy of the dataset produced the numbers.
327 + let audio_storage = storage::describe(&audio);
328 + storage::print_conditions(&[("audio", &audio_storage)], None);
329 +
322 330 let (truth, discarded, disputed) = load_truth(&annotations);
323 331 let index = index_audio(&audio);
324 332 println!(
M crates/audiofiles-bench/src/ingest.rs +19
@@ -32,6 +32,7 @@
32 32 use rayon::prelude::*;
33 33
34 34 use crate::report::{Report, peak_rss_mb};
35 + use crate::storage;
35 36
36 37 /// How many samples to analyse after import.
37 38 ///
@@ -240,6 +241,24 @@
240 241 return;
241 242 }
242 243
244 + // Described after the vault exists so its path canonicalizes, and reported
245 + // as two roles because corpus and vault are routinely on different drives:
246 + // reading from the external disk while writing to the internal one is a
247 + // third measurement, distinct from either drive on its own.
248 + let corpus_storage = storage::describe(corpus);
249 + let vault_storage = storage::describe(vault);
250 + report.set_storage("corpus", &corpus_storage);
251 + report.set_storage("vault", &vault_storage);
252 + let corpus_bytes: u64 = files
253 + .iter()
254 + .filter_map(|p| std::fs::metadata(p).ok())
255 + .map(|m| m.len())
256 + .sum();
257 + storage::print_conditions(
258 + &[("corpus", &corpus_storage), ("vault", &vault_storage)],
259 + Some(corpus_bytes),
260 + );
261 +
243 262 let db = match Database::open(vault.join("audiofiles.db")) {
244 263 Ok(db) => db,
245 264 Err(e) => {
M crates/audiofiles-bench/src/main.rs +47 -1
@@ -10,11 +10,19 @@
10 10 //!
11 11 //! Env: `AF_BENCH_CORPUS` (corpus root, default `<repo>/samples`),
12 12 //! `AF_BENCH_VAULT` (scratch vault for ingest), `AF_BENCH_FSL10K` (FSL10K root
13 - //! for accuracy), `AF_BENCH_BATCH`, `AF_BENCH_LIMIT`.
13 + //! for accuracy), `AF_BENCH_BATCH`, `AF_BENCH_LIMIT`, `AF_BENCH_ANALYZE`,
14 + //! `AF_BENCH_JSON` (machine-readable output path).
15 + //!
16 + //! Every mode opens with a conditions block naming the drive behind each path.
17 + //! These numbers are I/O bound, so they belong to a drive as much as to the
18 + //! code; see `storage`.
14 19
15 20 mod accuracy;
16 21 mod ingest;
17 22 mod report;
23 + mod storage;
24 +
25 + use crate::report::Report;
18 26
19 27 use std::collections::HashMap;
20 28 use std::path::{Path, PathBuf};
@@ -207,6 +215,23 @@
207 215 files
208 216 }
209 217
218 + /// Total bytes of audio under `dir`, for the page-cache warning.
219 + ///
220 + /// Returns None rather than 0 when the directory is unreadable, so a failure
221 + /// to measure does not print as "this corpus fits in RAM".
222 + fn dir_bytes(dir: &Path) -> Option<u64> {
223 + if !dir.exists() {
224 + return None;
225 + }
226 + Some(
227 + walkdir(dir)
228 + .iter()
229 + .filter_map(|p| std::fs::metadata(p).ok())
230 + .map(|m| m.len())
231 + .sum(),
232 + )
233 + }
234 +
210 235 fn walkdir(dir: &Path) -> Vec<PathBuf> {
211 236 let mut out = Vec::new();
212 237 if let Ok(entries) = std::fs::read_dir(dir) {
@@ -298,6 +323,11 @@
298 323 println!("╚══════════════════════════════════════════════════════════════╝");
299 324 println!();
300 325
326 + let mut report = Report::new("analysis");
327 + let corpus_storage = storage::describe(&samples_dir);
328 + report.set_storage("corpus", &corpus_storage);
329 + storage::print_conditions(&[("corpus", &corpus_storage)], dir_bytes(&samples_dir));
330 +
301 331 // Section 1: Per-Stage Timing (representative sample set)
302 332 println!("━━━ 1. PER-STAGE TIMING ━━━");
303 333 println!();
@@ -525,6 +555,20 @@
525 555 _ => "-".to_string(),
526 556 };
527 557 println!(" {fmt:<8} {count:>6} {mean:>10.2} {p95:>10.2} {max:>10.2} {rel:>9}");
558 +
559 + // Lowercased so the JSON keys stay snake_case like every other
560 + // metric. The file count travels with the timing because a mean
561 + // over a different number of files is a different measurement.
562 + let key = fmt.to_lowercase();
563 + report.set(
564 + &format!("decode_{key}_mean_ms"),
565 + (mean * 100.0).round() / 100.0,
566 + );
567 + report.set(
568 + &format!("decode_{key}_p95_ms"),
569 + (p95 * 100.0).round() / 100.0,
570 + );
571 + report.set(&format!("decode_{key}_files"), count);
528 572 }
529 573 println!();
530 574 println!(" Same audio in every row (44.1 kHz stereo), so the spread is codec cost.");
@@ -888,6 +932,8 @@
888 932 n + total_classified
889 933 );
890 934 println!("═══════════════════════════════════════════════════════════════");
935 +
936 + report.write();
891 937 }
892 938
893 939 // Library Scaling
M crates/audiofiles-bench/src/report.rs +21
@@ -26,6 +26,7 @@
26 26 pub(crate) struct Report {
27 27 mode: &'static str,
28 28 metrics: Map<String, Value>,
29 + storage: Map<String, Value>,
29 30 }
30 31
31 32 impl Report {
@@ -33,9 +34,20 @@
33 34 Self {
34 35 mode,
35 36 metrics: Map::new(),
37 + storage: Map::new(),
36 38 }
37 39 }
38 40
41 + /// Record the drive backing a path, under a role name (`corpus`, `vault`).
42 + ///
43 + /// Keyed by role rather than collapsed into one entry because a run can
44 + /// straddle two drives: reading a corpus from the external disk while
45 + /// writing the vault to the internal one is a different measurement from
46 + /// either drive alone, and the JSON has to say which.
47 + pub(crate) fn set_storage(&mut self, role: &str, storage: &crate::storage::Storage) {
48 + self.storage.insert(role.to_string(), storage.to_json());
49 + }
50 +
39 51 /// Record one metric. Keys are `snake_case` and carry their unit
40 52 /// (`_ms`, `_mb_s`, `_pct`) so a diff is readable without the schema.
41 53 pub(crate) fn set(&mut self, key: &str, value: impl Into<Value>) {
@@ -59,6 +71,15 @@
59 71 if let Some(rss) = peak_rss_mb() {
60 72 root.insert("peak_rss_mb".into(), rss.into());
61 73 }
74 + // Conditions sit next to the numbers, not inside metrics: they are what
75 + // the run was taken under, and a comparison tool has to read them
76 + // before deciding whether two runs are comparable at all.
77 + if !self.storage.is_empty() {
78 + root.insert("storage".into(), Value::Object(self.storage.clone()));
79 + }
80 + if let Some(mem) = crate::storage::mem_total_mb() {
81 + root.insert("mem_total_mb".into(), mem.round().into());
82 + }
62 83 root.insert("metrics".into(), Value::Object(self.metrics.clone()));
63 84
64 85 let text = serde_json::to_string_pretty(&Value::Object(root)).unwrap_or_default();
A crates/audiofiles-bench/src/storage.rs +320
@@ -1,0 +1,320 @@
1 + //! What drive a benchmark path actually sits on.
2 + //!
3 + //! Import and decode are I/O bound, so a run's numbers belong to a drive as
4 + //! much as to the code. Two runs of the same commit against the internal NVMe
5 + //! and against the exFAT test drive differ by more than any optimisation is
6 + //! likely to buy, and nothing in the output distinguished them: a saved
7 + //! baseline recorded throughput without recording what produced it.
8 + //!
9 + //! This records the conditions alongside the numbers so runs can be compared
10 + //! across drives, and so a comparison across *different* drives is visible as
11 + //! such rather than read as a regression.
12 + //!
13 + //! Linux only. Every field is optional and a missing one means "could not
14 + //! determine", never a default value.
15 +
16 + use std::path::{Path, PathBuf};
17 +
18 + use serde_json::{Map, Value};
19 +
20 + /// The backing store for one path.
21 + pub(crate) struct Storage {
22 + pub(crate) path: PathBuf,
23 + mount_point: Option<String>,
24 + fs_type: Option<String>,
25 + source: Option<String>,
26 + /// Whole-disk kernel name (`sda`), not the partition (`sda1`): the
27 + /// queue and device attributes hang off the disk.
28 + disk: Option<String>,
29 + model: Option<String>,
30 + rotational: Option<bool>,
31 + /// USB link speed in Mb/s, when the disk is behind USB. The distinction
32 + /// between 10000 and 480 here is the difference between a real
33 + /// measurement and a meaningless one, and a bad cable silently causes it.
34 + usb_speed_mbps: Option<u32>,
35 + }
36 +
37 + impl Storage {
38 + /// One line, for the printed conditions block.
39 + pub(crate) fn summary(&self) -> String {
40 + let mut parts: Vec<String> = Vec::new();
41 + if let Some(fs) = &self.fs_type {
42 + parts.push(fs.clone());
43 + }
44 + if let Some(src) = &self.source {
45 + parts.push(format!("on {src}"));
46 + }
47 + if let Some(model) = &self.model {
48 + parts.push(format!("({model})"));
49 + }
50 + match self.rotational {
51 + Some(true) => parts.push("rotational".into()),
52 + Some(false) => parts.push("solid-state".into()),
53 + None => {}
54 + }
55 + if let Some(mbps) = self.usb_speed_mbps {
56 + parts.push(format!("USB {mbps} Mb/s"));
57 + }
58 + if parts.is_empty() {
59 + "unknown".into()
60 + } else {
61 + parts.join(", ")
62 + }
63 + }
64 +
65 + /// True when this path is behind a USB link slower than 5 Gb/s.
66 + ///
67 + /// An Apple USB-C cable negotiates USB 2.0 on this drive, which turns
68 + /// every throughput number into a measurement of the cable.
69 + pub(crate) fn is_slow_usb(&self) -> bool {
70 + self.usb_speed_mbps.is_some_and(|s| s < 5000)
71 + }
72 +
73 + pub(crate) fn to_json(&self) -> Value {
74 + let mut m = Map::new();
75 + m.insert("path".into(), self.path.display().to_string().into());
76 + insert_opt(&mut m, "mount_point", self.mount_point.clone());
77 + insert_opt(&mut m, "fs_type", self.fs_type.clone());
78 + insert_opt(&mut m, "source", self.source.clone());
79 + insert_opt(&mut m, "disk", self.disk.clone());
80 + insert_opt(&mut m, "model", self.model.clone());
81 + if let Some(r) = self.rotational {
82 + m.insert("rotational".into(), r.into());
83 + }
84 + if let Some(s) = self.usb_speed_mbps {
85 + m.insert("usb_speed_mbps".into(), s.into());
86 + }
87 + Value::Object(m)
88 + }
89 + }
90 +
91 + fn insert_opt(m: &mut Map<String, Value>, key: &str, val: Option<String>) {
92 + if let Some(v) = val {
93 + m.insert(key.into(), v.into());
94 + }
95 + }
96 +
97 + /// Describe the drive backing `path`.
98 + ///
99 + /// Never fails: an undeterminable field is left unset rather than guessed, so
100 + /// a report from a machine this cannot read is still a valid report.
101 + pub(crate) fn describe(path: &Path) -> Storage {
102 + let canonical = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
103 + let mut storage = Storage {
104 + path: canonical.clone(),
105 + mount_point: None,
106 + fs_type: None,
107 + source: None,
108 + disk: None,
109 + model: None,
110 + rotational: None,
111 + usb_speed_mbps: None,
112 + };
113 +
114 + let Some(mount) = find_mount(&canonical) else {
115 + return storage;
116 + };
117 + storage.mount_point = Some(mount.point);
118 + storage.fs_type = Some(mount.fs_type);
119 + storage.source = Some(mount.source);
120 +
121 + if let Some(disk) = disk_for_dev(&mount.dev) {
122 + storage.rotational = read_trimmed(&format!("/sys/block/{disk}/queue/rotational"))
123 + .and_then(|v| v.parse::<u8>().ok())
124 + .map(|v| v == 1);
125 + storage.model = read_trimmed(&format!("/sys/block/{disk}/device/model"));
126 + storage.usb_speed_mbps = usb_speed_for_disk(&disk);
127 + storage.disk = Some(disk);
128 + }
129 +
130 + storage
131 + }
132 +
133 + struct Mount {
134 + point: String,
135 + fs_type: String,
136 + source: String,
137 + /// `major:minor`, which is how /sys/dev/block is keyed.
138 + dev: String,
139 + }
140 +
141 + /// Find the mount entry whose mount point is the longest prefix of `path`.
142 + ///
143 + /// Longest wins because mount points nest: /media/max/T9 sits under /, and the
144 + /// shorter match would attribute the external drive's numbers to the root
145 + /// filesystem.
146 + fn find_mount(path: &Path) -> Option<Mount> {
147 + let text = std::fs::read_to_string("/proc/self/mountinfo").ok()?;
148 + let mut best: Option<Mount> = None;
149 + let mut best_len = 0usize;
150 +
151 + for line in text.lines() {
152 + // Optional fields sit between the mount options and a "-" separator,
153 + // so the tail cannot be indexed from the start of the line.
154 + let (head, tail) = line.split_once(" - ")?;
155 + let head: Vec<&str> = head.split_whitespace().collect();
156 + let tail: Vec<&str> = tail.split_whitespace().collect();
157 + if head.len() < 5 || tail.len() < 2 {
158 + continue;
159 + }
160 + let point = unescape(head[4]);
161 + if !path.starts_with(&point) {
162 + continue;
163 + }
164 + if point.len() < best_len {
165 + continue;
166 + }
167 + best_len = point.len();
168 + best = Some(Mount {
169 + point,
170 + fs_type: tail[0].to_string(),
171 + source: unescape(tail[1]),
172 + dev: head[2].to_string(),
173 + });
174 + }
175 + best
176 + }
177 +
178 + /// mountinfo octal-escapes space, tab, newline and backslash in paths.
179 + fn unescape(s: &str) -> String {
180 + if !s.contains('\\') {
181 + return s.to_string();
182 + }
183 + let mut out = String::with_capacity(s.len());
184 + let bytes = s.as_bytes();
185 + let mut i = 0;
186 + while i < bytes.len() {
187 + if bytes[i] == b'\\' && i + 3 < bytes.len() {
188 + let oct = &s[i + 1..i + 4];
189 + if let Ok(v) = u8::from_str_radix(oct, 8) {
190 + out.push(v as char);
191 + i += 4;
192 + continue;
193 + }
194 + }
195 + out.push(bytes[i] as char);
196 + i += 1;
197 + }
198 + out
199 + }
200 +
201 + /// Map `major:minor` to the whole-disk kernel name.
202 + ///
203 + /// /sys/dev/block/<maj>:<min> resolves to the partition; the queue and device
204 + /// attributes live on its parent disk, so a partition walks up one level.
205 + fn disk_for_dev(dev: &str) -> Option<String> {
206 + let link = std::fs::read_link(format!("/sys/dev/block/{dev}")).ok()?;
207 + let name = link.file_name()?.to_string_lossy().to_string();
208 + // A partition directory contains "partition"; a whole disk does not.
209 + let resolved = format!("/sys/dev/block/{dev}");
210 + if Path::new(&format!("{resolved}/partition")).exists() {
211 + let parent = link.parent()?.file_name()?.to_string_lossy().to_string();
212 + return Some(parent);
213 + }
214 + Some(name)
215 + }
216 +
217 + /// USB link speed in Mb/s for a disk, or None when it is not behind USB.
218 + ///
219 + /// Walks up the sysfs device chain to the first ancestor that looks like a USB
220 + /// device (has both `speed` and `idVendor`). Intermediate nodes such as the
221 + /// SCSI host have no speed, and the USB interface node has no idVendor.
222 + fn usb_speed_for_disk(disk: &str) -> Option<u32> {
223 + let link = std::fs::read_link(format!("/sys/block/{disk}")).ok()?;
224 + let mut dir = Path::new("/sys/block").join(link).canonicalize().ok()?;
225 + loop {
226 + if dir.join("idVendor").exists()
227 + && let Some(speed) = read_trimmed(&dir.join("speed").display().to_string())
228 + && let Ok(v) = speed.parse::<f64>()
229 + {
230 + return Some(v.round() as u32);
231 + }
232 + dir = dir.parent()?.to_path_buf();
233 + if dir == Path::new("/sys") || dir == Path::new("/") {
234 + return None;
235 + }
236 + }
237 + }
238 +
239 + /// Print the conditions block that heads every mode's output.
240 + ///
241 + /// `corpus_bytes` is what the run will actually read, when the caller already
242 + /// knows it. It drives the page-cache warning, which matters more than it
243 + /// looks: a corpus that fits in RAM is served from memory on the second run,
244 + /// so two drives measured back to back can look identical no matter how far
245 + /// apart they really are.
246 + pub(crate) fn print_conditions(roles: &[(&str, &Storage)], corpus_bytes: Option<u64>) {
247 + println!("━━━ CONDITIONS ━━━");
248 + println!();
249 + for (role, storage) in roles {
250 + println!(" {:<8} {}", role, storage.path.display());
251 + println!(" {:<8} {}", "", storage.summary());
252 + if storage.is_slow_usb() {
253 + println!(
254 + " {:<8} WARNING: this link is USB 2.0 speed. Every timing here is a",
255 + ""
256 + );
257 + println!(
258 + " {:<8} measurement of the cable. Replace it before recording.",
259 + ""
260 + );
261 + }
262 + }
263 + if let Some(mem) = mem_total_mb() {
264 + println!(" {:<8} {:.0} MB RAM", "memory", mem);
265 + if let Some(bytes) = corpus_bytes {
266 + let corpus_mb = bytes as f64 / 1e6;
267 + if corpus_mb < mem * 0.5 {
268 + println!(
269 + " {:<8} corpus is {:.0} MB against {:.0} MB RAM, so a repeat run reads",
270 + "", corpus_mb, mem
271 + );
272 + println!(
273 + " {:<8} from page cache rather than the drive. Compare drives cold.",
274 + ""
275 + );
276 + }
277 + }
278 + }
279 + println!();
280 + }
281 +
282 + fn read_trimmed(path: &str) -> Option<String> {
283 + std::fs::read_to_string(path)
284 + .ok()
285 + .map(|s| s.trim().to_string())
286 + .filter(|s| !s.is_empty())
287 + }
288 +
289 + /// Total system RAM in MB.
290 + ///
291 + /// Recorded because it decides whether a corpus fits in page cache, which is
292 + /// the difference between timing a drive and timing memory.
293 + pub(crate) fn mem_total_mb() -> Option<f64> {
294 + let text = std::fs::read_to_string("/proc/meminfo").ok()?;
295 + let line = text.lines().find(|l| l.starts_with("MemTotal:"))?;
296 + let kb: f64 = line.split_whitespace().nth(1)?.parse().ok()?;
297 + Some(kb / 1024.0)
298 + }
299 +
300 + #[cfg(test)]
301 + mod tests {
302 + use super::*;
303 +
304 + #[test]
305 + fn unescapes_octal_paths() {
306 + assert_eq!(unescape("/media/max/My\\040Drive"), "/media/max/My Drive");
307 + assert_eq!(unescape("/plain/path"), "/plain/path");
308 + }
309 +
310 + #[test]
311 + fn describes_a_real_path_without_panicking() {
312 + // Root always exists and is always mounted, so the mount lookup must
313 + // succeed here. A silent None would mean the mountinfo parse broke,
314 + // which is exactly the failure this test exists to catch.
315 + let s = describe(Path::new("/"));
316 + assert!(s.fs_type.is_some(), "no filesystem type for /");
317 + assert!(s.mount_point.is_some(), "no mount point for /");
318 + assert_ne!(s.summary(), "unknown");
319 + }
320 + }