Skip to main content

max / audiofiles

Add ingest and accuracy benchmarks plus corpus tooling The analysis bench measured per-file DSP cost against a samples/ tree that is gitignored and had to be rebuilt by hand on every machine. Two layers went unmeasured. scripts/corpus.py fetches and lays out that tree with per-dataset toggles. Only the two CC-BY sources are on by default; NSynth and FSD50K are opt-in, and the manifest records per-dataset licences because docs/ml_classifier.md keeps training-data copyright surface out of the binary deliberately. Files matching two class keywords are dropped rather than resolved by rule order: these packs contain layered hits ("Kick-Hat", "Snare-Cymbal") and a wrong label reads as a classifier error forever. bench "ingest" measures the vault layer: import throughput per batch, dedup, and the query latency behind the browser list and filter panel. The file list is shuffled deterministically because path order groups by class, and classes differ in mean file size, so a files/s trend across batches would otherwise track composition rather than scale. It creates vfs nodes explicitly: store.import writes the blob and samples row but no node, and search_global over zero nodes returns instantly and measures nothing. bench "accuracy" scores tempo and key against FSL10K ground truth, which is the only corpus here carrying expert annotations. Tempo uses the MIREX Acc1/Acc2 pair so octave errors are visible as their own number. Annotator disagreements are dropped and counted rather than resolved. Key coverage is reported alongside accuracy so a zero cannot be mistaken for a real score when the cause is a parser or gating problem. The corpus root is now AF_BENCH_CORPUS, defaulting to the old in-repo path, since on a dev box it usually lives on an external drive.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-29 16:04 UTC
Signed with PGP, not checked
Commit: 5a6ee2a1b55697aadbe9a0c4d1f6289642a464a3
Parent: 69a616d
5 files changed, +1204 insertions, -2 deletions
M .gitignore +3
@@ -43,3 +43,6 @@
43 43
44 44 # Private working files — live in _private/, synced via Syncthing
45 45 todo.md
46 +
47 + # Python bytecode from scripts/
48 + __pycache__/
M crates/audiofiles-bench/src/main.rs +60 -2
@@ -3,7 +3,17 @@
3 3 //! Measures per-stage timing, throughput, resource usage, and classification accuracy
4 4 //! against labeled training data.
5 5 //!
6 - //! Usage: `cargo run --release -p audiofiles-bench`
6 + //! Usage:
7 + //! `cargo run --release -p audiofiles-bench` analysis pipeline
8 + //! `cargo run --release -p audiofiles-bench -- ingest` vault ingest + queries
9 + //! `cargo run --release -p audiofiles-bench -- accuracy` bpm/key vs ground truth
10 + //!
11 + //! Env: `AF_BENCH_CORPUS` (corpus root, default `<repo>/samples`),
12 + //! `AF_BENCH_VAULT` (scratch vault for ingest), `AF_BENCH_FSL10K` (FSL10K root
13 + //! for accuracy), `AF_BENCH_BATCH`, `AF_BENCH_LIMIT`.
14 +
15 + mod accuracy;
16 + mod ingest;
7 17
8 18 use std::collections::HashMap;
9 19 use std::path::{Path, PathBuf};
@@ -219,14 +229,62 @@
219 229 values[idx.min(values.len() - 1)]
220 230 }
221 231
232 + /// Locate the corpus.
233 + ///
234 + /// The corpus is gitignored and rebuilt per machine (see `scripts/corpus.py`),
235 + /// and on a dev box it is usually on an external drive rather than in the
236 + /// checkout, so the in-repo path is only the fallback.
237 + fn corpus_dir() -> PathBuf {
238 + if let Ok(dir) = std::env::var("AF_BENCH_CORPUS") {
239 + return PathBuf::from(dir);
240 + }
241 + PathBuf::from(env!("CARGO_MANIFEST_DIR"))
242 + .parent()
243 + .unwrap()
244 + .parent()
245 + .unwrap()
246 + .join("samples")
247 + }
248 +
222 249 fn main() {
250 + let args: Vec<String> = std::env::args().skip(1).collect();
251 + let samples_dir = corpus_dir();
223 252 let project_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
224 253 .parent()
225 254 .unwrap()
226 255 .parent()
227 256 .unwrap()
228 257 .to_path_buf();
229 - let samples_dir = project_root.join("samples");
258 +
259 + if args.first().map(String::as_str) == Some("ingest") {
260 + let vault = std::env::var("AF_BENCH_VAULT").map_or_else(
261 + |_| std::env::temp_dir().join("af-bench-vault"),
262 + PathBuf::from,
263 + );
264 + let batch = std::env::var("AF_BENCH_BATCH")
265 + .ok()
266 + .and_then(|v| v.parse().ok())
267 + .unwrap_or(500);
268 + let limit = std::env::var("AF_BENCH_LIMIT")
269 + .ok()
270 + .and_then(|v| v.parse().ok());
271 + ingest::run(&samples_dir, &vault, batch, limit);
272 + return;
273 + }
274 +
275 + if args.first().map(String::as_str) == Some("accuracy") {
276 + let Ok(root) = std::env::var("AF_BENCH_FSL10K") else {
277 + eprintln!("set AF_BENCH_FSL10K to the extracted FSL10K root");
278 + eprintln!("fetch it with: ./scripts/corpus.py --datasets fsl10k");
279 + std::process::exit(1);
280 + };
281 + let limit = std::env::var("AF_BENCH_LIMIT")
282 + .ok()
283 + .and_then(|v| v.parse().ok());
284 + accuracy::run(&PathBuf::from(root), limit);
285 + return;
286 + }
287 +
230 288 let training_dir = samples_dir.join("training");
231 289 let test_suite_dir = samples_dir.join("test-suite");
232 290
A crates/audiofiles-bench/src/accuracy.rs +449
@@ -1,0 +1,449 @@
1 + //! BPM and key accuracy against FSL10K ground truth.
2 + //!
3 + //! Section 5 of the analysis bench already scores classification against folder
4 + //! labels. This scores the other two things the pipeline emits and that nothing
5 + //! else can check: `bpm::detect_bpm_key`'s tempo and key.
6 + //!
7 + //! Ground truth comes from the Freesound Loop Dataset (CC-BY 4.0), which is the
8 + //! only corpus here carrying expert tempo and key annotations. Fetch it with
9 + //! `scripts/corpus.py --datasets fsl10k`, then point this at the extracted root:
10 + //!
11 + //! AF_BENCH_FSL10K=/media/max/T9/af-corpus/_raw/fsl10k \
12 + //! cargo run --release -p audiofiles-bench -- accuracy
13 + //!
14 + //! Tempo is scored with the standard MIREX pair rather than a single number,
15 + //! because a tempo estimator that is "wrong" is usually wrong by a factor of
16 + //! two, and collapsing that into one accuracy figure hides which kind of wrong
17 + //! it is:
18 + //! Acc1 = within 4% of ground truth.
19 + //! Acc2 = Acc1, or within 4% of truth scaled by 1/3, 1/2, 2, or 3.
20 + //! A large Acc2-minus-Acc1 gap means octave errors, which are a different fix
21 + //! from being diffusely inaccurate.
22 +
23 + use std::collections::HashMap;
24 + use std::path::{Path, PathBuf};
25 +
26 + use audiofiles_core::analysis::{bpm, decode};
27 + use rayon::prelude::*;
28 +
29 + /// Ground truth for one sound, after consensus across annotators.
30 + struct Truth {
31 + bpm: f64,
32 + /// Pitch class 0..=11 with mode, or None when the annotators marked the
33 + /// loop as having no key (drum loops, percussion, fx).
34 + key: Option<(u8, Mode)>,
35 + }
36 +
37 + #[derive(Clone, Copy, PartialEq, Eq, Debug)]
38 + enum Mode {
39 + Major,
40 + Minor,
41 + }
42 +
43 + /// Parse a note name to a pitch class.
44 + ///
45 + /// Comparing pitch classes rather than strings sidesteps enharmonic spelling
46 + /// entirely: the annotations use sharps only, but nothing guarantees the
47 + /// detector does, and "A#" and "Bb" are the same key.
48 + fn pitch_class(note: &str) -> Option<u8> {
49 + let s = note.trim().to_lowercase();
50 + let mut chars = s.chars();
51 + let base = match chars.next()? {
52 + 'c' => 0,
53 + 'd' => 2,
54 + 'e' => 4,
55 + 'f' => 5,
56 + 'g' => 7,
57 + 'a' => 9,
58 + 'b' => 11,
59 + _ => return None,
60 + };
61 + let accidental: i32 = match chars.next() {
62 + Some('#' | 's') => 1,
63 + Some('b' | 'f') => -1,
64 + None => 0,
65 + _ => return None,
66 + };
67 + Some(((base + accidental).rem_euclid(12)) as u8)
68 + }
69 +
70 + /// Parse a detected key string into a pitch class and mode.
71 + ///
72 + /// Handles both shapes on purpose. `detect_bpm_key` now normalises to
73 + /// "A minor" / "C major", but before that it passed through
74 + /// `stratum_dsp::Key::name()` unchanged, which is "C" / "F#" for major and
75 + /// "Am" / "C#m" for minor. Vaults analysed before migration 034 can still hold
76 + /// the compact form, and accepting only one spelling silently scores 0%
77 + /// forever rather than failing loudly, so both are parsed.
78 + fn parse_app_key(s: &str) -> Option<(u8, Mode)> {
79 + let s = s.trim();
80 + if let Some((note, word)) = s.rsplit_once(' ') {
81 + let mode = match word.trim().to_lowercase().as_str() {
82 + "minor" | "min" => Mode::Minor,
83 + "major" | "maj" => Mode::Major,
84 + _ => return None,
85 + };
86 + return Some((pitch_class(note)?, mode));
87 + }
88 + // Bare form: trailing 'm' means minor, otherwise major. Checked after the
89 + // spaced form so "F major" is not read as a note named "F majo" + m.
90 + if let Some(note) = s.strip_suffix('m') {
91 + return Some((pitch_class(note)?, Mode::Minor));
92 + }
93 + Some((pitch_class(s)?, Mode::Major))
94 + }
95 +
96 + /// Load and reconcile annotations.
97 + ///
98 + /// Half the annotated sounds in FSL10K carry more than one annotator's
99 + /// judgement, and annotators disagree. Rather than picking one arbitrarily,
100 + /// disagreements are dropped and counted: scoring a detector against a
101 + /// contested label measures the disagreement, not the detector.
102 + fn load_truth(annotations: &Path) -> (HashMap<String, Truth>, usize, usize) {
103 + let mut by_sound: HashMap<String, Vec<serde_json::Value>> = HashMap::new();
104 + let mut discarded = 0usize;
105 +
106 + let mut stack = vec![annotations.to_path_buf()];
107 + while let Some(dir) = stack.pop() {
108 + let Ok(entries) = std::fs::read_dir(&dir) else {
109 + continue;
110 + };
111 + for entry in entries.flatten() {
112 + let path = entry.path();
113 + if path.is_dir() {
114 + stack.push(path);
115 + continue;
116 + }
117 + let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
118 + continue;
119 + };
120 + let Some(id) = name
121 + .strip_prefix("sound-")
122 + .and_then(|n| n.strip_suffix(".json"))
123 + else {
124 + continue;
125 + };
126 + let Ok(text) = std::fs::read_to_string(&path) else {
127 + continue;
128 + };
129 + let Ok(v) = serde_json::from_str::<serde_json::Value>(&text) else {
130 + continue;
131 + };
132 + if v.get("discard").and_then(serde_json::Value::as_bool) == Some(true) {
133 + discarded += 1;
134 + continue;
135 + }
136 + by_sound.entry(id.to_string()).or_default().push(v);
137 + }
138 + }
139 +
140 + let mut truth = HashMap::new();
141 + let mut disputed = 0usize;
142 +
143 + for (id, annots) in by_sound {
144 + // BPM is stored as a string in these files.
145 + let bpms: Vec<f64> = annots
146 + .iter()
147 + .filter_map(|a| a.get("bpm").and_then(serde_json::Value::as_str))
148 + .filter_map(|s| s.trim().parse::<f64>().ok())
149 + .filter(|b| *b > 0.0)
150 + .collect();
151 + if bpms.is_empty() {
152 + continue;
153 + }
154 + // Annotators agree if they are all within 4% of the first, the same
155 + // tolerance the scoring uses.
156 + let first = bpms[0];
157 + if bpms.iter().any(|b| (b - first).abs() / first > 0.04) {
158 + disputed += 1;
159 + continue;
160 + }
161 + let bpm_truth = bpms.iter().sum::<f64>() / bpms.len() as f64;
162 +
163 + let keys: Vec<Option<(u8, Mode)>> = annots
164 + .iter()
165 + .map(|a| {
166 + let k = a.get("key").and_then(serde_json::Value::as_str)?;
167 + let m = a.get("mode").and_then(serde_json::Value::as_str)?;
168 + let mode = match m {
169 + "min" => Mode::Minor,
170 + "maj" => Mode::Major,
171 + _ => return None,
172 + };
173 + Some((pitch_class(k)?, mode))
174 + })
175 + .collect();
176 + // "none"/"unknown" parse to None, which is itself a meaningful answer
177 + // (the loop has no key), so only disagreement between Some values is a
178 + // dispute.
179 + let key_truth = if keys.iter().all(|k| *k == keys[0]) {
180 + keys[0]
181 + } else {
182 + None
183 + };
184 +
185 + truth.insert(
186 + id,
187 + Truth {
188 + bpm: bpm_truth,
189 + key: key_truth,
190 + },
191 + );
192 + }
193 +
194 + (truth, discarded, disputed)
195 + }
196 +
197 + /// Index audio files by Freesound sound ID (the filename stem).
198 + fn index_audio(root: &Path) -> HashMap<String, PathBuf> {
199 + let mut out = HashMap::new();
200 + let mut stack = vec![root.to_path_buf()];
201 + while let Some(dir) = stack.pop() {
202 + let Ok(entries) = std::fs::read_dir(&dir) else {
203 + continue;
204 + };
205 + for entry in entries.flatten() {
206 + let path = entry.path();
207 + if path.is_dir() {
208 + stack.push(path);
209 + continue;
210 + }
211 + let is_audio = path.extension().and_then(|e| e.to_str()).is_some_and(|e| {
212 + matches!(
213 + e.to_lowercase().as_str(),
214 + "wav" | "flac" | "mp3" | "ogg" | "aif" | "aiff"
215 + )
216 + });
217 + if !is_audio {
218 + continue;
219 + }
220 + if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
221 + // FSL10K names audio by sound ID, sometimes with a suffix after
222 + // the ID; take the leading digit run.
223 + let id: String = stem.chars().take_while(char::is_ascii_digit).collect();
224 + if !id.is_empty() {
225 + out.insert(id, path);
226 + }
227 + }
228 + }
229 + }
230 + out
231 + }
232 +
233 + fn within(a: f64, b: f64, tol: f64) -> bool {
234 + b > 0.0 && (a - b).abs() / b <= tol
235 + }
236 +
237 + struct Scored {
238 + bpm_acc1: bool,
239 + bpm_acc2: bool,
240 + /// Set only when both truth and detection carry a key.
241 + key_exact: Option<bool>,
242 + key_related: Option<bool>,
243 + /// Tracked separately so a zero key score can be attributed. "No key was
244 + /// scored" has three very different causes -- the loop has no key in the
245 + /// ground truth, the detector returned nothing, or the detected string
246 + /// failed to parse -- and collapsing them hides a parser bug as a result.
247 + had_truth_key: bool,
248 + had_detected_key: bool,
249 + detected_key_unparsed: bool,
250 + }
251 +
252 + fn score(detected_bpm: Option<f64>, detected_key: Option<&str>, truth: &Truth) -> Scored {
253 + let (acc1, acc2) = match detected_bpm {
254 + Some(d) => {
255 + let a1 = within(d, truth.bpm, 0.04);
256 + // Octave and triplet confusions: the classic failure mode of
257 + // autocorrelation tempo estimators.
258 + let a2 = a1
259 + || [1.0 / 3.0, 0.5, 2.0, 3.0]
260 + .iter()
261 + .any(|f| within(d, truth.bpm * f, 0.04));
262 + (a1, a2)
263 + }
264 + None => (false, false),
265 + };
266 +
267 + let (key_exact, key_related) = match (truth.key, detected_key.and_then(parse_app_key)) {
268 + (Some((tp, tm)), Some((dp, dm))) => {
269 + let exact = tp == dp && tm == dm;
270 + // Related = relative major/minor, or a perfect fifth apart in the
271 + // same mode. These are the confusions a chroma-based estimator makes
272 + // because the pitch content genuinely overlaps.
273 + let relative = match tm {
274 + Mode::Major => dm == Mode::Minor && dp == (tp + 9) % 12,
275 + Mode::Minor => dm == Mode::Major && dp == (tp + 3) % 12,
276 + };
277 + let fifth = tm == dm && (dp == (tp + 7) % 12 || dp == (tp + 5) % 12);
278 + (Some(exact), Some(exact || relative || fifth))
279 + }
280 + _ => (None, None),
281 + };
282 +
283 + Scored {
284 + bpm_acc1: acc1,
285 + bpm_acc2: acc2,
286 + key_exact,
287 + key_related,
288 + had_truth_key: truth.key.is_some(),
289 + had_detected_key: detected_key.is_some(),
290 + detected_key_unparsed: detected_key.is_some_and(|k| parse_app_key(k).is_none()),
291 + }
292 + }
293 +
294 + pub(crate) fn run(root: &Path, limit: Option<usize>) {
295 + println!("━━━ BPM + KEY ACCURACY (FSL10K ground truth) ━━━");
296 + println!();
297 +
298 + // The zip lays out FSL10K/audio/...; accept either the zip root or the
299 + // directory that directly contains audio/ and annotations/.
300 + let annotations = [
301 + "annotations",
302 + "FSL10K/annotations",
303 + "../fsl10k-annotations/annotations",
304 + ]
305 + .iter()
306 + .map(|p| root.join(p))
307 + .find(|p| p.is_dir());
308 + let audio = ["FSL10K/audio", "audio", "FSL10K/FSL10K/audio"]
309 + .iter()
310 + .map(|p| root.join(p))
311 + .find(|p| p.is_dir());
312 +
313 + let (Some(annotations), Some(audio)) = (annotations, audio) else {
314 + eprintln!(
315 + "could not find annotations/ and audio/ under {}",
316 + root.display()
317 + );
318 + eprintln!("fetch with: ./scripts/corpus.py --datasets fsl10k");
319 + return;
320 + };
321 +
322 + let (truth, discarded, disputed) = load_truth(&annotations);
323 + let index = index_audio(&audio);
324 + println!(
325 + " annotations: {} sounds with agreed ground truth",
326 + truth.len()
327 + );
328 + println!(
329 + " {discarded} annotation(s) flagged discard, {disputed} sound(s) dropped as disputed"
330 + );
331 + println!(" audio files indexed: {}", index.len());
332 +
333 + let mut pairs: Vec<(&String, &Truth, &PathBuf)> = truth
334 + .iter()
335 + .filter_map(|(id, t)| index.get(id).map(|p| (id, t, p)))
336 + .collect();
337 + pairs.sort_by_key(|(id, _, _)| (*id).clone());
338 + if let Some(lim) = limit {
339 + pairs.truncate(lim);
340 + }
341 +
342 + if pairs.is_empty() {
343 + eprintln!("no annotated sound matched an audio file");
344 + return;
345 + }
346 + println!(" scoring {} loops", pairs.len());
347 + println!();
348 +
349 + let results: Vec<Scored> = pairs
350 + .par_iter()
351 + .filter_map(|(_, t, path)| {
352 + let decoded = decode::decode_to_mono(path).ok()?;
353 + // 2.0 matches the analysis pipeline's min_duration gate, so this
354 + // measures what the app actually runs rather than a variant.
355 + let r = bpm::detect_bpm_key(&decoded.samples, decoded.sample_rate, 2.0);
356 + Some(score(r.bpm, r.key.as_deref(), t))
357 + })
358 + .collect();
359 +
360 + let n = results.len() as f64;
361 + let acc1 = results.iter().filter(|r| r.bpm_acc1).count() as f64 / n * 100.0;
362 + let acc2 = results.iter().filter(|r| r.bpm_acc2).count() as f64 / n * 100.0;
363 +
364 + println!(" TEMPO ({} scored)", results.len());
365 + println!(" Acc1 (within 4%) {acc1:>6.1}%");
366 + println!(" Acc2 (octave/triplet ok) {acc2:>6.1}%");
367 + let gap = acc2 - acc1;
368 + println!(" octave-error gap {gap:>6.1}%");
369 + if gap > 15.0 {
370 + println!(" ^ large gap: most misses are half/double time, not noise");
371 + }
372 + println!();
373 +
374 + let with_truth = results.iter().filter(|r| r.had_truth_key).count();
375 + let with_detected = results.iter().filter(|r| r.had_detected_key).count();
376 + let unparsed = results.iter().filter(|r| r.detected_key_unparsed).count();
377 +
378 + let keyed: Vec<&Scored> = results.iter().filter(|r| r.key_exact.is_some()).collect();
379 + println!(" KEY coverage");
380 + println!(
381 + " loops with ground-truth key {with_truth:>5} / {}",
382 + results.len()
383 + );
384 + println!(
385 + " loops with detected key {with_detected:>5} / {}",
386 + results.len()
387 + );
388 + if unparsed > 0 {
389 + println!(" detected but UNPARSED {unparsed:>5} <- parser/format mismatch");
390 + }
391 + println!(" scorable (both present) {:>5}", keyed.len());
392 +
393 + // Precision on keyless material, which the accuracy figures cannot show.
394 + // Most loops in a sample library are drums, and annotators mark those as
395 + // having no key. A detector that emits a key anyway is not inaccurate by
396 + // the measures above -- those only score loops that have a key -- but it
397 + // fills the library with confident wrong labels, and a key filter that
398 + // returns drum loops is a user-visible bug.
399 + let keyless = results.len() - with_truth;
400 + let spurious = results
401 + .iter()
402 + .filter(|r| !r.had_truth_key && r.had_detected_key)
403 + .count();
404 + if keyless > 0 {
405 + let rate = spurious as f64 / keyless as f64 * 100.0;
406 + println!(" key emitted on keyless loops {spurious:>5} / {keyless} ({rate:.1}%)");
407 + }
408 + println!();
409 +
410 + if keyed.is_empty() {
411 + println!(" KEY: nothing scorable. Check the coverage lines above for why.");
412 + } else {
413 + let kn = keyed.len() as f64;
414 + let exact = keyed.iter().filter(|r| r.key_exact == Some(true)).count() as f64 / kn * 100.0;
415 + let related =
416 + keyed.iter().filter(|r| r.key_related == Some(true)).count() as f64 / kn * 100.0;
417 + println!(" KEY ({} scored, of {} total)", keyed.len(), results.len());
418 + println!(" exact {exact:>6.1}%");
419 + println!(" exact or relative/fifth {related:>6.1}%");
420 + // Chance is 1/24 for exact over 12 pitch classes x 2 modes.
421 + println!(" (chance for exact is 4.2%)");
422 + }
423 + }
424 +
425 + #[cfg(test)]
426 + mod tests {
427 + use super::{Mode, parse_app_key, pitch_class};
428 +
429 + #[test]
430 + fn parses_stratum_key_format() {
431 + // The format detect_bpm_key actually emits.
432 + assert_eq!(parse_app_key("C"), Some((0, Mode::Major)));
433 + assert_eq!(parse_app_key("F#"), Some((6, Mode::Major)));
434 + assert_eq!(parse_app_key("Am"), Some((9, Mode::Minor)));
435 + assert_eq!(parse_app_key("C#m"), Some((1, Mode::Minor)));
436 + }
437 +
438 + #[test]
439 + fn parses_documented_key_format() {
440 + assert_eq!(parse_app_key("A minor"), Some((9, Mode::Minor)));
441 + assert_eq!(parse_app_key("C major"), Some((0, Mode::Major)));
442 + }
443 +
444 + #[test]
445 + fn enharmonics_are_the_same_pitch_class() {
446 + assert_eq!(pitch_class("a#"), pitch_class("bb"));
447 + assert_eq!(pitch_class("c#"), pitch_class("db"));
448 + }
449 + }
A crates/audiofiles-bench/src/ingest.rs +387
@@ -1,0 +1,387 @@
1 + //! Ingest and query benchmarks: the vault layer rather than the DSP layer.
2 + //!
3 + //! The analysis bench in `main.rs` measures per-file DSP cost. This measures
4 + //! what happens to a vault as it fills up: import throughput, dedup, and the
5 + //! query latency that backs the browser UI.
6 + //!
7 + //! Two properties of the store make scale worth measuring rather than
8 + //! assuming. Blobs live in one flat directory (`store_blob_path` is
9 + //! `root.join("{hash}.{ext}")`, no fanout), so a 100k-sample vault is 100k
10 + //! entries in a single directory, and directory-lookup cost is filesystem
11 + //! dependent. And the DB runs in WAL mode with several worker connections, so
12 + //! insert cost moves with index depth.
13 + //!
14 + //! Reported per batch rather than as one average, because the number that
15 + //! matters is whether throughput is flat or degrading as the vault grows.
16 +
17 + use std::path::{Path, PathBuf};
18 + use std::time::Instant;
19 +
20 + use audiofiles_core::db::Database;
21 + use audiofiles_core::id_types::SampleHash;
22 + use audiofiles_core::search::{self, SearchFilter, SearchScope};
23 + use audiofiles_core::store::SampleStore;
24 + use audiofiles_core::vfs;
25 +
26 + /// Deterministic reorder of the file list.
27 + ///
28 + /// Without this the list arrives sorted by path, which groups files by class,
29 + /// and classes have very different mean file sizes. Batch N would then differ
30 + /// from batch 1 in content as well as in vault size, so a files/s trend across
31 + /// batches would measure file-size composition rather than scaling behaviour.
32 + /// FNV-1a over the path keeps it deterministic without pulling in `rand`.
33 + fn shuffle_deterministic(files: &mut [PathBuf]) {
34 + fn fnv1a(s: &str) -> u64 {
35 + let mut h: u64 = 0xcbf2_9ce4_8422_2325;
36 + for b in s.as_bytes() {
37 + h ^= u64::from(*b);
38 + h = h.wrapping_mul(0x100_0000_01b3);
39 + }
40 + h
41 + }
42 + files.sort_by_key(|p| fnv1a(&p.to_string_lossy()));
43 + }
44 +
45 + /// One batch of imports.
46 + struct BatchStat {
47 + /// Cumulative sample count in the vault after this batch.
48 + cumulative: usize,
49 + files: usize,
50 + bytes: u64,
51 + elapsed_s: f64,
52 + }
53 +
54 + impl BatchStat {
55 + fn files_per_sec(&self) -> f64 {
56 + if self.elapsed_s <= 0.0 {
57 + return 0.0;
58 + }
59 + self.files as f64 / self.elapsed_s
60 + }
61 +
62 + fn mb_per_sec(&self) -> f64 {
63 + if self.elapsed_s <= 0.0 {
64 + return 0.0;
65 + }
66 + (self.bytes as f64 / 1e6) / self.elapsed_s
67 + }
68 + }
69 +
70 + /// Collect audio files under `dir`, recursively.
71 + fn collect(dir: &Path, out: &mut Vec<PathBuf>) {
72 + let Ok(entries) = std::fs::read_dir(dir) else {
73 + return;
74 + };
75 + for entry in entries.flatten() {
76 + let path = entry.path();
77 + if path.is_dir() {
78 + collect(&path, out);
79 + } else if path.extension().and_then(|e| e.to_str()).is_some_and(|e| {
80 + matches!(
81 + e.to_lowercase().as_str(),
82 + "wav" | "aif" | "aiff" | "flac" | "mp3" | "ogg"
83 + )
84 + }) {
85 + out.push(path);
86 + }
87 + }
88 + }
89 +
90 + fn count_samples(db: &Database) -> i64 {
91 + db.conn()
92 + .query_row("SELECT count(*) FROM samples", [], |r| r.get(0))
93 + .unwrap_or(-1)
94 + }
95 +
96 + /// Time a query, returning milliseconds. Runs it `reps` times and takes the
97 + /// median, since a single cold query mostly measures page-cache state.
98 + fn time_query(reps: usize, mut f: impl FnMut()) -> f64 {
99 + let mut times: Vec<f64> = Vec::with_capacity(reps);
100 + for _ in 0..reps {
101 + let t = Instant::now();
102 + f();
103 + times.push(t.elapsed().as_secs_f64() * 1000.0);
104 + }
105 + times.sort_by(f64::total_cmp);
106 + times[times.len() / 2]
107 + }
108 +
109 + /// Measure the query paths the browser list and filter panel depend on.
110 + fn report_query_latency(db: &Database) {
111 + let n = count_samples(db);
112 + let nodes: i64 = db
113 + .conn()
114 + .query_row("SELECT count(*) FROM vfs_nodes", [], |r| r.get(0))
115 + .unwrap_or(-1);
116 + // Printed together on purpose: a large sample count with zero nodes means
117 + // the timings below are measuring empty result sets.
118 + println!(" {n} samples / {nodes} vfs nodes (median of 5):");
119 + let rows = search::search_global(
120 + db,
121 + &SearchFilter {
122 + scope: SearchScope::Global,
123 + ..Default::default()
124 + },
125 + )
126 + .map_or(0, |r| r.len());
127 + // search.rs caps every result set at SEARCH_RESULT_LIMIT (500), so list
128 + // latency is bounded by design no matter how large the vault gets. The
129 + // scan and sort underneath it are not bounded, which is what these numbers
130 + // actually track.
131 + println!(" unfiltered search returns {rows} rows (capped at 500 by SEARCH_RESULT_LIMIT)");
132 +
133 + let analyzed: i64 = db
134 + .conn()
135 + .query_row("SELECT count(*) FROM audio_analysis", [], |r| r.get(0))
136 + .unwrap_or(0);
137 + if analyzed == 0 {
138 + println!(" NOTE: audio_analysis is empty, so the class and bpm filters below");
139 + println!(" match nothing and their timings are not meaningful. Run");
140 + println!(" the analysis pipeline over this vault to benchmark them.");
141 + }
142 +
143 + let ms = time_query(5, || {
144 + let _ = count_samples(db);
145 + });
146 + println!(" count(*) {ms:>8.2} ms");
147 +
148 + let mut filter = SearchFilter {
149 + scope: SearchScope::Global,
150 + ..Default::default()
151 + };
152 + let ms = time_query(5, || {
153 + let _ = search::search_global(db, &filter);
154 + });
155 + println!(" search_global (no filter) {ms:>6.2} ms <- worst-case list load");
156 +
157 + filter.text_query = "kick".to_string();
158 + let ms = time_query(5, || {
159 + let _ = search::search_global(db, &filter);
160 + });
161 + println!(" search_global (text) {ms:>6.2} ms <- search box keystroke");
162 +
163 + filter.text_query.clear();
164 + filter.classifications = vec!["kick".to_string()];
165 + let ms = time_query(5, || {
166 + let _ = search::search_global(db, &filter);
167 + });
168 + println!(" search_global (class) {ms:>6.2} ms <- filter panel");
169 +
170 + filter.classifications.clear();
171 + filter.bpm_min = Some(120.0);
172 + filter.bpm_max = Some(130.0);
173 + let ms = time_query(5, || {
174 + let _ = search::search_global(db, &filter);
175 + });
176 + println!(" search_global (bpm range) {ms:>6.2} ms");
177 + }
178 +
179 + /// Run the ingest benchmark against `corpus`, building a scratch vault at
180 + /// `vault`. Any existing scratch vault is removed first so runs are comparable.
181 + pub(crate) fn run(corpus: &Path, vault: &Path, batch: usize, limit: Option<usize>) {
182 + println!("━━━ INGEST AT SCALE ━━━");
183 + println!();
184 + println!(" corpus: {}", corpus.display());
185 + println!(" vault: {}", vault.display());
186 +
187 + let mut files = Vec::new();
188 + collect(corpus, &mut files);
189 + files.sort();
190 + shuffle_deterministic(&mut files);
191 + if let Some(lim) = limit {
192 + files.truncate(lim);
193 + }
194 + if files.is_empty() {
195 + eprintln!("no audio files under {}", corpus.display());
196 + return;
197 + }
198 + println!(" files: {}", files.len());
199 + println!();
200 +
201 + if vault.exists()
202 + && let Err(e) = std::fs::remove_dir_all(vault)
203 + {
204 + eprintln!("could not clear scratch vault: {e}");
205 + return;
206 + }
207 + let samples_dir = vault.join("samples");
208 + if let Err(e) = std::fs::create_dir_all(&samples_dir) {
209 + eprintln!("could not create scratch vault: {e}");
210 + return;
211 + }
212 +
213 + let db = match Database::open(vault.join("audiofiles.db")) {
214 + Ok(db) => db,
215 + Err(e) => {
216 + // Worth surfacing loudly: on a filesystem that cannot support WAL
217 + // this is exactly where a vault fails, and the app surfaces it as a
218 + // generic init error.
219 + eprintln!("Database::open failed (WAL unsupported on this fs?): {e}");
220 + return;
221 + }
222 + };
223 + let store = match SampleStore::new(&samples_dir) {
224 + Ok(s) => s,
225 + Err(e) => {
226 + eprintln!("SampleStore::new failed: {e}");
227 + return;
228 + }
229 + };
230 +
231 + // `store.import` writes the blob and the `samples` row but no VFS node.
232 + // The browser's import workflow creates those separately, and every query
233 + // the UI runs goes through `vfs_nodes`. Without them `search_global`
234 + // returns an empty set instantly and the query numbers below would be
235 + // measuring nothing.
236 + let vfs_id = match vfs::create_vfs(&db, "bench") {
237 + Ok(id) => id,
238 + Err(e) => {
239 + eprintln!("could not create bench vfs: {e}");
240 + return;
241 + }
242 + };
243 +
244 + println!(" batch cumulative files/s MB/s elapsed");
245 + println!(" ---------------------------------------------------------");
246 +
247 + let mut stats: Vec<BatchStat> = Vec::new();
248 + let mut cumulative = 0usize;
249 + let mut failures = 0usize;
250 + let mut link_failures = 0usize;
251 +
252 + for chunk in files.chunks(batch) {
253 + let mut bytes = 0u64;
254 + let start = Instant::now();
255 + for path in chunk {
256 + match store.import(path, &db) {
257 + Ok(hash) => {
258 + // Name links by index: sample names must be unique among
259 + // siblings, and the corpus has repeated basenames across
260 + // packs.
261 + let name = format!(
262 + "{cumulative:06}_{}",
263 + path.file_name().unwrap_or_default().to_string_lossy()
264 + );
265 + if vfs::create_sample_link(
266 + &db,
267 + vfs_id,
268 + None,
269 + &name,
270 + &SampleHash::from_trusted(hash),
271 + )
272 + .is_err()
273 + {
274 + link_failures += 1;
275 + }
276 + bytes += std::fs::metadata(path).map_or(0, |m| m.len());
277 + cumulative += 1;
278 + }
279 + Err(_) => failures += 1,
280 + }
281 + }
282 + let stat = BatchStat {
283 + cumulative,
284 + files: chunk.len(),
285 + bytes,
286 + elapsed_s: start.elapsed().as_secs_f64(),
287 + };
288 + println!(
289 + " {:>5} {:>10} {:>10.1} {:>10.1} {:>7.2}s",
290 + stats.len() + 1,
291 + stat.cumulative,
292 + stat.files_per_sec(),
293 + stat.mb_per_sec(),
294 + stat.elapsed_s,
295 + );
296 + stats.push(stat);
297 + }
298 +
299 + println!();
300 + if failures > 0 {
301 + println!(" {failures} file(s) failed to import");
302 + }
303 + if link_failures > 0 {
304 + println!(" {link_failures} vfs link(s) failed -- query numbers below undercount");
305 + }
306 +
307 + // Degradation is the actual question. Comparing first batch to last is the
308 + // cheapest signal that the flat blob directory or an index has started to
309 + // bite; a flat profile means it has not.
310 + if stats.len() >= 2 {
311 + let first = stats[0].files_per_sec();
312 + let last = stats[stats.len() - 1].files_per_sec();
313 + let delta = if first > 0.0 {
314 + (last - first) / first * 100.0
315 + } else {
316 + 0.0
317 + };
318 + println!(" first batch: {first:.1} files/s");
319 + println!(" last batch: {last:.1} files/s ({delta:+.1}%)");
320 + if delta < -25.0 {
321 + println!(" ^ throughput degraded as the vault grew");
322 + }
323 + }
324 +
325 + let total_files: usize = stats.iter().map(|s| s.files).sum();
326 + let total_bytes: u64 = stats.iter().map(|s| s.bytes).sum();
327 + let total_s: f64 = stats.iter().map(|s| s.elapsed_s).sum();
328 + println!();
329 + println!(
330 + " total: {total_files} files, {:.2} GB in {total_s:.1}s ({:.1} files/s, {:.1} MB/s)",
331 + total_bytes as f64 / 1e9,
332 + total_files as f64 / total_s,
333 + (total_bytes as f64 / 1e6) / total_s,
334 + );
335 +
336 + // Dedup: re-importing the same files must hit the content-addressed store
337 + // and skip the copy. If this is not dramatically faster, dedup is not
338 + // working and every duplicate costs a full hash-and-copy.
339 + println!();
340 + println!("━━━ DEDUP (re-import of identical content) ━━━");
341 + println!();
342 + let blobs_before = std::fs::read_dir(&samples_dir).map_or(0, std::iter::Iterator::count);
343 + let rows_before = count_samples(&db);
344 +
345 + let redo: Vec<&PathBuf> = files.iter().take(batch.min(files.len())).collect();
346 + let start = Instant::now();
347 + for path in &redo {
348 + let _ = store.import(path, &db);
349 + }
350 + let redo_s = start.elapsed().as_secs_f64();
351 +
352 + let blobs_after = std::fs::read_dir(&samples_dir).map_or(0, std::iter::Iterator::count);
353 + let rows_after = count_samples(&db);
354 +
355 + println!(
356 + " re-imported {} files in {redo_s:.2}s ({:.1} files/s)",
357 + redo.len(),
358 + redo.len() as f64 / redo_s.max(1e-9)
359 + );
360 + println!(" blobs on disk: {blobs_before} -> {blobs_after} (want: unchanged)");
361 + println!(" sample rows: {rows_before} -> {rows_after}");
362 +
363 + // Distinguishing these two matters: equal blob counts with equal row counts
364 + // means full dedup; equal blobs with more rows means the blob was reused but
365 + // a duplicate row was still written.
366 + if blobs_after == blobs_before {
367 + println!(" blob dedup: OK (no new blobs written)");
368 + } else {
369 + println!(
370 + " blob dedup: {} new blob(s) written",
371 + blobs_after - blobs_before
372 + );
373 + }
374 +
375 + println!();
376 + println!("━━━ QUERY LATENCY (backs the browser UI) ━━━");
377 + println!();
378 + report_query_latency(&db);
379 +
380 + // Left in place deliberately: the vault is the artifact to point the app at
381 + // for eyeballing UI responsiveness at this size.
382 + println!();
383 + println!(
384 + " scratch vault left at {} for UI inspection",
385 + vault.display()
386 + );
387 + }
A scripts/corpus.py +305
@@ -1,0 +1,305 @@
1 + #!/usr/bin/env python3
2 + """Fetch and lay out the benchmark sample corpus.
3 +
4 + audiofiles-bench scores classification against folder labels, so it needs a
5 + corpus laid out as samples/training/<class>/ and samples/test-suite/<kind>/.
6 + That tree is gitignored and has to be rebuilt per machine. This script fetches
7 + the source datasets and maps them into that layout.
8 +
9 + Only CC-BY datasets are enabled by default. The large ones are opt-in because
10 + they are tens of gigabytes and, for NSynth, 16 kHz mono, which skews per-file
11 + decode timings away from what a real 44.1/48 kHz library costs.
12 +
13 + ./scripts/corpus.py --list
14 + ./scripts/corpus.py --dest /media/max/T9/af-corpus
15 + ./scripts/corpus.py --dest /media/max/T9/af-corpus --datasets nsynth
16 +
17 + Datasets land in <dest>/_downloads (archives), <dest>/_raw (extracted), and
18 + <dest>/samples (the layout the bench reads). Point the bench at it with
19 + AF_BENCH_CORPUS=<dest>/samples.
20 + """
21 +
22 + import argparse
23 + import json
24 + import shutil
25 + import subprocess
26 + import sys
27 + import urllib.request
28 + from pathlib import Path
29 +
30 + # Dataset registry.
31 + #
32 + # `default` marks the ones that run without --datasets. The two defaults are
33 + # both CC-BY 4.0, which matters here: docs/ml_classifier.md keeps a trained
34 + # model out of the binary specifically to avoid training-data copyright
35 + # surface, so anything that could feed a shipped model needs a clean license.
36 + DATASETS = {
37 + "reverb-drums": {
38 + "default": True,
39 + "size": "570 MB",
40 + "count": "1,786 files",
41 + "license": "CC-BY 4.0",
42 + "url": "https://archive.org/download/reverb-drum-machines-complete-collection/"
43 + "Reverb%20Drum%20Machines%20_%20The%20Complete%20Collection.7z",
44 + "archive": "reverb-drums.7z",
45 + "desc": "Reverb drum machine packs. Instrument in the filename; the "
46 + "only default source of labeled kick/snare/hihat one-shots.",
47 + },
48 + "fsl10k": {
49 + "default": True,
50 + "size": "8.8 GB",
51 + "count": "9,455 loops",
52 + "license": "CC-BY 4.0",
53 + "url": "https://zenodo.org/api/records/3967852/files/FSL10K.zip/content",
54 + "archive": "FSL10K.zip",
55 + "extra": {
56 + "annotations.zip": "https://zenodo.org/api/records/3967852/files/annotations.zip/content"
57 + },
58 + "desc": "Freesound Loop Dataset. The only source here with ground-truth "
59 + "tempo and key, so it is what BPM/key accuracy can be scored against.",
60 + },
61 + "nsynth": {
62 + "default": False,
63 + "size": "~30 GB",
64 + "count": "305,979 notes",
65 + "license": "CC-BY 4.0",
66 + "url": "http://download.magenta.tensorflow.org/datasets/nsynth/nsynth-train.jsonwav.tar.gz",
67 + "archive": "nsynth-train.jsonwav.tar.gz",
68 + "desc": "NSynth. Use for count-scale (300k rows, 300k blobs in one flat "
69 + "dir). 16 kHz mono, so do not mix its timings into a throughput number.",
70 + },
71 + "fsd50k": {
72 + "default": False,
73 + "size": "~30 GB",
74 + "count": "51,197 clips",
75 + "license": "CC-BY (per-clip varies, includes CC-BY-NC)",
76 + "url": "https://zenodo.org/api/records/4060432/files/FSD50K.dev_audio.zip/content",
77 + "archive": "FSD50K.dev_audio.zip",
78 + "desc": "FSD50K, 200 AudioSet classes. Contains CC-BY-NC clips: fine for "
79 + "measurement, not for anything that ships.",
80 + },
81 + "percussive": {
82 + "default": False,
83 + "size": "119 MB",
84 + "count": "10,254 sounds",
85 + "license": "Attribution",
86 + "url": "https://zenodo.org/api/records/3665275/files/one_shot_percussive_sounds.zip/content",
87 + "archive": "one_shot_percussive_sounds.zip",
88 + "desc": "Freesound one-shot percussion. No class labels and 16 kHz "
89 + "normalized to 1 s, so it is bulk filler, not accuracy material.",
90 + },
91 + }
92 +
93 + # Filename/dirname keyword -> bench training class.
94 + #
95 + # Ordered, first match wins. Order is load-bearing: "loop" has to be tested
96 + # before any instrument keyword because loop files in these packs are named
97 + # "<machine> Loop3.wav" with no instrument in the name at all, and several
98 + # instrument keywords are substrings of each other.
99 + CLASS_RULES = [
100 + (["kick", "bassdrum", "bass drum", " bd", "_bd", "kik"], "kick"),
101 + (["snare", " sd", "_sd", "rimshot", "rim shot", " rim", "_rim",
102 + "sidestick", "side stick", "stick"], "snare"),
103 + (["hihat", "hi hat", "hi-hat", "chh", "ohh", " hh", "_hh", "hat"], "hihat"),
104 + (["cymbal", "crash", "ride", "splash", "china", "gong"], "cymbal"),
105 + (["clap", "handclap", " cp", "_cp"], "clap"),
106 + (["tom"], "tom"),
107 + (
108 + [
109 + "perc", "cowbell", "clave", "clava", "maraca", "tambor", "tambour",
110 + "bongo", "conga", "guiro", "shaker", "shake", "block", "triangle",
111 + "agogo", "cabasa", "timbale", "whistle", "bell", "scratch",
112 + "chime", "click", "beep", "quijada", "steel drum",
113 + ],
114 + "percussion",
115 + ),
116 + ]
117 +
118 + TRAINING_CLASSES = ["kick", "snare", "hihat", "cymbal", "clap", "tom", "percussion"]
119 +
120 +
121 + def classify_name(path: Path) -> tuple[str | None, str]:
122 + """Map a source file to a training class from its name and parent dirs.
123 +
124 + Returns (class, status) where status is one of "ok", "loop", "ambiguous",
125 + "unlabeled".
126 +
127 + Matches against the filename plus its two parent directories, since these
128 + packs sometimes put the instrument in a subdirectory ("..._Tom/") and
129 + sometimes only in the filename ("... Tom3.wav").
130 +
131 + Files matching more than one class are dropped rather than resolved by rule
132 + order. These packs contain genuinely ambiguous names -- "Kick_Cowbell.wav",
133 + "Tom-Cymbal.wav" -- and first-match-wins would silently assign one of the
134 + two at random. A wrong label is worse than a missing one here, because the
135 + bench reports accuracy against these folders and a mislabeled file looks
136 + like a classifier error forever.
137 + """
138 + hay = " ".join([path.name, path.parent.name, path.parent.parent.name]).lower()
139 +
140 + # Loops carry no instrument in the name at all in these packs, so they are
141 + # checked first and routed away from the training set entirely.
142 + if "loop" in hay:
143 + return None, "loop"
144 +
145 + matched = {cls for keywords, cls in CLASS_RULES if any(k in hay for k in keywords)}
146 + if len(matched) > 1:
147 + return None, "ambiguous"
148 + if len(matched) == 1:
149 + return matched.pop(), "ok"
150 + return None, "unlabeled"
151 +
152 +
153 + def run(cmd: list[str]) -> None:
154 + subprocess.run(cmd, check=True)
155 +
156 +
157 + def download(url: str, dest: Path) -> None:
158 + if dest.exists() and dest.stat().st_size > 0:
159 + print(f" have {dest.name} ({dest.stat().st_size / 1e9:.2f} GB), skipping")
160 + return
161 + print(f" downloading {dest.name} ...")
162 + tmp = dest.with_suffix(dest.suffix + ".part")
163 + urllib.request.urlretrieve(url, tmp)
164 + tmp.rename(dest)
165 + print(f" got {dest.name} ({dest.stat().st_size / 1e9:.2f} GB)")
166 +
167 +
168 + def extract(archive: Path, into: Path, marker: Path | None = None) -> None:
169 + """Extract `archive` into `into`.
170 +
171 + `marker` is the path whose existence means this archive is already
172 + unpacked. It defaults to "`into` is non-empty", which is right for the first
173 + archive into a directory but wrong for a second one (annotations landing
174 + next to audio), where the directory is already full.
175 + """
176 + done = marker.exists() if marker is not None else (into.exists() and any(into.iterdir()))
177 + if done:
178 + print(f" already extracted {archive.name}, skipping")
179 + return
180 + into.mkdir(parents=True, exist_ok=True)
181 + print(f" extracting {archive.name} ...")
182 + name = archive.name.lower()
183 + if name.endswith(".7z"):
184 + run(["7z", "x", "-y", f"-o{into}", str(archive)])
185 + elif name.endswith(".zip"):
186 + run(["unzip", "-q", "-o", str(archive), "-d", str(into)])
187 + elif name.endswith((".tar.gz", ".tgz")):
188 + run(["tar", "xzf", str(archive), "-C", str(into)])
189 + else:
190 + sys.exit(f"unknown archive type: {archive.name}")
191 +
192 +
193 + def build_training(raw: Path, samples: Path) -> dict[str, int]:
194 + """Copy labeled one-shots into samples/training/<class>/.
195 +
196 + Copies rather than symlinks: the corpus is expected to live on the exFAT
197 + test drive, which has no symlink support at all.
198 + """
199 + counts: dict[str, int] = {c: 0 for c in TRAINING_CLASSES}
200 + counts["_loops"] = 0
201 + counts["_ambiguous"] = 0
202 + counts["_unlabeled"] = 0
203 +
204 + training = samples / "training"
205 + loops = samples / "test-suite" / "genres" / "loops"
206 + for c in TRAINING_CLASSES:
207 + (training / c).mkdir(parents=True, exist_ok=True)
208 + loops.mkdir(parents=True, exist_ok=True)
209 +
210 + dropped: list[str] = []
211 +
212 + for src in sorted(raw.rglob("*")):
213 + if not src.is_file() or src.suffix.lower() not in (".wav", ".aif", ".aiff", ".flac"):
214 + continue
215 + cls, status = classify_name(src)
216 + if status == "loop":
217 + dest_dir, key = loops, "_loops"
218 + elif status == "ok":
219 + dest_dir, key = training / cls, cls
220 + else:
221 + counts[f"_{status}"] += 1
222 + dropped.append(f"{status}: {src.name}")
223 + continue
224 +
225 + # Flatten with a pack-qualified name so same-named files across packs
226 + # do not collide (every pack has a "Kick.wav").
227 + flat = f"{src.parent.parent.name}__{src.name}".replace("/", "_")
228 + dest = dest_dir / flat
229 + if not dest.exists():
230 + shutil.copy2(src, dest)
231 + counts[key] += 1
232 +
233 + # Written out rather than just counted: when accuracy looks off, the first
234 + # question is always whether the corpus or the classifier is wrong.
235 + (samples / "DROPPED.txt").write_text("\n".join(sorted(dropped)) + "\n")
236 +
237 + return counts
238 +
239 +
240 + def main() -> None:
241 + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
242 + ap.add_argument("--dest", type=Path, default=Path("/media/max/T9/af-corpus"))
243 + ap.add_argument("--datasets", help="comma-separated; default = the CC-BY defaults")
244 + ap.add_argument("--list", action="store_true", help="show the registry and exit")
245 + ap.add_argument("--no-build", action="store_true", help="fetch and extract only")
246 + args = ap.parse_args()
247 +
248 + if args.list:
249 + print(f"{'dataset':<14} {'size':>8} {'default':<8} {'license':<38} count")
250 + for name, d in DATASETS.items():
251 + print(
252 + f"{name:<14} {d['size']:>8} {str(d['default']):<8} "
253 + f"{d['license']:<38} {d['count']}"
254 + )
255 + print(f"{'':>14} {d['desc']}")
256 + return
257 +
258 + if args.datasets:
259 + wanted = [d.strip() for d in args.datasets.split(",")]
260 + unknown = [d for d in wanted if d not in DATASETS]
261 + if unknown:
262 + sys.exit(f"unknown dataset(s): {', '.join(unknown)}")
263 + else:
264 + wanted = [n for n, d in DATASETS.items() if d["default"]]
265 +
266 + downloads = args.dest / "_downloads"
267 + raw = args.dest / "_raw"
268 + samples = args.dest / "samples"
269 + downloads.mkdir(parents=True, exist_ok=True)
270 +
271 + manifest = {"datasets": [], "layout": str(samples)}
272 +
273 + for name in wanted:
274 + d = DATASETS[name]
275 + print(f"\n=== {name} ({d['size']}, {d['license']}) ===")
276 + archive = downloads / d["archive"]
277 + download(d["url"], archive)
278 + extract(archive, raw / name)
279 + # Extras unpack alongside the main archive, not into their own tree:
280 + # the accuracy bench expects annotations/ next to audio/ under one root.
281 + for extra_name, extra_url in d.get("extra", {}).items():
282 + extra_path = downloads / extra_name
283 + download(extra_url, extra_path)
284 + if extra_path.suffix.lower() in (".zip", ".7z", ".gz"):
285 + extract(extra_path, raw / name, marker=raw / name / Path(extra_name).stem)
286 + manifest["datasets"].append(
287 + {"name": name, "license": d["license"], "source": d["url"]}
288 + )
289 +
290 + if not args.no_build and "reverb-drums" in wanted:
291 + print("\n=== building training layout ===")
292 + counts = build_training(raw / "reverb-drums", samples)
293 + for k, v in counts.items():
294 + print(f" {k:<14} {v}")
295 + manifest["training_counts"] = counts
296 +
297 + # Provenance matters here: the corpus mixes licenses, and anything that
298 + # feeds a model needs that recorded rather than reconstructed later.
299 + (args.dest / "MANIFEST.json").write_text(json.dumps(manifest, indent=2))
300 + print(f"\nwrote {args.dest / 'MANIFEST.json'}")
301 + print(f"point the bench at it: export AF_BENCH_CORPUS={samples}")
302 +
303 +
304 + if __name__ == "__main__":
305 + main()