Skip to main content

max / audiofiles

Build a per-format decode corpus and make section 2 comparable Section 2 of the analysis bench has never produced a number. It reads test-suite/formats/{wav,aiff,mp3,flac}, which corpus.py never built, so it printed four empty rows. It could not be fed from the datasets either: every source corpus is wav-only, across reverb-drums, FSL10K and NSynth alike. corpus.py now transcodes a strided subset of the loops into all four formats via ffmpeg. Loops rather than one-shots because one-shots run about 0.2s, short enough that open-and-probe overhead hides the codec difference. Every arm is re-encoded, wav included, and normalized to 44.1 kHz stereo, so the only variable between rows is the container and codec rather than the sample rate and bit depth mix the source packs carry. A source that fails to encode is dropped from all four arms, since a file present in three would change which audio each row averages over. collect_audio_files now sorts before truncating to the limit. It walked in read_dir order, which is arbitrary, so taking 100 of the 120 files picked a different subset per format and the means compared different loops. That is the same class of error as the batch composition bias. Section 2 also reports per-format file counts, warns when the arms disagree or when a decode fails, and prints cost relative to wav.
Author: Max Johnson <me@maxj.phd> · 2026-07-29 17:02 UTC
Signed with PGP, not checked
Commit: 2d5234a63a8ce2be9695136e8c4fdef3828a8c5c
Parent: 2a9a2a1
2 files changed, +198 insertions, -25 deletions
M scripts/corpus.py +103
@@ -17,6 +17,10 @@
17 17 Datasets land in <dest>/_downloads (archives), <dest>/_raw (extracted), and
18 18 <dest>/samples (the layout the bench reads). Point the bench at it with
19 19 AF_BENCH_CORPUS=<dest>/samples.
20 +
21 + Every source dataset is wav-only, so the per-format decode section of the bench
22 + gets its non-wav files from a transcode step (needs ffmpeg on PATH) rather than
23 + from a download. See build_formats.
20 24 """
21 25
22 26 import argparse
@@ -237,12 +241,96 @@
237 241 return counts
238 242
239 243
244 + # Encoder settings per format, as ffmpeg output arguments.
245 + #
246 + # Every arm is re-encoded, including wav. Copying the source wav instead would
247 + # leave that one row measuring a different signal shape from the other three,
248 + # since the source packs mix sample rates, bit depths, and channel counts. All
249 + # four are normalized to the same 44.1 kHz stereo PCM so the only thing that
250 + # differs between rows is the container and codec.
251 + FORMAT_ENCODERS = {
252 + "wav": ["-c:a", "pcm_s16le"],
253 + "aiff": ["-c:a", "pcm_s16be"],
254 + "flac": ["-c:a", "flac"],
255 + # Lossy by nature, so its decoded output is not bit-identical to the other
256 + # three. That is inherent to comparing codecs and does not affect the
257 + # timing, which is what this corpus exists to measure.
258 + "mp3": ["-c:a", "libmp3lame", "-b:a", "320k"],
259 + }
260 +
261 +
262 + def build_formats(samples: Path, count: int) -> dict[str, int]:
263 + """Transcode a fixed subset of loops into samples/test-suite/formats/<ext>/.
264 +
265 + Section 2 of the bench times decode per format, which needs the same audio
266 + in every format. Nothing in the source datasets provides that: every corpus
267 + file, across all three datasets, is wav. Pointing the section at unrelated
268 + per-format files instead would measure content rather than codec.
269 +
270 + Sourced from the loops rather than the one-shots because the one-shots run
271 + about 0.2s, short enough that open-and-probe overhead dominates and the
272 + codec difference disappears into it. Loops run about 2.5s.
273 + """
274 + src_dir = samples / "test-suite" / "genres" / "loops"
275 + sources = sorted(p for p in src_dir.glob("*.wav") if p.is_file())
276 + if not sources:
277 + print(f" no loops under {src_dir}, skipping")
278 + return {}
279 +
280 + # Stride rather than head: the flattened names sort by pack, so the first N
281 + # would all come from one drum machine and share its recording character.
282 + if len(sources) > count:
283 + stride = len(sources) / count
284 + sources = [sources[int(i * stride)] for i in range(count)]
285 +
286 + out_root = samples / "test-suite" / "formats"
287 + for ext in FORMAT_ENCODERS:
288 + (out_root / ext).mkdir(parents=True, exist_ok=True)
289 +
290 + counts = {ext: 0 for ext in FORMAT_ENCODERS}
291 + failed = 0
292 + for src in sources:
293 + outputs = {ext: out_root / ext / f"{src.stem}.{ext}" for ext in FORMAT_ENCODERS}
294 + if all(p.exists() for p in outputs.values()):
295 + for ext in FORMAT_ENCODERS:
296 + counts[ext] += 1
297 + continue
298 +
299 + written: list[Path] = []
300 + for ext, enc in FORMAT_ENCODERS.items():
301 + dest = outputs[ext]
302 + cmd = ["ffmpeg", "-v", "error", "-y", "-i", str(src),
303 + "-ac", "2", "-ar", "44100", *enc, str(dest)]
304 + try:
305 + subprocess.run(cmd, check=True)
306 + except subprocess.CalledProcessError:
307 + break
308 + written.append(dest)
309 + else:
310 + for ext in FORMAT_ENCODERS:
311 + counts[ext] += 1
312 + continue
313 +
314 + # All-or-nothing. A file present in three formats but not the fourth
315 + # would silently change which audio each row averages over, so the
316 + # per-format means would no longer be comparable.
317 + for p in written:
318 + p.unlink(missing_ok=True)
319 + failed += 1
320 +
321 + if failed:
322 + print(f" dropped {failed} source file(s) that would not encode to all formats")
323 + return counts
324 +
325 +
240 326 def main() -> None:
241 327 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
242 328 ap.add_argument("--dest", type=Path, default=Path("/media/max/T9/af-corpus"))
243 329 ap.add_argument("--datasets", help="comma-separated; default = the CC-BY defaults")
244 330 ap.add_argument("--list", action="store_true", help="show the registry and exit")
245 331 ap.add_argument("--no-build", action="store_true", help="fetch and extract only")
332 + ap.add_argument("--formats-count", type=int, default=120,
333 + help="loops to transcode into each format for the decode bench")
246 334 args = ap.parse_args()
247 335
248 336 if args.list:
@@ -294,6 +382,21 @@
294 382 print(f" {k:<14} {v}")
295 383 manifest["training_counts"] = counts
296 384
385 + # Runs off the built layout, not off a dataset, so it is gated on the loops
386 + # existing rather than on which datasets were requested.
387 + if not args.no_build and (samples / "test-suite" / "genres" / "loops").exists():
388 + print("\n=== building format corpus ===")
389 + if shutil.which("ffmpeg") is None:
390 + print(" ffmpeg not found, skipping (section 2 of the bench will report no files)")
391 + else:
392 + fmt_counts = build_formats(samples, args.formats_count)
393 + for k, v in fmt_counts.items():
394 + print(f" {k:<14} {v}")
395 + # Recorded as derived so the license question does not come up
396 + # again: these are transcodes of corpus files, not a new dataset.
397 + manifest["format_counts"] = fmt_counts
398 + manifest["format_corpus"] = "transcoded from test-suite/genres/loops"
399 +
297 400 # Provenance matters here: the corpus mixes licenses, and anything that
298 401 # feeds a model needs that recorded rather than reconstructed later.
299 402 (args.dest / "MANIFEST.json").write_text(json.dumps(manifest, indent=2))
M crates/audiofiles-bench/src/main.rs +95 -25
@@ -181,12 +181,22 @@
181 181 audio_extensions().iter().any(|ext| lower.ends_with(ext))
182 182 }
183 183
184 + /// Collect audio files under `dir`, sorted, then truncated to `limit`.
185 + ///
186 + /// The sort is load-bearing, not tidiness. `walkdir` yields `read_dir` order,
187 + /// which is arbitrary (and on the exFAT test drive, not even stable across
188 + /// machines). Truncating an unsorted list to a limit picks an arbitrary subset,
189 + /// so the per-format decode section would average each format over a different
190 + /// set of loops and report codec differences that are really length
191 + /// differences. Sorting first makes every caller's selection reproducible.
184 192 fn collect_audio_files(dir: &Path, limit: Option<usize>) -> Vec<PathBuf> {
185 193 let mut files: Vec<PathBuf> = Vec::new();
186 194 if !dir.exists() {
187 195 return files;
188 196 }
189 - for entry in walkdir(dir) {
197 + let mut entries = walkdir(dir);
198 + entries.sort();
199 + for entry in entries {
190 200 if let Some(lim) = limit
191 201 && files.len() >= lim
192 202 {
@@ -426,6 +436,10 @@
426 436 println!("━━━ 2. FORMAT-SPECIFIC DECODE PERFORMANCE ━━━");
427 437 println!();
428 438
439 + // These directories hold the same loops transcoded four ways, built by
440 + // scripts/corpus.py. They are not four separate datasets: every source
441 + // dataset is wav-only, and timing unrelated files per format would measure
442 + // content rather than codec.
429 443 let format_dirs: Vec<(&str, PathBuf)> = vec![
430 444 ("WAV", test_suite_dir.join("formats/wav")),
431 445 ("AIFF", test_suite_dir.join("formats/aiff")),
@@ -433,33 +447,89 @@
433 447 ("FLAC", test_suite_dir.join("formats/flac")),
434 448 ];
435 449
436 - println!(
437 - " {:<8} {:>6} {:>10} {:>10} {:>10}",
438 - "Format", "Files", "Mean(ms)", "P95(ms)", "Max(ms)"
439 - );
440 - println!(" {}", "─".repeat(50));
450 + let format_files: Vec<(&str, Vec<PathBuf>)> = format_dirs
451 + .iter()
452 + .map(|(fmt, dir)| (*fmt, collect_audio_files(dir, Some(100))))
453 + .collect();
441 454
442 - for (fmt, dir) in &format_dirs {
443 - let files = collect_audio_files(dir, Some(100));
444 - if files.is_empty() {
445 - println!(" {:<8} {:>6} {:>10} {:>10} {:>10}", fmt, 0, "-", "-", "-");
446 - continue;
455 + if format_files.iter().all(|(_, f)| f.is_empty()) {
456 + println!(
457 + " no format corpus found under {}",
458 + test_suite_dir.join("formats").display()
459 + );
460 + println!(" build it with: ./scripts/corpus.py --dest <corpus root>");
461 + println!();
462 + } else {
463 + // A ratio across arms of different sizes compares different audio. This
464 + // is the failure this section is most likely to have and least likely
465 + // to look like one, so it is stated rather than assumed.
466 + let counts: Vec<usize> = format_files.iter().map(|(_, f)| f.len()).collect();
467 + if counts.iter().any(|c| *c != counts[0]) {
468 + println!(" WARNING: format arms hold different file counts {counts:?}.");
469 + println!(" Per-format means are not comparable; rebuild the format corpus.");
470 + println!();
447 471 }
448 - let mut decode_times: Vec<f64> = files
449 - .par_iter()
450 - .filter_map(|f| {
451 - let t = Instant::now();
452 - decode::decode_to_mono(f).ok()?;
453 - Some(t.elapsed().as_secs_f64() * 1000.0)
454 - })
455 - .collect();
456 - let count = decode_times.len();
457 - let mean = decode_times.iter().sum::<f64>() / count as f64;
458 - let p95 = percentile(&mut decode_times, 95.0);
459 - let max = percentile(&mut decode_times, 100.0);
460 - println!(" {fmt:<8} {count:>6} {mean:>10.2} {p95:>10.2} {max:>10.2}");
472 +
473 + println!(
474 + " {:<8} {:>6} {:>10} {:>10} {:>10} {:>9}",
475 + "Format", "Files", "Mean(ms)", "P95(ms)", "Max(ms)", "vs WAV"
476 + );
477 + println!(" {}", "─".repeat(60));
478 +
479 + let mut wav_mean: Option<f64> = None;
480 + for (fmt, files) in &format_files {
481 + if files.is_empty() {
482 + println!(
483 + " {:<8} {:>6} {:>10} {:>10} {:>10} {:>9}",
484 + fmt, 0, "-", "-", "-", "-"
485 + );
486 + continue;
487 + }
488 + let mut decode_times: Vec<f64> = files
489 + .par_iter()
490 + .filter_map(|f| {
491 + let t = Instant::now();
492 + decode::decode_to_mono(f).ok()?;
493 + Some(t.elapsed().as_secs_f64() * 1000.0)
494 + })
495 + .collect();
496 +
497 + // Decode failures shrink this arm silently, which is the same
498 + // comparability problem as a short directory.
499 + if decode_times.len() != files.len() {
500 + println!(
501 + " WARNING: {fmt} decoded {} of {} files",
502 + decode_times.len(),
503 + files.len()
504 + );
505 + }
506 + if decode_times.is_empty() {
507 + println!(
508 + " {:<8} {:>6} {:>10} {:>10} {:>10} {:>9}",
509 + fmt, 0, "-", "-", "-", "-"
510 + );
511 + continue;
512 + }
513 +
514 + let count = decode_times.len();
515 + let mean = decode_times.iter().sum::<f64>() / count as f64;
516 + let p95 = percentile(&mut decode_times, 95.0);
517 + let max = percentile(&mut decode_times, 100.0);
518 + // WAV is first in the list, so its mean is set before any other
519 + // row needs it.
520 + if *fmt == "WAV" {
521 + wav_mean = Some(mean);
522 + }
523 + let rel = match wav_mean {
524 + Some(w) if w > 0.0 => format!("{:.2}x", mean / w),
525 + _ => "-".to_string(),
526 + };
527 + println!(" {fmt:<8} {count:>6} {mean:>10.2} {p95:>10.2} {max:>10.2} {rel:>9}");
528 + }
529 + println!();
530 + println!(" Same audio in every row (44.1 kHz stereo), so the spread is codec cost.");
531 + println!();
461 532 }
462 - println!();
463 533
464 534 // Section 3: Throughput
465 535 println!("━━━ 3. THROUGHPUT ━━━");