Skip to main content

max / audiofiles

23.8 KB · 562 lines History Blame Raw
1 #!/usr/bin/env python3
2 """Fetch and lay out the benchmark sample corpus.
3
4 audiofiles-bench times the analysis pipeline over a corpus laid out as
5 samples/training/<class>/ and samples/test-suite/<kind>/. The per-class folders
6 are also the filename-derived ground truth the browse-axes measurements used.
7 That tree is gitignored and has to be rebuilt per machine. This script fetches
8 the source datasets and maps them into that layout.
9
10 The large datasets are opt-in because they are tens of gigabytes and, for
11 NSynth, 16 kHz mono, which skews per-file decode timings away from what a real
12 44.1/48 kHz library costs.
13
14 Nothing here is licence-clean as a whole. Two of the five mix per-sound
15 licences, including CC-BY-NC, so the registry's `license` field describes the
16 mix rather than claiming one label. See summarize_fsl10k_licenses.
17
18 ./scripts/corpus.py --list
19 ./scripts/corpus.py --dest /media/max/T9/af-corpus
20 ./scripts/corpus.py --dest /media/max/T9/af-corpus --datasets nsynth
21
22 Datasets land in <dest>/_downloads (archives), <dest>/_raw (extracted), and
23 <dest>/samples (the layout the bench reads). Point the bench at it with
24 AF_BENCH_CORPUS=<dest>/samples.
25
26 Every source dataset is wav-only, so the per-format decode section of the bench
27 gets its non-wav files from a transcode step (needs ffmpeg on PATH) rather than
28 from a download. See build_formats.
29 """
30
31 import argparse
32 import json
33 import shutil
34 import subprocess
35 import sys
36 import urllib.request
37 from pathlib import Path
38
39 # Dataset registry.
40 #
41 # `default` marks the ones that run without --datasets. It is a size-and-time
42 # judgement and says nothing about licence: fsl10k is a default and a sixth of
43 # it is CC-BY-NC. The two questions were conflated here until 2026-08-08, when
44 # the entry still claimed a flat "CC-BY 4.0" for a corpus with four licences in
45 # it.
46 #
47 # Nothing derived from any of this ships today. The bundled `.afcl` layer that
48 # would have was retired on 2026-08-08 (docs/ml_classifier.md, "What ships in
49 # the binary, and under what licence"), so every dataset here is measurement
50 # material only, which the NC clips are fine for. The licence is recorded
51 # accurately anyway: that position has already flipped once, and reconstructing
52 # provenance after the fact is what this file exists to avoid.
53 DATASETS = {
54 "reverb-drums": {
55 "default": True,
56 "size": "570 MB",
57 "count": "1,786 files",
58 "license": "CC-BY 4.0",
59 "url": "https://archive.org/download/reverb-drum-machines-complete-collection/"
60 "Reverb%20Drum%20Machines%20_%20The%20Complete%20Collection.7z",
61 "archive": "reverb-drums.7z",
62 "desc": "Reverb drum machine packs. Instrument in the filename; the "
63 "only default source of labeled kick/snare/hihat one-shots.",
64 },
65 "fsl10k": {
66 "default": True,
67 "size": "8.8 GB",
68 "count": "9,455 loops",
69 "license": "per sound: CC0, CC-BY, CC-BY-NC, Sampling+",
70 "url": "https://zenodo.org/api/records/3967852/files/FSL10K.zip/content",
71 "archive": "FSL10K.zip",
72 "extra": {
73 "annotations.zip": "https://zenodo.org/api/records/3967852/files/annotations.zip/content"
74 },
75 "desc": "Freesound Loop Dataset. The only source here with ground-truth "
76 "tempo and key, so it is what BPM/key accuracy can be scored against. "
77 "Zenodo publishes it as CC-BY; the sounds are not. 1,436 of 9,493 are "
78 "NC or Sampling+: fine for measurement, not for anything that ships. "
79 "Select on metadata.json, never on this entry.",
80 },
81 "nsynth": {
82 "default": False,
83 "size": "~30 GB",
84 "count": "305,979 notes",
85 "license": "CC-BY 4.0",
86 "url": "http://download.magenta.tensorflow.org/datasets/nsynth/nsynth-train.jsonwav.tar.gz",
87 "archive": "nsynth-train.jsonwav.tar.gz",
88 "desc": "NSynth. Use for count-scale (300k rows, 300k blobs in one flat "
89 "dir). 16 kHz mono, so do not mix its timings into a throughput number.",
90 },
91 "fsd50k": {
92 "default": False,
93 "size": "~30 GB",
94 "count": "51,197 clips",
95 "license": "CC-BY (per-clip varies, includes CC-BY-NC)",
96 "url": "https://zenodo.org/api/records/4060432/files/FSD50K.dev_audio.zip/content",
97 "archive": "FSD50K.dev_audio.zip",
98 "desc": "FSD50K, 200 AudioSet classes. Contains CC-BY-NC clips: fine for "
99 "measurement, not for anything that ships.",
100 },
101 "percussive": {
102 "default": False,
103 "size": "119 MB",
104 "count": "10,254 sounds",
105 "license": "Attribution",
106 "url": "https://zenodo.org/api/records/3665275/files/one_shot_percussive_sounds.zip/content",
107 "archive": "one_shot_percussive_sounds.zip",
108 "desc": "Freesound one-shot percussion. No class labels and 16 kHz "
109 "normalized to 1 s, so it is bulk filler, not accuracy material.",
110 },
111 }
112
113 # Freesound licence URLs as they appear in FSL10K's metadata.json, mapped to a
114 # short name and whether a derived work under that licence could ship.
115 #
116 # Sampling+ is counted as cannot-ship rather than argued about: it permits
117 # sampling into a new work but bars advertising use and verbatim
118 # redistribution, which is ambiguous for a corpus-derived artifact and worth
119 # 221 sounds out of 9,493.
120 FSL10K_LICENSES = {
121 "http://creativecommons.org/publicdomain/zero/1.0/": ("CC0 1.0", True),
122 "http://creativecommons.org/licenses/by/3.0/": ("CC-BY 3.0", True),
123 "http://creativecommons.org/licenses/by-nc/3.0/": ("CC-BY-NC 3.0", False),
124 "http://creativecommons.org/licenses/sampling+/1.0/": ("Sampling+ 1.0", False),
125 }
126
127
128 def summarize_fsl10k_licenses(raw: Path) -> dict | None:
129 """Count FSL10K's per-sound licences off its metadata.json.
130
131 The dataset-level label cannot answer the licence question for this corpus
132 and the registry no longer pretends it can, so the real mix is measured on
133 every run and recorded in the manifest. Anything that later selects sounds
134 out of FSL10K reads this, or reads metadata.json itself; the point is that
135 it does not read a single label off the registry entry.
136
137 An unrecognised licence URL counts as cannot-ship and is named in the
138 result. Defaulting the other way would let a new Freesound licence into a
139 shippable set silently, which is the exact failure this function exists to
140 close.
141
142 Returns None when the metadata is not there -- --no-build runs and partial
143 extractions are normal, and a missing count is better than a wrong one.
144 """
145 meta = raw / "metadata.json"
146 if not meta.is_file():
147 return None
148 try:
149 sounds = json.loads(meta.read_text())
150 except (OSError, json.JSONDecodeError) as e:
151 print(f" warning: cannot read {meta} ({e}); licence mix not recorded")
152 return None
153 if not isinstance(sounds, dict):
154 return None
155
156 counts: dict[str, int] = {}
157 unrecognised: dict[str, int] = {}
158 shippable = 0
159 for sound in sounds.values():
160 url = sound.get("license") if isinstance(sound, dict) else None
161 name, ok = FSL10K_LICENSES.get(url, (None, False))
162 if name is None:
163 name = str(url)
164 unrecognised[name] = unrecognised.get(name, 0) + 1
165 counts[name] = counts.get(name, 0) + 1
166 shippable += ok
167
168 summary = {
169 "total": len(sounds),
170 "by_license": dict(sorted(counts.items())),
171 "shippable": shippable,
172 "excluded": len(sounds) - shippable,
173 "source": "metadata.json, per sound",
174 }
175 if unrecognised:
176 summary["unrecognised"] = dict(sorted(unrecognised.items()))
177 return summary
178
179
180 # Filename/dirname keyword -> bench training class.
181 #
182 # Ordered, first match wins. Order is load-bearing: "loop" has to be tested
183 # before any instrument keyword because loop files in these packs are named
184 # "<machine> Loop3.wav" with no instrument in the name at all, and several
185 # instrument keywords are substrings of each other.
186 CLASS_RULES = [
187 (["kick", "bassdrum", "bass drum", " bd", "_bd", "kik"], "kick"),
188 (["snare", " sd", "_sd", "rimshot", "rim shot", " rim", "_rim",
189 "sidestick", "side stick", "stick"], "snare"),
190 (["hihat", "hi hat", "hi-hat", "chh", "ohh", " hh", "_hh", "hat"], "hihat"),
191 (["cymbal", "crash", "ride", "splash", "china", "gong"], "cymbal"),
192 (["clap", "handclap", " cp", "_cp"], "clap"),
193 (["tom"], "tom"),
194 (
195 [
196 "perc", "cowbell", "clave", "clava", "maraca", "tambor", "tambour",
197 "bongo", "conga", "guiro", "shaker", "shake", "block", "triangle",
198 "agogo", "cabasa", "timbale", "whistle", "bell", "scratch",
199 "chime", "click", "beep", "quijada", "steel drum",
200 ],
201 "percussion",
202 ),
203 ]
204
205 TRAINING_CLASSES = ["kick", "snare", "hihat", "cymbal", "clap", "tom", "percussion"]
206
207 # The one dataset that fills samples/training/, and so the one the labels are
208 # derived from. Recorded in the manifest as `training_source` because the
209 # `datasets` list answers a different question -- what was fetched -- and the
210 # two diverge as soon as anything else is downloaded into the same corpus.
211 TRAINING_DATASET = "reverb-drums"
212
213
214 def classify_name(path: Path) -> tuple[str | None, str]:
215 """Map a source file to a training class from its name and parent dirs.
216
217 Returns (class, status) where status is one of "ok", "loop", "ambiguous",
218 "unlabeled".
219
220 Matches against the filename plus its two parent directories, since these
221 packs sometimes put the instrument in a subdirectory ("..._Tom/") and
222 sometimes only in the filename ("... Tom3.wav").
223
224 Files matching more than one class are dropped rather than resolved by rule
225 order. These packs contain genuinely ambiguous names -- "Kick_Cowbell.wav",
226 "Tom-Cymbal.wav" -- and first-match-wins would silently assign one of the
227 two at random. A wrong label is worse than a missing one here: these folders
228 are the ground truth the browse-axes measurements are read against, so a
229 mislabeled file looks like a real result forever.
230 """
231 hay = " ".join([path.name, path.parent.name, path.parent.parent.name]).lower()
232
233 # Loops carry no instrument in the name at all in these packs, so they are
234 # checked first and routed away from the training set entirely.
235 if "loop" in hay:
236 return None, "loop"
237
238 matched = {cls for keywords, cls in CLASS_RULES if any(k in hay for k in keywords)}
239 if len(matched) > 1:
240 return None, "ambiguous"
241 if len(matched) == 1:
242 return matched.pop(), "ok"
243 return None, "unlabeled"
244
245
246 def run(cmd: list[str]) -> None:
247 subprocess.run(cmd, check=True)
248
249
250 def download(url: str, dest: Path) -> None:
251 if dest.exists() and dest.stat().st_size > 0:
252 print(f" have {dest.name} ({dest.stat().st_size / 1e9:.2f} GB), skipping")
253 return
254 print(f" downloading {dest.name} ...")
255 tmp = dest.with_suffix(dest.suffix + ".part")
256 urllib.request.urlretrieve(url, tmp)
257 tmp.rename(dest)
258 print(f" got {dest.name} ({dest.stat().st_size / 1e9:.2f} GB)")
259
260
261 def extract(archive: Path, into: Path, marker: Path | None = None) -> None:
262 """Extract `archive` into `into`.
263
264 `marker` is the path whose existence means this archive is already
265 unpacked. It defaults to "`into` is non-empty", which is right for the first
266 archive into a directory but wrong for a second one (annotations landing
267 next to audio), where the directory is already full.
268 """
269 done = marker.exists() if marker is not None else (into.exists() and any(into.iterdir()))
270 if done:
271 print(f" already extracted {archive.name}, skipping")
272 return
273 into.mkdir(parents=True, exist_ok=True)
274 print(f" extracting {archive.name} ...")
275 name = archive.name.lower()
276 if name.endswith(".7z"):
277 run(["7z", "x", "-y", f"-o{into}", str(archive)])
278 elif name.endswith(".zip"):
279 run(["unzip", "-q", "-o", str(archive), "-d", str(into)])
280 elif name.endswith((".tar.gz", ".tgz")):
281 run(["tar", "xzf", str(archive), "-C", str(into)])
282 else:
283 sys.exit(f"unknown archive type: {archive.name}")
284
285
286 def build_training(raw: Path, samples: Path) -> dict[str, int]:
287 """Copy labeled one-shots into samples/training/<class>/.
288
289 Copies rather than symlinks: the corpus is expected to live on the exFAT
290 test drive, which has no symlink support at all.
291 """
292 counts: dict[str, int] = {c: 0 for c in TRAINING_CLASSES}
293 counts["_loops"] = 0
294 counts["_ambiguous"] = 0
295 counts["_unlabeled"] = 0
296
297 training = samples / "training"
298 loops = samples / "test-suite" / "genres" / "loops"
299 for c in TRAINING_CLASSES:
300 (training / c).mkdir(parents=True, exist_ok=True)
301 loops.mkdir(parents=True, exist_ok=True)
302
303 dropped: list[str] = []
304
305 for src in sorted(raw.rglob("*")):
306 if not src.is_file() or src.suffix.lower() not in (".wav", ".aif", ".aiff", ".flac"):
307 continue
308 cls, status = classify_name(src)
309 if status == "loop":
310 dest_dir, key = loops, "_loops"
311 elif status == "ok":
312 dest_dir, key = training / cls, cls
313 else:
314 counts[f"_{status}"] += 1
315 dropped.append(f"{status}: {src.name}")
316 continue
317
318 # Flatten with a pack-qualified name so same-named files across packs
319 # do not collide (every pack has a "Kick.wav").
320 flat = f"{src.parent.parent.name}__{src.name}".replace("/", "_")
321 dest = dest_dir / flat
322 if not dest.exists():
323 shutil.copy2(src, dest)
324 counts[key] += 1
325
326 # Written out rather than just counted: when a measurement looks off, the
327 # first question is always whether the corpus or the code is wrong.
328 (samples / "DROPPED.txt").write_text("\n".join(sorted(dropped)) + "\n")
329
330 return counts
331
332
333 # Encoder settings per format, as ffmpeg output arguments.
334 #
335 # Every arm is re-encoded, including wav. Copying the source wav instead would
336 # leave that one row measuring a different signal shape from the other three,
337 # since the source packs mix sample rates, bit depths, and channel counts. All
338 # four are normalized to the same 44.1 kHz stereo PCM so the only thing that
339 # differs between rows is the container and codec.
340 FORMAT_ENCODERS = {
341 "wav": ["-c:a", "pcm_s16le"],
342 "aiff": ["-c:a", "pcm_s16be"],
343 "flac": ["-c:a", "flac"],
344 # Lossy by nature, so its decoded output is not bit-identical to the other
345 # three. That is inherent to comparing codecs and does not affect the
346 # timing, which is what this corpus exists to measure.
347 "mp3": ["-c:a", "libmp3lame", "-b:a", "320k"],
348 }
349
350
351 def build_formats(samples: Path, count: int) -> dict[str, int]:
352 """Transcode a fixed subset of loops into samples/test-suite/formats/<ext>/.
353
354 Section 2 of the bench times decode per format, which needs the same audio
355 in every format. Nothing in the source datasets provides that: every corpus
356 file, across all three datasets, is wav. Pointing the section at unrelated
357 per-format files instead would measure content rather than codec.
358
359 Sourced from the loops rather than the one-shots because the one-shots run
360 about 0.2s, short enough that open-and-probe overhead dominates and the
361 codec difference disappears into it. Loops run about 2.5s.
362 """
363 src_dir = samples / "test-suite" / "genres" / "loops"
364 sources = sorted(p for p in src_dir.glob("*.wav") if p.is_file())
365 if not sources:
366 print(f" no loops under {src_dir}, skipping")
367 return {}
368
369 # Stride rather than head: the flattened names sort by pack, so the first N
370 # would all come from one drum machine and share its recording character.
371 if len(sources) > count:
372 stride = len(sources) / count
373 sources = [sources[int(i * stride)] for i in range(count)]
374
375 out_root = samples / "test-suite" / "formats"
376 for ext in FORMAT_ENCODERS:
377 (out_root / ext).mkdir(parents=True, exist_ok=True)
378
379 counts = {ext: 0 for ext in FORMAT_ENCODERS}
380 failed = 0
381 for src in sources:
382 outputs = {ext: out_root / ext / f"{src.stem}.{ext}" for ext in FORMAT_ENCODERS}
383 if all(p.exists() for p in outputs.values()):
384 for ext in FORMAT_ENCODERS:
385 counts[ext] += 1
386 continue
387
388 written: list[Path] = []
389 for ext, enc in FORMAT_ENCODERS.items():
390 dest = outputs[ext]
391 cmd = ["ffmpeg", "-v", "error", "-y", "-i", str(src),
392 "-ac", "2", "-ar", "44100", *enc, str(dest)]
393 try:
394 subprocess.run(cmd, check=True)
395 except subprocess.CalledProcessError:
396 break
397 written.append(dest)
398 else:
399 for ext in FORMAT_ENCODERS:
400 counts[ext] += 1
401 continue
402
403 # All-or-nothing. A file present in three formats but not the fourth
404 # would silently change which audio each row averages over, so the
405 # per-format means would no longer be comparable.
406 for p in written:
407 p.unlink(missing_ok=True)
408 failed += 1
409
410 if failed:
411 print(f" dropped {failed} source file(s) that would not encode to all formats")
412 return counts
413
414
415 def load_manifest(path: Path) -> dict:
416 """Read the existing manifest, or start an empty one.
417
418 A run fetches the datasets it was asked for and rebuilds only what those
419 datasets feed, so it knows about a slice of the corpus rather than all of
420 it. Writing a manifest built from that slice alone is how the attribution
421 got lost: a `--datasets nsynth` run left samples/training/ full of
422 reverb-drums files and replaced the credit with a dataset the labels never
423 came from. Both are CC-BY 4.0 so the licence class survived it, but CC-BY
424 asks for credit to the work actually used. Merge instead.
425
426 A corrupt manifest is not fatal: this run is about to rewrite it with
427 whatever it knows, and refusing to proceed would leave the bad file in
428 place. The old contents are lost, which is the reason for the warning.
429 """
430 if not path.exists():
431 return {}
432 try:
433 doc = json.loads(path.read_text())
434 except (OSError, json.JSONDecodeError) as e:
435 print(f" warning: cannot read {path} ({e}); starting a fresh manifest")
436 return {}
437 return doc if isinstance(doc, dict) else {}
438
439
440 def merge_datasets(old: list, new: list) -> list:
441 """Union two dataset lists by name, with this run's entry winning.
442
443 This run's entry wins because it was just fetched from the URL it names,
444 whereas the recorded one may predate a registry edit. Sorted so a re-run
445 over the same corpus produces a byte-identical manifest.
446 """
447 by_name = {d["name"]: d for d in old if isinstance(d, dict) and "name" in d}
448 for d in new:
449 by_name[d["name"]] = d
450 return [by_name[k] for k in sorted(by_name)]
451
452
453 def main() -> None:
454 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
455 ap.add_argument("--dest", type=Path, default=Path("/media/max/T9/af-corpus"))
456 ap.add_argument("--datasets", help="comma-separated; default = the small ones")
457 ap.add_argument("--list", action="store_true", help="show the registry and exit")
458 ap.add_argument("--no-build", action="store_true", help="fetch and extract only")
459 ap.add_argument("--formats-count", type=int, default=120,
460 help="loops to transcode into each format for the decode bench")
461 args = ap.parse_args()
462
463 if args.list:
464 print(f"{'dataset':<14} {'size':>8} {'default':<8} {'license':<46} count")
465 for name, d in DATASETS.items():
466 print(
467 f"{name:<14} {d['size']:>8} {str(d['default']):<8} "
468 f"{d['license']:<46} {d['count']}"
469 )
470 print(f"{'':>14} {d['desc']}")
471 return
472
473 if args.datasets:
474 wanted = [d.strip() for d in args.datasets.split(",")]
475 unknown = [d for d in wanted if d not in DATASETS]
476 if unknown:
477 sys.exit(f"unknown dataset(s): {', '.join(unknown)}")
478 else:
479 wanted = [n for n, d in DATASETS.items() if d["default"]]
480
481 downloads = args.dest / "_downloads"
482 raw = args.dest / "_raw"
483 samples = args.dest / "samples"
484 downloads.mkdir(parents=True, exist_ok=True)
485
486 manifest_path = args.dest / "MANIFEST.json"
487 manifest = load_manifest(manifest_path)
488 manifest["layout"] = str(samples)
489 fetched: list[dict] = []
490
491 for name in wanted:
492 d = DATASETS[name]
493 print(f"\n=== {name} ({d['size']}, {d['license']}) ===")
494 archive = downloads / d["archive"]
495 download(d["url"], archive)
496 extract(archive, raw / name)
497 # Extras unpack alongside the main archive, not into their own tree:
498 # the accuracy bench expects annotations/ next to audio/ under one root.
499 for extra_name, extra_url in d.get("extra", {}).items():
500 extra_path = downloads / extra_name
501 download(extra_url, extra_path)
502 if extra_path.suffix.lower() in (".zip", ".7z", ".gz"):
503 extract(extra_path, raw / name, marker=raw / name / Path(extra_name).stem)
504
505 entry = {"name": name, "license": d["license"], "source": d["url"]}
506 # FSL10K is the one dataset here shipping a per-sound licence field, so
507 # it is the one where the manifest can carry something better than the
508 # registry's prose. Measured rather than copied: the registry entry is
509 # what was wrong in the first place.
510 if name == "fsl10k":
511 summary = summarize_fsl10k_licenses(raw / name)
512 if summary is not None:
513 entry["per_sound_licenses"] = summary
514 print(
515 f" licences: {summary['shippable']} shippable, "
516 f"{summary['excluded']} excluded, of {summary['total']}"
517 )
518 for lic, n in summary["by_license"].items():
519 print(f" {lic:<16} {n}")
520 fetched.append(entry)
521
522 manifest["datasets"] = merge_datasets(manifest.get("datasets", []), fetched)
523
524 if not args.no_build and TRAINING_DATASET in wanted:
525 print("\n=== building training layout ===")
526 counts = build_training(raw / TRAINING_DATASET, samples)
527 for k, v in counts.items():
528 print(f" {k:<14} {v}")
529 manifest["training_counts"] = counts
530 # Names the dataset the labels came from, which is what anything
531 # shipping those labels would have to credit. Nothing reads it as of
532 # 2026-08-08: the bundled .afcl generator that required it was retired
533 # with the layer itself. Kept because it costs a line and the credit is
534 # unreconstructable once the corpus is rebuilt with other datasets in
535 # it, which is how it got lost the first time (see load_manifest).
536 manifest["training_source"] = TRAINING_DATASET
537
538 # Runs off the built layout, not off a dataset, so it is gated on the loops
539 # existing rather than on which datasets were requested.
540 if not args.no_build and (samples / "test-suite" / "genres" / "loops").exists():
541 print("\n=== building format corpus ===")
542 if shutil.which("ffmpeg") is None:
543 print(" ffmpeg not found, skipping (section 2 of the bench will report no files)")
544 else:
545 fmt_counts = build_formats(samples, args.formats_count)
546 for k, v in fmt_counts.items():
547 print(f" {k:<14} {v}")
548 # Recorded as derived so the license question does not come up
549 # again: these are transcodes of corpus files, not a new dataset.
550 manifest["format_counts"] = fmt_counts
551 manifest["format_corpus"] = "transcoded from test-suite/genres/loops"
552
553 # Provenance matters here: the corpus mixes licenses, and anything that
554 # feeds a model needs that recorded rather than reconstructed later.
555 manifest_path.write_text(json.dumps(manifest, indent=2))
556 print(f"\nwrote {manifest_path}")
557 print(f"point the bench at it: export AF_BENCH_CORPUS={samples}")
558
559
560 if __name__ == "__main__":
561 main()
562