Skip to main content

max / audiofiles

17.1 KB · 468 lines History Blame Raw
1 //! BPM and key accuracy against FSL10K ground truth.
2 //!
3 //! The pipeline emits two things a corpus can check it on, and this is where both
4 //! are scored: `bpm::detect_bpm_key`'s tempo and key. (The analysis bench used to
5 //! carry a third, accuracy of the single sample-class label, until that label was
6 //! retired; see `docs/ml_classifier.md`.)
7 //!
8 //! Ground truth comes from the Freesound Loop Dataset, which is the only corpus
9 //! here carrying expert tempo and key annotations. Zenodo publishes FSL10K as
10 //! CC-BY, but its sounds carry four different Freesound licences and 1,436 of
11 //! the 9,493 are CC-BY-NC or Sampling+. That is fine for what happens here --
12 //! this mode decodes audio, scores a number, and emits nothing derived from the
13 //! samples -- and it is not fine for anything that ships. Anything that selects
14 //! sounds out of this corpus rather than measuring over it reads the per-sound
15 //! licence from `metadata.json`; `scripts/corpus.py` records the mix in the
16 //! corpus MANIFEST.json.
17 //!
18 //! Fetch it with `scripts/corpus.py --datasets fsl10k`, then point this at the
19 //! extracted root:
20 //!
21 //! AF_BENCH_FSL10K=/media/max/T9/af-corpus/_raw/fsl10k \
22 //! cargo run --release -p audiofiles-bench -- accuracy
23 //!
24 //! Tempo is scored with the standard MIREX pair rather than a single number,
25 //! because a tempo estimator that is "wrong" is usually wrong by a factor of
26 //! two, and collapsing that into one accuracy figure hides which kind of wrong
27 //! it is:
28 //! Acc1 = within 4% of ground truth.
29 //! Acc2 = Acc1, or within 4% of truth scaled by 1/3, 1/2, 2, or 3.
30 //! A large Acc2-minus-Acc1 gap means octave errors, which are a different fix
31 //! from being diffusely inaccurate.
32
33 use std::collections::HashMap;
34 use std::path::{Path, PathBuf};
35
36 use audiofiles_core::analysis::{bpm, decode};
37 use rayon::prelude::*;
38
39 use crate::storage;
40
41 /// Ground truth for one sound, after consensus across annotators.
42 struct Truth {
43 bpm: f64,
44 /// Pitch class 0..=11 with mode, or None when the annotators marked the
45 /// loop as having no key (drum loops, percussion, fx).
46 key: Option<(u8, Mode)>,
47 }
48
49 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
50 enum Mode {
51 Major,
52 Minor,
53 }
54
55 /// Parse a note name to a pitch class.
56 ///
57 /// Comparing pitch classes rather than strings sidesteps enharmonic spelling
58 /// entirely: the annotations use sharps only, but nothing guarantees the
59 /// detector does, and "A#" and "Bb" are the same key.
60 fn pitch_class(note: &str) -> Option<u8> {
61 let s = note.trim().to_lowercase();
62 let mut chars = s.chars();
63 let base = match chars.next()? {
64 'c' => 0,
65 'd' => 2,
66 'e' => 4,
67 'f' => 5,
68 'g' => 7,
69 'a' => 9,
70 'b' => 11,
71 _ => return None,
72 };
73 let accidental: i32 = match chars.next() {
74 Some('#' | 's') => 1,
75 Some('b' | 'f') => -1,
76 None => 0,
77 _ => return None,
78 };
79 Some(((base + accidental).rem_euclid(12)) as u8)
80 }
81
82 /// Parse a detected key string into a pitch class and mode.
83 ///
84 /// Handles both shapes on purpose. `detect_bpm_key` now normalises to
85 /// "A minor" / "C major", but before that it passed through
86 /// `stratum_dsp::Key::name()` unchanged, which is "C" / "F#" for major and
87 /// "Am" / "C#m" for minor. Vaults analysed before migration 034 can still hold
88 /// the compact form, and accepting only one spelling silently scores 0%
89 /// forever rather than failing loudly, so both are parsed.
90 fn parse_app_key(s: &str) -> Option<(u8, Mode)> {
91 let s = s.trim();
92 if let Some((note, word)) = s.rsplit_once(' ') {
93 let mode = match word.trim().to_lowercase().as_str() {
94 "minor" | "min" => Mode::Minor,
95 "major" | "maj" => Mode::Major,
96 _ => return None,
97 };
98 return Some((pitch_class(note)?, mode));
99 }
100 // Bare form: trailing 'm' means minor, otherwise major. Checked after the
101 // spaced form so "F major" is not read as a note named "F majo" + m.
102 if let Some(note) = s.strip_suffix('m') {
103 return Some((pitch_class(note)?, Mode::Minor));
104 }
105 Some((pitch_class(s)?, Mode::Major))
106 }
107
108 /// Load and reconcile annotations.
109 ///
110 /// Half the annotated sounds in FSL10K carry more than one annotator's
111 /// judgement, and annotators disagree. Rather than picking one arbitrarily,
112 /// disagreements are dropped and counted: scoring a detector against a
113 /// contested label measures the disagreement, not the detector.
114 fn load_truth(annotations: &Path) -> (HashMap<String, Truth>, usize, usize) {
115 let mut by_sound: HashMap<String, Vec<serde_json::Value>> = HashMap::new();
116 let mut discarded = 0usize;
117
118 let mut stack = vec![annotations.to_path_buf()];
119 while let Some(dir) = stack.pop() {
120 let Ok(entries) = std::fs::read_dir(&dir) else {
121 continue;
122 };
123 for entry in entries.flatten() {
124 let path = entry.path();
125 if path.is_dir() {
126 stack.push(path);
127 continue;
128 }
129 let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
130 continue;
131 };
132 let Some(id) = name
133 .strip_prefix("sound-")
134 .and_then(|n| n.strip_suffix(".json"))
135 else {
136 continue;
137 };
138 let Ok(text) = std::fs::read_to_string(&path) else {
139 continue;
140 };
141 let Ok(v) = serde_json::from_str::<serde_json::Value>(&text) else {
142 continue;
143 };
144 if v.get("discard").and_then(serde_json::Value::as_bool) == Some(true) {
145 discarded += 1;
146 continue;
147 }
148 by_sound.entry(id.to_string()).or_default().push(v);
149 }
150 }
151
152 let mut truth = HashMap::new();
153 let mut disputed = 0usize;
154
155 for (id, annots) in by_sound {
156 // BPM is stored as a string in these files.
157 let bpms: Vec<f64> = annots
158 .iter()
159 .filter_map(|a| a.get("bpm").and_then(serde_json::Value::as_str))
160 .filter_map(|s| s.trim().parse::<f64>().ok())
161 .filter(|b| *b > 0.0)
162 .collect();
163 if bpms.is_empty() {
164 continue;
165 }
166 // Annotators agree if they are all within 4% of the first, the same
167 // tolerance the scoring uses.
168 let first = bpms[0];
169 if bpms.iter().any(|b| (b - first).abs() / first > 0.04) {
170 disputed += 1;
171 continue;
172 }
173 let bpm_truth = bpms.iter().sum::<f64>() / bpms.len() as f64;
174
175 let keys: Vec<Option<(u8, Mode)>> = annots
176 .iter()
177 .map(|a| {
178 let k = a.get("key").and_then(serde_json::Value::as_str)?;
179 let m = a.get("mode").and_then(serde_json::Value::as_str)?;
180 let mode = match m {
181 "min" => Mode::Minor,
182 "maj" => Mode::Major,
183 _ => return None,
184 };
185 Some((pitch_class(k)?, mode))
186 })
187 .collect();
188 // "none"/"unknown" parse to None, which is itself a meaningful answer
189 // (the loop has no key), so only disagreement between Some values is a
190 // dispute.
191 let key_truth = if keys.iter().all(|k| *k == keys[0]) {
192 keys[0]
193 } else {
194 None
195 };
196
197 truth.insert(
198 id,
199 Truth {
200 bpm: bpm_truth,
201 key: key_truth,
202 },
203 );
204 }
205
206 (truth, discarded, disputed)
207 }
208
209 /// Index audio files by Freesound sound ID (the filename stem).
210 fn index_audio(root: &Path) -> HashMap<String, PathBuf> {
211 let mut out = HashMap::new();
212 let mut stack = vec![root.to_path_buf()];
213 while let Some(dir) = stack.pop() {
214 let Ok(entries) = std::fs::read_dir(&dir) else {
215 continue;
216 };
217 for entry in entries.flatten() {
218 let path = entry.path();
219 if path.is_dir() {
220 stack.push(path);
221 continue;
222 }
223 let is_audio = path.extension().and_then(|e| e.to_str()).is_some_and(|e| {
224 matches!(
225 e.to_lowercase().as_str(),
226 "wav" | "flac" | "mp3" | "ogg" | "aif" | "aiff"
227 )
228 });
229 if !is_audio {
230 continue;
231 }
232 if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
233 // FSL10K names audio by sound ID, sometimes with a suffix after
234 // the ID; take the leading digit run.
235 let id: String = stem.chars().take_while(char::is_ascii_digit).collect();
236 if !id.is_empty() {
237 out.insert(id, path);
238 }
239 }
240 }
241 }
242 out
243 }
244
245 fn within(a: f64, b: f64, tol: f64) -> bool {
246 b > 0.0 && (a - b).abs() / b <= tol
247 }
248
249 struct Scored {
250 bpm_acc1: bool,
251 bpm_acc2: bool,
252 /// Set only when both truth and detection carry a key.
253 key_exact: Option<bool>,
254 key_related: Option<bool>,
255 /// Tracked separately so a zero key score can be attributed. "No key was
256 /// scored" has three very different causes -- the loop has no key in the
257 /// ground truth, the detector returned nothing, or the detected string
258 /// failed to parse -- and collapsing them hides a parser bug as a result.
259 had_truth_key: bool,
260 had_detected_key: bool,
261 detected_key_unparsed: bool,
262 }
263
264 fn score(detected_bpm: Option<f64>, detected_key: Option<&str>, truth: &Truth) -> Scored {
265 let (acc1, acc2) = match detected_bpm {
266 Some(d) => {
267 let a1 = within(d, truth.bpm, 0.04);
268 // Octave and triplet confusions: the classic failure mode of
269 // autocorrelation tempo estimators.
270 let a2 = a1
271 || [1.0 / 3.0, 0.5, 2.0, 3.0]
272 .iter()
273 .any(|f| within(d, truth.bpm * f, 0.04));
274 (a1, a2)
275 }
276 None => (false, false),
277 };
278
279 let (key_exact, key_related) = match (truth.key, detected_key.and_then(parse_app_key)) {
280 (Some((tp, tm)), Some((dp, dm))) => {
281 let exact = tp == dp && tm == dm;
282 // Related = relative major/minor, or a perfect fifth apart in the
283 // same mode. These are the confusions a chroma-based estimator makes
284 // because the pitch content genuinely overlaps.
285 let relative = match tm {
286 Mode::Major => dm == Mode::Minor && dp == (tp + 9) % 12,
287 Mode::Minor => dm == Mode::Major && dp == (tp + 3) % 12,
288 };
289 let fifth = tm == dm && (dp == (tp + 7) % 12 || dp == (tp + 5) % 12);
290 (Some(exact), Some(exact || relative || fifth))
291 }
292 _ => (None, None),
293 };
294
295 Scored {
296 bpm_acc1: acc1,
297 bpm_acc2: acc2,
298 key_exact,
299 key_related,
300 had_truth_key: truth.key.is_some(),
301 had_detected_key: detected_key.is_some(),
302 detected_key_unparsed: detected_key.is_some_and(|k| parse_app_key(k).is_none()),
303 }
304 }
305
306 pub(crate) fn run(root: &Path, limit: Option<usize>) {
307 println!("━━━ BPM + KEY ACCURACY (FSL10K ground truth) ━━━");
308 println!();
309
310 // The zip lays out FSL10K/audio/...; accept either the zip root or the
311 // directory that directly contains audio/ and annotations/.
312 let annotations = [
313 "annotations",
314 "FSL10K/annotations",
315 "../fsl10k-annotations/annotations",
316 ]
317 .iter()
318 .map(|p| root.join(p))
319 .find(|p| p.is_dir());
320 let audio = ["FSL10K/audio", "audio", "FSL10K/FSL10K/audio"]
321 .iter()
322 .map(|p| root.join(p))
323 .find(|p| p.is_dir());
324
325 let (Some(annotations), Some(audio)) = (annotations, audio) else {
326 eprintln!(
327 "could not find annotations/ and audio/ under {}",
328 root.display()
329 );
330 eprintln!("fetch with: ./scripts/corpus.py --datasets fsl10k");
331 return;
332 };
333
334 // Printed but not serialised: this mode emits no timings, so the drive
335 // cannot change its results. It is recorded for provenance, so a scorecard
336 // says which copy of the dataset produced the numbers.
337 let audio_storage = storage::describe(&audio);
338 storage::print_conditions(&[("audio", &audio_storage)], None);
339
340 let (truth, discarded, disputed) = load_truth(&annotations);
341 let index = index_audio(&audio);
342 println!(
343 " annotations: {} sounds with agreed ground truth",
344 truth.len()
345 );
346 println!(
347 " {discarded} annotation(s) flagged discard, {disputed} sound(s) dropped as disputed"
348 );
349 println!(" audio files indexed: {}", index.len());
350
351 let mut pairs: Vec<(&String, &Truth, &PathBuf)> = truth
352 .iter()
353 .filter_map(|(id, t)| index.get(id).map(|p| (id, t, p)))
354 .collect();
355 pairs.sort_by_key(|(id, _, _)| (*id).clone());
356 if let Some(lim) = limit {
357 pairs.truncate(lim);
358 }
359
360 if pairs.is_empty() {
361 eprintln!("no annotated sound matched an audio file");
362 return;
363 }
364 println!(" scoring {} loops", pairs.len());
365 println!();
366
367 let results: Vec<Scored> = pairs
368 .par_iter()
369 .filter_map(|(_, t, path)| {
370 let decoded = decode::decode_to_mono(path).ok()?;
371 // 2.0 matches the analysis pipeline's min_duration gate, so this
372 // measures what the app actually runs rather than a variant.
373 let r = bpm::detect_bpm_key(&decoded.samples, decoded.sample_rate, 2.0);
374 Some(score(r.bpm, r.key.as_deref(), t))
375 })
376 .collect();
377
378 let n = results.len() as f64;
379 let acc1 = results.iter().filter(|r| r.bpm_acc1).count() as f64 / n * 100.0;
380 let acc2 = results.iter().filter(|r| r.bpm_acc2).count() as f64 / n * 100.0;
381
382 println!(" TEMPO ({} scored)", results.len());
383 println!(" Acc1 (within 4%) {acc1:>6.1}%");
384 println!(" Acc2 (octave/triplet ok) {acc2:>6.1}%");
385 let gap = acc2 - acc1;
386 println!(" octave-error gap {gap:>6.1}%");
387 if gap > 15.0 {
388 println!(" ^ large gap: most misses are half/double time, not noise");
389 }
390 println!();
391
392 let with_truth = results.iter().filter(|r| r.had_truth_key).count();
393 let with_detected = results.iter().filter(|r| r.had_detected_key).count();
394 let unparsed = results.iter().filter(|r| r.detected_key_unparsed).count();
395
396 let keyed: Vec<&Scored> = results.iter().filter(|r| r.key_exact.is_some()).collect();
397 println!(" KEY coverage");
398 println!(
399 " loops with ground-truth key {with_truth:>5} / {}",
400 results.len()
401 );
402 println!(
403 " loops with detected key {with_detected:>5} / {}",
404 results.len()
405 );
406 if unparsed > 0 {
407 println!(" detected but UNPARSED {unparsed:>5} <- parser/format mismatch");
408 }
409 println!(" scorable (both present) {:>5}", keyed.len());
410
411 // Precision on keyless material, which the accuracy figures cannot show.
412 // Most loops in a sample library are drums, and annotators mark those as
413 // having no key. A detector that emits a key anyway is not inaccurate by
414 // the measures above -- those only score loops that have a key -- but it
415 // fills the library with confident wrong labels, and a key filter that
416 // returns drum loops is a user-visible bug.
417 let keyless = results.len() - with_truth;
418 let spurious = results
419 .iter()
420 .filter(|r| !r.had_truth_key && r.had_detected_key)
421 .count();
422 if keyless > 0 {
423 let rate = spurious as f64 / keyless as f64 * 100.0;
424 println!(" key emitted on keyless loops {spurious:>5} / {keyless} ({rate:.1}%)");
425 }
426 println!();
427
428 if keyed.is_empty() {
429 println!(" KEY: nothing scorable. Check the coverage lines above for why.");
430 } else {
431 let kn = keyed.len() as f64;
432 let exact = keyed.iter().filter(|r| r.key_exact == Some(true)).count() as f64 / kn * 100.0;
433 let related =
434 keyed.iter().filter(|r| r.key_related == Some(true)).count() as f64 / kn * 100.0;
435 println!(" KEY ({} scored, of {} total)", keyed.len(), results.len());
436 println!(" exact {exact:>6.1}%");
437 println!(" exact or relative/fifth {related:>6.1}%");
438 // Chance is 1/24 for exact over 12 pitch classes x 2 modes.
439 println!(" (chance for exact is 4.2%)");
440 }
441 }
442
443 #[cfg(test)]
444 mod tests {
445 use super::{Mode, parse_app_key, pitch_class};
446
447 #[test]
448 fn parses_stratum_key_format() {
449 // The format detect_bpm_key actually emits.
450 assert_eq!(parse_app_key("C"), Some((0, Mode::Major)));
451 assert_eq!(parse_app_key("F#"), Some((6, Mode::Major)));
452 assert_eq!(parse_app_key("Am"), Some((9, Mode::Minor)));
453 assert_eq!(parse_app_key("C#m"), Some((1, Mode::Minor)));
454 }
455
456 #[test]
457 fn parses_documented_key_format() {
458 assert_eq!(parse_app_key("A minor"), Some((9, Mode::Minor)));
459 assert_eq!(parse_app_key("C major"), Some((0, Mode::Major)));
460 }
461
462 #[test]
463 fn enharmonics_are_the_same_pitch_class() {
464 assert_eq!(pitch_class("a#"), pitch_class("bb"));
465 assert_eq!(pitch_class("c#"), pitch_class("db"));
466 }
467 }
468