Skip to main content

max / audiofiles

16.9 KB · 444 lines History Blame Raw
1 //! Per-class threshold calibration: what operating point does a class support?
2 //!
3 //! The first run of `layer-eval` graded all seven classes against one global
4 //! threshold ([`DEFAULT_AUTO_THRESHOLD`], 0.85) and read the failure as the layer
5 //! being weak. Most of it was the threshold being wrong for six of the seven
6 //! classes.
7 //!
8 //! A score is the share of the k-nearest neighbourhood's kernel weight carrying a
9 //! tag, so 0.85 asks for roughly 13 of 15 neighbours to agree. Whether a class
10 //! can reach that is bounded by how many of its own members fall inside a fixed
11 //! `k`, which is a property of class size and local density rather than of how
12 //! distinguishable the sound is. Measured: hi-hat (109 files) and snare (188) sit
13 //! in the same top-1 band, 71.6% against 76.6%, and their recall at 0.85 differs
14 //! by a factor of nine. The score is well behaved inside a class and not
15 //! comparable across classes.
16 //!
17 //! So the question a ship gate should ask is not "what does this class do at
18 //! 0.85". It is "what is the most permissive threshold at which this class still
19 //! meets the precision we require, and what recall does it buy there". That
20 //! threshold is exactly a `tag_policy` row, which the layer format can already
21 //! carry and the export currently declines to ship.
22 //!
23 //! [`DEFAULT_AUTO_THRESHOLD`]: audiofiles_core::analysis::exemplar::DEFAULT_AUTO_THRESHOLD
24
25 /// Per-class counts at one score threshold.
26 #[derive(Default, Clone, Copy)]
27 pub(crate) struct Counts {
28 pub(crate) tp: usize,
29 pub(crate) fp: usize,
30 pub(crate) fn_: usize,
31 }
32
33 impl Counts {
34 /// Of what fired, how much was right. `None` when nothing fired, which is a
35 /// different fact from firing and being wrong, and the two must not print
36 /// the same.
37 pub(crate) fn precision(self) -> Option<f64> {
38 let predicted = self.tp + self.fp;
39 (predicted > 0).then(|| self.tp as f64 / predicted as f64)
40 }
41
42 /// Of what should have fired, how much did.
43 pub(crate) fn recall(self) -> Option<f64> {
44 let actual = self.tp + self.fn_;
45 (actual > 0).then(|| self.tp as f64 / actual as f64)
46 }
47
48 pub(crate) fn fired(self) -> usize {
49 self.tp + self.fp
50 }
51
52 pub(crate) fn actual(self) -> usize {
53 self.tp + self.fn_
54 }
55
56 pub(crate) fn add(&mut self, other: Self) {
57 self.tp += other.tp;
58 self.fp += other.fp;
59 self.fn_ += other.fn_;
60 }
61 }
62
63 /// One test sample's evidence for one class: what the layer scored it, whether it
64 /// really is that class, and which fold produced it.
65 ///
66 /// The fold travels with the point because calibrating a threshold on the same
67 /// predictions it is then scored against is threshold-fitting on the test set.
68 /// See [`out_of_fold`].
69 #[derive(Clone, Copy)]
70 pub(crate) struct Point {
71 pub(crate) score: f64,
72 pub(crate) actual: bool,
73 pub(crate) fold: usize,
74 }
75
76 /// A threshold and what shipping it would deliver.
77 #[derive(Clone, Copy)]
78 pub(crate) struct OperatingPoint {
79 pub(crate) threshold: f64,
80 pub(crate) counts: Counts,
81 }
82
83 /// z for a one-sided 95% lower bound.
84 const WILSON_Z: f64 = 1.645;
85
86 /// Lower bound of the Wilson score interval for `successes / trials`.
87 ///
88 /// Why a bound rather than the ratio: the first calibrated run picked, for each
89 /// class, the threshold whose *observed* precision just cleared 95%, and held-out
90 /// precision then came in at 78.6% to 94.9%. Every class undershot. That is not
91 /// bad luck, it is what selecting the most permissive point that clears a bar
92 /// does: the point that clears it by the narrowest margin is the one most likely
93 /// to have cleared it by chance, and picking the extreme of a noisy set is
94 /// selection bias with a known direction.
95 ///
96 /// The bound removes the ad-hoc part of the fix. Instead of "meet 95% and also
97 /// have at least N predictions", a class must be 95%-confident of being above the
98 /// bar, which asks for more evidence from a small sample and less from a large
99 /// one, on the same scale. 19 of 20 correct reads as 76% here; 190 of 200 reads
100 /// as 92%.
101 fn wilson_lower_bound(successes: usize, trials: usize) -> f64 {
102 if trials == 0 {
103 return 0.0;
104 }
105 let n = trials as f64;
106 let p = successes as f64 / n;
107 let z2 = WILSON_Z * WILSON_Z;
108 let denom = 1.0 + z2 / n;
109 let center = p + z2 / (2.0 * n);
110 let margin = WILSON_Z * (p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt();
111 ((center - margin) / denom).max(0.0)
112 }
113
114 /// Counts at a fixed threshold. Mirrors the runtime rule: a tag applies when its
115 /// score is at or above the threshold.
116 pub(crate) fn counts_at(points: &[Point], threshold: f64) -> Counts {
117 let mut c = Counts::default();
118 for p in points {
119 match (p.score >= threshold, p.actual) {
120 (true, true) => c.tp += 1,
121 (true, false) => c.fp += 1,
122 (false, true) => c.fn_ += 1,
123 (false, false) => {}
124 }
125 }
126 c
127 }
128
129 /// The most permissive threshold at which this class is 95%-confident of holding
130 /// `target_precision`, with at least `min_support` predictions behind it.
131 ///
132 /// Most permissive means lowest, because a lower threshold is more recall, and
133 /// recall is what we are buying once precision is fixed. Walks the score-sorted
134 /// points once, which visits every threshold that can produce a distinct split.
135 ///
136 /// The bar is [`wilson_lower_bound`], not the observed ratio: see there for why
137 /// the observed ratio systematically overstates what a chosen threshold delivers
138 /// on data it did not see.
139 ///
140 /// Two further guards:
141 ///
142 /// - `min_support`. A floor under the bound, so a class cannot ship a policy off
143 /// a handful of predictions even when the arithmetic allows it.
144 /// - A threshold of zero is refused. It would fire on every sample including the
145 /// ones the index scored nothing for, which is not a classifier.
146 ///
147 /// Precision is not monotone in the threshold, so a lower qualifying threshold
148 /// may sit below a stretch that does not qualify. That is fine and deliberate:
149 /// what ships is a single threshold, and the numbers reported are the ones
150 /// measured at it.
151 pub(crate) fn operating_point(
152 points: &[Point],
153 target_precision: f64,
154 min_support: usize,
155 ) -> Option<OperatingPoint> {
156 let positives = points.iter().filter(|p| p.actual).count();
157 if positives == 0 {
158 return None;
159 }
160
161 let mut sorted: Vec<&Point> = points.iter().collect();
162 sorted.sort_by(|a, b| b.score.total_cmp(&a.score));
163
164 let mut tp = 0usize;
165 let mut fp = 0usize;
166 let mut best: Option<OperatingPoint> = None;
167
168 let mut i = 0;
169 while i < sorted.len() {
170 // Every point sharing this score has to be taken with it: no threshold
171 // can separate two samples that scored the same.
172 let score = sorted[i].score;
173 #[allow(
174 clippy::float_cmp,
175 reason = "exact equality is the point: a threshold cannot split two \
176 samples that scored bit-identically, so the group boundary \
177 has to be exact rather than within a tolerance"
178 )]
179 while i < sorted.len() && sorted[i].score == score {
180 if sorted[i].actual {
181 tp += 1;
182 } else {
183 fp += 1;
184 }
185 i += 1;
186 }
187 if score <= 0.0 {
188 break;
189 }
190 let fired = tp + fp;
191 if fired < min_support {
192 continue;
193 }
194 if wilson_lower_bound(tp, fired) >= target_precision {
195 // Descending walk, so each qualifying point is more permissive than
196 // the last. Keep overwriting and the survivor is the lowest.
197 best = Some(OperatingPoint {
198 threshold: score,
199 counts: Counts {
200 tp,
201 fp,
202 fn_: positives - tp,
203 },
204 });
205 }
206 }
207 best
208 }
209
210 /// Calibrate and evaluate without letting a class pick its threshold from the
211 /// samples it is then graded on.
212 ///
213 /// For each fold: choose the threshold from every *other* fold's points, then
214 /// count this fold's points at it. Summing across folds gives per-class numbers
215 /// that no threshold was fitted to. A fold whose calibration finds no qualifying
216 /// threshold contributes its positives as misses, because a policy that cannot be
217 /// derived is a policy that does not ship and a tag that never fires.
218 ///
219 /// Returns the summed counts and the thresholds chosen, one per fold that found
220 /// one. The spread of those thresholds is worth reading: a class whose threshold
221 /// swings between folds is one whose operating point is not a stable property of
222 /// the class.
223 pub(crate) fn out_of_fold(
224 points: &[Point],
225 folds: usize,
226 target_precision: f64,
227 min_support: usize,
228 ) -> (Counts, Vec<f64>) {
229 let mut total = Counts::default();
230 let mut thresholds = Vec::new();
231
232 for fold in 0..folds {
233 let calibration: Vec<Point> = points.iter().filter(|p| p.fold != fold).copied().collect();
234 let held_out: Vec<Point> = points.iter().filter(|p| p.fold == fold).copied().collect();
235 if held_out.is_empty() {
236 continue;
237 }
238
239 // The calibration set is smaller than the whole, so scale the support
240 // floor with it. Otherwise a class that clears `min_support` overall
241 // fails to calibrate on 4/5 of the data for arithmetic reasons.
242 let scaled_support = (min_support * (folds - 1)).div_ceil(folds).max(1);
243 match operating_point(&calibration, target_precision, scaled_support) {
244 Some(op) => {
245 total.add(counts_at(&held_out, op.threshold));
246 thresholds.push(op.threshold);
247 }
248 None => {
249 // No threshold: nothing fires, so every positive here is a miss.
250 total.fn_ += held_out.iter().filter(|p| p.actual).count();
251 }
252 }
253 }
254 (total, thresholds)
255 }
256
257 /// Mean, and the spread, of the per-fold thresholds.
258 pub(crate) fn threshold_spread(thresholds: &[f64]) -> Option<(f64, f64, f64)> {
259 if thresholds.is_empty() {
260 return None;
261 }
262 let mean = thresholds.iter().sum::<f64>() / thresholds.len() as f64;
263 let min = thresholds.iter().copied().fold(f64::INFINITY, f64::min);
264 let max = thresholds.iter().copied().fold(f64::NEG_INFINITY, f64::max);
265 Some((mean, min, max))
266 }
267
268 #[cfg(test)]
269 mod tests {
270 use super::*;
271
272 fn pts(spec: &[(f64, bool)]) -> Vec<Point> {
273 spec.iter()
274 .enumerate()
275 .map(|(i, &(score, actual))| Point {
276 score,
277 actual,
278 fold: i % 5,
279 })
280 .collect()
281 }
282
283 #[test]
284 fn counts_at_matches_the_runtime_at_or_above_rule() {
285 let p = pts(&[(0.9, true), (0.5, true), (0.5, false), (0.1, true)]);
286 let c = counts_at(&p, 0.5);
287 assert_eq!((c.tp, c.fp, c.fn_), (2, 1, 1), "0.5 fires at exactly 0.5");
288 }
289
290 #[test]
291 fn operating_point_takes_the_most_permissive_qualifying_threshold() {
292 // Perfectly ordered: every positive above every negative. Asserted as a
293 // property rather than a magic threshold, because what "most permissive"
294 // resolves to depends on the bound, the target and the sample size, and
295 // a hardcoded number would be testing this fixture's arithmetic.
296 let spec: Vec<(f64, bool)> = (0..100)
297 .map(|i| (0.9 - f64::from(i) * 0.001, true))
298 .chain((0..100).map(|i| (0.2 - f64::from(i) * 0.001, false)))
299 .collect();
300 let p = pts(&spec);
301 let op = operating_point(&p, 0.95, 1).unwrap();
302
303 // What it reports is what that threshold actually does.
304 let at = counts_at(&p, op.threshold);
305 assert_eq!((at.tp, at.fp), (op.counts.tp, op.counts.fp));
306 assert!(wilson_lower_bound(at.tp, at.fired()) >= 0.95);
307
308 // And nothing lower qualifies, which is what makes it the most permissive.
309 for lower in p
310 .iter()
311 .map(|x| x.score)
312 .filter(|s| *s < op.threshold && *s > 0.0)
313 {
314 let c = counts_at(&p, lower);
315 assert!(
316 wilson_lower_bound(c.tp, c.fired()) < 0.95,
317 "threshold {lower} also qualifies, so {} was not the lowest",
318 op.threshold
319 );
320 }
321 }
322
323 #[test]
324 fn operating_point_refuses_a_threshold_backed_by_too_few_predictions() {
325 // Two perfect predictions then a mess. The bound alone rejects this:
326 // 2 of 2 is not evidence of 95%, and the first run read cymbal that way.
327 let p = pts(&[
328 (0.99, true),
329 (0.98, true),
330 (0.5, false),
331 (0.5, false),
332 (0.5, false),
333 (0.4, true),
334 ]);
335 assert!(
336 operating_point(&p, 0.95, 1).is_none(),
337 "two perfect predictions are not 95% confidence of 95% precision"
338 );
339 }
340
341 #[test]
342 fn the_bound_demands_more_evidence_from_a_smaller_sample() {
343 // The property the whole calibration rests on: the same observed ratio
344 // qualifies at scale and does not qualify on a handful.
345 assert!(wilson_lower_bound(19, 20) < 0.85);
346 assert!(wilson_lower_bound(190, 200) > 0.9);
347 // Monotone in sample size at a fixed ratio.
348 assert!(wilson_lower_bound(9, 10) < wilson_lower_bound(90, 100));
349 // Degenerate inputs stay in range rather than producing a NaN that
350 // would silently compare false against every target.
351 assert!(wilson_lower_bound(0, 0).abs() < f64::EPSILON);
352 assert!(wilson_lower_bound(0, 5) >= 0.0);
353 assert!(wilson_lower_bound(5, 5) <= 1.0);
354 }
355
356 #[test]
357 fn operating_point_never_returns_a_zero_threshold() {
358 // Every sample scores zero: the index had nothing to say. A zero
359 // threshold would "apply" the tag to all of them.
360 let spec: Vec<(f64, bool)> = (0..40).map(|i| (0.0, i % 2 == 0)).collect();
361 assert!(operating_point(&pts(&spec), 0.4, 1).is_none());
362 }
363
364 #[test]
365 fn operating_point_is_none_when_precision_is_unreachable() {
366 let p = pts(&[(0.9, false), (0.8, false), (0.7, true)]);
367 assert!(operating_point(&p, 0.95, 1).is_none());
368 }
369
370 #[test]
371 fn tied_scores_are_taken_together() {
372 // A threshold cannot split two equal scores, so the walk must not report
373 // a precision that only holds if it does. A clean run of positives, then
374 // a tied group of one positive and five negatives: taking the group
375 // whole fails the bar, and splitting it would pass.
376 let mut spec: Vec<(f64, bool)> = (0..100)
377 .map(|i| (0.9 - f64::from(i) * 0.001, true))
378 .collect();
379 spec.push((0.5, true));
380 spec.extend((0..5).map(|_| (0.5, false)));
381
382 let op = operating_point(&pts(&spec), 0.95, 1).unwrap();
383 assert!(
384 op.threshold > 0.5,
385 "the tie was split; got {}",
386 op.threshold
387 );
388 }
389
390 #[test]
391 fn out_of_fold_counts_an_uncalibratable_fold_as_misses() {
392 // Nothing separates these, so no fold finds a threshold and every
393 // positive must be a miss rather than silently vanishing.
394 let p = pts(&[
395 (0.5, true),
396 (0.5, false),
397 (0.5, true),
398 (0.5, false),
399 (0.5, true),
400 (0.5, false),
401 (0.5, true),
402 (0.5, false),
403 (0.5, true),
404 (0.5, false),
405 ]);
406 let (counts, thresholds) = out_of_fold(&p, 5, 0.95, 1);
407 assert!(thresholds.is_empty());
408 assert_eq!(counts.tp, 0);
409 assert_eq!(counts.actual(), 5, "all five positives are accounted for");
410 }
411
412 #[test]
413 fn out_of_fold_is_not_more_optimistic_than_the_data_supports() {
414 // Separable data: out-of-fold should recover it, since the threshold
415 // learned on four folds transfers to the fifth.
416 let mut spec: Vec<(f64, bool)> = (0..100)
417 .map(|i| (0.9 - f64::from(i) * 0.001, true))
418 .collect();
419 spec.extend((0..100).map(|i| (0.3 - f64::from(i) * 0.001, false)));
420 let p = pts(&spec);
421 let (counts, thresholds) = out_of_fold(&p, 5, 0.95, 4);
422 assert_eq!(thresholds.len(), 5, "every fold calibrates");
423 // Precision holds on folds the threshold never saw, which is the whole
424 // contract. Not 100%: the target is 0.95, so the most permissive
425 // qualifying threshold deliberately admits a few negatives, and demanding
426 // perfection here would be asserting a stricter bar than was asked for.
427 assert!(
428 counts.precision().unwrap() >= 0.95,
429 "{:?}",
430 counts.precision()
431 );
432 assert!(counts.recall().unwrap() > 0.9, "{:?}", counts.recall());
433 }
434
435 #[test]
436 fn threshold_spread_reports_min_and_max() {
437 let (mean, min, max) = threshold_spread(&[0.4, 0.6, 0.5]).unwrap();
438 assert!((mean - 0.5).abs() < 1e-9);
439 assert!((min - 0.4).abs() < 1e-9);
440 assert!((max - 0.6).abs() < 1e-9);
441 assert!(threshold_spread(&[]).is_none());
442 }
443 }
444