Skip to main content

max / audiofiles

12.4 KB · 344 lines History Blame Raw
1 //! Does a detector's confidence predict whether it was right?
2 //!
3 //! `detect_bpm_key` returns a `bpm_confidence` and a `key_confidence` beside
4 //! every answer, and nothing has ever checked that they mean anything. Both are
5 //! read as facts downstream, and a confidence that does not rank correct calls
6 //! above wrong ones is worse than no confidence at all: it is a number the UI
7 //! shows and a gate could be built on, carrying no information.
8 //!
9 //! Two different questions, answered by two different shapes here.
10 //!
11 //! [`discrimination`] and [`bands`] ask whether the number ranks at all, over
12 //! calls that have a right answer to be measured against. Discrimination is one
13 //! scalar and bands are where it came from, because a scalar near 0.5 does not
14 //! say whether the confidence is flat or non-monotone.
15 //!
16 //! [`gate_sweep`] asks the design question the spurious-key problem needs: the
17 //! detector emits a key on 90.8% of loops annotators call keyless, so a gate
18 //! that only lets a key through above some confidence has to be priced. Its
19 //! population is not the scorable loops -- it is every loop, because what a
20 //! gate costs is measured on the keyed ones and what it buys is measured on the
21 //! keyless ones.
22 //!
23 //! Thresholds and band edges are taken from the observed quantiles rather than
24 //! from a fixed grid. A detector whose confidences all sit between 0.62 and
25 //! 0.71 would give an equal-width sweep one populated row and nine empty ones,
26 //! and that is a plausible thing for these confidences to do.
27
28 /// Whether two confidences are the same number.
29 ///
30 /// Exact equality, and deliberately so: a tie here means the detector returned
31 /// the identical value twice, not two values close enough to round together.
32 /// Ties drive midranks and band boundaries, and a tolerance would merge
33 /// distinct confidences into one block and change both.
34 #[expect(
35 clippy::float_cmp,
36 reason = "a tie is bit-equality; a tolerance would merge distinct confidences"
37 )]
38 fn tied(a: f64, b: f64) -> bool {
39 a == b
40 }
41
42 /// One detection with a right answer to be scored against.
43 #[derive(Clone, Copy)]
44 pub(crate) struct Call {
45 pub(crate) conf: f64,
46 pub(crate) correct: bool,
47 }
48
49 /// Probability that a randomly chosen correct call carries a higher confidence
50 /// than a randomly chosen wrong one, ties counting half. `None` when everything
51 /// is right or everything is wrong, which is no evidence either way rather than
52 /// a score of any particular value.
53 ///
54 /// 0.5 is a confidence that ranks no better than a coin. This is AUROC, by way
55 /// of the Mann-Whitney U it equals; computed off the ranks rather than by
56 /// counting pairs so 2,755 loops does not become 3.8M comparisons.
57 pub(crate) fn discrimination(calls: &[Call]) -> Option<f64> {
58 let correct = calls.iter().filter(|c| c.correct).count();
59 let wrong = calls.len() - correct;
60 if correct == 0 || wrong == 0 {
61 return None;
62 }
63
64 let mut sorted: Vec<Call> = calls.to_vec();
65 sorted.sort_by(|a, b| a.conf.total_cmp(&b.conf));
66
67 // Midranks, so a block of equal confidences contributes the half-credit
68 // ties are supposed to get instead of whatever order the sort left them in.
69 let mut rank_sum = 0.0;
70 let mut i = 0;
71 while i < sorted.len() {
72 let mut j = i;
73 while j + 1 < sorted.len() && tied(sorted[j + 1].conf, sorted[i].conf) {
74 j += 1;
75 }
76 // Ranks are 1-based over i..=j, so their mean is this.
77 let midrank = (i + j) as f64 / 2.0 + 1.0;
78 rank_sum += sorted[i..=j].iter().filter(|c| c.correct).count() as f64 * midrank;
79 i = j + 1;
80 }
81
82 let (n_c, n_w) = (correct as f64, wrong as f64);
83 Some((rank_sum - n_c * (n_c + 1.0) / 2.0) / (n_c * n_w))
84 }
85
86 /// One confidence band and how often the detector was right inside it.
87 pub(crate) struct Band {
88 pub(crate) low: f64,
89 pub(crate) high: f64,
90 pub(crate) n: usize,
91 pub(crate) correct: usize,
92 }
93
94 impl Band {
95 pub(crate) fn accuracy(&self) -> f64 {
96 self.correct as f64 / self.n as f64
97 }
98 }
99
100 /// Split `calls` into up to `groups` bands of roughly equal population and
101 /// report accuracy in each. Calibration is the claim that accuracy rises with
102 /// the band, so this is the table that claim is read off.
103 ///
104 /// Equal population rather than equal width: a band nobody landed in reports a
105 /// ratio over zero, and enough of those turn the table into a row of blanks
106 /// with the whole detector inside one cell.
107 pub(crate) fn bands(calls: &[Call], groups: usize) -> Vec<Band> {
108 if calls.is_empty() || groups == 0 {
109 return Vec::new();
110 }
111 let mut sorted: Vec<Call> = calls.to_vec();
112 sorted.sort_by(|a, b| a.conf.total_cmp(&b.conf));
113
114 let mut out = Vec::new();
115 let mut start = 0;
116 for g in 0..groups {
117 if start >= sorted.len() {
118 break;
119 }
120 let mut end = (sorted.len() * (g + 1)) / groups;
121 // A tie must not straddle a boundary, or two bands report overlapping
122 // confidence ranges and neither one means anything.
123 while end < sorted.len() && end > start && tied(sorted[end - 1].conf, sorted[end].conf) {
124 end += 1;
125 }
126 if end <= start {
127 continue;
128 }
129 let slice = &sorted[start..end];
130 out.push(Band {
131 low: slice[0].conf,
132 high: slice[slice.len() - 1].conf,
133 n: slice.len(),
134 correct: slice.iter().filter(|c| c.correct).count(),
135 });
136 start = end;
137 }
138 out
139 }
140
141 /// One loop's evidence for the key gate.
142 #[derive(Clone, Copy)]
143 pub(crate) struct KeyCall {
144 /// The confidence the detector attached to the key it emitted.
145 pub(crate) conf: f64,
146 /// Whether the annotators say this loop has a key at all.
147 pub(crate) has_truth_key: bool,
148 /// Whether the emitted key was exactly right. Only meaningful when
149 /// `has_truth_key`; a keyless loop has no key to match.
150 pub(crate) exact: bool,
151 }
152
153 /// What a gate at one threshold would deliver.
154 pub(crate) struct GatePoint {
155 pub(crate) threshold: f64,
156 /// Keys kept on loops annotators say are keyed.
157 pub(crate) kept_true: usize,
158 /// Of those, how many were exactly right.
159 pub(crate) kept_exact: usize,
160 /// Keys kept on loops annotators say are keyless. Every one is spurious.
161 pub(crate) kept_spurious: usize,
162 }
163
164 impl GatePoint {
165 /// Of the keyed loops that had a key emitted, the share this gate still
166 /// lets through. What the gate costs.
167 pub(crate) fn retention(&self, total_true: usize) -> Option<f64> {
168 (total_true > 0).then(|| self.kept_true as f64 / total_true as f64)
169 }
170
171 /// The share of spurious keys this gate removes. What the gate buys.
172 pub(crate) fn suppression(&self, total_spurious: usize) -> Option<f64> {
173 (total_spurious > 0).then(|| 1.0 - self.kept_spurious as f64 / total_spurious as f64)
174 }
175
176 /// Of every key that survives the gate, the share that is exactly right.
177 /// A spurious key counts against this, which is the point: it is what a
178 /// user reading a key field would experience.
179 pub(crate) fn precision(&self) -> Option<f64> {
180 let kept = self.kept_true + self.kept_spurious;
181 (kept > 0).then(|| self.kept_exact as f64 / kept as f64)
182 }
183 }
184
185 /// Price a gate at each of `steps` quantiles of the observed confidences.
186 ///
187 /// `calls` is every loop the detector emitted a key for, keyed and keyless
188 /// alike. A loop with no emitted key is not a gate decision and must be left
189 /// out, or the sweep reports the gate suppressing keys it was never offered.
190 pub(crate) fn gate_sweep(calls: &[KeyCall], steps: usize) -> Vec<GatePoint> {
191 if calls.is_empty() || steps == 0 {
192 return Vec::new();
193 }
194 let mut confs: Vec<f64> = calls.iter().map(|c| c.conf).collect();
195 confs.sort_by(f64::total_cmp);
196
197 let mut thresholds: Vec<f64> = (0..steps)
198 .map(|s| confs[(confs.len() - 1) * s / steps])
199 .collect();
200 thresholds.dedup();
201
202 thresholds
203 .into_iter()
204 .map(|threshold| {
205 let kept = calls.iter().filter(|c| c.conf >= threshold);
206 let mut point = GatePoint {
207 threshold,
208 kept_true: 0,
209 kept_exact: 0,
210 kept_spurious: 0,
211 };
212 for call in kept {
213 if call.has_truth_key {
214 point.kept_true += 1;
215 point.kept_exact += usize::from(call.exact);
216 } else {
217 point.kept_spurious += 1;
218 }
219 }
220 point
221 })
222 .collect()
223 }
224
225 #[cfg(test)]
226 mod tests {
227 use super::{Call, KeyCall, bands, discrimination, gate_sweep, tied};
228
229 fn call(conf: f64, correct: bool) -> Call {
230 Call { conf, correct }
231 }
232
233 #[test]
234 fn a_perfect_ranking_scores_one() {
235 let calls = [call(0.1, false), call(0.2, false), call(0.9, true)];
236 assert_eq!(discrimination(&calls), Some(1.0));
237 }
238
239 #[test]
240 fn a_reversed_ranking_scores_zero() {
241 let calls = [call(0.9, false), call(0.1, true)];
242 assert_eq!(discrimination(&calls), Some(0.0));
243 }
244
245 #[test]
246 fn a_confidence_that_says_nothing_scores_half() {
247 // Every call carries the same number, so it cannot rank anything.
248 let calls = [
249 call(0.7, true),
250 call(0.7, false),
251 call(0.7, true),
252 call(0.7, false),
253 ];
254 assert_eq!(discrimination(&calls), Some(0.5));
255 }
256
257 #[test]
258 fn one_sided_evidence_is_no_score_rather_than_a_default() {
259 // Everything correct: nothing to rank against, and reporting 0.5 or 1.0
260 // would both read as a measurement that was made.
261 assert!(discrimination(&[call(0.1, true), call(0.9, true)]).is_none());
262 assert!(discrimination(&[]).is_none());
263 }
264
265 #[test]
266 fn bands_split_by_population_and_carry_their_range() {
267 let calls: Vec<Call> = (0..10).map(|i| call(i as f64 / 10.0, i >= 5)).collect();
268 let out = bands(&calls, 2);
269 assert_eq!(out.len(), 2);
270 assert_eq!((out[0].n, out[0].correct), (5, 0));
271 assert_eq!((out[1].n, out[1].correct), (5, 5));
272 assert!(tied(out[0].low, 0.0));
273 assert!(tied(out[1].high, 0.9));
274 }
275
276 #[test]
277 fn a_tie_does_not_straddle_a_band_boundary() {
278 // Four of one value and two of another into three bands: splitting the
279 // block of 0.5s would have two bands both reporting 0.5..0.5.
280 let calls: Vec<Call> = [0.5, 0.5, 0.5, 0.5, 0.8, 0.9]
281 .iter()
282 .map(|c| call(*c, true))
283 .collect();
284 let out = bands(&calls, 3);
285 for pair in out.windows(2) {
286 assert!(pair[0].high < pair[1].low, "bands overlap");
287 }
288 assert_eq!(out.iter().map(|b| b.n).sum::<usize>(), calls.len());
289 }
290
291 #[test]
292 fn a_gate_trades_spurious_keys_against_real_ones() {
293 let calls = [
294 KeyCall {
295 conf: 0.2,
296 has_truth_key: false,
297 exact: false,
298 },
299 KeyCall {
300 conf: 0.4,
301 has_truth_key: false,
302 exact: false,
303 },
304 KeyCall {
305 conf: 0.6,
306 has_truth_key: true,
307 exact: false,
308 },
309 KeyCall {
310 conf: 0.8,
311 has_truth_key: true,
312 exact: true,
313 },
314 ];
315 let sweep = gate_sweep(&calls, 4);
316 let open = &sweep[0];
317 assert_eq!(open.kept_spurious, 2);
318 assert_eq!(open.suppression(2), Some(0.0));
319 assert_eq!(open.retention(2), Some(1.0));
320 // Wide open, half the emitted keys are on keyless loops and only one of
321 // four is exactly right.
322 assert_eq!(open.precision(), Some(0.25));
323
324 let tightest = sweep.last().expect("a sweep has rows");
325 assert!(tightest.threshold >= 0.6);
326 assert_eq!(tightest.kept_spurious, 0);
327 assert_eq!(tightest.suppression(2), Some(1.0));
328 }
329
330 #[test]
331 fn a_gate_sweep_over_one_confidence_is_a_single_row() {
332 // Every key emitted with the same confidence: no threshold separates
333 // anything, and ten identical rows would read as ten measurements.
334 let calls: Vec<KeyCall> = (0..8)
335 .map(|i| KeyCall {
336 conf: 0.7,
337 has_truth_key: i % 2 == 0,
338 exact: false,
339 })
340 .collect();
341 assert_eq!(gate_sweep(&calls, 10).len(), 1);
342 }
343 }
344