|
1 |
+ |
//! Does a detector's confidence predict whether it was right?
|
|
2 |
+ |
//!
|
|
3 |
+ |
//! `detect_bpm_key` returns a `bpm_confidence` and a `key_confidence` beside
|
|
4 |
+ |
//! every answer, and nothing has ever checked that they mean anything. Both are
|
|
5 |
+ |
//! read as facts downstream, and a confidence that does not rank correct calls
|
|
6 |
+ |
//! above wrong ones is worse than no confidence at all: it is a number the UI
|
|
7 |
+ |
//! shows and a gate could be built on, carrying no information.
|
|
8 |
+ |
//!
|
|
9 |
+ |
//! Two different questions, answered by two different shapes here.
|
|
10 |
+ |
//!
|
|
11 |
+ |
//! [`discrimination`] and [`bands`] ask whether the number ranks at all, over
|
|
12 |
+ |
//! calls that have a right answer to be measured against. Discrimination is one
|
|
13 |
+ |
//! scalar and bands are where it came from, because a scalar near 0.5 does not
|
|
14 |
+ |
//! say whether the confidence is flat or non-monotone.
|
|
15 |
+ |
//!
|
|
16 |
+ |
//! [`gate_sweep`] asks the design question the spurious-key problem needs: the
|
|
17 |
+ |
//! detector emits a key on 90.8% of loops annotators call keyless, so a gate
|
|
18 |
+ |
//! that only lets a key through above some confidence has to be priced. Its
|
|
19 |
+ |
//! population is not the scorable loops -- it is every loop, because what a
|
|
20 |
+ |
//! gate costs is measured on the keyed ones and what it buys is measured on the
|
|
21 |
+ |
//! keyless ones.
|
|
22 |
+ |
//!
|
|
23 |
+ |
//! Thresholds and band edges are taken from the observed quantiles rather than
|
|
24 |
+ |
//! from a fixed grid. A detector whose confidences all sit between 0.62 and
|
|
25 |
+ |
//! 0.71 would give an equal-width sweep one populated row and nine empty ones,
|
|
26 |
+ |
//! and that is a plausible thing for these confidences to do.
|
|
27 |
+ |
|
|
28 |
+ |
/// Whether two confidences are the same number.
|
|
29 |
+ |
///
|
|
30 |
+ |
/// Exact equality, and deliberately so: a tie here means the detector returned
|
|
31 |
+ |
/// the identical value twice, not two values close enough to round together.
|
|
32 |
+ |
/// Ties drive midranks and band boundaries, and a tolerance would merge
|
|
33 |
+ |
/// distinct confidences into one block and change both.
|
|
34 |
+ |
#[expect(
|
|
35 |
+ |
clippy::float_cmp,
|
|
36 |
+ |
reason = "a tie is bit-equality; a tolerance would merge distinct confidences"
|
|
37 |
+ |
)]
|
|
38 |
+ |
fn tied(a: f64, b: f64) -> bool {
|
|
39 |
+ |
a == b
|
|
40 |
+ |
}
|
|
41 |
+ |
|
|
42 |
+ |
/// One detection with a right answer to be scored against.
|
|
43 |
+ |
#[derive(Clone, Copy)]
|
|
44 |
+ |
pub(crate) struct Call {
|
|
45 |
+ |
pub(crate) conf: f64,
|
|
46 |
+ |
pub(crate) correct: bool,
|
|
47 |
+ |
}
|
|
48 |
+ |
|
|
49 |
+ |
/// Probability that a randomly chosen correct call carries a higher confidence
|
|
50 |
+ |
/// than a randomly chosen wrong one, ties counting half. `None` when everything
|
|
51 |
+ |
/// is right or everything is wrong, which is no evidence either way rather than
|
|
52 |
+ |
/// a score of any particular value.
|
|
53 |
+ |
///
|
|
54 |
+ |
/// 0.5 is a confidence that ranks no better than a coin. This is AUROC, by way
|
|
55 |
+ |
/// of the Mann-Whitney U it equals; computed off the ranks rather than by
|
|
56 |
+ |
/// counting pairs so 2,755 loops does not become 3.8M comparisons.
|
|
57 |
+ |
pub(crate) fn discrimination(calls: &[Call]) -> Option<f64> {
|
|
58 |
+ |
let correct = calls.iter().filter(|c| c.correct).count();
|
|
59 |
+ |
let wrong = calls.len() - correct;
|
|
60 |
+ |
if correct == 0 || wrong == 0 {
|
|
61 |
+ |
return None;
|
|
62 |
+ |
}
|
|
63 |
+ |
|
|
64 |
+ |
let mut sorted: Vec<Call> = calls.to_vec();
|
|
65 |
+ |
sorted.sort_by(|a, b| a.conf.total_cmp(&b.conf));
|
|
66 |
+ |
|
|
67 |
+ |
// Midranks, so a block of equal confidences contributes the half-credit
|
|
68 |
+ |
// ties are supposed to get instead of whatever order the sort left them in.
|
|
69 |
+ |
let mut rank_sum = 0.0;
|
|
70 |
+ |
let mut i = 0;
|
|
71 |
+ |
while i < sorted.len() {
|
|
72 |
+ |
let mut j = i;
|
|
73 |
+ |
while j + 1 < sorted.len() && tied(sorted[j + 1].conf, sorted[i].conf) {
|
|
74 |
+ |
j += 1;
|
|
75 |
+ |
}
|
|
76 |
+ |
// Ranks are 1-based over i..=j, so their mean is this.
|
|
77 |
+ |
let midrank = (i + j) as f64 / 2.0 + 1.0;
|
|
78 |
+ |
rank_sum += sorted[i..=j].iter().filter(|c| c.correct).count() as f64 * midrank;
|
|
79 |
+ |
i = j + 1;
|
|
80 |
+ |
}
|
|
81 |
+ |
|
|
82 |
+ |
let (n_c, n_w) = (correct as f64, wrong as f64);
|
|
83 |
+ |
Some((rank_sum - n_c * (n_c + 1.0) / 2.0) / (n_c * n_w))
|
|
84 |
+ |
}
|
|
85 |
+ |
|
|
86 |
+ |
/// One confidence band and how often the detector was right inside it.
|
|
87 |
+ |
pub(crate) struct Band {
|
|
88 |
+ |
pub(crate) low: f64,
|
|
89 |
+ |
pub(crate) high: f64,
|
|
90 |
+ |
pub(crate) n: usize,
|
|
91 |
+ |
pub(crate) correct: usize,
|
|
92 |
+ |
}
|
|
93 |
+ |
|
|
94 |
+ |
impl Band {
|
|
95 |
+ |
pub(crate) fn accuracy(&self) -> f64 {
|
|
96 |
+ |
self.correct as f64 / self.n as f64
|
|
97 |
+ |
}
|
|
98 |
+ |
}
|
|
99 |
+ |
|
|
100 |
+ |
/// Split `calls` into up to `groups` bands of roughly equal population and
|
|
101 |
+ |
/// report accuracy in each. Calibration is the claim that accuracy rises with
|
|
102 |
+ |
/// the band, so this is the table that claim is read off.
|
|
103 |
+ |
///
|
|
104 |
+ |
/// Equal population rather than equal width: a band nobody landed in reports a
|
|
105 |
+ |
/// ratio over zero, and enough of those turn the table into a row of blanks
|
|
106 |
+ |
/// with the whole detector inside one cell.
|
|
107 |
+ |
pub(crate) fn bands(calls: &[Call], groups: usize) -> Vec<Band> {
|
|
108 |
+ |
if calls.is_empty() || groups == 0 {
|
|
109 |
+ |
return Vec::new();
|
|
110 |
+ |
}
|
|
111 |
+ |
let mut sorted: Vec<Call> = calls.to_vec();
|
|
112 |
+ |
sorted.sort_by(|a, b| a.conf.total_cmp(&b.conf));
|
|
113 |
+ |
|
|
114 |
+ |
let mut out = Vec::new();
|
|
115 |
+ |
let mut start = 0;
|
|
116 |
+ |
for g in 0..groups {
|
|
117 |
+ |
if start >= sorted.len() {
|
|
118 |
+ |
break;
|
|
119 |
+ |
}
|
|
120 |
+ |
let mut end = (sorted.len() * (g + 1)) / groups;
|
|
121 |
+ |
// A tie must not straddle a boundary, or two bands report overlapping
|
|
122 |
+ |
// confidence ranges and neither one means anything.
|
|
123 |
+ |
while end < sorted.len() && end > start && tied(sorted[end - 1].conf, sorted[end].conf) {
|
|
124 |
+ |
end += 1;
|
|
125 |
+ |
}
|
|
126 |
+ |
if end <= start {
|
|
127 |
+ |
continue;
|
|
128 |
+ |
}
|
|
129 |
+ |
let slice = &sorted[start..end];
|
|
130 |
+ |
out.push(Band {
|
|
131 |
+ |
low: slice[0].conf,
|
|
132 |
+ |
high: slice[slice.len() - 1].conf,
|
|
133 |
+ |
n: slice.len(),
|
|
134 |
+ |
correct: slice.iter().filter(|c| c.correct).count(),
|
|
135 |
+ |
});
|
|
136 |
+ |
start = end;
|
|
137 |
+ |
}
|
|
138 |
+ |
out
|
|
139 |
+ |
}
|
|
140 |
+ |
|
|
141 |
+ |
/// One loop's evidence for the key gate.
|
|
142 |
+ |
#[derive(Clone, Copy)]
|
|
143 |
+ |
pub(crate) struct KeyCall {
|
|
144 |
+ |
/// The confidence the detector attached to the key it emitted.
|
|
145 |
+ |
pub(crate) conf: f64,
|
|
146 |
+ |
/// Whether the annotators say this loop has a key at all.
|
|
147 |
+ |
pub(crate) has_truth_key: bool,
|
|
148 |
+ |
/// Whether the emitted key was exactly right. Only meaningful when
|
|
149 |
+ |
/// `has_truth_key`; a keyless loop has no key to match.
|
|
150 |
+ |
pub(crate) exact: bool,
|
|
151 |
+ |
}
|
|
152 |
+ |
|
|
153 |
+ |
/// What a gate at one threshold would deliver.
|
|
154 |
+ |
pub(crate) struct GatePoint {
|
|
155 |
+ |
pub(crate) threshold: f64,
|
|
156 |
+ |
/// Keys kept on loops annotators say are keyed.
|
|
157 |
+ |
pub(crate) kept_true: usize,
|
|
158 |
+ |
/// Of those, how many were exactly right.
|
|
159 |
+ |
pub(crate) kept_exact: usize,
|
|
160 |
+ |
/// Keys kept on loops annotators say are keyless. Every one is spurious.
|
|
161 |
+ |
pub(crate) kept_spurious: usize,
|
|
162 |
+ |
}
|
|
163 |
+ |
|
|
164 |
+ |
impl GatePoint {
|
|
165 |
+ |
/// Of the keyed loops that had a key emitted, the share this gate still
|
|
166 |
+ |
/// lets through. What the gate costs.
|
|
167 |
+ |
pub(crate) fn retention(&self, total_true: usize) -> Option<f64> {
|
|
168 |
+ |
(total_true > 0).then(|| self.kept_true as f64 / total_true as f64)
|
|
169 |
+ |
}
|
|
170 |
+ |
|
|
171 |
+ |
/// The share of spurious keys this gate removes. What the gate buys.
|
|
172 |
+ |
pub(crate) fn suppression(&self, total_spurious: usize) -> Option<f64> {
|
|
173 |
+ |
(total_spurious > 0).then(|| 1.0 - self.kept_spurious as f64 / total_spurious as f64)
|
|
174 |
+ |
}
|
|
175 |
+ |
|
|
176 |
+ |
/// Of every key that survives the gate, the share that is exactly right.
|
|
177 |
+ |
/// A spurious key counts against this, which is the point: it is what a
|
|
178 |
+ |
/// user reading a key field would experience.
|
|
179 |
+ |
pub(crate) fn precision(&self) -> Option<f64> {
|
|
180 |
+ |
let kept = self.kept_true + self.kept_spurious;
|
|
181 |
+ |
(kept > 0).then(|| self.kept_exact as f64 / kept as f64)
|
|
182 |
+ |
}
|
|
183 |
+ |
}
|
|
184 |
+ |
|
|
185 |
+ |
/// Price a gate at each of `steps` quantiles of the observed confidences.
|
|
186 |
+ |
///
|
|
187 |
+ |
/// `calls` is every loop the detector emitted a key for, keyed and keyless
|
|
188 |
+ |
/// alike. A loop with no emitted key is not a gate decision and must be left
|
|
189 |
+ |
/// out, or the sweep reports the gate suppressing keys it was never offered.
|
|
190 |
+ |
pub(crate) fn gate_sweep(calls: &[KeyCall], steps: usize) -> Vec<GatePoint> {
|
|
191 |
+ |
if calls.is_empty() || steps == 0 {
|
|
192 |
+ |
return Vec::new();
|
|
193 |
+ |
}
|
|
194 |
+ |
let mut confs: Vec<f64> = calls.iter().map(|c| c.conf).collect();
|
|
195 |
+ |
confs.sort_by(f64::total_cmp);
|
|
196 |
+ |
|
|
197 |
+ |
let mut thresholds: Vec<f64> = (0..steps)
|
|
198 |
+ |
.map(|s| confs[(confs.len() - 1) * s / steps])
|
|
199 |
+ |
.collect();
|
|
200 |
+ |
thresholds.dedup();
|
|
201 |
+ |
|
|
202 |
+ |
thresholds
|
|
203 |
+ |
.into_iter()
|
|
204 |
+ |
.map(|threshold| {
|
|
205 |
+ |
let kept = calls.iter().filter(|c| c.conf >= threshold);
|
|
206 |
+ |
let mut point = GatePoint {
|
|
207 |
+ |
threshold,
|
|
208 |
+ |
kept_true: 0,
|
|
209 |
+ |
kept_exact: 0,
|
|
210 |
+ |
kept_spurious: 0,
|
|
211 |
+ |
};
|
|
212 |
+ |
for call in kept {
|
|
213 |
+ |
if call.has_truth_key {
|
|
214 |
+ |
point.kept_true += 1;
|
|
215 |
+ |
point.kept_exact += usize::from(call.exact);
|
|
216 |
+ |
} else {
|
|
217 |
+ |
point.kept_spurious += 1;
|
|
218 |
+ |
}
|
|
219 |
+ |
}
|
|
220 |
+ |
point
|
|
221 |
+ |
})
|
|
222 |
+ |
.collect()
|
|
223 |
+ |
}
|
|
224 |
+ |
|
|
225 |
+ |
#[cfg(test)]
|
|
226 |
+ |
mod tests {
|
|
227 |
+ |
use super::{Call, KeyCall, bands, discrimination, gate_sweep, tied};
|
|
228 |
+ |
|
|
229 |
+ |
fn call(conf: f64, correct: bool) -> Call {
|
|
230 |
+ |
Call { conf, correct }
|
|
231 |
+ |
}
|
|
232 |
+ |
|
|
233 |
+ |
#[test]
|
|
234 |
+ |
fn a_perfect_ranking_scores_one() {
|
|
235 |
+ |
let calls = [call(0.1, false), call(0.2, false), call(0.9, true)];
|
|
236 |
+ |
assert_eq!(discrimination(&calls), Some(1.0));
|
|
237 |
+ |
}
|
|
238 |
+ |
|
|
239 |
+ |
#[test]
|
|
240 |
+ |
fn a_reversed_ranking_scores_zero() {
|
|
241 |
+ |
let calls = [call(0.9, false), call(0.1, true)];
|
|
242 |
+ |
assert_eq!(discrimination(&calls), Some(0.0));
|
|
243 |
+ |
}
|
|
244 |
+ |
|
|
245 |
+ |
#[test]
|
|
246 |
+ |
fn a_confidence_that_says_nothing_scores_half() {
|
|
247 |
+ |
// Every call carries the same number, so it cannot rank anything.
|
|
248 |
+ |
let calls = [
|
|
249 |
+ |
call(0.7, true),
|
|
250 |
+ |
call(0.7, false),
|
|
251 |
+ |
call(0.7, true),
|
|
252 |
+ |
call(0.7, false),
|
|
253 |
+ |
];
|
|
254 |
+ |
assert_eq!(discrimination(&calls), Some(0.5));
|
|
255 |
+ |
}
|
|
256 |
+ |
|
|
257 |
+ |
#[test]
|
|
258 |
+ |
fn one_sided_evidence_is_no_score_rather_than_a_default() {
|
|
259 |
+ |
// Everything correct: nothing to rank against, and reporting 0.5 or 1.0
|
|
260 |
+ |
// would both read as a measurement that was made.
|
|
261 |
+ |
assert!(discrimination(&[call(0.1, true), call(0.9, true)]).is_none());
|
|
262 |
+ |
assert!(discrimination(&[]).is_none());
|
|
263 |
+ |
}
|
|
264 |
+ |
|
|
265 |
+ |
#[test]
|
|
266 |
+ |
fn bands_split_by_population_and_carry_their_range() {
|
|
267 |
+ |
let calls: Vec<Call> = (0..10).map(|i| call(i as f64 / 10.0, i >= 5)).collect();
|
|
268 |
+ |
let out = bands(&calls, 2);
|
|
269 |
+ |
assert_eq!(out.len(), 2);
|
|
270 |
+ |
assert_eq!((out[0].n, out[0].correct), (5, 0));
|
|
271 |
+ |
assert_eq!((out[1].n, out[1].correct), (5, 5));
|
|
272 |
+ |
assert!(tied(out[0].low, 0.0));
|
|
273 |
+ |
assert!(tied(out[1].high, 0.9));
|
|
274 |
+ |
}
|
|
275 |
+ |
|
|
276 |
+ |
#[test]
|
|
277 |
+ |
fn a_tie_does_not_straddle_a_band_boundary() {
|
|
278 |
+ |
// Four of one value and two of another into three bands: splitting the
|
|
279 |
+ |
// block of 0.5s would have two bands both reporting 0.5..0.5.
|
|
280 |
+ |
let calls: Vec<Call> = [0.5, 0.5, 0.5, 0.5, 0.8, 0.9]
|
|
281 |
+ |
.iter()
|
|
282 |
+ |
.map(|c| call(*c, true))
|
|
283 |
+ |
.collect();
|
|
284 |
+ |
let out = bands(&calls, 3);
|
|
285 |
+ |
for pair in out.windows(2) {
|
|
286 |
+ |
assert!(pair[0].high < pair[1].low, "bands overlap");
|
|
287 |
+ |
}
|
|
288 |
+ |
assert_eq!(out.iter().map(|b| b.n).sum::<usize>(), calls.len());
|
|
289 |
+ |
}
|
|
290 |
+ |
|
|
291 |
+ |
#[test]
|
|
292 |
+ |
fn a_gate_trades_spurious_keys_against_real_ones() {
|
|
293 |
+ |
let calls = [
|
|
294 |
+ |
KeyCall {
|
|
295 |
+ |
conf: 0.2,
|
|
296 |
+ |
has_truth_key: false,
|
|
297 |
+ |
exact: false,
|
|
298 |
+ |
},
|
|
299 |
+ |
KeyCall {
|
|
300 |
+ |
conf: 0.4,
|
|
301 |
+ |
has_truth_key: false,
|
|
302 |
+ |
exact: false,
|
|
303 |
+ |
},
|
|
304 |
+ |
KeyCall {
|
|
305 |
+ |
conf: 0.6,
|
|
306 |
+ |
has_truth_key: true,
|
|
307 |
+ |
exact: false,
|
|
308 |
+ |
},
|
|
309 |
+ |
KeyCall {
|
|
310 |
+ |
conf: 0.8,
|
|
311 |
+ |
has_truth_key: true,
|
|
312 |
+ |
exact: true,
|
|
313 |
+ |
},
|
|
314 |
+ |
];
|
|
315 |
+ |
let sweep = gate_sweep(&calls, 4);
|
|
316 |
+ |
let open = &sweep[0];
|
|
317 |
+ |
assert_eq!(open.kept_spurious, 2);
|
|
318 |
+ |
assert_eq!(open.suppression(2), Some(0.0));
|
|
319 |
+ |
assert_eq!(open.retention(2), Some(1.0));
|
|
320 |
+ |
// Wide open, half the emitted keys are on keyless loops and only one of
|
|
321 |
+ |
// four is exactly right.
|
|
322 |
+ |
assert_eq!(open.precision(), Some(0.25));
|
|
323 |
+ |
|
|
324 |
+ |
let tightest = sweep.last().expect("a sweep has rows");
|
|
325 |
+ |
assert!(tightest.threshold >= 0.6);
|
|
326 |
+ |
assert_eq!(tightest.kept_spurious, 0);
|
|
327 |
+ |
assert_eq!(tightest.suppression(2), Some(1.0));
|
|
328 |
+ |
}
|
|
329 |
+ |
|
|
330 |
+ |
#[test]
|
|
331 |
+ |
fn a_gate_sweep_over_one_confidence_is_a_single_row() {
|
|
332 |
+ |
// Every key emitted with the same confidence: no threshold separates
|
|
333 |
+ |
// anything, and ten identical rows would read as ten measurements.
|
|
334 |
+ |
let calls: Vec<KeyCall> = (0..8)
|
|
335 |
+ |
.map(|i| KeyCall {
|
|
336 |
+ |
conf: 0.7,
|
|
337 |
+ |
has_truth_key: i % 2 == 0,
|
|
338 |
+ |
exact: false,
|
|
339 |
+ |
})
|
|
340 |
+ |
.collect();
|
|
341 |
+ |
assert_eq!(gate_sweep(&calls, 10).len(), 1);
|
|
342 |
+ |
}
|
|
343 |
+ |
}
|