Skip to main content

max / audiofiles

60.3 KB · 1601 lines History Blame Raw
1 //! Does the layer give the same answer twice? The queue-stability meter.
2 //!
3 //! [`crate::layer_eval`] asks whether an answer is right. This asks whether it is
4 //! the same answer next week. For a layer whose model is the user's own library
5 //! those are independent properties, and only the second one is measured here: a
6 //! consistently wrong answer is perfectly stable, so nothing below can be inferred
7 //! from the 95.5% / 97.4% family-resolution accuracy figures, and nothing below
8 //! substitutes for them.
9 //!
10 //! # Why this is an attention gate and not a safety gate
11 //!
12 //! Exemplars from outside the user's own labelling are **suggest-only** (decided
13 //! 2026-08-07 for the bundled layer, and inherited by imported layers when that
14 //! was retired): they never auto-apply, they populate a review queue. That kills
15 //! the failure this
16 //! measurement was originally filed against. The apply path is monotonic
17 //! (`apply_policy` skips tags the sample already has, `apply_tag_sourced` is
18 //! INSERT OR IGNORE, only `remove_tags_by_source` removes), so a changed answer
19 //! used to mean the sample kept its old tag *and* gained the new one, permanently.
20 //! Nothing is written unasked now, so that cannot happen.
21 //!
22 //! What survives is cheaper and still real: a queue whose contents reshuffle
23 //! between runs spends the user's attention twice. Someone who read 340 low
24 //! suggestions last month should not be handed a substantially different 340 this
25 //! month for the same unchanged samples. So the unit of measurement is the
26 //! **queue entry**, not the written tag: a sample's answer is its top-scoring tag
27 //! if that tag clears the review threshold, and `silent` otherwise.
28 //!
29 //! # The bar, written before the first run
30 //!
31 //! Same discipline as the ship gate, and for the same reason: a number chosen
32 //! after reading one is not a bar. See [`BAR`].
33 //!
34 //! # What varies, and why those four
35 //!
36 //! Each measurement holds the probe set fixed and varies one property of the
37 //! index, because a flip rate is meaningless without saying what moved.
38 //!
39 //! 1. [`composition`] — same size, different class mix. The user whose library is
40 //! mostly kicks and the user whose library is mostly cymbals are running the
41 //! same code against different models.
42 //! 2. [`size`] — same mix, growing index. A library grows by accretion, so this is
43 //! the shape of every real user's second month.
44 //! 3. [`deployment_shape`] — the mixed index the app actually builds: the user's
45 //! own labels at 1.0 beside an imported layer at `DEFAULT_IMPORT_WEIGHT` 0.5.
46 //! Never measured before this module, and the most decision-relevant of the
47 //! four. Every earlier number came off a uniform-weight index, where the weight
48 //! is a constant multiplier inside a sum divided by its own total and therefore
49 //! cancels. It does not cancel here, and neither does the second-order effect:
50 //! `build_index` fits the standardization params over local and imported
51 //! exemplars together, so a user's labels move the space distances are measured
52 //! in. This is the difference between a queue that helps and one that repeats
53 //! what the user already knows.
54 //! 4. [`feedback`] — the queue changes its own inputs. An accepted suggestion
55 //! becomes a user label at weight 1.0 and re-enters the index, so working
56 //! through the queue rewrites the rest of it. Converges or oscillates is a
57 //! question no other measurement here can answer.
58 //!
59 //! Usage: `cargo run --release -p audiofiles-bench -- layer-stability`
60 //! Env: `AF_BENCH_CORPUS`, `AF_BENCH_VAULT`, `AF_BENCH_EVAL_K` (first entry wins,
61 //! default `DEFAULT_K`), `AF_BENCH_EVAL_LABELS`, `AF_BENCH_STABILITY_PROBE`
62 //! (probe fraction denominator, default 5), `AF_BENCH_JSON`.
63
64 use std::collections::{BTreeMap, BTreeSet, HashMap};
65 use std::path::Path;
66
67 use audiofiles_core::analysis::config::AnalysisConfig;
68 use audiofiles_core::analysis::exemplar::{self, DEFAULT_K, DEFAULT_REVIEW_THRESHOLD};
69 use audiofiles_core::analysis::features::FEATURE_VERSION;
70 use audiofiles_core::rules::{RuleContext, RuleField};
71 use audiofiles_core::{rules, starter_rules};
72
73 use crate::families::{self, LabelSpace};
74 use crate::labelled;
75 use crate::report::Report;
76 use crate::rows::{self, Row};
77
78 /// The bar, stated before the first run.
79 ///
80 /// Per-run family-label flip rate on a fixed held-out probe set, measured at the
81 /// deployment weight, counting only flips between two answers that both cleared
82 /// the review threshold. A sample moving between suggested and silent costs the
83 /// user nothing (the queue is shorter or longer, not wrong), so it is reported
84 /// separately as churn rather than counted here.
85 ///
86 /// 2% is proposed rather than settled: on a 170-sample probe set it is three
87 /// samples, which is the resolution this corpus supports and not a claim that 2%
88 /// is where a user stops noticing. Max accepts or moves it.
89 const BAR: f64 = 0.02;
90
91 /// A sample either has a queue entry or it does not.
92 ///
93 /// The threshold is [`DEFAULT_REVIEW_THRESHOLD`] and not the auto threshold on
94 /// purpose: under suggest-only nothing auto-applies, so the review threshold is
95 /// the only line that decides whether the user ever sees the answer.
96 #[derive(Clone, PartialEq, Eq)]
97 enum Answer {
98 Silent,
99 Tag(String),
100 }
101
102 impl Answer {
103 fn of(index: &exemplar::ExemplarIndex, vector: &[f64], k: usize) -> Self {
104 index
105 .score(vector, k, None)
106 .first()
107 .filter(|s| s.score >= DEFAULT_REVIEW_THRESHOLD)
108 .map_or(Self::Silent, |s| Self::Tag(s.tag.clone()))
109 }
110
111 fn tag(&self) -> Option<&str> {
112 match self {
113 Self::Silent => None,
114 Self::Tag(t) => Some(t),
115 }
116 }
117 }
118
119 /// One index variant's answer for every probe sample, in probe order.
120 type Answers = Vec<Answer>;
121
122 /// How two runs' queues differ.
123 struct Churn {
124 /// Probe samples suggested in both runs. The denominator of [`Self::flip_rate`].
125 both: usize,
126 /// Suggested in both, and the tag changed. What the bar is set against.
127 flips: usize,
128 /// Silent then suggested: the queue grew.
129 appeared: usize,
130 /// Suggested then silent: the queue shrank.
131 vanished: usize,
132 }
133
134 impl Churn {
135 fn between(before: &Answers, after: &Answers) -> Self {
136 let mut churn = Self {
137 both: 0,
138 flips: 0,
139 appeared: 0,
140 vanished: 0,
141 };
142 for (x, y) in before.iter().zip(after) {
143 match (x.tag(), y.tag()) {
144 (Some(was), Some(now)) => {
145 churn.both += 1;
146 if was != now {
147 churn.flips += 1;
148 }
149 }
150 (None, Some(_)) => churn.appeared += 1,
151 (Some(_), None) => churn.vanished += 1,
152 (None, None) => {}
153 }
154 }
155 churn
156 }
157
158 /// `None` when nothing was suggested in both runs, which is not a flip rate of
159 /// zero: it is a pair of runs with no overlapping queue to compare.
160 fn flip_rate(&self) -> Option<f64> {
161 (self.both > 0).then(|| self.flips as f64 / self.both as f64)
162 }
163
164 /// The flip rate, but only when enough samples stood behind it to mean
165 /// anything. See [`MIN_COMPARABLE`].
166 fn comparable_flip_rate(&self) -> Option<f64> {
167 self.flip_rate().filter(|_| self.both >= MIN_COMPARABLE)
168 }
169
170 fn passes(&self) -> bool {
171 self.flip_rate().is_some_and(|r| r <= BAR)
172 }
173 }
174
175 /// Fewest overlapping queue entries a pair of runs needs before its flip rate
176 /// counts toward a verdict.
177 ///
178 /// Same job as `layer_eval`'s `min_support` and learned the same way. The first
179 /// run of [`feedback`] ended with seven samples still open and two of them
180 /// flipping, which is 28.6% and became the worst figure in the whole report. It
181 /// is one sample either way. Rates over thin tails are still printed — a tail
182 /// that thrashes is worth seeing — but they do not decide a pass.
183 const MIN_COMPARABLE: usize = 30;
184
185 fn pct(v: Option<f64>) -> String {
186 v.map_or_else(|| "-".to_string(), |x| format!("{:.1}%", x * 100.0))
187 }
188
189 fn round4(v: f64) -> f64 {
190 (v * 10_000.0).round() / 10_000.0
191 }
192
193 /// Score every probe row against one index.
194 fn answers(index: &exemplar::ExemplarIndex, probe: &[&Row], k: usize) -> Answers {
195 probe
196 .iter()
197 .map(|r| Answer::of(index, &r.vector, k))
198 .collect()
199 }
200
201 /// Build an index from `local` (weight 1.0) plus `imported` (the layer weight) and
202 /// score the probe against it.
203 fn run_variant(local: &[&Row], imported: &[&Row], probe: &[&Row], k: usize, what: &str) -> Answers {
204 let db = match rows::mixed_db(local, imported) {
205 Ok(db) => db,
206 Err(e) => {
207 eprintln!("{what}: {e}");
208 std::process::exit(1);
209 }
210 };
211 let index = match exemplar::build_index(&db) {
212 Ok(i) => i,
213 Err(e) => {
214 eprintln!("{what}: build_index: {e}");
215 std::process::exit(1);
216 }
217 };
218 answers(&index, probe, k)
219 }
220
221 /// Take `n` rows of class `class` from `pool`, in pool order.
222 ///
223 /// Prefix rather than a sample: pool order is content-hash order, which is already
224 /// arbitrary with respect to the pack a file came from, so this needs no RNG and
225 /// two runs produce the same variants. It also makes the size sweep nested by
226 /// construction, which is the property [`size`] wants: a growing library accretes,
227 /// it does not resample.
228 fn take_class<'a>(pool: &[&'a Row], class: &str, n: usize) -> Vec<&'a Row> {
229 pool.iter()
230 .filter(|r| r.truth.as_deref() == Some(class))
231 .take(n)
232 .copied()
233 .collect()
234 }
235
236 /// Split a pool into two by stratified round-robin, so both halves carry the same
237 /// class mix and neither is a prefix of the other.
238 fn halve<'a>(pool: &[&'a Row]) -> (Vec<&'a Row>, Vec<&'a Row>) {
239 let mut seen: HashMap<&str, usize> = HashMap::new();
240 let mut a = Vec::new();
241 let mut b = Vec::new();
242 for r in pool {
243 let key = r.truth.as_deref().unwrap_or("");
244 let n = seen.entry(key).or_default();
245 if (*n).is_multiple_of(2) {
246 a.push(*r);
247 } else {
248 b.push(*r);
249 }
250 *n += 1;
251 }
252 (a, b)
253 }
254
255 pub(crate) fn run(
256 corpus: &Path,
257 vault: &Path,
258 config: &AnalysisConfig,
259 k: usize,
260 probe_denominator: usize,
261 space: LabelSpace,
262 ) {
263 println!("━━━ CLASSIFIER LAYER STABILITY ━━━");
264 println!();
265 println!(" corpus {}", corpus.display());
266 println!(" scratch {}", vault.display());
267 println!(" features v{FEATURE_VERSION}");
268 println!(" k {k}");
269 println!(" labels {}", space.describe());
270 println!(
271 " answer top-scoring tag at score >= {DEFAULT_REVIEW_THRESHOLD} (the review\n \
272 threshold), else silent. Under suggest-only that line is what decides\n \
273 whether the user ever sees the answer."
274 );
275 println!();
276 println!(
277 " Bar: per-run family-label flip rate <= {:.0}% on the fixed probe set, at",
278 BAR * 100.0
279 );
280 println!(" the deployment weight, counting only flips between two suggested answers.");
281 println!(" Silent <-> suggested is reported as churn and is not a flip: the queue got");
282 println!(" longer or shorter, it did not contradict itself.");
283 println!();
284 println!(" Flip rate and error rate are independent. A consistently wrong answer is");
285 println!(" perfectly stable, so nothing here can be read off the accuracy figures and");
286 println!(" nothing here substitutes for them.");
287 println!();
288
289 let built = match labelled::build_vault(corpus, vault, config) {
290 Ok(v) => v,
291 Err(e) => {
292 eprintln!("corpus: {e}");
293 std::process::exit(1);
294 }
295 };
296 let (all, dropped) = match rows::load_rows(&built.db, space) {
297 Ok(r) => r,
298 Err(e) => {
299 eprintln!("reading the vault back: {e}");
300 std::process::exit(1);
301 }
302 };
303 if dropped.unprojectable > 0 {
304 println!(
305 " {} row(s) carry no label in this space ({}) and are excluded from",
306 dropped.unprojectable,
307 dropped
308 .origins
309 .iter()
310 .cloned()
311 .collect::<Vec<_>>()
312 .join(", ")
313 );
314 println!(" every index and every probe:");
315 println!("{}", families::DROPPED_NOTE);
316 println!();
317 }
318
319 // Probe set: one stratified slice, held out of every index in every
320 // measurement below. Fixed on purpose. A probe set that moved between
321 // variants would mix "the layer changed its mind" with "we asked about
322 // different samples", which is the whole thing this module exists to separate.
323 let fold_of = rows::assign_folds(&all, probe_denominator);
324 let probe: Vec<&Row> = all
325 .iter()
326 .zip(&fold_of)
327 .filter(|(_, f)| **f == Some(0))
328 .map(|(r, _)| r)
329 .collect();
330 let pool: Vec<&Row> = all
331 .iter()
332 .zip(&fold_of)
333 .filter(|(_, f)| **f != Some(0))
334 .map(|(r, _)| r)
335 .collect();
336
337 let classes: Vec<String> = all
338 .iter()
339 .filter_map(|r| r.truth.clone())
340 .collect::<BTreeSet<_>>()
341 .into_iter()
342 .collect();
343
344 println!(
345 " {} row(s) scoreable: {} held out as the fixed probe, {} in the pool every",
346 all.len(),
347 probe.len(),
348 pool.len()
349 );
350 println!(" index is drawn from. No probe sample is ever an exemplar.");
351 for c in &classes {
352 let in_probe = probe
353 .iter()
354 .filter(|r| r.truth.as_deref() == Some(c.as_str()))
355 .count();
356 let in_pool = pool
357 .iter()
358 .filter(|r| r.truth.as_deref() == Some(c.as_str()))
359 .count();
360 println!(
361 " {:<14} probe {:>4} pool {:>4}",
362 families::label_for(space, c),
363 in_probe,
364 in_pool
365 );
366 }
367 println!();
368
369 if probe.is_empty() || pool.is_empty() {
370 eprintln!("nothing to measure: probe or pool is empty");
371 std::process::exit(1);
372 }
373
374 let mut report = Report::new("layer-stability");
375 report.set("label_space", format!("{space:?}"));
376 report.set("k", k);
377 report.set("feat_version", FEATURE_VERSION);
378 report.set("bar_flip_rate", BAR);
379 report.set("review_threshold", DEFAULT_REVIEW_THRESHOLD);
380 report.set("import_weight", rows::IMPORT_WEIGHT);
381 report.set("probe", probe.len());
382 report.set("pool", pool.len());
383
384 review_threshold_note(&pool, &probe, &classes, k, space, &mut report);
385 let miss = value_add_population(&probe, &mut report);
386 let m1 = composition(&pool, &probe, &classes, k, space, &mut report);
387 let m2 = size(&pool, &probe, &classes, k, &mut report);
388 let m3 = deployment_shape(
389 &pool,
390 &probe,
391 k,
392 space,
393 "3. DEPLOYMENT SHAPE",
394 "deploy",
395 &mut report,
396 );
397 let m3b = value_add_deployment(&pool, &miss, k, space, &mut report);
398 let m4 = feedback(&pool, k, space, &mut report);
399
400 let mut outcomes = vec![m1, m2, m3];
401 outcomes.extend(m3b);
402 outcomes.push(m4);
403 verdict(&outcomes, space, &classes, &mut report);
404 report.write();
405 }
406
407 /// Whether the review threshold is doing any work at this label resolution.
408 ///
409 /// It is not, on a two-class corpus, and that has to be said before any number
410 /// below is read. A sample's score for a tag is that tag's share of the
411 /// neighbourhood's kernel weight, so when every exemplar carries exactly one of
412 /// two tags the two scores sum to 1 and the higher one is at or above 0.5 by
413 /// arithmetic. `silent` is unreachable, every probe sample is always a queue
414 /// entry, and the `appeared` / `vanished` columns are structurally zero.
415 ///
416 /// Two consequences a reader must not miss:
417 ///
418 /// - The queue-length half of "does the queue hold still" is untested here. Only
419 /// the contents were measured, because the length cannot move.
420 /// - Any metric defined as "the layer answers where the user's own labels do not"
421 /// is identically zero for the same reason, whatever the layer is worth. That
422 /// is why [`deployment_shape`] measures the layer's contribution as a change in
423 /// the answer and its correctness rather than as an answer appearing.
424 ///
425 /// This resolves at three classes or more and is not a property of the layer, so
426 /// it is a scope note on the corpus rather than a finding about the code.
427 fn review_threshold_note(
428 pool: &[&Row],
429 probe: &[&Row],
430 classes: &[String],
431 k: usize,
432 space: LabelSpace,
433 report: &mut Report,
434 ) {
435 println!("━━━ IS THE REVIEW THRESHOLD BINDING? ━━━");
436 println!();
437
438 let full = run_variant(pool, &[], probe, k, "full pool");
439 let silent = full.iter().filter(|a| a.tag().is_none()).count();
440 println!(
441 " Against the full pool, {} of {} probe samples fall below the {} review",
442 silent,
443 probe.len(),
444 DEFAULT_REVIEW_THRESHOLD
445 );
446 println!(" threshold and stay out of the queue.");
447 println!();
448
449 report.set("silent_at_full_pool", silent);
450 report.set("classes", classes.len());
451
452 if silent == 0 && classes.len() == 2 {
453 println!(" Zero, and it is arithmetic rather than luck. A score is a tag's share of");
454 println!(" the neighbourhood's kernel weight; with every exemplar carrying exactly one");
455 println!(" of two tags the two shares sum to 1, so the larger is always at or above");
456 println!(" 0.5. On this corpus `silent` is unreachable and the threshold decides");
457 println!(" nothing.");
458 println!();
459 println!(" What that costs the measurements below, stated plainly:");
460 println!();
461 println!(" - The queue can only change its CONTENTS, never its LENGTH. The churn");
462 println!(" columns are structurally zero and prove nothing.");
463 println!(" - 'The layer answers where the user's labels do not' is identically zero");
464 println!(" for every layer, good or useless. Measurement 3 therefore reads the");
465 println!(" layer's contribution off the answer and its correctness instead.");
466 println!();
467 println!(" Both resolve at three classes or more, so this is a limit of the drums-");
468 println!(" only corpus and not a property of the layer. Phase C is where it lifts.");
469 report.set("review_threshold_binding", false);
470 } else {
471 report.set("review_threshold_binding", true);
472 }
473 println!();
474 let _ = space;
475 }
476
477 /// A named measurement's answer to "did it pass".
478 struct Outcome {
479 name: &'static str,
480 /// `None` when the measurement could not be run at all, which is not a pass.
481 worst: Option<f64>,
482 note: String,
483 }
484
485 // The value-add population
486
487 /// How much of the probe set the filename rules already answer.
488 ///
489 /// The layer's stated job is libraries whose filenames say nothing.
490 /// `starter_rules` labels 97.7% of this corpus correctly off the name alone, and
491 /// every measurement of this layer so far — accuracy and now stability — runs on
492 /// exactly that population. So the number that matters is measured on the
493 /// complement, and this reports whether the complement is big enough to measure
494 /// on at all.
495 ///
496 /// Returns the complement — the probe samples no filename rule answers — so
497 /// [`deployment_shape`] can be re-run on it. Empty when it is too thin to carry a
498 /// rate, which is a real possibility on a corpus whose folder labels were derived
499 /// from these same filenames. Phase C is where it stops being marginal: NSynth
500 /// names are `bass_synthetic_033-052-100`-shaped and carry no instrument keyword
501 /// the starter pack knows.
502 fn value_add_population<'a>(probe: &[&'a Row], report: &mut Report) -> Vec<&'a Row> {
503 println!("━━━ THE VALUE-ADD POPULATION ━━━");
504 println!();
505
506 let Some(name_rules) = filename_rules() else {
507 println!(" could not seed the starter rules; skipped");
508 println!();
509 return Vec::new();
510 };
511
512 let mut hit = 0usize;
513 let mut miss: Vec<&Row> = Vec::new();
514 for r in probe {
515 let ctx = RuleContext {
516 name: r.name.clone(),
517 ..RuleContext::default()
518 };
519 if name_rules
520 .iter()
521 .any(|rule| rules::rule_matches(rule, &ctx))
522 {
523 hit += 1;
524 } else {
525 miss.push(r);
526 }
527 }
528
529 let rate = hit as f64 / probe.len() as f64;
530 println!(
531 " {} of {} probe samples ({}) already carry a filename a starter rule fires on.",
532 hit,
533 probe.len(),
534 pct(Some(rate))
535 );
536 println!(
537 " The layer's job is the other {}: libraries whose filenames say nothing.",
538 miss.len()
539 );
540 println!(" This counts only rules reading the filename, and only whether one FIRES —");
541 println!(" not whether it fires correctly. It is a different quantity from the 97.7%");
542 println!(" in `af-browse-axes`, which is an accuracy over the whole corpus, so the two");
543 println!(" are not comparable and the gap between them is not a regression.");
544 println!();
545 if miss.len() < MIN_COMPARABLE {
546 println!(" Too thin to carry a flip rate, and thin by construction: this corpus's");
547 println!(" folder labels were derived from these same filenames, so a rule-miss here");
548 println!(" is close to a corpus artifact. Every number below is therefore measured on");
549 println!(" the population the rules already answer, which is the easy half.");
550 println!(" Phase C (24cd7747) is where this lifts.");
551 println!();
552 report.set("rule_hit", hit);
553 report.set("rule_miss", miss.len());
554 report.set("rule_hit_rate", round4(rate));
555 return Vec::new();
556 }
557 println!(" Large enough to carry a rate. Measurement 3 is re-run on it below, because");
558 println!(" a layer that is stable and useful only where the filename already said the");
559 println!(" answer is stable and useful nowhere that matters.");
560 println!();
561
562 report.set("rule_hit", hit);
563 report.set("rule_miss", miss.len());
564 report.set("rule_hit_rate", round4(rate));
565 miss
566 }
567
568 /// The starter pack's filename rules, enabled, as the app would evaluate them.
569 ///
570 /// Seeded into a throwaway in-memory vault rather than reconstructed from
571 /// `starter_rules::rules()` by hand: the seeding path is what assigns ids and
572 /// priorities, and guard rules depend on that order. Rules touching any field but
573 /// the name are dropped — this asks what the *filename* already answers, and a
574 /// rule reading spectral centroid is the classifier's own input wearing a
575 /// different hat.
576 fn filename_rules() -> Option<Vec<rules::Rule>> {
577 let db = audiofiles_core::db::Database::open_in_memory().ok()?;
578 starter_rules::seed(&db).ok()?;
579 let all = rules::list_rules(&db).ok()?;
580 Some(
581 all.into_iter()
582 .filter(|r| {
583 !r.conditions.is_empty() && r.conditions.iter().all(|c| c.field == RuleField::Name)
584 })
585 .collect(),
586 )
587 }
588
589 // 1. Composition
590
591 /// Same index size, different class mix.
592 ///
593 /// Two users with libraries of the same size but different contents are running
594 /// the same code against different models, and this is the spread between them.
595 /// The variants are held to one common size so composition is not confounded with
596 /// [`size`]: an index that is both bigger and differently shaped explains nothing.
597 ///
598 /// On this corpus the label space is two families, so the mix is varied both
599 /// between them (`low`-heavy against `bright`-heavy) and *within* `low` by corpus
600 /// origin (all-kick against all-tom). The second is the sharper test: `low` is
601 /// kick plus tom and `af-coarse-families` leaves open whether those behave as one
602 /// family, so a user whose low end is all toms and one whose low end is all kicks
603 /// are the realistic worst case for a family-level answer.
604 fn composition(
605 pool: &[&Row],
606 probe: &[&Row],
607 classes: &[String],
608 k: usize,
609 space: LabelSpace,
610 report: &mut Report,
611 ) -> Outcome {
612 println!("━━━ 1. COMPOSITION: same size, different mix ━━━");
613 println!();
614
615 let per_class: Vec<usize> = classes
616 .iter()
617 .map(|c| {
618 pool.iter()
619 .filter(|r| r.truth.as_deref() == Some(c.as_str()))
620 .count()
621 })
622 .collect();
623 let smallest = per_class.iter().copied().min().unwrap_or(0);
624 if smallest < 20 || classes.len() < 2 {
625 println!(" the pool cannot supply two differently-shaped indexes of a common size");
626 println!();
627 return Outcome {
628 name: "composition",
629 worst: None,
630 note: "not runnable on this corpus".into(),
631 };
632 }
633
634 // Common size: what the most lopsided mix can afford. 80/20 over the
635 // smallest class is the binding constraint.
636 let minor = smallest / 4;
637 let major = smallest;
638 let total = major + minor;
639
640 let mut variants: Vec<(String, Vec<&Row>)> = Vec::new();
641 let balanced_each = total / classes.len();
642 variants.push((
643 "balanced".into(),
644 classes
645 .iter()
646 .flat_map(|c| take_class(pool, c, balanced_each))
647 .collect(),
648 ));
649 for heavy in classes {
650 let mut v = Vec::new();
651 for c in classes {
652 let n = if c == heavy {
653 major
654 } else {
655 minor / (classes.len() - 1).max(1)
656 };
657 v.extend(take_class(pool, c, n));
658 }
659 variants.push((format!("{}-heavy", families::label_for(space, heavy)), v));
660 }
661 // Within-family origin variants, where the corpus has more than one folder
662 // behind a class.
663 for (label, origin) in origin_variants(pool, classes) {
664 let mut v = Vec::new();
665 for c in classes {
666 let want = if Some(c.as_str()) == origin.class.as_deref() {
667 pool.iter()
668 .filter(|r| r.truth.as_deref() == Some(c.as_str()) && r.origin == origin.folder)
669 .take(balanced_each)
670 .copied()
671 .collect()
672 } else {
673 take_class(pool, c, balanced_each)
674 };
675 v.extend(want);
676 }
677 variants.push((label, v));
678 }
679
680 println!(" Each index holds one class mix at a common size; the probe set never moves.");
681 println!();
682 println!(" {:<22} {:>8} class mix", "variant", "n");
683 println!(" {}", "─".repeat(72));
684 let mut computed: Vec<(String, Answers)> = Vec::new();
685 for (name, exemplars) in &variants {
686 let mix: Vec<String> = classes
687 .iter()
688 .map(|c| {
689 let n = exemplars
690 .iter()
691 .filter(|r| r.truth.as_deref() == Some(c.as_str()))
692 .count();
693 format!("{} {}", families::label_for(space, c), n)
694 })
695 .collect();
696 println!(" {:<22} {:>8} {}", name, exemplars.len(), mix.join(", "));
697 let a = run_variant(exemplars, &[], probe, k, name);
698 computed.push((name.clone(), a));
699 }
700 println!();
701
702 let worst = print_pairwise(&computed, report, "composition");
703 Outcome {
704 name: "composition",
705 worst,
706 note: format!("{} variants", computed.len()),
707 }
708 }
709
710 /// A within-class origin split worth building a variant for.
711 struct OriginVariant {
712 class: Option<String>,
713 folder: String,
714 }
715
716 /// Classes the corpus fills from more than one folder, and the folders behind
717 /// them. `low` is kick plus tom; an all-kick and an all-tom index are the same
718 /// class holding two different things.
719 fn origin_variants(pool: &[&Row], classes: &[String]) -> Vec<(String, OriginVariant)> {
720 let mut out = Vec::new();
721 for c in classes {
722 let mut folders: BTreeMap<&str, usize> = BTreeMap::new();
723 for r in pool
724 .iter()
725 .filter(|r| r.truth.as_deref() == Some(c.as_str()))
726 {
727 *folders.entry(r.origin.as_str()).or_default() += 1;
728 }
729 if folders.len() < 2 {
730 continue;
731 }
732 for (folder, n) in folders {
733 if n < 20 {
734 continue;
735 }
736 out.push((
737 format!("{folder}-only"),
738 OriginVariant {
739 class: Some(c.clone()),
740 folder: folder.to_string(),
741 },
742 ));
743 }
744 }
745 out
746 }
747
748 /// Every pair of variants' flip rate, and the worst of them.
749 fn print_pairwise(computed: &[(String, Answers)], report: &mut Report, key: &str) -> Option<f64> {
750 println!(" Pairwise flip rate (both suggested, tag changed):");
751 println!();
752 println!(
753 " {:<22} {:<22} {:>7} {:>8} {:>9} {:>9}",
754 "a", "b", "both", "flips", "rate", "churn"
755 );
756 println!(" {}", "─".repeat(84));
757
758 let mut worst: Option<f64> = None;
759 for i in 0..computed.len() {
760 for j in (i + 1)..computed.len() {
761 let c = Churn::between(&computed[i].1, &computed[j].1);
762 let rate = c.flip_rate();
763 if let Some(r) = c.comparable_flip_rate() {
764 worst = Some(worst.map_or(r, |w: f64| w.max(r)));
765 }
766 println!(
767 " {:<22} {:<22} {:>7} {:>8} {:>9} {:>9}",
768 computed[i].0,
769 computed[j].0,
770 c.both,
771 c.flips,
772 pct(rate),
773 format!("+{} -{}", c.appeared, c.vanished)
774 );
775 }
776 }
777 println!();
778 match worst {
779 Some(w) => {
780 println!(
781 " worst pairwise flip rate {} ({} the {:.0}% bar)",
782 pct(Some(w)),
783 if w <= BAR { "within" } else { "OVER" },
784 BAR * 100.0
785 );
786 report.set(&format!("{key}_worst_flip_rate"), round4(w));
787 }
788 None => println!(" no pair had a suggested answer in common; nothing to compare"),
789 }
790 println!();
791 worst
792 }
793
794 // 2. Size
795
796 /// Same mix, growing index: what a user's second month looks like.
797 ///
798 /// Nested by construction — each index is a prefix of the next, so growth is
799 /// accretion rather than resampling and a flip is the layer changing its mind
800 /// about a sample rather than an artifact of two unrelated draws. The number to
801 /// read is the *consecutive* rate: whether it decays toward the bar says whether
802 /// the answer stabilises, and where it crosses says at what library size.
803 fn size(
804 pool: &[&Row],
805 probe: &[&Row],
806 classes: &[String],
807 k: usize,
808 report: &mut Report,
809 ) -> Outcome {
810 println!("━━━ 2. SIZE: same mix, growing index ━━━");
811 println!();
812 println!(" Each index is a prefix of the next: a library accretes, it does not");
813 println!(" resample. Read the consecutive column — the pairwise-with-full column is");
814 println!(" the same information seen from the end state.");
815 println!();
816
817 const FRACTIONS: &[f64] = &[0.1, 0.2, 0.4, 0.6, 0.8, 1.0];
818 let mut computed: Vec<(String, Answers, usize)> = Vec::new();
819 for f in FRACTIONS {
820 let exemplars: Vec<&Row> = classes
821 .iter()
822 .flat_map(|c| {
823 let have = pool
824 .iter()
825 .filter(|r| r.truth.as_deref() == Some(c.as_str()))
826 .count();
827 #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
828 take_class(pool, c, (have as f64 * f).round() as usize)
829 })
830 .collect();
831 if exemplars.is_empty() {
832 continue;
833 }
834 let label = format!("{:.0}%", f * 100.0);
835 let a = run_variant(&exemplars, &[], probe, k, &label);
836 computed.push((label, a, exemplars.len()));
837 }
838 if computed.len() < 2 {
839 println!(" the pool is too small to sweep");
840 println!();
841 return Outcome {
842 name: "size",
843 worst: None,
844 note: "not runnable on this corpus".into(),
845 };
846 }
847
848 println!(
849 " {:<8} {:>8} {:>8} {:>8} {:>10} {:>12}",
850 "index", "n", "queued", "flips", "vs prev", "vs full"
851 );
852 println!(" {}", "─".repeat(60));
853
854 let full = &computed[computed.len() - 1].1;
855 let mut worst_consecutive: Option<f64> = None;
856 for (i, (label, a, n)) in computed.iter().enumerate() {
857 let queued = a.iter().filter(|x| x.tag().is_some()).count();
858 let prev = (i > 0).then(|| Churn::between(&computed[i - 1].1, a));
859 let vs_full = Churn::between(a, full);
860 if let Some(r) = prev.as_ref().and_then(Churn::comparable_flip_rate) {
861 worst_consecutive = Some(worst_consecutive.map_or(r, |w: f64| w.max(r)));
862 }
863 println!(
864 " {:<8} {:>8} {:>8} {:>8} {:>10} {:>12}",
865 label,
866 n,
867 queued,
868 prev.as_ref().map_or(0, |c| c.flips),
869 prev.as_ref()
870 .map_or_else(|| "-".to_string(), |c| pct(c.flip_rate())),
871 pct(vs_full.flip_rate()),
872 );
873 report.set(
874 &format!("size_{}_queued", label.trim_end_matches('%')),
875 queued,
876 );
877 }
878 println!();
879
880 // Where it settles: the first index size from which every later step stays
881 // within the bar. "Stabilises" has to mean it stays stable, not that one step
882 // happened to be quiet.
883 let settles = (0..computed.len() - 1).find(|&i| {
884 (i..computed.len() - 1).all(|j| Churn::between(&computed[j].1, &computed[j + 1].1).passes())
885 });
886 let settle_note = settles.map_or_else(
887 || "never settles".to_string(),
888 |i| format!("settles at {} exemplars", computed[i].2),
889 );
890 match settles {
891 Some(i) => {
892 println!(
893 " Settles at {} ({} exemplars): every step from there stays within the bar.",
894 computed[i].0, computed[i].2
895 );
896 report.set("size_settles_at", computed[i].2);
897 }
898 None => {
899 println!(" Never settles: no index size after which every step stays within the");
900 println!(" bar. On this pool that is a statement about the pool as much as the");
901 println!(" layer — the largest index here is still small.");
902 }
903 }
904 println!();
905 if let Some(w) = worst_consecutive {
906 report.set("size_worst_consecutive_flip_rate", round4(w));
907 }
908
909 Outcome {
910 name: "size",
911 worst: worst_consecutive,
912 note: format!("{} steps, {settle_note}", computed.len()),
913 }
914 }
915
916 // 3. Deployment shape
917
918 /// The mixed index the app actually builds, and the one nothing had measured.
919 ///
920 /// An imported layer sits at [`rows::IMPORT_WEIGHT`] beneath the user's own labels
921 /// at 1.0. Every earlier number came off a uniform index where that weight
922 /// cancels; here it does not, and neither does `build_index` fitting the
923 /// standardization params over both populations together.
924 ///
925 /// Two questions, and the second is the one the ship decision turns on:
926 ///
927 /// - **Stability.** As the user's own labels accumulate beside a fixed layer, does
928 /// the queue for the samples they have *not* labelled hold still? That is the
929 /// consecutive flip rate, measured at the deployment weight, and it is what
930 /// [`BAR`] is set against.
931 /// - **Value add.** At each user-library size, what does the layer contribute over
932 /// the user's own labels alone? Measured as the probe samples the mixed index
933 /// answers and the user-only index does not, and how often that answer is right.
934 /// A layer whose contribution goes to zero by the time a user has a few hundred
935 /// labels is a first-week feature, which is a fine thing to be and a different
936 /// thing from what it is currently described as.
937 fn deployment_shape(
938 pool: &[&Row],
939 probe: &[&Row],
940 k: usize,
941 space: LabelSpace,
942 population: &str,
943 key_prefix: &str,
944 report: &mut Report,
945 ) -> Outcome {
946 println!(
947 "━━━ {population}: user labels at 1.0, layer at {} ━━━",
948 rows::IMPORT_WEIGHT
949 );
950 println!();
951
952 let (layer_rows, user_pool) = halve(pool);
953 println!(
954 " The pool splits stratified into a simulated imported layer ({} exemplars,\n \
955 imported at {}) and a user pool ({} labels, weight 1.0) the user's own\n \
956 library is drawn from. Same class mix on both sides.",
957 layer_rows.len(),
958 rows::IMPORT_WEIGHT,
959 user_pool.len()
960 );
961 println!();
962
963 const USER_FRACTIONS: &[f64] = &[0.0, 0.05, 0.1, 0.25, 0.5, 1.0];
964 let classes: Vec<String> = user_pool
965 .iter()
966 .filter_map(|r| r.truth.clone())
967 .collect::<BTreeSet<_>>()
968 .into_iter()
969 .collect();
970
971 println!(
972 " {:<8} {:>7} {:>9} {:>9} {:>8} {:>9} {:>9} {:>7}",
973 "user", "labels", "vs prev", "vs layer", "differs", "mixed ok", "user ok", "delta"
974 );
975 println!(" {}", "─".repeat(74));
976
977 let mut computed: Vec<(String, Answers)> = Vec::new();
978 let mut worst_consecutive: Option<f64> = None;
979 let mut last_value_add: Option<(usize, f64)> = None;
980
981 for f in USER_FRACTIONS {
982 let user: Vec<&Row> = classes
983 .iter()
984 .flat_map(|c| {
985 let refs: Vec<&Row> = user_pool.clone();
986 let have = refs
987 .iter()
988 .filter(|r| r.truth.as_deref() == Some(c.as_str()))
989 .count();
990 #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
991 let n = (have as f64 * f).round() as usize;
992 take_class(&refs, c, n)
993 })
994 .collect();
995
996 let label = format!("{:.0}%", f * 100.0);
997 let mixed = run_variant(&user, &layer_rows, probe, k, &label);
998 // The counterfactual: the same user labels with no layer at all. This is
999 // what the layer has to beat to be worth shipping.
1000 let user_only = run_variant(&user, &[], probe, k, &format!("{label} user-only"));
1001
1002 let queued = mixed.iter().filter(|a| a.tag().is_some()).count();
1003 let prev = computed.last().map(|(_, a)| Churn::between(a, &mixed));
1004 if let Some(r) = prev.as_ref().and_then(Churn::comparable_flip_rate) {
1005 worst_consecutive = Some(worst_consecutive.map_or(r, |w: f64| w.max(r)));
1006 }
1007 let vs_layer_only = computed
1008 .first()
1009 .map(|(_, a)| Churn::between(a, &mixed))
1010 .and_then(|c| c.flip_rate());
1011
1012 // What the layer contributes, read off the answer rather than off an
1013 // answer appearing. "The layer answers where the user's labels do not"
1014 // is identically zero at two classes whatever the layer is worth (see
1015 // `review_threshold_note`), so the contribution is measured as: how often
1016 // the two disagree, and whether the layer's presence leaves the probe
1017 // more often right. The delta is the number the ship decision turns on —
1018 // a layer that changes answers without improving them is spending the
1019 // user's attention for nothing.
1020 let differs = (0..probe.len())
1021 .filter(|&i| mixed[i].tag() != user_only[i].tag())
1022 .count();
1023 let accuracy = |a: &Answers| {
1024 a.iter()
1025 .zip(probe)
1026 .filter(|(x, r)| x.tag().is_some() && x.tag() == r.truth.as_deref())
1027 .count() as f64
1028 / probe.len() as f64
1029 };
1030 let mixed_ok = accuracy(&mixed);
1031 let user_ok = accuracy(&user_only);
1032 last_value_add = Some((differs, mixed_ok - user_ok));
1033
1034 println!(
1035 " {:<8} {:>7} {:>9} {:>9} {:>8} {:>9} {:>9} {:>+7.1}",
1036 label,
1037 user.len(),
1038 prev.as_ref()
1039 .map_or_else(|| "-".to_string(), |c| pct(c.flip_rate())),
1040 pct(vs_layer_only),
1041 differs,
1042 pct(Some(mixed_ok)),
1043 pct(Some(user_ok)),
1044 (mixed_ok - user_ok) * 100.0,
1045 );
1046
1047 let key = label.trim_end_matches('%');
1048 report.set(&format!("{key_prefix}_user{key}_queued"), queued);
1049 report.set(&format!("{key_prefix}_user{key}_differs"), differs);
1050 report.set(
1051 &format!("{key_prefix}_user{key}_mixed_accuracy"),
1052 round4(mixed_ok),
1053 );
1054 report.set(
1055 &format!("{key_prefix}_user{key}_user_accuracy"),
1056 round4(user_ok),
1057 );
1058
1059 computed.push((label, mixed));
1060 }
1061 println!();
1062 println!(" 'differs' is probe samples where the mixed index and the same user labels");
1063 println!(" alone give different answers; 'delta' is what the layer's presence does to");
1064 println!(" accuracy, in points. Together they are the layer's contribution surviving");
1065 println!(" contact with a user's own data.");
1066 println!();
1067 println!(" The 0% row is the layer alone, which is the cold-start case and the only row");
1068 println!(" where 'user ok' is not a real alternative: a user with no labels has nothing");
1069 println!(" to fall back on.");
1070 println!();
1071
1072 if let Some((differs, delta)) = last_value_add {
1073 println!(
1074 " At a full user library the layer changes {differs} answer(s) and moves accuracy\n \
1075 by {:+.1} points.",
1076 delta * 100.0
1077 );
1078 if differs == 0 {
1079 println!();
1080 println!(" Zero is the strong version of the cold-start reading: once the user has");
1081 println!(" their own labels the layer is not merely adding little, it is changing");
1082 println!(" nothing at all. Worth shipping for the first week, and the documentation");
1083 println!(" has to say that rather than describe an ongoing contribution.");
1084 } else if delta.abs() < 0.005 {
1085 println!();
1086 println!(" Answers move and accuracy does not. That is the worst shape available:");
1087 println!(" the user re-reads a queue for no gain.");
1088 }
1089 println!();
1090 }
1091
1092 if let Some(w) = worst_consecutive {
1093 report.set(
1094 &format!("{key_prefix}_worst_consecutive_flip_rate"),
1095 round4(w),
1096 );
1097 println!(
1098 " worst consecutive flip rate at the deployment weight {} ({} the {:.0}% bar)",
1099 pct(Some(w)),
1100 if w <= BAR { "within" } else { "OVER" },
1101 BAR * 100.0
1102 );
1103 println!();
1104 }
1105
1106 let _ = space;
1107 Outcome {
1108 name: "deployment shape",
1109 worst: worst_consecutive,
1110 note: format!("layer {} + user pool {}", layer_rows.len(), user_pool.len()),
1111 }
1112 }
1113
1114 /// [`deployment_shape`] again, restricted to the probe samples no filename rule
1115 /// answers.
1116 ///
1117 /// The layer's stated job is libraries whose filenames say nothing, and every
1118 /// number in this file up to here is measured on the population the starter rules
1119 /// already handle. This is the same measurement on the population that motivates
1120 /// the feature. It is the one to read first when the two disagree.
1121 fn value_add_deployment(
1122 pool: &[&Row],
1123 miss: &[&Row],
1124 k: usize,
1125 space: LabelSpace,
1126 report: &mut Report,
1127 ) -> Option<Outcome> {
1128 if miss.is_empty() {
1129 return None;
1130 }
1131 println!(
1132 " Probe restricted to the {} samples no filename rule answers.",
1133 miss.len()
1134 );
1135 println!();
1136 let mut o = deployment_shape(
1137 pool,
1138 miss,
1139 k,
1140 space,
1141 "3b. DEPLOYMENT SHAPE, VALUE-ADD POPULATION ONLY",
1142 "valueadd",
1143 report,
1144 );
1145 o.name = "value-add deployment";
1146 Some(o)
1147 }
1148
1149 // 4. Accepted-tag feedback
1150
1151 /// The queue changes its own inputs.
1152 ///
1153 /// An accepted suggestion becomes a user label at weight 1.0 and re-enters the
1154 /// index, so working through the queue rewrites the rest of it. This is the one
1155 /// property no held-out measurement can see, because the thing that moves is the
1156 /// index and the thing being measured is the same set of samples.
1157 ///
1158 /// Simulated as a user actually working: build the mixed index, queue the
1159 /// unlabelled library, accept the highest-scoring batch, fold those in as weight
1160 /// 1.0 labels, rebuild, repeat. Two acceptance policies, because they bracket
1161 /// real behaviour and they fail differently:
1162 ///
1163 /// - **accept-all** — the user trusts the queue. Wrong suggestions become wrong
1164 /// exemplars at full weight, so this is where a feedback loop would compound.
1165 /// - **accept-correct** — an oracle user who accepts only right answers. If this
1166 /// one oscillates, the instability is in the mechanism and not in the user's
1167 /// mistakes.
1168 ///
1169 /// What is measured per round, over the samples still unlabelled and suggested
1170 /// before and after: the flip rate (does it decay), and per sample whether its
1171 /// answer has changed more than once across the whole run (oscillation, which a
1172 /// decaying average can hide).
1173 fn feedback(pool: &[&Row], k: usize, space: LabelSpace, report: &mut Report) -> Outcome {
1174 println!("━━━ 4. ACCEPTED-TAG FEEDBACK: the queue rewrites itself ━━━");
1175 println!();
1176
1177 let (layer_rows, user_pool) = halve(pool);
1178 // A small seed, because the interesting case is a user who has barely
1179 // labelled anything and is leaning on the queue to bootstrap.
1180 let seed_n = (user_pool.len() / 20).max(4);
1181 let seed: Vec<&Row> = user_pool.iter().take(seed_n).copied().collect();
1182 let library: Vec<&Row> = user_pool.iter().skip(seed_n).copied().collect();
1183 let batch = (library.len() / 8).max(1);
1184
1185 println!(
1186 " Layer {} at {}, user seed {} at 1.0, library {} unlabelled. Each round",
1187 layer_rows.len(),
1188 rows::IMPORT_WEIGHT,
1189 seed.len(),
1190 library.len()
1191 );
1192 println!(" accepts the {batch} highest-scoring suggestions and rebuilds the index.");
1193 println!();
1194
1195 if library.len() < 20 {
1196 println!(" library too small to work through");
1197 println!();
1198 return Outcome {
1199 name: "feedback",
1200 worst: None,
1201 note: "not runnable on this corpus".into(),
1202 };
1203 }
1204
1205 let mut worst: Option<f64> = None;
1206 for accept_all in [true, false] {
1207 let policy = if accept_all {
1208 "accept-all"
1209 } else {
1210 "accept-correct"
1211 };
1212 println!(" {policy}:");
1213 println!();
1214 println!(
1215 " {:<7} {:>8} {:>8} {:>8} {:>9} {:>9}",
1216 "round", "labels", "open", "queued", "flips", "rate"
1217 );
1218 println!(" {}", "─".repeat(56));
1219
1220 // `accepted` are library rows folded in as user labels; `open` are the
1221 // rest, and only they are measured — a sample the user has labelled is no
1222 // longer a queue entry, so its answer moving is not churn.
1223 let mut accepted: Vec<&Row> = Vec::new();
1224 let mut open: Vec<&Row> = library.clone();
1225 let mut previous: Option<HashMap<&str, Answer>> = None;
1226 let mut changes: HashMap<&str, usize> = HashMap::new();
1227 let mut round = 0usize;
1228
1229 while !open.is_empty() {
1230 let local: Vec<&Row> = seed.iter().chain(&accepted).copied().collect();
1231 let db = match rows::mixed_db(&local, &layer_rows) {
1232 Ok(db) => db,
1233 Err(e) => {
1234 eprintln!("{policy} round {round}: {e}");
1235 std::process::exit(1);
1236 }
1237 };
1238 let index = match exemplar::build_index(&db) {
1239 Ok(i) => i,
1240 Err(e) => {
1241 eprintln!("{policy} round {round}: build_index: {e}");
1242 std::process::exit(1);
1243 }
1244 };
1245
1246 // Score every open sample. `exclude_hash` is unnecessary: an open
1247 // sample is not in the index, which is the point of holding it out.
1248 let scored: Vec<(&Row, Option<exemplar::TagScore>)> = open
1249 .iter()
1250 .map(|r| {
1251 let top = index
1252 .score(&r.vector, k, None)
1253 .into_iter()
1254 .next()
1255 .filter(|s| s.score >= DEFAULT_REVIEW_THRESHOLD);
1256 (*r, top)
1257 })
1258 .collect();
1259
1260 let now: HashMap<&str, Answer> = scored
1261 .iter()
1262 .map(|(r, top)| {
1263 (
1264 r.hash.as_str(),
1265 top.as_ref()
1266 .map_or(Answer::Silent, |s| Answer::Tag(s.tag.clone())),
1267 )
1268 })
1269 .collect();
1270 let queued = now.values().filter(|a| a.tag().is_some()).count();
1271
1272 // Flips are counted only over samples open in both rounds: a sample
1273 // that left the library because it was accepted did not change its
1274 // mind, it stopped being a question.
1275 let (both, flips) = previous.as_ref().map_or((0, 0), |before| {
1276 let mut b = 0;
1277 let mut f = 0;
1278 for (hash, after) in &now {
1279 let Some(prev) = before.get(hash) else {
1280 continue;
1281 };
1282 if let (Some(p), Some(q)) = (prev.tag(), after.tag()) {
1283 b += 1;
1284 if p != q {
1285 f += 1;
1286 *changes.entry(*hash).or_default() += 1;
1287 }
1288 }
1289 }
1290 (b, f)
1291 });
1292 let rate = (both > 0).then(|| flips as f64 / both as f64);
1293 // A tail of a handful of stubborn samples produces enormous rates off
1294 // one flip. Printed, marked, and kept out of the verdict.
1295 let thin = both > 0 && both < MIN_COMPARABLE;
1296 if let Some(r) = rate.filter(|_| round > 0 && !thin) {
1297 worst = Some(worst.map_or(r, |w: f64| w.max(r)));
1298 }
1299
1300 println!(
1301 " {:<7} {:>8} {:>8} {:>8} {:>9} {:>9}{}",
1302 round,
1303 local.len(),
1304 open.len(),
1305 queued,
1306 flips,
1307 rate.map_or_else(|| "-".to_string(), |r| pct(Some(r))),
1308 if thin { " (thin)" } else { "" }
1309 );
1310
1311 // Accept the batch: highest-scoring suggestions first, which is the
1312 // order the review screen presents and therefore the order a user
1313 // works in.
1314 let mut candidates: Vec<(&Row, &exemplar::TagScore)> = scored
1315 .iter()
1316 .filter_map(|(r, top)| top.as_ref().map(|s| (*r, s)))
1317 .filter(|(r, s)| accept_all || Some(s.tag.as_str()) == r.truth.as_deref())
1318 .collect();
1319 candidates.sort_by(|a, b| b.1.score.total_cmp(&a.1.score));
1320 let taking: Vec<&str> = candidates
1321 .iter()
1322 .take(batch)
1323 .map(|(r, _)| r.hash.as_str())
1324 .collect();
1325 if taking.is_empty() {
1326 println!(" nothing left above the review threshold; the queue is dry");
1327 break;
1328 }
1329 // Accepting the layer's answer means the row enters the index with
1330 // the tag the layer proposed, right or wrong. Using the row's own
1331 // corpus tags instead would quietly make every accepted label
1332 // correct, which is the accept-all failure mode being simulated.
1333 let taken: BTreeSet<&str> = taking.iter().copied().collect();
1334 for (r, s) in candidates
1335 .iter()
1336 .filter(|(r, _)| taken.contains(r.hash.as_str()))
1337 {
1338 accepted.push(accepted_row(r, &s.tag));
1339 }
1340 open.retain(|r| !taken.contains(r.hash.as_str()));
1341 previous = Some(now);
1342 round += 1;
1343 if round > 20 {
1344 break;
1345 }
1346 }
1347
1348 let oscillating = changes.values().filter(|n| **n > 1).count();
1349 println!();
1350 println!(
1351 " (thin) marks a round with fewer than {MIN_COMPARABLE} entries still open. Those rates"
1352 );
1353 println!(" are one sample either way and do not decide the verdict.");
1354 println!(" {oscillating} sample(s) changed answer more than once across the run.");
1355 if oscillating == 0 {
1356 println!(" No oscillation: a sample that moved, moved once and stayed.");
1357 } else {
1358 println!(" Oscillation is the failure a decaying average hides: the mean flip");
1359 println!(" rate can fall while individual entries keep swapping back.");
1360 }
1361 println!();
1362 report.set(
1363 &format!("feedback_{}_oscillating", policy.replace('-', "_")),
1364 oscillating,
1365 );
1366 }
1367
1368 if let Some(w) = worst {
1369 report.set("feedback_worst_flip_rate", round4(w));
1370 }
1371 let _ = space;
1372 Outcome {
1373 name: "feedback",
1374 worst,
1375 note: format!("batch {batch}"),
1376 }
1377 }
1378
1379 /// A row as it enters the index after the user accepts `tag` for it.
1380 ///
1381 /// Leaks a `Row` deliberately: the accepted set is built round by round and has to
1382 /// outlive the loop iteration that created it, while every other row in the run is
1383 /// borrowed from the corpus. The run is a bounded number of rounds over a bounded
1384 /// library, so the leak is bounded by the library size and the process exits
1385 /// immediately after.
1386 fn accepted_row(r: &Row, tag: &str) -> &'static Row {
1387 Box::leak(Box::new(Row {
1388 hash: r.hash.clone(),
1389 vector: r.vector.clone(),
1390 tags: vec![tag.to_string()],
1391 truth: r.truth.clone(),
1392 origin: r.origin.clone(),
1393 name: r.name.clone(),
1394 }))
1395 }
1396
1397 // Verdict
1398
1399 fn verdict(outcomes: &[Outcome], space: LabelSpace, classes: &[String], report: &mut Report) {
1400 println!("━━━ VERDICT ━━━");
1401 println!();
1402 println!(" {:<20} {:>12}", "measurement", "worst flip");
1403 println!(" {}", "─".repeat(64));
1404
1405 let mut failed = Vec::new();
1406 let mut unrun = Vec::new();
1407 for o in outcomes {
1408 let state = match o.worst {
1409 Some(w) if w <= BAR => "within the bar",
1410 Some(_) => {
1411 failed.push(o.name);
1412 "OVER THE BAR"
1413 }
1414 None => {
1415 unrun.push(o.name);
1416 "not measured"
1417 }
1418 };
1419 println!(
1420 " {:<20} {:>12} {state} ({})",
1421 o.name,
1422 o.worst.map_or_else(|| "-".to_string(), |w| pct(Some(w))),
1423 o.note
1424 );
1425 }
1426 println!();
1427
1428 report.set("failed_measurements", failed.len());
1429 report.set("unrun_measurements", unrun.len());
1430
1431 if failed.is_empty() && unrun.is_empty() {
1432 println!(" PASS on every measurement that ran.");
1433 } else if failed.is_empty() {
1434 println!(
1435 " PASS on what ran; {} not measurable on this corpus: {}.",
1436 unrun.len(),
1437 unrun.join(", ")
1438 );
1439 } else {
1440 println!(" FAIL: {}.", failed.join(", "));
1441 }
1442 println!();
1443
1444 // Scope, restated at the bottom where a verdict gets quoted from.
1445 let (covered, uncovered) = families::covered_families(classes);
1446 if space != LabelSpace::Instrument && !uncovered.is_empty() {
1447 println!(
1448 " SCOPE: this corpus reaches {} of 7 families ({}). It says nothing about",
1449 covered.len(),
1450 covered.join(", ")
1451 );
1452 println!(
1453 " {}, and a stability number measured over two",
1454 uncovered.join(", ")
1455 );
1456 println!(" classes is an easier question than the shipped layer will face: fewer");
1457 println!(" classes means fewer things an answer can flip to. Read every figure above");
1458 println!(" as a floor on the flip rate, not an estimate of it.");
1459 println!();
1460 }
1461 println!(" And the standing caveat: this measures whether the answer is the SAME, not");
1462 println!(" whether it is RIGHT. The two are independent. layer-eval is the other half.");
1463 println!();
1464 }
1465
1466 /// Probe fraction denominator from the env: 5 means one in five rows is held out.
1467 pub(crate) fn probe_denominator_from_env() -> usize {
1468 std::env::var("AF_BENCH_STABILITY_PROBE")
1469 .ok()
1470 .and_then(|v| v.parse().ok())
1471 .filter(|n| *n >= 2)
1472 .unwrap_or(5)
1473 }
1474
1475 /// The single `k` this mode runs at.
1476 ///
1477 /// Stability is measured at the `k` that ships, not swept: `k` is a global
1478 /// constant at runtime, so a user never experiences two of them, and sweeping it
1479 /// here would answer a question about the mechanism when the bar is about the
1480 /// product. `AF_BENCH_EVAL_K`'s first entry overrides it so the two modes can be
1481 /// pointed at the same non-default `k` for a comparison.
1482 pub(crate) fn k_from_env() -> usize {
1483 std::env::var("AF_BENCH_EVAL_K")
1484 .ok()
1485 .and_then(|v| v.split(',').next()?.trim().parse().ok())
1486 .filter(|k| *k > 0)
1487 .unwrap_or(DEFAULT_K)
1488 }
1489
1490 #[cfg(test)]
1491 mod tests {
1492 use super::*;
1493
1494 fn ans(tags: &[Option<&str>]) -> Answers {
1495 tags.iter()
1496 .map(|t| t.map_or(Answer::Silent, |x| Answer::Tag(x.into())))
1497 .collect()
1498 }
1499
1500 #[test]
1501 fn churn_separates_a_flip_from_a_queue_getting_longer() {
1502 // The distinction the whole module turns on: a sample that gains an
1503 // answer costs the user a read, a sample that changes its answer costs
1504 // them a read they already did.
1505 let a = ans(&[Some("low"), Some("low"), None, Some("low")]);
1506 let b = ans(&[Some("low"), Some("bright"), Some("low"), None]);
1507 let c = Churn::between(&a, &b);
1508 assert_eq!(c.both, 2);
1509 assert_eq!(c.flips, 1);
1510 assert_eq!(c.appeared, 1);
1511 assert_eq!(c.vanished, 1);
1512 assert_eq!(c.flip_rate(), Some(0.5));
1513 }
1514
1515 #[test]
1516 fn no_overlapping_queue_is_not_a_flip_rate_of_zero() {
1517 // Two runs that never both suggested anything have no measured
1518 // agreement. Reporting 0% would read as perfect stability.
1519 let a = ans(&[Some("low"), None]);
1520 let b = ans(&[None, Some("low")]);
1521 let c = Churn::between(&a, &b);
1522 assert_eq!(c.flip_rate(), None);
1523 assert!(!c.passes(), "an unmeasurable pair must not pass");
1524 }
1525
1526 #[test]
1527 fn the_review_threshold_is_what_gates_an_answer() {
1528 // Not the auto threshold: under suggest-only nothing auto-applies, so a
1529 // score between review and auto is a queue entry and must count. If the
1530 // two ever collapse, every `Answer` here silently becomes an auto-apply
1531 // decision and the module is measuring the wrong line.
1532 const {
1533 assert!(DEFAULT_REVIEW_THRESHOLD < exemplar::DEFAULT_AUTO_THRESHOLD);
1534 }
1535 }
1536
1537 #[test]
1538 fn halving_keeps_the_class_mix_on_both_sides() {
1539 let rows: Vec<Row> = (0..10)
1540 .map(|i| Row {
1541 hash: format!("h{i}"),
1542 vector: Vec::new(),
1543 tags: Vec::new(),
1544 truth: Some(if i < 6 { "low" } else { "bright" }.to_string()),
1545 origin: String::new(),
1546 name: String::new(),
1547 })
1548 .collect();
1549 let refs: Vec<&Row> = rows.iter().collect();
1550 let (a, b) = halve(&refs);
1551 let low = |v: &[&Row]| {
1552 v.iter()
1553 .filter(|r| r.truth.as_deref() == Some("low"))
1554 .count()
1555 };
1556 assert_eq!(low(&a), 3);
1557 assert_eq!(low(&b), 3);
1558 assert_eq!(a.len() + b.len(), 10);
1559 }
1560
1561 #[test]
1562 fn take_class_is_a_prefix_so_the_size_sweep_nests() {
1563 // The size sweep's whole claim is that a bigger index contains the
1564 // smaller one. If this resampled, a flip would be an artifact of the draw.
1565 let rows: Vec<Row> = (0..6)
1566 .map(|i| Row {
1567 hash: format!("h{i}"),
1568 vector: Vec::new(),
1569 tags: Vec::new(),
1570 truth: Some("low".to_string()),
1571 origin: String::new(),
1572 name: String::new(),
1573 })
1574 .collect();
1575 let refs: Vec<&Row> = rows.iter().collect();
1576 let small = take_class(&refs, "low", 2);
1577 let big = take_class(&refs, "low", 4);
1578 assert_eq!(small.len(), 2);
1579 assert_eq!(big.len(), 4);
1580 assert!(
1581 small.iter().zip(&big).all(|(s, b)| std::ptr::eq(*s, *b)),
1582 "the smaller index must be a prefix of the larger"
1583 );
1584 }
1585
1586 #[test]
1587 fn the_filename_rules_are_name_only() {
1588 // A rule reading spectral centroid would be the classifier's own input
1589 // deciding what counts as the population the classifier adds value to.
1590 let rules = filename_rules().expect("the starter pack seeds");
1591 assert!(!rules.is_empty());
1592 for r in &rules {
1593 assert!(
1594 r.conditions.iter().all(|c| c.field == RuleField::Name),
1595 "{} reads a field other than the name",
1596 r.name
1597 );
1598 }
1599 }
1600 }
1601