//! Cross-validated evaluation of the exemplar k-NN layer over the labelled corpus.
//!
//! This is the meter the bundled `.afcl` never had, and the reason it was
//! eventually retired rather than shipped (2026-08-08, wiki `af-likeness-web`).
//! The generator is gone; this outlived it because what it measures is the
//! exemplar k-NN itself, which the app still runs on.
//!
//! What it measures: stratified k-fold cross-validation of [`exemplar`] over the
//! same corpus the layer is built from, at the same `k` the app uses at runtime.
//! Each fold builds a real `ExemplarIndex` from the other folds and scores this
//! fold's samples against it, so no sample is ever a neighbour of itself and the
//! standardization params are fitted on training data alone.
//!
//! Why it is not one accuracy figure: the retired threshold classifier scored
//! 33.4% strict with two of its seven classes unreachable by any rule, and a
//! single number is exactly what hid that. Everything here is per class.
//!
//! # The gate changed after the first run, and that needs saying out loud
//!
//! Run 1 (2026-08-06, `02395cb`) graded every class on precision and recall at
//! [`DEFAULT_AUTO_THRESHOLD`], the global 0.85 the app ships. It failed: four of
//! seven classes had auto recall under 7%, percussion never fired at all.
//!
//! Reading the run said the threshold was doing most of the failing. A score is
//! the share of the k=15 neighbourhood's kernel weight carrying a tag, so what a
//! class can reach depends on how many of its own members sit inside a fixed `k`.
//! Hi-hat (109 files) and snare (188) are equally separable by top-1 (71.6%
//! against 76.6%) and differ ninefold in recall at 0.85. And 0.85 is itself an
//! unvalidated constant: it predates any measurement of this layer.
//!
//! So the gate below asks a different question, not an easier one. Old: "at the
//! threshold we happen to ship, is each class good enough?" New: "at the
//! precision we actually require, what threshold does each class need, and is the
//! recall there worth shipping?" The precision bar went **up**, 0.80 to 0.95,
//! because that is what auto-applying a tag into someone's library unasked
//! deserves; the per-class threshold is what stops class size being graded as if
//! it were quality. Run 1's numbers stay in `docs/ml_classifier.md` so the change
//! is auditable rather than quietly overwritten.
//!
//! The output still reports the shipped defaults, because "what happens if this
//! ships unchanged" remains a real question with a bad answer.
//!
//! # And then the whole question changed, which needs saying louder
//!
//! Runs 1 and 2 (2026-08-06, `02395cb` and `50fe541`) graded seven **specific drum
//! instruments**. Both halves of that were settled against on 2026-07-29 and this
//! module did it anyway: audiofiles classifies at coarse family resolution because
//! instrument identity is not in these features (33.4% with ~40 tuned thresholds
//! against 92.4% for families on one unfitted cut, and instrument labels are not
//! perceptually coherent), and drums-only is the defect the whole scope effort
//! exists to fix rather than the scope to measure within. The machinery below was
//! never the problem: fold split, index construction, calibration, the Wilson
//! bound and the confusion matrix are all label-agnostic. The label mapping was.
//!
//! So the corpus is now projected onto families as it is read back
//! ([`crate::families`]), and everything is graded at that resolution. Two things
//! follow that a reader should not have to infer:
//!
//! - The drum corpus reaches **two of seven** families, `low` and `drum-bright`.
//!   Nothing here is a verdict on the layer. Five families are unmeasured.
//! - Runs 1 and 2 answer a retired question. They are not superseded results, they
//!   are results for something else, and `docs/ml_classifier.md` says so.
//!
//! Usage: `cargo run --release -p audiofiles-bench -- layer-eval`
//! Env: `AF_BENCH_CORPUS`, `AF_BENCH_VAULT`, `AF_BENCH_EVAL_FOLDS` (default 5),
//! `AF_BENCH_EVAL_K` (comma-separated sweep, default `5,10,15,25,50`),
//! `AF_BENCH_EVAL_LABELS` (`family` default, `family-tom-split`, `instrument`),
//! `AF_BENCH_JSON`.
//!
//! [`DEFAULT_AUTO_THRESHOLD`]: audiofiles_core::analysis::exemplar::DEFAULT_AUTO_THRESHOLD

use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;

use audiofiles_core::analysis::config::AnalysisConfig;
use audiofiles_core::analysis::exemplar::{
    self, DEFAULT_AUTO_THRESHOLD, DEFAULT_K, DEFAULT_REVIEW_THRESHOLD,
};
use audiofiles_core::analysis::features::FEATURE_VERSION;

use crate::calibration::{self, Counts, Point};
use crate::families::{self, LabelSpace};
use crate::labelled;
use crate::report::Report;
use crate::rows::{self, Row};

/// Default fold count.
///
/// Five over ~1000 files leaves every training fold within a fifth of the size of
/// the layer that actually ships, so the measured neighbourhood density is close
/// to the real one. Fewer folds would train on visibly less data than ships and
/// understate the layer; many more would leave the smallest class (clap, 48
/// files) with single-digit test sets whose per-class recall moves in 10% steps.
const DEFAULT_FOLDS: usize = 5;

/// Neighbour counts to sweep.
///
/// `k` is a global constant at runtime and the score is a share of it, so it is
/// the lever that decides whether a small class can reach any threshold at all.
/// Sweeping it costs nothing (the corpus is imported and analysed once, and only
/// the scoring repeats) and it is the difference between "this class is hard" and
/// "this class is outnumbered inside a window we chose".
const DEFAULT_K_SWEEP: &[usize] = &[5, 10, 15, 25, 50];

/// Score thresholds the sweep tables report, spanning both shipped defaults.
const SWEEP_THRESHOLDS: &[f64] = &[0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9];

/// The ship gate. See the module header for why it is not run 1's gate.
struct Gate {
    /// Precision each class must reach for the layer to auto-apply it, as a 95%
    /// lower bound rather than an observed ratio (see [`calibration`]). A tag
    /// written into a library unasked should be right 19 times in 20.
    target_precision: f64,
    /// Recall at that precision, below which a per-class threshold is not worth
    /// shipping: the tag fires too rarely to be a head start.
    min_recall: f64,
    /// Floor under the confidence bound. The bound already rejects a precision
    /// claimed off a handful of predictions (run 1 read cymbal as 100% precise
    /// on two); this stops a policy shipping off a thin sample regardless.
    min_support: usize,
    /// Macro-averaged top-1 recall: the threshold-free separability floor. Below
    /// this the feature space is not distinguishing these classes at all and no
    /// per-class calibration rescues it.
    top1_macro_recall: f64,
}

const GATE: Gate = Gate {
    target_precision: 0.95,
    min_recall: 0.40,
    min_support: 15,
    top1_macro_recall: 0.60,
};

/// What one test sample produced under one `k`.
struct Prediction {
    truth: String,
    /// Highest-scoring tag, or `None` when the index returned nothing.
    top1: Option<String>,
    /// Score per class, for threshold sweeps.
    scores: BTreeMap<String, f64>,
    /// The fold this sample was held out of, so a threshold is never calibrated
    /// on the predictions it is graded against.
    fold: usize,
    /// Corpus folder(s) behind the truth label. See [`Row::origin`].
    origin: String,
}

fn pct(v: Option<f64>) -> String {
    v.map_or_else(|| "-".to_string(), |x| format!("{:.1}%", x * 100.0))
}

/// Every test sample's evidence for one class. A class absent from a sample's
/// scores scored zero for it, which is a real observation and not a gap.
fn class_points(predictions: &[Prediction], class: &str) -> Vec<Point> {
    predictions
        .iter()
        .map(|p| Point {
            score: p.scores.get(class).copied().unwrap_or(0.0),
            actual: p.truth == class,
            fold: p.fold,
        })
        .collect()
}

pub(crate) fn run(
    corpus: &Path,
    vault: &Path,
    config: &AnalysisConfig,
    folds: usize,
    k_sweep: &[usize],
    space: LabelSpace,
) {
    println!("━━━ CLASSIFIER LAYER EVALUATION ━━━");
    println!();
    println!("  corpus   {}", corpus.display());
    println!("  scratch  {}", vault.display());
    println!("  features v{FEATURE_VERSION}");
    println!("  k        {DEFAULT_K} (runtime default), sweeping {k_sweep:?}");
    println!("  folds    {folds}, stratified by class");
    println!("  labels   {}", space.describe());
    println!();
    println!("  Gate:");
    println!(
        "    per-class precision, 95%-confident, at a per-class threshold >= {:.0}%",
        GATE.target_precision * 100.0
    );
    println!(
        "    per-class recall at that threshold                           >= {:.0}%",
        GATE.min_recall * 100.0
    );
    println!(
        "    predictions behind that precision                            >= {}",
        GATE.min_support
    );
    println!(
        "    macro-averaged top-1 recall                                  >= {:.0}%",
        GATE.top1_macro_recall * 100.0
    );
    println!("    every class calibratable, in every fold");
    println!();
    println!("  Thresholds are calibrated on the folds a sample is NOT in, so no");
    println!("  class picks its operating point from the data it is graded on.");
    println!();

    let built = match labelled::build_vault(corpus, vault, config) {
        Ok(v) => v,
        Err(e) => {
            eprintln!("corpus: {e}");
            std::process::exit(1);
        }
    };

    let (rows, dropped) = match rows::load_rows(&built.db, space) {
        Ok(r) => r,
        Err(e) => {
            eprintln!("reading the vault back: {e}");
            std::process::exit(1);
        }
    };
    if dropped.unprojectable > 0 {
        println!();
        println!(
            "  {} row(s) carry no label in this space ({}) and are excluded from",
            dropped.unprojectable,
            dropped
                .origins
                .iter()
                .cloned()
                .collect::<Vec<_>>()
                .join(", ")
        );
        println!("  training and testing both:");
        println!("{}", families::DROPPED_NOTE);
    }
    let ambiguous = rows.iter().filter(|r| r.truth.is_none()).count();
    let testable = rows.len() - ambiguous;
    if testable == 0 {
        eprintln!("no single-labelled samples to test");
        std::process::exit(1);
    }
    println!();
    println!("  {} scoreable row(s) in the vault", rows.len());
    if ambiguous > 0 {
        // Not a silent drop: the same file under two class folders is a corpus
        // problem, and the count is how anyone notices it grew.
        println!(
            "  {ambiguous} carry more than one class tag (duplicate audio across folders);\n  \
             they train in every fold and are never tested"
        );
    }

    let classes: BTreeSet<String> = rows.iter().filter_map(|r| r.truth.clone()).collect();
    let classes: Vec<String> = classes.into_iter().collect();
    let fold_of = rows::assign_folds(&rows, folds);

    // Cross-validation. The index is built once per fold and scored at every `k`,
    // because building it is the expensive half and `k` only enters at scoring.
    let mut by_k: BTreeMap<usize, Vec<Prediction>> =
        k_sweep.iter().map(|k| (*k, Vec::new())).collect();
    for fold in 0..folds {
        let train: Vec<&Row> = rows
            .iter()
            .zip(&fold_of)
            .filter(|(_, f)| **f != Some(fold))
            .map(|(r, _)| r)
            .collect();
        let test: Vec<&Row> = rows
            .iter()
            .zip(&fold_of)
            .filter(|(_, f)| **f == Some(fold))
            .map(|(r, _)| r)
            .collect();

        let db = match rows::local_db(&train) {
            Ok(db) => db,
            Err(e) => {
                eprintln!("fold {fold}: {e}");
                std::process::exit(1);
            }
        };
        let index = match exemplar::build_index(&db) {
            Ok(i) => i,
            Err(e) => {
                eprintln!("fold {fold}: build_index: {e}");
                std::process::exit(1);
            }
        };
        println!(
            "  fold {fold}: {} exemplars, {} held out",
            index.len(),
            test.len()
        );

        for row in test {
            for &k in k_sweep {
                // No `exclude_hash`: the row is not in this index at all, which
                // is the property the fold split exists to give.
                let scored = index.score(&row.vector, k, None);
                let top1 = scored.first().map(|s| s.tag.clone());
                let scores = scored.into_iter().map(|s| (s.tag, s.score)).collect();
                by_k.entry(k).or_default().push(Prediction {
                    truth: row.truth.clone().unwrap_or_default(),
                    top1,
                    scores,
                    fold,
                    origin: row.origin.clone(),
                });
            }
        }
    }
    println!();

    let mut report = Report::new("layer-eval");
    report.set("label_space", format!("{space:?}"));
    report.set("dropped_unprojectable", dropped.unprojectable);
    report.set("folds", folds);
    report.set("k", DEFAULT_K);
    report.set("feat_version", FEATURE_VERSION);
    report.set("exemplars_total", rows.len());
    report.set("ambiguous_excluded", ambiguous);
    report.set("gate_target_precision", GATE.target_precision);
    report.set("gate_min_recall", GATE.min_recall);
    report.set("gate_min_support", GATE.min_support);
    report.set("gate_top1_macro_recall", GATE.top1_macro_recall);

    let default_k = by_k
        .get(&DEFAULT_K)
        .expect("the sweep always contains the runtime k");
    report.set("tested", default_k.len());

    let top1 = print_confusion(default_k, &classes, space, &mut report);
    print_origin_breakdown(default_k, &classes, space, &mut report);
    print_shipped_defaults(default_k, &classes, space, &mut report);
    print_threshold_sweep(default_k, &classes, space);
    let calibrated = print_calibration(default_k, &classes, folds, space, &mut report);
    if k_sweep.len() > 1 {
        print_k_sweep(&by_k, &classes, folds, &mut report);
        print_nested_k(&rows, &fold_of, &classes, k_sweep, folds, &mut report);
    }
    print_verdict(&classes, &top1, &calibrated, space, &mut report);
    report.write();
}

/// Top-1 confusion matrix, and per-class top-1 recall. Returns the per-class counts.
fn print_confusion(
    predictions: &[Prediction],
    classes: &[String],
    space: LabelSpace,
    report: &mut Report,
) -> BTreeMap<String, Counts> {
    println!("━━━ TOP-1 CONFUSION (k = {DEFAULT_K}) ━━━");
    println!();
    println!("  Rows are the corpus label, columns the highest-scoring tag.");
    println!("  Threshold-free: this is what the layer would say if forced to pick,");
    println!("  so it measures separability rather than any policy over it.");
    println!();

    let width = classes
        .iter()
        .map(|c| families::label_for(space, c).len().max(5))
        .collect::<Vec<_>>();

    print!("  {:<12}", "true \\ pred");
    for (c, w) in classes.iter().zip(&width) {
        print!(" {:>w$}", families::label_for(space, c), w = w);
    }
    println!(" {:>6} {:>8}", "(none)", "recall");
    println!(
        "  {}",
        "─".repeat(12 + width.iter().map(|w| w + 1).sum::<usize>() + 16)
    );

    let mut counts: BTreeMap<String, Counts> = BTreeMap::new();
    let mut never_predicted: Vec<&str> = Vec::new();

    for truth in classes {
        let mine: Vec<&Prediction> = predictions.iter().filter(|p| &p.truth == truth).collect();
        print!("  {:<12}", families::label_for(space, truth));
        let mut correct = 0usize;
        for (pred, w) in classes.iter().zip(&width) {
            let n = mine
                .iter()
                .filter(|p| p.top1.as_deref() == Some(pred.as_str()))
                .count();
            if pred == truth {
                correct = n;
            }
            print!(" {n:>w$}");
        }
        let none = mine.iter().filter(|p| p.top1.is_none()).count();
        let recall = if mine.is_empty() {
            None
        } else {
            Some(correct as f64 / mine.len() as f64)
        };
        println!(" {:>6} {:>8}", none, pct(recall));

        // Precision needs the whole column, so it is counted here rather than
        // inside the row loop.
        let predicted_as = predictions
            .iter()
            .filter(|p| p.top1.as_deref() == Some(truth.as_str()))
            .count();
        if predicted_as == 0 {
            never_predicted.push(families::label_for(space, truth));
        }
        counts.insert(
            truth.clone(),
            Counts {
                tp: correct,
                fp: predicted_as - correct,
                fn_: mine.len() - correct,
            },
        );
    }
    println!();

    let macro_recall = macro_average(classes, &counts, Counts::recall);
    let micro = counts.values().map(|c| c.tp).sum::<usize>() as f64 / predictions.len() as f64;
    println!("  macro-averaged recall  {}", pct(macro_recall));
    println!("  overall top-1 accuracy {}", pct(Some(micro)));
    if never_predicted.is_empty() {
        println!("  Every class is predicted at least once.");
    } else {
        println!();
        println!(
            "  NEVER PREDICTED: {}. This class is unreachable, not merely weak.",
            never_predicted.join(", ")
        );
    }
    println!();

    if let Some(m) = macro_recall {
        report.set("top1_macro_recall", round4(m));
    }
    report.set("top1_accuracy", round4(micro));
    report.set("never_predicted", never_predicted.len());
    for (tag, c) in &counts {
        let label = families::label_for(space, tag);
        if let Some(r) = c.recall() {
            report.set(&format!("top1_{label}_recall"), round4(r));
        }
        if let Some(p) = c.precision() {
            report.set(&format!("top1_{label}_precision"), round4(p));
        }
    }
    counts
}

/// Per-class top-1 recall broken out by the corpus folder each sample came from.
///
/// A family is only a family if its members behave like one. `low` is kick plus
/// tom, and `af-coarse-families` calls that the one open split worth measuring:
/// tom sits between `low` and `bass` on centroid (p25-p75 904-1996 against kick's
/// 397-885) and is 23% of the drum corpus, so folding it in silently assumes the
/// answer. If toms are recovered as `low` about as often as kicks are, the fold
/// holds; if they are systematically lost, `low` is two things wearing one label.
///
/// This is the measurement, not a proposal to add a `tom` family. Reading it
/// against `AF_BENCH_EVAL_LABELS=family-tom-split`, which grades tom as its own
/// class, is what separates "tom is hard" from "tom is not low".
fn print_origin_breakdown(
    predictions: &[Prediction],
    classes: &[String],
    space: LabelSpace,
    report: &mut Report,
) {
    let origins: BTreeSet<&str> = predictions.iter().map(|p| p.origin.as_str()).collect();
    // Nothing to say when every class is one folder: the table would be the
    // recall column of the confusion matrix, transposed.
    if origins.len() <= classes.len() {
        return;
    }

    println!("━━━ BY CORPUS ORIGIN (k = {DEFAULT_K}) ━━━");
    println!();
    println!("  The same top-1 answers, grouped by the folder the sample came from");
    println!("  rather than by the class it was projected onto. A family whose");
    println!("  members disagree here is not one family.");
    println!();
    println!(
        "  {:<14} {:<14} {:>6} {:>9}   most common wrong answer",
        "origin", "projects to", "n", "recall"
    );
    println!("  {}", "─".repeat(76));

    for origin in origins {
        let mine: Vec<&Prediction> = predictions.iter().filter(|p| p.origin == origin).collect();
        let Some(truth) = mine.first().map(|p| p.truth.clone()) else {
            continue;
        };
        let correct = mine
            .iter()
            .filter(|p| p.top1.as_deref() == Some(truth.as_str()))
            .count();
        let recall = correct as f64 / mine.len() as f64;

        // Where the misses go, which is the informative half: a tom read as
        // drum-bright says something different from a tom the index has no
        // answer for at all.
        let mut wrong: BTreeMap<&str, usize> = BTreeMap::new();
        for p in &mine {
            match p.top1.as_deref() {
                Some(t) if t != truth => *wrong.entry(t).or_default() += 1,
                None => *wrong.entry("(none)").or_default() += 1,
                _ => {}
            }
        }
        let worst = wrong.iter().max_by_key(|(_, n)| **n).map_or_else(
            || "-".to_string(),
            |(t, n)| {
                let label = if *t == "(none)" {
                    "(none)"
                } else {
                    families::label_for(space, t)
                };
                format!("{label} ({n})")
            },
        );

        println!(
            "  {:<14} {:<14} {:>6} {:>9}   {worst}",
            origin,
            families::label_for(space, &truth),
            mine.len(),
            pct(Some(recall)),
        );
        report.set(&format!("origin_{origin}_recall"), round4(recall));
        report.set(&format!("origin_{origin}_n"), mine.len());
    }
    println!();
}

/// What the layer does today, unchanged: one global auto threshold for every
/// class. Kept because it is the status quo the ship decision is against.
fn print_shipped_defaults(
    predictions: &[Prediction],
    classes: &[String],
    space: LabelSpace,
    report: &mut Report,
) {
    println!("━━━ AT THE SHIPPED DEFAULTS (one global threshold) ━━━");
    println!();

    let mut counts: BTreeMap<String, Counts> = BTreeMap::new();
    for class in classes {
        counts.insert(
            class.clone(),
            calibration::counts_at(&class_points(predictions, class), DEFAULT_AUTO_THRESHOLD),
        );
    }

    println!(
        "  {:<12} {:>7} {:>8} {:>10} {:>9}   auto {DEFAULT_AUTO_THRESHOLD:.2} / review {DEFAULT_REVIEW_THRESHOLD:.2}",
        "class", "n", "fired", "precision", "recall"
    );
    println!("  {}", "─".repeat(50));
    for class in classes {
        let c = counts[class];
        println!(
            "  {:<12} {:>7} {:>8} {:>10} {:>9}",
            families::label_for(space, class),
            c.actual(),
            c.fired(),
            pct(c.precision()),
            pct(c.recall()),
        );
    }
    println!("  {}", "─".repeat(50));
    println!(
        "  {:<12} {:>7} {:>8} {:>10} {:>9}",
        "macro",
        predictions.len(),
        counts.values().map(|c| c.fired()).sum::<usize>(),
        pct(macro_average(classes, &counts, Counts::precision)),
        pct(macro_average(classes, &counts, Counts::recall)),
    );
    println!();
    let silent = predictions
        .iter()
        .filter(|p| !p.scores.values().any(|s| *s >= DEFAULT_AUTO_THRESHOLD))
        .count();
    println!(
        "  {silent} of {} samples ({:.0}%) get no tag at all.",
        predictions.len(),
        silent as f64 / predictions.len() as f64 * 100.0
    );
    println!();

    for (tag, c) in &counts {
        let label = families::label_for(space, tag);
        if let Some(p) = c.precision() {
            report.set(&format!("shipped_{label}_precision"), round4(p));
        }
        if let Some(r) = c.recall() {
            report.set(&format!("shipped_{label}_recall"), round4(r));
        }
    }
    report.set("shipped_silent_samples", silent);
}

/// Precision and recall for every class across a range of thresholds.
///
/// This is the evidence that one global threshold cannot serve seven classes: read
/// down a column and the same number means a different thing in every row.
fn print_threshold_sweep(predictions: &[Prediction], classes: &[String], space: LabelSpace) {
    let points: BTreeMap<&String, Vec<Point>> = classes
        .iter()
        .map(|c| (c, class_points(predictions, c)))
        .collect();

    for (title, metric) in [
        ("PRECISION", Counts::precision as fn(Counts) -> Option<f64>),
        ("RECALL", Counts::recall as fn(Counts) -> Option<f64>),
    ] {
        println!("━━━ {title} BY THRESHOLD (k = {DEFAULT_K}) ━━━");
        println!();
        print!("  {:<12}", "class");
        for t in SWEEP_THRESHOLDS {
            print!(" {t:>7.2}");
        }
        println!();
        println!("  {}", "─".repeat(12 + SWEEP_THRESHOLDS.len() * 8));
        for class in classes {
            print!("  {:<12}", families::label_for(space, class));
            for t in SWEEP_THRESHOLDS {
                let c = calibration::counts_at(&points[class], *t);
                print!(" {:>7}", pct(metric(c)));
            }
            println!();
        }
        println!();
    }
    println!("  A dash is a class that fires nothing at that threshold, which is not");
    println!("  the same as firing and being wrong. Read the two tables together: a");
    println!("  class holding high precision far down the range has headroom the");
    println!("  global 0.85 is not spending.");
    println!();
}

/// The per-class operating points the data supports, and what they deliver on
/// folds they were not calibrated on.
fn print_calibration(
    predictions: &[Prediction],
    classes: &[String],
    folds: usize,
    space: LabelSpace,
    report: &mut Report,
) -> BTreeMap<String, Counts> {
    println!(
        "━━━ CALIBRATED OPERATING POINTS (target precision {:.0}%) ━━━",
        GATE.target_precision * 100.0
    );
    println!();
    println!("  The most permissive threshold at which each class still meets the");
    println!("  precision bar, and what it buys. These are `tag_policy` rows: what a");
    println!("  layer would ship if it carried its own thresholds instead of");
    println!("  inheriting one global pair.");
    println!();
    println!(
        "  {:<12} {:>7} {:>10} {:>10} {:>9} {:>18}",
        "class", "n", "threshold", "precision", "recall", "held-out recall"
    );
    println!("  {}", "─".repeat(70));

    let mut out_of_fold: BTreeMap<String, Counts> = BTreeMap::new();
    for class in classes {
        let points = class_points(predictions, class);
        let in_sample =
            calibration::operating_point(&points, GATE.target_precision, GATE.min_support);
        let (oof, thresholds) =
            calibration::out_of_fold(&points, folds, GATE.target_precision, GATE.min_support);
        out_of_fold.insert(class.clone(), oof);

        let label = families::label_for(space, class);
        match in_sample {
            Some(op) => println!(
                "  {:<12} {:>7} {:>10.3} {:>10} {:>9} {:>18}",
                label,
                op.counts.actual(),
                op.threshold,
                pct(op.counts.precision()),
                pct(op.counts.recall()),
                pct(oof.recall()),
            ),
            None => println!(
                "  {:<12} {:>7} {:>10} {:>10} {:>9} {:>18}",
                label,
                points.iter().filter(|p| p.actual).count(),
                "none",
                "-",
                "-",
                pct(oof.recall()),
            ),
        }

        if let Some((mean, min, max)) = calibration::threshold_spread(&thresholds) {
            report.set(&format!("calibrated_{label}_threshold"), round4(mean));
            if (max - min) > 0.15 {
                // A threshold that moves this much between folds is not a stable
                // property of the class, and shipping the mean would be fiction.
                println!(
                    "  {:<12} threshold unstable across folds: {min:.2} to {max:.2}",
                    ""
                );
            }
        }
        if thresholds.len() < folds {
            println!(
                "  {:<12} {} of {folds} folds found no qualifying threshold",
                "",
                folds - thresholds.len()
            );
        }
        if let Some(p) = oof.precision() {
            report.set(&format!("calibrated_{label}_precision"), round4(p));
        }
        if let Some(r) = oof.recall() {
            report.set(&format!("calibrated_{label}_recall"), round4(r));
        }
        report.set(&format!("calibrated_{label}_fired"), oof.fired());
    }
    println!("  {}", "─".repeat(70));
    println!(
        "  {:<12} {:>7} {:>10} {:>10} {:>9} {:>18}",
        "macro",
        predictions.len(),
        "",
        pct(macro_average(classes, &out_of_fold, Counts::precision)),
        "",
        pct(macro_average(classes, &out_of_fold, Counts::recall)),
    );
    println!();
    println!("  The last column is the honest one: thresholds chosen on four folds,");
    println!("  measured on the fifth. The gap between it and the recall column is");
    println!("  how much of the calibration was fitting noise.");
    println!();

    if let Some(p) = macro_average(classes, &out_of_fold, Counts::precision) {
        report.set("calibrated_macro_precision", round4(p));
    }
    if let Some(r) = macro_average(classes, &out_of_fold, Counts::recall) {
        report.set("calibrated_macro_recall", round4(r));
    }
    out_of_fold
}

/// Does `k` move the classes a fixed window was starving?
fn print_k_sweep(
    by_k: &BTreeMap<usize, Vec<Prediction>>,
    classes: &[String],
    folds: usize,
    report: &mut Report,
) {
    println!("━━━ k SWEEP ━━━");
    println!();
    println!("  A score is a share of k neighbours, so k decides whether a small");
    println!("  class can reach any threshold at all. Calibrated columns are");
    println!("  held-out, per class, at the target precision.");
    println!();
    println!(
        "  {:>5} {:>12} {:>14} {:>16} {:>14} {:>12}",
        "k", "top-1 macro", "calib. recall", "calib. precision", "worst class", "uncalib."
    );
    println!("  {}", "─".repeat(78));

    for (k, predictions) in by_k {
        let mut top1: BTreeMap<String, Counts> = BTreeMap::new();
        let mut calibrated: BTreeMap<String, Counts> = BTreeMap::new();
        let mut uncalibratable = 0usize;
        for class in classes {
            let mine = predictions.iter().filter(|p| &p.truth == class).count();
            let correct = predictions
                .iter()
                .filter(|p| &p.truth == class && p.top1.as_deref() == Some(class.as_str()))
                .count();
            let predicted_as = predictions
                .iter()
                .filter(|p| p.top1.as_deref() == Some(class.as_str()))
                .count();
            top1.insert(
                class.clone(),
                Counts {
                    tp: correct,
                    fp: predicted_as - correct,
                    fn_: mine - correct,
                },
            );

            let points = class_points(predictions, class);
            let (oof, thresholds) =
                calibration::out_of_fold(&points, folds, GATE.target_precision, GATE.min_support);
            if thresholds.is_empty() {
                uncalibratable += 1;
            }
            calibrated.insert(class.clone(), oof);
        }

        let worst = classes
            .iter()
            .map(|c| calibrated[c].recall().unwrap_or(0.0))
            .fold(f64::INFINITY, f64::min);
        let marker = if *k == DEFAULT_K { " <- runtime" } else { "" };
        println!(
            "  {:>5} {:>12} {:>14} {:>16} {:>14} {:>12}{marker}",
            k,
            pct(macro_average(classes, &top1, Counts::recall)),
            pct(macro_average(classes, &calibrated, Counts::recall)),
            pct(macro_average(classes, &calibrated, Counts::precision)),
            pct(Some(worst)),
            uncalibratable,
        );

        report.set(
            &format!("k{k}_top1_macro_recall"),
            round4(macro_average(classes, &top1, Counts::recall).unwrap_or(0.0)),
        );
        report.set(
            &format!("k{k}_calibrated_macro_recall"),
            round4(macro_average(classes, &calibrated, Counts::recall).unwrap_or(0.0)),
        );
        report.set(&format!("k{k}_uncalibratable_classes"), uncalibratable);
    }
    println!();
    println!("  `uncalib.` counts classes for which no fold found a threshold meeting");
    println!("  the precision bar. Those are the classes a shipped layer cannot apply");
    println!("  at all, whatever the global default is set to.");
    println!();
    println!("  CAVEAT: this table selects k on the data it reports. Thresholds are");
    println!("  calibrated out of fold, k is not, so reading the best row here and");
    println!("  shipping that k would be choosing a hyperparameter on the test set.");
    println!("  It is evidence about the mechanism. The outer fold below is the");
    println!("  number to quote instead.");
    println!();
}

/// Select `k` under an outer fold, so the selection is never scored on the data
/// it was made from.
///
/// The sweep above is the standard trap: it fits thresholds out of fold and then
/// picks `k` by reading every row of the result. That is choosing a
/// hyperparameter on the test set, and the honest correction is the same shape as
/// the one that fixed the thresholds — hold the selection out too.
///
/// Procedure. For each outer fold: take the other folds as a training corpus,
/// split THOSE by an inner fold, select the `k` with the best inner macro
/// calibrated recall, then build one index over the whole training corpus and
/// score the outer fold at the selected `k`. Nothing about the outer fold is
/// visible to the selection, so the pooled result is what "select k this way"
/// generalises to.
///
/// Indexes are rebuilt rather than reused from `by_k`: a prediction there was
/// made by an index containing the outer test fold, so recycling them would leak
/// exactly what this exists to stop. That costs an extra `folds * folds` index
/// builds and no extra analysis, which is the cheap half.
///
/// What to read: if the outer number matches the best row of the sweep, the
/// sweep was not being flattered by its own selection and the mechanism finding
/// stands. If it comes in below, the difference is the selection bias, and the
/// outer number is the one that would survive contact with a user's library.
fn print_nested_k(
    rows: &[Row],
    outer_of: &[Option<usize>],
    classes: &[String],
    k_sweep: &[usize],
    folds: usize,
    report: &mut Report,
) {
    println!("━━━ k UNDER AN OUTER FOLD ━━━");
    println!();
    println!("  k selected inside each outer fold's training corpus, then scored on the");
    println!("  outer fold it never saw. This is the k-sweep number with the selection");
    println!("  bias removed.");
    println!();
    println!(
        "  {:>7} {:>10} {:>10} {:>16} {:>14}",
        "outer", "train", "k chosen", "inner recall", "outer recall"
    );
    println!("  {}", "─".repeat(64));

    let mut pooled: Vec<Prediction> = Vec::new();
    let mut chosen: Vec<usize> = Vec::new();

    for outer in 0..folds {
        let train: Vec<&Row> = rows
            .iter()
            .zip(outer_of)
            .filter(|(_, f)| **f != Some(outer))
            .map(|(r, _)| r)
            .collect();
        let test: Vec<&Row> = rows
            .iter()
            .zip(outer_of)
            .filter(|(_, f)| **f == Some(outer))
            .map(|(r, _)| r)
            .collect();
        if test.is_empty() {
            continue;
        }

        // Inner CV over the training corpus only.
        let train_rows: Vec<Row> = train.iter().map(|r| clone_row(r)).collect();
        let inner_of = rows::assign_folds(&train_rows, folds);
        let mut inner: BTreeMap<usize, Vec<Prediction>> =
            k_sweep.iter().map(|k| (*k, Vec::new())).collect();
        for i in 0..folds {
            let itrain: Vec<&Row> = train_rows
                .iter()
                .zip(&inner_of)
                .filter(|(_, f)| **f != Some(i))
                .map(|(r, _)| r)
                .collect();
            let itest: Vec<&Row> = train_rows
                .iter()
                .zip(&inner_of)
                .filter(|(_, f)| **f == Some(i))
                .map(|(r, _)| r)
                .collect();
            let Ok(db) = rows::local_db(&itrain) else {
                continue;
            };
            let Ok(index) = exemplar::build_index(&db) else {
                continue;
            };
            for row in itest {
                for &k in k_sweep {
                    push_prediction(inner.entry(k).or_default(), &index, row, k, i);
                }
            }
        }

        // Selection criterion: macro calibrated recall, the same quantity the
        // sweep table ranks on, so the two are comparable.
        let score_of = |preds: &Vec<Prediction>| {
            let mut calibrated: BTreeMap<String, Counts> = BTreeMap::new();
            for class in classes {
                let points = class_points(preds, class);
                let (oof, _) = calibration::out_of_fold(
                    &points,
                    folds,
                    GATE.target_precision,
                    GATE.min_support,
                );
                calibrated.insert(class.clone(), oof);
            }
            macro_average(classes, &calibrated, Counts::recall).unwrap_or(0.0)
        };
        let Some((best_k, inner_recall)) = k_sweep
            .iter()
            .map(|k| (*k, score_of(&inner[k])))
            // total_cmp then the smaller k, so a tie ships the tighter
            // neighbourhood rather than whichever the map iterated first.
            .max_by(|a, b| a.1.total_cmp(&b.1).then(b.0.cmp(&a.0)))
        else {
            continue;
        };
        chosen.push(best_k);

        // Score the outer fold at the selected k, from an index over the whole
        // training corpus.
        let Ok(db) = rows::local_db(&train) else {
            continue;
        };
        let Ok(index) = exemplar::build_index(&db) else {
            continue;
        };
        let mut outer_preds: Vec<Prediction> = Vec::new();
        for row in &test {
            push_prediction(&mut outer_preds, &index, row, best_k, outer);
        }
        let outer_recall = {
            let mut top1: BTreeMap<String, Counts> = BTreeMap::new();
            for class in classes {
                let mine = outer_preds.iter().filter(|p| &p.truth == class).count();
                let correct = outer_preds
                    .iter()
                    .filter(|p| &p.truth == class && p.top1.as_deref() == Some(class.as_str()))
                    .count();
                let predicted_as = outer_preds
                    .iter()
                    .filter(|p| p.top1.as_deref() == Some(class.as_str()))
                    .count();
                top1.insert(
                    class.clone(),
                    Counts {
                        tp: correct,
                        fp: predicted_as - correct,
                        fn_: mine - correct,
                    },
                );
            }
            macro_average(classes, &top1, Counts::recall)
        };

        println!(
            "  {:>7} {:>10} {:>10} {:>16} {:>14}",
            outer,
            train.len(),
            best_k,
            pct(Some(inner_recall)),
            pct(outer_recall),
        );
        pooled.extend(outer_preds);
    }
    println!();

    if chosen.is_empty() {
        println!("  no outer fold completed");
        println!();
        return;
    }

    let mut top1: BTreeMap<String, Counts> = BTreeMap::new();
    for class in classes {
        let mine = pooled.iter().filter(|p| &p.truth == class).count();
        let correct = pooled
            .iter()
            .filter(|p| &p.truth == class && p.top1.as_deref() == Some(class.as_str()))
            .count();
        let predicted_as = pooled
            .iter()
            .filter(|p| p.top1.as_deref() == Some(class.as_str()))
            .count();
        top1.insert(
            class.clone(),
            Counts {
                tp: correct,
                fp: predicted_as - correct,
                fn_: mine - correct,
            },
        );
    }
    let pooled_recall = macro_average(classes, &top1, Counts::recall);
    let agreed: BTreeSet<usize> = chosen.iter().copied().collect();

    println!(
        "  Pooled outer macro top-1 recall {} over {} predictions.",
        pct(pooled_recall),
        pooled.len()
    );
    if agreed.len() == 1 {
        let k = *agreed.iter().next().unwrap_or(&DEFAULT_K);
        println!("  Every outer fold selected k = {k}. A selection that does not move with");
        println!("  the training data is one the corpus supports, not one it happened onto.");
        if k != DEFAULT_K {
            println!(
                "  It is NOT the shipped k ({DEFAULT_K}). That is a real finding, not a rounding:"
            );
            println!("  the runtime constant predates every measurement of this layer.");
        }
        report.set("nested_k_selected", k);
    } else {
        let spread: Vec<String> = agreed.iter().map(usize::to_string).collect();
        println!("  Outer folds disagreed on k: {}.", spread.join(", "));
        println!("  A selection that moves with the training data is not a property of the");
        println!("  corpus, and shipping any single one of these is a coin toss dressed as a");
        println!("  measurement. Read the sweep as a mechanism finding and leave k alone.");
        report.set("nested_k_disagreed", agreed.len());
    }
    if let Some(r) = pooled_recall {
        report.set("nested_k_outer_macro_recall", round4(r));
    }
    println!();
}

/// Score one row against one index and push the prediction.
fn push_prediction(
    into: &mut Vec<Prediction>,
    index: &exemplar::ExemplarIndex,
    row: &Row,
    k: usize,
    fold: usize,
) {
    // No `exclude_hash`: the row is not in this index at all, which is the
    // property the fold split exists to give.
    let scored = index.score(&row.vector, k, None);
    into.push(Prediction {
        truth: row.truth.clone().unwrap_or_default(),
        top1: scored.first().map(|s| s.tag.clone()),
        scores: scored.into_iter().map(|s| (s.tag, s.score)).collect(),
        fold,
        origin: row.origin.clone(),
    });
}

/// A `Row` copy, so the inner split can own its training corpus.
///
/// `assign_folds` takes `&[Row]` rather than `&[&Row]`, and the inner fold is
/// assigned over a subset that only exists as borrows. Copying ~800 short vectors
/// once per outer fold is cheaper than threading a second lifetime through the
/// split.
fn clone_row(r: &Row) -> Row {
    Row {
        hash: r.hash.clone(),
        vector: r.vector.clone(),
        tags: r.tags.clone(),
        truth: r.truth.clone(),
        origin: r.origin.clone(),
        name: r.name.clone(),
    }
}

fn macro_average(
    classes: &[String],
    counts: &BTreeMap<String, Counts>,
    metric: fn(Counts) -> Option<f64>,
) -> Option<f64> {
    // Classes with no metric (nothing predicted, so no precision) count as zero
    // rather than being skipped. Dropping them would let a layer raise its macro
    // average by predicting a class less often.
    let vals: Vec<f64> = classes
        .iter()
        .map(|c| counts.get(c).and_then(|c| metric(*c)).unwrap_or(0.0))
        .collect();
    (!vals.is_empty()).then(|| vals.iter().sum::<f64>() / vals.len() as f64)
}

fn round4(v: f64) -> f64 {
    (v * 10000.0).round() / 10000.0
}

/// Score the run against [`GATE`] and say plainly whether it passes.
fn print_verdict(
    classes: &[String],
    top1: &BTreeMap<String, Counts>,
    calibrated: &BTreeMap<String, Counts>,
    space: LabelSpace,
    report: &mut Report,
) {
    println!("━━━ VERDICT ━━━");
    println!();

    let mut failures: Vec<String> = Vec::new();

    for class in classes {
        let label = families::label_for(space, class);
        let c = calibrated[class];
        if c.fired() == 0 {
            failures.push(format!(
                "{label}: no threshold reaches {:.0}% precision, so the layer cannot apply it at all",
                GATE.target_precision * 100.0
            ));
            continue;
        }
        if let Some(r) = c.recall()
            && r < GATE.min_recall
        {
            failures.push(format!(
                "{label}: held-out recall {} at the precision bar, below the {:.0}% gate",
                pct(Some(r)),
                GATE.min_recall * 100.0
            ));
        }
        if let Some(p) = c.precision()
            && p < GATE.target_precision
        {
            // The threshold met the bar in calibration and missed it held out,
            // which means the operating point does not generalise.
            failures.push(format!(
                "{label}: held-out precision {} below the {:.0}% bar its threshold was calibrated to",
                pct(Some(p)),
                GATE.target_precision * 100.0
            ));
        }
        if top1.get(class).is_some_and(|t| t.fired() == 0) {
            failures.push(format!("{label}: never the top-1 answer for any sample"));
        }
    }

    let macro_recall = macro_average(classes, top1, Counts::recall).unwrap_or(0.0);
    if macro_recall < GATE.top1_macro_recall {
        failures.push(format!(
            "macro top-1 recall {} below the {:.0}% gate",
            pct(Some(macro_recall)),
            GATE.top1_macro_recall * 100.0
        ));
    }

    if failures.is_empty() {
        println!("  PASS, on a per-class policy. Shipping this means shipping the");
        println!("  calibrated thresholds with it: set `include_policy` in afcl_gen and");
        println!("  export the tag_policy rows, or the layer inherits the global 0.85");
        println!("  and none of the above holds.");
    } else {
        println!("  FAIL on {} criterion/criteria:", failures.len());
        for f in &failures {
            println!("    - {f}");
        }
    }
    println!();

    // Printed on pass and on fail both, because the scope caveat is not a
    // consolation for a failure: a pass here is the more dangerous of the two to
    // read as a verdict on the layer.
    print_scope(classes, space);

    report.set("gate_pass", failures.is_empty());
    report.set("gate_failures", failures.len());
}

/// What this corpus can and cannot support a claim about.
///
/// The gate above says whether the classes present are shippable. It cannot say
/// anything about the classes absent, and the absent ones are the majority: the
/// corpus is drum one-shots, so it reaches two of the seven families and five have
/// no material at all. A layer that answers `drum-bright` confidently for a vocal
/// it has never seen passes every criterion above.
fn print_scope(classes: &[String], space: LabelSpace) {
    if space == LabelSpace::Instrument {
        println!("  SCOPE: this run grades specific drum instruments, which is the");
        println!("  retired question (wiki af-browse-axes, 2026-07-29). Kept runnable so");
        println!("  the results already written up stay reproducible. Do not extend it.");
        println!();
        return;
    }

    let (covered, uncovered) = families::covered_families(classes);
    println!(
        "  SCOPE: {} of {} families measured: {}.",
        covered.len(),
        families::FAMILIES.len(),
        covered.join(", ")
    );
    if !uncovered.is_empty() {
        println!("  No corpus material for: {}.", uncovered.join(", "));
        println!("  The layer has never been asked about them and will answer with the");
        println!("  nearest drum it knows. Nothing above is a verdict on the layer;");
        println!("  widening the corpus is the next phase.");
    }
    println!();
}

/// Fold count from the environment, or [`DEFAULT_FOLDS`].
pub(crate) fn folds_from_env() -> usize {
    std::env::var("AF_BENCH_EVAL_FOLDS")
        .ok()
        .and_then(|v| v.parse().ok())
        .filter(|n| *n >= 2)
        .unwrap_or(DEFAULT_FOLDS)
}

/// Neighbour counts to sweep, always including the runtime default so the
/// detailed sections have something to report.
pub(crate) fn k_sweep_from_env() -> Vec<usize> {
    let mut ks: Vec<usize> = std::env::var("AF_BENCH_EVAL_K").map_or_else(
        |_| DEFAULT_K_SWEEP.to_vec(),
        |v| {
            v.split(',')
                .filter_map(|s| s.trim().parse().ok())
                .filter(|k| *k > 0)
                .collect()
        },
    );
    ks.push(DEFAULT_K);
    ks.sort_unstable();
    ks.dedup();
    ks
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn macro_average_counts_an_unpredicted_class_as_zero() {
        // Skipping it would let a layer raise its macro precision by predicting
        // a hard class less often, which is backwards.
        let classes = vec!["a".to_string(), "b".to_string()];
        let mut counts = BTreeMap::new();
        counts.insert(
            "a".to_string(),
            Counts {
                tp: 10,
                fp: 0,
                fn_: 0,
            },
        );
        counts.insert(
            "b".to_string(),
            Counts {
                tp: 0,
                fp: 0,
                fn_: 10,
            },
        );
        assert_eq!(
            macro_average(&classes, &counts, Counts::precision),
            Some(0.5)
        );
    }

    #[test]
    fn class_points_score_an_absent_class_as_zero() {
        // A class missing from a sample's scores is a real zero, not a gap: the
        // index considered it and gave it no neighbourhood weight. Treating it
        // as missing would drop true negatives and inflate precision.
        let p = vec![Prediction {
            truth: "instrument.drum.kick".into(),
            top1: Some("instrument.drum.kick".into()),
            scores: BTreeMap::from([("instrument.drum.kick".to_string(), 0.9)]),
            fold: 0,
            origin: "kick".into(),
        }];
        let pts = class_points(&p, "instrument.drum.snare");
        assert_eq!(pts.len(), 1);
        assert!((pts[0].score - 0.0).abs() < f64::EPSILON);
        assert!(!pts[0].actual);
    }

    #[test]
    fn k_sweep_always_contains_the_runtime_k() {
        // Safe to set: this test does not read the env, it checks the invariant
        // the parser must hold whatever the env said.
        let ks = k_sweep_from_env();
        assert!(ks.contains(&DEFAULT_K));
        assert!(ks.windows(2).all(|w| w[0] < w[1]), "sorted and deduped");
    }
}