max / audiofiles
4 files changed,
+1034 insertions,
-338 deletions
| @@ -68,20 +68,20 @@ | |||
| 68 | 68 | //! | |
| 69 | 69 | //! [`DEFAULT_AUTO_THRESHOLD`]: audiofiles_core::analysis::exemplar::DEFAULT_AUTO_THRESHOLD | |
| 70 | 70 | ||
| 71 | - | use std::collections::{BTreeMap, BTreeSet, HashMap}; | |
| 71 | + | use std::collections::{BTreeMap, BTreeSet}; | |
| 72 | 72 | use std::path::Path; | |
| 73 | 73 | ||
| 74 | 74 | use audiofiles_core::analysis::config::AnalysisConfig; | |
| 75 | 75 | use audiofiles_core::analysis::exemplar::{ | |
| 76 | 76 | self, DEFAULT_AUTO_THRESHOLD, DEFAULT_K, DEFAULT_REVIEW_THRESHOLD, | |
| 77 | 77 | }; | |
| 78 | - | use audiofiles_core::analysis::features::{FEATURE_VERSION, NUM_FEATURES}; | |
| 79 | - | use audiofiles_core::db::Database; | |
| 78 | + | use audiofiles_core::analysis::features::FEATURE_VERSION; | |
| 80 | 79 | ||
| 81 | 80 | use crate::calibration::{self, Counts, Point}; | |
| 82 | 81 | use crate::families::{self, LabelSpace}; | |
| 83 | 82 | use crate::labelled; | |
| 84 | 83 | use crate::report::Report; | |
| 84 | + | use crate::rows::{self, Row}; | |
| 85 | 85 | ||
| 86 | 86 | /// Default fold count. | |
| 87 | 87 | /// | |
| @@ -130,25 +130,6 @@ | |||
| 130 | 130 | top1_macro_recall: 0.60, | |
| 131 | 131 | }; | |
| 132 | 132 | ||
| 133 | - | /// One corpus sample: its vector, its labels, and the class it is scored against. | |
| 134 | - | struct Row { | |
| 135 | - | hash: String, | |
| 136 | - | vector: Vec<f64>, | |
| 137 | - | /// Labels in the evaluation's space, so the index under test carries the tags | |
| 138 | - | /// being graded. | |
| 139 | - | tags: Vec<String>, | |
| 140 | - | /// The single class this row is ground truth for, or `None` when the corpus | |
| 141 | - | /// gave it more than one. Content-addressed import collapses a file that | |
| 142 | - | /// appears in two class folders into one row carrying both tags; its true | |
| 143 | - | /// class is undecidable, so it trains but is never tested. | |
| 144 | - | truth: Option<String>, | |
| 145 | - | /// The corpus folder(s) this row came from, before projection. Kept so a | |
| 146 | - | /// family's members can be broken out by where they came from: `low` is kick | |
| 147 | - | /// plus tom, and whether the layer recovers a tom as readily as a kick is the | |
| 148 | - | /// open split `af-coarse-families` asks about. | |
| 149 | - | origin: String, | |
| 150 | - | } | |
| 151 | - | ||
| 152 | 133 | /// What one test sample produced under one `k`. | |
| 153 | 134 | struct Prediction { | |
| 154 | 135 | truth: String, | |
| @@ -167,166 +148,6 @@ | |||
| 167 | 148 | v.map_or_else(|| "-".to_string(), |x| format!("{:.1}%", x * 100.0)) | |
| 168 | 149 | } | |
| 169 | 150 | ||
| 170 | - | /// What `load_rows` set aside, so no exclusion is silent. | |
| 171 | - | struct Dropped { | |
| 172 | - | /// Rows whose every corpus tag projects nowhere in this label space. | |
| 173 | - | unprojectable: usize, | |
| 174 | - | /// Corpus labels those rows came from. | |
| 175 | - | origins: BTreeSet<String>, | |
| 176 | - | } | |
| 177 | - | ||
| 178 | - | /// Read the analysed corpus back out of the scratch vault as scoreable rows, | |
| 179 | - | /// projected onto `space`. | |
| 180 | - | /// | |
| 181 | - | /// The projection happens here rather than in [`labelled`] on purpose. The vault | |
| 182 | - | /// is ground truth at the finest resolution the corpus carries, which is what | |
| 183 | - | /// `afcl_gen` exports from; grading at a coarser resolution is a property of the | |
| 184 | - | /// evaluation, not of the corpus, so it belongs on the read side. That also keeps | |
| 185 | - | /// the exported layer byte-identical and leaves the retired instrument question | |
| 186 | - | /// runnable instead of deleted. | |
| 187 | - | fn load_rows(db: &Database, space: LabelSpace) -> Result<(Vec<Row>, Dropped), String> { | |
| 188 | - | let conn = db.conn(); | |
| 189 | - | ||
| 190 | - | let mut tags_by_hash: HashMap<String, Vec<String>> = HashMap::new(); | |
| 191 | - | { | |
| 192 | - | let mut stmt = conn | |
| 193 | - | .prepare("SELECT sample_hash, tag FROM tags") | |
| 194 | - | .map_err(|e| format!("tags query: {e}"))?; | |
| 195 | - | let rows = stmt | |
| 196 | - | .query_map([], |row| { | |
| 197 | - | Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) | |
| 198 | - | }) | |
| 199 | - | .map_err(|e| format!("tags query: {e}"))?; | |
| 200 | - | for r in rows { | |
| 201 | - | let (hash, tag) = r.map_err(|e| format!("tags row: {e}"))?; | |
| 202 | - | tags_by_hash.entry(hash).or_default().push(tag); | |
| 203 | - | } | |
| 204 | - | } | |
| 205 | - | ||
| 206 | - | let mut out = Vec::new(); | |
| 207 | - | let mut dropped = Dropped { | |
| 208 | - | unprojectable: 0, | |
| 209 | - | origins: BTreeSet::new(), | |
| 210 | - | }; | |
| 211 | - | let mut stmt = conn | |
| 212 | - | .prepare("SELECT hash, vector FROM sample_features WHERE feat_version = ?1 ORDER BY hash") | |
| 213 | - | .map_err(|e| format!("features query: {e}"))?; | |
| 214 | - | let rows = stmt | |
| 215 | - | .query_map([FEATURE_VERSION], |row| { | |
| 216 | - | Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) | |
| 217 | - | }) | |
| 218 | - | .map_err(|e| format!("features query: {e}"))?; | |
| 219 | - | for r in rows { | |
| 220 | - | let (hash, json) = r.map_err(|e| format!("features row: {e}"))?; | |
| 221 | - | let Some(mut corpus_tags) = tags_by_hash.remove(&hash) else { | |
| 222 | - | continue; | |
| 223 | - | }; | |
| 224 | - | corpus_tags.sort(); | |
| 225 | - | corpus_tags.dedup(); | |
| 226 | - | let vector: Vec<f64> = | |
| 227 | - | serde_json::from_str(&json).map_err(|e| format!("vector for {hash}: {e}"))?; | |
| 228 | - | // The index drops these too (`is_usable_vector`), so counting them as | |
| 229 | - | // testable would score the layer on samples it never sees. | |
| 230 | - | if vector.len() != NUM_FEATURES || !vector.iter().all(|x| x.is_finite()) { | |
| 231 | - | continue; | |
| 232 | - | } | |
| 233 | - | ||
| 234 | - | // Project, then dedup again: two instrument tags landing in one family is | |
| 235 | - | // not ambiguity, it is the coarser question being easier. A file in both | |
| 236 | - | // the kick and tom folders has no instrument truth and a perfectly good | |
| 237 | - | // family one. | |
| 238 | - | let mut tags: Vec<String> = corpus_tags | |
| 239 | - | .iter() | |
| 240 | - | .filter_map(|t| families::project(space, t)) | |
| 241 | - | .map(str::to_string) | |
| 242 | - | .collect(); | |
| 243 | - | tags.sort(); | |
| 244 | - | tags.dedup(); | |
| 245 | - | if tags.is_empty() { | |
| 246 | - | dropped.unprojectable += 1; | |
| 247 | - | dropped.origins.extend( | |
| 248 | - | corpus_tags | |
| 249 | - | .iter() | |
| 250 | - | .map(|t| labelled::label_for_tag(t).to_string()), | |
| 251 | - | ); | |
| 252 | - | continue; | |
| 253 | - | } | |
| 254 | - | ||
| 255 | - | let truth = (tags.len() == 1).then(|| tags[0].clone()); | |
| 256 | - | let origin = corpus_tags | |
| 257 | - | .iter() | |
| 258 | - | .map(|t| labelled::label_for_tag(t)) | |
| 259 | - | .collect::<Vec<_>>() | |
| 260 | - | .join("+"); | |
| 261 | - | out.push(Row { | |
| 262 | - | hash, | |
| 263 | - | vector, | |
| 264 | - | tags, | |
| 265 | - | truth, | |
| 266 | - | origin, | |
| 267 | - | }); | |
| 268 | - | } | |
| 269 | - | Ok((out, dropped)) | |
| 270 | - | } | |
| 271 | - | ||
| 272 | - | /// Assign each row a fold, stratified by class. | |
| 273 | - | /// | |
| 274 | - | /// Round-robin down each class's hash-sorted list rather than a shuffle: the | |
| 275 | - | /// content hash is already an arbitrary order with respect to the source pack a | |
| 276 | - | /// file came from, so this needs no RNG and two runs over the same corpus produce | |
| 277 | - | /// the same folds. Ambiguous rows get no fold; they train everywhere. | |
| 278 | - | fn assign_folds(rows: &[Row], folds: usize) -> Vec<Option<usize>> { | |
| 279 | - | let mut seen_per_class: HashMap<&str, usize> = HashMap::new(); | |
| 280 | - | rows.iter() | |
| 281 | - | .map(|r| { | |
| 282 | - | let truth = r.truth.as_deref()?; | |
| 283 | - | let n = seen_per_class.entry(truth).or_default(); | |
| 284 | - | let fold = *n % folds; | |
| 285 | - | *n += 1; | |
| 286 | - | Some(fold) | |
| 287 | - | }) | |
| 288 | - | .collect() | |
| 289 | - | } | |
| 290 | - | ||
| 291 | - | /// Build an in-memory vault holding just the training rows, so the index under | |
| 292 | - | /// test is built by the same `build_index` the app calls. | |
| 293 | - | /// | |
| 294 | - | /// Every exemplar here is `local` (weight 1.0) where the shipped ones arrive from | |
| 295 | - | /// an imported layer at a lower weight. That does not change a score: the weight | |
| 296 | - | /// is a constant multiplier inside a sum that is then divided by its own total, | |
| 297 | - | /// so a uniform weight cancels. It would matter only against a mixed index of | |
| 298 | - | /// user labels plus the layer, which is a different measurement (how the layer | |
| 299 | - | /// behaves beside a user's own data) and not what a ship gate turns on. | |
| 300 | - | fn training_db(rows: &[&Row]) -> Result<Database, String> { | |
| 301 | - | let db = Database::open_in_memory().map_err(|e| format!("open_in_memory: {e}"))?; | |
| 302 | - | db.transaction(|_tx| { | |
| 303 | - | for row in rows { | |
| 304 | - | db.conn().execute( | |
| 305 | - | "INSERT INTO samples (hash, original_name, file_extension, file_size, import_date, last_modified) | |
| 306 | - | VALUES (?1, ?1, 'wav', 1, 0, 0)", | |
| 307 | - | [&row.hash], | |
| 308 | - | )?; | |
| 309 | - | let json = serde_json::to_string(&row.vector).unwrap_or_default(); | |
| 310 | - | db.conn().execute( | |
| 311 | - | "INSERT INTO sample_features (hash, feat_version, vector, computed_at) VALUES (?1, ?2, ?3, 0)", | |
| 312 | - | rusqlite::params![&row.hash, FEATURE_VERSION, json], | |
| 313 | - | )?; | |
| 314 | - | } | |
| 315 | - | Ok(()) | |
| 316 | - | }) | |
| 317 | - | .map_err(|e| format!("seeding the fold index: {e}"))?; | |
| 318 | - | // Tags outside the transaction: `add_tag` is the public path and manages its | |
| 319 | - | // own writes, and this is an in-memory DB where the fsync cost it avoids does | |
| 320 | - | // not exist. | |
| 321 | - | for row in rows { | |
| 322 | - | for tag in &row.tags { | |
| 323 | - | audiofiles_core::tags::add_tag(&db, &row.hash, tag) | |
| 324 | - | .map_err(|e| format!("tagging the fold index: {e}"))?; | |
| 325 | - | } | |
| 326 | - | } | |
| 327 | - | Ok(db) | |
| 328 | - | } | |
| 329 | - | ||
| 330 | 151 | /// Every test sample's evidence for one class. A class absent from a sample's | |
| 331 | 152 | /// scores scored zero for it, which is a real observation and not a gap. | |
| 332 | 153 | fn class_points(predictions: &[Prediction], class: &str) -> Vec<Point> { | |
| @@ -388,7 +209,7 @@ | |||
| 388 | 209 | } | |
| 389 | 210 | }; | |
| 390 | 211 | ||
| 391 | - | let (rows, dropped) = match load_rows(&built.db, space) { | |
| 212 | + | let (rows, dropped) = match rows::load_rows(&built.db, space) { | |
| 392 | 213 | Ok(r) => r, | |
| 393 | 214 | Err(e) => { | |
| 394 | 215 | eprintln!("reading the vault back: {e}"); | |
| @@ -429,7 +250,7 @@ | |||
| 429 | 250 | ||
| 430 | 251 | let classes: BTreeSet<String> = rows.iter().filter_map(|r| r.truth.clone()).collect(); | |
| 431 | 252 | let classes: Vec<String> = classes.into_iter().collect(); | |
| 432 | - | let fold_of = assign_folds(&rows, folds); | |
| 253 | + | let fold_of = rows::assign_folds(&rows, folds); | |
| 433 | 254 | ||
| 434 | 255 | // Cross-validation. The index is built once per fold and scored at every `k`, | |
| 435 | 256 | // because building it is the expensive half and `k` only enters at scoring. | |
| @@ -449,7 +270,7 @@ | |||
| 449 | 270 | .map(|(r, _)| r) | |
| 450 | 271 | .collect(); | |
| 451 | 272 | ||
| 452 | - | let db = match training_db(&train) { | |
| 273 | + | let db = match rows::local_db(&train) { | |
| 453 | 274 | Ok(db) => db, | |
| 454 | 275 | Err(e) => { | |
| 455 | 276 | eprintln!("fold {fold}: {e}"); | |
| @@ -1173,64 +994,6 @@ | |||
| 1173 | 994 | mod tests { | |
| 1174 | 995 | use super::*; | |
| 1175 | 996 | ||
| 1176 | - | fn row(hash: &str, truth: Option<&str>) -> Row { | |
| 1177 | - | Row { | |
| 1178 | - | hash: hash.to_string(), | |
| 1179 | - | vector: vec![0.0; NUM_FEATURES], | |
| 1180 | - | tags: truth.map(|t| vec![t.to_string()]).unwrap_or_default(), | |
| 1181 | - | truth: truth.map(str::to_string), | |
| 1182 | - | origin: truth.map_or_else(String::new, |t| labelled::label_for_tag(t).to_string()), | |
| 1183 | - | } | |
| 1184 | - | } | |
| 1185 | - | ||
| 1186 | - | #[test] | |
| 1187 | - | fn folds_are_stratified_and_ambiguous_rows_get_none() { | |
| 1188 | - | let mut rows: Vec<Row> = (0..10) | |
| 1189 | - | .map(|i| row(&format!("k{i}"), Some("instrument.drum.kick"))) | |
| 1190 | - | .collect(); | |
| 1191 | - | rows.extend((0..4).map(|i| row(&format!("s{i}"), Some("instrument.drum.snare")))); | |
| 1192 | - | rows.push(row("dup", None)); | |
| 1193 | - | ||
| 1194 | - | let folds = assign_folds(&rows, 5); | |
| 1195 | - | assert_eq!( | |
| 1196 | - | folds.last().copied().flatten(), | |
| 1197 | - | None, | |
| 1198 | - | "ambiguous row trains everywhere" | |
| 1199 | - | ); | |
| 1200 | - | ||
| 1201 | - | // Every fold holds two of the ten kicks: stratification, not a global | |
| 1202 | - | // round-robin that would leave a fold without any snares. | |
| 1203 | - | for f in 0..5 { | |
| 1204 | - | let kicks = rows | |
| 1205 | - | .iter() | |
| 1206 | - | .zip(&folds) | |
| 1207 | - | .filter(|(r, fold)| { | |
| 1208 | - | **fold == Some(f) && r.truth.as_deref() == Some("instrument.drum.kick") | |
| 1209 | - | }) | |
| 1210 | - | .count(); | |
| 1211 | - | assert_eq!(kicks, 2, "fold {f}"); | |
| 1212 | - | } | |
| 1213 | - | // Four snares over five folds: one fold is short, and none holds two. | |
| 1214 | - | for f in 0..5 { | |
| 1215 | - | let snares = rows | |
| 1216 | - | .iter() | |
| 1217 | - | .zip(&folds) | |
| 1218 | - | .filter(|(r, fold)| { | |
| 1219 | - | **fold == Some(f) && r.truth.as_deref() == Some("instrument.drum.snare") | |
| 1220 | - | }) | |
| 1221 | - | .count(); | |
| 1222 | - | assert!(snares <= 1, "fold {f} holds {snares} snares"); | |
| 1223 | - | } | |
| 1224 | - | } | |
| 1225 | - | ||
| 1226 | - | #[test] | |
| 1227 | - | fn folds_are_deterministic() { | |
| 1228 | - | let rows: Vec<Row> = (0..20) | |
| 1229 | - | .map(|i| row(&format!("k{i}"), Some("instrument.drum.kick"))) | |
| 1230 | - | .collect(); | |
| 1231 | - | assert_eq!(assign_folds(&rows, 5), assign_folds(&rows, 5)); | |
| 1232 | - | } | |
| 1233 | - | ||
| 1234 | 997 | #[test] | |
| 1235 | 998 | fn macro_average_counts_an_unpredicted_class_as_zero() { | |
| 1236 | 999 | // Skipping it would let a layer raise its macro precision by predicting | |
| @@ -1259,100 +1022,6 @@ | |||
| 1259 | 1022 | ); | |
| 1260 | 1023 | } | |
| 1261 | 1024 | ||
| 1262 | - | #[test] | |
| 1263 | - | fn a_fold_index_scores_a_held_out_sample_from_its_neighbours() { | |
| 1264 | - | // End to end over the real index: two tight clusters, one held out | |
| 1265 | - | // sample, and the index must answer with the cluster it sits in. | |
| 1266 | - | let mut rows = Vec::new(); | |
| 1267 | - | for i in 0..5 { | |
| 1268 | - | let mut r = row(&format!("k{i}"), Some("instrument.drum.kick")); | |
| 1269 | - | r.vector = vec![0.01 * f64::from(i); NUM_FEATURES]; | |
| 1270 | - | rows.push(r); | |
| 1271 | - | } | |
| 1272 | - | for i in 0..5 { | |
| 1273 | - | let mut r = row(&format!("s{i}"), Some("instrument.drum.snare")); | |
| 1274 | - | r.vector = vec![100.0 + f64::from(i); NUM_FEATURES]; | |
| 1275 | - | rows.push(r); | |
| 1276 | - | } | |
| 1277 | - | let refs: Vec<&Row> = rows.iter().collect(); | |
| 1278 | - | let db = training_db(&refs).unwrap(); | |
| 1279 | - | let index = exemplar::build_index(&db).unwrap(); | |
| 1280 | - | assert_eq!(index.len(), 10); | |
| 1281 | - | ||
| 1282 | - | let query = vec![0.02; NUM_FEATURES]; | |
| 1283 | - | let scored = index.score(&query, DEFAULT_K, None); | |
| 1284 | - | assert_eq!( | |
| 1285 | - | scored.first().map(|s| s.tag.as_str()), | |
| 1286 | - | Some("instrument.drum.kick") | |
| 1287 | - | ); | |
| 1288 | - | } | |
| 1289 | - | ||
| 1290 | - | #[test] | |
| 1291 | - | fn load_rows_marks_a_multi_tagged_sample_ambiguous() { | |
| 1292 | - | let rows = [ | |
| 1293 | - | row("a", Some("instrument.drum.kick")), | |
| 1294 | - | Row { | |
| 1295 | - | hash: "b".into(), | |
| 1296 | - | vector: vec![0.5; NUM_FEATURES], | |
| 1297 | - | tags: vec![ | |
| 1298 | - | "instrument.drum.kick".into(), | |
| 1299 | - | "instrument.drum.snare".into(), | |
| 1300 | - | ], | |
| 1301 | - | truth: None, | |
| 1302 | - | origin: "kick+snare".into(), | |
| 1303 | - | }, | |
| 1304 | - | ]; | |
| 1305 | - | let refs: Vec<&Row> = rows.iter().collect(); | |
| 1306 | - | let db = training_db(&refs).unwrap(); | |
| 1307 | - | ||
| 1308 | - | let (read, dropped) = load_rows(&db, LabelSpace::Instrument).unwrap(); | |
| 1309 | - | assert_eq!(read.len(), 2); | |
| 1310 | - | assert_eq!(dropped.unprojectable, 0); | |
| 1311 | - | let b = read.iter().find(|r| r.hash == "b").unwrap(); | |
| 1312 | - | assert_eq!(b.truth, None, "two class tags means no ground truth"); | |
| 1313 | - | assert_eq!(b.tags.len(), 2, "but it still trains with both labels"); | |
| 1314 | - | } | |
| 1315 | - | ||
| 1316 | - | #[test] | |
| 1317 | - | fn the_family_projection_resolves_an_instrument_level_ambiguity() { | |
| 1318 | - | // A file in both the kick and tom folders has no instrument truth and a | |
| 1319 | - | // perfectly good family one: the coarser question is the easier question, | |
| 1320 | - | // and that has to show up as a testable row rather than a dropped one. | |
| 1321 | - | let rows = [Row { | |
| 1322 | - | hash: "a".into(), | |
| 1323 | - | vector: vec![0.5; NUM_FEATURES], | |
| 1324 | - | tags: vec!["instrument.drum.kick".into(), "instrument.drum.tom".into()], | |
| 1325 | - | truth: None, | |
| 1326 | - | origin: "kick+tom".into(), | |
| 1327 | - | }]; | |
| 1328 | - | let refs: Vec<&Row> = rows.iter().collect(); | |
| 1329 | - | let db = training_db(&refs).unwrap(); | |
| 1330 | - | ||
| 1331 | - | let (read, _) = load_rows(&db, LabelSpace::Family).unwrap(); | |
| 1332 | - | assert_eq!(read[0].truth.as_deref(), Some("family.low")); | |
| 1333 | - | assert_eq!(read[0].origin, "kick+tom"); | |
| 1334 | - | } | |
| 1335 | - | ||
| 1336 | - | #[test] | |
| 1337 | - | fn percussion_leaves_the_family_run_entirely() { | |
| 1338 | - | // Not merely untested: an exemplar carrying a guessed family label would | |
| 1339 | - | // pull the index toward that guess for every other sample too. | |
| 1340 | - | let rows = [Row { | |
| 1341 | - | hash: "p".into(), | |
| 1342 | - | vector: vec![0.5; NUM_FEATURES], | |
| 1343 | - | tags: vec!["instrument.percussion".into()], | |
| 1344 | - | truth: Some("instrument.percussion".into()), | |
| 1345 | - | origin: "percussion".into(), | |
| 1346 | - | }]; | |
| 1347 | - | let refs: Vec<&Row> = rows.iter().collect(); | |
| 1348 | - | let db = training_db(&refs).unwrap(); | |
| 1349 | - | ||
| 1350 | - | let (read, dropped) = load_rows(&db, LabelSpace::Family).unwrap(); | |
| 1351 | - | assert!(read.is_empty(), "it must not train either"); | |
| 1352 | - | assert_eq!(dropped.unprojectable, 1); | |
| 1353 | - | assert!(dropped.origins.contains("percussion")); | |
| 1354 | - | } | |
| 1355 | - | ||
| 1356 | 1025 | #[test] | |
| 1357 | 1026 | fn class_points_score_an_absent_class_as_zero() { | |
| 1358 | 1027 | // A class missing from a sample's scores is a real zero, not a gap: the |
| @@ -13,12 +13,18 @@ | |||
| 13 | 13 | //! `cargo run --release -p audiofiles-bench -- layout` blob layout migration | |
| 14 | 14 | //! `cargo run --release -p audiofiles-bench -- afcl` build the official layer | |
| 15 | 15 | //! `cargo run --release -p audiofiles-bench -- layer-eval` cross-validate that layer | |
| 16 | + | //! `cargo run --release -p audiofiles-bench -- layer-stability` does it answer twice | |
| 16 | 17 | //! | |
| 17 | 18 | //! Two modes are not measurements. `layout` is a checker: it fabricates flat | |
| 18 | 19 | //! vaults, sweeps them, and exits non-zero if any scenario fails. `afcl` is a | |
| 19 | 20 | //! generator: it turns the labelled corpus into the bundled official `.afcl`. | |
| 20 | 21 | //! Both live here because corpus walking and scratch-vault fabrication do. | |
| 21 | 22 | //! | |
| 23 | + | //! `layer-eval` and `layer-stability` are the two meters for what `afcl` | |
| 24 | + | //! generates, and they measure independent properties: whether an answer is right, | |
| 25 | + | //! and whether it is the same answer next week. A consistently wrong answer is | |
| 26 | + | //! perfectly stable, so neither can be inferred from the other. | |
| 27 | + | //! | |
| 22 | 28 | //! `layer-eval` is the meter for what `afcl` generates: stratified k-fold | |
| 23 | 29 | //! cross-validation of the k-NN layer over the same corpus, per class, against a | |
| 24 | 30 | //! ship gate written down before the first run. It shares `afcl`'s corpus import | |
| @@ -30,7 +36,8 @@ | |||
| 30 | 36 | //! `AF_BENCH_ANALYZE`, `AF_BENCH_LAYOUT_N`, `AF_BENCH_JSON` (machine-readable | |
| 31 | 37 | //! output path), `AF_AFCL_OUT` (where `afcl` writes the layer), | |
| 32 | 38 | //! `AF_BENCH_EVAL_FOLDS` and `AF_BENCH_EVAL_K` (folds and the neighbour-count | |
| 33 | - | //! sweep for `layer-eval`), | |
| 39 | + | //! sweep for `layer-eval`), `AF_BENCH_STABILITY_PROBE` (probe fraction for | |
| 40 | + | //! `layer-stability`), | |
| 34 | 41 | //! `AF_BENCH_STAGES` (files per per-stage probe during `ingest`, 0 = off). | |
| 35 | 42 | //! | |
| 36 | 43 | //! Section 1 times the analysis stages per file, against the corpus and no | |
| @@ -53,6 +60,8 @@ | |||
| 53 | 60 | mod layer_eval; | |
| 54 | 61 | mod layout; | |
| 55 | 62 | mod report; | |
| 63 | + | mod rows; | |
| 64 | + | mod stability; | |
| 56 | 65 | mod storage; | |
| 57 | 66 | ||
| 58 | 67 | use crate::report::Report; | |
| @@ -357,6 +366,24 @@ | |||
| 357 | 366 | return; | |
| 358 | 367 | } | |
| 359 | 368 | ||
| 369 | + | if args.first().map(String::as_str) == Some("layer-stability") { | |
| 370 | + | // Its own scratch vault again, so a stability run and an eval run can sit | |
| 371 | + | // side by side without either rebuilding the other's corpus. | |
| 372 | + | let vault = std::env::var("AF_BENCH_VAULT").map_or_else( | |
| 373 | + | |_| std::env::temp_dir().join("af-layer-stability"), | |
| 374 | + | PathBuf::from, | |
| 375 | + | ); | |
| 376 | + | stability::run( | |
| 377 | + | &samples_dir, | |
| 378 | + | &vault, | |
| 379 | + | &full_pipeline_config(), | |
| 380 | + | stability::k_from_env(), | |
| 381 | + | stability::probe_denominator_from_env(), | |
| 382 | + | families::LabelSpace::from_env(), | |
| 383 | + | ); | |
| 384 | + | return; | |
| 385 | + | } | |
| 386 | + | ||
| 360 | 387 | if args.first().map(String::as_str) == Some("accuracy") { | |
| 361 | 388 | let Ok(root) = std::env::var("AF_BENCH_FSL10K") else { | |
| 362 | 389 | eprintln!("set AF_BENCH_FSL10K to the extracted FSL10K root"); |
| @@ -1,0 +1,515 @@ | |||
| 1 | + | //! Scoreable corpus rows, and the in-memory indexes built from them. | |
| 2 | + | //! | |
| 3 | + | //! Extracted from [`crate::layer_eval`] when [`crate::stability`] arrived and | |
| 4 | + | //! needed the same four things: read the analysed vault back as (vector, labels, | |
| 5 | + | //! truth) rows projected onto a label space, split them without an RNG, and build | |
| 6 | + | //! a real `ExemplarIndex` from an arbitrary subset. Same reason [`crate::labelled`] | |
| 7 | + | //! exists: the step is subtle enough that two copies would drift, and the | |
| 8 | + | //! subtleties are all about what is silently excluded. | |
| 9 | + | //! | |
| 10 | + | //! What lives here is label-space-agnostic and policy-free. The gate belongs to | |
| 11 | + | //! `layer_eval`, the bar to `stability`; neither is a property of a row. | |
| 12 | + | ||
| 13 | + | use std::collections::{BTreeSet, HashMap}; | |
| 14 | + | ||
| 15 | + | use audiofiles_core::analysis::afcl::{ | |
| 16 | + | self, Afcl, AfclExemplar, AfclManifest, DEFAULT_IMPORT_WEIGHT, | |
| 17 | + | }; | |
| 18 | + | use audiofiles_core::analysis::features::{FEATURE_VERSION, NUM_FEATURES}; | |
| 19 | + | use audiofiles_core::db::Database; | |
| 20 | + | ||
| 21 | + | use crate::families::{self, LabelSpace}; | |
| 22 | + | use crate::labelled; | |
| 23 | + | ||
| 24 | + | /// One corpus sample: its vector, its labels, and the class it is scored against. | |
| 25 | + | pub(crate) struct Row { | |
| 26 | + | pub(crate) hash: String, | |
| 27 | + | pub(crate) vector: Vec<f64>, | |
| 28 | + | /// Labels in the evaluation's space, so the index under test carries the tags | |
| 29 | + | /// being graded. | |
| 30 | + | pub(crate) tags: Vec<String>, | |
| 31 | + | /// The single class this row is ground truth for, or `None` when the corpus | |
| 32 | + | /// gave it more than one. Content-addressed import collapses a file that | |
| 33 | + | /// appears in two class folders into one row carrying both tags; its true | |
| 34 | + | /// class is undecidable, so it trains but is never tested. | |
| 35 | + | pub(crate) truth: Option<String>, | |
| 36 | + | /// The corpus folder(s) this row came from, before projection. Kept so a | |
| 37 | + | /// family's members can be broken out by where they came from: `low` is kick | |
| 38 | + | /// plus tom, and whether the layer recovers a tom as readily as a kick is the | |
| 39 | + | /// open split `af-coarse-families` asks about. | |
| 40 | + | pub(crate) origin: String, | |
| 41 | + | /// The sample's filename, as imported. Only the filename rules read it, and | |
| 42 | + | /// only to answer whether this sample is in the population the layer exists | |
| 43 | + | /// to serve: `starter_rules` already labels 97.7% of this corpus off its | |
| 44 | + | /// name, and every accuracy number so far was measured on exactly that | |
| 45 | + | /// population. See [`crate::stability`]. | |
| 46 | + | pub(crate) name: String, | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | /// What [`load_rows`] set aside, so no exclusion is silent. | |
| 50 | + | pub(crate) struct Dropped { | |
| 51 | + | /// Rows whose every corpus tag projects nowhere in this label space. | |
| 52 | + | pub(crate) unprojectable: usize, | |
| 53 | + | /// Corpus labels those rows came from. | |
| 54 | + | pub(crate) origins: BTreeSet<String>, | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | /// Read the analysed corpus back out of the scratch vault as scoreable rows, | |
| 58 | + | /// projected onto `space`. | |
| 59 | + | /// | |
| 60 | + | /// The projection happens here rather than in [`labelled`] on purpose. The vault | |
| 61 | + | /// is ground truth at the finest resolution the corpus carries, which is what | |
| 62 | + | /// `afcl_gen` exports from; grading at a coarser resolution is a property of the | |
| 63 | + | /// evaluation, not of the corpus, so it belongs on the read side. That also keeps | |
| 64 | + | /// the exported layer byte-identical and leaves the retired instrument question | |
| 65 | + | /// runnable instead of deleted. | |
| 66 | + | pub(crate) fn load_rows(db: &Database, space: LabelSpace) -> Result<(Vec<Row>, Dropped), String> { | |
| 67 | + | let conn = db.conn(); | |
| 68 | + | ||
| 69 | + | let mut tags_by_hash: HashMap<String, Vec<String>> = HashMap::new(); | |
| 70 | + | { | |
| 71 | + | let mut stmt = conn | |
| 72 | + | .prepare("SELECT sample_hash, tag FROM tags") | |
| 73 | + | .map_err(|e| format!("tags query: {e}"))?; | |
| 74 | + | let rows = stmt | |
| 75 | + | .query_map([], |row| { | |
| 76 | + | Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) | |
| 77 | + | }) | |
| 78 | + | .map_err(|e| format!("tags query: {e}"))?; | |
| 79 | + | for r in rows { | |
| 80 | + | let (hash, tag) = r.map_err(|e| format!("tags row: {e}"))?; | |
| 81 | + | tags_by_hash.entry(hash).or_default().push(tag); | |
| 82 | + | } | |
| 83 | + | } | |
| 84 | + | ||
| 85 | + | // Names are a left join in spirit: a row with no `samples` entry is one this | |
| 86 | + | // module fabricated in a test, and it still has to be scoreable. | |
| 87 | + | let mut name_by_hash: HashMap<String, String> = HashMap::new(); | |
| 88 | + | { | |
| 89 | + | let mut stmt = conn | |
| 90 | + | .prepare("SELECT hash, original_name FROM samples") | |
| 91 | + | .map_err(|e| format!("names query: {e}"))?; | |
| 92 | + | let rows = stmt | |
| 93 | + | .query_map([], |row| { | |
| 94 | + | Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) | |
| 95 | + | }) | |
| 96 | + | .map_err(|e| format!("names query: {e}"))?; | |
| 97 | + | for r in rows { | |
| 98 | + | let (hash, name) = r.map_err(|e| format!("names row: {e}"))?; | |
| 99 | + | name_by_hash.insert(hash, name); | |
| 100 | + | } | |
| 101 | + | } | |
| 102 | + | ||
| 103 | + | let mut out = Vec::new(); | |
| 104 | + | let mut dropped = Dropped { | |
| 105 | + | unprojectable: 0, | |
| 106 | + | origins: BTreeSet::new(), | |
| 107 | + | }; | |
| 108 | + | let mut stmt = conn | |
| 109 | + | .prepare("SELECT hash, vector FROM sample_features WHERE feat_version = ?1 ORDER BY hash") | |
| 110 | + | .map_err(|e| format!("features query: {e}"))?; | |
| 111 | + | let rows = stmt | |
| 112 | + | .query_map([FEATURE_VERSION], |row| { | |
| 113 | + | Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) | |
| 114 | + | }) | |
| 115 | + | .map_err(|e| format!("features query: {e}"))?; | |
| 116 | + | for r in rows { | |
| 117 | + | let (hash, json) = r.map_err(|e| format!("features row: {e}"))?; | |
| 118 | + | let Some(mut corpus_tags) = tags_by_hash.remove(&hash) else { | |
| 119 | + | continue; | |
| 120 | + | }; | |
| 121 | + | corpus_tags.sort(); | |
| 122 | + | corpus_tags.dedup(); | |
| 123 | + | let vector: Vec<f64> = | |
| 124 | + | serde_json::from_str(&json).map_err(|e| format!("vector for {hash}: {e}"))?; | |
| 125 | + | // The index drops these too (`is_usable_vector`), so counting them as | |
| 126 | + | // testable would score the layer on samples it never sees. | |
| 127 | + | if vector.len() != NUM_FEATURES || !vector.iter().all(|x| x.is_finite()) { | |
| 128 | + | continue; | |
| 129 | + | } | |
| 130 | + | ||
| 131 | + | // Project, then dedup again: two instrument tags landing in one family is | |
| 132 | + | // not ambiguity, it is the coarser question being easier. A file in both | |
| 133 | + | // the kick and tom folders has no instrument truth and a perfectly good | |
| 134 | + | // family one. | |
| 135 | + | let mut tags: Vec<String> = corpus_tags | |
| 136 | + | .iter() | |
| 137 | + | .filter_map(|t| families::project(space, t)) | |
| 138 | + | .map(str::to_string) | |
| 139 | + | .collect(); | |
| 140 | + | tags.sort(); | |
| 141 | + | tags.dedup(); | |
| 142 | + | if tags.is_empty() { | |
| 143 | + | dropped.unprojectable += 1; | |
| 144 | + | dropped.origins.extend( | |
| 145 | + | corpus_tags | |
| 146 | + | .iter() | |
| 147 | + | .map(|t| labelled::label_for_tag(t).to_string()), | |
| 148 | + | ); | |
| 149 | + | continue; | |
| 150 | + | } | |
| 151 | + | ||
| 152 | + | let truth = (tags.len() == 1).then(|| tags[0].clone()); | |
| 153 | + | let origin = corpus_tags | |
| 154 | + | .iter() | |
| 155 | + | .map(|t| labelled::label_for_tag(t)) | |
| 156 | + | .collect::<Vec<_>>() | |
| 157 | + | .join("+"); | |
| 158 | + | let name = name_by_hash.get(&hash).cloned().unwrap_or_default(); | |
| 159 | + | out.push(Row { | |
| 160 | + | hash, | |
| 161 | + | vector, | |
| 162 | + | tags, | |
| 163 | + | truth, | |
| 164 | + | origin, | |
| 165 | + | name, | |
| 166 | + | }); | |
| 167 | + | } | |
| 168 | + | Ok((out, dropped)) | |
| 169 | + | } | |
| 170 | + | ||
| 171 | + | /// Assign each row a fold, stratified by class. | |
| 172 | + | /// | |
| 173 | + | /// Round-robin down each class's hash-sorted list rather than a shuffle: the | |
| 174 | + | /// content hash is already an arbitrary order with respect to the source pack a | |
| 175 | + | /// file came from, so this needs no RNG and two runs over the same corpus produce | |
| 176 | + | /// the same folds. Ambiguous rows get no fold; they train everywhere. | |
| 177 | + | pub(crate) fn assign_folds(rows: &[Row], folds: usize) -> Vec<Option<usize>> { | |
| 178 | + | let mut seen_per_class: HashMap<&str, usize> = HashMap::new(); | |
| 179 | + | rows.iter() | |
| 180 | + | .map(|r| { | |
| 181 | + | let truth = r.truth.as_deref()?; | |
| 182 | + | let n = seen_per_class.entry(truth).or_default(); | |
| 183 | + | let fold = *n % folds; | |
| 184 | + | *n += 1; | |
| 185 | + | Some(fold) | |
| 186 | + | }) | |
| 187 | + | .collect() | |
| 188 | + | } | |
| 189 | + | ||
| 190 | + | /// Build an in-memory vault holding just `local` rows, so the index under test is | |
| 191 | + | /// built by the same `build_index` the app calls. | |
| 192 | + | /// | |
| 193 | + | /// Every exemplar here is `local` (weight 1.0) where the shipped ones arrive from | |
| 194 | + | /// an imported layer at a lower weight. That does not change a score: the weight | |
| 195 | + | /// is a constant multiplier inside a sum that is then divided by its own total, | |
| 196 | + | /// so a uniform weight cancels. It matters only against a mixed index of user | |
| 197 | + | /// labels plus the layer, which is [`mixed_db`]. | |
| 198 | + | pub(crate) fn local_db(local: &[&Row]) -> Result<Database, String> { | |
| 199 | + | mixed_db(local, &[]) | |
| 200 | + | } | |
| 201 | + | ||
| 202 | + | /// Build an in-memory vault holding `local` rows at weight 1.0 and `imported` | |
| 203 | + | /// rows as an enabled `.afcl` layer at [`DEFAULT_IMPORT_WEIGHT`]. | |
| 204 | + | /// | |
| 205 | + | /// This is the deployment shape, and it is not a uniform index: the weight only | |
| 206 | + | /// cancels when every exemplar carries the same one. Here the ratio is real, and | |
| 207 | + | /// so is the second-order effect that makes this worth simulating rather than | |
| 208 | + | /// reasoning about — `build_index` fits the standardization means and standard | |
| 209 | + | /// deviations over local **and** imported exemplars together, so adding a user's | |
| 210 | + | /// labels moves the space every distance is measured in. | |
| 211 | + | /// | |
| 212 | + | /// The layer goes in through `afcl::import` rather than a hand-written INSERT so | |
| 213 | + | /// the weight, the enabled flag and the `feat_version` gate are the ones the app | |
| 214 | + | /// applies. Imported exemplars carry vector + tags and no hash, exactly as a | |
| 215 | + | /// shipped layer does. | |
| 216 | + | pub(crate) fn mixed_db(local: &[&Row], imported: &[&Row]) -> Result<Database, String> { | |
| 217 | + | let db = Database::open_in_memory().map_err(|e| format!("open_in_memory: {e}"))?; | |
| 218 | + | db.transaction(|_tx| { | |
| 219 | + | for row in local { | |
| 220 | + | db.conn().execute( | |
| 221 | + | "INSERT INTO samples (hash, original_name, file_extension, file_size, import_date, last_modified) | |
| 222 | + | VALUES (?1, ?2, 'wav', 1, 0, 0)", | |
| 223 | + | rusqlite::params![&row.hash, &row.name], | |
| 224 | + | )?; | |
| 225 | + | let json = serde_json::to_string(&row.vector).unwrap_or_default(); | |
| 226 | + | db.conn().execute( | |
| 227 | + | "INSERT INTO sample_features (hash, feat_version, vector, computed_at) VALUES (?1, ?2, ?3, 0)", | |
| 228 | + | rusqlite::params![&row.hash, FEATURE_VERSION, json], | |
| 229 | + | )?; | |
| 230 | + | } | |
| 231 | + | Ok(()) | |
| 232 | + | }) | |
| 233 | + | .map_err(|e| format!("seeding the index: {e}"))?; | |
| 234 | + | // Tags outside the transaction: `add_tag` is the public path and manages its | |
| 235 | + | // own writes, and this is an in-memory DB where the fsync cost it avoids does | |
| 236 | + | // not exist. | |
| 237 | + | for row in local { | |
| 238 | + | for tag in &row.tags { | |
| 239 | + | audiofiles_core::tags::add_tag(&db, &row.hash, tag) | |
| 240 | + | .map_err(|e| format!("tagging the index: {e}"))?; | |
| 241 | + | } | |
| 242 | + | } | |
| 243 | + | ||
| 244 | + | if !imported.is_empty() { | |
| 245 | + | let exemplars: Vec<AfclExemplar> = imported | |
| 246 | + | .iter() | |
| 247 | + | .map(|r| AfclExemplar { | |
| 248 | + | vector: r.vector.clone(), | |
| 249 | + | tags: r.tags.clone(), | |
| 250 | + | }) | |
| 251 | + | .collect(); | |
| 252 | + | let layer = Afcl { | |
| 253 | + | manifest: AfclManifest { | |
| 254 | + | afcl_version: afcl::AFCL_VERSION, | |
| 255 | + | feat_version: FEATURE_VERSION, | |
| 256 | + | name: "simulated bundled layer".into(), | |
| 257 | + | description: "built by audiofiles-bench, never written to disk".into(), | |
| 258 | + | kind: afcl::LayerKind::Official.as_str().to_string(), | |
| 259 | + | created_at: 0, | |
| 260 | + | license_note: String::new(), | |
| 261 | + | exemplar_count: exemplars.len(), | |
| 262 | + | rule_count: 0, | |
| 263 | + | policy_count: 0, | |
| 264 | + | }, | |
| 265 | + | exemplars, | |
| 266 | + | rules: Vec::new(), | |
| 267 | + | policy: Vec::new(), | |
| 268 | + | }; | |
| 269 | + | let summary = afcl::import(&db, &layer, Some(afcl::BUNDLED_SOURCE)) | |
| 270 | + | .map_err(|e| format!("importing the simulated layer: {e}"))?; | |
| 271 | + | // An exemplar dropped at import is one the measurement thinks it has and | |
| 272 | + | // does not. `load_rows` already applied the same finiteness filter, so a | |
| 273 | + | // mismatch here means the two disagree and the run is not what it says. | |
| 274 | + | if summary.exemplars != imported.len() { | |
| 275 | + | return Err(format!( | |
| 276 | + | "the simulated layer imported {} of {} exemplars", | |
| 277 | + | summary.exemplars, | |
| 278 | + | imported.len() | |
| 279 | + | )); | |
| 280 | + | } | |
| 281 | + | } | |
| 282 | + | Ok(db) | |
| 283 | + | } | |
| 284 | + | ||
| 285 | + | /// The weight an imported layer carries beside the user's own labels, restated | |
| 286 | + | /// here so a report can print the ratio it was measured at. | |
| 287 | + | pub(crate) const IMPORT_WEIGHT: f64 = DEFAULT_IMPORT_WEIGHT; | |
| 288 | + | ||
| 289 | + | #[cfg(test)] | |
| 290 | + | mod tests { | |
| 291 | + | use super::*; | |
| 292 | + | ||
| 293 | + | fn row(hash: &str, tag: &str) -> Row { | |
| 294 | + | Row { | |
| 295 | + | hash: hash.into(), | |
| 296 | + | vector: vec![0.5; NUM_FEATURES], | |
| 297 | + | tags: vec![tag.into()], | |
| 298 | + | truth: Some(tag.into()), | |
| 299 | + | origin: labelled::label_for_tag(tag).into(), | |
| 300 | + | name: format!("{hash}.wav"), | |
| 301 | + | } | |
| 302 | + | } | |
| 303 | + | ||
| 304 | + | #[test] | |
| 305 | + | fn folds_are_stratified_and_ambiguous_rows_get_none() { | |
| 306 | + | let mut rows: Vec<Row> = (0..10) | |
| 307 | + | .map(|i| row(&format!("k{i}"), "instrument.drum.kick")) | |
| 308 | + | .collect(); | |
| 309 | + | rows.extend((0..4).map(|i| row(&format!("s{i}"), "instrument.drum.snare"))); | |
| 310 | + | rows.push(Row { | |
| 311 | + | hash: "dup".into(), | |
| 312 | + | vector: vec![0.0; NUM_FEATURES], | |
| 313 | + | tags: Vec::new(), | |
| 314 | + | truth: None, | |
| 315 | + | origin: String::new(), | |
| 316 | + | name: "dup.wav".into(), | |
| 317 | + | }); | |
| 318 | + | ||
| 319 | + | let folds = assign_folds(&rows, 5); | |
| 320 | + | assert_eq!( | |
| 321 | + | folds.last().copied().flatten(), | |
| 322 | + | None, | |
| 323 | + | "ambiguous row trains everywhere" | |
| 324 | + | ); | |
| 325 | + | ||
| 326 | + | // Every fold holds two of the ten kicks: stratification, not a global | |
| 327 | + | // round-robin that would leave a fold without any snares. | |
| 328 | + | for f in 0..5 { | |
| 329 | + | let kicks = rows | |
| 330 | + | .iter() | |
| 331 | + | .zip(&folds) | |
| 332 | + | .filter(|(r, fold)| { | |
| 333 | + | **fold == Some(f) && r.truth.as_deref() == Some("instrument.drum.kick") | |
| 334 | + | }) | |
| 335 | + | .count(); | |
| 336 | + | assert_eq!(kicks, 2, "fold {f}"); | |
| 337 | + | } | |
| 338 | + | // Four snares over five folds: one fold is short, and none holds two. | |
| 339 | + | for f in 0..5 { | |
| 340 | + | let snares = rows | |
| 341 | + | .iter() | |
| 342 | + | .zip(&folds) | |
| 343 | + | .filter(|(r, fold)| { | |
| 344 | + | **fold == Some(f) && r.truth.as_deref() == Some("instrument.drum.snare") | |
| 345 | + | }) | |
| 346 | + | .count(); | |
| 347 | + | assert!(snares <= 1, "fold {f} holds {snares} snares"); | |
| 348 | + | } | |
| 349 | + | } | |
| 350 | + | ||
| 351 | + | #[test] | |
| 352 | + | fn folds_are_deterministic() { | |
| 353 | + | let rows: Vec<Row> = (0..20) | |
| 354 | + | .map(|i| row(&format!("k{i}"), "instrument.drum.kick")) | |
| 355 | + | .collect(); | |
| 356 | + | assert_eq!(assign_folds(&rows, 5), assign_folds(&rows, 5)); | |
| 357 | + | } | |
| 358 | + | ||
| 359 | + | #[test] | |
| 360 | + | fn a_fold_index_scores_a_held_out_sample_from_its_neighbours() { | |
| 361 | + | // End to end over the real index: two tight clusters, one held out | |
| 362 | + | // sample, and the index must answer with the cluster it sits in. | |
| 363 | + | let mut rows = Vec::new(); | |
| 364 | + | for i in 0..5 { | |
| 365 | + | let mut r = row(&format!("k{i}"), "instrument.drum.kick"); | |
| 366 | + | r.vector = vec![0.01 * f64::from(i); NUM_FEATURES]; | |
| 367 | + | rows.push(r); | |
| 368 | + | } | |
| 369 | + | for i in 0..5 { | |
| 370 | + | let mut r = row(&format!("s{i}"), "instrument.drum.snare"); | |
| 371 | + | r.vector = vec![100.0 + f64::from(i); NUM_FEATURES]; | |
| 372 | + | rows.push(r); | |
| 373 | + | } | |
| 374 | + | let refs: Vec<&Row> = rows.iter().collect(); | |
| 375 | + | let db = local_db(&refs).unwrap(); | |
| 376 | + | let index = audiofiles_core::analysis::exemplar::build_index(&db).unwrap(); | |
| 377 | + | assert_eq!(index.len(), 10); | |
| 378 | + | ||
| 379 | + | let query = vec![0.02; NUM_FEATURES]; | |
| 380 | + | let scored = index.score(&query, audiofiles_core::analysis::exemplar::DEFAULT_K, None); | |
| 381 | + | assert_eq!( | |
| 382 | + | scored.first().map(|s| s.tag.as_str()), | |
| 383 | + | Some("instrument.drum.kick") | |
| 384 | + | ); | |
| 385 | + | } | |
| 386 | + | ||
| 387 | + | #[test] | |
| 388 | + | fn load_rows_marks_a_multi_tagged_sample_ambiguous() { | |
| 389 | + | let rows = [ | |
| 390 | + | row("a", "instrument.drum.kick"), | |
| 391 | + | Row { | |
| 392 | + | hash: "b".into(), | |
| 393 | + | vector: vec![0.5; NUM_FEATURES], | |
| 394 | + | tags: vec![ | |
| 395 | + | "instrument.drum.kick".into(), | |
| 396 | + | "instrument.drum.snare".into(), | |
| 397 | + | ], | |
| 398 | + | truth: None, | |
| 399 | + | origin: "kick+snare".into(), | |
| 400 | + | name: "b.wav".into(), | |
| 401 | + | }, | |
| 402 | + | ]; | |
| 403 | + | let refs: Vec<&Row> = rows.iter().collect(); | |
| 404 | + | let db = local_db(&refs).unwrap(); | |
| 405 | + | ||
| 406 | + | let (read, dropped) = load_rows(&db, LabelSpace::Instrument).unwrap(); | |
| 407 | + | assert_eq!(read.len(), 2); | |
| 408 | + | assert_eq!(dropped.unprojectable, 0); | |
| 409 | + | let b = read.iter().find(|r| r.hash == "b").unwrap(); | |
| 410 | + | assert_eq!(b.truth, None, "two class tags means no ground truth"); | |
| 411 | + | assert_eq!(b.tags.len(), 2, "but it still trains with both labels"); | |
| 412 | + | } | |
| 413 | + | ||
| 414 | + | #[test] | |
| 415 | + | fn the_family_projection_resolves_an_instrument_level_ambiguity() { | |
| 416 | + | // A file in both the kick and tom folders has no instrument truth and a | |
| 417 | + | // perfectly good family one: the coarser question is the easier question, | |
| 418 | + | // and that has to show up as a testable row rather than a dropped one. | |
| 419 | + | let rows = [Row { | |
| 420 | + | hash: "a".into(), | |
| 421 | + | vector: vec![0.5; NUM_FEATURES], | |
| 422 | + | tags: vec!["instrument.drum.kick".into(), "instrument.drum.tom".into()], | |
| 423 | + | truth: None, | |
| 424 | + | origin: "kick+tom".into(), | |
| 425 | + | name: "a.wav".into(), | |
| 426 | + | }]; | |
| 427 | + | let refs: Vec<&Row> = rows.iter().collect(); | |
| 428 | + | let db = local_db(&refs).unwrap(); | |
| 429 | + | ||
| 430 | + | let (read, _) = load_rows(&db, LabelSpace::Family).unwrap(); | |
| 431 | + | assert_eq!(read[0].truth.as_deref(), Some("family.low")); | |
| 432 | + | assert_eq!(read[0].origin, "kick+tom"); | |
| 433 | + | } | |
| 434 | + | ||
| 435 | + | #[test] | |
| 436 | + | fn percussion_leaves_the_family_run_entirely() { | |
| 437 | + | // Not merely untested: an exemplar carrying a guessed family label would | |
| 438 | + | // pull the index toward that guess for every other sample too. | |
| 439 | + | let rows = [Row { | |
| 440 | + | hash: "p".into(), | |
| 441 | + | vector: vec![0.5; NUM_FEATURES], | |
| 442 | + | tags: vec!["instrument.percussion".into()], | |
| 443 | + | truth: Some("instrument.percussion".into()), | |
| 444 | + | origin: "percussion".into(), | |
| 445 | + | name: "p.wav".into(), | |
| 446 | + | }]; | |
| 447 | + | let refs: Vec<&Row> = rows.iter().collect(); | |
| 448 | + | let db = local_db(&refs).unwrap(); | |
| 449 | + | ||
| 450 | + | let (read, dropped) = load_rows(&db, LabelSpace::Family).unwrap(); | |
| 451 | + | assert!(read.is_empty(), "it must not train either"); | |
| 452 | + | assert_eq!(dropped.unprojectable, 1); | |
| 453 | + | assert!(dropped.origins.contains("percussion")); | |
| 454 | + | } | |
| 455 | + | ||
| 456 | + | #[test] | |
| 457 | + | fn load_rows_carries_the_filename_back() { | |
| 458 | + | // The rule-miss population is keyed off this, and a silently empty name | |
| 459 | + | // would read as "no filename rule fires" for the whole corpus, which is | |
| 460 | + | // the opposite of the truth. | |
| 461 | + | let rows = [row("a", "instrument.drum.kick")]; | |
| 462 | + | let refs: Vec<&Row> = rows.iter().collect(); | |
| 463 | + | let db = local_db(&refs).unwrap(); | |
| 464 | + | let (read, _) = load_rows(&db, LabelSpace::Instrument).unwrap(); | |
| 465 | + | assert_eq!(read[0].name, "a.wav"); | |
| 466 | + | } | |
| 467 | + | ||
| 468 | + | #[test] | |
| 469 | + | fn a_mixed_index_holds_both_populations() { | |
| 470 | + | let local: Vec<Row> = (0..3) | |
| 471 | + | .map(|i| row(&format!("l{i}"), "instrument.drum.kick")) | |
| 472 | + | .collect(); | |
| 473 | + | let imported: Vec<Row> = (0..4) | |
| 474 | + | .map(|i| row(&format!("i{i}"), "instrument.drum.snare")) | |
| 475 | + | .collect(); | |
| 476 | + | let l: Vec<&Row> = local.iter().collect(); | |
| 477 | + | let i: Vec<&Row> = imported.iter().collect(); | |
| 478 | + | let db = mixed_db(&l, &i).unwrap(); | |
| 479 | + | ||
| 480 | + | let index = audiofiles_core::analysis::exemplar::build_index(&db).unwrap(); | |
| 481 | + | assert_eq!(index.len(), 7, "local and imported both reach the index"); | |
| 482 | + | ||
| 483 | + | let layers = afcl::list_layers(&db).unwrap(); | |
| 484 | + | assert_eq!(layers.len(), 1); | |
| 485 | + | assert!(layers[0].enabled, "a disabled layer contributes nothing"); | |
| 486 | + | assert!((layers[0].weight - IMPORT_WEIGHT).abs() < 1e-9); | |
| 487 | + | } | |
| 488 | + | ||
| 489 | + | #[test] | |
| 490 | + | fn an_imported_exemplar_outvoted_by_local_ones_shows_the_weight_is_live() { | |
| 491 | + | // The property `mixed_db` exists for. Two exemplars equidistant from the | |
| 492 | + | // query, one local and one imported: the local tag must score 1/(1+0.5) | |
| 493 | + | // of the neighbourhood rather than half of it. If the weight were being | |
| 494 | + | // cancelled away this would come back 0.5. | |
| 495 | + | let mut near_local = row("l0", "instrument.drum.kick"); | |
| 496 | + | near_local.vector = vec![0.0; NUM_FEATURES]; | |
| 497 | + | let mut near_imported = row("i0", "instrument.drum.snare"); | |
| 498 | + | near_imported.vector = vec![0.0; NUM_FEATURES]; | |
| 499 | + | ||
| 500 | + | let db = mixed_db(&[&near_local], &[&near_imported]).unwrap(); |
Lines truncated
| @@ -1,0 +1,1598 @@ | |||
| 1 | + | //! Does the layer give the same answer twice? The queue-stability meter. | |
| 2 | + | //! | |
| 3 | + | //! [`crate::layer_eval`] asks whether an answer is right. This asks whether it is | |
| 4 | + | //! the same answer next week. For a layer whose model is the user's own library | |
| 5 | + | //! those are independent properties, and only the second one is measured here: a | |
| 6 | + | //! consistently wrong answer is perfectly stable, so nothing below can be inferred | |
| 7 | + | //! from the 95.5% / 97.4% family-resolution accuracy figures, and nothing below | |
| 8 | + | //! substitutes for them. | |
| 9 | + | //! | |
| 10 | + | //! # Why this is an attention gate and not a safety gate | |
| 11 | + | //! | |
| 12 | + | //! The bundled layer ships **suggest-only** (decided 2026-08-07): it never | |
| 13 | + | //! auto-applies, it populates a review queue. That kills the failure this | |
| 14 | + | //! measurement was originally filed against. The apply path is monotonic | |
| 15 | + | //! (`apply_policy` skips tags the sample already has, `apply_tag_sourced` is | |
| 16 | + | //! INSERT OR IGNORE, only `remove_tags_by_source` removes), so a changed answer | |
| 17 | + | //! used to mean the sample kept its old tag *and* gained the new one, permanently. | |
| 18 | + | //! Nothing is written unasked now, so that cannot happen. | |
| 19 | + | //! | |
| 20 | + | //! What survives is cheaper and still real: a queue whose contents reshuffle | |
| 21 | + | //! between runs spends the user's attention twice. Someone who read 340 low | |
| 22 | + | //! suggestions last month should not be handed a substantially different 340 this | |
| 23 | + | //! month for the same unchanged samples. So the unit of measurement is the | |
| 24 | + | //! **queue entry**, not the written tag: a sample's answer is its top-scoring tag | |
| 25 | + | //! if that tag clears the review threshold, and `silent` otherwise. | |
| 26 | + | //! | |
| 27 | + | //! # The bar, written before the first run | |
| 28 | + | //! | |
| 29 | + | //! Same discipline as the ship gate, and for the same reason: a number chosen | |
| 30 | + | //! after reading one is not a bar. See [`BAR`]. | |
| 31 | + | //! | |
| 32 | + | //! # What varies, and why those four | |
| 33 | + | //! | |
| 34 | + | //! Each measurement holds the probe set fixed and varies one property of the | |
| 35 | + | //! index, because a flip rate is meaningless without saying what moved. | |
| 36 | + | //! | |
| 37 | + | //! 1. [`composition`] — same size, different class mix. The user whose library is | |
| 38 | + | //! mostly kicks and the user whose library is mostly cymbals are running the | |
| 39 | + | //! same code against different models. | |
| 40 | + | //! 2. [`size`] — same mix, growing index. A library grows by accretion, so this is | |
| 41 | + | //! the shape of every real user's second month. | |
| 42 | + | //! 3. [`deployment_shape`] — the mixed index the app actually builds: the user's | |
| 43 | + | //! own labels at 1.0 beside the bundled layer at `DEFAULT_IMPORT_WEIGHT` 0.5. | |
| 44 | + | //! Never measured before this module, and the most decision-relevant of the | |
| 45 | + | //! four. Every earlier number came off a uniform-weight index, where the weight | |
| 46 | + | //! is a constant multiplier inside a sum divided by its own total and therefore | |
| 47 | + | //! cancels. It does not cancel here, and neither does the second-order effect: | |
| 48 | + | //! `build_index` fits the standardization params over local and imported | |
| 49 | + | //! exemplars together, so a user's labels move the space distances are measured | |
| 50 | + | //! in. This is the difference between a queue that helps and one that repeats | |
| 51 | + | //! what the user already knows. | |
| 52 | + | //! 4. [`feedback`] — the queue changes its own inputs. An accepted suggestion | |
| 53 | + | //! becomes a user label at weight 1.0 and re-enters the index, so working | |
| 54 | + | //! through the queue rewrites the rest of it. Converges or oscillates is a | |
| 55 | + | //! question no other measurement here can answer. | |
| 56 | + | //! | |
| 57 | + | //! Usage: `cargo run --release -p audiofiles-bench -- layer-stability` | |
| 58 | + | //! Env: `AF_BENCH_CORPUS`, `AF_BENCH_VAULT`, `AF_BENCH_EVAL_K` (first entry wins, | |
| 59 | + | //! default `DEFAULT_K`), `AF_BENCH_EVAL_LABELS`, `AF_BENCH_STABILITY_PROBE` | |
| 60 | + | //! (probe fraction denominator, default 5), `AF_BENCH_JSON`. | |
| 61 | + | ||
| 62 | + | use std::collections::{BTreeMap, BTreeSet, HashMap}; | |
| 63 | + | use std::path::Path; | |
| 64 | + | ||
| 65 | + | use audiofiles_core::analysis::config::AnalysisConfig; | |
| 66 | + | use audiofiles_core::analysis::exemplar::{self, DEFAULT_K, DEFAULT_REVIEW_THRESHOLD}; | |
| 67 | + | use audiofiles_core::analysis::features::FEATURE_VERSION; | |
| 68 | + | use audiofiles_core::rules::{RuleContext, RuleField}; | |
| 69 | + | use audiofiles_core::{rules, starter_rules}; | |
| 70 | + | ||
| 71 | + | use crate::families::{self, LabelSpace}; | |
| 72 | + | use crate::labelled; | |
| 73 | + | use crate::report::Report; | |
| 74 | + | use crate::rows::{self, Row}; | |
| 75 | + | ||
| 76 | + | /// The bar, stated before the first run. | |
| 77 | + | /// | |
| 78 | + | /// Per-run family-label flip rate on a fixed held-out probe set, measured at the | |
| 79 | + | /// deployment weight, counting only flips between two answers that both cleared | |
| 80 | + | /// the review threshold. A sample moving between suggested and silent costs the | |
| 81 | + | /// user nothing (the queue is shorter or longer, not wrong), so it is reported | |
| 82 | + | /// separately as churn rather than counted here. | |
| 83 | + | /// | |
| 84 | + | /// 2% is proposed rather than settled: on a 170-sample probe set it is three | |
| 85 | + | /// samples, which is the resolution this corpus supports and not a claim that 2% | |
| 86 | + | /// is where a user stops noticing. Max accepts or moves it. | |
| 87 | + | const BAR: f64 = 0.02; | |
| 88 | + | ||
| 89 | + | /// A sample either has a queue entry or it does not. | |
| 90 | + | /// | |
| 91 | + | /// The threshold is [`DEFAULT_REVIEW_THRESHOLD`] and not the auto threshold on | |
| 92 | + | /// purpose: under suggest-only nothing auto-applies, so the review threshold is | |
| 93 | + | /// the only line that decides whether the user ever sees the answer. | |
| 94 | + | #[derive(Clone, PartialEq, Eq)] | |
| 95 | + | enum Answer { | |
| 96 | + | Silent, | |
| 97 | + | Tag(String), | |
| 98 | + | } | |
| 99 | + | ||
| 100 | + | impl Answer { | |
| 101 | + | fn of(index: &exemplar::ExemplarIndex, vector: &[f64], k: usize) -> Self { | |
| 102 | + | index | |
| 103 | + | .score(vector, k, None) | |
| 104 | + | .first() | |
| 105 | + | .filter(|s| s.score >= DEFAULT_REVIEW_THRESHOLD) | |
| 106 | + | .map_or(Self::Silent, |s| Self::Tag(s.tag.clone())) | |
| 107 | + | } | |
| 108 | + | ||
| 109 | + | fn tag(&self) -> Option<&str> { | |
| 110 | + | match self { | |
| 111 | + | Self::Silent => None, | |
| 112 | + | Self::Tag(t) => Some(t), | |
| 113 | + | } | |
| 114 | + | } | |
| 115 | + | } | |
| 116 | + | ||
| 117 | + | /// One index variant's answer for every probe sample, in probe order. | |
| 118 | + | type Answers = Vec<Answer>; | |
| 119 | + | ||
| 120 | + | /// How two runs' queues differ. | |
| 121 | + | struct Churn { | |
| 122 | + | /// Probe samples suggested in both runs. The denominator of [`Self::flip_rate`]. | |
| 123 | + | both: usize, | |
| 124 | + | /// Suggested in both, and the tag changed. What the bar is set against. | |
| 125 | + | flips: usize, | |
| 126 | + | /// Silent then suggested: the queue grew. | |
| 127 | + | appeared: usize, | |
| 128 | + | /// Suggested then silent: the queue shrank. | |
| 129 | + | vanished: usize, | |
| 130 | + | } | |
| 131 | + | ||
| 132 | + | impl Churn { | |
| 133 | + | fn between(before: &Answers, after: &Answers) -> Self { | |
| 134 | + | let mut churn = Self { | |
| 135 | + | both: 0, | |
| 136 | + | flips: 0, | |
| 137 | + | appeared: 0, | |
| 138 | + | vanished: 0, | |
| 139 | + | }; | |
| 140 | + | for (x, y) in before.iter().zip(after) { | |
| 141 | + | match (x.tag(), y.tag()) { | |
| 142 | + | (Some(was), Some(now)) => { | |
| 143 | + | churn.both += 1; | |
| 144 | + | if was != now { | |
| 145 | + | churn.flips += 1; | |
| 146 | + | } | |
| 147 | + | } | |
| 148 | + | (None, Some(_)) => churn.appeared += 1, | |
| 149 | + | (Some(_), None) => churn.vanished += 1, | |
| 150 | + | (None, None) => {} | |
| 151 | + | } | |
| 152 | + | } | |
| 153 | + | churn | |
| 154 | + | } | |
| 155 | + | ||
| 156 | + | /// `None` when nothing was suggested in both runs, which is not a flip rate of | |
| 157 | + | /// zero: it is a pair of runs with no overlapping queue to compare. | |
| 158 | + | fn flip_rate(&self) -> Option<f64> { | |
| 159 | + | (self.both > 0).then(|| self.flips as f64 / self.both as f64) | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | /// The flip rate, but only when enough samples stood behind it to mean | |
| 163 | + | /// anything. See [`MIN_COMPARABLE`]. | |
| 164 | + | fn comparable_flip_rate(&self) -> Option<f64> { | |
| 165 | + | self.flip_rate().filter(|_| self.both >= MIN_COMPARABLE) | |
| 166 | + | } | |
| 167 | + | ||
| 168 | + | fn passes(&self) -> bool { | |
| 169 | + | self.flip_rate().is_some_and(|r| r <= BAR) | |
| 170 | + | } | |
| 171 | + | } | |
| 172 | + | ||
| 173 | + | /// Fewest overlapping queue entries a pair of runs needs before its flip rate | |
| 174 | + | /// counts toward a verdict. | |
| 175 | + | /// | |
| 176 | + | /// Same job as `layer_eval`'s `min_support` and learned the same way. The first | |
| 177 | + | /// run of [`feedback`] ended with seven samples still open and two of them | |
| 178 | + | /// flipping, which is 28.6% and became the worst figure in the whole report. It | |
| 179 | + | /// is one sample either way. Rates over thin tails are still printed — a tail | |
| 180 | + | /// that thrashes is worth seeing — but they do not decide a pass. | |
| 181 | + | const MIN_COMPARABLE: usize = 30; | |
| 182 | + | ||
| 183 | + | fn pct(v: Option<f64>) -> String { | |
| 184 | + | v.map_or_else(|| "-".to_string(), |x| format!("{:.1}%", x * 100.0)) | |
| 185 | + | } | |
| 186 | + | ||
| 187 | + | fn round4(v: f64) -> f64 { | |
| 188 | + | (v * 10_000.0).round() / 10_000.0 | |
| 189 | + | } | |
| 190 | + | ||
| 191 | + | /// Score every probe row against one index. | |
| 192 | + | fn answers(index: &exemplar::ExemplarIndex, probe: &[&Row], k: usize) -> Answers { | |
| 193 | + | probe | |
| 194 | + | .iter() | |
| 195 | + | .map(|r| Answer::of(index, &r.vector, k)) | |
| 196 | + | .collect() | |
| 197 | + | } | |
| 198 | + | ||
| 199 | + | /// Build an index from `local` (weight 1.0) plus `imported` (the layer weight) and | |
| 200 | + | /// score the probe against it. | |
| 201 | + | fn run_variant(local: &[&Row], imported: &[&Row], probe: &[&Row], k: usize, what: &str) -> Answers { | |
| 202 | + | let db = match rows::mixed_db(local, imported) { | |
| 203 | + | Ok(db) => db, | |
| 204 | + | Err(e) => { | |
| 205 | + | eprintln!("{what}: {e}"); | |
| 206 | + | std::process::exit(1); | |
| 207 | + | } | |
| 208 | + | }; | |
| 209 | + | let index = match exemplar::build_index(&db) { | |
| 210 | + | Ok(i) => i, | |
| 211 | + | Err(e) => { | |
| 212 | + | eprintln!("{what}: build_index: {e}"); | |
| 213 | + | std::process::exit(1); | |
| 214 | + | } | |
| 215 | + | }; | |
| 216 | + | answers(&index, probe, k) | |
| 217 | + | } | |
| 218 | + | ||
| 219 | + | /// Take `n` rows of class `class` from `pool`, in pool order. | |
| 220 | + | /// | |
| 221 | + | /// Prefix rather than a sample: pool order is content-hash order, which is already | |
| 222 | + | /// arbitrary with respect to the pack a file came from, so this needs no RNG and | |
| 223 | + | /// two runs produce the same variants. It also makes the size sweep nested by | |
| 224 | + | /// construction, which is the property [`size`] wants: a growing library accretes, | |
| 225 | + | /// it does not resample. | |
| 226 | + | fn take_class<'a>(pool: &[&'a Row], class: &str, n: usize) -> Vec<&'a Row> { | |
| 227 | + | pool.iter() | |
| 228 | + | .filter(|r| r.truth.as_deref() == Some(class)) | |
| 229 | + | .take(n) | |
| 230 | + | .copied() | |
| 231 | + | .collect() | |
| 232 | + | } | |
| 233 | + | ||
| 234 | + | /// Split a pool into two by stratified round-robin, so both halves carry the same | |
| 235 | + | /// class mix and neither is a prefix of the other. | |
| 236 | + | fn halve<'a>(pool: &[&'a Row]) -> (Vec<&'a Row>, Vec<&'a Row>) { | |
| 237 | + | let mut seen: HashMap<&str, usize> = HashMap::new(); | |
| 238 | + | let mut a = Vec::new(); | |
| 239 | + | let mut b = Vec::new(); | |
| 240 | + | for r in pool { | |
| 241 | + | let key = r.truth.as_deref().unwrap_or(""); | |
| 242 | + | let n = seen.entry(key).or_default(); | |
| 243 | + | if (*n).is_multiple_of(2) { | |
| 244 | + | a.push(*r); | |
| 245 | + | } else { | |
| 246 | + | b.push(*r); | |
| 247 | + | } | |
| 248 | + | *n += 1; | |
| 249 | + | } | |
| 250 | + | (a, b) | |
| 251 | + | } | |
| 252 | + | ||
| 253 | + | pub(crate) fn run( | |
| 254 | + | corpus: &Path, | |
| 255 | + | vault: &Path, | |
| 256 | + | config: &AnalysisConfig, | |
| 257 | + | k: usize, | |
| 258 | + | probe_denominator: usize, | |
| 259 | + | space: LabelSpace, | |
| 260 | + | ) { | |
| 261 | + | println!("━━━ CLASSIFIER LAYER STABILITY ━━━"); | |
| 262 | + | println!(); | |
| 263 | + | println!(" corpus {}", corpus.display()); | |
| 264 | + | println!(" scratch {}", vault.display()); | |
| 265 | + | println!(" features v{FEATURE_VERSION}"); | |
| 266 | + | println!(" k {k}"); | |
| 267 | + | println!(" labels {}", space.describe()); | |
| 268 | + | println!( | |
| 269 | + | " answer top-scoring tag at score >= {DEFAULT_REVIEW_THRESHOLD} (the review\n \ | |
| 270 | + | threshold), else silent. Under suggest-only that line is what decides\n \ | |
| 271 | + | whether the user ever sees the answer." | |
| 272 | + | ); | |
| 273 | + | println!(); | |
| 274 | + | println!( | |
| 275 | + | " Bar: per-run family-label flip rate <= {:.0}% on the fixed probe set, at", | |
| 276 | + | BAR * 100.0 | |
| 277 | + | ); | |
| 278 | + | println!(" the deployment weight, counting only flips between two suggested answers."); | |
| 279 | + | println!(" Silent <-> suggested is reported as churn and is not a flip: the queue got"); | |
| 280 | + | println!(" longer or shorter, it did not contradict itself."); | |
| 281 | + | println!(); | |
| 282 | + | println!(" Flip rate and error rate are independent. A consistently wrong answer is"); | |
| 283 | + | println!(" perfectly stable, so nothing here can be read off the accuracy figures and"); | |
| 284 | + | println!(" nothing here substitutes for them."); | |
| 285 | + | println!(); | |
| 286 | + | ||
| 287 | + | let built = match labelled::build_vault(corpus, vault, config) { | |
| 288 | + | Ok(v) => v, | |
| 289 | + | Err(e) => { | |
| 290 | + | eprintln!("corpus: {e}"); | |
| 291 | + | std::process::exit(1); | |
| 292 | + | } | |
| 293 | + | }; | |
| 294 | + | let (all, dropped) = match rows::load_rows(&built.db, space) { | |
| 295 | + | Ok(r) => r, | |
| 296 | + | Err(e) => { | |
| 297 | + | eprintln!("reading the vault back: {e}"); | |
| 298 | + | std::process::exit(1); | |
| 299 | + | } | |
| 300 | + | }; | |
| 301 | + | if dropped.unprojectable > 0 { | |
| 302 | + | println!( | |
| 303 | + | " {} row(s) carry no label in this space ({}) and are excluded from", | |
| 304 | + | dropped.unprojectable, | |
| 305 | + | dropped | |
| 306 | + | .origins | |
| 307 | + | .iter() | |
| 308 | + | .cloned() | |
| 309 | + | .collect::<Vec<_>>() | |
| 310 | + | .join(", ") | |
| 311 | + | ); | |
| 312 | + | println!(" every index and every probe:"); | |
| 313 | + | println!("{}", families::DROPPED_NOTE); | |
| 314 | + | println!(); | |
| 315 | + | } | |
| 316 | + | ||
| 317 | + | // Probe set: one stratified slice, held out of every index in every | |
| 318 | + | // measurement below. Fixed on purpose. A probe set that moved between | |
| 319 | + | // variants would mix "the layer changed its mind" with "we asked about | |
| 320 | + | // different samples", which is the whole thing this module exists to separate. | |
| 321 | + | let fold_of = rows::assign_folds(&all, probe_denominator); | |
| 322 | + | let probe: Vec<&Row> = all | |
| 323 | + | .iter() | |
| 324 | + | .zip(&fold_of) | |
| 325 | + | .filter(|(_, f)| **f == Some(0)) | |
| 326 | + | .map(|(r, _)| r) | |
| 327 | + | .collect(); | |
| 328 | + | let pool: Vec<&Row> = all | |
| 329 | + | .iter() | |
| 330 | + | .zip(&fold_of) | |
| 331 | + | .filter(|(_, f)| **f != Some(0)) | |
| 332 | + | .map(|(r, _)| r) | |
| 333 | + | .collect(); | |
| 334 | + | ||
| 335 | + | let classes: Vec<String> = all | |
| 336 | + | .iter() | |
| 337 | + | .filter_map(|r| r.truth.clone()) | |
| 338 | + | .collect::<BTreeSet<_>>() | |
| 339 | + | .into_iter() | |
| 340 | + | .collect(); | |
| 341 | + | ||
| 342 | + | println!( | |
| 343 | + | " {} row(s) scoreable: {} held out as the fixed probe, {} in the pool every", | |
| 344 | + | all.len(), | |
| 345 | + | probe.len(), | |
| 346 | + | pool.len() | |
| 347 | + | ); | |
| 348 | + | println!(" index is drawn from. No probe sample is ever an exemplar."); | |
| 349 | + | for c in &classes { | |
| 350 | + | let in_probe = probe | |
| 351 | + | .iter() | |
| 352 | + | .filter(|r| r.truth.as_deref() == Some(c.as_str())) | |
| 353 | + | .count(); | |
| 354 | + | let in_pool = pool | |
| 355 | + | .iter() | |
| 356 | + | .filter(|r| r.truth.as_deref() == Some(c.as_str())) | |
| 357 | + | .count(); | |
| 358 | + | println!( | |
| 359 | + | " {:<14} probe {:>4} pool {:>4}", | |
| 360 | + | families::label_for(space, c), | |
| 361 | + | in_probe, | |
| 362 | + | in_pool | |
| 363 | + | ); | |
| 364 | + | } | |
| 365 | + | println!(); | |
| 366 | + | ||
| 367 | + | if probe.is_empty() || pool.is_empty() { | |
| 368 | + | eprintln!("nothing to measure: probe or pool is empty"); | |
| 369 | + | std::process::exit(1); | |
| 370 | + | } | |
| 371 | + | ||
| 372 | + | let mut report = Report::new("layer-stability"); | |
| 373 | + | report.set("label_space", format!("{space:?}")); | |
| 374 | + | report.set("k", k); | |
| 375 | + | report.set("feat_version", FEATURE_VERSION); | |
| 376 | + | report.set("bar_flip_rate", BAR); | |
| 377 | + | report.set("review_threshold", DEFAULT_REVIEW_THRESHOLD); | |
| 378 | + | report.set("import_weight", rows::IMPORT_WEIGHT); | |
| 379 | + | report.set("probe", probe.len()); | |
| 380 | + | report.set("pool", pool.len()); | |
| 381 | + | ||
| 382 | + | review_threshold_note(&pool, &probe, &classes, k, space, &mut report); | |
| 383 | + | let miss = value_add_population(&probe, &mut report); | |
| 384 | + | let m1 = composition(&pool, &probe, &classes, k, space, &mut report); | |
| 385 | + | let m2 = size(&pool, &probe, &classes, k, &mut report); | |
| 386 | + | let m3 = deployment_shape( | |
| 387 | + | &pool, | |
| 388 | + | &probe, | |
| 389 | + | k, | |
| 390 | + | space, | |
| 391 | + | "3. DEPLOYMENT SHAPE", | |
| 392 | + | "deploy", | |
| 393 | + | &mut report, | |
| 394 | + | ); | |
| 395 | + | let m3b = value_add_deployment(&pool, &miss, k, space, &mut report); | |
| 396 | + | let m4 = feedback(&pool, k, space, &mut report); | |
| 397 | + | ||
| 398 | + | let mut outcomes = vec![m1, m2, m3]; | |
| 399 | + | outcomes.extend(m3b); | |
| 400 | + | outcomes.push(m4); | |
| 401 | + | verdict(&outcomes, space, &classes, &mut report); | |
| 402 | + | report.write(); | |
| 403 | + | } | |
| 404 | + | ||
| 405 | + | /// Whether the review threshold is doing any work at this label resolution. | |
| 406 | + | /// | |
| 407 | + | /// It is not, on a two-class corpus, and that has to be said before any number | |
| 408 | + | /// below is read. A sample's score for a tag is that tag's share of the | |
| 409 | + | /// neighbourhood's kernel weight, so when every exemplar carries exactly one of | |
| 410 | + | /// two tags the two scores sum to 1 and the higher one is at or above 0.5 by | |
| 411 | + | /// arithmetic. `silent` is unreachable, every probe sample is always a queue | |
| 412 | + | /// entry, and the `appeared` / `vanished` columns are structurally zero. | |
| 413 | + | /// | |
| 414 | + | /// Two consequences a reader must not miss: | |
| 415 | + | /// | |
| 416 | + | /// - The queue-length half of "does the queue hold still" is untested here. Only | |
| 417 | + | /// the contents were measured, because the length cannot move. | |
| 418 | + | /// - Any metric defined as "the layer answers where the user's own labels do not" | |
| 419 | + | /// is identically zero for the same reason, whatever the layer is worth. That | |
| 420 | + | /// is why [`deployment_shape`] measures the layer's contribution as a change in | |
| 421 | + | /// the answer and its correctness rather than as an answer appearing. | |
| 422 | + | /// | |
| 423 | + | /// This resolves at three classes or more and is not a property of the layer, so | |
| 424 | + | /// it is a scope note on the corpus rather than a finding about the code. | |
| 425 | + | fn review_threshold_note( | |
| 426 | + | pool: &[&Row], | |
| 427 | + | probe: &[&Row], | |
| 428 | + | classes: &[String], | |
| 429 | + | k: usize, | |
| 430 | + | space: LabelSpace, | |
| 431 | + | report: &mut Report, | |
| 432 | + | ) { | |
| 433 | + | println!("━━━ IS THE REVIEW THRESHOLD BINDING? ━━━"); | |
| 434 | + | println!(); | |
| 435 | + | ||
| 436 | + | let full = run_variant(pool, &[], probe, k, "full pool"); | |
| 437 | + | let silent = full.iter().filter(|a| a.tag().is_none()).count(); | |
| 438 | + | println!( | |
| 439 | + | " Against the full pool, {} of {} probe samples fall below the {} review", | |
| 440 | + | silent, | |
| 441 | + | probe.len(), | |
| 442 | + | DEFAULT_REVIEW_THRESHOLD | |
| 443 | + | ); | |
| 444 | + | println!(" threshold and stay out of the queue."); | |
| 445 | + | println!(); | |
| 446 | + | ||
| 447 | + | report.set("silent_at_full_pool", silent); | |
| 448 | + | report.set("classes", classes.len()); | |
| 449 | + | ||
| 450 | + | if silent == 0 && classes.len() == 2 { | |
| 451 | + | println!(" Zero, and it is arithmetic rather than luck. A score is a tag's share of"); | |
| 452 | + | println!(" the neighbourhood's kernel weight; with every exemplar carrying exactly one"); | |
| 453 | + | println!(" of two tags the two shares sum to 1, so the larger is always at or above"); | |
| 454 | + | println!(" 0.5. On this corpus `silent` is unreachable and the threshold decides"); | |
| 455 | + | println!(" nothing."); | |
| 456 | + | println!(); | |
| 457 | + | println!(" What that costs the measurements below, stated plainly:"); | |
| 458 | + | println!(); | |
| 459 | + | println!(" - The queue can only change its CONTENTS, never its LENGTH. The churn"); | |
| 460 | + | println!(" columns are structurally zero and prove nothing."); | |
| 461 | + | println!(" - 'The layer answers where the user's labels do not' is identically zero"); | |
| 462 | + | println!(" for every layer, good or useless. Measurement 3 therefore reads the"); | |
| 463 | + | println!(" layer's contribution off the answer and its correctness instead."); | |
| 464 | + | println!(); | |
| 465 | + | println!(" Both resolve at three classes or more, so this is a limit of the drums-"); | |
| 466 | + | println!(" only corpus and not a property of the layer. Phase C is where it lifts."); | |
| 467 | + | report.set("review_threshold_binding", false); | |
| 468 | + | } else { | |
| 469 | + | report.set("review_threshold_binding", true); | |
| 470 | + | } | |
| 471 | + | println!(); | |
| 472 | + | let _ = space; | |
| 473 | + | } | |
| 474 | + | ||
| 475 | + | /// A named measurement's answer to "did it pass". | |
| 476 | + | struct Outcome { | |
| 477 | + | name: &'static str, | |
| 478 | + | /// `None` when the measurement could not be run at all, which is not a pass. | |
| 479 | + | worst: Option<f64>, | |
| 480 | + | note: String, | |
| 481 | + | } | |
| 482 | + | ||
| 483 | + | // The value-add population | |
| 484 | + | ||
| 485 | + | /// How much of the probe set the filename rules already answer. | |
| 486 | + | /// | |
| 487 | + | /// The layer's stated job is libraries whose filenames say nothing. | |
| 488 | + | /// `starter_rules` labels 97.7% of this corpus correctly off the name alone, and | |
| 489 | + | /// every measurement of this layer so far — accuracy and now stability — runs on | |
| 490 | + | /// exactly that population. So the number that matters is measured on the | |
| 491 | + | /// complement, and this reports whether the complement is big enough to measure | |
| 492 | + | /// on at all. | |
| 493 | + | /// | |
| 494 | + | /// Returns the complement — the probe samples no filename rule answers — so | |
| 495 | + | /// [`deployment_shape`] can be re-run on it. Empty when it is too thin to carry a | |
| 496 | + | /// rate, which is a real possibility on a corpus whose folder labels were derived | |
| 497 | + | /// from these same filenames. Phase C is where it stops being marginal: NSynth | |
| 498 | + | /// names are `bass_synthetic_033-052-100`-shaped and carry no instrument keyword | |
| 499 | + | /// the starter pack knows. | |
| 500 | + | fn value_add_population<'a>(probe: &[&'a Row], report: &mut Report) -> Vec<&'a Row> { |
Lines truncated