Skip to main content

max / audiofiles

Add an official layer kind and a corpus-to-layer .afcl generator Both halves of the official .afcl work, which came off hold on 2026-08-05 when the decision landed to ship a bundled default layer. ExportOptions gains a kind, as a typed LayerKind rather than a String. The import side was hand-rolling the same two-valued normalisation inline, so both halves now share one type and the export half cannot emit a kind the import half will not read. Default stays Imported, so no existing caller changes behaviour; the one exhaustive struct literal sets it explicitly, because a user's export is never the bundled default. An unknown kind from a future version still degrades to Imported rather than failing the parse. The generator is a new `afcl` mode in audiofiles-bench: import every labelled file into a throwaway vault, analyse it, tag it from the folder it came in, call build_export with kind = Official. Exemplars only. Rules and policy are excluded because the app already ships starter_rules in the binary, and exporting them would put a second, staler copy inside the layer for the user to review. Folder-to-tag mapping resolves against starter_rules::DRUM_CLASSES rather than carrying a second taxonomy, and an unmapped folder is fatal: a silently dropped class reads downstream as the classifier being bad at it rather than never having seen it. Proving run against the corpus on the T9: 1,049 files across 7 classes, imported in 2.4s, analysed in 5.5s, 1,096 KiB layer. kind = official, feat_version 5, every exemplar a 35-d vector with tags and no hash field, tag spread matching the corpus, no non-finite components. That run also exposed a licensing bug, so the generator refuses a manifest with no training_counts. samples/training/ is filled only by reverb-drums, but the manifest on the T9 lists only nsynth: corpus.py overwrites the manifest rather than merging it, so a later --datasets run dropped the reverb-drums credit. Both are CC-BY 4.0, but CC-BY requires crediting the work actually used, and this attribution ships inside the layer. The guard is a tripwire on the symptom; the corpus still needs rebuilding.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-08-05 16:21 UTC
Signed with PGP, not checked
Commit: 46717b07c5ed66941994153eda1a02b90b13dd7b
Parent: 831d504
5 files changed, +631 insertions, -14 deletions
M Cargo.lock +4 -4
@@ -7301,6 +7301,10 @@
7301 7301 name = "docengine"
7302 7302 version = "0.4.0"
7303 7303
7304 + [[patch.unused]]
7305 + name = "supernote-push"
7306 + version = "0.1.0"
7307 +
7304 7308 [[patch.unused]]
7305 7309 name = "kberg"
7306 7310 version = "0.1.0"
@@ -7308,7 +7312,3 @@
7308 7312 [[patch.unused]]
7309 7313 name = "painhours"
7310 7314 version = "0.1.0"
7311 -
7312 - [[patch.unused]]
7313 - name = "supernote-push"
7314 - version = "0.1.0"
M crates/audiofiles-bench/src/main.rs +28 -3
@@ -11,21 +11,25 @@
11 11 //! `cargo run --release -p audiofiles-bench -- ingest` vault ingest + queries
12 12 //! `cargo run --release -p audiofiles-bench -- accuracy` bpm/key vs ground truth
13 13 //! `cargo run --release -p audiofiles-bench -- layout` blob layout migration
14 + //! `cargo run --release -p audiofiles-bench -- afcl` build the official layer
14 15 //!
15 - //! `layout` is the one mode that is a checker rather than a measurement: it
16 - //! fabricates flat vaults, sweeps them, and exits non-zero if any scenario fails.
16 + //! Two modes are not measurements. `layout` is a checker: it fabricates flat
17 + //! vaults, sweeps them, and exits non-zero if any scenario fails. `afcl` is a
18 + //! generator: it turns the labelled corpus into the bundled official `.afcl`.
19 + //! Both live here because corpus walking and scratch-vault fabrication do.
17 20 //!
18 21 //! Env: `AF_BENCH_CORPUS` (corpus root, default `<repo>/samples`),
19 22 //! `AF_BENCH_VAULT` (scratch vault for ingest and layout), `AF_BENCH_FSL10K`
20 23 //! (FSL10K root for accuracy), `AF_BENCH_BATCH`, `AF_BENCH_LIMIT`,
21 24 //! `AF_BENCH_ANALYZE`, `AF_BENCH_LAYOUT_N`, `AF_BENCH_JSON` (machine-readable
22 - //! output path).
25 + //! output path), `AF_AFCL_OUT` (where `afcl` writes the layer).
23 26 //!
24 27 //! Every mode opens with a conditions block naming the drive behind each path.
25 28 //! These numbers are I/O bound, so they belong to a drive as much as to the
26 29 //! code; see `storage`.
27 30
28 31 mod accuracy;
32 + mod afcl_gen;
29 33 mod ingest;
30 34 mod layout;
31 35 mod report;
@@ -290,6 +294,27 @@
290 294 return;
291 295 }
292 296
297 + if args.first().map(String::as_str) == Some("afcl") {
298 + // Its own scratch vault for the same reason `layout` has one: this mode
299 + // deletes and rebuilds the vault, and pointing it at a corpus vault
300 + // someone had just imported would be an expensive surprise.
301 + let vault = std::env::var("AF_BENCH_VAULT")
302 + .map_or_else(|_| std::env::temp_dir().join("af-afcl-gen"), PathBuf::from);
303 + let out = std::env::var("AF_AFCL_OUT").map_or_else(
304 + |_| {
305 + PathBuf::from(env!("CARGO_MANIFEST_DIR"))
306 + .parent()
307 + .unwrap()
308 + .parent()
309 + .unwrap()
310 + .join("assets/official.afcl")
311 + },
312 + PathBuf::from,
313 + );
314 + afcl_gen::run(&samples_dir, &vault, &out, &full_pipeline_config());
315 + return;
316 + }
317 +
293 318 if args.first().map(String::as_str) == Some("accuracy") {
294 319 let Ok(root) = std::env::var("AF_BENCH_FSL10K") else {
295 320 eprintln!("set AF_BENCH_FSL10K to the extracted FSL10K root");
M crates/audiofiles-browser/src/state/classifier.rs +3 -1
@@ -335,7 +335,7 @@
335 335 /// Export the user's classifier data to `path` using the export-form selections,
336 336 /// runs on a worker thread (reads + serializes the whole library).
337 337 pub fn classifier_export_afcl(&mut self, path: &std::path::Path) {
338 - use audiofiles_core::analysis::afcl::ExportOptions;
338 + use audiofiles_core::analysis::afcl::{ExportOptions, LayerKind};
339 339 let opts = ExportOptions {
340 340 name: {
341 341 let n = self.classifier.export_name.trim();
@@ -347,6 +347,8 @@
347 347 },
348 348 description: String::new(),
349 349 license_note: String::new(),
350 + // A user's export is never the bundled default, whatever they name it.
351 + kind: LayerKind::Imported,
350 352 include_exemplars: self.classifier.export_include_exemplars,
351 353 include_rules: self.classifier.export_include_rules,
352 354 include_policy: self.classifier.export_include_policy,
M crates/audiofiles-core/src/analysis/afcl.rs +103 -6
@@ -87,12 +87,53 @@
87 87 pub policy: Vec<AfclPolicy>,
88 88 }
89 89
90 + /// What a layer is: a user's own share, or the bundled default that ships with the app.
91 + ///
92 + /// The manifest carries this as a free-text `String` rather than as this enum, so a file
93 + /// written by a future version naming a kind we don't know parses rather than failing.
94 + /// [`LayerKind::from_manifest`] is where an unknown kind degrades to [`Imported`], which
95 + /// is the conservative reading: an unrecognised layer gets the weight and the
96 + /// review-before-enable treatment that a stranger's file gets.
97 + ///
98 + /// [`Imported`]: LayerKind::Imported
99 + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
100 + pub enum LayerKind {
101 + /// A user-to-user share. The default, and what every user-driven export produces.
102 + #[default]
103 + Imported,
104 + /// The bundled default classifier, generated from the labelled corpus at build time.
105 + Official,
106 + }
107 +
108 + impl LayerKind {
109 + /// The manifest spelling, and the value stored in `classifier_layers.kind`.
110 + pub fn as_str(self) -> &'static str {
111 + match self {
112 + Self::Imported => "imported",
113 + Self::Official => "official",
114 + }
115 + }
116 +
117 + /// Read a manifest's `kind`. Anything unrecognised reads as [`Imported`].
118 + ///
119 + /// [`Imported`]: LayerKind::Imported
120 + pub fn from_manifest(kind: &str) -> Self {
121 + match kind {
122 + "official" => Self::Official,
123 + _ => Self::Imported,
124 + }
125 + }
126 + }
127 +
90 128 /// What to include when exporting.
91 129 #[derive(Debug, Clone)]
92 130 pub struct ExportOptions {
93 131 pub name: String,
94 132 pub description: String,
95 133 pub license_note: String,
134 + /// Almost always [`LayerKind::Imported`]. The generator that builds the bundled
135 + /// default layer is the only caller that sets [`LayerKind::Official`].
136 + pub kind: LayerKind,
96 137 pub include_exemplars: bool,
97 138 pub include_rules: bool,
98 139 pub include_policy: bool,
@@ -104,6 +145,7 @@
104 145 name: "My classifier".to_string(),
105 146 description: String::new(),
106 147 license_note: String::new(),
148 + kind: LayerKind::Imported,
107 149 include_exemplars: true,
108 150 include_rules: true,
109 151 include_policy: true,
@@ -171,7 +213,7 @@
171 213 feat_version: FEATURE_VERSION,
172 214 name: opts.name.clone(),
173 215 description: opts.description.clone(),
174 - kind: "imported".to_string(),
216 + kind: opts.kind.as_str().to_string(),
175 217 created_at: unix_now(),
176 218 license_note: opts.license_note.clone(),
177 219 exemplar_count: exemplars.len(),
@@ -306,11 +348,7 @@
306 348 }
307 349
308 350 let layer_id = new_layer_id();
309 - let kind = if afcl.manifest.kind == "official" {
310 - "official"
311 - } else {
312 - "imported"
313 - };
351 + let kind = LayerKind::from_manifest(&afcl.manifest.kind).as_str();
314 352 db.conn().execute(
315 353 "INSERT INTO classifier_layers (id, name, kind, weight, enabled, source, imported_at)
316 354 VALUES (?1, ?2, ?3, ?4, 1, ?5, ?6)",
@@ -600,6 +638,65 @@
600 638 assert!(afcl.policy.is_empty());
601 639 }
602 640
641 + #[test]
642 + fn export_defaults_to_imported_and_can_produce_official() {
643 + let db = Database::open_in_memory().unwrap();
644 + seed_library(&db);
645 +
646 + let user = build_export(&db, &ExportOptions::default()).unwrap();
647 + assert_eq!(user.manifest.kind, "imported");
648 +
649 + let official = build_export(
650 + &db,
651 + &ExportOptions {
652 + kind: LayerKind::Official,
653 + ..ExportOptions::default()
654 + },
655 + )
656 + .unwrap();
657 + assert_eq!(official.manifest.kind, "official");
658 + }
659 +
660 + #[test]
661 + fn official_kind_survives_the_round_trip_to_the_layer_row() {
662 + let src = Database::open_in_memory().unwrap();
663 + seed_library(&src);
664 + let json = export_to_string(
665 + &src,
666 + &ExportOptions {
667 + kind: LayerKind::Official,
668 + ..ExportOptions::default()
669 + },
670 + )
671 + .unwrap();
672 +
673 + let dst = Database::open_in_memory().unwrap();
674 + import_from_string(&dst, &json, Some("official.afcl")).unwrap();
675 +
676 + let layers = list_layers(&dst).unwrap();
677 + assert_eq!(layers.len(), 1);
678 + assert_eq!(layers[0].kind, "official");
679 + }
680 +
681 + #[test]
682 + fn unknown_manifest_kind_degrades_to_imported() {
683 + // A file from a future version naming a kind this build does not know must
684 + // parse and land as a stranger's layer, not fail the import.
685 + assert_eq!(LayerKind::from_manifest("official"), LayerKind::Official);
686 + assert_eq!(LayerKind::from_manifest("imported"), LayerKind::Imported);
687 + assert_eq!(LayerKind::from_manifest("curated"), LayerKind::Imported);
688 + assert_eq!(LayerKind::from_manifest(""), LayerKind::Imported);
689 +
690 + let src = Database::open_in_memory().unwrap();
691 + seed_library(&src);
692 + let mut afcl = build_export(&src, &ExportOptions::default()).unwrap();
693 + afcl.manifest.kind = "curated-by-someone-else".to_string();
694 +
695 + let dst = Database::open_in_memory().unwrap();
696 + import(&dst, &afcl, Some("future.afcl")).unwrap();
697 + assert_eq!(list_layers(&dst).unwrap()[0].kind, "imported");
698 + }
699 +
603 700 #[test]
604 701 fn round_trip_import_creates_layer() {
605 702 let src = Database::open_in_memory().unwrap();
A crates/audiofiles-bench/src/afcl_gen.rs +493
@@ -1,0 +1,493 @@
1 + //! Build the bundled official `.afcl` from the labelled corpus.
2 + //!
3 + //! Like `layout`, this is not a measurement. It lives in the bench crate because
4 + //! this is where corpus walking and scratch-vault fabrication already exist, and
5 + //! duplicating both in a second binary to keep the crate's name honest would cost
6 + //! more than the misnomer does.
7 + //!
8 + //! What it does: import every labelled file into a throwaway vault, analyse it so
9 + //! `sample_features` holds a vector, tag it from the folder it came in, then call
10 + //! [`build_export`] with `kind = "official"`. The exemplars in the resulting layer
11 + //! are (vector, tags) pairs and carry no audio and no hashes, which is the whole
12 + //! reason a `.afcl` can ship at all.
13 + //!
14 + //! The output is a build artifact, not a test fixture. Generate it deliberately,
15 + //! check it in, and regenerate it whenever `FEATURE_VERSION` moves: import gates
16 + //! on `feat_version` and a layer built under an old one is rejected outright, so a
17 + //! stale layer is worse than no layer.
18 +
19 + use std::collections::BTreeMap;
20 + use std::path::{Path, PathBuf};
21 + use std::time::Instant;
22 +
23 + use audiofiles_core::analysis::afcl::{ExportOptions, LayerKind, export_to_path};
24 + use audiofiles_core::analysis::features::FEATURE_VERSION;
25 + use audiofiles_core::analysis::{self, config::AnalysisConfig};
26 + use audiofiles_core::db::Database;
27 + use audiofiles_core::starter_rules::DRUM_CLASSES;
28 + use audiofiles_core::store::SampleStore;
29 + use audiofiles_core::tags;
30 +
31 + /// Corpus subdirectory holding the labelled one-shots, one folder per class.
32 + const TRAINING_SUBDIR: &str = "training";
33 +
34 + /// Corpus manifest naming the source datasets and their licences.
35 + const MANIFEST: &str = "MANIFEST.json";
36 +
37 + /// Map a corpus folder name to its canonical tag.
38 + ///
39 + /// The folder names and [`DRUM_CLASSES`] are the same taxonomy, so this resolves
40 + /// against that table rather than carrying a second copy of it. The only mismatch
41 + /// is punctuation: the table labels the class `hi-hat` and the corpus folder is
42 + /// `hihat`, so both sides are compared with `-` and spaces removed.
43 + fn tag_for_folder(folder: &str) -> Option<&'static str> {
44 + fn squash(s: &str) -> String {
45 + s.chars()
46 + .filter(|c| !matches!(c, '-' | ' ' | '_'))
47 + .flat_map(char::to_lowercase)
48 + .collect()
49 + }
50 + let want = squash(folder);
51 + DRUM_CLASSES
52 + .iter()
53 + .find(|c| squash(c.label) == want)
54 + .map(|c| c.tag)
55 + }
56 +
57 + /// Read the corpus manifest into an attribution line for `license_note`.
58 + ///
59 + /// This is the CC-BY attribution, and it is the only place it travels: an `.afcl`
60 + /// carries no audio, so the manifest is what records whose labels these derive
61 + /// from. A missing or unreadable manifest is fatal rather than defaulted, because
62 + /// a layer that ships without attribution is the one outcome worth failing over.
63 + ///
64 + /// Looked up in `corpus_root` and then in its parent, because `AF_BENCH_CORPUS`
65 + /// points at the `samples/` directory while `corpus.py` writes the manifest one
66 + /// level up beside it.
67 + fn manifest_path(corpus_root: &Path) -> Option<PathBuf> {
68 + let here = corpus_root.join(MANIFEST);
69 + if here.is_file() {
70 + return Some(here);
71 + }
72 + let beside = corpus_root.parent()?.join(MANIFEST);
73 + beside.is_file().then_some(beside)
74 + }
75 +
76 + fn license_note(corpus_root: &Path) -> Result<String, String> {
77 + let path = manifest_path(corpus_root).ok_or_else(|| {
78 + format!(
79 + "no {MANIFEST} in {} or its parent; corpus.py writes it beside samples/",
80 + corpus_root.display()
81 + )
82 + })?;
83 + let raw = std::fs::read_to_string(&path)
84 + .map_err(|e| format!("cannot read {}: {e}", path.display()))?;
85 + let doc: serde_json::Value = serde_json::from_str(&raw)
86 + .map_err(|e| format!("{} is not valid JSON: {e}", path.display()))?;
87 +
88 + let datasets = doc
89 + .get("datasets")
90 + .and_then(|d| d.as_array())
91 + .ok_or_else(|| format!("{} has no `datasets` array", path.display()))?;
92 + if datasets.is_empty() {
93 + return Err(format!("{} lists no datasets", path.display()));
94 + }
95 +
96 + // The manifest must actually describe the training data, not merely exist.
97 + //
98 + // `corpus.py` populates `samples/training/` only from `reverb-drums`, and
99 + // records `training_counts` in the same pass. It also OVERWRITES the manifest
100 + // rather than merging it, so a later `--datasets nsynth` run leaves the
101 + // training folders in place while replacing the attribution with a dataset
102 + // the labels did not come from. That is what the corpus on the T9 looks like
103 + // today: 1,049 reverb-drums one-shots under `training/`, and a manifest
104 + // naming only nsynth.
105 + //
106 + // Both are CC-BY 4.0, so this is not a licence-class error, but CC-BY
107 + // requires crediting the work actually used. Refuse rather than emit a
108 + // confident wrong credit into a file meant to ship.
109 + if doc.get("training_counts").is_none() {
110 + return Err(format!(
111 + "{} has no `training_counts`, so it does not describe the labelled \
112 + training data and its `datasets` list cannot be trusted as the \
113 + attribution. Rebuild the corpus with reverb-drums (which is what \
114 + fills samples/training/) so the manifest and the labels agree.",
115 + path.display()
116 + ));
117 + }
118 +
119 + let mut parts = Vec::new();
120 + for d in datasets {
121 + let name = d.get("name").and_then(|v| v.as_str()).unwrap_or("");
122 + let license = d.get("license").and_then(|v| v.as_str()).unwrap_or("");
123 + let source = d.get("source").and_then(|v| v.as_str()).unwrap_or("");
124 + if name.is_empty() || license.is_empty() {
125 + return Err(format!(
126 + "{} has a dataset entry missing name or license",
127 + path.display()
128 + ));
129 + }
130 + if source.is_empty() {
131 + parts.push(format!("{name} ({license})"));
132 + } else {
133 + parts.push(format!("{name} ({license}), {source}"));
134 + }
135 + }
136 + Ok(format!(
137 + "Labels derived from: {}. This layer carries feature vectors and labels only, no audio.",
138 + parts.join("; ")
139 + ))
140 + }
141 +
142 + /// Collect the labelled files as (path, tag) pairs, grouped for a stable report.
143 + fn collect_labelled(training: &Path) -> Result<BTreeMap<&'static str, Vec<PathBuf>>, String> {
144 + let entries = std::fs::read_dir(training)
145 + .map_err(|e| format!("cannot read {}: {e}", training.display()))?;
146 +
147 + let mut by_tag: BTreeMap<&'static str, Vec<PathBuf>> = BTreeMap::new();
148 + let mut unmapped = Vec::new();
149 + for entry in entries.flatten() {
150 + if !entry.path().is_dir() {
151 + continue;
152 + }
153 + let folder = entry.file_name().to_string_lossy().to_string();
154 + let Some(tag) = tag_for_folder(&folder) else {
155 + unmapped.push(folder);
156 + continue;
157 + };
158 + let mut files: Vec<PathBuf> = std::fs::read_dir(entry.path())
159 + .map_err(|e| format!("cannot read {}: {e}", entry.path().display()))?
160 + .flatten()
161 + .map(|f| f.path())
162 + .filter(|p| p.is_file())
163 + .collect();
164 + // Deterministic order so two runs over the same corpus produce the same
165 + // layer, which is what makes the checked-in artifact reviewable.
166 + files.sort();
167 + by_tag.entry(tag).or_default().extend(files);
168 + }
169 +
170 + if !unmapped.is_empty() {
171 + // Loud rather than silent: a folder nobody mapped is a class silently
172 + // missing from the shipped layer, which reads downstream as the
173 + // classifier being bad at that class rather than never having seen it.
174 + unmapped.sort();
175 + return Err(format!(
176 + "no tag mapping for corpus folder(s): {}. Add them to DRUM_CLASSES or move them out of {}",
177 + unmapped.join(", "),
178 + training.display()
179 + ));
180 + }
181 + if by_tag.is_empty() {
182 + return Err(format!(
183 + "no labelled class folders under {}",
184 + training.display()
185 + ));
186 + }
187 + Ok(by_tag)
188 + }
189 +
190 + /// Generate the official layer from `corpus` into `out`, using `vault` as scratch.
191 + pub(crate) fn run(corpus: &Path, vault: &Path, out: &Path, config: &AnalysisConfig) {
192 + println!("━━━ OFFICIAL .afcl GENERATION ━━━");
193 + println!();
194 + println!(" corpus {}", corpus.display());
195 + println!(" scratch {}", vault.display());
196 + println!(" output {}", out.display());
197 + println!(" features v{FEATURE_VERSION}");
198 + println!();
199 +
200 + let note = match license_note(corpus) {
201 + Ok(n) => n,
202 + Err(e) => {
203 + eprintln!("attribution: {e}");
204 + std::process::exit(1);
205 + }
206 + };
207 +
208 + let by_tag = match collect_labelled(&corpus.join(TRAINING_SUBDIR)) {
209 + Ok(m) => m,
210 + Err(e) => {
211 + eprintln!("corpus: {e}");
212 + std::process::exit(1);
213 + }
214 + };
215 +
216 + let total: usize = by_tag.values().map(Vec::len).sum();
217 + println!(
218 + " {} labelled file(s) across {} class(es):",
219 + total,
220 + by_tag.len()
221 + );
222 + for (tag, files) in &by_tag {
223 + println!(" {:<28} {:>5}", tag, files.len());
224 + }
225 + println!();
226 +
227 + // A fresh vault every run. The layer is a pure function of the corpus, and
228 + // leftovers from a previous run would silently widen it.
229 + if vault.exists()
230 + && let Err(e) = std::fs::remove_dir_all(vault)
231 + {
232 + eprintln!("could not clear scratch vault {}: {e}", vault.display());
233 + std::process::exit(1);
234 + }
235 + let samples_dir = vault.join("samples");
236 + if let Err(e) = std::fs::create_dir_all(&samples_dir) {
237 + eprintln!("could not create {}: {e}", samples_dir.display());
238 + std::process::exit(1);
239 + }
240 +
241 + let db = match Database::open(vault.join("audiofiles.db")) {
242 + Ok(db) => db,
243 + Err(e) => {
244 + eprintln!("Database::open failed: {e}");
245 + std::process::exit(1);
246 + }
247 + };
248 + let store = match SampleStore::new(&samples_dir) {
249 + Ok(s) => s,
250 + Err(e) => {
251 + eprintln!("SampleStore::new failed: {e}");
252 + std::process::exit(1);
253 + }
254 + };
255 +
256 + // Import and tag. No VFS nodes: nothing here runs a UI query, and the export
257 + // reads `sample_features` joined to `tags`, neither of which needs one.
258 + let start = Instant::now();
259 + let mut to_analyze: Vec<(String, PathBuf)> = Vec::with_capacity(total);
260 + let mut import_failures = 0usize;
261 + for (tag, files) in &by_tag {
262 + for path in files {
263 + match store.import(path, &db) {
264 + Ok(hash) => {
265 + if let Err(e) = tags::add_tag(&db, &hash, tag) {
266 + eprintln!(" tag {tag} on {}: {e}", path.display());
267 + import_failures += 1;
268 + continue;
269 + }
270 + to_analyze.push((hash, path.clone()));
271 + }
272 + Err(e) => {
273 + eprintln!(" import {}: {e}", path.display());
274 + import_failures += 1;
275 + }
276 + }
277 + }
278 + }
279 + println!(
280 + " imported {} file(s) in {:.1}s{}",
281 + to_analyze.len(),
282 + start.elapsed().as_secs_f64(),
283 + if import_failures > 0 {
284 + format!(", {import_failures} failed")
285 + } else {
286 + String::new()
287 + }
288 + );
289 +
290 + // Analyse everything. The ingest benchmark caps this because analysis is the
291 + // expensive stage and it only needs a sample; here every vector is the
292 + // payload, so there is no budget to apply.
293 + let start = Instant::now();
294 + let results: Vec<_> = to_analyze
295 + .iter()
296 + .filter_map(|(hash, path)| analysis::analyze_sample(hash, path, config).ok())
297 + .collect();
298 + let analyzed = results.len();
299 + if let Err(e) = analysis::save_analysis_batch(&db, &results) {
300 + eprintln!("save_analysis_batch failed: {e}");
301 + std::process::exit(1);
302 + }
303 + println!(
304 + " analysed {} file(s) in {:.1}s{}",
305 + analyzed,
306 + start.elapsed().as_secs_f64(),
307 + if analyzed < to_analyze.len() {
308 + format!(", {} failed to analyse", to_analyze.len() - analyzed)
309 + } else {
310 + String::new()
311 + }
312 + );
313 +
314 + // Exemplars only. Rules and policy are deliberately excluded: the app already
315 + // ships `starter_rules` in the binary, so exporting them here would put a
316 + // second, staler copy inside the layer for the user to review and enable.
317 + let opts = ExportOptions {
318 + name: "Official drum classifier".to_string(),
319 + description: format!(
320 + "Bundled default layer. {analyzed} exemplars across {} classes, built from the labelled corpus.",
321 + by_tag.len()
322 + ),
323 + license_note: note,
324 + kind: LayerKind::Official,
325 + include_exemplars: true,
326 + include_rules: false,
327 + include_policy: false,
328 + };
329 +
330 + if let Some(parent) = out.parent()
331 + && let Err(e) = std::fs::create_dir_all(parent)
332 + {
333 + eprintln!("could not create {}: {e}", parent.display());
334 + std::process::exit(1);
335 + }
336 + if let Err(e) = export_to_path(&db, out, &opts) {
337 + eprintln!("export failed: {e}");
338 + std::process::exit(1);
339 + }
340 +
341 + let bytes = std::fs::metadata(out).map_or(0, |m| m.len());
342 + println!();
343 + println!(
344 + " wrote {} ({:.1} KiB)",
345 + out.display(),
346 + bytes as f64 / 1024.0
347 + );
348 + println!();
349 + println!(" Layer is built for features v{FEATURE_VERSION}. Regenerate on every");
350 + println!(" FEATURE_VERSION bump: import rejects a layer built under another.");
351 +
352 + if analyzed == 0 {
353 + eprintln!();
354 + eprintln!(" nothing analysed, so the layer carries no exemplars");
355 + std::process::exit(1);
356 + }
357 + }
358 +
359 + #[cfg(test)]
360 + mod tests {
361 + use super::*;
362 +
363 + #[test]
364 + fn folder_names_map_to_canonical_tags() {
365 + assert_eq!(tag_for_folder("kick"), Some("instrument.drum.kick"));
366 + assert_eq!(tag_for_folder("snare"), Some("instrument.drum.snare"));
367 + assert_eq!(tag_for_folder("cymbal"), Some("instrument.drum.cymbal"));
368 + assert_eq!(tag_for_folder("clap"), Some("instrument.drum.clap"));
369 + assert_eq!(tag_for_folder("tom"), Some("instrument.drum.tom"));
370 + assert_eq!(tag_for_folder("percussion"), Some("instrument.percussion"));
371 + }
372 +
373 + #[test]
374 + fn hihat_folder_matches_the_hi_hat_label() {
375 + // The one place the corpus and DRUM_CLASSES disagree on spelling.
376 + assert_eq!(tag_for_folder("hihat"), Some("instrument.drum.hihat"));
377 + assert_eq!(tag_for_folder("hi-hat"), Some("instrument.drum.hihat"));
378 + assert_eq!(tag_for_folder("Hi Hat"), Some("instrument.drum.hihat"));
379 + }
380 +
381 + #[test]
382 + fn unknown_folder_has_no_tag() {
383 + assert_eq!(tag_for_folder("bass"), None);
384 + assert_eq!(tag_for_folder(""), None);
385 + }
386 +
387 + #[test]
388 + fn every_drum_class_is_reachable_from_some_folder_name() {
389 + // Guards the mapping against a DRUM_CLASSES entry whose label stops
390 + // resolving; without this, a renamed label silently drops a class.
391 + for class in DRUM_CLASSES {
392 + assert_eq!(
393 + tag_for_folder(class.label),
394 + Some(class.tag),
395 + "class {} no longer resolves from its own label",
396 + class.label
397 + );
398 + }
399 + }
400 +
401 + #[test]
402 + fn license_note_requires_a_manifest() {
403 + let dir = tempfile::tempdir().unwrap();
404 + assert!(license_note(dir.path()).is_err());
405 + }
406 +
407 + #[test]
408 + fn manifest_is_found_beside_the_samples_dir() {
409 + // The real corpus layout: AF_BENCH_CORPUS points at <root>/samples and
410 + // corpus.py writes MANIFEST.json at <root>.
411 + let dir = tempfile::tempdir().unwrap();
412 + let samples = dir.path().join("samples");
413 + std::fs::create_dir_all(&samples).unwrap();
414 + std::fs::write(
415 + dir.path().join(MANIFEST),
416 + r#"{"training_counts":{"kick":2},"datasets":[{"name":"nsynth","license":"CC-BY 4.0"}]}"#,
417 + )
418 + .unwrap();
419 + assert!(license_note(&samples).unwrap().contains("nsynth"));
420 + }
421 +
422 + #[test]
423 + fn license_note_names_every_dataset_and_its_licence() {
424 + let dir = tempfile::tempdir().unwrap();
425 + std::fs::write(
426 + dir.path().join(MANIFEST),
427 + r#"{"training_counts":{"kick":2},"datasets":[
428 + {"name":"nsynth","license":"CC-BY 4.0","source":"http://example.invalid/nsynth"},
429 + {"name":"other","license":"CC0 1.0"}
430 + ]}"#,
431 + )
432 + .unwrap();
433 + let note = license_note(dir.path()).unwrap();
434 + assert!(note.contains("nsynth (CC-BY 4.0)"));
435 + assert!(note.contains("http://example.invalid/nsynth"));
436 + assert!(note.contains("other (CC0 1.0)"));
437 + assert!(note.contains("no audio"));
438 + }
439 +
440 + #[test]
441 + fn license_note_rejects_a_dataset_missing_its_licence() {
442 + let dir = tempfile::tempdir().unwrap();
443 + std::fs::write(
444 + dir.path().join(MANIFEST),
445 + r#"{"training_counts":{"kick":2},"datasets":[{"name":"nsynth"}]}"#,
446 + )
447 + .unwrap();
448 + assert!(license_note(dir.path()).is_err());
449 + }
450 +
451 + #[test]
452 + fn license_note_rejects_a_manifest_that_does_not_describe_the_training_data() {
453 + // The live failure: corpus.py overwrites the manifest, so a later fetch
454 + // of a different dataset leaves training/ intact while replacing the
455 + // attribution. Without `training_counts` the datasets list is not
456 + // evidence of where the labels came from.
457 + let dir = tempfile::tempdir().unwrap();
458 + std::fs::write(
459 + dir.path().join(MANIFEST),
460 + r#"{"datasets":[{"name":"nsynth","license":"CC-BY 4.0"}]}"#,
461 + )
462 + .unwrap();
463 + let err = license_note(dir.path()).unwrap_err();
464 + assert!(err.contains("training_counts"), "{err}");
465 + }
466 +
467 + #[test]
468 + fn collect_labelled_rejects_an_unmapped_folder() {
469 + let dir = tempfile::tempdir().unwrap();
470 + let training = dir.path().join(TRAINING_SUBDIR);
471 + std::fs::create_dir_all(training.join("kick")).unwrap();
472 + std::fs::create_dir_all(training.join("didgeridoo")).unwrap();
473 + let err = collect_labelled(&training).unwrap_err();
474 + assert!(err.contains("didgeridoo"), "{err}");
475 + }
476 +
477 + #[test]
478 + fn collect_labelled_groups_files_under_their_tag() {
479 + let dir = tempfile::tempdir().unwrap();
480 + let training = dir.path().join(TRAINING_SUBDIR);
481 + std::fs::create_dir_all(training.join("kick")).unwrap();
482 + std::fs::create_dir_all(training.join("snare")).unwrap();
483 + std::fs::write(training.join("kick/b.wav"), b"x").unwrap();
484 + std::fs::write(training.join("kick/a.wav"), b"x").unwrap();
485 + std::fs::write(training.join("snare/c.wav"), b"x").unwrap();
486 +
487 + let got = collect_labelled(&training).unwrap();
488 + assert_eq!(got["instrument.drum.kick"].len(), 2);
489 + assert_eq!(got["instrument.drum.snare"].len(), 1);
490 + // Sorted, so the artifact is reproducible across runs.
491 + assert!(got["instrument.drum.kick"][0].ends_with("a.wav"));
492 + }
493 + }