//! `.afcl` classifier-layer file sharing (Phase 7 of the hybrid classifier). //! //! A `.afcl` ("AF classifier") is a portable, self-describing classifier-layer artifact. //! It carries **no audio**, only the non-reversible 35-d feature vectors plus labels and //! deterministic rules, so sharing one exposes nothing of the source library's audio. The //! same mechanism serves user-to-user sharing and the eventual bundled official classifier //! (which ships as a `kind = "official"` `.afcl`). //! //! The container is a single self-describing **JSON document** (dep-free; the design's //! optional zip/gzip is skipped, these files are small and a plain JSON `.afcl` stays //! diffable). An import creates a removable [`ClassifierLayer`]: //! //! - **Exemplars** land in `classifier_exemplars` (vector + tags only, no sample row), //! join the k-NN index weighted *below* the user's own `local` data, and only fill gaps //! or break ties (see [`super::exemplar::build_index`]). //! - **Rules** are added as ordinary `tag_rules` rows, **disabled**, grouped under the //! layer via `classifier_layer_rules` so the user reviews before enabling. //! - **Policy** thresholds are applied only for tags the user hasn't already configured //! (never clobbers the user's own thresholds). //! //! `feat_version` is gated at import: a `.afcl` built under a different feature layout is //! rejected. The whole layer is removable in one action. use std::collections::HashSet; use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; use serde::{Deserialize, Serialize}; use crate::db::Database; use crate::error::{CoreError, Result, io_err, unix_now}; use crate::rules::{self, NewRule, Rule}; use tracing::instrument; use super::classify::{FEATURE_VERSION, NUM_FEATURES}; /// On-disk `.afcl` format version (independent of `FEATURE_VERSION`). pub const AFCL_VERSION: u32 = 1; /// Default k-NN weight for an imported layer, below the user's own `local` (1.0), so /// imported exemplars fill gaps and break ties without overriding the user's labels. pub const DEFAULT_IMPORT_WEIGHT: f64 = 0.5; // ── File format ── /// `.afcl` manifest. `feat_version` MUST match the importing app's [`FEATURE_VERSION`]. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AfclManifest { pub afcl_version: u32, pub feat_version: u32, pub name: String, pub description: String, /// `"imported"` (user share) or `"official"` (bundled default classifier). pub kind: String, pub created_at: i64, /// Free-text note about the rights to the shared labels (UI surfaces it on export). pub license_note: String, pub exemplar_count: usize, pub rule_count: usize, pub policy_count: usize, } /// One shared exemplar: a feature vector and its labels (sample hash stripped). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AfclExemplar { pub vector: Vec, pub tags: Vec, } /// One shared per-tag policy. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AfclPolicy { pub tag: String, pub review_threshold: f64, pub auto_threshold: f64, } /// A complete `.afcl` document. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Afcl { pub manifest: AfclManifest, #[serde(default)] pub exemplars: Vec, #[serde(default)] pub rules: Vec, #[serde(default)] pub policy: Vec, } /// What to include when exporting. #[derive(Debug, Clone)] pub struct ExportOptions { pub name: String, pub description: String, pub license_note: String, pub include_exemplars: bool, pub include_rules: bool, pub include_policy: bool, } impl Default for ExportOptions { fn default() -> Self { Self { name: "My classifier".to_string(), description: String::new(), license_note: String::new(), include_exemplars: true, include_rules: true, include_policy: true, } } } /// A persisted classifier layer (a group of imported exemplars/rules). #[derive(Debug, Clone)] pub struct ClassifierLayer { pub id: String, pub name: String, pub kind: String, pub weight: f64, pub enabled: bool, pub source: Option, pub imported_at: i64, /// Live counts (joined at list time). pub exemplar_count: usize, pub rule_count: usize, } /// Outcome of an import. #[derive(Debug, Clone)] pub struct ImportSummary { pub layer_id: String, pub name: String, pub exemplars: usize, pub rules: usize, pub policies: usize, } // ── Layer ids ── static LAYER_COUNTER: AtomicU64 = AtomicU64::new(0); fn new_layer_id() -> String { let n = LAYER_COUNTER.fetch_add(1, Ordering::Relaxed); format!("layer-{:x}{:x}", unix_now(), n) } // ── Export ── /// Build an [`Afcl`] from the user's own (`local`) classifier data per `opts`. #[instrument(skip_all)] pub fn build_export(db: &Database, opts: &ExportOptions) -> Result { let exemplars = if opts.include_exemplars { local_exemplars(db)? } else { Vec::new() }; let rules = if opts.include_rules { local_rules(db)? } else { Vec::new() }; let policy = if opts.include_policy { local_policy(db)? } else { Vec::new() }; let manifest = AfclManifest { afcl_version: AFCL_VERSION, feat_version: FEATURE_VERSION, name: opts.name.clone(), description: opts.description.clone(), kind: "imported".to_string(), created_at: unix_now(), license_note: opts.license_note.clone(), exemplar_count: exemplars.len(), rule_count: rules.len(), policy_count: policy.len(), }; Ok(Afcl { manifest, exemplars, rules, policy, }) } /// Serialize the user's classifier data to a `.afcl` JSON string. pub fn export_to_string(db: &Database, opts: &ExportOptions) -> Result { let afcl = build_export(db, opts)?; serde_json::to_string_pretty(&afcl).map_err(|e| CoreError::Serialization(e.to_string())) } /// Write the user's classifier data to a `.afcl` file. pub fn export_to_path(db: &Database, path: &Path, opts: &ExportOptions) -> Result<()> { let json = export_to_string(db, opts)?; std::fs::write(path, json).map_err(|e| io_err(path, e))?; Ok(()) } /// The user's own (`local`) exemplars: current-version feature vectors carrying >=1 non-ML /// tag. Sample hashes are intentionally dropped, only vector + tags travel. fn local_exemplars(db: &Database) -> Result> { let conn = db.conn(); // Non-ML tags per local sample. let mut tags_by_hash: std::collections::HashMap> = std::collections::HashMap::new(); { let mut stmt = conn.prepare( "SELECT t.sample_hash, t.tag FROM tags t WHERE NOT EXISTS ( SELECT 1 FROM tag_provenance p WHERE p.sample_hash = t.sample_hash AND p.tag = t.tag AND p.source = 'ml')", )?; let rows = stmt.query_map([], |row| { Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) })?; for r in rows { let (hash, tag) = r?; tags_by_hash.entry(hash).or_default().push(tag); } } let mut out = Vec::new(); let mut stmt = conn.prepare("SELECT hash, vector FROM sample_features WHERE feat_version = ?1")?; let rows = stmt.query_map([FEATURE_VERSION], |row| { Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) })?; for r in rows { let (hash, json) = r?; let Some(tags) = tags_by_hash.remove(&hash) else { continue; }; let vector: Vec = serde_json::from_str(&json).map_err(|e| CoreError::Serialization(e.to_string()))?; if vector.len() == NUM_FEATURES { out.push(AfclExemplar { vector, tags }); } } Ok(out) } /// The user's own rules (those not belonging to any imported layer). fn local_rules(db: &Database) -> Result> { let imported: HashSet = imported_rule_ids(db)?; Ok(rules::list_rules(db)? .into_iter() .filter(|r| !imported.contains(&r.id)) .collect()) } fn local_policy(db: &Database) -> Result> { Ok(super::exemplar::list_policies(db)? .into_iter() .map(|(tag, p)| AfclPolicy { tag, review_threshold: p.review_threshold, auto_threshold: p.auto_threshold, }) .collect()) } fn imported_rule_ids(db: &Database) -> Result> { let mut stmt = db .conn() .prepare("SELECT rule_id FROM classifier_layer_rules")?; let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; Ok(rows.collect::, _>>()?) } // ── Import ── /// Parse and import a `.afcl` JSON string as a new removable layer. #[instrument(skip_all)] pub fn import_from_string( db: &Database, json: &str, source: Option<&str>, ) -> Result { let afcl: Afcl = serde_json::from_str(json).map_err(|e| CoreError::Serialization(e.to_string()))?; import(db, &afcl, source) } /// Read and import a `.afcl` file as a new removable layer. pub fn import_from_path(db: &Database, path: &Path) -> Result { let json = std::fs::read_to_string(path).map_err(|e| io_err(path, e))?; let source = path.file_name().and_then(|s| s.to_str()); import_from_string(db, &json, source) } /// Import a parsed `.afcl`, creating a new layer. Gated on `afcl_version`/`feat_version`. pub fn import(db: &Database, afcl: &Afcl, source: Option<&str>) -> Result { if afcl.manifest.afcl_version > AFCL_VERSION { return Err(CoreError::Incompatible(format!( "this .afcl needs a newer version of audiofiles (format v{}, supported v{})", afcl.manifest.afcl_version, AFCL_VERSION ))); } if afcl.manifest.feat_version != FEATURE_VERSION { return Err(CoreError::Incompatible(format!( "this .afcl was built for a different analysis version (features v{}, this app v{})", afcl.manifest.feat_version, FEATURE_VERSION ))); } let layer_id = new_layer_id(); let kind = if afcl.manifest.kind == "official" { "official" } else { "imported" }; db.conn().execute( "INSERT INTO classifier_layers (id, name, kind, weight, enabled, source, imported_at) VALUES (?1, ?2, ?3, ?4, 1, ?5, ?6)", rusqlite::params![ layer_id, afcl.manifest.name, kind, DEFAULT_IMPORT_WEIGHT, source, unix_now(), ], )?; // Exemplars (vector + tags only). let mut exemplars = 0; for ex in &afcl.exemplars { if ex.vector.len() != NUM_FEATURES { continue; } // Reject non-finite vectors at the trust boundary. A crafted or corrupt // `.afcl` carrying a NaN/Inf component would otherwise flow into the k-NN // index, where it poisons standardization means (one NaN -> every // standardized vector NaN) and reaches the distance-sort comparators. if ex.vector.iter().any(|v| !v.is_finite()) { tracing::warn!("skipping imported exemplar with non-finite feature vector"); continue; } let vector = serde_json::to_string(&ex.vector) .map_err(|e| CoreError::Serialization(e.to_string()))?; let tags = serde_json::to_string(&ex.tags).map_err(|e| CoreError::Serialization(e.to_string()))?; // Globally-unique TEXT id (layer id is unique) so it survives cross-device sync. let ex_id = format!("{layer_id}#{exemplars}"); db.conn().execute( "INSERT INTO classifier_exemplars (id, layer_id, feat_version, vector, tags) VALUES (?1, ?2, ?3, ?4, ?5)", rusqlite::params![ex_id, layer_id, FEATURE_VERSION, vector, tags], )?; exemplars += 1; } // Rules: created disabled, grouped under the layer for the user to review. let mut rule_count = 0; for r in &afcl.rules { let created = rules::create_rule( db, NewRule { name: r.name.clone(), enabled: false, priority: None, match_mode: r.match_mode, conditions: r.conditions.clone(), actions: r.actions.clone(), }, )?; db.conn().execute( "INSERT INTO classifier_layer_rules (layer_id, rule_id) VALUES (?1, ?2)", rusqlite::params![layer_id, created.id], )?; rule_count += 1; } // Policy: only for tags the user hasn't configured (never clobber the user's own). let mut policies = 0; let configured: HashSet = super::exemplar::list_policies(db)? .into_iter() .map(|(t, _)| t) .collect(); for p in &afcl.policy { if configured.contains(&p.tag) { continue; } super::exemplar::set_policy(db, &p.tag, p.review_threshold, p.auto_threshold)?; policies += 1; } Ok(ImportSummary { layer_id, name: afcl.manifest.name.clone(), exemplars, rules: rule_count, policies, }) } // ── Layer management ── /// All imported/official layers with live exemplar + rule counts, newest first. #[instrument(skip_all)] pub fn list_layers(db: &Database) -> Result> { let conn = db.conn(); let mut stmt = conn.prepare( "SELECT l.id, l.name, l.kind, l.weight, l.enabled, l.source, l.imported_at, (SELECT COUNT(*) FROM classifier_exemplars e WHERE e.layer_id = l.id), (SELECT COUNT(*) FROM classifier_layer_rules r WHERE r.layer_id = l.id) FROM classifier_layers l ORDER BY l.imported_at DESC, l.id DESC", )?; let rows = stmt.query_map([], |row| { Ok(ClassifierLayer { id: row.get(0)?, name: row.get(1)?, kind: row.get(2)?, weight: row.get(3)?, enabled: row.get::<_, i64>(4)? != 0, source: row.get(5)?, imported_at: row.get(6)?, exemplar_count: row.get::<_, i64>(7)? as usize, rule_count: row.get::<_, i64>(8)? as usize, }) })?; Ok(rows.collect::, _>>()?) } /// Remove a layer and everything it brought in: its imported rules (from `tag_rules`, /// reconciling their tags away), then the layer row (cascading exemplars + membership). #[instrument(skip_all)] pub fn remove_layer(db: &Database, layer_id: &str) -> Result<()> { let rule_ids: Vec = { let mut stmt = db .conn() .prepare("SELECT rule_id FROM classifier_layer_rules WHERE layer_id = ?1")?; let rows = stmt.query_map([layer_id], |row| row.get::<_, String>(0))?; rows.collect::, _>>()? }; // Delete children explicitly (not via FK cascade): recursive_triggers is off, so cascade // deletes wouldn't fire the sync triggers and the removal wouldn't propagate to other // devices. Membership first, then the rules (delete_rule reconciles their tags away and // is sync-logged), then the exemplars, then the layer. db.conn().execute( "DELETE FROM classifier_layer_rules WHERE layer_id = ?1", [layer_id], )?; for id in &rule_ids { rules::delete_rule(db, id)?; } db.conn().execute( "DELETE FROM classifier_exemplars WHERE layer_id = ?1", [layer_id], )?; db.conn() .execute("DELETE FROM classifier_layers WHERE id = ?1", [layer_id])?; Ok(()) } /// Enable or disable a layer (disabled layers contribute nothing to the k-NN index). pub fn set_layer_enabled(db: &Database, layer_id: &str, enabled: bool) -> Result<()> { db.conn().execute( "UPDATE classifier_layers SET enabled = ?2 WHERE id = ?1", rusqlite::params![layer_id, enabled as i64], )?; Ok(()) } /// Set a layer's k-NN weight (clamped to `[0, 1]`, at or below the user's own data). pub fn set_layer_weight(db: &Database, layer_id: &str, weight: f64) -> Result<()> { let weight = weight.clamp(0.0, 1.0); db.conn().execute( "UPDATE classifier_layers SET weight = ?2 WHERE id = ?1", rusqlite::params![layer_id, weight], )?; Ok(()) } /// An imported exemplar joining the k-NN index: a synthetic id (`layer:#`, no /// audio), its raw feature vector, tags, and the owning layer's weight. pub struct ImportedExemplar { pub id: String, pub vector: Vec, pub tags: Vec, pub weight: f64, } /// Every current-version exemplar from **enabled** layers, unioned into the k-NN index by /// [`super::exemplar::build_index`], weighted below the user's own `local` data. pub fn enabled_imported_exemplars(db: &Database) -> Result> { let conn = db.conn(); let mut stmt = conn.prepare( "SELECT e.id, e.vector, e.tags, l.weight FROM classifier_exemplars e JOIN classifier_layers l ON l.id = e.layer_id WHERE l.enabled = 1 AND e.feat_version = ?1", )?; let rows = stmt.query_map([FEATURE_VERSION], |row| { Ok(( row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?, row.get::<_, f64>(3)?, )) })?; let mut out = Vec::new(); for r in rows { let (id, vec_json, tags_json, weight) = r?; let vector: Vec = serde_json::from_str(&vec_json).map_err(|e| CoreError::Serialization(e.to_string()))?; let tags: Vec = serde_json::from_str(&tags_json) .map_err(|e| CoreError::Serialization(e.to_string()))?; if vector.len() == NUM_FEATURES { out.push(ImportedExemplar { id, vector, tags, weight, }); } } Ok(out) } #[cfg(test)] mod tests { use super::*; use crate::analysis::classify::NUM_FEATURES; fn insert(db: &Database, hash: &str, fill: f64, tags: &[&str]) { db.conn() .execute( "INSERT INTO samples (hash, original_name, file_extension, file_size, import_date, last_modified) \ VALUES (?1, ?1, 'wav', 1, 0, 0)", [hash], ) .unwrap(); let v = vec![fill; NUM_FEATURES]; let json = serde_json::to_string(&v).unwrap(); db.conn() .execute( "INSERT INTO sample_features (hash, feat_version, vector, computed_at) VALUES (?1, ?2, ?3, 0)", rusqlite::params![hash, FEATURE_VERSION, json], ) .unwrap(); for t in tags { crate::tags::add_tag(db, hash, t).unwrap(); } } fn seed_library(db: &Database) { insert(db, "k1", 0.0, &["instrument.drum.kick"]); insert(db, "k2", 0.1, &["instrument.drum.kick"]); insert(db, "s1", 9.0, &["instrument.drum.snare"]); super::super::exemplar::set_policy(db, "instrument.drum.kick", 0.4, 0.8).unwrap(); rules::create_rule( db, NewRule { name: "kicks folder".to_string(), enabled: true, priority: None, match_mode: crate::rules::MatchMode::All, conditions: vec![], actions: vec![crate::rules::RuleAction::AddTag( "instrument.drum.kick".to_string(), )], }, ) .unwrap(); } #[test] fn export_carries_exemplars_rules_policy_no_audio() { let db = Database::open_in_memory().unwrap(); seed_library(&db); let afcl = build_export(&db, &ExportOptions::default()).unwrap(); assert_eq!(afcl.manifest.feat_version, FEATURE_VERSION); assert_eq!(afcl.exemplars.len(), 3); assert_eq!(afcl.rules.len(), 1); assert!(afcl.policy.iter().any(|p| p.tag == "instrument.drum.kick")); // Vectors travel, hashes do not (AfclExemplar has no hash field). assert!( afcl.exemplars .iter() .all(|e| e.vector.len() == NUM_FEATURES) ); } #[test] fn export_selects_components() { let db = Database::open_in_memory().unwrap(); seed_library(&db); let opts = ExportOptions { include_exemplars: false, include_rules: true, include_policy: false, ..ExportOptions::default() }; let afcl = build_export(&db, &opts).unwrap(); assert!(afcl.exemplars.is_empty()); assert_eq!(afcl.rules.len(), 1); assert!(afcl.policy.is_empty()); } #[test] fn round_trip_import_creates_layer() { let src = Database::open_in_memory().unwrap(); seed_library(&src); let json = export_to_string(&src, &ExportOptions::default()).unwrap(); let dst = Database::open_in_memory().unwrap(); let summary = import_from_string(&dst, &json, Some("friend.afcl")).unwrap(); assert_eq!(summary.exemplars, 3); assert_eq!(summary.rules, 1); assert_eq!(summary.policies, 1); let layers = list_layers(&dst).unwrap(); assert_eq!(layers.len(), 1); assert_eq!(layers[0].exemplar_count, 3); assert_eq!(layers[0].rule_count, 1); assert!((layers[0].weight - DEFAULT_IMPORT_WEIGHT).abs() < 1e-9); // Imported rules arrive disabled. let imported_ids = imported_rule_ids(&dst).unwrap(); let rule = rules::list_rules(&dst).unwrap(); assert!( rule.iter() .any(|r| imported_ids.contains(&r.id) && !r.enabled) ); // Imported exemplars are available to the k-NN index. let imp = enabled_imported_exemplars(&dst).unwrap(); assert_eq!(imp.len(), 3); assert!( imp.iter() .all(|e| (e.weight - DEFAULT_IMPORT_WEIGHT).abs() < 1e-9) ); } #[test] fn import_rejects_non_finite_exemplar_vectors() { let src = Database::open_in_memory().unwrap(); seed_library(&src); let mut afcl = build_export(&src, &ExportOptions::default()).unwrap(); assert_eq!(afcl.exemplars.len(), 3); // Poison one exemplar's vector. A crafted `.afcl` with an overflowing // exponent (`1e999`) parses to inf; it must be skipped at the trust // boundary rather than stored and later poisoning the k-NN index. afcl.exemplars[0].vector = vec![f64::INFINITY; NUM_FEATURES]; let dst = Database::open_in_memory().unwrap(); let summary = import(&dst, &afcl, Some("crafted.afcl")).unwrap(); assert_eq!(summary.exemplars, 2, "the non-finite exemplar is rejected"); let imp = enabled_imported_exemplars(&dst).unwrap(); assert_eq!(imp.len(), 2); assert!(imp.iter().all(|e| e.vector.iter().all(|x| x.is_finite()))); } #[test] fn import_does_not_clobber_existing_policy() { let src = Database::open_in_memory().unwrap(); seed_library(&src); // kick policy 0.4/0.8 let json = export_to_string(&src, &ExportOptions::default()).unwrap(); let dst = Database::open_in_memory().unwrap(); // The user already set their own kick policy. super::super::exemplar::set_policy(&dst, "instrument.drum.kick", 0.6, 0.95).unwrap(); import_from_string(&dst, &json, None).unwrap(); let p = super::super::exemplar::get_policy(&dst, "instrument.drum.kick").unwrap(); assert!( (p.review_threshold - 0.6).abs() < 1e-9, "user policy preserved" ); assert!((p.auto_threshold - 0.95).abs() < 1e-9); } #[test] fn feat_version_mismatch_is_rejected() { let db = Database::open_in_memory().unwrap(); let mut afcl = build_export(&db, &ExportOptions::default()).unwrap(); afcl.manifest.feat_version = FEATURE_VERSION + 1; let json = serde_json::to_string(&afcl).unwrap(); assert!(import_from_string(&db, &json, None).is_err()); } #[test] fn remove_layer_undoes_everything() { let src = Database::open_in_memory().unwrap(); seed_library(&src); let json = export_to_string(&src, &ExportOptions::default()).unwrap(); let dst = Database::open_in_memory().unwrap(); let summary = import_from_string(&dst, &json, None).unwrap(); let rules_before = rules::list_rules(&dst).unwrap().len(); remove_layer(&dst, &summary.layer_id).unwrap(); assert!(list_layers(&dst).unwrap().is_empty()); assert!(enabled_imported_exemplars(&dst).unwrap().is_empty()); // The layer's imported rule is gone too. assert_eq!(rules::list_rules(&dst).unwrap().len(), rules_before - 1); } #[test] fn disabled_layer_contributes_no_exemplars() { let src = Database::open_in_memory().unwrap(); seed_library(&src); let json = export_to_string(&src, &ExportOptions::default()).unwrap(); let dst = Database::open_in_memory().unwrap(); let summary = import_from_string(&dst, &json, None).unwrap(); set_layer_enabled(&dst, &summary.layer_id, false).unwrap(); assert!(enabled_imported_exemplars(&dst).unwrap().is_empty()); set_layer_enabled(&dst, &summary.layer_id, true).unwrap(); assert_eq!(enabled_imported_exemplars(&dst).unwrap().len(), 3); } }