Skip to main content

max / audiofiles

Index name search with an FTS5 trigram table (scale cliff: per-keystroke scan) Text search ran `n.name LIKE '%query%'` against vfs_nodes on every keystroke — a full scan of the node table per character, the search scale cliff the audit flagged. M027 adds an FTS5 virtual table (`vfs_nodes_fts`) over vfs_nodes.name with the trigram tokenizer, which makes `LIKE '%query%'` index-accelerated while keeping identical substring semantics (same escaping, same results). search.rs now routes the text clause through `n.id IN (SELECT rowid FROM vfs_nodes_fts WHERE name LIKE ?)`. The index is kept in sync by triggers on every vfs_nodes insert/update-of-name/ delete — with no applying_remote guard, so it tracks remote sync applies too; the FTS table and its shadow tables are local-derived and never synced. Verified present in the bundled SQLite (3.45, trigram available). Tests: core 538 green — new tests assert the query plan reaches the FTS index and that rename + delete keep it synced; existing substring-search tests unchanged. browser 224, sync 52 green; clippy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-15 23:57 UTC
Commit: 80e778f8ae884e1d3bc84f778bba5ee44973fc91
Parent: d31bf85
2 files changed, +113 insertions, -8 deletions
M crates/audiofiles-core/src/db.rs +42 -7
@@ -1417,6 +1417,38 @@ BEGIN
1417 1417 END;
1418 1418 "#;
1419 1419
1420 + const MIGRATION_027: &str = r#"
1421 + -- FTS5 trigram index over vfs_nodes.name for fast substring search.
1422 + --
1423 + -- The trigram tokenizer lets `name LIKE '%query%'` use the index instead of
1424 + -- scanning all of vfs_nodes on every keystroke, which is the scale cliff the
1425 + -- ultra-fuzz audit flagged. The table is content-bearing (standalone) and keyed
1426 + -- by rowid = vfs_nodes.id, kept in sync by triggers that fire on EVERY vfs_nodes
1427 + -- change -- including remote sync applies (no applying_remote guard), because
1428 + -- the local derived index must always mirror the local rows. The FTS table and
1429 + -- its shadow tables are local-only and never synced (no sync_changelog triggers).
1430 + CREATE VIRTUAL TABLE IF NOT EXISTS vfs_nodes_fts USING fts5(
1431 + name,
1432 + tokenize = 'trigram'
1433 + );
1434 +
1435 + -- Backfill from existing rows.
1436 + INSERT INTO vfs_nodes_fts(rowid, name) SELECT id, name FROM vfs_nodes;
1437 +
1438 + CREATE TRIGGER IF NOT EXISTS vfs_nodes_fts_insert AFTER INSERT ON vfs_nodes BEGIN
1439 + INSERT INTO vfs_nodes_fts(rowid, name) VALUES (NEW.id, NEW.name);
1440 + END;
1441 +
1442 + CREATE TRIGGER IF NOT EXISTS vfs_nodes_fts_delete AFTER DELETE ON vfs_nodes BEGIN
1443 + DELETE FROM vfs_nodes_fts WHERE rowid = OLD.id;
1444 + END;
1445 +
1446 + CREATE TRIGGER IF NOT EXISTS vfs_nodes_fts_update AFTER UPDATE OF name ON vfs_nodes BEGIN
1447 + DELETE FROM vfs_nodes_fts WHERE rowid = OLD.id;
1448 + INSERT INTO vfs_nodes_fts(rowid, name) VALUES (NEW.id, NEW.name);
1449 + END;
1450 + "#;
1451 +
1420 1452 /// Register `hash_row_id(salt, key) -> TEXT` as a deterministic SQLite
1421 1453 /// function on the given connection. Used by the M018 sync triggers so the
1422 1454 /// `sync_changelog.row_id` field never carries cleartext content (tag strings,
@@ -1534,6 +1566,7 @@ impl Database {
1534 1566 MIGRATION_024,
1535 1567 MIGRATION_025,
1536 1568 MIGRATION_026,
1569 + MIGRATION_027,
1537 1570 ];
1538 1571
1539 1572 for (i, sql) in MIGRATIONS.iter().enumerate() {
@@ -1701,9 +1734,11 @@ mod tests {
1701 1734 fn open_in_memory_creates_all_tables() {
1702 1735 let db = Database::open_in_memory().unwrap();
1703 1736
1737 + // Exclude the FTS5 shadow tables (vfs_nodes_fts, _data, _idx, _docsize,
1738 + // _config) created by M027 — this asserts the set of logical tables.
1704 1739 let tables: Vec<String> = db
1705 1740 .conn()
1706 - .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name")
1741 + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE 'vfs_nodes_fts%' ORDER BY name")
1707 1742 .unwrap()
1708 1743 .query_map([], |row| row.get(0))
1709 1744 .unwrap()
@@ -1743,7 +1778,7 @@ mod tests {
1743 1778 .conn()
1744 1779 .query_row("PRAGMA user_version", [], |row| row.get(0))
1745 1780 .unwrap();
1746 - assert_eq!(version, 26);
1781 + assert_eq!(version, 27);
1747 1782 }
1748 1783
1749 1784 #[test]
@@ -1754,7 +1789,7 @@ mod tests {
1754 1789 .conn()
1755 1790 .query_row("PRAGMA user_version", [], |row| row.get(0))
1756 1791 .unwrap();
1757 - assert_eq!(version, 26);
1792 + assert_eq!(version, 27);
1758 1793 }
1759 1794
1760 1795 /// Open a fresh file-backed DB, close, reopen. The second open re-enters
@@ -1773,7 +1808,7 @@ mod tests {
1773 1808 .conn()
1774 1809 .query_row("PRAGMA user_version", [], |row| row.get(0))
1775 1810 .unwrap();
1776 - assert_eq!(version, 26);
1811 + assert_eq!(version, 27);
1777 1812 }
1778 1813
1779 1814 /// Simulates the worst-case recovery path: a prior partial migration left
@@ -1817,7 +1852,7 @@ mod tests {
1817 1852 .conn()
1818 1853 .query_row("PRAGMA user_version", [], |row| row.get(0))
1819 1854 .unwrap();
1820 - assert_eq!(version, 26);
1855 + assert_eq!(version, 27);
1821 1856 }
1822 1857
1823 1858 /// M018 contract: the `sync_changelog.row_id` for sensitive tables must
@@ -2039,7 +2074,7 @@ mod tests {
2039 2074 let initial_version: i32 = conn
2040 2075 .query_row("PRAGMA user_version", [], |row| row.get(0))
2041 2076 .unwrap();
2042 - assert_eq!(initial_version, 26);
2077 + assert_eq!(initial_version, 27);
2043 2078
2044 2079 let batch = format!(
2045 2080 "BEGIN;\n{}\nPRAGMA user_version = 999;\nCOMMIT;",
@@ -2102,7 +2137,7 @@ mod tests {
2102 2137 .conn()
2103 2138 .query_row("PRAGMA user_version", [], |row| row.get(0))
2104 2139 .unwrap();
2105 - assert_eq!(version, 26);
2140 + assert_eq!(version, 27);
2106 2141 }
2107 2142
2108 2143 #[test]
M crates/audiofiles-core/src/search.rs +71 -1
@@ -272,8 +272,12 @@ fn append_filter_clauses(
272 272 ) {
273 273 if !filter.text_query.is_empty() {
274 274 let pattern = format!("%{}%", tagtree::escape_like(&filter.text_query));
275 + // Route the substring match through the FTS5 trigram index (M027) instead
276 + // of a full `n.name LIKE` scan of vfs_nodes. The trigram tokenizer makes
277 + // `LIKE '%query%'` index-accelerated; semantics are identical to the old
278 + // direct LIKE (still a literal substring match, same escaping).
275 279 sql.push_str(&format!(
276 - " AND (n.name LIKE ?{idx} ESCAPE '\\')",
280 + " AND n.id IN (SELECT rowid FROM vfs_nodes_fts WHERE name LIKE ?{idx} ESCAPE '\\')",
277 281 idx = params.len() + 1
278 282 ));
279 283 params.push(Box::new(pattern));
@@ -386,6 +390,72 @@ mod tests {
386 390 }
387 391
388 392 #[test]
393 + fn fts_index_stays_synced_through_rename_and_delete() {
394 + let (db, vfs_id) = setup_with_samples();
395 + // setup has "kick.wav" (hash1) and "snare.wav" (hash2).
396 + let mut filter = SearchFilter::default();
397 + filter.text_query = "kick".to_string();
398 + let nodes = search_in_folder(&db, &filter, vfs_id, None).unwrap();
399 + assert_eq!(nodes.len(), 1, "kick.wav should match before rename");
400 + let kick_id = nodes[0].node.id;
401 +
402 + // Rename kick.wav -> hat.wav: the UPDATE trigger must re-index it.
403 + vfs::rename_node(&db, kick_id, "hat.wav").unwrap();
404 + assert_eq!(
405 + search_in_folder(&db, &filter, vfs_id, None).unwrap().len(),
406 + 0,
407 + "kick must no longer match after rename"
408 + );
409 + let mut hat = SearchFilter::default();
410 + hat.text_query = "hat".to_string();
411 + assert_eq!(
412 + search_in_folder(&db, &hat, vfs_id, None).unwrap().len(),
413 + 1,
414 + "hat must match after rename"
415 + );
416 +
417 + // Delete it: the DELETE trigger must drop it from the index.
418 + vfs::delete_node(&db, kick_id).unwrap();
419 + assert_eq!(
420 + search_in_folder(&db, &hat, vfs_id, None).unwrap().len(),
421 + 0,
422 + "hat must not match after delete"
423 + );
424 + }
425 +
426 + #[test]
427 + fn text_search_uses_the_fts_trigram_index() {
428 + let (db, vfs_id) = setup_with_samples();
429 + let mut filter = SearchFilter::default();
430 + filter.text_query = "kick".to_string();
431 +
432 + // Build the same SQL the search uses and confirm its plan reaches the
433 + // FTS virtual table rather than a linear scan of vfs_nodes by name.
434 + let mut sql = String::from(
435 + "SELECT n.id FROM vfs_nodes n
436 + LEFT JOIN audio_analysis a ON n.sample_hash = a.hash
437 + LEFT JOIN samples s ON n.sample_hash = s.hash
438 + WHERE n.vfs_id = ?1 AND n.parent_id IS ?2 AND s.deleted_at IS NULL",
439 + );
440 + let mut params: Vec<Box<dyn rusqlite::types::ToSql>> = vec![Box::new(vfs_id), Box::new(None::<NodeId>)];
441 + append_filter_clauses(&mut sql, &mut params, &filter);
442 +
443 + let plan: String = {
444 + let mut stmt = db.conn().prepare(&format!("EXPLAIN QUERY PLAN {sql}")).unwrap();
445 + let refs: Vec<&dyn rusqlite::types::ToSql> = params.iter().map(|p| p.as_ref()).collect();
446 + stmt.query_map(refs.as_slice(), |r| r.get::<_, String>(3))
447 + .unwrap()
448 + .map(|r| r.unwrap())
449 + .collect::<Vec<_>>()
450 + .join(" | ")
451 + };
452 + assert!(
453 + plan.contains("vfs_nodes_fts"),
454 + "query plan should hit the FTS index, got: {plan}"
455 + );
456 + }
457 +
458 + #[test]
389 459 fn folder_search_caps_at_result_limit() {
390 460 let db = Database::open_in_memory().unwrap();
391 461 let vfs_id = vfs::create_vfs(&db, "Big").unwrap();