Skip to main content

max / audiofiles

Persist the k-nearest-neighbour graph Similarity answered from a session-cached VP-tree, so the neighbourhood was computed and thrown away per ask and nothing could be built on top of it. M038 stores the k out-edges per sample plus the ranges they were computed under, on the waveform_data model: derived, no sync triggers. The search worker maintains it, because keeping it current needs a tree and that is the only place one is built. Reads fall back to the tree whenever the graph cannot answer, so cold, stale or half-built costs speed and never correctness.
Author: Max Johnson <me@maxj.phd> · 2026-08-09 15:20 UTC
Signed with PGP, not checked
Commit: 53175e1de64803eb4bfb3a0a6a981a715659f70e
Parent: 4df39eb
6 files changed, +627 insertions, -28 deletions
M Cargo.lock +4 -4
@@ -7313,6 +7313,10 @@
7313 7313 name = "quasi-webview"
7314 7314 version = "0.1.0"
7315 7315
7316 + [[patch.unused]]
7317 + name = "docengine"
7318 + version = "0.4.0"
7319 +
7316 7320 [[patch.unused]]
7317 7321 name = "kberg"
7318 7322 version = "0.1.0"
@@ -7320,7 +7324,3 @@
7320 7324 [[patch.unused]]
7321 7325 name = "painhours"
7322 7326 version = "0.1.0"
7323 -
7324 - [[patch.unused]]
7325 - name = "docengine"
7326 - version = "0.4.0"
M crates/audiofiles-core/src/db.rs +69 -6
@@ -1753,6 +1753,65 @@
1753 1753 CREATE INDEX IF NOT EXISTS idx_vfs_nodes_sort ON vfs_nodes(node_type, name);
1754 1754 ";
1755 1755
1756 + const MIGRATION_038: &str = r"
1757 + -- The persisted k-nearest-neighbour graph over the similarity features.
1758 + --
1759 + -- Derived data, on the waveform_data model (M004): recomputable from
1760 + -- audio_analysis, and machine-dependent besides, because the distances are
1761 + -- normalized against this library's global feature ranges. So it carries no
1762 + -- sync triggers at all, where every hand-entered table near it carries three.
1763 + --
1764 + -- What it buys: a similarity query answers from a table read instead of a
1765 + -- VP-tree build, and the neighbourhood becomes a structure other things can be
1766 + -- built on (chains, regions, 'what is unlike everything') rather than a ranking
1767 + -- computed and thrown away per ask.
1768 + CREATE TABLE IF NOT EXISTS sample_neighbours (
1769 + hash TEXT NOT NULL REFERENCES samples(hash) ON DELETE CASCADE,
1770 + neighbour_hash TEXT NOT NULL,
1771 + distance REAL NOT NULL,
1772 + rank INTEGER NOT NULL,
1773 + PRIMARY KEY (hash, neighbour_hash)
1774 + );
1775 + CREATE INDEX IF NOT EXISTS idx_sample_neighbours_rank ON sample_neighbours(hash, rank);
1776 + -- The back-edge index. A delete has to find every source pointing AT the gone
1777 + -- sample, which is the one access this table makes against the grain of its
1778 + -- primary key.
1779 + CREATE INDEX IF NOT EXISTS idx_sample_neighbours_back ON sample_neighbours(neighbour_hash);
1780 +
1781 + -- Single-row provenance: the k the edges were computed for, and the
1782 + -- normalization ranges they were computed under. A range that has since widened
1783 + -- invalidates every stored distance in principle, so the ranges are what the
1784 + -- refresh pass compares against to choose rebuild over incremental update.
1785 + CREATE TABLE IF NOT EXISTS neighbour_graph_meta (
1786 + id INTEGER PRIMARY KEY CHECK (id = 1),
1787 + k INTEGER NOT NULL,
1788 + ranges TEXT NOT NULL,
1789 + stale INTEGER NOT NULL DEFAULT 0,
1790 + built_at INTEGER NOT NULL
1791 + );
1792 +
1793 + -- Samples whose out-edges need recomputing: freshly analysed, or left short by
1794 + -- a deleted neighbour. Drained by the refresh pass on the next similarity
1795 + -- query, which is a point where a VP-tree is being built anyway. Persisted
1796 + -- rather than held in worker memory so a restart mid-import does not lose track
1797 + -- of which sources are behind.
1798 + CREATE TABLE IF NOT EXISTS neighbour_graph_dirty (
1799 + hash TEXT PRIMARY KEY REFERENCES samples(hash) ON DELETE CASCADE
1800 + );
1801 +
1802 + -- Out-edges cascade with the sample row. Back-edges do not: they name a hash
1803 + -- that is not the row's own, so no foreign key connects them to the delete.
1804 + -- Without this the table accumulates edges pointing at samples that are gone,
1805 + -- and the sources holding them are silently short of k.
1806 + CREATE TRIGGER IF NOT EXISTS neighbour_graph_delete_back_edges
1807 + AFTER DELETE ON samples
1808 + BEGIN
1809 + INSERT OR IGNORE INTO neighbour_graph_dirty (hash)
1810 + SELECT hash FROM sample_neighbours WHERE neighbour_hash = OLD.hash;
1811 + DELETE FROM sample_neighbours WHERE neighbour_hash = OLD.hash;
1812 + END;
1813 + ";
1814 +
1756 1815 /// Register `hash_row_id(salt, key) -> TEXT` as a deterministic SQLite
1757 1816 /// function on the given connection. Used by the M018 sync triggers so the
1758 1817 /// `sync_changelog.row_id` field never carries cleartext content (tag strings,
@@ -1944,6 +2003,7 @@
1944 2003 MIGRATION_035,
1945 2004 MIGRATION_036,
1946 2005 MIGRATION_037,
2006 + MIGRATION_038,
1947 2007 ];
1948 2008
1949 2009 for (i, sql) in MIGRATIONS.iter().enumerate() {
@@ -2346,7 +2406,10 @@
2346 2406 "edit_history",
2347 2407 "fingerprints",
2348 2408 "hlc_ledger",
2409 + "neighbour_graph_dirty",
2410 + "neighbour_graph_meta",
2349 2411 "sample_features",
2412 + "sample_neighbours",
2350 2413 "samples",
2351 2414 "sync_changelog",
2352 2415 "sync_state",
@@ -2370,7 +2433,7 @@
2370 2433 .conn()
2371 2434 .query_row("PRAGMA user_version", [], |row| row.get(0))
2372 2435 .unwrap();
2373 - assert_eq!(version, 37);
2436 + assert_eq!(version, 38);
2374 2437 }
2375 2438
2376 2439 #[test]
@@ -2381,7 +2444,7 @@
2381 2444 .conn()
2382 2445 .query_row("PRAGMA user_version", [], |row| row.get(0))
2383 2446 .unwrap();
2384 - assert_eq!(version, 37);
2447 + assert_eq!(version, 38);
2385 2448 }
2386 2449
2387 2450 #[test]
@@ -2565,7 +2628,7 @@
2565 2628 .conn()
2566 2629 .query_row("PRAGMA user_version", [], |row| row.get(0))
2567 2630 .unwrap();
2568 - assert_eq!(version, 37);
2631 + assert_eq!(version, 38);
2569 2632 }
2570 2633
2571 2634 /// Simulates the worst-case recovery path: a prior partial migration left
@@ -2609,7 +2672,7 @@
2609 2672 .conn()
2610 2673 .query_row("PRAGMA user_version", [], |row| row.get(0))
2611 2674 .unwrap();
2612 - assert_eq!(version, 37);
2675 + assert_eq!(version, 38);
2613 2676 }
2614 2677
2615 2678 /// M037 contract: the browse-list sort must not build a temp B-tree.
@@ -2880,7 +2943,7 @@
2880 2943 let initial_version: i32 = conn
2881 2944 .query_row("PRAGMA user_version", [], |row| row.get(0))
2882 2945 .unwrap();
2883 - assert_eq!(initial_version, 37);
2946 + assert_eq!(initial_version, 38);
2884 2947
2885 2948 let batch = format!("BEGIN;\n{bad_sql}\nPRAGMA user_version = 999;\nCOMMIT;");
2886 2949 let first_err = conn.execute_batch(&batch).unwrap_err();
@@ -2945,7 +3008,7 @@
2945 3008 .conn()
2946 3009 .query_row("PRAGMA user_version", [], |row| row.get(0))
2947 3010 .unwrap();
2948 - assert_eq!(version, 37);
3011 + assert_eq!(version, 38);
2949 3012 }
2950 3013
2951 3014 #[test]
M crates/audiofiles-core/src/similarity.rs +470 -2
@@ -1,8 +1,10 @@
1 1 //! Similarity search: find samples with similar audio features using weighted Euclidean distance.
2 2
3 3 use crate::db::Database;
4 - use crate::error::{CoreError, Result};
4 + use crate::error::{CoreError, Result, unix_now};
5 5 use crate::vp_tree::VpTree;
6 + use rusqlite::{Connection, params};
7 + use serde::{Deserialize, Serialize};
6 8 use tracing::instrument;
7 9
8 10 /// Audio feature vector for similarity comparison.
@@ -68,7 +70,11 @@
68 70 }
69 71
70 72 /// Normalization ranges for each feature dimension, learned from the dataset.
71 - #[derive(Debug, Clone, Default)]
73 + ///
74 + /// Round-trips through `neighbour_graph_meta.ranges` so a stored distance can be
75 + /// checked against the ranges it was computed under; `PartialEq` is what
76 + /// [`NeighbourGraph::refresh`] uses to notice that they have drifted.
77 + #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
72 78 struct NormRanges {
73 79 bpm: (f64, f64),
74 80 duration: (f64, f64),
@@ -493,6 +499,40 @@
493 499 })
494 500 .collect()
495 501 }
502 +
503 + /// Hash of the `i`th indexed sample, for `i` in `0..len()`.
504 + pub fn hash_at(&self, i: usize) -> &str {
505 + &self.tree.get(i).hash
506 + }
507 +
508 + /// Neighbours of the `i`th indexed sample, queried from its own stored
509 + /// entry rather than from raw features.
510 + ///
511 + /// Equivalent to [`find_similar`](Self::find_similar) with that sample's
512 + /// features, minus the redundant re-normalization; the graph builder walks
513 + /// every sample in the index, so that saving is the whole build.
514 + /// Self-exclusion is by tree index rather than by hash, so a distinct sample
515 + /// that happens to sit at distance zero is still a neighbour.
516 + pub fn neighbours_of(&self, i: usize, limit: usize) -> Vec<SimilarResult> {
517 + // Request limit+1: the query point is itself in the tree.
518 + let candidates = self
519 + .tree
520 + .find_nearest(self.tree.get(i), limit + 1, entry_distance);
521 + candidates
522 + .into_iter()
523 + .filter(|c| c.index != i)
524 + .take(limit)
525 + .map(|c| SimilarResult {
526 + hash: self.tree.get(c.index).hash.clone(),
527 + distance: c.distance,
528 + })
529 + .collect()
530 + }
531 +
532 + /// The normalization ranges this index was built under.
533 + fn ranges(&self) -> &NormRanges {
534 + &self.ranges
535 + }
496 536 }
497 537
498 538 /// Compute min/max ranges across all samples (no reference bias).
@@ -531,6 +571,366 @@
531 571 ranges
532 572 }
533 573
574 + // --- Persisted neighbour graph ---
575 +
576 + /// Out-edges stored per sample in the persisted neighbour graph.
577 + ///
578 + /// The read path answers any query up to this limit from `sample_neighbours`;
579 + /// past it the VP-tree is the only source. This is the similarity path's own
580 + /// constant and is unrelated to `analysis::exemplar::DEFAULT_K`, which belongs
581 + /// to the classifier: the two share no call site and never did.
582 + pub const GRAPH_K: usize = 50;
583 +
584 + /// Fraction of the library that has to be pending before [`NeighbourGraph::refresh`]
585 + /// stops updating sample by sample and rebuilds the whole graph instead.
586 + ///
587 + /// A cost trade, not a semantic one: both paths land on the same edges. One
588 + /// incremental update is a tree query plus a few dozen small writes, and a full
589 + /// rebuild is one tree query per sample with no per-source bookkeeping at all,
590 + /// so past roughly a quarter of the library the incremental path stops paying
591 + /// for itself. A first import takes the rebuild path, which is what it should do.
592 + const REBUILD_WHEN_DIRTY_FRACTION: usize = 4;
593 +
594 + /// What the graph was built with, from `neighbour_graph_meta`.
595 + struct GraphMeta {
596 + k: usize,
597 + ranges: NormRanges,
598 + stale: bool,
599 + }
600 +
601 + /// The stored k-nearest-neighbour graph: `sample_neighbours` plus the two small
602 + /// tables that say how far behind it is.
603 + ///
604 + /// The point is not caching a query, it is having the neighbourhood as a thing
605 + /// that exists. A VP-tree answers "what is near this" and forgets; a stored
606 + /// graph can be walked, so chains, regions and "what is near nothing" become
607 + /// reachable without a new index each time.
608 + ///
609 + /// Distances are exactly what [`SimilarityIndex::find_similar`] returns:
610 + /// weighted Euclidean on the min-max-normalized vector, no second scale layered
611 + /// on top. Nothing scores a sample as a share of its neighbourhood any more, so
612 + /// there is no denominator left for a standardisation to serve, and keeping the
613 + /// raw number is what makes "how far apart are these two really" answerable.
614 + ///
615 + /// The graph is derived and additive. Every read path falls back to the VP-tree
616 + /// when it cannot answer, so a graph that is cold, stale or half-built costs
617 + /// speed and never correctness.
618 + pub struct NeighbourGraph;
619 +
620 + impl NeighbourGraph {
621 + /// Read the single meta row. `None` means the graph was never built.
622 + fn meta(db: &Database) -> Result<Option<GraphMeta>> {
623 + let row = db.conn().query_row(
624 + "SELECT k, ranges, stale FROM neighbour_graph_meta WHERE id = 1",
625 + [],
626 + |r| {
627 + Ok((
628 + r.get::<_, i64>(0)?,
629 + r.get::<_, String>(1)?,
630 + r.get::<_, i64>(2)?,
631 + ))
632 + },
633 + );
634 + match row {
635 + Ok((k, ranges, stale)) => Ok(Some(GraphMeta {
636 + k: usize::try_from(k).unwrap_or(0),
637 + ranges: serde_json::from_str(&ranges)
638 + .map_err(|e| CoreError::Serialization(e.to_string()))?,
639 + stale: stale != 0,
640 + })),
641 + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
642 + Err(e) => Err(e.into()),
643 + }
644 + }
645 +
646 + /// Whether the graph cannot be trusted as it stands: never built, explicitly
647 + /// marked, or built for a smaller k than the code now asks for.
648 + pub fn is_stale(db: &Database) -> Result<bool> {
649 + Ok(match Self::meta(db)? {
650 + None => true,
651 + Some(m) => m.stale || m.k < GRAPH_K,
652 + })
653 + }
654 +
655 + /// Distrust the whole graph, for a caller that knows the library changed but
656 + /// not which samples. The next [`refresh`](Self::refresh) rebuilds it.
657 + pub fn mark_stale(db: &Database) -> Result<()> {
658 + db.conn()
659 + .execute("UPDATE neighbour_graph_meta SET stale = 1 WHERE id = 1", [])?;
660 + Ok(())
661 + }
662 +
663 + /// Queue samples whose out-edges need recomputing. Unknown hashes are
664 + /// dropped rather than rejected, so a caller can pass a batch without first
665 + /// checking which of it survived.
666 + pub fn mark_dirty(db: &Database, hashes: &[String]) -> Result<()> {
667 + if hashes.is_empty() {
668 + return Ok(());
669 + }
670 + db.transaction_core(|_tx| {
671 + let mut stmt = db.conn().prepare(
672 + "INSERT OR IGNORE INTO neighbour_graph_dirty (hash)
673 + SELECT hash FROM samples WHERE hash = ?1",
674 + )?;
675 + for hash in hashes {
676 + stmt.execute([hash])?;
677 + }
678 + Ok(())
679 + })
680 + }
681 +
682 + /// Stored neighbours of `hash`, or `None` when the graph cannot answer and
683 + /// the caller must fall back to the VP-tree.
684 + ///
685 + /// It declines for a limit past `GRAPH_K`, for a graph that is stale or has
686 + /// this sample queued for update, and for a row set shorter than asked that
687 + /// is not simply the whole library. That last case is what the count query
688 + /// is for: in a library of three samples, two neighbours IS the complete
689 + /// answer to a request for fifty, and treating it as a miss would mean the
690 + /// graph never answers a small library at all.
691 + #[instrument(skip_all, fields(hash = %hash))]
692 + pub fn neighbours(
693 + db: &Database,
694 + hash: &str,
695 + limit: usize,
696 + ) -> Result<Option<Vec<SimilarResult>>> {
697 + if limit == 0 {
698 + return Ok(Some(Vec::new()));
699 + }
700 + if limit > GRAPH_K {
701 + return Ok(None);
702 + }
703 + match Self::meta(db)? {
704 + Some(m) if !m.stale && m.k >= limit => {}
705 + _ => return Ok(None),
706 + }
707 + let queued: bool = db.conn().query_row(
708 + "SELECT EXISTS(SELECT 1 FROM neighbour_graph_dirty WHERE hash = ?1)",
709 + [hash],
710 + |r| r.get(0),
711 + )?;
712 + if queued {
713 + return Ok(None);
714 + }
715 +
716 + let mut stmt = db.conn().prepare(
717 + "SELECT neighbour_hash, distance FROM sample_neighbours
718 + WHERE hash = ?1 ORDER BY rank LIMIT ?2",
719 + )?;
720 + let rows: Vec<SimilarResult> = stmt
721 + .query_map(params![hash, limit as i64], |r| {
722 + Ok(SimilarResult {
723 + hash: r.get(0)?,
724 + distance: r.get(1)?,
725 + })
726 + })?
727 + .collect::<std::result::Result<_, _>>()?;
728 + if rows.len() >= limit {
729 + return Ok(Some(rows));
730 + }
731 +
732 + // Short of the limit. Only the whole-library case makes that an answer,
733 + // and only for a sample the graph could have covered in the first place.
734 + let (analysed, known): (i64, bool) = db.conn().query_row(
735 + "SELECT (SELECT COUNT(*) FROM audio_analysis),
736 + EXISTS(SELECT 1 FROM audio_analysis WHERE hash = ?1)",
737 + [hash],
738 + |r| Ok((r.get(0)?, r.get(1)?)),
739 + )?;
740 + if known && rows.len() as i64 + 1 >= analysed {
741 + Ok(Some(rows))
742 + } else {
743 + Ok(None)
744 + }
745 + }
746 +
747 + /// Rebuild every edge from `index`, replacing whatever was stored.
748 + #[instrument(skip_all)]
749 + pub fn rebuild(db: &Database, index: &SimilarityIndex) -> Result<()> {
750 + let ranges = serde_json::to_string(index.ranges())
751 + .map_err(|e| CoreError::Serialization(e.to_string()))?;
752 + let built_at = unix_now();
753 + db.transaction_core(|_tx| {
754 + let conn = db.conn();
755 + conn.execute("DELETE FROM sample_neighbours", [])?;
756 + conn.execute("DELETE FROM neighbour_graph_dirty", [])?;
757 + {
758 + let mut stmt = conn.prepare(
759 + "INSERT OR REPLACE INTO sample_neighbours (hash, neighbour_hash, distance, rank)
760 + VALUES (?1, ?2, ?3, ?4)",
761 + )?;
762 + for i in 0..index.len() {
763 + let hash = index.hash_at(i);
764 + for (rank, n) in index.neighbours_of(i, GRAPH_K).iter().enumerate() {
765 + stmt.execute(params![hash, n.hash, n.distance, rank as i64])?;
766 + }
767 + }
768 + }
769 + conn.execute(
770 + "INSERT OR REPLACE INTO neighbour_graph_meta (id, k, ranges, stale, built_at)
771 + VALUES (1, ?1, ?2, 0, ?3)",
772 + params![GRAPH_K as i64, ranges, built_at],
773 + )?;
774 + Ok(())
775 + })
776 + }
777 +
778 + /// Recompute one sample's out-edges, and offer it as a neighbour to each
779 + /// sample it landed near.
780 + ///
781 + /// At most k+1 sources are touched: the sample itself, plus a back-edge into
782 + /// each neighbour where the new distance beats that neighbour's current
783 + /// k-th. `index` need not contain `hash` — the query goes by features, and
784 + /// the metric is symmetric, so the back-edge distance is the same number
785 + /// already computed for the out-edge.
786 + #[instrument(skip_all, fields(hash = %hash))]
787 + pub fn insert(db: &Database, index: &SimilarityIndex, hash: &str) -> Result<()> {
788 + let features = load_features(db, hash)?;
789 + let outs = index.find_similar(hash, &features, GRAPH_K);
790 + db.transaction_core(|_tx| {
791 + let conn = db.conn();
792 + conn.execute("DELETE FROM sample_neighbours WHERE hash = ?1", [hash])?;
793 + {
794 + let mut stmt = conn.prepare(
795 + "INSERT OR REPLACE INTO sample_neighbours (hash, neighbour_hash, distance, rank)
796 + VALUES (?1, ?2, ?3, ?4)",
797 + )?;
798 + for (rank, n) in outs.iter().enumerate() {
799 + stmt.execute(params![hash, n.hash, n.distance, rank as i64])?;
800 + }
801 + }
802 + for n in &outs {
803 + Self::offer_back_edge(conn, &n.hash, hash, n.distance)?;
804 + }
805 + conn.execute("DELETE FROM neighbour_graph_dirty WHERE hash = ?1", [hash])?;
806 + Ok(())
807 + })
808 + }
809 +
810 + /// Write `neighbour` into `source`'s edge list if it earns a place, then
811 + /// trim the list back to k and renumber it.
812 + fn offer_back_edge(
813 + conn: &Connection,
814 + source: &str,
815 + neighbour: &str,
816 + distance: f64,
817 + ) -> Result<()> {
818 + let (count, worst): (i64, Option<f64>) = conn.query_row(
819 + "SELECT COUNT(*), MAX(distance) FROM sample_neighbours WHERE hash = ?1",
820 + [source],
821 + |r| Ok((r.get(0)?, r.get(1)?)),
822 + )?;
823 + let earns_a_place =
824 + usize::try_from(count).unwrap_or(0) < GRAPH_K || worst.is_some_and(|w| distance < w);
825 + if !earns_a_place {
826 + return Ok(());
827 + }
828 + // Rank goes in as a placeholder: the insert and the trim below both
829 + // reorder the list, so it is recomputed for the whole source afterwards.
830 + conn.execute(
831 + "INSERT OR REPLACE INTO sample_neighbours (hash, neighbour_hash, distance, rank)
832 + VALUES (?1, ?2, ?3, 0)",
833 + params![source, neighbour, distance],
834 + )?;
835 + conn.execute(
836 + "DELETE FROM sample_neighbours WHERE hash = ?1 AND neighbour_hash NOT IN (
837 + SELECT neighbour_hash FROM sample_neighbours WHERE hash = ?1
838 + ORDER BY distance, neighbour_hash LIMIT ?2)",
839 + params![source, GRAPH_K as i64],
840 + )?;
841 + Self::renumber(conn, source)
842 + }
843 +
844 + /// Recompute `rank` for one source from the stored distances. Ties break on
845 + /// `neighbour_hash` so the order is total and the ranks are unique.
846 + fn renumber(conn: &Connection, source: &str) -> Result<()> {
847 + conn.execute(
848 + "UPDATE sample_neighbours SET rank = (
849 + SELECT COUNT(*) FROM sample_neighbours AS other
850 + WHERE other.hash = sample_neighbours.hash
851 + AND (other.distance < sample_neighbours.distance
852 + OR (other.distance = sample_neighbours.distance
853 + AND other.neighbour_hash < sample_neighbours.neighbour_hash))
854 + ) WHERE hash = ?1",
855 + [source],
856 + )?;
857 + Ok(())
858 + }
859 +
860 + /// Drop a sample from the graph in both directions.
861 + ///
862 + /// The `neighbour_graph_delete_back_edges` trigger does this when the sample
863 + /// row itself goes; this is the explicit form, for a caller unlinking a
864 + /// sample while its row stays. Sources left short are queued rather than
865 + /// refilled here, because refilling needs a VP-tree and the refresh pass is
866 + /// where one exists.
867 + pub fn remove(db: &Database, hash: &str) -> Result<()> {
868 + db.transaction_core(|_tx| {
869 + let conn = db.conn();
870 + conn.execute(
871 + "INSERT OR IGNORE INTO neighbour_graph_dirty (hash)
872 + SELECT hash FROM sample_neighbours WHERE neighbour_hash = ?1",
873 + [hash],
874 + )?;
875 + conn.execute(
876 + "DELETE FROM sample_neighbours WHERE neighbour_hash = ?1",
877 + [hash],
878 + )?;
879 + conn.execute("DELETE FROM sample_neighbours WHERE hash = ?1", [hash])?;
880 + conn.execute("DELETE FROM neighbour_graph_dirty WHERE hash = ?1", [hash])?;
881 + Ok(())
882 + })
883 + }
884 +
885 + /// Bring the graph up to date against `index`, incrementally where that is
886 + /// cheaper and by full rebuild where it is not.
887 + ///
888 + /// Range drift decides it first. `index` was built from current data, so its
889 + /// ranges differing from the stored ones means every stored distance was
890 + /// computed under a normalization that no longer holds, and no amount of
891 + /// per-sample repair fixes that. Comparing the ranges is exact, which is why
892 + /// there is no threshold to guess at here: whether a widening insert
893 + /// actually reorders anyone's top-k is a number to measure before loosening
894 + /// this, not before shipping it.
895 + #[instrument(skip_all)]
896 + pub fn refresh(db: &Database, index: &SimilarityIndex) -> Result<()> {
897 + let needs_rebuild = match Self::meta(db)? {
898 + None => true,
899 + Some(m) => m.stale || m.k < GRAPH_K || m.ranges != *index.ranges(),
900 + };
901 + if needs_rebuild {
902 + return Self::rebuild(db, index);
903 + }
904 +
905 + let dirty: Vec<String> = {
906 + let mut stmt = db
907 + .conn()
908 + .prepare("SELECT hash FROM neighbour_graph_dirty")?;
909 + let rows = stmt.query_map([], |r| r.get(0))?;
910 + rows.collect::<std::result::Result<_, _>>()?
911 + };
912 + if dirty.is_empty() {
913 + return Ok(());
914 + }
915 + if dirty.len().saturating_mul(REBUILD_WHEN_DIRTY_FRACTION) >= index.len() {
916 + return Self::rebuild(db, index);
917 + }
918 + for hash in &dirty {
919 + match Self::insert(db, index, hash) {
920 + Ok(()) => {}
921 + // A sample that lost its analysis row between the mark and here
922 + // is not a failure: drop it from the queue and carry on.
923 + Err(CoreError::SampleNotFound(_)) => {
924 + db.conn()
925 + .execute("DELETE FROM neighbour_graph_dirty WHERE hash = ?1", [hash])?;
926 + }
927 + Err(e) => return Err(e),
928 + }
929 + }
930 + Ok(())
931 + }
932 + }
933 +
534 934 #[cfg(test)]
535 935 mod tests {
536 936 use super::*;
@@ -805,6 +1205,261 @@
805 1205 assert!(results.is_empty());
806 1206 }
807 1207
1208 + // --- NeighbourGraph tests ---
1209 +
1210 + /// A five-sample fixture whose pairwise distances are all distinct, so a
1211 + /// ranking disagreement is a real disagreement and never a tie broken two
1212 + /// ways. Only bpm and duration vary; the other ten dimensions are constant
1213 + /// across `insert_with_features`, which is what makes the arithmetic here
1214 + /// checkable by hand.
1215 + fn fixture_library() -> Database {
1216 + let db = Database::open_in_memory().unwrap();
1217 + insert_with_features(&db, "a", 100.0, 1.0);
1218 + insert_with_features(&db, "b", 120.0, 2.0);
1219 + insert_with_features(&db, "c", 150.0, 4.0);
1220 + insert_with_features(&db, "d", 180.0, 7.0);
1221 + insert_with_features(&db, "e", 200.0, 11.0);
1222 + db
1223 + }
1224 +
1225 + /// Every stored edge, in a comparable order.
1226 + fn stored_edges(db: &Database) -> Vec<(String, String, f64, i64)> {
1227 + let mut stmt = db
1228 + .conn()
1229 + .prepare(
1230 + "SELECT hash, neighbour_hash, distance, rank FROM sample_neighbours
1231 + ORDER BY hash, rank",
1232 + )
1233 + .unwrap();
1234 + stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))
1235 + .unwrap()
1236 + .collect::<std::result::Result<_, _>>()
1237 + .unwrap()
1238 + }
1239 +
1240 + /// The graph's ranking must be the brute-force oracle's ranking, for every
1241 + /// sample. This is the whole correctness claim: the table is a cache of an
1242 + /// answer that already has a right value, so it is checked against the
1243 + /// definition rather than against the index that produced it.
1244 + #[test]
1245 + fn graph_agrees_with_the_brute_force_oracle() {
1246 + let db = fixture_library();
1247 + let index = SimilarityIndex::build(&db).unwrap();
1248 + NeighbourGraph::rebuild(&db, &index).unwrap();
1249 +
1250 + for hash in ["a", "b", "c", "d", "e"] {
1251 + let oracle = find_similar(&db, hash, GRAPH_K).unwrap();
1252 + let stored = NeighbourGraph::neighbours(&db, hash, GRAPH_K)
1253 + .unwrap()
1254 + .unwrap_or_else(|| panic!("graph declined to answer for {hash}"));
1255 + let oracle_hashes: Vec<&str> = oracle.iter().map(|r| r.hash.as_str()).collect();
1256 + let stored_hashes: Vec<&str> = stored.iter().map(|r| r.hash.as_str()).collect();
1257 + assert_eq!(stored_hashes, oracle_hashes, "ranking differs for {hash}");
1258 + for (s, o) in stored.iter().zip(oracle.iter()) {
1259 + assert!(
1260 + (s.distance - o.distance).abs() < 1e-12,
1261 + "distance differs for {hash} -> {}: {} vs {}",
1262 + s.hash,
1263 + s.distance,
1264 + o.distance
1265 + );
1266 + }
1267 + }
1268 + }
1269 +
1270 + /// Patching one sample in must land on exactly the graph a rebuild would
1271 + /// have produced. If it does not, the incremental path is a second
1272 + /// implementation of the same idea rather than a cheaper route to it.
1273 + #[test]
1274 + fn incremental_insert_matches_a_full_rebuild() {
1275 + let db = fixture_library();
Lines truncated
M crates/audiofiles-browser/src/backend/search_worker.rs +65 -11
@@ -3,13 +3,20 @@
3 3 //!
4 4 //! Building a VP-tree over the whole library is CPU-bound (the ultra-fuzz audit
5 5 //! flagged the lazy first-query build freezing the egui frame on large
6 - //! libraries). This worker owns both indexes and its own read-only `Database`
7 - //! connection (WAL lets it read alongside the main connection), so the build and
8 - //! query happen entirely off the render thread. Results return via [`SearchEvent`]
9 - //! and are applied in `poll_workers`.
6 + //! libraries). This worker owns both indexes and its own `Database` connection
7 + //! (WAL lets it work alongside the main connection), so the build and query
8 + //! happen entirely off the render thread. Results return via [`SearchEvent`] and
9 + //! are applied in `poll_workers`.
10 10 //!
11 11 //! The indexes are cached across queries and dropped on [`SearchCommand::Invalidate`]
12 12 //! (sent when new analysis lands), so they rebuild from fresh data on the next query.
13 + //!
14 + //! Similarity has a second, longer-lived cache underneath that: the persisted
15 + //! [`NeighbourGraph`](similarity::NeighbourGraph), which survives restarts and
16 + //! answers without a tree at all. This worker is also where the graph is
17 + //! maintained, because keeping it current needs a VP-tree and this is the only
18 + //! place one is built. Invalidate queues the work; the next similarity query
19 + //! answers the user first and then does it.
13 20
14 21 use std::path::PathBuf;
15 22
@@ -23,8 +30,14 @@
23 30 FindSimilar { hash: String, limit: usize },
24 31 /// Find near-duplicates of `hash` by fingerprint (builds the index if needed).
25 32 FindNearDuplicates { hash: String, limit: usize },
26 - /// Drop both cached indexes so the next query rebuilds from fresh data.
27 - Invalidate,
33 + /// Drop both cached indexes so the next query rebuilds from fresh data, and
34 + /// queue `changed` for a neighbour-graph update.
35 + ///
36 + /// An empty `changed` means "something changed, unspecified", and is handled
37 + /// the only safe way: the whole graph is marked stale and rebuilt on the next
38 + /// query. Naming the hashes is what buys the incremental path, so a caller
39 + /// that knows them should pass them.
40 + Invalidate { changed: Vec<String> },
28 41 }
29 42
30 43 /// Event sent from the search worker back to the GUI thread.
@@ -41,9 +54,12 @@
41 54 /// Cancel command, so the runtime handle is used directly.
42 55 pub type SearchWorkerHandle = WorkerHandle<SearchCommand, SearchEvent>;
43 56
44 - /// Persistent worker state: the read-only DB connection and the two cached
57 + /// Persistent worker state: the worker's own DB connection and the two cached
45 58 /// VP-tree indexes, reused across queries and dropped on `Invalidate`.
46 59 ///
60 + /// The connection writes as well as reads: maintaining the neighbour graph needs
61 + /// a VP-tree, and this is the only place one is built.
62 + ///
47 63 /// Eviction policy: each index is bounded by library size (one VP-tree node per
48 64 /// analyzed sample, order ~100 MB at a million samples), and **at most one is
49 65 /// resident at a time**. A similarity query evicts the fingerprint index and vice
@@ -58,8 +74,8 @@
58 74 fingerprint_index: Option<fingerprint::FingerprintIndex>,
59 75 }
60 76
61 - /// Spawn the background search worker. It opens its own read-only connection to
62 - /// the database at `db_path` on the worker thread.
77 + /// Spawn the background search worker. It opens its own connection to the
78 + /// database at `db_path` on the worker thread.
63 79 pub fn spawn_search_worker(db_path: PathBuf) -> std::io::Result<SearchWorkerHandle> {
64 80 spawn_worker(
65 81 "search-worker",
@@ -82,13 +98,42 @@
82 98
83 99 fn search_step(state: &mut SearchState, cmd: SearchCommand, ctx: &WorkerCtx<SearchEvent>) {
84 100 match cmd {
85 - SearchCommand::Invalidate => {
101 + SearchCommand::Invalidate { changed } => {
86 102 state.similarity_index = None;
87 103 state.fingerprint_index = None;
104 + // Queue the graph work, do not do it. An import sends one of these
105 + // per analysis batch, and building a tree per batch is the cost this
106 + // whole thing exists to stop paying. The next similarity query is
107 + // where a tree gets built anyway, so that is where the graph catches
108 + // up. Failing to queue costs freshness, never correctness: the read
109 + // path falls back to the tree whenever the graph cannot answer.
110 + let queued = if changed.is_empty() {
111 + similarity::NeighbourGraph::mark_stale(&state.db)
112 + } else {
113 + similarity::NeighbourGraph::mark_dirty(&state.db, &changed)
114 + };
115 + if let Err(e) = queued {
116 + tracing::warn!("neighbour graph not marked for update: {e}");
117 + }
88 118 }
89 119 SearchCommand::FindSimilar { hash, limit } => {
90 120 // A malformed feature row that panics is caught by the runtime and
91 121 // reported as Error; the worker survives for the next query.
122 + // The stored graph answers first: for a library that has not changed
123 + // this is a table read and no VP-tree is built at all, which is the
124 + // point of the graph. Everything below runs only on a miss.
125 + match similarity::NeighbourGraph::neighbours(&state.db, &hash, limit) {
126 + Ok(Some(rows)) => {
127 + ctx.emit(SearchEvent::SimilarResults {
128 + source: hash,
129 + hashes: rows.into_iter().map(|r| r.hash).collect(),
130 + });
131 + return;
132 + }
133 + Ok(None) => {}
134 + Err(e) => tracing::warn!("neighbour graph read failed, using the tree: {e}"),
135 + }
136 +
92 137 // Evict the other index: at most one VP-tree resident at a time.
93 138 state.fingerprint_index = None;
94 139 let result: Result<Vec<String>, audiofiles_core::error::CoreError> = (|| {
@@ -114,6 +159,15 @@
114 159 error: e.to_string(),
115 160 },
116 161 });
162 +
163 + // Answer first, then catch the graph up on the tree that is now
164 + // resident. Best-effort: the GUI connection may hold the write lock,
165 + // and a refresh that loses that race just happens on the next query.
166 + if let Some(index) = state.similarity_index.as_ref()
167 + && let Err(e) = similarity::NeighbourGraph::refresh(&state.db, index)
168 + {
169 + tracing::warn!("neighbour graph refresh failed: {e}");
170 + }
117 171 }
118 172 SearchCommand::FindNearDuplicates { hash, limit } => {
119 173 // Evict the other index: at most one VP-tree resident at a time.
@@ -188,7 +242,7 @@
188 242 }
189 243
190 244 // Invalidate + a second query still respond (worker survived).
191 - assert!(handle.send(SearchCommand::Invalidate));
245 + assert!(handle.send(SearchCommand::Invalidate { changed: vec![] }));
192 246 assert!(handle.send(SearchCommand::FindNearDuplicates {
193 247 hash: "deadbeef".to_string(),
194 248 limit: 10,
M crates/audiofiles-browser/src/state/library.rs +12 -4
@@ -317,11 +317,19 @@
317 317
318 318 // --- Similarity search ---
319 319
320 - /// Start a similarity search for the given hash. The VP-tree build + query
321 - /// run on the search worker; the results land in `poll_workers` via
322 - /// [`apply_similarity_results`](Self::apply_similarity_results).
320 + /// Start a similarity search for the given hash. The lookup runs on the
321 + /// search worker, from the stored neighbour graph where it can answer and
322 + /// from a VP-tree build where it cannot; the results land in `poll_workers`
323 + /// via [`apply_similarity_results`](Self::apply_similarity_results).
324 + ///
325 + /// The limit is `GRAPH_K` rather than a number chosen here: past it the
326 + /// graph has no stored edge to answer with, so asking for more would fall
327 + /// back to a tree build every time.
323 328 pub fn find_similar(&mut self, hash: &str) {
324 - match self.backend.start_find_similar(hash, 50) {
329 + match self
330 + .backend
331 + .start_find_similar(hash, audiofiles_core::similarity::GRAPH_K)
332 + {
325 333 Ok(()) => {
326 334 self.search.latest_similarity_request = Some((hash.to_string(), false));
327 335 self.status = "Searching for similar samples...".to_string();
M crates/audiofiles-browser/src/backend/direct/analysis.rs +7 -1
@@ -24,10 +24,16 @@
24 24 // Invalidate the search worker's cached VP-trees once for the batch, new
25 25 // analysis data changes normalization ranges and may add fingerprints, so
26 26 // both indexes must rebuild on the next query.
27 + //
28 + // The hashes go with it. They are what lets the neighbour graph update
29 + // these samples in place instead of rebuilding itself; without them the
30 + // worker can only mark the whole graph stale.
27 31 if let Some(w) = self.search_worker.lock().as_ref() {
28 32 // Fire-and-forget cache drop; if the worker is gone it rebuilds
29 33 // on next spawn, so a dropped Invalidate is harmless (no busy flag).
30 - let _ = w.send(SearchCommand::Invalidate);
34 + let _ = w.send(SearchCommand::Invalidate {
35 + changed: results.iter().map(|r| r.hash.clone()).collect(),
36 + });
31 37 }
32 38 Ok(())
33 39 }