Skip to main content

max / audiofiles

Extract loose-files subsystem from store into store/loose_files Convert store.rs to store/mod.rs and peel the loose-files subsystem into a new store/loose_files.rs: the integrity/relocate/purge free functions plus the impl SampleStore block for import_loose_files and import_loose_files_hashed. loose_files uses use super::* (inheriting the parent's named imports and reaching its private helpers as a descendant) plus an explicit tracing::instrument import for the macro. mod.rs re-exports via pub use loose_files::*, so every store::* path across the browser and app crates is unchanged, and the loose-files tests stay in mod.rs and resolve through the re-export. relocate_sample stays in mod.rs as a general source_path repoint. No behavior change; build and clippy clean, all 640 core tests pass, dependent crates build.
Co-Authored-By
Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-13 17:50 UTC
Signed with PGP, not checked
Commit: 5a46f4a04d2799e9e7409f4b098125fa76a0f15d
Parent: 230e3e8
2 files changed, +235 insertions, -223 deletions
R crates/audiofiles-core/src/store.rs → crates/audiofiles-core/src/store/mod.rs +3 -223
@@ -23,6 +23,9 @@
23 23 use crate::error::{io_err, unix_now, CoreError, Result};
24 24 use tracing::instrument;
25 25
26 + mod loose_files;
27 + pub use loose_files::*;
28 +
26 29 /// Probe an audio file to extract its duration from metadata/headers.
27 30 /// Returns `None` if the duration cannot be determined without full decode.
28 31 fn probe_duration(path: &Path) -> Option<f64> {
@@ -775,229 +778,6 @@
775 778 Ok(())
776 779 }
777 780
778 - /// Check integrity of loose-files mode samples.
779 - ///
780 - /// Returns `(valid, missing)` — counts of source_path entries where the file
781 - /// exists vs. does not exist on disk.
782 - pub fn check_loose_files_integrity(db: &Database) -> Result<(usize, usize)> {
783 - let mut stmt = db.conn().prepare(
784 - "SELECT source_path FROM live_samples WHERE source_path IS NOT NULL",
785 - )?;
786 - let paths: Vec<String> = stmt
787 - .query_map([], |row| row.get(0))?
788 - .collect::<std::result::Result<Vec<_>, _>>()?;
789 -
790 - let mut valid = 0;
791 - let mut missing = 0;
792 - for p in &paths {
793 - if Path::new(p).exists() {
794 - valid += 1;
795 - } else {
796 - missing += 1;
797 - }
798 - }
799 - Ok((valid, missing))
800 - }
801 -
802 - /// Try to re-locate loose-files mode samples whose source files have moved.
803 - ///
804 - /// Walks `search_root` recursively, building a basename map of candidate
805 - /// files. For each missing sample, looks up its basename, then hash-verifies
806 - /// candidates (cheapest: size-check before re-hashing the full file). On hash
807 - /// match, the sample's `source_path` is updated to the new location.
808 - ///
809 - /// Returns `(relocated, still_missing)`. `relocated` counts samples whose
810 - /// `source_path` was successfully repointed; `still_missing` is the residual
811 - /// count for the dialog to surface so the user can run Locate again against a
812 - /// different directory.
813 - pub fn relocate_missing_loose_files(
814 - db: &Database,
815 - search_root: &Path,
816 - ) -> Result<(usize, usize)> {
817 - // 1. Gather missing samples — hash, basename of stored source_path, and
818 - // the recorded file_size (used for cheap pre-filter before re-hashing).
819 - let mut stmt = db.conn().prepare(
820 - "SELECT hash, source_path, file_size FROM live_samples \
821 - WHERE source_path IS NOT NULL",
822 - )?;
823 - let rows: Vec<(String, String, i64)> = stmt
824 - .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
825 - .collect::<std::result::Result<Vec<_>, _>>()?;
826 - let missing: Vec<(String, String, i64)> = rows
827 - .into_iter()
828 - .filter(|(_, source_path, _)| !Path::new(source_path).exists())
829 - .collect();
830 -
831 - if missing.is_empty() {
832 - return Ok((0, 0));
833 - }
834 -
835 - // 2. Walk search_root, build basename -> Vec<PathBuf>. Lowercased so a
836 - // case-changed filesystem (e.g. moved between macOS/Linux) still
837 - // matches. Bounded by what the filesystem returns; large trees walk
838 - // once and stay in memory for the duration of this call.
839 - let mut candidates: std::collections::HashMap<String, Vec<PathBuf>> =
840 - std::collections::HashMap::new();
841 - let mut dirs = vec![search_root.to_path_buf()];
842 - while let Some(d) = dirs.pop() {
843 - let Ok(entries) = std::fs::read_dir(&d) else { continue };
844 - for entry in entries.flatten() {
845 - let path = entry.path();
846 - if path.is_dir() {
847 - if !crate::util::is_macos_metadata_dir(&path) {
848 - dirs.push(path);
849 - }
850 - } else if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
851 - candidates
852 - .entry(name.to_lowercase())
853 - .or_default()
854 - .push(path);
855 - }
856 - }
857 - }
858 -
859 - // 3. For each missing sample, check candidates with matching basename.
860 - // Size check filters out same-name-different-file collisions before we
861 - // spend cycles hashing. Hash verify is the authoritative match.
862 - let mut relocated_pairs: Vec<(String, String)> = Vec::new();
863 - for (hash, source_path, file_size) in &missing {
864 - let Some(basename) = Path::new(source_path)
865 - .file_name()
866 - .and_then(|n| n.to_str())
867 - else { continue };
868 - let key = basename.to_lowercase();
869 - let Some(paths) = candidates.get(&key) else { continue };
870 - for cand in paths {
871 - let Ok(md) = std::fs::metadata(cand) else { continue };
872 - if md.len() as i64 != *file_size {
873 - continue;
874 - }
875 - // Hash verify. Bail on first match; sample hashes are unique so a
876 - // second match for the same hash would be redundant.
877 - let Ok(mut file) = fs::File::open(cand) else { continue };
878 - let mut hasher = Sha256::new();
879 - let mut buf = [0u8; 8192];
880 - let ok = loop {
881 - let Ok(n) = file.read(&mut buf) else { break false };
882 - if n == 0 {
883 - break true;
884 - }
885 - hasher.update(&buf[..n]);
886 - };
887 - if !ok {
888 - continue;
889 - }
890 - let computed = format!("{:x}", hasher.finalize());
891 - if computed == *hash {
892 - let abs = cand
893 - .canonicalize()
894 - .map(|p| p.to_string_lossy().to_string())
895 - .unwrap_or_else(|_| cand.to_string_lossy().to_string());
896 - relocated_pairs.push((hash.clone(), abs));
897 - break;
898 - }
899 - }
900 - }
901 -
902 - // 4. Atomic update of all relocated source_paths.
903 - let relocated = relocated_pairs.len();
904 - if relocated > 0 {
905 - db.transaction(|_tx| {
906 - for (hash, new_path) in &relocated_pairs {
907 - db.conn().execute(
908 - "UPDATE samples SET source_path = ?1 WHERE hash = ?2",
909 - rusqlite::params![new_path, hash],
910 - )?;
911 - }
912 - Ok(())
913 - })?;
914 - }
915 -
916 - let still_missing = missing.len() - relocated;
917 - Ok((relocated, still_missing))
918 - }
919 -
920 - /// Delete all loose-files mode samples whose source files no longer exist on disk.
921 - ///
922 - /// Returns the number of samples purged. CASCADE handles VFS nodes, tags, etc.
923 - pub fn purge_missing_loose_files(db: &Database) -> Result<usize> {
924 - let mut stmt = db.conn().prepare(
925 - "SELECT hash, source_path FROM live_samples WHERE source_path IS NOT NULL",
926 - )?;
927 - let rows: Vec<(String, String)> = stmt
928 - .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
929 - .collect::<std::result::Result<Vec<_>, _>>()?;
930 -
931 - // Collect hashes to purge, then delete atomically in one transaction.
932 - let to_purge: Vec<&str> = rows
933 - .iter()
934 - .filter(|(_, source_path)| !Path::new(source_path).exists())
935 - .map(|(hash, _)| hash.as_str())
936 - .collect();
937 - let purged = to_purge.len();
938 -
939 - if purged > 0 {
940 - db.transaction(|_tx| {
941 - for hash in &to_purge {
942 - db.conn()
943 - .execute("DELETE FROM samples WHERE hash = ?1", [hash])?;
944 - }
945 - Ok(())
946 - })?;
947 - }
948 -
949 - Ok(purged)
950 - }
951 -
952 - impl SampleStore {
953 - /// Import a file in loose-files mode: hash it but do NOT copy to the store.
954 - ///
955 - /// Records the original absolute path as `source_path` in the database.
956 - /// The file stays where it is on disk.
957 - #[instrument(skip_all)]
958 - pub fn import_loose_files(&self, path: &Path, db: &Database) -> Result<String> {
959 - let (hash, file_size) = hash_file(path)?;
960 - self.import_loose_files_hashed(path, &hash, file_size, db)?;
961 - Ok(hash)
962 - }
963 -
964 - /// Loose-files import with a pre-computed hash (see [`import_hashed`]). Records
965 - /// the source path; no blob copy.
966 - ///
967 - /// [`import_hashed`]: SampleStore::import_hashed
968 - #[instrument(skip_all)]
969 - pub fn import_loose_files_hashed(
970 - &self,
971 - path: &Path,
972 - hash: &str,
973 - file_size: i64,
974 - db: &Database,
975 - ) -> Result<()> {
976 - let ext = crate::util::get_extension(path);
977 - let original_name = clamp_original_name(crate::util::get_filename(path, "unknown"));
978 -
979 - // Probe duration from file headers (cheap, no full decode)
980 - let duration = probe_duration(path);
981 -
982 - // Resolve absolute path for storage
983 - let abs_path = path
984 - .canonicalize()
985 - .map_err(|e| io_err(path, e))?
986 - .to_string_lossy()
987 - .to_string();
988 -
989 - // Insert into DB with source_path (ignore if hash already exists)
990 - let now = unix_now();
991 - db.conn().execute(
992 - "INSERT OR IGNORE INTO samples (hash, original_name, file_extension, file_size, import_date, last_modified, duration, source_path)
993 - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
994 - rusqlite::params![hash, original_name, ext, file_size, now, now, duration, abs_path],
995 - )?;
996 -
997 - Ok(())
998 - }
999 - }
1000 -
1001 781 /// Hash a file's bytes with SHA-256, returning the hex digest and its size.
1002 782 ///
1003 783 /// Pure (no store/DB side effects), so a batch can be hashed in parallel before
A crates/audiofiles-core/src/store/loose_files.rs +232
@@ -1,0 +1,232 @@
1 + //! Loose-files mode: import samples by reference (recording their on-disk
2 + //! `source_path`) instead of copying blobs into the content-addressed store,
3 + //! plus the integrity / relocate / purge operations over those references.
4 + //!
5 + //! Split out of the parent `store` module; every path stays `store::*` via the
6 + //! parent's `pub use loose_files::*`.
7 +
8 + use super::*;
9 + use tracing::instrument;
10 +
11 + /// Check integrity of loose-files mode samples.
12 + ///
13 + /// Returns `(valid, missing)` — counts of source_path entries where the file
14 + /// exists vs. does not exist on disk.
15 + pub fn check_loose_files_integrity(db: &Database) -> Result<(usize, usize)> {
16 + let mut stmt = db.conn().prepare(
17 + "SELECT source_path FROM live_samples WHERE source_path IS NOT NULL",
18 + )?;
19 + let paths: Vec<String> = stmt
20 + .query_map([], |row| row.get(0))?
21 + .collect::<std::result::Result<Vec<_>, _>>()?;
22 +
23 + let mut valid = 0;
24 + let mut missing = 0;
25 + for p in &paths {
26 + if Path::new(p).exists() {
27 + valid += 1;
28 + } else {
29 + missing += 1;
30 + }
31 + }
32 + Ok((valid, missing))
33 + }
34 +
35 + /// Try to re-locate loose-files mode samples whose source files have moved.
36 + ///
37 + /// Walks `search_root` recursively, building a basename map of candidate
38 + /// files. For each missing sample, looks up its basename, then hash-verifies
39 + /// candidates (cheapest: size-check before re-hashing the full file). On hash
40 + /// match, the sample's `source_path` is updated to the new location.
41 + ///
42 + /// Returns `(relocated, still_missing)`. `relocated` counts samples whose
43 + /// `source_path` was successfully repointed; `still_missing` is the residual
44 + /// count for the dialog to surface so the user can run Locate again against a
45 + /// different directory.
46 + pub fn relocate_missing_loose_files(
47 + db: &Database,
48 + search_root: &Path,
49 + ) -> Result<(usize, usize)> {
50 + // 1. Gather missing samples — hash, basename of stored source_path, and
51 + // the recorded file_size (used for cheap pre-filter before re-hashing).
52 + let mut stmt = db.conn().prepare(
53 + "SELECT hash, source_path, file_size FROM live_samples \
54 + WHERE source_path IS NOT NULL",
55 + )?;
56 + let rows: Vec<(String, String, i64)> = stmt
57 + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
58 + .collect::<std::result::Result<Vec<_>, _>>()?;
59 + let missing: Vec<(String, String, i64)> = rows
60 + .into_iter()
61 + .filter(|(_, source_path, _)| !Path::new(source_path).exists())
62 + .collect();
63 +
64 + if missing.is_empty() {
65 + return Ok((0, 0));
66 + }
67 +
68 + // 2. Walk search_root, build basename -> Vec<PathBuf>. Lowercased so a
69 + // case-changed filesystem (e.g. moved between macOS/Linux) still
70 + // matches. Bounded by what the filesystem returns; large trees walk
71 + // once and stay in memory for the duration of this call.
72 + let mut candidates: std::collections::HashMap<String, Vec<PathBuf>> =
73 + std::collections::HashMap::new();
74 + let mut dirs = vec![search_root.to_path_buf()];
75 + while let Some(d) = dirs.pop() {
76 + let Ok(entries) = std::fs::read_dir(&d) else { continue };
77 + for entry in entries.flatten() {
78 + let path = entry.path();
79 + if path.is_dir() {
80 + if !crate::util::is_macos_metadata_dir(&path) {
81 + dirs.push(path);
82 + }
83 + } else if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
84 + candidates
85 + .entry(name.to_lowercase())
86 + .or_default()
87 + .push(path);
88 + }
89 + }
90 + }
91 +
92 + // 3. For each missing sample, check candidates with matching basename.
93 + // Size check filters out same-name-different-file collisions before we
94 + // spend cycles hashing. Hash verify is the authoritative match.
95 + let mut relocated_pairs: Vec<(String, String)> = Vec::new();
96 + for (hash, source_path, file_size) in &missing {
97 + let Some(basename) = Path::new(source_path)
98 + .file_name()
99 + .and_then(|n| n.to_str())
100 + else { continue };
101 + let key = basename.to_lowercase();
102 + let Some(paths) = candidates.get(&key) else { continue };
103 + for cand in paths {
104 + let Ok(md) = std::fs::metadata(cand) else { continue };
105 + if md.len() as i64 != *file_size {
106 + continue;
107 + }
108 + // Hash verify. Bail on first match; sample hashes are unique so a
109 + // second match for the same hash would be redundant.
110 + let Ok(mut file) = fs::File::open(cand) else { continue };
111 + let mut hasher = Sha256::new();
112 + let mut buf = [0u8; 8192];
113 + let ok = loop {
114 + let Ok(n) = file.read(&mut buf) else { break false };
115 + if n == 0 {
116 + break true;
117 + }
118 + hasher.update(&buf[..n]);
119 + };
120 + if !ok {
121 + continue;
122 + }
123 + let computed = format!("{:x}", hasher.finalize());
124 + if computed == *hash {
125 + let abs = cand
126 + .canonicalize()
127 + .map(|p| p.to_string_lossy().to_string())
128 + .unwrap_or_else(|_| cand.to_string_lossy().to_string());
129 + relocated_pairs.push((hash.clone(), abs));
130 + break;
131 + }
132 + }
133 + }
134 +
135 + // 4. Atomic update of all relocated source_paths.
136 + let relocated = relocated_pairs.len();
137 + if relocated > 0 {
138 + db.transaction(|_tx| {
139 + for (hash, new_path) in &relocated_pairs {
140 + db.conn().execute(
141 + "UPDATE samples SET source_path = ?1 WHERE hash = ?2",
142 + rusqlite::params![new_path, hash],
143 + )?;
144 + }
145 + Ok(())
146 + })?;
147 + }
148 +
149 + let still_missing = missing.len() - relocated;
150 + Ok((relocated, still_missing))
151 + }
152 +
153 + /// Delete all loose-files mode samples whose source files no longer exist on disk.
154 + ///
155 + /// Returns the number of samples purged. CASCADE handles VFS nodes, tags, etc.
156 + pub fn purge_missing_loose_files(db: &Database) -> Result<usize> {
157 + let mut stmt = db.conn().prepare(
158 + "SELECT hash, source_path FROM live_samples WHERE source_path IS NOT NULL",
159 + )?;
160 + let rows: Vec<(String, String)> = stmt
161 + .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
162 + .collect::<std::result::Result<Vec<_>, _>>()?;
163 +
164 + // Collect hashes to purge, then delete atomically in one transaction.
165 + let to_purge: Vec<&str> = rows
166 + .iter()
167 + .filter(|(_, source_path)| !Path::new(source_path).exists())
168 + .map(|(hash, _)| hash.as_str())
169 + .collect();
170 + let purged = to_purge.len();
171 +
172 + if purged > 0 {
173 + db.transaction(|_tx| {
174 + for hash in &to_purge {
175 + db.conn()
176 + .execute("DELETE FROM samples WHERE hash = ?1", [hash])?;
177 + }
178 + Ok(())
179 + })?;
180 + }
181 +
182 + Ok(purged)
183 + }
184 +
185 + impl SampleStore {
186 + /// Import a file in loose-files mode: hash it but do NOT copy to the store.
187 + ///
188 + /// Records the original absolute path as `source_path` in the database.
189 + /// The file stays where it is on disk.
190 + #[instrument(skip_all)]
191 + pub fn import_loose_files(&self, path: &Path, db: &Database) -> Result<String> {
192 + let (hash, file_size) = hash_file(path)?;
193 + self.import_loose_files_hashed(path, &hash, file_size, db)?;
194 + Ok(hash)
195 + }
196 +
197 + /// Loose-files import with a pre-computed hash (see [`import_hashed`]). Records
198 + /// the source path; no blob copy.
199 + ///
200 + /// [`import_hashed`]: SampleStore::import_hashed
201 + #[instrument(skip_all)]
202 + pub fn import_loose_files_hashed(
203 + &self,
204 + path: &Path,
205 + hash: &str,
206 + file_size: i64,
207 + db: &Database,
208 + ) -> Result<()> {
209 + let ext = crate::util::get_extension(path);
210 + let original_name = clamp_original_name(crate::util::get_filename(path, "unknown"));
211 +
212 + // Probe duration from file headers (cheap, no full decode)
213 + let duration = probe_duration(path);
214 +
215 + // Resolve absolute path for storage
216 + let abs_path = path
217 + .canonicalize()
218 + .map_err(|e| io_err(path, e))?
219 + .to_string_lossy()
220 + .to_string();
221 +
222 + // Insert into DB with source_path (ignore if hash already exists)
223 + let now = unix_now();
224 + db.conn().execute(
225 + "INSERT OR IGNORE INTO samples (hash, original_name, file_extension, file_size, import_date, last_modified, duration, source_path)
226 + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
227 + rusqlite::params![hash, original_name, ext, file_size, now, now, duration, abs_path],
228 + )?;
229 +
230 + Ok(())
231 + }
232 + }