Skip to main content

max / audiofiles

Parallel-hash files during import (scale cliff: serial SHA-256) SHA-256 over the whole file is the dominant per-file import cost and was fully serial. The hash is pure (no store/DB side effects), so it's now computed in parallel: store gains hash_file() + hash_files_parallel() (rayon), and import() is split into hash_file() + import_hashed() (and import_loose_files into import_loose_files_hashed). The import worker pre-hashes each directory level's audio files in parallel (prehash_level), then records them serially — every blob copy and DB write stays serial, so the content-addressed store invariants (atomic temp+rename, orphan-cleanup-on-insert-failure, dedup) are untouched. A file the pre-hash pass somehow missed falls back to hashing inline. Tests: core 541 green (hash_file matches import + rejects zero-byte/non-audio; import_hashed lands the same blob+row as import; hash_files_parallel aligns results), browser 225 green (new end-to-end flat-import test through the prehash path skips non-audio and imports all). clippy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-16 00:18 UTC
Commit: dcf342edd294a62482e534f6334ced500ca173a9
Parent: 80e778f
2 files changed, +246 insertions, -72 deletions
M crates/audiofiles-browser/src/import.rs +112 -10
@@ -4,6 +4,7 @@
4 4 //! dedicated thread with its own DB connection, communicating via channels. Between files
5 5 //! it checks for cancellation, keeping the UI responsive during large imports.
6 6
7 + use std::collections::HashMap;
7 8 use std::fs;
8 9 use std::path::{Path, PathBuf};
9 10 use std::sync::{mpsc, Mutex};
@@ -207,7 +208,34 @@ fn count_audio_files(
207 208 Some((count, total_bytes))
208 209 }
209 210
210 - /// Import a single file into store + VFS, returning the result.
211 + /// A file's SHA-256 hash and size, pre-computed by the parallel hash pass, or
212 + /// the error message from that hashing attempt.
213 + type PreHash = Result<(String, i64), String>;
214 +
215 + /// Hash a directory level's audio files in parallel (rayon), keyed by path. The
216 + /// serial record step then looks each up, so the dominant per-file cost (SHA-256
217 + /// over the whole file) overlaps across cores while every store/DB mutation stays
218 + /// serial.
219 + fn prehash_level(paths: &[PathBuf]) -> HashMap<PathBuf, PreHash> {
220 + let files: Vec<PathBuf> = paths
221 + .iter()
222 + .filter(|p| p.is_file() && is_audio_file(p))
223 + .cloned()
224 + .collect();
225 + files
226 + .iter()
227 + .cloned()
228 + .zip(
229 + audiofiles_core::store::hash_files_parallel(&files)
230 + .into_iter()
231 + .map(|r| r.map_err(|e| e.to_string())),
232 + )
233 + .collect()
234 + }
235 +
236 + /// Import a single file into store + VFS, returning the result. `prehashed` is
237 + /// the file's pre-computed `(hash, size)` from [`prehash_level`]; `None` falls
238 + /// back to hashing inline (so a file the pre-hash pass missed still imports).
211 239 fn import_single_file(
212 240 path: &Path,
213 241 vfs_id: VfsId,
@@ -215,12 +243,18 @@ fn import_single_file(
215 243 store: &SampleStore,
216 244 db: &Database,
217 245 loose_files: bool,
246 + prehashed: Option<PreHash>,
218 247 ) -> Result<ImportFileResult, CoreError> {
219 - let hash = if loose_files {
220 - store.import_loose_files(path, db)?
221 - } else {
222 - store.import(path, db)?
248 + let (hash, file_size) = match prehashed {
249 + Some(Ok((h, sz))) => (h, sz),
250 + Some(Err(msg)) => return Err(CoreError::Internal(msg)),
251 + None => audiofiles_core::store::hash_file(path)?,
223 252 };
253 + if loose_files {
254 + store.import_loose_files_hashed(path, &hash, file_size, db)?;
255 + } else {
256 + store.import_hashed(path, &hash, file_size, db)?;
257 + }
224 258 let name = audiofiles_core::util::get_filename(path, "unknown");
225 259 let ext = audiofiles_core::util::get_extension(path);
226 260
@@ -274,7 +308,14 @@ impl ImportContext<'_> {
274 308 }
275 309
276 310 /// Import a single audio file, updating counters and sending error events.
277 - fn process_file(&mut self, path: &Path, vfs_id: VfsId, parent_id: Option<NodeId>) {
311 + /// `prehashed` is the file's pre-computed hash from [`prehash_level`].
312 + fn process_file(
313 + &mut self,
314 + path: &Path,
315 + vfs_id: VfsId,
316 + parent_id: Option<NodeId>,
317 + prehashed: Option<PreHash>,
318 + ) {
278 319 let name = audiofiles_core::util::get_filename(path, "unknown");
279 320 self.send_progress(name);
280 321
@@ -284,7 +325,15 @@ impl ImportContext<'_> {
284 325 // that exits Importing. A panicked file is counted as an error and the
285 326 // import continues to completion.
286 327 let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
287 - import_single_file(path, vfs_id, parent_id, self.store, self.db, self.loose_files)
328 + import_single_file(
329 + path,
330 + vfs_id,
331 + parent_id,
332 + self.store,
333 + self.db,
334 + self.loose_files,
335 + prehashed,
336 + )
288 337 }));
289 338 match outcome {
290 339 Ok(Ok(ImportFileResult::Imported(hash, ext))) => {
@@ -332,6 +381,9 @@ fn import_directory_recursive(
332 381 let mut paths: Vec<PathBuf> = entries.flatten().map(|e| e.path()).collect();
333 382 paths.sort();
334 383
384 + // Hash this level's audio files in parallel before the serial record loop.
385 + let mut hashes = prehash_level(&paths);
386 +
335 387 for path in paths {
336 388 if ctx.is_cancelled() {
337 389 return true;
@@ -371,7 +423,7 @@ fn import_directory_recursive(
371 423 return true;
372 424 }
373 425 } else if path.is_file() && is_audio_file(&path) {
374 - ctx.process_file(&path, vfs_id, parent_id);
426 + ctx.process_file(&path, vfs_id, parent_id, hashes.remove(&path));
375 427 }
376 428 }
377 429
@@ -397,6 +449,9 @@ fn import_directory_flat(
397 449 let mut paths: Vec<PathBuf> = entries.flatten().map(|e| e.path()).collect();
398 450 paths.sort();
399 451
452 + // Hash this level's audio files in parallel before the serial record loop.
453 + let mut hashes = prehash_level(&paths);
454 +
400 455 for path in paths {
401 456 if ctx.is_cancelled() {
402 457 return true;
@@ -412,7 +467,7 @@ fn import_directory_flat(
412 467 return true;
413 468 }
414 469 } else if path.is_file() && is_audio_file(&path) {
415 - ctx.process_file(&path, vfs_id, parent_id);
470 + ctx.process_file(&path, vfs_id, parent_id, hashes.remove(&path));
416 471 }
417 472 }
418 473
@@ -440,6 +495,10 @@ fn import_structured(
440 495 let mut paths: Vec<PathBuf> = entries.flatten().map(|e| e.path()).collect();
441 496 paths.sort();
442 497
498 + // Pre-hash root-level files in parallel; subdirectory files are pre-hashed
499 + // per-level inside import_directory_recursive.
500 + let mut hashes = prehash_level(&paths);
501 +
443 502 let mut folders = Vec::new();
444 503
445 504 for path in paths {
@@ -497,7 +556,7 @@ fn import_structured(
497 556 }
498 557 } else if path.is_file() && is_audio_file(&path) {
499 558 // Root-level files — import directly, no folder grouping
500 - ctx.process_file(&path, vfs_id, parent_id);
559 + ctx.process_file(&path, vfs_id, parent_id, hashes.remove(&path));
501 560 }
502 561 }
503 562
@@ -721,6 +780,49 @@ mod tests {
721 780 }
722 781
723 782 #[test]
783 + fn flat_import_via_prehash_imports_all_audio_files() {
784 + let tmp = tempfile::TempDir::new().unwrap();
785 + let src = tmp.path().join("src");
786 + std::fs::create_dir(&src).unwrap();
787 + for name in ["a.wav", "b.wav", "c.wav"] {
788 + std::fs::write(src.join(name), format!("audio-{name}")).unwrap();
789 + }
790 + // A non-audio file must be skipped, not counted/imported.
791 + std::fs::write(src.join("readme.txt"), "nope").unwrap();
792 +
793 + let db = Database::open_in_memory().unwrap();
794 + let store = SampleStore::new(tmp.path().join("store")).unwrap();
795 + let vfs_id = vfs::create_vfs(&db, "T").unwrap();
796 +
797 + let (event_tx, _event_rx) = mpsc::channel();
798 + let (_cmd_tx, cmd_rx) = mpsc::channel();
799 + let (mut completed, mut errors, mut dups, mut imported) = (0, 0, 0, Vec::new());
800 + let mut ctx = ImportContext {
801 + store: &store,
802 + db: &db,
803 + event_tx: &event_tx,
804 + cmd_rx: &cmd_rx,
805 + completed: &mut completed,
806 + total: 3,
807 + errors: &mut errors,
808 + duplicates: &mut dups,
809 + imported: &mut imported,
810 + loose_files: false,
811 + };
812 +
813 + let cancelled = import_directory_flat(&src, vfs_id, None, &mut ctx);
814 + assert!(!cancelled);
815 + assert_eq!(completed, 3, "all three audio files imported");
816 + assert_eq!(errors, 0);
817 +
818 + let count: i64 = db
819 + .conn()
820 + .query_row("SELECT COUNT(*) FROM samples", [], |r| r.get(0))
821 + .unwrap();
822 + assert_eq!(count, 3, "three content-addressed rows, txt skipped");
823 + }
824 +
825 + #[test]
724 826 fn spawn_and_drop_does_not_hang() {
725 827 let dir = tempfile::TempDir::new().unwrap();
726 828 let db_path = dir.path().join("test.db");
M crates/audiofiles-core/src/store.rs +134 -62
@@ -110,36 +110,24 @@ impl SampleStore {
110 110 /// insert into DB. Returns the hex SHA-256 hash.
111 111 #[instrument(skip_all)]
112 112 pub fn import(&self, path: &Path, db: &Database) -> Result<String> {
113 - if !crate::util::is_audio_file(path) {
114 - return Err(CoreError::Internal(format!(
115 - "not a supported audio file: {}",
116 - path.display()
117 - )));
118 - }
119 -
120 - let mut file = fs::File::open(path).map_err(|e| io_err(path, e))?;
121 - let metadata = file.metadata().map_err(|e| io_err(path, e))?;
122 - let file_size = metadata.len() as i64;
123 -
124 - if file_size == 0 {
125 - return Err(CoreError::Internal(format!(
126 - "cannot import zero-byte file: {}",
127 - path.display()
128 - )));
129 - }
130 -
131 - // Stream through SHA-256
132 - let mut hasher = Sha256::new();
133 - let mut buf = [0u8; 8192];
134 - loop {
135 - let n = file.read(&mut buf).map_err(|e| io_err(path, e))?;
136 - if n == 0 {
137 - break;
138 - }
139 - hasher.update(&buf[..n]);
140 - }
141 - let hash = format!("{:x}", hasher.finalize());
113 + let (hash, file_size) = hash_file(path)?;
114 + self.import_hashed(path, &hash, file_size, db)?;
115 + Ok(hash)
116 + }
142 117
118 + /// Import a file whose SHA-256 hash and size were already computed (e.g. by a
119 + /// parallel pre-hash pass). Does the serial side-effecting work — content-
120 + /// addressed blob copy + DB insert + orphan cleanup — exactly as [`import`].
121 + /// Splitting the pure hash out lets the import pipeline hash a batch in
122 + /// parallel while keeping every store/DB mutation serial.
123 + #[instrument(skip_all)]
124 + pub fn import_hashed(
125 + &self,
126 + path: &Path,
127 + hash: &str,
128 + file_size: i64,
129 + db: &Database,
130 + ) -> Result<()> {
143 131 // Resolve the blob extension from an existing row if this hash is already
144 132 // known. The store is content-addressed (one blob per hash), so identical
145 133 // bytes imported under a different extension must reuse the existing blob
@@ -150,7 +138,7 @@ impl SampleStore {
150 138 // falls back to the incoming file's extension.
151 139 let ext = match db.conn().query_row(
152 140 "SELECT file_extension FROM samples WHERE hash = ?1",
153 - [&hash],
141 + [hash],
154 142 |row| row.get::<_, String>(0),
155 143 ) {
156 144 Ok(existing) => existing,
@@ -173,7 +161,7 @@ impl SampleStore {
173 161 // Mirrors the atomic temp+rename the sync download path uses. The pid
174 162 // suffix keeps two concurrent imports of the same hash from colliding on
175 163 // the temp file.
176 - let dest = self.sample_path(&hash, &ext)?;
164 + let dest = self.sample_path(hash, &ext)?;
177 165 let needs_write = match fs::metadata(&dest) {
178 166 Ok(m) => m.len() != file_size as u64,
179 167 Err(_) => true,
@@ -207,7 +195,7 @@ impl SampleStore {
207 195 return Err(CoreError::Db(e));
208 196 }
209 197
210 - Ok(hash)
198 + Ok(())
211 199 }
212 200
213 201 /// Check if a sample file exists in the store.
@@ -645,36 +633,23 @@ impl SampleStore {
645 633 /// The file stays where it is on disk.
646 634 #[instrument(skip_all)]
647 635 pub fn import_loose_files(&self, path: &Path, db: &Database) -> Result<String> {
648 - if !crate::util::is_audio_file(path) {
649 - return Err(CoreError::Internal(format!(
650 - "not a supported audio file: {}",
651 - path.display()
652 - )));
653 - }
654 -
655 - let mut file = fs::File::open(path).map_err(|e| io_err(path, e))?;
656 - let metadata = file.metadata().map_err(|e| io_err(path, e))?;
657 - let file_size = metadata.len() as i64;
658 -
659 - if file_size == 0 {
660 - return Err(CoreError::Internal(format!(
661 - "cannot import zero-byte file: {}",
662 - path.display()
663 - )));
664 - }
665 -
666 - // Stream through SHA-256
667 - let mut hasher = Sha256::new();
668 - let mut buf = [0u8; 8192];
669 - loop {
670 - let n = file.read(&mut buf).map_err(|e| io_err(path, e))?;
671 - if n == 0 {
672 - break;
673 - }
674 - hasher.update(&buf[..n]);
675 - }
676 - let hash = format!("{:x}", hasher.finalize());
636 + let (hash, file_size) = hash_file(path)?;
637 + self.import_loose_files_hashed(path, &hash, file_size, db)?;
638 + Ok(hash)
639 + }
677 640
641 + /// Loose-files import with a pre-computed hash (see [`import_hashed`]). Records
642 + /// the source path; no blob copy.
643 + ///
644 + /// [`import_hashed`]: SampleStore::import_hashed
645 + #[instrument(skip_all)]
646 + pub fn import_loose_files_hashed(
647 + &self,
648 + path: &Path,
649 + hash: &str,
650 + file_size: i64,
651 + db: &Database,
652 + ) -> Result<()> {
678 653 let ext = crate::util::get_extension(path);
679 654 let original_name = clamp_original_name(crate::util::get_filename(path, "unknown"));
680 655
@@ -696,8 +671,54 @@ impl SampleStore {
696 671 rusqlite::params![hash, original_name, ext, file_size, now, now, duration, abs_path],
697 672 )?;
698 673
699 - Ok(hash)
674 + Ok(())
675 + }
676 + }
677 +
678 + /// Hash a file's bytes with SHA-256, returning the hex digest and its size.
679 + ///
680 + /// Pure (no store/DB side effects), so a batch can be hashed in parallel before
681 + /// the serial copy+record step. Applies the same guards [`SampleStore::import`]
682 + /// did inline: rejects non-audio and zero-byte files.
683 + pub fn hash_file(path: &Path) -> Result<(String, i64)> {
684 + if !crate::util::is_audio_file(path) {
685 + return Err(CoreError::Internal(format!(
686 + "not a supported audio file: {}",
687 + path.display()
688 + )));
700 689 }
690 +
691 + let mut file = fs::File::open(path).map_err(|e| io_err(path, e))?;
692 + let metadata = file.metadata().map_err(|e| io_err(path, e))?;
693 + let file_size = metadata.len() as i64;
694 +
695 + if file_size == 0 {
696 + return Err(CoreError::Internal(format!(
697 + "cannot import zero-byte file: {}",
698 + path.display()
699 + )));
700 + }
701 +
702 + let mut hasher = Sha256::new();
703 + let mut buf = [0u8; 8192];
704 + loop {
705 + let n = file.read(&mut buf).map_err(|e| io_err(path, e))?;
706 + if n == 0 {
707 + break;
708 + }
709 + hasher.update(&buf[..n]);
710 + }
711 + Ok((format!("{:x}", hasher.finalize()), file_size))
712 + }
713 +
714 + /// Hash a batch of files in parallel (rayon). Each result is aligned to the
715 + /// corresponding entry in `paths`. Hashing is the dominant per-file import cost
716 + /// (SHA-256 over the whole file) and is embarrassingly parallel — every side
717 + /// effect (blob copy, DB insert) stays serial in the caller, so the content-
718 + /// addressed store invariants are untouched.
719 + pub fn hash_files_parallel(paths: &[PathBuf]) -> Vec<Result<(String, i64)>> {
720 + use rayon::prelude::*;
721 + paths.par_iter().map(|p| hash_file(p)).collect()
701 722 }
702 723
703 724 #[cfg(test)]
@@ -738,6 +759,57 @@ mod tests {
738 759 }
739 760
740 761 #[test]
762 + fn hash_file_matches_import_and_rejects_bad_input() {
763 + let (dir, db, store) = setup();
764 + let src = create_test_file(&dir, "kick.wav", b"fake audio data");
765 +
766 + // hash_file's digest equals the hash import() records.
767 + let (hash, size) = hash_file(&src).unwrap();
768 + assert_eq!(size, "fake audio data".len() as i64);
769 + let imported = store.import(&src, &db).unwrap();
770 + assert_eq!(hash, imported);
771 +
772 + // Zero-byte and non-audio files are rejected (same guards as import).
773 + let empty = create_test_file(&dir, "empty.wav", b"");
774 + assert!(hash_file(&empty).is_err());
775 + let txt = create_test_file(&dir, "notes.txt", b"hello");
776 + assert!(hash_file(&txt).is_err());
777 + }
778 +
779 + #[test]
780 + fn import_hashed_matches_serial_import() {
781 + let (dir, db, store) = setup();
782 + let src = create_test_file(&dir, "snare.wav", b"some audio bytes");
783 +
784 + // Pre-hash then record — the prehashed path must land the same blob + row
785 + // as the all-in-one import().
786 + let (hash, size) = hash_file(&src).unwrap();
787 + store.import_hashed(&src, &hash, size, &db).unwrap();
788 +
789 + assert!(store.exists(&hash, "wav").unwrap());
790 + let count: i64 = db
791 + .conn()
792 + .query_row("SELECT COUNT(*) FROM samples WHERE hash = ?1", [&hash], |r| r.get(0))
793 + .unwrap();
794 + assert_eq!(count, 1);
795 + }
796 +
797 + #[test]
798 + fn hash_files_parallel_aligns_results() {
799 + let (dir, _db, _store) = setup();
800 + let a = create_test_file(&dir, "a.wav", b"aaaa");
801 + let b = create_test_file(&dir, "b.wav", b"bbbbbb");
802 + let bad = create_test_file(&dir, "z.txt", b"nope"); // non-audio -> Err
803 +
804 + let results = hash_files_parallel(&[a.clone(), b.clone(), bad.clone()]);
805 + assert_eq!(results.len(), 3);
806 + // Aligned to input order; each Ok hash equals a direct hash_file call.
807 + assert_eq!(results[0].as_ref().unwrap().0, hash_file(&a).unwrap().0);
808 + assert_eq!(results[1].as_ref().unwrap().1, 6);
809 + assert!(results[2].is_err());
810 + }
811 +
812 + #[test]
741 813 fn import_creates_file_and_row() {
742 814 let (dir, db, store) = setup();
743 815 let src = create_test_file(&dir, "kick.wav", b"fake audio data");