//! Background worker for the blob-directory layout migration. //! //! Mirrors the pattern in `cleanup.rs`: a dedicated thread with its own Database + //! SampleStore, communicating via channels, with the GUI thread polling events each //! frame. The work itself lives in `audiofiles_core::store::layout`; this is only //! the off-GUI-thread wrapper plus progress throttling. //! //! Why it must be off the GUI thread rather than a startup step: the sweep is one //! rename per blob on a filesystem whose metadata operations are the reason the //! migration exists at all. On the measured 289k-file library that is minutes at //! best, so doing it inline at vault open would present as a hang. use std::path::PathBuf; use tracing::{error, info, instrument, warn}; use audiofiles_core::config_key::ConfigKey; use audiofiles_core::db::Database; use audiofiles_core::store::SampleStore; use audiofiles_core::store::layout::{self, LayoutMigration}; use audiofiles_core::vfs_mirror::{MirrorConfig, sync_mirror}; use audiofiles_core::worker_runtime::{WorkerCtx, WorkerHandle, spawn_worker}; /// Emit at most one [`LayoutEvent::Progress`] per this many blobs handled. /// /// The event channel is bounded (4096) and applies backpressure, so an unthrottled /// per-blob emit over a 289k-blob sweep would have the worker waiting on a GUI that /// redraws at most 60 times a second, turning a progress bar into a brake. One /// event per 256 blobs is far finer than a human can perceive on a bar that takes /// minutes to fill. const PROGRESS_STRIDE: usize = 256; /// Command sent from the GUI thread to the layout worker. pub enum LayoutCommand { /// Start (or resume) relocating flat blobs into their shard directories. Migrate, /// Cancel the running migration (sets the worker's cancel flag synchronously). Cancel, } /// Event sent from the layout worker back to the GUI thread. pub enum LayoutEvent { /// Progress through the blobs enumerated at the start of this pass. Progress { completed: usize, total: usize }, /// The pass finished, was cancelled, or failed to start. Complete { /// Blobs relocated into a shard. moved: usize, /// Redundant flat blobs discarded because the shard already matched. deduped: usize, /// Blobs that could not be relocated. errors: usize, /// Stopped early on request. The vault stays resolvable and resumable. cancelled: bool, /// The vault is now fully sharded and recorded as such. completed: bool, /// The VFS mirror was rebuilt after a completed migration. mirror_rebuilt: bool, }, } impl LayoutEvent { /// A `Complete` reporting that nothing ran, for the failure paths that must /// still emit a terminal event so the GUI's busy flag clears. fn failed() -> Self { LayoutEvent::Complete { moved: 0, deduped: 0, errors: 1, cancelled: false, completed: false, mirror_rebuilt: false, } } } /// Handle for communicating with the background layout worker. pub struct LayoutHandle(WorkerHandle); impl LayoutHandle { /// Poll for the next event without blocking. pub fn try_recv(&self) -> Option { self.0.try_recv() } /// Send a command to the worker. Returns false if the worker is no longer /// alive, so callers don't treat a dropped command as accepted (and then wait /// forever for a terminal event that cannot arrive). pub fn send(&self, cmd: LayoutCommand) -> bool { if matches!(cmd, LayoutCommand::Cancel) { self.0.request_cancel(); } self.0.send(cmd) } } /// Per-worker state: its own DB connection + store. struct LayoutWorker { db: Database, store: SampleStore, } /// Spawn the background layout-migration worker. #[instrument(skip_all)] pub fn spawn_layout_worker(db_path: PathBuf, store_root: PathBuf) -> std::io::Result { let handle = spawn_worker( "layout-worker", move || -> Result { let db = Database::open(&db_path)?; let store = SampleStore::new(&store_root)?; Ok(LayoutWorker { db, store }) }, |e| { error!("Layout worker failed to open DB/store: {e}"); LayoutEvent::failed() }, |_state| LayoutEvent::failed(), layout_step, )?; Ok(LayoutHandle(handle)) } #[allow( clippy::needless_pass_by_value, reason = "signature dictated by worker_runtime::spawn_worker step-fn contract (FnMut(&mut State, Cmd, &WorkerCtx))" )] fn layout_step(worker: &mut LayoutWorker, cmd: LayoutCommand, ctx: &WorkerCtx) { // Cancel: the flag was already set synchronously by the handle, so the queued // command itself is a no-op. if matches!(cmd, LayoutCommand::Cancel) { return; } // A stale cancel from a previous pass must not abort this one. ctx.reset_cancel(); let report = match layout::migrate_to_sharded( &worker.store, &worker.db, ctx.cancel_flag(), &mut |completed, total| { if completed % PROGRESS_STRIDE == 0 || completed == total { ctx.emit(LayoutEvent::Progress { completed, total }); } }, ) { Ok(report) => report, Err(e) => { error!("Layout migration failed: {e}"); ctx.emit(LayoutEvent::failed()); return; } }; // The mirror's symlinks point at resolved blob paths, so every link to a // relocated blob is now stale. Rebuilding is only worth doing once the sweep is // actually complete: mid-migration the resolver still finds the un-moved blobs, // so a partial rebuild would be work thrown away on the next pass. let mirror_rebuilt = report.completed && report.moved > 0 && rebuild_mirror(worker); let LayoutMigration { moved, deduped, errors, cancelled, completed, } = report; info!( moved, deduped, errors, cancelled, completed, mirror_rebuilt, "layout migration pass finished" ); ctx.emit(LayoutEvent::Complete { moved, deduped, errors, cancelled, completed, mirror_rebuilt, }); } /// Rebuild the VFS mirror if one is configured. Returns whether it ran and /// succeeded. /// /// Best-effort: the migration itself has already committed, and a mirror is a /// derived convenience tree, so a failure here is logged and reported rather than /// turned into a migration failure the user would be invited to retry. fn rebuild_mirror(worker: &LayoutWorker) -> bool { let enabled = worker .db .get_config(ConfigKey::MirrorEnabled) .ok() .flatten() .is_some_and(|v| v == "true" || v == "1"); if !enabled { return false; } let Some(mirror_root) = worker.db.get_config(ConfigKey::MirrorPath).ok().flatten() else { return false; }; let config = MirrorConfig { mirror_root: PathBuf::from(mirror_root), store_root: worker.store.root().to_path_buf(), }; match sync_mirror(&worker.db, &config) { Ok(stats) => { info!( links_created = stats.links_created, entries_removed = stats.entries_removed, "layout migration: mirror rebuilt" ); true } Err(e) => { warn!("layout migration: mirror rebuild failed: {e}"); false } } } /// Whether this vault has flat blobs left to relocate. /// /// Cheap (one `read_dir`) and safe to call at vault open to decide whether to /// dispatch [`LayoutCommand::Migrate`] at all. Checks the filesystem rather than /// trusting the recorded layout alone, so a vault whose sweep was interrupted /// before it could record completion still gets picked up. pub fn migration_pending(db: &Database, store_root: &std::path::Path) -> bool { if matches!(layout::recorded_layout(db), Ok(layout::BlobLayout::Sharded)) { return false; } layout::count_flat_blobs(store_root).unwrap_or(0) > 0 } #[cfg(test)] mod tests { use super::*; use audiofiles_core::store::legacy_flat_blob_path; const HASH: &str = "aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"; fn poll_complete(handle: &LayoutHandle) -> LayoutEvent { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); while std::time::Instant::now() < deadline { while let Some(ev) = handle.try_recv() { if matches!(ev, LayoutEvent::Complete { .. }) { return ev; } } std::thread::sleep(std::time::Duration::from_millis(5)); } panic!("layout worker did not report Complete within 10s"); } #[test] fn spawn_and_drop_does_not_hang() { let dir = tempfile::TempDir::new().unwrap(); let db_path = dir.path().join("audiofiles.db"); let store_root = dir.path().join("store"); std::fs::create_dir_all(&store_root).unwrap(); let _db = Database::open(&db_path).unwrap(); let handle = spawn_layout_worker(db_path, store_root).unwrap(); assert!(handle.try_recv().is_none()); drop(handle); } #[test] fn worker_relocates_a_flat_blob() { let dir = tempfile::TempDir::new().unwrap(); let db_path = dir.path().join("audiofiles.db"); let store_root = dir.path().join("store"); std::fs::create_dir_all(&store_root).unwrap(); let db = Database::open(&db_path).unwrap(); std::fs::write(legacy_flat_blob_path(&store_root, HASH, "wav"), b"bytes").unwrap(); assert!(migration_pending(&db, &store_root)); drop(db); let handle = spawn_layout_worker(db_path.clone(), store_root.clone()).unwrap(); assert!(handle.send(LayoutCommand::Migrate)); match poll_complete(&handle) { LayoutEvent::Complete { moved, errors, completed, cancelled, .. } => { assert_eq!(moved, 1); assert_eq!(errors, 0); assert!(completed); assert!(!cancelled); } LayoutEvent::Progress { .. } => unreachable!("filtered by poll_complete"), } drop(handle); assert!(store_root.join("aa").join(format!("{HASH}.wav")).is_file()); assert!(!legacy_flat_blob_path(&store_root, HASH, "wav").exists()); let db = Database::open(&db_path).unwrap(); assert!(!migration_pending(&db, &store_root)); } #[test] fn empty_store_completes_without_work() { let dir = tempfile::TempDir::new().unwrap(); let db_path = dir.path().join("audiofiles.db"); let store_root = dir.path().join("store"); std::fs::create_dir_all(&store_root).unwrap(); let db = Database::open(&db_path).unwrap(); // A fresh vault has no blobs, so nothing is pending even though the layout // has never been recorded. assert!(!migration_pending(&db, &store_root)); drop(db); let handle = spawn_layout_worker(db_path, store_root).unwrap(); assert!(handle.send(LayoutCommand::Migrate)); match poll_complete(&handle) { LayoutEvent::Complete { moved, errors, completed, mirror_rebuilt, .. } => { assert_eq!(moved, 0); assert_eq!(errors, 0); assert!(completed, "an empty root is trivially sharded"); assert!(!mirror_rebuilt, "nothing moved, so no rebuild"); } LayoutEvent::Progress { .. } => unreachable!("filtered by poll_complete"), } } #[test] fn cancel_before_run_reports_cancelled_and_leaves_the_blob() { let dir = tempfile::TempDir::new().unwrap(); let db_path = dir.path().join("audiofiles.db"); let store_root = dir.path().join("store"); std::fs::create_dir_all(&store_root).unwrap(); let _db = Database::open(&db_path).unwrap(); std::fs::write(legacy_flat_blob_path(&store_root, HASH, "wav"), b"bytes").unwrap(); let handle = spawn_layout_worker(db_path, store_root.clone()).unwrap(); // Cancel sets the flag synchronously, so the Migrate queued behind it aborts // at its first check rather than running to completion. assert!(handle.send(LayoutCommand::Cancel)); assert!(handle.send(LayoutCommand::Migrate)); // Migrate resets the cancel flag at its start (so a stale cancel cannot // wedge every future pass), which means this run legitimately completes. // The point of the test is that the sequence terminates with a real event // and the data survives either way. match poll_complete(&handle) { LayoutEvent::Complete { errors, .. } => assert_eq!(errors, 0), LayoutEvent::Progress { .. } => unreachable!("filtered by poll_complete"), } drop(handle); let found = store_root.join("aa").join(format!("{HASH}.wav")).is_file() || legacy_flat_blob_path(&store_root, HASH, "wav").is_file(); assert!(found, "the blob must exist in one layout or the other"); } #[test] fn layout_event_variants_constructible() { let _ = LayoutEvent::Progress { completed: 1, total: 2, }; let _ = LayoutEvent::failed(); } }