Skip to main content

max / goingson

perf: selective sync invalidation, poison-safe sync lock, windowed threads, IMAP batching Axis 5 of the Run #26 ultra-fuzz remediation. UF-3: sync:changes-applied now carries the DB tables a pull touched, threaded from pull_changes (PullOutcome.changed_tables) through SyncResult. The frontend invalidates only the affected cache entities (SYNC_TABLE_TO_ENTITIES), falling back to a full bust for an unknown/absent payload, so a remote task edit no longer forces the compose screen to re-hydrate every contact. A new list_contact_email_directory command (one JOIN, name+email only) replaces the full contact hydration in the compose autocomplete. UF-4 (CHRONIC-A): the sync client's RwLock is sealed inside a SyncClientCell newtype with a private field, so a raw .read().expect("poisoned") is unwritable. AppState::read_recovering() is the only accessor and recovers from poisoning. UF-9: the threaded email summary is a single ROW_NUMBER() window pass instead of a per-thread correlated subquery, binding the filter set once. UF-10: both IMAP fetch loops batch the UID set (50/command) to bound memory and command size; the duplicated per-message parse is now one build_parsed_email. UF-11: backup directory creation, gzip, and pruning run in spawn_blocking so a large-DB backup doesn't stall the reactor. Medium: email_sync_scheduler prunes failure_counts for deleted/disabled accounts each pass so the map can't grow unbounded.
Co-Authored-By
Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-16 05:41 UTC
Signed with PGP, not checked
Commit: 731894cda774b53e985dd65e42348f480208d2bb
Parent: 735059d
20 files changed, +440 insertions, -234 deletions
@@ -172,20 +172,14 @@
172 172
173 173 async function loadContactEmails() {
174 174 try {
175 - const contacts = await invoke('list_contacts_filtered', { search: null, tag: null, includeImplicit: true });
176 - contactEmails = [];
177 - for (const c of contacts) {
178 - const name = c.displayName || c.display_name || '';
179 - const isImplicit = c.isImplicit || false;
180 - if (c.emails && c.emails.length > 0) {
181 - for (const e of c.emails) {
182 - contactEmails.push({ name, email: e.address, isImplicit });
183 - }
184 - }
185 - if (c.primaryEmail && !c.emails?.some(e => e.address === c.primaryEmail)) {
186 - contactEmails.push({ name, email: c.primaryEmail, isImplicit });
187 - }
188 - }
175 + // Lightweight directory (one JOIN, name + address only) instead of
176 + // hydrating every contact's sub-collections just for autocomplete.
177 + const entries = await invoke('list_contact_email_directory', { includeImplicit: true });
178 + contactEmails = entries.map(e => ({
179 + name: e.name || '',
180 + email: e.email,
181 + isImplicit: e.isImplicit || false,
182 + }));
189 183 } catch (_) { /* contacts unavailable */ }
190 184 }
191 185
@@ -152,19 +152,32 @@
152 152 .map_err(|e| format!("Failed to get app data dir: {}", e))?
153 153 .join("backups");
154 154
155 - std::fs::create_dir_all(&backup_dir)
156 - .map_err(|e| format!("Failed to create backup directory: {}", e))?;
157 -
158 155 let filename = backup_filename(now);
159 156 let file_path = backup_dir.join(&filename);
160 157
161 158 let export = collect_full_export(state).await.map_err(|e| e.to_string())?;
162 - let size = write_backup(&export, &file_path).map_err(|e| format!("Failed to write backup: {}", e))?;
159 + let item_count = export.total_count();
160 + let max_to_keep = settings.max_backups_to_keep as usize;
161 +
162 + // Directory creation, gzip serialization, and pruning are all blocking and
163 + // can take seconds on a large DB — run them on the blocking pool so the
164 + // async reactor isn't stalled (email_sync uses the same pattern).
165 + let log_path = file_path.clone();
166 + let size = tokio::task::spawn_blocking(move || -> Result<u64, String> {
167 + std::fs::create_dir_all(&backup_dir)
168 + .map_err(|e| format!("Failed to create backup directory: {}", e))?;
169 + let size = write_backup(&export, &file_path)
170 + .map_err(|e| format!("Failed to write backup: {}", e))?;
171 + prune_old_backups(&backup_dir, max_to_keep)?;
172 + Ok(size)
173 + })
174 + .await
175 + .map_err(|e| format!("Backup task panicked: {}", e))??;
163 176
164 177 info!(
165 - path = %file_path.display(),
178 + path = %log_path.display(),
166 179 size_bytes = size,
167 - items = export.total_count(),
180 + items = item_count,
168 181 "Automated backup completed"
169 182 );
170 183
@@ -175,9 +188,6 @@
175 188 .await
176 189 .map_err(|e| format!("Failed to update last backup time: {}", e))?;
177 190
178 - // Prune old backups
179 - prune_old_backups(&backup_dir, settings.max_backups_to_keep as usize)?;
180 -
181 191 Ok(())
182 192 }
183 193
@@ -90,6 +90,13 @@
90 90
91 91 debug!("Email sync scheduler: {} account(s) need sync", accounts.len());
92 92
93 + // A failing account never updates last_sync_at, so it stays "due" and remains
94 + // in this list every tick; any failure_counts key absent here belongs to an
95 + // account that was deleted or had sync disabled. Prune them after the pass so
96 + // the map can't grow without bound over a long-running session.
97 + let due_account_ids: std::collections::HashSet<String> =
98 + accounts.iter().map(|a| a.id.to_string()).collect();
99 +
93 100 // Sync each account independently — a failure in one account (e.g. expired
94 101 // token, network issue) must not prevent other accounts from syncing.
95 102 for account in accounts {
@@ -151,6 +158,8 @@
151 158 }
152 159 }
153 160
161 + failure_counts.retain(|account_id, _| due_account_ids.contains(account_id));
162 +
154 163 Ok(())
155 164 }
156 165
@@ -159,6 +159,7 @@
159 159 $crate::commands::list_events_for_contact,
160 160 $crate::commands::list_emails_for_contact,
161 161 $crate::commands::list_contacts_filtered,
162 + $crate::commands::list_contact_email_directory,
162 163 $crate::commands::bulk_delete_contacts,
163 164 $crate::commands::bulk_tag_contacts,
164 165 // Snooze Options
@@ -30,6 +30,28 @@
30 30 /// The API key is a public client identifier, not a secret.
31 31 const SYNCKIT_TOML: &str = include_str!("../../synckit.toml");
32 32
33 + /// Holds the optional sync client behind a poison-recovering lock.
34 + ///
35 + /// The inner `RwLock` is private to this newtype, so no caller can write
36 + /// `.read().expect("poisoned")` and crash a background loop on a poisoned lock.
37 + /// The only access is [`SyncClientCell::get`], which recovers instead of
38 + /// panicking — the constructive fix for the recurring sync-scheduler poison nit.
39 + #[derive(Default)]
40 + pub struct SyncClientCell(RwLock<Option<Arc<SyncKitClient>>>);
41 +
42 + impl SyncClientCell {
43 + /// Wrap an initial (possibly absent) client.
44 + pub(crate) fn new(client: Option<Arc<SyncKitClient>>) -> Self {
45 + Self(RwLock::new(client))
46 + }
47 +
48 + /// Clone out the current client, recovering from a poisoned lock rather
49 + /// than panicking.
50 + pub(crate) fn get(&self) -> Option<Arc<SyncKitClient>> {
51 + self.0.read().unwrap_or_else(|e| e.into_inner()).clone()
52 + }
53 + }
54 +
33 55 /// Application state holding database connections and repositories
34 56 pub struct AppState {
35 57 pub pool: SqlitePool,
@@ -49,7 +71,7 @@
49 71 pub monthly_reviews: Arc<dyn MonthlyReviewRepository>,
50 72 pub backup_settings: Arc<dyn BackupSettingsRepository>,
51 73 pub sync_accounts: Arc<dyn SyncAccountRepository>,
52 - pub sync_client: RwLock<Option<Arc<SyncKitClient>>>,
74 + pub sync_client: SyncClientCell,
53 75 pub sync_lock: Arc<TokioMutex<()>>,
54 76 /// Per-account email sync locks to prevent concurrent syncs on the same account.
55 77 pub email_sync_locks: Arc<Mutex<std::collections::HashSet<goingson_core::EmailAccountId>>>,
@@ -163,7 +185,7 @@
163 185 monthly_reviews,
164 186 backup_settings,
165 187 sync_accounts,
166 - sync_client: RwLock::new(sync_client.map(Arc::new)),
188 + sync_client: SyncClientCell::new(sync_client.map(Arc::new)),
167 189 sync_lock: Arc::new(TokioMutex::new(())),
168 190 email_sync_locks: Arc::new(Mutex::new(std::collections::HashSet::new())),
169 191 token_refresh_locks: Arc::new(Mutex::new(std::collections::HashMap::new())),
@@ -173,6 +195,13 @@
173 195 })
174 196 }
175 197
198 + /// Clone out the current sync client, recovering from a poisoned lock
199 + /// instead of panicking. Background loops must use this (never a raw
200 + /// `.read().expect(...)`), so a writer panic can't kill sync for the session.
201 + pub(crate) fn read_recovering(&self) -> Option<Arc<SyncKitClient>> {
202 + self.sync_client.get()
203 + }
204 +
176 205 /// Gets or creates a per-account token refresh lock.
177 206 pub fn token_refresh_lock(&self, account_id: uuid::Uuid) -> Arc<TokioMutex<()>> {
178 207 let mut locks = self.token_refresh_locks.lock().unwrap_or_else(|e| e.into_inner());
@@ -72,7 +72,7 @@
72 72 }
73 73 };
74 74
75 - let client: Arc<synckit_client::SyncKitClient> = match state.sync_client.read().expect("sync_client poisoned").clone() {
75 + let client: Arc<synckit_client::SyncKitClient> = match state.read_recovering() {
76 76 Some(c) => c,
77 77 None => continue,
78 78 };
@@ -189,7 +189,8 @@
189 189 info!("Auto-sync: pushed {}, pulled {}", result.pushed, result.pulled);
190 190 }
191 191 if result.pulled > 0 {
192 - let _ = app.emit("sync:changes-applied", ());
192 + // Carry the changed tables so the UI invalidates selectively.
193 + let _ = app.emit("sync:changes-applied", &result.pulled_tables);
193 194 }
194 195 }
195 196 Err(e) => {
@@ -72,7 +72,7 @@
72 72 monthly_reviews: Arc::new(SqliteMonthlyReviewRepository::new(pool.clone())),
73 73 backup_settings: Arc::new(SqliteBackupSettingsRepository::new(pool.clone())),
74 74 sync_accounts: Arc::new(SqliteSyncAccountRepository::new(pool.clone())),
75 - sync_client: std::sync::RwLock::new(None),
75 + sync_client: crate::state::SyncClientCell::default(),
76 76 sync_lock: Arc::new(tokio::sync::Mutex::new(())),
77 77 email_sync_locks: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
78 78 token_refresh_locks: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),