Skip to main content

max / goingson

Count export rows instead of listing them The export summary called list_all on five repositories and took .len(), pulling every row into memory (email bodies included) to produce five integers. It is a single batched COUNT query now, on the stats repository alongside the dashboard counts. The restore path deserializes straight off the decompressing reader rather than reading the whole decompressed JSON into a String first, so it no longer holds the text and the parsed structure at the same time.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-26 14:55 UTC
Signed with PGP, not checked
Commit: efff2d189f527d3c6bdaa10e471395643743fbdd
Parent: 280c0bd
5 files changed, +271 insertions, -23 deletions
M crates/db-sqlite/tests/stats_repo_tests.rs +187 -4
@@ -4,12 +4,13 @@
4 4
5 5 use chrono::{Duration, Utc};
6 6 use goingson_core::{
7 - EmailRepository, EventRepository, NewEmail, NewEvent, NewProject, NewTask, Priority,
8 - ProjectRepository, ProjectStatus, ProjectType, Recurrence, StatsRepository, TaskCrud,
7 + ContactRepository, EmailRepository, EventRepository, NewContact, NewEmail, NewEvent,
8 + NewProject, NewTask, Priority, ProjectRepository, ProjectStatus, ProjectType, Recurrence,
9 + StatsRepository, TaskCrud,
9 10 };
10 11 use goingson_db_sqlite::{
11 - SqliteEmailRepository, SqliteEventRepository, SqliteProjectRepository, SqliteStatsRepository,
12 - SqliteTaskRepository,
12 + SqliteContactRepository, SqliteEmailRepository, SqliteEventRepository, SqliteProjectRepository,
13 + SqliteStatsRepository, SqliteTaskRepository,
13 14 };
14 15
15 16 #[tokio::test]
@@ -285,3 +286,185 @@
285 286 );
286 287 assert!(stats.high_urgency_tasks.is_empty());
287 288 }
289 +
290 + /// `get_export_counts` must agree with the `list_all` methods it stands in for.
291 + ///
292 + /// The export summary used to be `list_all(..).len()` on five repositories,
293 + /// which loaded every row (email bodies included) to produce five integers. It
294 + /// is a COUNT query now, so the filters live in two places: each `list_all`
295 + /// WHERE clause and the counts query. This test populates the rows those
296 + /// filters disagree about -- a deleted task, a draft email, an archived email,
297 + /// an implicit contact -- so a filter that drifts on one side fails here rather
298 + /// than quietly showing the user the wrong number before a backup.
299 + #[tokio::test]
300 + async fn export_counts_match_list_all() {
301 + let pool = common::setup_test_db().await;
302 + let user_id = common::create_test_user(&pool).await;
303 +
304 + let projects = SqliteProjectRepository::new(pool.clone());
305 + let tasks = SqliteTaskRepository::new(pool.clone());
306 + let events = SqliteEventRepository::new(pool.clone());
307 + let emails = SqliteEmailRepository::new(pool.clone());
308 + let contacts = SqliteContactRepository::new(pool.clone());
309 + let stats = SqliteStatsRepository::new(pool.clone());
310 +
311 + let now = Utc::now();
312 +
313 + projects
314 + .create(
315 + user_id,
316 + NewProject {
317 + name: "Counted".to_string(),
318 + description: String::new(),
319 + project_type: ProjectType::default(),
320 + status: ProjectStatus::Active,
321 + },
322 + )
323 + .await
324 + .expect("create project");
325 +
326 + tasks
327 + .create(user_id, NewTask::builder("Kept").build())
328 + .await
329 + .expect("create kept task");
330 + // Soft-deleted: excluded by list_all's `status != 'Deleted'`.
331 + let doomed = tasks
332 + .create(user_id, NewTask::builder("Deleted").build())
333 + .await
334 + .expect("create doomed task");
335 + tasks.delete(doomed.id, user_id).await.expect("delete task");
336 +
337 + events
338 + .create(
339 + user_id,
340 + NewEvent {
341 + user_id: Some(user_id),
342 + project_id: None,
343 + contact_id: None,
344 + title: "Counted".into(),
345 + description: String::new(),
346 + start_time: now,
347 + end_time: Some(now + Duration::hours(1)),
348 + location: None,
349 + linked_task_id: None,
350 + recurrence: Recurrence::None,
351 + recurrence_rule: None,
352 + block_type: None,
353 + reminder_offsets_seconds: Vec::new(),
354 + },
355 + )
356 + .await
357 + .expect("create event");
358 +
359 + let plain = emails
360 + .create(
361 + user_id,
362 + NewEmail {
363 + project_id: None,
364 + from_address: "a@example.com".to_string(),
365 + to_address: "b@example.com".to_string(),
366 + subject: "Plain".to_string(),
367 + body: "body".to_string(),
368 + is_read: false,
369 + received_at: Some(now),
370 + },
371 + )
372 + .await
373 + .expect("create email");
374 + // Archived: still counted, because the export passes include_archived = true.
375 + let archived = emails
376 + .create(
377 + user_id,
378 + NewEmail {
379 + project_id: None,
380 + from_address: "a@example.com".to_string(),
381 + to_address: "b@example.com".to_string(),
382 + subject: "Archived".to_string(),
383 + body: "body".to_string(),
384 + is_read: true,
385 + received_at: Some(now),
386 + },
387 + )
388 + .await
389 + .expect("create archived email");
390 + emails
391 + .archive(archived.id, user_id)
392 + .await
393 + .expect("archive email");
394 + // Draft: excluded by list_all's `is_draft = 0`.
395 + emails
396 + .save_draft(
397 + goingson_core::EmailId::new(),
398 + user_id,
399 + "a@example.com",
400 + "b@example.com",
401 + None,
402 + None,
403 + "Draft",
404 + "body",
405 + None,
406 + None,
407 + None,
408 + None,
409 + )
410 + .await
411 + .expect("save draft");
412 + assert_ne!(plain.id, archived.id);
413 +
414 + for (name, is_implicit) in [("Explicit", false), ("Implicit", true)] {
415 + contacts
416 + .create(
417 + user_id,
418 + NewContact {
419 + display_name: name.to_string(),
420 + nickname: None,
421 + company: None,
422 + title: None,
423 + notes: String::new(),
424 + tags: vec![],
425 + birthday: None,
426 + timezone: None,
427 + is_implicit,
428 + },
429 + )
430 + .await
431 + .expect("create contact");
432 + }
433 +
434 + let counts = stats
435 + .get_export_counts(user_id)
436 + .await
437 + .expect("get_export_counts");
438 +
439 + assert_eq!(
440 + counts.projects as usize,
441 + projects.list_all(user_id).await.unwrap().len(),
442 + "project count"
443 + );
444 + assert_eq!(
445 + counts.tasks as usize,
446 + tasks.list_all(user_id).await.unwrap().len(),
447 + "task count excludes the deleted task"
448 + );
449 + assert_eq!(
450 + counts.events as usize,
451 + events.list_all(user_id).await.unwrap().len(),
452 + "event count"
453 + );
454 + assert_eq!(
455 + counts.emails as usize,
456 + emails.list_all(user_id, true).await.unwrap().len(),
457 + "email count keeps the archived email and drops the draft"
458 + );
459 + assert_eq!(
460 + counts.contacts as usize,
461 + contacts.list_all(user_id).await.unwrap().len(),
462 + "contact count excludes the implicit contact"
463 + );
464 +
465 + // Pin the absolute numbers too, so a filter that drifts on *both* sides at
466 + // once (making the equalities above vacuously true) still fails.
467 + assert_eq!(counts.tasks, 1, "one kept task");
468 + assert_eq!(counts.emails, 2, "plain + archived, not the draft");
469 + assert_eq!(counts.contacts, 1, "one explicit contact");
470 + }
M src-tauri/src/commands/export.rs +8 -12
@@ -107,20 +107,16 @@
107 107 pub async fn get_export_summary(
108 108 state: State<'_, Arc<AppState>>,
109 109 ) -> Result<ExportSummaryResponse, ApiError> {
110 - let (projects, tasks, events, emails, contacts) = tokio::join!(
111 - state.projects.list_all(DESKTOP_USER_ID),
112 - state.tasks.list_all(DESKTOP_USER_ID),
113 - state.events.list_all(DESKTOP_USER_ID),
114 - state.emails.list_all(DESKTOP_USER_ID, true),
115 - state.contacts.list_all(DESKTOP_USER_ID),
116 - );
110 + // COUNT queries, not list_all + len(): counting this way used to pull every
111 + // row (email bodies included) into memory to produce five integers.
112 + let counts = state.stats.get_export_counts(DESKTOP_USER_ID).await?;
117 113
118 114 Ok(ExportSummaryResponse {
119 - project_count: projects?.len(),
120 - task_count: tasks?.len(),
121 - event_count: events?.len(),
122 - email_count: emails?.len(),
123 - contact_count: contacts?.len(),
115 + project_count: counts.projects as usize,
116 + task_count: counts.tasks as usize,
117 + event_count: counts.events as usize,
118 + email_count: counts.emails as usize,
119 + contact_count: counts.contacts as usize,
124 120 })
125 121 }
126 122
M src-tauri/src/export/backup.rs +6 -6
@@ -3,7 +3,7 @@
3 3 //! Provides compressed JSON backup creation and restoration for all GoingsOn data.
4 4
5 5 use std::fs::File;
6 - use std::io::{BufWriter, Read, Write};
6 + use std::io::{BufReader, BufWriter, Read, Write};
7 7 use std::path::Path;
8 8
9 9 use chrono::{DateTime, Utc};
@@ -191,12 +191,12 @@
191 191 let file = File::open(path.as_ref())?;
192 192 let decoder = GzDecoder::new(file);
193 193 // Limit decompressed size to 500 MB to prevent decompression bombs
194 - let mut limited = decoder.take(500 * 1024 * 1024);
194 + let limited = decoder.take(500 * 1024 * 1024);
195 195
196 - let mut json = String::new();
197 - limited.read_to_string(&mut json)?;
198 -
199 - let export: FullExport = serde_json::from_str(&json)?;
196 + // Deserialize straight off the decompressing reader. Reading the whole
197 + // decompressed JSON into a String first meant holding the text and the
198 + // parsed structure at once, doubling peak memory on a large restore.
199 + let export: FullExport = serde_json::from_reader(BufReader::new(limited))?;
200 200
201 201 if !export.is_compatible() {
202 202 return Err(BackupError::IncompatibleVersion {
M crates/core/src/repository/stats.rs +27
@@ -34,9 +34,36 @@
34 34 pub due: Option<String>,
35 35 }
36 36
37 + /// Row counts for the export/backup summary.
38 + ///
39 + /// Each field counts exactly what the matching `list_all` on that repository
40 + /// would return, so the summary shown before an export matches the export.
41 + /// Keeping these as COUNT queries is the point: the summary used to call
42 + /// `list_all` and take `.len()`, which materialized every email body in memory
43 + /// to produce five integers.
44 + #[derive(Debug, Clone, serde::Serialize)]
45 + pub struct ExportCounts {
46 + /// Projects (all of them).
47 + pub projects: i64,
48 + /// Tasks, excluding those in the `Deleted` status.
49 + pub tasks: i64,
50 + /// Events (all of them).
51 + pub events: i64,
52 + /// Emails, excluding drafts, including archived.
53 + pub emails: i64,
54 + /// Contacts, excluding implicit ones.
55 + pub contacts: i64,
56 + }
57 +
37 58 /// Repository for dashboard statistics.
38 59 #[async_trait]
39 60 pub trait StatsRepository: Send + Sync {
61 + /// Counts the rows an export would write, without loading them.
62 + ///
63 + /// Filters must stay in step with the corresponding `list_all` methods; the
64 + /// `export_counts_match_list_all` tests pin that parity.
65 + async fn get_export_counts(&self, user_id: UserId) -> Result<ExportCounts>;
66 +
40 67 /// Computes aggregated dashboard statistics for a user.
41 68 ///
42 69 /// Calendar-day windows (`today_start`, `tomorrow_start`, `week_end`) are
M crates/db-sqlite/src/repository/stats_repo.rs +43 -1
@@ -8,7 +8,9 @@
8 8
9 9 use async_trait::async_trait;
10 10 use chrono::{DateTime, Duration, Utc};
11 - use goingson_core::{CoreError, DashboardStats, HighUrgencyTask, Result, StatsRepository, UserId};
11 + use goingson_core::{
12 + CoreError, DashboardStats, ExportCounts, HighUrgencyTask, Result, StatsRepository, UserId,
13 + };
12 14 use sqlx::SqlitePool;
13 15
14 16 use crate::utils::{format_datetime, parse_datetime};
@@ -31,6 +33,46 @@
31 33
32 34 #[async_trait]
33 35 impl StatsRepository for SqliteStatsRepository {
36 + #[tracing::instrument(skip_all)]
37 + async fn get_export_counts(&self, user_id: UserId) -> Result<ExportCounts> {
38 + // Same single-query shape as the dashboard counts. Each predicate mirrors
39 + // the WHERE clause of the matching list_all: tasks skip 'Deleted', emails
40 + // skip drafts (and keep archived, since the export passes
41 + // include_archived = true), contacts skip implicit rows.
42 + #[derive(sqlx::FromRow)]
43 + struct CountsRow {
44 + projects: i64,
45 + tasks: i64,
46 + events: i64,
47 + emails: i64,
48 + contacts: i64,
49 + }
50 +
51 + let row: CountsRow = sqlx::query_as(
52 + "SELECT \
53 + (SELECT COUNT(*) FROM projects WHERE user_id = ?1) AS projects, \
54 + (SELECT COUNT(*) FROM tasks WHERE user_id = ?1 \
55 + AND status != 'Deleted') AS tasks, \
56 + (SELECT COUNT(*) FROM events WHERE user_id = ?1) AS events, \
57 + (SELECT COUNT(*) FROM emails WHERE user_id = ?1 \
58 + AND is_draft = 0) AS emails, \
59 + (SELECT COUNT(*) FROM contacts WHERE user_id = ?1 \
60 + AND is_implicit = 0) AS contacts",
61 + )
62 + .bind(user_id.to_string())
63 + .fetch_one(&self.pool)
64 + .await
65 + .map_err(CoreError::database)?;
66 +
67 + Ok(ExportCounts {
68 + projects: row.projects,
69 + tasks: row.tasks,
70 + events: row.events,
71 + emails: row.emails,
72 + contacts: row.contacts,
73 + })
74 + }
75 +
34 76 #[tracing::instrument(skip_all)]
35 77 async fn get_dashboard_stats(
36 78 &self,