Skip to main content

max / makenotwork

mt: bound directory count + tighten search query (ultra-fuzz P1, P4) P1: list_communities/list_archived_communities used a row-multiplying categories-threads join + GROUP BY COUNT(DISTINCT), forcing aggregation of every community's threads before the LIMIT. Rewrite the per-community category/thread counts as scalar subqueries in the target list, evaluated only for the page's output rows. Migration-free. P4: search_threads parsed websearch_to_tsquery 5x inline. Hoist it into a one-row q CTE cross-joined into both match CTEs, and replace the t.id NOT IN (subquery) anti-join with NOT EXISTS. (/search is already rate-limited via search_rate_limit; the audit note there was inaccurate.) P2 (profile endorsement/post counts) assessed and left as-is: bounded per-(user, community), PK/idx_posts_author-driven, not a table scan; removing the residual category-chain hop needs a community_id denorm on threads/posts, disproportionate at this scale (audit sanctioned 'accept'). New directory_counts_categories_and_threads_per_community test; 252 integration + 136 lib unit green; clippy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-19 17:51 UTC
Commit: cd0acf55b810d659201ebe42d39cf10b2d03d85c
Parent: a0494c2
2 files changed, +59 insertions, -17 deletions
M multithreaded/crates/mt-db/src/queries.rs +31 -17
@@ -183,15 +183,20 @@ pub struct CommunityListRow {
183 183 #[tracing::instrument(skip_all)]
184 184 pub async fn list_communities(pool: &PgPool, limit: i64, offset: i64) -> Result<Vec<CommunityListRow>, sqlx::Error> {
185 185 sqlx::query_as::<_, CommunityListRow>(
186 + // Per-community counts as scalar subqueries rather than a
187 + // categories⋈threads join + GROUP BY COUNT(DISTINCT): the join
188 + // multiplied rows (community × categories × threads) and forced
189 + // aggregating *every* community's threads before LIMIT. As scalar
190 + // subqueries in the target list they're evaluated only for the page's
191 + // output rows (after ORDER BY + LIMIT), bounding the work to one page.
186 192 "SELECT co.name, co.slug, co.description,
187 - COUNT(DISTINCT c.id) AS category_count,
188 - COUNT(DISTINCT t.id) AS thread_count
193 + (SELECT COUNT(*) FROM categories c WHERE c.community_id = co.id) AS category_count,
194 + (SELECT COUNT(*) FROM threads t
195 + JOIN categories c2 ON c2.id = t.category_id
196 + WHERE c2.community_id = co.id) AS thread_count
189 197 FROM communities co
190 - LEFT JOIN categories c ON c.community_id = co.id
191 - LEFT JOIN threads t ON t.category_id = c.id
192 198 WHERE co.suspended_at IS NULL
193 199 AND co.state <> 'archived'
194 - GROUP BY co.id
195 200 ORDER BY co.name
196 201 LIMIT $1 OFFSET $2",
197 202 )
@@ -220,15 +225,15 @@ pub async fn list_archived_communities(
220 225 offset: i64,
221 226 ) -> Result<Vec<CommunityListRow>, sqlx::Error> {
222 227 sqlx::query_as::<_, CommunityListRow>(
228 + // Scalar-subquery counts (see `list_communities`) — bounded to the page.
223 229 "SELECT co.name, co.slug, co.description,
224 - COUNT(DISTINCT c.id) AS category_count,
225 - COUNT(DISTINCT t.id) AS thread_count
230 + (SELECT COUNT(*) FROM categories c WHERE c.community_id = co.id) AS category_count,
231 + (SELECT COUNT(*) FROM threads t
232 + JOIN categories c2 ON c2.id = t.category_id
233 + WHERE c2.community_id = co.id) AS thread_count
226 234 FROM communities co
227 - LEFT JOIN categories c ON c.community_id = co.id
228 - LEFT JOIN threads t ON t.category_id = c.id
229 235 WHERE co.suspended_at IS NULL
230 236 AND co.state = 'archived'
231 - GROUP BY co.id
232 237 ORDER BY co.name
233 238 LIMIT $1 OFFSET $2",
234 239 )
@@ -1477,7 +1482,14 @@ pub async fn search_threads(
1477 1482 limit: i64,
1478 1483 ) -> Result<Vec<SearchResultRow>, sqlx::Error> {
1479 1484 sqlx::query_as::<_, SearchResultRow>(
1480 - "WITH thread_matches AS (
1485 + // `q` parses the tsquery once (it was parsed 5× inline); cross-joining
1486 + // the single-row CTE feeds it to every rank/match without re-parsing.
1487 + // post_matches uses NOT EXISTS rather than NOT IN (better anti-join,
1488 + // and NULL-safe).
1489 + "WITH q AS (
1490 + SELECT websearch_to_tsquery('english', $1) AS tsq
1491 + ),
1492 + thread_matches AS (
1481 1493 SELECT t.id AS thread_id,
1482 1494 t.title AS thread_title,
1483 1495 COALESCE(u.display_name, u.username) AS author_username,
@@ -1487,15 +1499,16 @@ pub async fn search_threads(
1487 1499 c.slug AS category_slug,
1488 1500 LEFT(t.title, 200) AS snippet,
1489 1501 t.last_activity_at,
1490 - (ts_rank(t.search_tsv, websearch_to_tsquery('english', $1)) * 2.0
1502 + (ts_rank(t.search_tsv, q.tsq) * 2.0
1491 1503 + similarity(t.title, $2)) AS rank
1492 1504 FROM threads t
1493 1505 JOIN categories c ON c.id = t.category_id
1494 1506 JOIN communities co ON co.id = c.community_id
1495 1507 JOIN users u ON u.mnw_account_id = t.author_id
1508 + CROSS JOIN q
1496 1509 WHERE t.deleted_at IS NULL
1497 1510 AND co.suspended_at IS NULL
1498 - AND (t.search_tsv @@ websearch_to_tsquery('english', $1)
1511 + AND (t.search_tsv @@ q.tsq
1499 1512 OR similarity(t.title, $2) > 0.1)
1500 1513 AND ($3::text IS NULL OR co.slug = $3)
1501 1514 ),
@@ -1510,19 +1523,20 @@ pub async fn search_threads(
1510 1523 c.slug AS category_slug,
1511 1524 LEFT(p.body_markdown, 200) AS snippet,
1512 1525 t.last_activity_at,
1513 - ts_rank(p.search_tsv, websearch_to_tsquery('english', $1)) AS rank
1526 + ts_rank(p.search_tsv, q.tsq) AS rank
1514 1527 FROM posts p
1515 1528 JOIN threads t ON t.id = p.thread_id
1516 1529 JOIN categories c ON c.id = t.category_id
1517 1530 JOIN communities co ON co.id = c.community_id
1518 1531 JOIN users pu ON pu.mnw_account_id = t.author_id
1532 + CROSS JOIN q
1519 1533 WHERE t.deleted_at IS NULL
1520 1534 AND co.suspended_at IS NULL
1521 1535 AND p.removed_at IS NULL
1522 - AND p.search_tsv @@ websearch_to_tsquery('english', $1)
1536 + AND p.search_tsv @@ q.tsq
1523 1537 AND ($3::text IS NULL OR co.slug = $3)
1524 - AND t.id NOT IN (SELECT thread_id FROM thread_matches)
1525 - ORDER BY t.id, ts_rank(p.search_tsv, websearch_to_tsquery('english', $1)) DESC
1538 + AND NOT EXISTS (SELECT 1 FROM thread_matches tm WHERE tm.thread_id = t.id)
1539 + ORDER BY t.id, ts_rank(p.search_tsv, q.tsq) DESC
1526 1540 )
1527 1541 SELECT * FROM thread_matches
1528 1542 UNION ALL
M multithreaded/tests/workflows/pagination.rs +28
@@ -1,6 +1,34 @@
1 1 use crate::harness::TestHarness;
2 2
3 3 #[tokio::test]
4 + async fn directory_counts_categories_and_threads_per_community() {
5 + // Pins the scalar-subquery rewrite of list_communities (P1): per-community
6 + // category_count and thread_count must match the old COUNT(DISTINCT) join.
7 + let mut h = TestHarness::new().await;
8 + let comm = h.create_community("Counted", "counted").await;
9 + let cat_a = h.create_category(comm, "Alpha", "alpha").await;
10 + let cat_b = h.create_category(comm, "Beta", "beta").await;
11 + let author = h.login_as("writer").await;
12 + h.add_membership(author, comm, "member").await;
13 + // 2 threads in alpha, 1 in beta → 3 threads, 2 categories.
14 + h.create_thread_with_post(cat_a, author, "T1", "b").await;
15 + h.create_thread_with_post(cat_a, author, "T2", "b").await;
16 + h.create_thread_with_post(cat_b, author, "T3", "b").await;
17 +
18 + // A second community with no content stays at zero — proves counts are
19 + // per-community, not global.
20 + h.create_community("Empty", "empty").await;
21 +
22 + let rows = mt_db::queries::list_communities(&h.db, 25, 0).await.unwrap();
23 + let counted = rows.iter().find(|r| r.slug == "counted").expect("counted present");
24 + assert_eq!(counted.category_count, 2, "two categories");
25 + assert_eq!(counted.thread_count, 3, "three threads across categories");
26 + let empty = rows.iter().find(|r| r.slug == "empty").expect("empty present");
27 + assert_eq!(empty.category_count, 0);
28 + assert_eq!(empty.thread_count, 0);
29 + }
30 +
31 + #[tokio::test]
4 32 async fn directory_pagination_shows_limited_communities() {
5 33 let mut h = TestHarness::new().await;
6 34