Skip to main content

max / makenotwork

perf: stream CSV exports + close hot/cold-path inefficiencies (ultra-fuzz Run 4, A+) Phase 4 (Performance). Fixes the one SERIOUS finding plus the localized perf gaps. - S1 (SERIOUS) unbounded export queries -> streamed, bounded exports. Every CSV export (sales, purchases, splits, followers+subscribers, subscriptions) now pages its query (LIMIT/OFFSET, stable (created_at,id) order) and streams the body chunk-by-chunk through a bounded channel: peak memory is one batch and the DB connection is released between batches, instead of loading a creator's whole history into one String from one unbounded full-table query. The no-JS HTMX data-URI fallback (which can't stream) materializes a bounded prefix. An overall EXPORT_MAX_ROWS ceiling bounds worst-case OFFSET cost. Admin/internal callers keep a convenience wrapper that accumulates, now capped. (Keyset pagination is the documented future optimization.) - M-Perf1 (S2) urlhaus host-extraction walked the mmap on the async runtime; split check_urlhaus into extract + lookup and run the page-fault-prone extraction in spawn_blocking on the large-file scan path. - fetch-all-then-find -> indexed point lookups: update_repo_visibility uses get_repo_by_id; begin_rotation uses sync_device_belongs. - spool free-space TOCTOU: reserve headroom for all SCAN_WORKER_COUNT workers, so concurrent spool downloads can't collectively overrun the floor between the check and the write. - health dashboard: the four DB count probes now run via tokio::join! (each still records its own latency) instead of serially. Accepted residuals (Max, 2026-06-23), documented at the sites: the scheduler holds 1/25 pool connections for a tick (dedicated lock side-pool deferred - deploy-critical lifecycle, small gain); the daily item-purge cron gathers all expired keys before its batch delete (bounding couples gather+delete on a low-value cron path). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-24 00:08 UTC
Commit: c114130e6e0086f03f532db97f0226f2107f419b
Parent: cc53cb1
20 files changed, +836 insertions, -380 deletions
A server/.sqlx/query-169ed0d8f74cc69847d5d63b6a1d3b3f4d277cd3540036c7d5278ddc139a3340.json +72
@@ -0,0 +1,72 @@
1 + {
2 + "db_name": "PostgreSQL",
3 + "query": "\n SELECT p.title AS project_title, t.name AS tier_name, t.price_cents,\n u.username, s.status AS \"status: super::SubscriptionStatus\",\n s.current_period_start AS \"current_period_start: chrono::DateTime<chrono::Utc>\",\n s.current_period_end AS \"current_period_end: chrono::DateTime<chrono::Utc>\",\n s.canceled_at AS \"canceled_at: chrono::DateTime<chrono::Utc>\",\n s.created_at AS \"created_at: chrono::DateTime<chrono::Utc>\"\n FROM subscriptions s\n JOIN users u ON u.id = s.subscriber_id\n JOIN subscription_tiers t ON t.id = s.tier_id\n JOIN projects p ON p.id = s.project_id\n WHERE s.project_id IN (SELECT id FROM projects WHERE user_id = $1)\n ORDER BY s.created_at DESC, s.id DESC\n LIMIT $2 OFFSET $3\n ",
4 + "describe": {
5 + "columns": [
6 + {
7 + "ordinal": 0,
8 + "name": "project_title",
9 + "type_info": "Varchar"
10 + },
11 + {
12 + "ordinal": 1,
13 + "name": "tier_name",
14 + "type_info": "Varchar"
15 + },
16 + {
17 + "ordinal": 2,
18 + "name": "price_cents",
19 + "type_info": "Int4"
20 + },
21 + {
22 + "ordinal": 3,
23 + "name": "username",
24 + "type_info": "Varchar"
25 + },
26 + {
27 + "ordinal": 4,
28 + "name": "status: super::SubscriptionStatus",
29 + "type_info": "Varchar"
30 + },
31 + {
32 + "ordinal": 5,
33 + "name": "current_period_start: chrono::DateTime<chrono::Utc>",
34 + "type_info": "Timestamptz"
35 + },
36 + {
37 + "ordinal": 6,
38 + "name": "current_period_end: chrono::DateTime<chrono::Utc>",
39 + "type_info": "Timestamptz"
40 + },
41 + {
42 + "ordinal": 7,
43 + "name": "canceled_at: chrono::DateTime<chrono::Utc>",
44 + "type_info": "Timestamptz"
45 + },
46 + {
47 + "ordinal": 8,
48 + "name": "created_at: chrono::DateTime<chrono::Utc>",
49 + "type_info": "Timestamptz"
50 + }
51 + ],
52 + "parameters": {
53 + "Left": [
54 + "Uuid",
55 + "Int8",
56 + "Int8"
57 + ]
58 + },
59 + "nullable": [
60 + false,
61 + false,
62 + false,
63 + false,
64 + false,
65 + true,
66 + true,
67 + true,
68 + false
69 + ]
70 + },
71 + "hash": "169ed0d8f74cc69847d5d63b6a1d3b3f4d277cd3540036c7d5278ddc139a3340"
72 + }
A server/.sqlx/query-17188cab9291709226c6f4586bccfbbd878a4c98b27bb844e8b48c56edd7981a.json +150
@@ -0,0 +1,150 @@
1 + {
2 + "db_name": "PostgreSQL",
3 + "query": "\n SELECT\n id AS \"id: TransactionId\", buyer_id AS \"buyer_id: UserId\", seller_id AS \"seller_id: UserId\",\n item_id AS \"item_id: ItemId\", amount_cents AS \"amount_cents: Cents\", platform_fee_cents AS \"platform_fee_cents: Cents\",\n currency, status AS \"status: super::TransactionStatus\", stripe_payment_intent_id, stripe_checkout_session_id,\n created_at AS \"created_at: chrono::DateTime<chrono::Utc>\", completed_at AS \"completed_at: chrono::DateTime<chrono::Utc>\",\n item_title, seller_username, share_contact, project_id AS \"project_id: ProjectId\",\n parent_transaction_id AS \"parent_transaction_id: TransactionId\", promo_code_id AS \"promo_code_id: PromoCodeId\",\n guest_email, claim_token AS \"claim_token: ClaimToken\", claimed_by AS \"claimed_by: UserId\",\n download_token AS \"download_token: DownloadToken\"\n FROM transactions WHERE buyer_id = $1\n ORDER BY created_at DESC, id DESC\n LIMIT $2 OFFSET $3\n ",
4 + "describe": {
5 + "columns": [
6 + {
7 + "ordinal": 0,
8 + "name": "id: TransactionId",
9 + "type_info": "Uuid"
10 + },
11 + {
12 + "ordinal": 1,
13 + "name": "buyer_id: UserId",
14 + "type_info": "Uuid"
15 + },
16 + {
17 + "ordinal": 2,
18 + "name": "seller_id: UserId",
19 + "type_info": "Uuid"
20 + },
21 + {
22 + "ordinal": 3,
23 + "name": "item_id: ItemId",
24 + "type_info": "Uuid"
25 + },
26 + {
27 + "ordinal": 4,
28 + "name": "amount_cents: Cents",
29 + "type_info": "Int4"
30 + },
31 + {
32 + "ordinal": 5,
33 + "name": "platform_fee_cents: Cents",
34 + "type_info": "Int4"
35 + },
36 + {
37 + "ordinal": 6,
38 + "name": "currency",
39 + "type_info": "Varchar"
40 + },
41 + {
42 + "ordinal": 7,
43 + "name": "status: super::TransactionStatus",
44 + "type_info": "Varchar"
45 + },
46 + {
47 + "ordinal": 8,
48 + "name": "stripe_payment_intent_id",
49 + "type_info": "Varchar"
50 + },
51 + {
52 + "ordinal": 9,
53 + "name": "stripe_checkout_session_id",
54 + "type_info": "Varchar"
55 + },
56 + {
57 + "ordinal": 10,
58 + "name": "created_at: chrono::DateTime<chrono::Utc>",
59 + "type_info": "Timestamptz"
60 + },
61 + {
62 + "ordinal": 11,
63 + "name": "completed_at: chrono::DateTime<chrono::Utc>",
64 + "type_info": "Timestamptz"
65 + },
66 + {
67 + "ordinal": 12,
68 + "name": "item_title",
69 + "type_info": "Varchar"
70 + },
71 + {
72 + "ordinal": 13,
73 + "name": "seller_username",
74 + "type_info": "Varchar"
75 + },
76 + {
77 + "ordinal": 14,
78 + "name": "share_contact",
79 + "type_info": "Bool"
80 + },
81 + {
82 + "ordinal": 15,
83 + "name": "project_id: ProjectId",
84 + "type_info": "Uuid"
85 + },
86 + {
87 + "ordinal": 16,
88 + "name": "parent_transaction_id: TransactionId",
89 + "type_info": "Uuid"
90 + },
91 + {
92 + "ordinal": 17,
93 + "name": "promo_code_id: PromoCodeId",
94 + "type_info": "Uuid"
95 + },
96 + {
97 + "ordinal": 18,
98 + "name": "guest_email",
99 + "type_info": "Varchar"
100 + },
101 + {
102 + "ordinal": 19,
103 + "name": "claim_token: ClaimToken",
104 + "type_info": "Uuid"
105 + },
106 + {
107 + "ordinal": 20,
108 + "name": "claimed_by: UserId",
109 + "type_info": "Uuid"
110 + },
111 + {
112 + "ordinal": 21,
113 + "name": "download_token: DownloadToken",
114 + "type_info": "Uuid"
115 + }
116 + ],
117 + "parameters": {
118 + "Left": [
119 + "Uuid",
120 + "Int8",
121 + "Int8"
122 + ]
123 + },
124 + "nullable": [
125 + false,
126 + true,
127 + true,
128 + true,
129 + false,
130 + false,
131 + false,
132 + false,
133 + true,
134 + true,
135 + false,
136 + true,
137 + true,
138 + true,
139 + false,
140 + true,
141 + true,
142 + true,
143 + true,
144 + true,
145 + true,
146 + true
147 + ]
148 + },
149 + "hash": "17188cab9291709226c6f4586bccfbbd878a4c98b27bb844e8b48c56edd7981a"
150 + }
A server/.sqlx/query-5013a9ea2c85fe65193f873cfc5d32bdccbedb7de2af30694a52c37df3c592e5.json +48
@@ -0,0 +1,48 @@
1 + {
2 + "db_name": "PostgreSQL",
3 + "query": "\n SELECT u.username, u.display_name, t.name AS tier_name,\n s.status AS \"status: super::SubscriptionStatus\",\n s.created_at AS \"created_at: chrono::DateTime<chrono::Utc>\"\n FROM subscriptions s\n JOIN users u ON u.id = s.subscriber_id\n JOIN subscription_tiers t ON t.id = s.tier_id\n WHERE s.project_id IN (SELECT id FROM projects WHERE user_id = $1)\n ORDER BY s.created_at DESC, s.id DESC\n LIMIT $2 OFFSET $3\n ",
4 + "describe": {
5 + "columns": [
6 + {
7 + "ordinal": 0,
8 + "name": "username",
9 + "type_info": "Varchar"
10 + },
11 + {
12 + "ordinal": 1,
13 + "name": "display_name",
14 + "type_info": "Varchar"
15 + },
16 + {
17 + "ordinal": 2,
18 + "name": "tier_name",
19 + "type_info": "Varchar"
20 + },
21 + {
22 + "ordinal": 3,
23 + "name": "status: super::SubscriptionStatus",
24 + "type_info": "Varchar"
25 + },
26 + {
27 + "ordinal": 4,
28 + "name": "created_at: chrono::DateTime<chrono::Utc>",
29 + "type_info": "Timestamptz"
30 + }
31 + ],
32 + "parameters": {
33 + "Left": [
34 + "Uuid",
35 + "Int8",
36 + "Int8"
37 + ]
38 + },
39 + "nullable": [
40 + false,
41 + true,
42 + false,
43 + false,
44 + false
45 + ]
46 + },
47 + "hash": "5013a9ea2c85fe65193f873cfc5d32bdccbedb7de2af30694a52c37df3c592e5"
48 + }
D server/.sqlx/query-92512050fa13a3ad75ae5e31b848ac1040a9f9c69eecdeef2e03adb90dcbfc69.json -70
@@ -1,70 +0,0 @@
1 - {
2 - "db_name": "PostgreSQL",
3 - "query": "\n SELECT p.title AS project_title, t.name AS tier_name, t.price_cents,\n u.username, s.status AS \"status: super::SubscriptionStatus\",\n s.current_period_start AS \"current_period_start: chrono::DateTime<chrono::Utc>\",\n s.current_period_end AS \"current_period_end: chrono::DateTime<chrono::Utc>\",\n s.canceled_at AS \"canceled_at: chrono::DateTime<chrono::Utc>\",\n s.created_at AS \"created_at: chrono::DateTime<chrono::Utc>\"\n FROM subscriptions s\n JOIN users u ON u.id = s.subscriber_id\n JOIN subscription_tiers t ON t.id = s.tier_id\n JOIN projects p ON p.id = s.project_id\n WHERE s.project_id IN (SELECT id FROM projects WHERE user_id = $1)\n ORDER BY s.created_at DESC\n ",
4 - "describe": {
5 - "columns": [
6 - {
7 - "ordinal": 0,
8 - "name": "project_title",
9 - "type_info": "Varchar"
10 - },
11 - {
12 - "ordinal": 1,
13 - "name": "tier_name",
14 - "type_info": "Varchar"
15 - },
16 - {
17 - "ordinal": 2,
18 - "name": "price_cents",
19 - "type_info": "Int4"
20 - },
21 - {
22 - "ordinal": 3,
23 - "name": "username",
24 - "type_info": "Varchar"
25 - },
26 - {
27 - "ordinal": 4,
28 - "name": "status: super::SubscriptionStatus",
29 - "type_info": "Varchar"
30 - },
31 - {
32 - "ordinal": 5,
33 - "name": "current_period_start: chrono::DateTime<chrono::Utc>",
34 - "type_info": "Timestamptz"
35 - },
36 - {
37 - "ordinal": 6,
38 - "name": "current_period_end: chrono::DateTime<chrono::Utc>",
39 - "type_info": "Timestamptz"
40 - },
41 - {
42 - "ordinal": 7,
43 - "name": "canceled_at: chrono::DateTime<chrono::Utc>",
44 - "type_info": "Timestamptz"
45 - },
46 - {
47 - "ordinal": 8,
48 - "name": "created_at: chrono::DateTime<chrono::Utc>",
49 - "type_info": "Timestamptz"
50 - }
51 - ],
52 - "parameters": {
53 - "Left": [
54 - "Uuid"
55 - ]
56 - },
57 - "nullable": [
58 - false,
59 - false,
60 - false,
61 - false,
62 - false,
63 - true,
64 - true,
65 - true,
66 - false
67 - ]
68 - },
69 - "hash": "92512050fa13a3ad75ae5e31b848ac1040a9f9c69eecdeef2e03adb90dcbfc69"
70 - }
D server/.sqlx/query-b0e71c186ef4f316174cd31b3903d03e801d8be57576766dbd3fa9a677979ca5.json -52
@@ -1,52 +0,0 @@
1 - {
2 - "db_name": "PostgreSQL",
3 - "query": "\n SELECT\n t.created_at AS \"created_at: chrono::DateTime<chrono::Utc>\",\n t.item_id AS \"item_id: ItemId\",\n t.item_title,\n t.amount_cents AS \"amount_cents: Cents\",\n t.status AS \"status: super::TransactionStatus\",\n CASE WHEN t.share_contact AND NOT EXISTS (\n SELECT 1 FROM contact_revocations cr\n WHERE cr.buyer_id = t.buyer_id AND cr.seller_id = t.seller_id\n ) THEN u.email ELSE NULL END as buyer_email\n FROM transactions t\n LEFT JOIN users u ON u.id = t.buyer_id\n WHERE t.seller_id = $1\n ORDER BY t.created_at DESC\n ",
4 - "describe": {
5 - "columns": [
6 - {
7 - "ordinal": 0,
8 - "name": "created_at: chrono::DateTime<chrono::Utc>",
9 - "type_info": "Timestamptz"
10 - },
11 - {
12 - "ordinal": 1,
13 - "name": "item_id: ItemId",
14 - "type_info": "Uuid"
15 - },
16 - {
17 - "ordinal": 2,
18 - "name": "item_title",
19 - "type_info": "Varchar"
20 - },
21 - {
22 - "ordinal": 3,
23 - "name": "amount_cents: Cents",
24 - "type_info": "Int4"
25 - },
26 - {
27 - "ordinal": 4,
28 - "name": "status: super::TransactionStatus",
29 - "type_info": "Varchar"
30 - },
31 - {
32 - "ordinal": 5,
33 - "name": "buyer_email",
34 - "type_info": "Varchar"
35 - }
36 - ],
37 - "parameters": {
38 - "Left": [
39 - "Uuid"
40 - ]
41 - },
42 - "nullable": [
43 - false,
44 - true,
45 - true,
46 - false,
47 - false,
48 - null
49 - ]
50 - },
51 - "hash": "b0e71c186ef4f316174cd31b3903d03e801d8be57576766dbd3fa9a677979ca5"
52 - }
D server/.sqlx/query-b5fa4c20ec1e09cb5a89fee1674d3ce34fb8016490fc5b286bb171a7d34f8b2c.json -46
@@ -1,46 +0,0 @@
1 - {
2 - "db_name": "PostgreSQL",
3 - "query": "\n SELECT u.username, u.display_name, t.name AS tier_name,\n s.status AS \"status: super::SubscriptionStatus\",\n s.created_at AS \"created_at: chrono::DateTime<chrono::Utc>\"\n FROM subscriptions s\n JOIN users u ON u.id = s.subscriber_id\n JOIN subscription_tiers t ON t.id = s.tier_id\n WHERE s.project_id IN (SELECT id FROM projects WHERE user_id = $1)\n ORDER BY s.created_at DESC\n ",
4 - "describe": {
5 - "columns": [
6 - {
7 - "ordinal": 0,
8 - "name": "username",
9 - "type_info": "Varchar"
10 - },
11 - {
12 - "ordinal": 1,
13 - "name": "display_name",
14 - "type_info": "Varchar"
15 - },
16 - {
17 - "ordinal": 2,
18 - "name": "tier_name",
19 - "type_info": "Varchar"
20 - },
21 - {
22 - "ordinal": 3,
23 - "name": "status: super::SubscriptionStatus",
24 - "type_info": "Varchar"
25 - },
26 - {
27 - "ordinal": 4,
28 - "name": "created_at: chrono::DateTime<chrono::Utc>",
29 - "type_info": "Timestamptz"
30 - }
31 - ],
32 - "parameters": {
33 - "Left": [
34 - "Uuid"
35 - ]
36 - },
37 - "nullable": [
38 - false,
39 - true,
40 - false,
41 - false,
42 - false
43 - ]
44 - },
45 - "hash": "b5fa4c20ec1e09cb5a89fee1674d3ce34fb8016490fc5b286bb171a7d34f8b2c"
46 - }
A server/.sqlx/query-c9b1c6374241b92f67625311ba06f033367de4cf3822ffae9ba04b3dbe3cb9e0.json +54
@@ -0,0 +1,54 @@
1 + {
2 + "db_name": "PostgreSQL",
3 + "query": "\n SELECT\n t.created_at AS \"created_at: chrono::DateTime<chrono::Utc>\",\n t.item_id AS \"item_id: ItemId\",\n t.item_title,\n t.amount_cents AS \"amount_cents: Cents\",\n t.status AS \"status: super::TransactionStatus\",\n CASE WHEN t.share_contact AND NOT EXISTS (\n SELECT 1 FROM contact_revocations cr\n WHERE cr.buyer_id = t.buyer_id AND cr.seller_id = t.seller_id\n ) THEN u.email ELSE NULL END as buyer_email\n FROM transactions t\n LEFT JOIN users u ON u.id = t.buyer_id\n WHERE t.seller_id = $1\n ORDER BY t.created_at DESC, t.id DESC\n LIMIT $2 OFFSET $3\n ",
4 + "describe": {
5 + "columns": [
6 + {
7 + "ordinal": 0,
8 + "name": "created_at: chrono::DateTime<chrono::Utc>",
9 + "type_info": "Timestamptz"
10 + },
11 + {
12 + "ordinal": 1,
13 + "name": "item_id: ItemId",
14 + "type_info": "Uuid"
15 + },
16 + {
17 + "ordinal": 2,
18 + "name": "item_title",
19 + "type_info": "Varchar"
20 + },
21 + {
22 + "ordinal": 3,
23 + "name": "amount_cents: Cents",
24 + "type_info": "Int4"
25 + },
26 + {
27 + "ordinal": 4,
28 + "name": "status: super::TransactionStatus",
29 + "type_info": "Varchar"
30 + },
31 + {
32 + "ordinal": 5,
33 + "name": "buyer_email",
34 + "type_info": "Varchar"
35 + }
36 + ],
37 + "parameters": {
38 + "Left": [
39 + "Uuid",
40 + "Int8",
41 + "Int8"
42 + ]
43 + },
44 + "nullable": [
45 + false,
46 + true,
47 + true,
48 + false,
49 + false,
50 + null
51 + ]
52 + },
53 + "hash": "c9b1c6374241b92f67625311ba06f033367de4cf3822ffae9ba04b3dbe3cb9e0"
54 + }
M server/src/db/follows.rs +13 -2
@@ -229,9 +229,17 @@ pub async fn get_followed_tag_ids(pool: &PgPool, follower_id: UserId) -> Result<
229 229 ///
230 230 /// Returns username, display_name, what they follow (user/project), and when.
231 231 #[tracing::instrument(skip_all)]
232 - pub async fn get_followers_for_export(
232 + /// One page of a creator's followers for CSV export, newest first.
233 + ///
234 + /// Paginated (`LIMIT`/`OFFSET`) so the export streams in bounded batches rather
235 + /// than materializing every follower in one query, and so the per-row email
236 + /// `EXISTS` reveal runs only for a bounded page at a time (ultra-fuzz Run 4 S1).
237 + /// Stable `(created_at, follower_id)` ordering keeps OFFSET batches consistent.
238 + pub async fn get_followers_for_export_page(
233 239 pool: &PgPool,
234 240 user_id: UserId,
241 + limit: i64,
242 + offset: i64,
235 243 ) -> Result<Vec<FollowerExportRow>> {
236 244 let rows = sqlx::query_as::<_, FollowerExportRow>(
237 245 r#"
@@ -255,10 +263,13 @@ pub async fn get_followers_for_export(
255 263 JOIN users u ON u.id = f.follower_id
256 264 WHERE (f.target_type = 'user' AND f.target_id = $1)
257 265 OR (f.target_type = 'project' AND f.target_id IN (SELECT id FROM projects WHERE user_id = $1))
258 - ORDER BY f.created_at DESC
266 + ORDER BY f.created_at DESC, f.follower_id DESC
267 + LIMIT $2 OFFSET $3
259 268 "#,
260 269 )
261 270 .bind(user_id)
271 + .bind(limit)
272 + .bind(offset)
262 273 .fetch_all(pool)
263 274 .await?;
264 275
M server/src/db/project_members.rs +12 -2
@@ -323,9 +323,16 @@ pub async fn count_splits_for_recipient(pool: &PgPool, recipient_id: UserId) ->
323 323 /// - The recipient (collaborator receiving a share), or
324 324 /// - The seller/tip recipient (owner who owes collaborators)
325 325 #[tracing::instrument(skip(pool))]
326 - pub async fn get_splits_for_export(
326 + /// One page of a user's revenue splits for CSV export, newest first.
327 + ///
328 + /// Paginated so the splits export streams in bounded batches instead of loading
329 + /// every split in one query (ultra-fuzz Run 4 S1). Stable `(created_at, id)`
330 + /// ordering keeps OFFSET batches consistent.
331 + pub async fn get_splits_for_export_page(
327 332 pool: &PgPool,
328 333 user_id: UserId,
334 + limit: i64,
335 + offset: i64,
329 336 ) -> Result<Vec<DbSplitExportRow>> {
330 337 let rows = sqlx::query_as::<_, DbSplitExportRow>(
331 338 r#"
@@ -338,10 +345,13 @@ pub async fn get_splits_for_export(
338 345 LEFT JOIN tips tip ON tip.id = rs.tip_id
339 346 WHERE rs.recipient_id = $1
340 347 OR COALESCE(t.seller_id, tip.recipient_id) = $1
341 - ORDER BY rs.created_at DESC
348 + ORDER BY rs.created_at DESC, rs.id DESC
349 + LIMIT $2 OFFSET $3
342 350 "#,
343 351 )
344 352 .bind(user_id)
353 + .bind(limit)
354 + .bind(offset)
345 355 .fetch_all(pool)
346 356 .await?;
347 357
M server/src/db/subscriptions.rs +20 -4
@@ -664,9 +664,14 @@ pub async fn get_project_subscriber_count(pool: &PgPool, project_id: ProjectId)
664 664 ///
665 665 /// Returns username, display_name, tier name, subscription status, and when.
666 666 #[tracing::instrument(skip_all)]
667 - pub async fn get_project_subscribers_for_export(
667 + /// One page of a creator's project subscribers for CSV export, newest first.
668 + /// Paginated for bounded-memory streaming (ultra-fuzz Run 4 S1); stable
669 + /// `(created_at, id)` ordering keeps OFFSET batches consistent.
670 + pub async fn get_project_subscribers_for_export_page(
668 671 pool: &PgPool,
669 672 user_id: UserId,
673 + limit: i64,
674 + offset: i64,
670 675 ) -> Result<Vec<SubscriberExportRow>> {
671 676 let rows = sqlx::query_as!(
672 677 SubscriberExportRow,
@@ -678,9 +683,12 @@ pub async fn get_project_subscribers_for_export(
678 683 JOIN users u ON u.id = s.subscriber_id
679 684 JOIN subscription_tiers t ON t.id = s.tier_id
680 685 WHERE s.project_id IN (SELECT id FROM projects WHERE user_id = $1)
681 - ORDER BY s.created_at DESC
686 + ORDER BY s.created_at DESC, s.id DESC
687 + LIMIT $2 OFFSET $3
682 688 "#,
683 689 user_id as UserId,
690 + limit,
691 + offset,
684 692 )
685 693 .fetch_all(pool)
686 694 .await?;
@@ -693,9 +701,14 @@ pub async fn get_project_subscribers_for_export(
693 701 /// Returns project name, tier name, price, subscriber username, status,
694 702 /// billing period dates, and cancellation date.
695 703 #[tracing::instrument(skip_all)]
696 - pub async fn get_subscriptions_for_export(
704 + /// One page of a creator's subscriptions for CSV export, newest first.
705 + /// Paginated for bounded-memory streaming (ultra-fuzz Run 4 S1); stable
706 + /// `(created_at, id)` ordering keeps OFFSET batches consistent.
707 + pub async fn get_subscriptions_for_export_page(
697 708 pool: &PgPool,
698 709 user_id: UserId,
710 + limit: i64,
711 + offset: i64,
699 712 ) -> Result<Vec<SubscriptionExportRow>> {
700 713 let rows = sqlx::query_as!(
701 714 SubscriptionExportRow,
@@ -711,9 +724,12 @@ pub async fn get_subscriptions_for_export(
711 724 JOIN subscription_tiers t ON t.id = s.tier_id
712 725 JOIN projects p ON p.id = s.project_id
713 726 WHERE s.project_id IN (SELECT id FROM projects WHERE user_id = $1)
714 - ORDER BY s.created_at DESC
727 + ORDER BY s.created_at DESC, s.id DESC
728 + LIMIT $2 OFFSET $3
715 729 "#,
716 730 user_id as UserId,
731 + limit,
732 + offset,
717 733 )
718 734 .fetch_all(pool)
719 735 .await?;
M server/src/db/transactions.rs +79 -2
@@ -359,6 +359,43 @@ pub async fn get_transactions_by_buyer(
359 359 Ok(txs)
360 360 }
361 361
362 + /// One page of a buyer's purchases for CSV export, newest first.
363 + ///
364 + /// Paginated so the purchases export streams in bounded batches rather than
365 + /// loading the buyer's whole history with `limit: None` (ultra-fuzz Run 4 S1).
366 + /// Stable `(created_at, id)` ordering keeps OFFSET batches consistent.
367 + pub async fn get_buyer_transactions_for_export_page(
368 + pool: &PgPool,
369 + buyer_id: UserId,
370 + limit: i64,
371 + offset: i64,
372 + ) -> Result<Vec<DbTransaction>> {
373 + let txs = sqlx::query_as!(
374 + DbTransaction,
375 + r#"
376 + SELECT
377 + id AS "id: TransactionId", buyer_id AS "buyer_id: UserId", seller_id AS "seller_id: UserId",
378 + item_id AS "item_id: ItemId", amount_cents AS "amount_cents: Cents", platform_fee_cents AS "platform_fee_cents: Cents",
379 + currency, status AS "status: super::TransactionStatus", stripe_payment_intent_id, stripe_checkout_session_id,
380 + created_at AS "created_at: chrono::DateTime<chrono::Utc>", completed_at AS "completed_at: chrono::DateTime<chrono::Utc>",
381 + item_title, seller_username, share_contact, project_id AS "project_id: ProjectId",
382 + parent_transaction_id AS "parent_transaction_id: TransactionId", promo_code_id AS "promo_code_id: PromoCodeId",
383 + guest_email, claim_token AS "claim_token: ClaimToken", claimed_by AS "claimed_by: UserId",
384 + download_token AS "download_token: DownloadToken"
385 + FROM transactions WHERE buyer_id = $1
386 + ORDER BY created_at DESC, id DESC
387 + LIMIT $2 OFFSET $3
388 + "#,
389 + buyer_id as UserId,
390 + limit,
391 + offset,
392 + )
393 + .fetch_all(pool)
394 + .await?;
395 +
396 + Ok(txs)
397 + }
398 +
362 399 /// List transactions where the user is the seller, newest first.
363 400 ///
364 401 /// Pass `limit: None` for all rows (exports), or `Some(n)` for dashboard display.
@@ -1100,9 +1137,19 @@ pub async fn get_seller_contacts(
1100 1137 /// Respects contact revocations: if a buyer revoked sharing, their email
1101 1138 /// is hidden even if `share_contact` was true on the transaction.
1102 1139 #[tracing::instrument(skip_all)]
1103 - pub async fn get_seller_transactions_for_export(
1140 + /// One page of a seller's sales for CSV export, newest first.
1141 + ///
1142 + /// Paginated (`LIMIT`/`OFFSET`) so the export streams in bounded batches instead
1143 + /// of loading the seller's entire transaction history into memory in one query
1144 + /// (ultra-fuzz Run 4 S1). The `(created_at, id)` ordering is stable so OFFSET
1145 + /// batches don't reorder. (Keyset pagination would avoid OFFSET's deep-scan cost
1146 + /// and is the future optimization; OFFSET is sufficient at current scale and
1147 + /// keeps peak memory + per-query result bounded, which is the DoS fix.)
1148 + pub async fn get_seller_transactions_for_export_page(
1104 1149 pool: &PgPool,
1105 1150 seller_id: UserId,
1151 + limit: i64,
1152 + offset: i64,
1106 1153 ) -> Result<Vec<DbTransactionExportRow>> {
1107 1154 let rows = sqlx::query_as!(
1108 1155 DbTransactionExportRow,
@@ -1120,9 +1167,12 @@ pub async fn get_seller_transactions_for_export(
1120 1167 FROM transactions t
1121 1168 LEFT JOIN users u ON u.id = t.buyer_id
1122 1169 WHERE t.seller_id = $1
1123 - ORDER BY t.created_at DESC
1170 + ORDER BY t.created_at DESC, t.id DESC
1171 + LIMIT $2 OFFSET $3
1124 1172 "#,
1125 1173 seller_id as UserId,
1174 + limit,
1175 + offset,
1126 1176 )
1127 1177 .fetch_all(pool)
1128 1178 .await?;
@@ -1130,6 +1180,33 @@ pub async fn get_seller_transactions_for_export(
1130 1180 Ok(rows)
1131 1181 }
1132 1182
1183 + /// All of a seller's export rows accumulated into one Vec, bounded to
1184 + /// `EXPORT_ACCUMULATE_CAP` rows. For admin / internal-API callers that need the
1185 + /// full set in memory; the public creator-facing export streams page-by-page via
1186 + /// [`get_seller_transactions_for_export_page`] instead of materializing here.
1187 + pub async fn get_seller_transactions_for_export(
1188 + pool: &PgPool,
1189 + seller_id: UserId,
1190 + ) -> Result<Vec<DbTransactionExportRow>> {
1191 + /// Page size for the accumulating fetch.
1192 + const PAGE: i64 = 5_000;
1193 + /// Cap so even an admin/internal export can't load an unbounded result set.
1194 + const EXPORT_ACCUMULATE_CAP: usize = 1_000_000;
1195 +
1196 + let mut all = Vec::new();
1197 + let mut offset = 0i64;
1198 + loop {
1199 + let page = get_seller_transactions_for_export_page(pool, seller_id, PAGE, offset).await?;
1200 + let n = page.len();
1201 + all.extend(page);
1202 + offset += n as i64;
1203 + if (n as i64) < PAGE || all.len() >= EXPORT_ACCUMULATE_CAP {
1204 + break;
1205 + }
1206 + }
1207 + Ok(all)
1208 + }
1209 +
1133 1210 /// Platform-wide revenue stats: total completed revenue, completed count, refunded count.
1134 1211 #[tracing::instrument(skip_all)]
1135 1212 pub async fn get_platform_revenue_stats(pool: &PgPool) -> Result<(i64, i64, i64)> {
M server/src/routes/api/exports/mod.rs +294 -160
@@ -5,10 +5,14 @@ mod content;
5 5 pub(super) use content::export_content;
6 6
7 7 use axum::{
8 + body::Body,
8 9 extract::State,
9 10 http::header::HeaderMap,
10 11 response::{IntoResponse, Response},
11 12 };
13 + use bytes::Bytes;
14 + use tokio::sync::mpsc;
15 + use tokio_stream::{wrappers::ReceiverStream, StreamExt};
12 16 use crate::{
13 17 auth::AuthUser,
14 18 db,
@@ -18,6 +22,90 @@ use crate::{
18 22 AppState,
19 23 };
20 24
25 + /// Rows per page for streamed CSV exports.
26 + const EXPORT_BATCH: i64 = 2_000;
27 + /// Hard ceiling on rows in one export, bounding worst-case OFFSET scan cost.
28 + const EXPORT_MAX_ROWS: usize = 1_000_000;
29 +
30 + /// Spawn a producer that pages a single export query and streams CSV chunks
31 + /// (header first) into a bounded channel. Peak memory is one batch and the DB
32 + /// connection is released between batches, so a huge export no longer loads the
33 + /// whole result set into one `String` from one unbounded query (ultra-fuzz Run 4
34 + /// S1). `page(limit, offset)` returns the formatted CSV for that page and its row
35 + /// count; a short page ends the stream. OFFSET pagination is sufficient at
36 + /// current scale; keyset is the future optimization (see the `_page` queries).
37 + fn spawn_paginated_csv<F, Fut>(header: &'static str, mut page: F) -> mpsc::Receiver<Bytes>
38 + where
39 + F: FnMut(i64, i64) -> Fut + Send + 'static,
40 + Fut: std::future::Future<Output = Result<(String, usize)>> + Send,
41 + {
42 + let (tx, rx) = mpsc::channel::<Bytes>(4);
43 + tokio::spawn(async move {
44 + if tx.send(Bytes::from_static(header.as_bytes())).await.is_err() {
45 + return;
46 + }
47 + let mut offset = 0i64;
48 + let mut total = 0usize;
49 + loop {
50 + let (chunk, n) = match page(EXPORT_BATCH, offset).await {
51 + Ok(p) => p,
52 + Err(e) => {
53 + tracing::error!(error = ?e, "csv export page failed mid-stream");
54 + break;
55 + }
56 + };
57 + if n > 0 && tx.send(Bytes::from(chunk)).await.is_err() {
58 + return; // client disconnected; stop producing
59 + }
60 + offset += n as i64;
61 + total += n;
62 + if (n as i64) < EXPORT_BATCH {
63 + break;
64 + }
65 + if total >= EXPORT_MAX_ROWS {
66 + let _ = tx
67 + .send(Bytes::from_static(
68 + b"# export truncated at row limit; contact support for a full export\n",
69 + ))
70 + .await;
71 + break;
72 + }
73 + }
74 + });
75 + rx
76 + }
77 +
78 + /// Turn a CSV chunk stream into a response. The non-HTMX path (the JS
79 + /// `fetch().then(r => r.blob())` download) gets a true streamed attachment with
80 + /// flat memory. The no-JS HTMX fallback can't stream a `data:` URI, so it
81 + /// materializes a bounded prefix and notes the truncation.
82 + async fn finish_csv(is_htmx: bool, filename: &str, mut rx: mpsc::Receiver<Bytes>) -> Result<Response> {
83 + if is_htmx {
84 + const HTMX_BYTE_CAP: usize = 4 * 1024 * 1024;
85 + let mut body = String::new();
86 + while let Some(chunk) = rx.recv().await {
87 + body.push_str(&String::from_utf8_lossy(&chunk));
88 + if body.len() >= HTMX_BYTE_CAP {
89 + body.push_str("\n# Export truncated. Enable JavaScript to download the full file.\n");
90 + break;
91 + }
92 + }
93 + // Dropping `rx` here stops the producer (its next send errors out).
94 + let data_uri = format!("data:text/csv;charset=utf-8,{}", urlencoding::encode(&body));
95 + return Ok(ExportDownloadTemplate {
96 + data_uri,
97 + filename: filename.to_string(),
98 + }
99 + .into_response());
100 + }
101 + let stream = ReceiverStream::new(rx).map(Ok::<Bytes, std::convert::Infallible>);
102 + Response::builder()
103 + .header("Content-Type", "text/csv")
104 + .header("Content-Disposition", format!("attachment; filename=\"{filename}\""))
105 + .body(Body::from_stream(stream))
106 + .context("build streaming export response")
107 + }
108 +
21 109 /// Return an inline error message for HTMX export requests instead of
22 110 /// letting the error propagate to the JSON error layer (which would swap
23 111 /// raw JSON text into the status div).
@@ -330,41 +418,40 @@ pub(super) async fn export_sales(
330 418 AuthUser(user): AuthUser,
331 419 ) -> Result<Response> {
332 420 let is_htmx = is_htmx_request(&headers);
333 -
334 - // Get all sales with conditional buyer email
335 - let transactions = db::transactions::get_seller_transactions_for_export(&state.db, user.id).await?;
336 -
337 - let mut csv_content = String::from("Date,Item ID,Item Title,Amount,Status,Buyer Email\n");
338 - for tx in &transactions {
339 - let item_title = tx.item_title.as_deref().unwrap_or("[Deleted]");
340 - let item_id_str = tx.item_id
341 - .map(|id| id.to_string())
342 - .unwrap_or_else(|| "[Deleted]".to_string());
343 - let buyer_email = tx.buyer_email.as_deref().unwrap_or("");
344 -
345 - csv_content.push_str(&format!(
346 - "{},{},{},{:.2},{},{}\n",
347 - tx.created_at.format("%Y-%m-%d %H:%M:%S"),
348 - item_id_str,
349 - sanitize_csv_cell(item_title),
350 - tx.amount_cents.as_f64() / 100.0,
351 - sanitize_csv_cell(&tx.status.to_string()),
352 - sanitize_csv_cell(buyer_email)
353 - ));
354 - }
355 -
356 - if is_htmx {
357 - let data_uri = format!(
358 - "data:text/csv;charset=utf-8,{}",
359 - urlencoding::encode(&csv_content)
360 - );
361 - return Ok(ExportDownloadTemplate {
362 - data_uri,
363 - filename: "makenot-work-sales.csv".to_string(),
364 - }.into_response());
365 - }
366 -
367 - download_response(csv_content.into_bytes(), "makenot-work-sales.csv", "text/csv")
421 + let pool = state.db.clone();
422 + let uid = user.id;
423 + let rx = spawn_paginated_csv(
424 + "Date,Item ID,Item Title,Amount,Status,Buyer Email\n",
425 + move |limit, offset| {
426 + let pool = pool.clone();
427 + async move {
428 + let rows = db::transactions::get_seller_transactions_for_export_page(
429 + &pool, uid, limit, offset,
430 + )
431 + .await?;
432 + let mut buf = String::new();
433 + for tx in &rows {
434 + let item_title = tx.item_title.as_deref().unwrap_or("[Deleted]");
435 + let item_id_str = tx
436 + .item_id
437 + .map(|id| id.to_string())
438 + .unwrap_or_else(|| "[Deleted]".to_string());
439 + let buyer_email = tx.buyer_email.as_deref().unwrap_or("");
440 + buf.push_str(&format!(
441 + "{},{},{},{:.2},{},{}\n",
442 + tx.created_at.format("%Y-%m-%d %H:%M:%S"),
443 + item_id_str,
444 + sanitize_csv_cell(item_title),
445 + tx.amount_cents.as_f64() / 100.0,
446 + sanitize_csv_cell(&tx.status.to_string()),
447 + sanitize_csv_cell(buyer_email)
448 + ));
449 + }
450 + Ok((buf, rows.len()))
451 + }
452 + },
453 + );
454 + finish_csv(is_htmx, "makenot-work-sales.csv", rx).await
368 455 }
369 456
370 457 /// Export revenue splits as a downloadable CSV file.
@@ -375,35 +462,33 @@ pub(super) async fn export_splits(
375 462 AuthUser(user): AuthUser,
376 463 ) -> Result<Response> {
377 464 let is_htmx = is_htmx_request(&headers);
378 -
379 - let splits = db::project_members::get_splits_for_export(&state.db, user.id).await?;
380 -
381 - let mut csv_content = String::from("Date,Type,Direction,Recipient,Amount,Split %\n");
382 - for split in &splits {
383 - let direction = if split.recipient_id == user.id { "incoming" } else { "outgoing" };
384 - csv_content.push_str(&format!(
385 - "{},{},{},{},{:.2},{}\n",
386 - split.created_at.format("%Y-%m-%d %H:%M:%S"),
387 - sanitize_csv_cell(&split.source_type),
388 - direction,
389 - sanitize_csv_cell(&split.recipient_username),
390 - split.amount_cents.as_f64() / 100.0,
391 - split.split_percent,
392 - ));
393 - }
394 -
395 - if is_htmx {
396 - let data_uri = format!(
397 - "data:text/csv;charset=utf-8,{}",
398 - urlencoding::encode(&csv_content)
399 - );
400 - return Ok(ExportDownloadTemplate {
401 - data_uri,
402 - filename: "makenot-work-splits.csv".to_string(),
403 - }.into_response());
404 - }
405 -
406 - download_response(csv_content.into_bytes(), "makenot-work-splits.csv", "text/csv")
465 + let pool = state.db.clone();
466 + let uid = user.id;
467 + let rx = spawn_paginated_csv(
468 + "Date,Type,Direction,Recipient,Amount,Split %\n",
469 + move |limit, offset| {
470 + let pool = pool.clone();
471 + async move {
472 + let splits =
473 + db::project_members::get_splits_for_export_page(&pool, uid, limit, offset).await?;
474 + let mut buf = String::new();
475 + for split in &splits {
476 + let direction = if split.recipient_id == uid { "incoming" } else { "outgoing" };
477 + buf.push_str(&format!(
478 + "{},{},{},{},{:.2},{}\n",
479 + split.created_at.format("%Y-%m-%d %H:%M:%S"),
480 + sanitize_csv_cell(&split.source_type),
481 + direction,
482 + sanitize_csv_cell(&split.recipient_username),
483 + split.amount_cents.as_f64() / 100.0,
484 + split.split_percent,
485 + ));
486 + }
487 + Ok((buf, splits.len()))
488 + }
489 + },
490 + );
491 + finish_csv(is_htmx, "makenot-work-splits.csv", rx).await
407 492 }
408 493
409 494 /// Export all purchase transactions as a downloadable CSV file.
@@ -414,57 +499,56 @@ pub(super) async fn export_purchases(
414 499 AuthUser(user): AuthUser,
415 500 ) -> Result<Response> {
416 501 let is_htmx = is_htmx_request(&headers);
417 -
418 - // Get all purchases (transactions where user is the buyer)
419 - let transactions = db::transactions::get_transactions_by_buyer(&state.db, user.id, None).await?;
420 -
421 - // Batch-fetch titles for transactions missing the denormalized item_title
422 - let missing_title_ids: Vec<db::ItemId> = transactions.iter()
423 - .filter(|tx| tx.item_title.is_none())
424 - .filter_map(|tx| tx.item_id)
425 - .collect();
426 - let title_lookup: std::collections::HashMap<db::ItemId, String> =
427 - db::items::get_item_titles_batch(&state.db, &missing_title_ids)
428 - .await?
429 - .into_iter()
430 - .collect();
431 -
432 - let mut csv_content = String::from("Date,Item ID,Item Title,Amount,Status\n");
433 - for tx in &transactions {
434 - let item_title = if let Some(title) = &tx.item_title {
435 - title.clone()
436 - } else if let Some(item_id) = tx.item_id {
437 - title_lookup.get(&item_id).cloned().unwrap_or_else(|| "[Deleted]".to_string())
438 - } else {
439 - "[Deleted]".to_string()
440 - };
441 -
442 - let item_id_str = tx.item_id
443 - .map(|id| id.to_string())
444 - .unwrap_or_else(|| "[Deleted]".to_string());
445 -
446 - csv_content.push_str(&format!(
447 - "{},{},{},{:.2},{}\n",
448 - tx.created_at.format("%Y-%m-%d %H:%M:%S"),
449 - item_id_str,
450 - sanitize_csv_cell(&item_title),
451 - tx.amount_cents.as_f64() / 100.0,
452 - sanitize_csv_cell(&tx.status.to_string())
453 - ));
454 - }
455 -
456 - if is_htmx {
457 - let data_uri = format!(
458 - "data:text/csv;charset=utf-8,{}",
459 - urlencoding::encode(&csv_content)
460 - );
461 - return Ok(ExportDownloadTemplate {
462 - data_uri,
463 - filename: "makenot-work-purchases.csv".to_string(),
464 - }.into_response());
465 - }
466 -
467 - download_response(csv_content.into_bytes(), "makenot-work-purchases.csv", "text/csv")
502 + let pool = state.db.clone();
503 + let uid = user.id;
504 + let rx = spawn_paginated_csv(
505 + "Date,Item ID,Item Title,Amount,Status\n",
506 + move |limit, offset| {
507 + let pool = pool.clone();
508 + async move {
509 + let transactions =
510 + db::transactions::get_buyer_transactions_for_export_page(&pool, uid, limit, offset)
511 + .await?;
512 + // Batch-fetch titles only for this page's transactions missing
513 + // the denormalized item_title.
514 + let missing_title_ids: Vec<db::ItemId> = transactions
515 + .iter()
516 + .filter(|tx| tx.item_title.is_none())
517 + .filter_map(|tx| tx.item_id)
518 + .collect();
519 + let title_lookup: std::collections::HashMap<db::ItemId, String> =
520 + db::items::get_item_titles_batch(&pool, &missing_title_ids)
521 + .await?
522 + .into_iter()
523 + .collect();
524 +
525 + let mut buf = String::new();
526 + for tx in &transactions {
527 + let item_title = if let Some(title) = &tx.item_title {
528 + title.clone()
529 + } else if let Some(item_id) = tx.item_id {
530 + title_lookup.get(&item_id).cloned().unwrap_or_else(|| "[Deleted]".to_string())
531 + } else {
532 + "[Deleted]".to_string()
533 + };
534 + let item_id_str = tx
535 + .item_id
536 + .map(|id| id.to_string())
537 + .unwrap_or_else(|| "[Deleted]".to_string());
538 + buf.push_str(&format!(
539 + "{},{},{},{:.2},{}\n",
540 + tx.created_at.format("%Y-%m-%d %H:%M:%S"),
541 + item_id_str,
542 + sanitize_csv_cell(&item_title),
543 + tx.amount_cents.as_f64() / 100.0,
544 + sanitize_csv_cell(&tx.status.to_string())
545 + ));
546 + }
547 + Ok((buf, transactions.len()))
548 + }
549 + },
550 + );
551 + finish_csv(is_htmx, "makenot-work-purchases.csv", rx).await
468 552 }
469 553
470 554 /// Export followers and subscribers as a downloadable CSV file.
@@ -475,48 +559,76 @@ pub(super) async fn export_followers(
475 559 AuthUser(user): AuthUser,
476 560 ) -> Result<Response> {
477 561 let is_htmx = is_htmx_request(&headers);
562 + let pool = state.db.clone();
563 + let uid = user.id;
564 +
565 + // Two-section CSV (followers, then subscribers); each section pages
566 + // independently so the whole thing streams in bounded batches (Run 4 S1).
567 + let (tx, rx) = mpsc::channel::<Bytes>(4);
568 + tokio::spawn(async move {
569 + if tx
570 + .send(Bytes::from_static(b"Section,Username,Display Name,Email,Type,Status,Since\n"))
571 + .await
572 + .is_err()
573 + {
574 + return;
575 + }
478 576
479 - let followers = db::follows::get_followers_for_export(&state.db, user.id).await?;
480 - let subscribers = db::subscriptions::get_project_subscribers_for_export(&state.db, user.id).await?;
481 -
482 - let mut csv_content = String::from("Section,Username,Display Name,Email,Type,Status,Since\n");
483 -
484 - for f in &followers {
485 - csv_content.push_str(&format!(
486 - "Follower,{},{},{},{},{},{}\n",
487 - sanitize_csv_cell(&f.username),
488 - sanitize_csv_cell(f.display_name.as_deref().unwrap_or("")),
489 - sanitize_csv_cell(f.email.as_deref().unwrap_or("")),
490 - f.target_type,
491 - "",
492 - f.created_at.format("%Y-%m-%d %H:%M:%S"),
493 - ));
494 - }
495 -
496 - for s in &subscribers {
497 - csv_content.push_str(&format!(
498 - "Subscriber,{},{},{},{},{},{}\n",
499 - sanitize_csv_cell(&s.username),
500 - sanitize_csv_cell(s.display_name.as_deref().unwrap_or("")),
501 - "",
502 - sanitize_csv_cell(&s.tier_name),
503 - s.status,
504 - s.created_at.format("%Y-%m-%d %H:%M:%S"),
505 - ));
506 - }
507 -
508 - if is_htmx {
509 - let data_uri = format!(
510 - "data:text/csv;charset=utf-8,{}",
511 - urlencoding::encode(&csv_content)
512 - );
513 - return Ok(ExportDownloadTemplate {
514 - data_uri,
515 - filename: "makenot-work-followers.csv".to_string(),
516 - }.into_response());
517 - }
577 + let mut offset = 0i64;
578 + let mut total = 0usize;
579 + loop {
580 + let rows = match db::follows::get_followers_for_export_page(&pool, uid, EXPORT_BATCH, offset).await {
581 + Ok(r) => r,
582 + Err(e) => { tracing::error!(error = ?e, "followers export page failed"); break; }
583 + };
584 + if !rows.is_empty() {
585 + let mut buf = String::new();
586 + for f in &rows {
587 + buf.push_str(&format!(
588 + "Follower,{},{},{},{},{},{}\n",
589 + sanitize_csv_cell(&f.username),
590 + sanitize_csv_cell(f.display_name.as_deref().unwrap_or("")),
591 + sanitize_csv_cell(f.email.as_deref().unwrap_or("")),
592 + f.target_type,
593 + "",
594 + f.created_at.format("%Y-%m-%d %H:%M:%S"),
595 + ));
596 + }
597 + if tx.send(Bytes::from(buf)).await.is_err() { return; }
598 + }
599 + offset += rows.len() as i64;
600 + total += rows.len();
601 + if (rows.len() as i64) < EXPORT_BATCH || total >= EXPORT_MAX_ROWS { break; }
602 + }
518 603
519 - download_response(csv_content.into_bytes(), "makenot-work-followers.csv", "text/csv")
604 + let mut offset = 0i64;
605 + let mut total = 0usize;
606 + loop {
607 + let rows = match db::subscriptions::get_project_subscribers_for_export_page(&pool, uid, EXPORT_BATCH, offset).await {
608 + Ok(r) => r,
609 + Err(e) => { tracing::error!(error = ?e, "subscribers export page failed"); break; }
610 + };
611 + if !rows.is_empty() {
612 + let mut buf = String::new();
613 + for s in &rows {
614 + buf.push_str(&format!(
615 + "Subscriber,{},{},{},{},{},{}\n",
616 + sanitize_csv_cell(&s.username),
617 + sanitize_csv_cell(s.display_name.as_deref().unwrap_or("")),
618 + "",
619 + sanitize_csv_cell(&s.tier_name),
620 + s.status,
621 + s.created_at.format("%Y-%m-%d %H:%M:%S"),
622 + ));
623 + }
624 + if tx.send(Bytes::from(buf)).await.is_err() { return; }
625 + }
626 + offset += rows.len() as i64;
627 + total += rows.len();
628 + if (rows.len() as i64) < EXPORT_BATCH || total >= EXPORT_MAX_ROWS { break; }
629 + }
630 + });
631 + finish_csv(is_htmx, "makenot-work-followers.csv", rx).await
520 632 }
521 633
522 634 /// Export subscriptions as a downloadable CSV file with full detail.
@@ -527,47 +639,40 @@ pub(super) async fn export_subscriptions(
527 639 AuthUser(user): AuthUser,
528 640 ) -> Result<Response> {
529 641 let is_htmx = is_htmx_request(&headers);
530 -
531 - let subscriptions =
532 - db::subscriptions::get_subscriptions_for_export(&state.db, user.id).await?;
533 -
534 - let mut csv_content = String::from(
642 + let pool = state.db.clone();
643 + let uid = user.id;
644 + let rx = spawn_paginated_csv(
535 645 "Project,Tier,Price,Username,Status,Period Start,Period End,Canceled At,Created At\n",
646 + move |limit, offset| {
647 + let pool = pool.clone();
648 + async move {
649 + let subscriptions = db::subscriptions::get_subscriptions_for_export_page(
650 + &pool, uid, limit, offset,
651 + )
652 + .await?;
653 + let fmt_opt = |dt: Option<chrono::DateTime<chrono::Utc>>| -> String {
654 + dt.map(|d| d.format("%Y-%m-%d %H:%M:%S").to_string()).unwrap_or_default()
655 + };
656 + let mut buf = String::new();
657 + for s in &subscriptions {
658 + buf.push_str(&format!(
659 + "{},{},{},{},{},{},{},{},{}\n",
660 + sanitize_csv_cell(&s.project_title),
661 + sanitize_csv_cell(&s.tier_name),
662 + crate::formatting::format_dollars_plain(s.price_cents),
663 + sanitize_csv_cell(&s.username),
664 + sanitize_csv_cell(&s.status.to_string()),
665 + fmt_opt(s.current_period_start),
666 + fmt_opt(s.current_period_end),
667 + fmt_opt(s.canceled_at),
668 + s.created_at.format("%Y-%m-%d %H:%M:%S"),
669 + ));
Lines truncated
M server/src/routes/api/projects.rs +3 -3
@@ -575,10 +575,10 @@ pub(super) async fn update_repo_visibility(
575 575 ) -> Result<impl IntoResponse> {
576 576 user.check_not_suspended()?;
577 577
578 - let repo = db::git_repos::get_repos_by_user(&state.db, user.id)
578 + // Indexed lookup by primary key, not fetch-all-then-find over the user's
579 + // whole repo set (ultra-fuzz Run 4 Perf). Ownership is still enforced below.
580 + let repo = db::git_repos::get_repo_by_id(&state.db, repo_id)
579 581 .await?
580 - .into_iter()
581 - .find(|r| r.id == repo_id)
582 582 .ok_or(AppError::NotFound)?;
583 583
584 584 if repo.user_id != user.id {
M server/src/routes/pages/public/health/mod.rs +29 -28
@@ -237,34 +237,35 @@ async fn collect_health(state: &AppState) -> HealthData {
237 237 }
238 238 }
239 239
240 - // Database health checks
241 - let db_test_users = run_test("Count users", || async {
242 - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM users")
243 - .fetch_one(&state.db)
244 - .await
245 - .is_ok()
246 - }).await;
247 -
248 - let db_test_projects = run_test("Count projects", || async {
249 - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM projects")
250 - .fetch_one(&state.db)
251 - .await
252 - .is_ok()
253 - }).await;
254 -
255 - let db_test_items = run_test("Count items", || async {
256 - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM items")
257 - .fetch_one(&state.db)
258 - .await
259 - .is_ok()
260 - }).await;
261 -
262 - let db_test_transactions = run_test("Count transactions", || async {
263 - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM transactions")
264 - .fetch_one(&state.db)
265 - .await
266 - .is_ok()
267 - }).await;
240 + // Database health probes. Run concurrently (each still records its own
241 + // latency) instead of serially, so the dashboard's worst-case wait is the
242 + // slowest single probe, not their sum (ultra-fuzz Run 4 Perf).
243 + let (db_test_users, db_test_projects, db_test_items, db_test_transactions) = tokio::join!(
244 + run_test("Count users", || async {
245 + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM users")
246 + .fetch_one(&state.db)
247 + .await
248 + .is_ok()
249 + }),
250 + run_test("Count projects", || async {
251 + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM projects")
252 + .fetch_one(&state.db)
253 + .await
254 + .is_ok()
255 + }),
256 + run_test("Count items", || async {
257 + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM items")
258 + .fetch_one(&state.db)
259 + .await
260 + .is_ok()
261 + }),
262 + run_test("Count transactions", || async {
263 + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM transactions")
264 + .fetch_one(&state.db)
265 + .await
266 + .is_ok()
267 + }),
268 + );
268 269
269 270 // Get actual counts
270 271 let stats = db::health::get_health_stats(&state.db).await.unwrap_or(db::health::DbHealthStats {
M server/src/routes/synckit/sync.rs +3 -3
@@ -683,9 +683,9 @@ pub(super) async fn begin_rotation(
683 683 ));
684 684 }
685 685
686 - // Verify device belongs to this user + app
687 - let devices = db::synckit::get_sync_devices(&state.db, sync_user.app_id, sync_user.user_id).await?;
688 - if !devices.iter().any(|d| d.id == req.device_id) {
686 + // Verify device belongs to this user + app via an indexed point lookup, not a
687 + // fetch-all-then-scan (ultra-fuzz Run 4 Perf).
688 + if !db::synckit::sync_device_belongs(&state.db, req.device_id, sync_user.app_id, sync_user.user_id).await? {
689 689 return Err(AppError::BadRequest("Unknown device".to_string()));
690 690 }
691 691
M server/src/scanning/mod.rs +10 -1
@@ -457,7 +457,16 @@ impl ScanPipeline {
457 457 let urlhaus_key = self.abuse_ch_auth_key.clone();
458 458 let urlhaus_fut = async move {
459 459 if urlhaus_enabled {
460 - urlhaus::check_urlhaus(&urlhaus_map, urlhaus_key.as_deref()).await
460 + // Host extraction walks the memory-mapped buffer and can
461 + // page-fault; run it on the blocking pool so it doesn't stall a
462 + // tokio worker, then do only the network lookups async
463 + // (ultra-fuzz Run 4 Perf S2).
464 + let hosts = tokio::task::spawn_blocking(move || {
465 + urlhaus::extract_unique_hosts(&urlhaus_map[..], urlhaus::MAX_HOSTS_PER_FILE)
466 + })
467 + .await
468 + .unwrap_or_default();
469 + urlhaus::check_urlhaus_hosts(hosts, urlhaus_key.as_deref()).await
461 470 } else {
462 471 LayerResult {
463 472 layer: "urlhaus",
M server/src/scanning/spool.rs +13 -4
@@ -9,7 +9,10 @@
9 9 //! Nothing in the scan pipeline calls this module yet; it is wired up in
10 10 //! later chunks of the scanner-streaming refactor.
11 11
12 - use crate::constants::{SCAN_SPOOL_FREE_RESERVE_BYTES, SCAN_SPOOL_MAX_BYTES, SCAN_SPOOL_ORPHAN_AGE_SECS};
12 + use crate::constants::{
13 + SCAN_SPOOL_FREE_RESERVE_BYTES, SCAN_SPOOL_MAX_BYTES, SCAN_SPOOL_ORPHAN_AGE_SECS,
14 + SCAN_WORKER_COUNT,
15 + };
13 16 use s3_storage::ByteStream;
14 17 use std::path::{Path, PathBuf};
15 18 use tokio::fs::{File, OpenOptions};
@@ -74,10 +77,16 @@ pub fn check_free_space(spool_dir: &Path, expected_size: u64) -> Result<(), Stri
74 77 };
75 78 let free = fs2::available_space(probe)
76 79 .map_err(|e| format!("statvfs {}: {e}", probe.display()))?;
77 - if free.saturating_sub(expected_size) < SCAN_SPOOL_FREE_RESERVE_BYTES {
80 + // The check and the write aren't atomic: with `SCAN_WORKER_COUNT` workers, up
81 + // to that many spool downloads can pass this check and then write at once.
82 + // Reserve headroom for ALL of them rather than just this one, so concurrent
83 + // workers can't collectively overrun the free-space floor (ultra-fuzz Run 4
84 + // Perf TOCTOU). Conservative by at most `(workers-1) * expected`.
85 + let concurrent_reserve = expected_size.saturating_mul(SCAN_WORKER_COUNT as u64);
86 + if free.saturating_sub(concurrent_reserve) < SCAN_SPOOL_FREE_RESERVE_BYTES {
78 87 return Err(format!(
79 - "scan spool volume short on space: free={} expected={} reserve={}",
80 - free, expected_size, SCAN_SPOOL_FREE_RESERVE_BYTES
88 + "scan spool volume short on space: free={} expected={} workers={} reserve={}",
89 + free, expected_size, SCAN_WORKER_COUNT, SCAN_SPOOL_FREE_RESERVE_BYTES
81 90 ));
82 91 }
83 92 Ok(())
M server/src/scanning/urlhaus.rs +21 -3
@@ -23,7 +23,7 @@ pub const ERROR_POLICY: ErrorPolicy = ErrorPolicy::FailOpen;
23 23
24 24 const URLHAUS_API_URL: &str = "https://urlhaus-api.abuse.ch/v1/host/";
25 25 /// Cap per-upload host lookups to keep within free-tier quotas and bound work.
26 - const MAX_HOSTS_PER_FILE: usize = 5;
26 + pub(crate) const MAX_HOSTS_PER_FILE: usize = 5;
27 27 /// Cap the byte window we scan for URLs; URLs in the wild fit easily here.
28 28 const MAX_SCAN_BYTES: usize = 4 * 1024 * 1024;
29 29 /// Minimum printable-ASCII run length to consider as a potential string.
@@ -31,6 +31,25 @@ const MIN_RUN_LEN: usize = 6;
31 31
32 32 /// Check the file for URLs that appear in URLhaus's known-bad index.
33 33 pub async fn check_urlhaus(data: &[u8], auth_key: Option<&str>) -> LayerResult {
34 + if auth_key.is_none() {
35 + return LayerResult {
36 + layer: "urlhaus",
37 + verdict: LayerVerdict::Skip,
38 + detail: Some("No abuse.ch Auth-Key configured".to_string()),
39 + };
40 + }
41 + // In-memory path (small uploads, tests): extraction is cheap on a heap
42 + // buffer. The large-file mmap path extracts inside `spawn_blocking` and calls
43 + // `check_urlhaus_hosts` directly (ultra-fuzz Run 4 Perf S2).
44 + let hosts = extract_unique_hosts(data, MAX_HOSTS_PER_FILE);
45 + check_urlhaus_hosts(hosts, auth_key).await
46 + }
47 +
48 + /// URLhaus lookup over already-extracted hosts. Split from [`check_urlhaus`] so
49 + /// the mmap scan path can run the page-fault-prone host extraction off the async
50 + /// runtime (in `spawn_blocking`) and pass the result here, leaving only the
51 + /// network lookups on the runtime.
52 + pub(crate) async fn check_urlhaus_hosts(hosts: Vec<String>, auth_key: Option<&str>) -> LayerResult {
34 53 let Some(key) = auth_key else {
35 54 return LayerResult {
36 55 layer: "urlhaus",
@@ -39,7 +58,6 @@ pub async fn check_urlhaus(data: &[u8], auth_key: Option<&str>) -> LayerResult {
39 58 };
40 59 };
41 60
42 - let hosts = extract_unique_hosts(data, MAX_HOSTS_PER_FILE);
43 61 if hosts.is_empty() {
44 62 return LayerResult {
45 63 layer: "urlhaus",
@@ -160,7 +178,7 @@ fn classify_urlhaus_response(body: &serde_json::Value) -> HostVerdict {
160 178
161 179 /// Pull printable-ASCII URL hosts out of the byte buffer. Cap at `max` unique
162 180 /// hosts to bound per-upload work and free-tier quota use.
163 - fn extract_unique_hosts(data: &[u8], max: usize) -> Vec<String> {
181 + pub(crate) fn extract_unique_hosts(data: &[u8], max: usize) -> Vec<String> {
164 182 let scan = if data.len() > MAX_SCAN_BYTES { &data[..MAX_SCAN_BYTES] } else { data };
165 183
166 184 let mut hosts: HashSet<String> = HashSet::new();
M server/src/scheduler/cleanup.rs +7
@@ -309,6 +309,13 @@ pub(super) async fn scrub_stale_ip_addresses(state: &AppState) {
309 309 /// Permanently delete items that were soft-deleted more than 7 days ago.
310 310 /// Cleans up S3 objects (item files + version files) and decrements storage
311 311 /// before DB deletion to prevent orphaned storage and accounting drift.
312 + ///
313 + /// Accepted residual (ultra-fuzz Run 4 Perf, decision 2026-06-23): this gathers
314 + /// all expired-item S3 keys into one Vec before the batch delete. It is a daily
315 + /// cron off the request path; steady-state (7-day window) is small, so the only
316 + /// large allocation is transient, after a rare mass-delete event. Bounding it
317 + /// per-tick means coupling a LIMIT on the key-gather to the same slice as the
318 + /// CASCADE delete; deferred as low-value for a cron path. Revisit if it matters.
312 319 pub(super) async fn purge_expired_deleted_items(state: &AppState) {
313 320 // Collect S3 keys from items AND their versions before CASCADE delete destroys the data
314 321 let mut all_s3_keys: Vec<(String, String)> = Vec::new();
M server/src/scheduler/mod.rs +8
@@ -96,6 +96,14 @@ pub fn spawn_scheduler(
96 96 // Pin advisory lock to a dedicated connection held for the entire tick.
97 97 // pg_try_advisory_lock is session-scoped — holding the connection prevents
98 98 // another instance from acquiring the lock until this tick completes.
99 + //
100 + // Accepted residual (ultra-fuzz Run 4 Perf, decision 2026-06-23): this
101 + // borrows 1 of the request pool's connections for the tick's duration
102 + // (effective serving capacity 24/25 during a tick). The architecturally
103 + // cleaner shape is a dedicated 1-connection side-pool for the lock, but
104 + // that change touches the deploy-critical scheduler's connection
105 + // lifecycle for a small steady-state gain; revisit if pool-pressure
106 + // metrics show it matters.
99 107 let mut lock_conn = match state.db.acquire().await {
100 108 Ok(conn) => conn,
101 109 Err(e) => {