Skip to main content

max / makenotwork

Background Stripe fan-outs and close perf cliffs (ultra-fuzz Run 6 Performance) S1/S2/R6-Perf-M1: extract payments::fan_ops::spawn_fan_sub_fanout, a single helper that runs a creator's per-fan-subscription Stripe fan-out on the bounded background queue. Wire it into creator pause, webhook re-subscribe unpause, and admin suspend/unsuspend/terminate so none of those hot paths block on N serial Stripe round-trips (the webhook one no longer risks a timeout/retry storm). S3: cap YARA input at SCAN_YARA_MAX_BYTES (512 MiB) so a large object no longer faults its whole mmap resident per scan; SHA-256 and ClamAV still see the full file. S4: stop the decompression-bomb walk at min(absolute, compressed*MAX_RATIO) instead of always expanding to the 2 GB absolute cap (single-stream and per-entry paths); the ratio verdict fires identically on the early count. S6: batch the git push-refs issue writes into db::issues::create_comments + update_issue_statuses (two queries) instead of one UPDATE + INSERT per referenced commit. R6-Perf-M2: stream export_contacts through spawn_paginated_csv like its sibling exports. R6-Perf-M3: bound the scan spool write at claimed size + slack, not the global 8 GiB ceiling. R6-Perf-M4: creator_stats counts projects/items with COUNT(*) rather than fetching rows to .len(). R6-Perf-M5: suggest_slugs resolves availability with one slug = ANY($1) query instead of up to 9 point queries.
Co-Authored-By
Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-24 15:39 UTC
Signed with PGP, not checked
Commit: a57614cdc8cb85c4e5ae5bb5844fccfc73c1524c
Parent: f2dec50
17 files changed, +458 insertions, -145 deletions
M server/src/constants.rs +13
@@ -221,6 +221,19 @@
221 221 /// pending object. The scanner refuses if `statvfs(free) - expected_size`
222 222 /// would drop below this threshold.
223 223 pub const SCAN_SPOOL_FREE_RESERVE_BYTES: u64 = 2 * 1024 * 1024 * 1024;
224 + /// Slack added to a scan job's claimed object size when bounding how many bytes
225 + /// the spool writer will accept. The S3 object size is authoritative, but a
226 + /// small margin absorbs benign content-length/multipart rounding without
227 + /// letting an under-reported object stream the full `SCAN_SPOOL_MAX_BYTES` to
228 + /// scratch before the writer aborts.
229 + pub const SCAN_SPOOL_SLACK_BYTES: u64 = 16 * 1024 * 1024; // 16 MiB
230 + /// Maximum number of bytes fed to YARA in a single scan. yara-x's `Scanner`
231 + /// walks the whole slice, which demand-pages the entire mmap resident — so an
232 + /// 8 GiB object would otherwise pin 8 GiB of page cache per scan (×
233 + /// `SCAN_MAX_CONCURRENT`). Malware signatures cluster near a file's start, and
234 + /// ClamAV (streamed, uncapped) is the full-file backstop, so scanning a generous
235 + /// prefix is the right trade. Above this, YARA sees the prefix and logs the cap.
236 + pub const SCAN_YARA_MAX_BYTES: usize = 512 * 1024 * 1024; // 512 MiB
224 237
225 238 // -- Caddy on-demand TLS --
226 239 // Caps concurrent cache-miss DB lookups in `/api/domains/caddy-ask`. Cache hits
M server/src/db/issues.rs +73
@@ -257,6 +257,79 @@
257 257 Ok(comment)
258 258 }
259 259
260 + /// Bulk-insert issue comments in a single statement, then touch each affected
261 + /// issue's `updated_at` once. The git-push processor would otherwise issue one
262 + /// INSERT (and one updated_at UPDATE) per referenced commit, ~100-150 sequential
263 + /// writes per push (ultra-fuzz Run 6 S6). All comments share one author.
264 + #[tracing::instrument(skip_all)]
265 + pub async fn create_comments(
266 + pool: &PgPool,
267 + author_id: UserId,
268 + comments: &[(IssueId, String, String)],
269 + ) -> Result<()> {
270 + if comments.is_empty() {
271 + return Ok(());
272 + }
273 + let issue_ids: Vec<sqlx::types::Uuid> =
274 + comments.iter().map(|(id, _, _)| *id.as_uuid()).collect();
275 + let bodies_md: Vec<&str> = comments.iter().map(|(_, md, _)| md.as_str()).collect();
276 + let bodies_html: Vec<&str> = comments.iter().map(|(_, _, html)| html.as_str()).collect();
277 +
278 + sqlx::query(
279 + r#"
280 + INSERT INTO issue_comments (issue_id, author_user_id, body_markdown, body_html)
281 + SELECT u.issue_id, $1, u.body_md, u.body_html
282 + FROM UNNEST($2::uuid[], $3::text[], $4::text[]) AS u(issue_id, body_md, body_html)
283 + "#,
284 + )
285 + .bind(author_id)
286 + .bind(&issue_ids)
287 + .bind(&bodies_md)
288 + .bind(&bodies_html)
289 + .execute(pool)
290 + .await?;
291 +
292 + // Touch updated_at once per distinct affected issue.
293 + let mut distinct = issue_ids;
294 + distinct.sort_unstable();
295 + distinct.dedup();
296 + sqlx::query("UPDATE issues SET updated_at = NOW() WHERE id = ANY($1)")
297 + .bind(&distinct)
298 + .execute(pool)
299 + .await?;
300 +
301 + Ok(())
302 + }
303 +
304 + /// Bulk status update for the git-push processor. Each entry is `(issue, final
305 + /// status)`; the caller resolves repeated references to one final state so each
306 + /// issue appears at most once.
307 + #[tracing::instrument(skip_all)]
308 + pub async fn update_issue_statuses(
309 + pool: &PgPool,
310 + updates: &[(IssueId, IssueStatus)],
311 + ) -> Result<()> {
312 + if updates.is_empty() {
313 + return Ok(());
314 + }
315 + let ids: Vec<sqlx::types::Uuid> = updates.iter().map(|(id, _)| *id.as_uuid()).collect();
316 + let statuses: Vec<String> = updates.iter().map(|(_, s)| s.to_string()).collect();
317 +
318 + sqlx::query(
319 + r#"
320 + UPDATE issues AS i SET status = u.status, updated_at = NOW()
321 + FROM UNNEST($1::uuid[], $2::text[]) AS u(id, status)
322 + WHERE i.id = u.id
323 + "#,
324 + )
325 + .bind(&ids)
326 + .bind(&statuses)
327 + .execute(pool)
328 + .await?;
329 +
330 + Ok(())
331 + }
332 +
260 333 /// List all comments on an issue with author usernames.
261 334 #[tracing::instrument(skip_all)]
262 335 pub async fn list_comments(
M server/src/db/projects.rs +30
@@ -123,6 +123,36 @@
123 123 Ok(projects)
124 124 }
125 125
126 + /// Of the given candidate slugs, return those already taken by a project owned
127 + /// by `user_id`. One indexed `slug = ANY($2)` query replaces a per-candidate
128 + /// point-query loop (ultra-fuzz Run 6 R6-Perf-M5).
129 + #[tracing::instrument(skip_all)]
130 + pub async fn filter_taken_slugs(
131 + pool: &PgPool,
132 + user_id: UserId,
133 + slugs: &[String],
134 + ) -> Result<Vec<String>> {
135 + let taken: Vec<String> =
136 + sqlx::query_scalar("SELECT slug FROM projects WHERE user_id = $1 AND slug = ANY($2)")
137 + .bind(user_id)
138 + .bind(slugs)
139 + .fetch_all(pool)
140 + .await?;
141 + Ok(taken)
142 + }
143 +
144 + /// Count a user's projects without materializing the rows. For callers that only
145 + /// need the total (e.g. stats), this avoids fetching up to 500 full rows just to
146 + /// `.len()` them (ultra-fuzz Run 6 R6-Perf-M4).
147 + #[tracing::instrument(skip_all)]
148 + pub async fn count_projects_by_user(pool: &PgPool, user_id: UserId) -> Result<i64> {
149 + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM projects WHERE user_id = $1")
150 + .bind(user_id)
151 + .fetch_one(pool)
152 + .await?;
153 + Ok(count)
154 + }
155 +
126 156 /// Partially update a project's fields (COALESCE keeps existing values when `None`).
127 157 ///
128 158 /// When `features` is `Some`, the project_type is auto-derived from the new features.
M server/src/db/transactions.rs +43
@@ -1132,6 +1132,49 @@
1132 1132 Ok(rows)
1133 1133 }
1134 1134
1135 + /// One page of a seller's sharing-opted-in contacts for CSV export, newest
1136 + /// first. Paginated so the contacts export streams in bounded batches like its
1137 + /// sibling exports instead of buffering one capped query into a single `String`
1138 + /// (ultra-fuzz Run 6 R6-Perf-M2). `(MAX(completed_at), buyer_id)` ordering is
1139 + /// stable so OFFSET batches don't reorder.
1140 + #[tracing::instrument(skip_all)]
1141 + pub async fn get_seller_contacts_page(
1142 + pool: &PgPool,
1143 + seller_id: UserId,
1144 + limit: i64,
1145 + offset: i64,
1146 + ) -> Result<Vec<DbContactRow>> {
1147 + let rows = sqlx::query_as::<_, DbContactRow>(
1148 + r#"
1149 + SELECT
1150 + u.username,
1151 + u.email,
1152 + COUNT(*) AS total_purchases,
1153 + COALESCE(SUM(t.amount_cents), 0)::BIGINT AS total_spent_cents,
1154 + MAX(t.completed_at) AS last_purchase_at
1155 + FROM transactions t
1156 + JOIN users u ON u.id = t.buyer_id
1157 + WHERE t.seller_id = $1
1158 + AND t.status = 'completed'
1159 + AND t.share_contact = true
1160 + AND NOT EXISTS (
1161 + SELECT 1 FROM contact_revocations cr
1162 + WHERE cr.buyer_id = t.buyer_id AND cr.seller_id = t.seller_id
1163 + )
1164 + GROUP BY t.buyer_id, u.username, u.email
1165 + ORDER BY MAX(t.completed_at) DESC, t.buyer_id
1166 + LIMIT $2 OFFSET $3
1167 + "#,
1168 + )
1169 + .bind(seller_id)
1170 + .bind(limit)
1171 + .bind(offset)
1172 + .fetch_all(pool)
1173 + .await?;
1174 +
1175 + Ok(rows)
1176 + }
1177 +
1135 1178 /// Get seller transactions for CSV export, with conditional buyer email.
1136 1179 ///
1137 1180 /// Respects contact revocations: if a buyer revoked sharing, their email
M server/src/payments/mod.rs +1
@@ -17,6 +17,7 @@
17 17 mod checkout;
18 18 mod checkout_metadata;
19 19 mod connect;
20 + pub mod fan_ops;
20 21 pub mod synckit_app_pricing;
21 22 pub mod synckit_billing;
22 23 mod webhooks;
M server/src/scanning/archive.rs +24 -3
@@ -106,6 +106,19 @@
106 106 /// `Err(detail)` is a mid-stream decode failure: the caller decides what that
107 107 /// means for each dimension (ZIP bomb accounting uses a conservative estimate
108 108 /// and continues; a single-stream compressor fails closed).
109 + /// Byte limit at which `tee_decompress` should stop expanding a stream for bomb
110 + /// accounting: the smaller of the 2 GB absolute cap and `compressed * MAX_RATIO`.
111 + /// A stream that blows past its ratio budget is flagged as a bomb regardless, so
112 + /// there is no reason to keep decompressing it up to the full 2 GB first. Falls
113 + /// back to the absolute cap when the compressed size is unknown (0).
114 + fn entry_bomb_stop_limit(compressed_size: u64) -> u64 {
115 + if compressed_size == 0 {
116 + return constants::SCAN_ZIP_MAX_UNCOMPRESSED;
117 + }
118 + let ratio_limit = compressed_size.saturating_mul(constants::SCAN_ZIP_MAX_RATIO as u64);
119 + constants::SCAN_ZIP_MAX_UNCOMPRESSED.min(ratio_limit)
120 + }
121 +
109 122 fn tee_decompress(
110 123 reader: &mut dyn Read,
111 124 bomb_abs_limit: u64,
@@ -308,12 +321,16 @@
308 321 }
309 322
310 323 // Decompress the entry exactly once, teeing the full size (bomb) and the
311 - // buffered prefix (content).
324 + // buffered prefix (content). Stop early once the entry blows past
325 + // `compressed * MAX_RATIO` rather than always decompressing to the 2 GB
326 + // absolute cap — a high-ratio entry is flagged below regardless, so there
327 + // is no reason to expand it fully first.
328 + let entry_stop_limit = entry_bomb_stop_limit(entry_compressed);
312 329 let want_content = content && nested_res.is_none();
313 330 let (counted, content_buf, decode_err) = match archive.by_index(i) {
314 331 Ok(mut entry) => match tee_decompress(
315 332 &mut entry,
316 - constants::SCAN_ZIP_MAX_UNCOMPRESSED,
333 + entry_stop_limit,
317 334 want_content,
318 335 &mut interior_budget,
319 336 ) {
@@ -496,8 +513,12 @@
496 513 let ratio_limit = compressed_size.saturating_mul(constants::SCAN_ZIP_MAX_RATIO as u64);
497 514 let mut interior_budget: u64 = constants::SCAN_ZIP_MAX_UNCOMPRESSED;
498 515
516 + // Stop expanding once the stream blows past `compressed * MAX_RATIO` rather
517 + // than always running to the 2 GB absolute cap; the ratio verdict below fires
518 + // identically on the early-stopped count.
519 + let stop_limit = entry_bomb_stop_limit(compressed_size);
499 520 let (counted, content_buf, decode_err) =
500 - match tee_decompress(decoder.as_mut(), abs_limit, content, &mut interior_budget) {
521 + match tee_decompress(decoder.as_mut(), stop_limit, content, &mut interior_budget) {
501 522 Ok((c, cb)) => (c, cb, None),
502 523 Err((c, why)) => (c, None, Some(why)),
503 524 };
M server/src/scanning/mod.rs +17 -1
@@ -547,7 +547,23 @@
547 547 layers.push(archive_layer);
548 548 layers.push(archive_nested_layer);
549 549 layers.push(match self.yara_rules {
550 - Some(ref rules) => yara::scan_with_yara(rules, data),
550 + Some(ref rules) => {
551 + // Cap YARA's input: it walks the whole slice, faulting the entire
552 + // mmap resident. ClamAV (streamed, uncapped) is the full-file
553 + // backstop, so scanning a generous prefix bounds peak RAM without
554 + // surrendering the floor.
555 + let yara_input = if data.len() > crate::constants::SCAN_YARA_MAX_BYTES {
556 + tracing::warn!(
557 + full_bytes = data.len(),
558 + capped_bytes = crate::constants::SCAN_YARA_MAX_BYTES,
559 + "YARA input capped; scanning prefix only (ClamAV scans the full file)"
560 + );
561 + &data[..crate::constants::SCAN_YARA_MAX_BYTES]
562 + } else {
563 + data
564 + };
565 + yara::scan_with_yara(rules, yara_input)
566 + }
551 567 None => LayerResult {
552 568 layer: "yara",
553 569 verdict: LayerVerdict::Skip,
M server/src/scanning/spool.rs +11 -3
@@ -11,7 +11,7 @@
11 11
12 12 use crate::constants::{
13 13 SCAN_SPOOL_FREE_RESERVE_BYTES, SCAN_SPOOL_MAX_BYTES, SCAN_SPOOL_ORPHAN_AGE_SECS,
14 - SCAN_WORKER_COUNT,
14 + SCAN_SPOOL_SLACK_BYTES, SCAN_WORKER_COUNT,
15 15 };
16 16 use s3_storage::ByteStream;
17 17 use std::path::{Path, PathBuf};
@@ -148,6 +148,14 @@
148 148 // — funnels through a single cleanup that unlinks the partial file. The
149 149 // earlier code only removed it on the over-cap branch, leaking a partial
150 150 // tempfile on a write/flush/stream error until the orphan reaper swept it.
151 + // Bound the streamed bytes by the *claimed* object size plus a small slack,
152 + // not just the global ceiling: an object that under-reports its size must not
153 + // be allowed to stream the full `SCAN_SPOOL_MAX_BYTES` to scratch before we
154 + // abort (Run 6 R6-Perf-M3). `check_free_space`/worker already cap
155 + // `expected_size` at the global max, so this is always the tighter bound.
156 + let write_cap = expected_size
157 + .saturating_add(SCAN_SPOOL_SLACK_BYTES)
158 + .min(SCAN_SPOOL_MAX_BYTES);
151 159 let write_result: Result<(), String> = async {
152 160 let mut written: u64 = 0;
153 161 while let Some(chunk) = stream
@@ -156,9 +164,9 @@
156 164 .map_err(|e| format!("read S3 stream: {e}"))?
157 165 {
158 166 written += chunk.len() as u64;
159 - if written > SCAN_SPOOL_MAX_BYTES {
167 + if written > write_cap {
160 168 return Err(format!(
161 - "S3 stream exceeded SCAN_SPOOL_MAX_BYTES ({SCAN_SPOOL_MAX_BYTES} bytes); aborting scan spool"
169 + "S3 stream exceeded expected size bound ({write_cap} bytes, claimed {expected_size}); aborting scan spool"
162 170 ));
163 171 }
164 172 file.write_all(&chunk)
M server/src/db/items/mod.rs +18
@@ -218,6 +218,24 @@
218 218 Ok(items)
219 219 }
220 220
221 + /// Count a user's (non-deleted) items without materializing the rows. For
222 + /// callers that only need the total, this avoids fetching up to 500 full rows
223 + /// just to `.len()` them (ultra-fuzz Run 6 R6-Perf-M4).
224 + #[tracing::instrument(skip_all)]
225 + pub async fn count_items_by_user(pool: &PgPool, user_id: UserId) -> Result<i64> {
226 + let count: i64 = sqlx::query_scalar(
227 + r#"
228 + SELECT COUNT(*) FROM items i
229 + JOIN projects p ON i.project_id = p.id
230 + WHERE p.user_id = $1 AND i.deleted_at IS NULL
231 + "#,
232 + )
233 + .bind(user_id)
234 + .fetch_one(pool)
235 + .await?;
236 + Ok(count)
237 + }
238 +
221 239 /// Count items per project for all projects owned by a user.
222 240 ///
223 241 /// Returns `(project_id, count)` tuples. Used by the CLI to avoid N+1 queries.
M server/src/routes/admin/users.rs +36 -19
M server/src/routes/api/validate.rs +20 -16
M server/src/routes/git_issues/push_refs.rs +33 -45
M server/src/routes/api/exports/mod.rs +25 -26
M server/src/routes/api/internal/creators.rs +4 -4
M server/src/routes/api/users/profile.rs +11 -14
M server/src/routes/stripe/webhook/checkout.rs +12 -14
A server/src/payments/fan_ops.rs +87