Skip to main content

max / makenotwork

server: bound per-tick scheduler sweeps; anchor request-timeout exemptions - PERF-S1: cleanup_orphaned_uploads ran an unbounded SELECT + serial per-row S3 delete every tick, inside the tick-wide advisory lock — a backlog could stall the scheduler past its 30s alert ceiling. Cap the query at 200 rows, oldest-first; the next tick drains the rest. - PERF-S2: escalate_stale_refunds had the same unbounded per-tick query in the same locked window. Cap at 100, oldest-first (escalation is idempotent). - PERF M-1: timeout_exempt matched substrings, so contains("/export") also exempted the /dashboard/export page and any creator slug holding the token, and "/sync/builds" matched nothing. Anchor to real route prefixes. Ultra-fuzz Run #23, Performance axis. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-18 22:00 UTC
Commit: 349dc7f21011d9bc590b6b549baeb9df4a7b7008
Parent: 75b315b
3 files changed, +64 insertions, -9 deletions
M server/src/db/pending_refunds.rs +10 -1
@@ -101,7 +101,14 @@ pub struct StaleRefund {
101 101 pub created_at: chrono::DateTime<chrono::Utc>,
102 102 }
103 103
104 - /// Get pending refunds older than `age` that have not been matched or escalated.
104 + /// Per-tick cap on the stale-refund escalation sweep. It runs every scheduler
105 + /// tick under the tick-wide advisory lock; escalation is idempotent (sets
106 + /// `escalated_at`), so a bound here drains a backlog across ticks instead of
107 + /// letting one unbounded query stall the tick (PERF-S2, Run #23).
108 + pub const STALE_REFUND_BATCH: i64 = 100;
109 +
110 + /// Get up to [`STALE_REFUND_BATCH`] pending refunds older than `age` that have
111 + /// not been matched or escalated, oldest first.
105 112 pub async fn get_stale_refunds(
106 113 pool: &PgPool,
107 114 age: chrono::Duration,
@@ -115,9 +122,11 @@ pub async fn get_stale_refunds(
115 122 AND escalated_at IS NULL
116 123 AND created_at < $1
117 124 ORDER BY created_at
125 + LIMIT $2
118 126 "#,
119 127 )
120 128 .bind(cutoff)
129 + .bind(STALE_REFUND_BATCH)
121 130 .fetch_all(pool)
122 131 .await?;
123 132
M server/src/db/pending_uploads.rs +11 -2
@@ -58,16 +58,25 @@ pub async fn remove_pending_upload<'e>(
58 58 Ok(())
59 59 }
60 60
61 - /// Fetch presigned uploads older than `max_age` that were never confirmed.
61 + /// Per-tick cap on the orphan-upload reaper. The reaper runs every scheduler
62 + /// tick under the tick-wide advisory lock and deletes serially (one S3 round-
63 + /// trip per row), so an unbounded result set lets a backlog wedge the tick
64 + /// (PERF-S1, Run #23). Oldest-first + a bound drains across ticks instead.
65 + pub const STALE_UPLOAD_BATCH: i64 = 200;
66 +
67 + /// Fetch up to [`STALE_UPLOAD_BATCH`] presigned uploads older than `max_age`
68 + /// that were never confirmed, oldest first.
62 69 pub async fn get_stale_pending_uploads(
63 70 pool: &PgPool,
64 71 max_age: chrono::Duration,
65 72 ) -> Result<Vec<(String, String)>> {
66 73 let cutoff = chrono::Utc::now() - max_age;
67 74 let rows: Vec<(String, String)> = sqlx::query_as(
68 - "SELECT s3_key, bucket FROM pending_uploads WHERE created_at < $1",
75 + "SELECT s3_key, bucket FROM pending_uploads WHERE created_at < $1 \
76 + ORDER BY created_at LIMIT $2",
69 77 )
70 78 .bind(cutoff)
79 + .bind(STALE_UPLOAD_BATCH)
71 80 .fetch_all(pool)
72 81 .await?;
73 82 Ok(rows)
M server/src/lib.rs +43 -6
@@ -260,14 +260,22 @@ pub fn build_app(
260 260 const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120);
261 261
262 262 /// Routes exempt from [`REQUEST_TIMEOUT`]: ones that legitimately run long or
263 - /// stream open-ended bodies. git smart-HTTP (clone/pack), data + content
264 - /// exports, the SyncKit SSE push channel, and OTA / build artifact transfer.
263 + /// stream open-ended bodies. git smart-HTTP (clone/pack), data exports, the
264 + /// SyncKit SSE push channel, and OTA artifact transfer.
265 + ///
266 + /// Anchored to real route prefixes, not substrings (PERF M-1, Run #23): the old
267 + /// `contains("/export")` also exempted the `/dashboard/export` page and any
268 + /// future creator-controlled path containing the token, silently widening the
269 + /// un-timed-out surface. The `/sync/builds` substring matched no route (build
270 + /// artifacts move over the OTA path, already covered).
265 271 fn timeout_exempt(path: &str) -> bool {
266 272 path.starts_with("/git")
267 - || path.contains("/export")
268 - || path.contains("/sync/subscribe")
269 - || path.contains("/sync/ota")
270 - || path.contains("/sync/builds")
273 + || path.starts_with("/api/export/")
274 + || path.starts_with("/api/internal/creator/export/")
275 + || path.starts_with("/api/sync/subscribe")
276 + || path.starts_with("/api/v1/sync/subscribe")
277 + || path.starts_with("/api/sync/ota")
278 + || path.starts_with("/api/v1/sync/ota")
271 279 }
272 280
273 281 /// Global request timeout with per-route opt-out. A single hung handler used to
@@ -293,6 +301,35 @@ async fn request_timeout_middleware(
293 301 }
294 302 }
295 303
304 + #[cfg(test)]
305 + mod timeout_exempt_tests {
306 + use super::timeout_exempt;
307 +
308 + #[test]
309 + fn exempts_only_anchored_long_running_routes() {
310 + // Genuinely long / streaming routes stay exempt.
311 + for p in [
312 + "/git/foo/bar.git/info/refs",
313 + "/api/export/content",
314 + "/api/internal/creator/export/sales",
315 + "/api/sync/subscribe",
316 + "/api/v1/sync/subscribe",
317 + "/api/sync/ota/apps/x/releases",
318 + "/api/v1/sync/ota/slug/macos/arm64/1.0.0",
319 + ] {
320 + assert!(timeout_exempt(p), "{p} should be exempt");
321 + }
322 + // The substring-match hazard: these contain a token but must NOT be exempt.
323 + for p in [
324 + "/dashboard/export", // a normal page, not a long export
325 + "/u/somecreator/export-notes", // creator-controlled slug
326 + "/items/sync/ota-recap", // happens to contain the token mid-path
327 + ] {
328 + assert!(!timeout_exempt(p), "{p} must not be exempt");
329 + }
330 + }
331 + }
332 +
296 333 /// Middleware that sets security headers on all responses.
297 334 /// Embed routes (`/embed/`) get permissive frame headers for iframe embedding.
298 335 async fn security_headers_middleware(