Skip to main content

max / makenotwork

Add global request timeout; background the content export Global TimeoutLayer-equivalent: a path-aware middleware bounds every non-exempt handler's response generation to 120s and returns 504 if it blows past, so a handler wedged on a stuck upstream can no longer pin a connection indefinitely. git smart-HTTP, exports, the SyncKit SSE channel, and OTA/build artifact transfer are exempt (they legitimately run long or stream open-ended bodies). Background the content export: it built the multi-GB ZIP synchronously inside the request, holding the EXPORT_LIMITER permit and a connection for the whole job — a wedged S3 GET pinned both indefinitely. The handler now collects the file list (fast), returns 202 immediately, and hands the download + zip + multipart upload to the background pool, which emails a presigned link when ready (or a failure notice). Build-artifact exports were already off the request path; this matches them. Tests updated: content export now asserts 202 + awaits the emailed link, then verifies the uploaded ZIP's contents. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-17 01:13 UTC
Commit: f4d08f02ef5b8dcef2100cf59a90bb0b498e36bd
Parent: 9c8aec8
5 files changed, +237 insertions, -78 deletions
M server/src/email/templates/monetization.rs +54
@@ -190,6 +190,60 @@ Enter this code at checkout to apply it toward any purchase on the platform.{exp
190 190 self.transport.send_email(to_email, subject, &body).await
191 191 }
192 192
193 + /// Notify a creator that their backgrounded content export is ready, with a
194 + /// time-limited download link. The export ZIP is built off the request path,
195 + /// so the link arrives by email rather than inline in the dashboard.
196 + pub async fn send_content_export_ready(
197 + &self,
198 + to_email: &str,
199 + to_name: Option<&str>,
200 + download_url: &str,
201 + ) -> Result<()> {
202 + let subject = "Your content export is ready";
203 + let body = format!(
204 + r#"Hi{name},
205 +
206 + Your content export is ready to download:
207 +
208 + {url}
209 +
210 + This link expires in one hour. If it expires before you download, start a new export from your dashboard.
211 +
212 + - Makenotwork"#,
213 + name = crate::email::greeting(to_name),
214 + url = download_url,
215 + );
216 +
217 + self.transport.send_email(to_email, subject, &body).await
218 + }
219 +
220 + /// Notify a creator that their content export could not be completed, so a
221 + /// silently-failed background job doesn't leave them waiting indefinitely.
222 + pub async fn send_content_export_failed(
223 + &self,
224 + to_email: &str,
225 + to_name: Option<&str>,
226 + reason: Option<&str>,
227 + ) -> Result<()> {
228 + let subject = "Your content export could not be completed";
229 + let reason_line = reason
230 + .map(|r| format!("\n\n{r}"))
231 + .unwrap_or_default();
232 + let body = format!(
233 + r#"Hi{name},
234 +
235 + We were unable to finish preparing your content export. No changes were made to your content.{reason}
236 +
237 + Please try again from your dashboard; if it keeps failing, contact info@makenot.work.
238 +
239 + - Makenotwork"#,
240 + name = crate::email::greeting(to_name),
241 + reason = reason_line,
242 + );
243 +
244 + self.transport.send_email(to_email, subject, &body).await
245 + }
246 +
193 247 /// Send a Fan+ cancellation email.
194 248 pub async fn send_fan_plus_cancelled(
195 249 &self,
M server/src/lib.rs +42 -1
@@ -238,7 +238,8 @@ pub fn build_app(
238 238 // process lifetime (Run #14 CHRONIC 1). Guarded by `Once` internally.
239 239 crate::rate_limit::start_governor_sweeper();
240 240
241 - app.layer(middleware::from_fn_with_state(state.clone(), access_gate::access_gate_middleware))
241 + app.layer(middleware::from_fn(request_timeout_middleware))
242 + .layer(middleware::from_fn_with_state(state.clone(), access_gate::access_gate_middleware))
242 243 .layer(middleware::from_fn_with_state(state.clone(), security_headers_middleware))
243 244 .layer(middleware::from_fn(metrics::cache_control_middleware))
244 245 .layer(middleware::from_fn(metrics::metrics_middleware))
@@ -252,6 +253,46 @@ pub fn build_app(
252 253 .layer(middleware::from_fn_with_state(state.clone(), routes::user_pages::dispatch))
253 254 }
254 255
256 + /// Wall-clock ceiling on response generation. Generous — every normal page/API
257 + /// handler finishes in well under this; the bound exists to catch a handler that
258 + /// wedges on a stuck upstream (S3 GET, a hung DB call with no per-query timeout)
259 + /// rather than to pace healthy traffic.
260 + const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120);
261 +
262 + /// Routes exempt from [`REQUEST_TIMEOUT`]: ones that legitimately run long or
263 + /// stream open-ended bodies. git smart-HTTP (clone/pack), data + content
264 + /// exports, the SyncKit SSE push channel, and OTA / build artifact transfer.
265 + fn timeout_exempt(path: &str) -> bool {
266 + path.starts_with("/git")
267 + || path.contains("/export")
268 + || path.contains("/sync/subscribe")
269 + || path.contains("/sync/ota")
270 + || path.contains("/sync/builds")
271 + }
272 +
273 + /// Global request timeout with per-route opt-out. A single hung handler used to
274 + /// pin a connection (and its DB conn / scan permit) indefinitely; this bounds
275 + /// every non-exempt handler to [`REQUEST_TIMEOUT`] and returns 504 if it blows
276 + /// past it. Streaming-body handlers are unaffected anyway (the body flows after
277 + /// this future returns), but they are listed in [`timeout_exempt`] for clarity.
278 + async fn request_timeout_middleware(
279 + request: axum::http::Request<axum::body::Body>,
280 + next: middleware::Next,
281 + ) -> axum::response::Response {
282 + use axum::response::IntoResponse;
283 + if timeout_exempt(request.uri().path()) {
284 + return next.run(request).await;
285 + }
286 + match tokio::time::timeout(REQUEST_TIMEOUT, next.run(request)).await {
287 + Ok(response) => response,
288 + Err(_) => (
289 + axum::http::StatusCode::GATEWAY_TIMEOUT,
290 + "Request timed out",
291 + )
292 + .into_response(),
293 + }
294 + }
295 +
255 296 /// Middleware that sets security headers on all responses.
256 297 /// Embed routes (`/embed/`) get permissive frame headers for iframe embedding.
257 298 async fn security_headers_middleware(
M server/src/routes/api/exports/content.rs +85 -62
@@ -8,7 +8,7 @@ use std::io::Write;
8 8 use axum::{
9 9 extract::{Query, State},
10 10 http::header::HeaderMap,
11 - response::{IntoResponse, Response},
11 + response::Response,
12 12 };
13 13 use serde::Deserialize;
14 14 use zip::write::SimpleFileOptions;
@@ -18,7 +18,6 @@ use crate::{
18 18 db,
19 19 error::{AppError, Result, ResultExt},
20 20 helpers::is_htmx_request,
21 - templates::ExportContentReadyTemplate,
22 21 AppState,
23 22 };
24 23
@@ -57,19 +56,13 @@ pub(in crate::routes::api) async fn export_content(
57 56 ) -> Result<Response> {
58 57 let is_htmx = is_htmx_request(&headers);
59 58
60 - // Hold a concurrency permit for the lifetime of the export so a burst can't
61 - // saturate the blocking pool. Acquired before any DB/S3 work, so a queued
62 - // request holds no connection while it waits.
63 - let _export_permit = EXPORT_LIMITER
64 - .acquire()
65 - .await
66 - .expect("export limiter semaphore is never closed");
67 -
68 59 let s3 = state.s3.as_ref().ok_or_else(|| {
69 60 AppError::ServiceUnavailable("File storage is not configured".to_string())
70 61 })?;
71 62
72 - // Collect all S3 keys from items, versions, and insertions
63 + // Collect all S3 keys from items, versions, and insertions. These are fast
64 + // indexed reads, done up front so an empty export is reported immediately;
65 + // the heavy download + zip + multipart upload runs in the background (below).
73 66 let item_keys = db::items::get_user_s3_keys(&state.db, user.id).await?;
74 67 let version_keys = db::versions::get_user_version_s3_keys(&state.db, user.id).await?;
75 68
@@ -134,13 +127,73 @@ pub(in crate::routes::api) async fn export_content(
134 127 return Err(AppError::BadRequest("No content files to export.".to_string()));
135 128 }
136 129
137 - // Write ZIP to a temporary file, downloading files one at a time.
138 - // Peak memory is O(largest_single_file) — the ZIP itself lives on disk.
139 - let s3_clone = s3.clone();
130 + // Hand the heavy work to the background pool: download every file, build the
131 + // ZIP on disk, multipart-upload it, and email a presigned link. The request
132 + // returns now instead of holding a connection + export permit for the whole
133 + // multi-GB job — a wedged S3 GET used to pin both indefinitely.
134 + let s3 = s3.clone();
135 + let email = state.email.clone();
140 136 let username = user.username.to_string();
137 + let to_email = user.email.clone();
138 + let to_name = user.display_name.clone();
139 + let user_id = user.id;
140 + state.bg.spawn("content_export", async move {
141 + match build_content_export(&s3, user_id, &username, files).await {
142 + Ok(download_url) => {
143 + if let Err(e) = email
144 + .send_content_export_ready(&to_email, to_name.as_deref(), &download_url)
145 + .await
146 + {
147 + tracing::error!(error = ?e, "failed to send content-export-ready email");
148 + }
149 + }
150 + Err(reason) => {
151 + tracing::warn!(user_id = %user_id, reason = %reason, "content export failed");
152 + if let Err(e) = email
153 + .send_content_export_failed(&to_email, to_name.as_deref(), Some(&reason))
154 + .await
155 + {
156 + tracing::error!(error = ?e, "failed to send content-export-failed email");
157 + }
158 + }
159 + }
160 + });
161 +
162 + let message = format!(
163 + "Preparing your content export. We'll email a download link to {} when it's ready.",
164 + user.email
165 + );
166 + if is_htmx {
167 + return Ok(super::export_pending_html(&message));
168 + }
169 + Response::builder()
170 + .status(axum::http::StatusCode::ACCEPTED)
171 + .body(message.into())
172 + .context("build export accepted response")
173 + }
141 174
142 - let tmp_dir = tempfile::tempdir()
143 - .context("create temp dir for export")?;
175 + /// Build the content-export ZIP off the request path: download every file,
176 + /// zip to a tempfile, multipart-upload, and return a 1-hour presigned download
177 + /// URL. Holds the [`EXPORT_LIMITER`] permit for its lifetime so a burst can't
178 + /// saturate the blocking pool. On any failure returns a user-facing reason
179 + /// string (emailed to the creator); never panics the background task.
180 + async fn build_content_export(
181 + s3: &std::sync::Arc<dyn crate::storage::StorageBackend>,
182 + user_id: db::UserId,
183 + username: &str,
184 + files: Vec<(String, String, Option<i64>)>,
185 + ) -> std::result::Result<String, String> {
186 + // Hold a concurrency permit for the lifetime of the export so a burst can't
187 + // saturate the blocking pool. Acquired here (off the request path), so a
188 + // queued export holds no DB connection while it waits.
189 + let _export_permit = EXPORT_LIMITER
190 + .acquire()
191 + .await
192 + .expect("export limiter semaphore is never closed");
193 +
194 + let s3_clone = s3.clone();
195 +
196 + let tmp_dir = tempfile::tempdir().map_err(|e| format!("create temp dir for export: {e}"))?;
144 197 let zip_path = tmp_dir.path().join("export.zip");
145 198
146 199 {
@@ -158,8 +211,8 @@ pub(in crate::routes::api) async fn export_content(
158 211 },
159 212 )
160 213 .await
161 - .context("join zip create task")?
162 - .context("create export zip file")?;
214 + .map_err(|e| format!("join zip create task: {e}"))?
215 + .map_err(|e| format!("create export zip file: {e}"))?;
163 216
164 217 let options = SimpleFileOptions::default()
165 218 .compression_method(zip::CompressionMethod::Stored);
@@ -186,11 +239,7 @@ pub(in crate::routes::api) async fn export_content(
186 239 continue;
187 240 }
188 241 if total_size + size > MAX_TOTAL_SIZE {
189 - let msg = "Content export exceeds 2 GB limit. Try exporting a single project instead.";
190 - if is_htmx {
191 - return Ok(export_error_html(msg));
192 - }
193 - return Err(AppError::BadRequest(msg.to_string()));
242 + return Err("Content export exceeds the 2 GB limit. Try exporting a single project instead.".to_string());
194 243 }
195 244 }
196 245
@@ -198,11 +247,7 @@ pub(in crate::routes::api) async fn export_content(
198 247 Ok(data) => {
199 248 total_size += data.len() as u64;
200 249 if total_size > MAX_TOTAL_SIZE {
201 - let msg = "Content export exceeds 2 GB limit. Try exporting a single project instead.";
202 - if is_htmx {
203 - return Ok(export_error_html(msg));
204 - }
205 - return Err(AppError::BadRequest(msg.to_string()));
250 + return Err("Content export exceeds the 2 GB limit. Try exporting a single project instead.".to_string());
206 251 }
207 252 let file_size = data.len() as i64;
208 253 // Move writer + file bytes onto the blocking pool, write, get
@@ -217,8 +262,8 @@ pub(in crate::routes::api) async fn export_content(
217 262 },
218 263 )
219 264 .await
220 - .context("join zip write task")?
221 - .context("write file into export zip")?;
265 + .map_err(|e| format!("join zip write task: {e}"))?
266 + .map_err(|e| format!("write file into export zip: {e}"))?;
222 267 manifest.push((zip_path_entry.clone(), file_size));
223 268 }
224 269 Err(e) => {
@@ -229,11 +274,7 @@ pub(in crate::routes::api) async fn export_content(
229 274 }
230 275
231 276 if manifest.is_empty() {
232 - let msg = "Could not download any files from storage. Please try again later.";
233 - if is_htmx {
234 - return Ok(export_error_html(msg));
235 - }
236 - return Err(AppError::Storage(msg.to_string()));
277 + return Err("Could not download any files from storage. Please try again later.".to_string());
237 278 }
238 279
239 280 // Build README.txt as the last ZIP entry (cheap string work, async side)
@@ -271,43 +312,25 @@ pub(in crate::routes::api) async fn export_content(
271 312 Ok(())
272 313 })
273 314 .await
274 - .context("join zip finalize task")?
275 - .context("finalize export zip")?;
315 + .map_err(|e| format!("join zip finalize task: {e}"))?
316 + .map_err(|e| format!("finalize export zip: {e}"))?;
276 317 }
277 318
278 319 // Upload ZIP to S3 via multipart upload (streams from disk in 10 MB parts)
279 320 let timestamp = chrono::Utc::now().format("%Y%m%d-%H%M%S");
280 - let export_key = format!("{}/exports/content-{}.zip", user.id, timestamp);
321 + let export_key = format!("{}/exports/content-{}.zip", user_id, timestamp);
281 322 if let Err(e) = s3.upload_multipart(&export_key, "application/zip", &zip_path).await {
282 323 tracing::error!(error = ?e, "Failed to upload content export ZIP to S3");
283 - if is_htmx {
284 - return Ok(export_error_html("Failed to prepare download. Please try again."));
285 - }
286 - return Err(e);
324 + return Err("Failed to upload the export to storage.".to_string());
287 325 }
288 326
289 327 // Generate presigned download URL (1 hour)
290 - let download_url = match s3.presign_download(&export_key, Some(3600)).await {
291 - Ok(url) => url,
292 - Err(e) => {
293 - tracing::error!(error = ?e, "Failed to generate presigned URL for content export");
294 - if is_htmx {
295 - return Ok(export_error_html("Export created but download link failed. Please try again."));
296 - }
297 - return Err(e);
298 - }
299 - };
300 -
301 - if is_htmx {
302 - return Ok(ExportContentReadyTemplate { download_url }.into_response());
303 - }
328 + let download_url = s3.presign_download(&export_key, Some(3600)).await.map_err(|e| {
329 + tracing::error!(error = ?e, "Failed to generate presigned URL for content export");
330 + "Export created but the download link could not be generated.".to_string()
331 + })?;
304 332
305 - // Direct API call: redirect to presigned URL
306 - Response::builder()
307 - .status(303)
308 - .header("Location", &download_url)
309 - .body("".into())
310 - .context("build export redirect response")
333 + Ok(download_url)
311 334 }
312 335
313 336 /// Extract file extension from an S3 key (e.g. "user/item/audio/track.mp3" -> "mp3").
M server/src/routes/api/exports/mod.rs +13
@@ -32,6 +32,19 @@ pub(crate) fn export_error_html(message: &str) -> Response {
32 32 .into_response()
33 33 }
34 34
35 + /// HTMX success panel for a queued (backgrounded) export — the link is delivered
36 + /// by email when the job finishes rather than inline.
37 + pub(crate) fn export_pending_html(message: &str) -> Response {
38 + axum::response::Html(
39 + FormStatusTemplate {
40 + success: true,
41 + message: message.to_string(),
42 + }
43 + .render_string(),
44 + )
45 + .into_response()
46 + }
47 +
35 48 /// Build an HTTP response for a downloadable file attachment.
36 49 fn download_response(content: Vec<u8>, filename: &str, content_type: &str) -> Result<Response> {
37 50 Response::builder()
M server/tests/workflows/exports.rs +43 -15
@@ -245,16 +245,43 @@ async fn upload_audio(h: &mut TestHarness, item_id: &str, file_name: &str, bytes
245 245 s3_key
246 246 }
247 247
248 - /// Pull the export object key out of the 303 redirect Location
249 - /// (`http://test-storage/<key>`, per the mock presigner).
250 - fn export_key_from_location(loc: &str) -> &str {
251 - loc.strip_prefix("http://test-storage/")
252 - .unwrap_or_else(|| panic!("unexpected export Location: {loc}"))
248 + /// Pull the export object key out of a presigned URL (`http://test-storage/<key>`,
249 + /// per the mock presigner) found in the export-ready email.
250 + fn export_key_from_url(url: &str) -> &str {
251 + url.strip_prefix("http://test-storage/")
252 + .unwrap_or_else(|| panic!("unexpected export URL: {url}"))
253 + }
254 +
255 + /// Content export now runs in the background and emails a link. Poll the mock
256 + /// email outbox until the "content export is ready" email lands, then return the
257 + /// export object key parsed out of its download URL.
258 + async fn await_export_key(h: &TestHarness) -> String {
259 + let mock = h.mock_email.as_ref().expect("mock email transport required");
260 + for _ in 0..100 {
261 + if let Some(email) = mock
262 + .sent()
263 + .iter()
264 + .find(|e| e.subject.contains("content export is ready"))
265 + {
266 + let start = email
267 + .body
268 + .find("http://test-storage/")
269 + .expect("export email must contain a download URL");
270 + let url: String = email.body[start..]
271 + .split_whitespace()
272 + .next()
273 + .unwrap()
274 + .to_string();
275 + return export_key_from_url(&url).to_string();
276 + }
277 + tokio::time::sleep(std::time::Duration::from_millis(50)).await;
278 + }
279 + panic!("export-ready email never arrived");
253 280 }
254 281
255 282 #[tokio::test]
256 283 async fn content_export_zips_files_and_uploads_to_s3() {
257 - let mut h = TestHarness::with_storage().await;
284 + let mut h = TestHarness::with_mocks().await;
258 285 let setup = h.create_creator_with_item("ctexport", "audio", 0).await;
259 286 h.trust_user(setup.user_id).await;
260 287 h.grant_tier(setup.user_id, "small_files").await;
@@ -262,23 +289,24 @@ async fn content_export_zips_files_and_uploads_to_s3() {
262 289 const AUDIO: &[u8] = b"FAKE-MP3-AUDIO-CONTENT-CTEXPORT-0123456789";
263 290 upload_audio(&mut h, &setup.item_id, "track.mp3", AUDIO).await;
264 291
292 + // The request returns immediately (202) and the zip is built in the
293 + // background; the download link arrives by email.
265 294 let resp = h.client.post_form("/api/export/content", "").await;
266 - assert_eq!(resp.status.as_u16(), 303, "content export should redirect to the zip: {} {}", resp.status, resp.text);
295 + assert_eq!(resp.status.as_u16(), 202, "content export should be accepted for background processing: {} {}", resp.status, resp.text);
267 296
268 - let loc = resp.header("location").expect("export must set a Location header");
269 - let export_key = export_key_from_location(loc);
297 + let export_key = await_export_key(&h).await;
270 298 assert!(
271 299 export_key.starts_with(&format!("{}/exports/content-", setup.user_id)),
272 300 "export key must live under the user's exports prefix: {export_key}"
273 301 );
274 302 assert!(export_key.ends_with(".zip"), "export key must be a .zip: {export_key}");
275 303
276 - // The handler actually uploaded the archive to (mock) S3.
304 + // The background job actually uploaded the archive to (mock) S3.
277 305 let zip = h
278 306 .storage
279 307 .as_ref()
280 308 .unwrap()
281 - .download_object(export_key)
309 + .download_object(&export_key)
282 310 .await
283 311 .expect("export zip must be uploaded to S3");
284 312 assert!(zip.len() > 4 && &zip[..4] == b"PK\x03\x04", "must be a real ZIP archive ({} bytes)", zip.len());
@@ -300,7 +328,7 @@ async fn content_export_with_no_files_returns_error() {
300 328
301 329 #[tokio::test]
302 330 async fn content_export_scoped_to_project_excludes_other_projects() {
303 - let mut h = TestHarness::with_storage().await;
331 + let mut h = TestHarness::with_mocks().await;
304 332 let setup = h.create_creator_with_item("ctscope", "audio", 0).await;
305 333 h.trust_user(setup.user_id).await;
306 334 h.grant_tier(setup.user_id, "small_files").await;
@@ -332,14 +360,14 @@ async fn content_export_scoped_to_project_excludes_other_projects() {
332 360 "",
333 361 )
334 362 .await;
335 - assert_eq!(resp.status.as_u16(), 303, "scoped export should redirect: {} {}", resp.status, resp.text);
363 + assert_eq!(resp.status.as_u16(), 202, "scoped export should be accepted for background processing: {} {}", resp.status, resp.text);
336 364
337 - let loc = resp.header("location").unwrap().to_string();
365 + let export_key = await_export_key(&h).await;
338 366 let zip = h
339 367 .storage
340 368 .as_ref()
341 369 .unwrap()
342 - .download_object(export_key_from_location(&loc))
370 + .download_object(&export_key)
343 371 .await
344 372 .unwrap();
345 373 assert!(contains_bytes(&zip, AUDIO_A), "scoped export must include project A's file");