Skip to main content

max / makenotwork

Fix PERF-1: bound content-export memory for legacy NULL-size rows The content-export per-file 500MB cap was gated on `if let Some(size) = db_size`, so a legacy row with NULL file_size_bytes skipped the check entirely and its whole object was buffered into a Vec<u8> by download_object before the post-download 2GB total guard could fire — a single legacy object of arbitrary size could OOM the heap. Resolve each file's size BEFORE downloading: prefer the DB column, fall back to a cheap S3 HEAD (object_size) for NULL rows, and skip a row whose size can't be resolved (HEAD error / object gone). The per-file and running-total caps now apply unconditionally, so nothing is buffered before its size is bounded. The post-download total check stays as a backstop. Regression test: a legacy NULL-size row is still exported with its size resolved via HEAD (included, not dropped or buffered blind). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-18 01:12 UTC
Commit: 998f99ae6099fd1613e42ed42d4e2caa2b9818b3
Parent: b19a53f
2 files changed, +73 insertions, -17 deletions
M server/src/routes/api/exports/content.rs +33 -17
@@ -224,23 +224,39 @@ async fn build_content_export(
224 224 let mut skipped: Vec<String> = Vec::new();
225 225
226 226 for (s3_key, zip_path_entry, db_size) in &files {
227 - // Per-file size pre-check BEFORE downloading so a single 20 GB video
228 - // can't blow the heap before the post-download total check fires.
229 - // The size comes from the DB column written at upload-confirm — no
230 - // S3 HEAD round-trip. The post-download total guard below is the
231 - // backstop for any row with a missing (None) size.
232 - if let Some(size) = db_size {
233 - let size = (*size).max(0) as u64;
234 - if size > MAX_FILE_SIZE {
235 - skipped.push(format!(
236 - "{} (exceeds 500 MB per-file export cap)",
237 - zip_path_entry
238 - ));
239 - continue;
240 - }
241 - if total_size + size > MAX_TOTAL_SIZE {
242 - return Err("Content export exceeds the 2 GB limit. Try exporting a single project instead.".to_string());
243 - }
227 + // Resolve the file size BEFORE downloading so a single huge object
228 + // can't blow the heap before any guard fires. Prefer the DB column
229 + // written at upload-confirm; fall back to a cheap S3 HEAD for legacy
230 + // rows that predate it (a `None` size). A row whose size can't be
231 + // resolved (HEAD error / object gone) is skipped rather than blindly
232 + // buffered — without this, a legacy `None`-size object of arbitrary
233 + // size landed fully in RAM before the post-download total check could
234 + // fire (PERF-1).
235 + let size = match db_size {
236 + Some(s) => (*s).max(0) as u64,
237 + None => match s3_clone.object_size(s3_key).await {
238 + Ok(Some(s)) => s.max(0) as u64,
239 + Ok(None) => {
240 + tracing::warn!(s3_key = %s3_key, "export: object has no size (missing?) — skipping");
241 + skipped.push(zip_path_entry.clone());
242 + continue;
243 + }
244 + Err(e) => {
245 + tracing::warn!(s3_key = %s3_key, error = %e, "export: HEAD failed — skipping");
246 + skipped.push(zip_path_entry.clone());
247 + continue;
248 + }
249 + },
250 + };
251 + if size > MAX_FILE_SIZE {
252 + skipped.push(format!(
253 + "{} (exceeds 500 MB per-file export cap)",
254 + zip_path_entry
255 + ));
256 + continue;
257 + }
258 + if total_size + size > MAX_TOTAL_SIZE {
259 + return Err("Content export exceeds the 2 GB limit. Try exporting a single project instead.".to_string());
244 260 }
245 261
246 262 match s3_clone.download_object(s3_key).await {
M server/tests/workflows/exports.rs +40
@@ -373,3 +373,43 @@ async fn content_export_scoped_to_project_excludes_other_projects() {
373 373 assert!(contains_bytes(&zip, AUDIO_A), "scoped export must include project A's file");
374 374 assert!(!contains_bytes(&zip, AUDIO_B), "scoped export must EXCLUDE project B's file");
375 375 }
376 +
377 + // PERF-1: a legacy row with NULL file_size_bytes (predating the column) must not
378 + // bypass the per-file size guard. The export now resolves its size with an S3
379 + // HEAD before downloading, so the file is still included (and capped) rather than
380 + // dropped or buffered blind. Before the fix, the `None` size skipped the guard
381 + // entirely and the whole object landed in RAM ahead of the post-download check.
382 + #[tokio::test]
383 + async fn content_export_resolves_legacy_null_size_via_head() {
384 + let mut h = TestHarness::with_mocks().await;
385 + let setup = h.create_creator_with_item("ctexnull", "audio", 0).await;
386 + h.trust_user(setup.user_id).await;
387 + h.grant_tier(setup.user_id, "small_files").await;
388 +
389 + const AUDIO: &[u8] = b"FAKE-MP3-LEGACY-NULL-SIZE-9876543210";
390 + upload_audio(&mut h, &setup.item_id, "legacy.mp3", AUDIO).await;
391 +
392 + // Simulate a legacy row predating the file_size_bytes column.
393 + sqlx::query("UPDATE items SET audio_file_size_bytes = NULL WHERE id = $1::uuid")
394 + .bind(&setup.item_id)
395 + .execute(&h.db)
396 + .await
397 + .unwrap();
398 +
399 + let resp = h.client.post_form("/api/export/content", "").await;
400 + assert_eq!(resp.status.as_u16(), 202, "export should be accepted: {} {}", resp.status, resp.text);
401 +
402 + let export_key = await_export_key(&h).await;
403 + let zip = h
404 + .storage
405 + .as_ref()
406 + .unwrap()
407 + .download_object(&export_key)
408 + .await
409 + .expect("export zip must be uploaded");
410 + assert!(zip.len() > 4 && &zip[..4] == b"PK\x03\x04", "must be a real ZIP");
411 + assert!(
412 + contains_bytes(&zip, AUDIO),
413 + "a legacy NULL-size file must still be exported (size resolved via S3 HEAD)"
414 + );
415 + }