Skip to main content

max / makenotwork

perf: stream content-export files through a disk spool, not a full RAM buffer Each file was downloaded into a Vec<u8> (up to 500 MB) before being written into the zip; with three concurrent exports that was ~1.5 GB of transient heap. Stream each object to a per-file spool on disk, then io::copy it into the zip on the blocking pool, so peak memory is one streaming chunk regardless of file size. The zip already spooled to a tempfile; this makes the file body genuinely O(stream) too, matching the scan pipeline's large-file handling. Per-file cap is re-enforced mid-stream as a backstop for legacy null-size rows. ultra-fuzz Run 5 S2. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-24 01:44 UTC
Commit: dfde7a93bce979f929715c6c769b1c06276d1d49
Parent: 3e42941
1 file changed, +88 insertions, -23 deletions
M server/src/routes/api/exports/content.rs +88 -23
@@ -172,6 +172,54 @@ pub(in crate::routes::api) async fn export_content(
172 172 .context("build export accepted response")
173 173 }
174 174
175 + /// Failure modes of [`spool_s3_object_to_file`].
176 + enum SpoolError {
177 + /// The object exceeded the per-file cap mid-stream (never fully buffered).
178 + TooLarge,
179 + /// An S3 or filesystem error; the message is log-only (never user-facing).
180 + Io(String),
181 + }
182 +
183 + /// Stream an S3 object to `dest`, returning the number of bytes written. Aborts
184 + /// with [`SpoolError::TooLarge`] the moment the running total exceeds
185 + /// `max_bytes`, so a mis-sized object can't fill the disk. Peak memory is one
186 + /// streaming chunk — this is what keeps the export's footprint at O(chunk)
187 + /// rather than O(file).
188 + async fn spool_s3_object_to_file(
189 + s3: &std::sync::Arc<dyn crate::storage::StorageBackend>,
190 + s3_key: &str,
191 + dest: &std::path::Path,
192 + max_bytes: u64,
193 + ) -> std::result::Result<u64, SpoolError> {
194 + use tokio::io::AsyncWriteExt;
195 +
196 + let mut stream = s3
197 + .download_stream(s3_key)
198 + .await
199 + .map_err(|e| SpoolError::Io(e.to_string()))?;
200 + let mut file = tokio::fs::File::create(dest)
201 + .await
202 + .map_err(|e| SpoolError::Io(e.to_string()))?;
203 + let mut written: u64 = 0;
204 + while let Some(chunk) = stream
205 + .try_next()
206 + .await
207 + .map_err(|e| SpoolError::Io(e.to_string()))?
208 + {
209 + written += chunk.len() as u64;
210 + if written > max_bytes {
211 + return Err(SpoolError::TooLarge);
212 + }
213 + file.write_all(&chunk)
214 + .await
215 + .map_err(|e| SpoolError::Io(e.to_string()))?;
216 + }
217 + file.flush()
218 + .await
219 + .map_err(|e| SpoolError::Io(e.to_string()))?;
220 + Ok(written)
221 + }
222 +
175 223 /// Build the content-export ZIP off the request path: download every file,
176 224 /// zip to a tempfile, multipart-upload, and return a 1-hour presigned download
177 225 /// URL. Holds the [`EXPORT_LIMITER`] permit for its lifetime so a burst can't
@@ -259,34 +307,51 @@ async fn build_content_export(
259 307 return Err("Content export exceeds the 2 GB limit. Try exporting a single project instead.".to_string());
260 308 }
261 309
262 - match s3_clone.download_object(s3_key).await {
263 - Ok(data) => {
264 - total_size += data.len() as u64;
265 - if total_size > MAX_TOTAL_SIZE {
266 - return Err("Content export exceeds the 2 GB limit. Try exporting a single project instead.".to_string());
267 - }
268 - let file_size = data.len() as i64;
269 - // Move writer + file bytes onto the blocking pool, write, get
270 - // the writer back. `data` drops inside the task afterward, so
271 - // only one file is in RAM at a time.
272 - let entry = zip_path_entry.clone();
273 - zip = tokio::task::spawn_blocking(
274 - move || -> std::result::Result<_, zip::result::ZipError> {
275 - zip.start_file(&entry, options)?;
276 - zip.write_all(&data)?;
277 - Ok(zip)
278 - },
279 - )
280 - .await
281 - .map_err(|e| format!("join zip write task: {e}"))?
282 - .map_err(|e| format!("write file into export zip: {e}"))?;
283 - manifest.push((zip_path_entry.clone(), file_size));
310 + // Stream the object to a per-file spool on disk instead of buffering
311 + // the whole (up to 500 MB) file in RAM. Peak memory is one streaming
312 + // chunk, not the full object — three concurrent exports previously
313 + // held ~1.5 GB of file bytes between them. The scan pipeline streams
314 + // large files the same way; the exporter now matches it (Run #5 Perf
315 + // S2). The MAX_FILE_SIZE guard is re-enforced mid-stream as a backstop
316 + // for legacy rows whose size couldn't be resolved above.
317 + let part_path = tmp_dir.path().join(format!("part-{}", manifest.len()));
318 + let part_bytes = match spool_s3_object_to_file(&s3_clone, s3_key, &part_path, MAX_FILE_SIZE).await {
319 + Ok(n) => n,
320 + Err(SpoolError::TooLarge) => {
321 + skipped.push(format!("{} (exceeds 500 MB per-file export cap)", zip_path_entry));
322 + let _ = tokio::fs::remove_file(&part_path).await;
323 + continue;
284 324 }
285 - Err(e) => {
325 + Err(SpoolError::Io(e)) => {
286 326 tracing::warn!("Failed to download S3 key {}: {}", s3_key, e);
287 327 skipped.push(zip_path_entry.clone());
328 + let _ = tokio::fs::remove_file(&part_path).await;
329 + continue;
288 330 }
331 + };
332 + total_size += part_bytes;
333 + if total_size > MAX_TOTAL_SIZE {
334 + return Err("Content export exceeds the 2 GB limit. Try exporting a single project instead.".to_string());
289 335 }
336 + let file_size = part_bytes as i64;
337 + // Copy the spooled file into the zip on the blocking pool (the zip
338 + // crate's IO is synchronous). The reader streams from disk via
339 + // io::copy, so RAM stays at the copy buffer size, not the file size.
340 + let entry = zip_path_entry.clone();
341 + let copy_path = part_path.clone();
342 + zip = tokio::task::spawn_blocking(
343 + move || -> std::result::Result<_, zip::result::ZipError> {
344 + zip.start_file(&entry, options)?;
345 + let mut reader = std::fs::File::open(&copy_path)?;
346 + std::io::copy(&mut reader, &mut zip)?;
347 + Ok(zip)
348 + },
349 + )
350 + .await
351 + .map_err(|e| format!("join zip write task: {e}"))?
352 + .map_err(|e| format!("write file into export zip: {e}"))?;
353 + let _ = tokio::fs::remove_file(&part_path).await;
354 + manifest.push((zip_path_entry.clone(), file_size));
290 355 }
291 356
292 357 if manifest.is_empty() {