Skip to main content

max / makenotwork

12.2 KB · 337 lines History Blame Raw
1 //! Content file export: ZIP archive of audio, covers, videos, versions, and insertions.
2 //!
3 //! Writes the ZIP to a temporary file and uploads via S3 multipart upload,
4 //! so peak memory is O(single_file) regardless of total export size.
5
6 use std::io::Write;
7
8 use axum::{
9 extract::{Query, State},
10 http::header::HeaderMap,
11 response::{IntoResponse, Response},
12 };
13 use serde::Deserialize;
14 use zip::write::SimpleFileOptions;
15
16 use crate::{
17 auth::AuthUser,
18 db,
19 error::{AppError, Result, ResultExt},
20 helpers::is_htmx_request,
21 templates::ExportContentReadyTemplate,
22 AppState,
23 };
24
25 use super::export_error_html;
26
27 /// Query parameters for the content export endpoint.
28 #[derive(Deserialize)]
29 pub(in crate::routes::api) struct ContentExportQuery {
30 /// When set, only export files from this project (useful for large
31 /// catalogs or to stay within the 2GB per-export memory limit).
32 pub project_id: Option<db::ProjectId>,
33 }
34
35 /// Export content files as a ZIP archive uploaded to S3.
36 ///
37 /// Collects audio, covers, version downloads, and insertion clips,
38 /// bundles them with a README.txt manifest, uploads to S3 as a
39 /// temporary export, and returns a presigned download link.
40 ///
41 /// Pass `?project_id=<uuid>` to limit the export to a single project
42 /// (insertions are user-scoped and always excluded from per-project exports).
43 #[tracing::instrument(skip_all, name = "exports::export_content")]
44 pub(in crate::routes::api) async fn export_content(
45 State(state): State<AppState>,
46 headers: HeaderMap,
47 Query(query): Query<ContentExportQuery>,
48 AuthUser(user): AuthUser,
49 ) -> Result<Response> {
50 let is_htmx = is_htmx_request(&headers);
51
52 let s3 = state.s3.as_ref().ok_or_else(|| {
53 AppError::ServiceUnavailable("File storage is not configured".to_string())
54 })?;
55
56 // Collect all S3 keys from items, versions, and insertions
57 let item_keys = db::items::get_user_s3_keys(&state.db, user.id).await?;
58 let version_keys = db::versions::get_user_version_s3_keys(&state.db, user.id).await?;
59
60 // Build the list of (s3_key, zip_path) pairs
61 let mut files: Vec<(String, String)> = Vec::new();
62
63 for item in &item_keys {
64 if let Some(pid) = query.project_id
65 && item.project_id != pid
66 {
67 continue;
68 }
69 let slug = item.project_slug.as_str();
70 let title = sanitize_filename(&item.title);
71 if let Some(ref key) = item.audio_s3_key {
72 let ext = extension_from_key(key);
73 files.push((key.clone(), format!("projects/{}/{}.{}", slug, title, ext)));
74 }
75 if let Some(ref key) = item.cover_s3_key {
76 let ext = extension_from_key(key);
77 files.push((key.clone(), format!("projects/{}/{}-cover.{}", slug, title, ext)));
78 }
79 if let Some(ref key) = item.video_s3_key {
80 let ext = extension_from_key(key);
81 files.push((key.clone(), format!("projects/{}/{}-video.{}", slug, title, ext)));
82 }
83 }
84
85 for ver in &version_keys {
86 if let Some(pid) = query.project_id
87 && ver.project_id != pid
88 {
89 continue;
90 }
91 if let Some(ref key) = ver.s3_key {
92 let slug = ver.project_slug.as_str();
93 let title = sanitize_filename(&ver.item_title);
94 let fname = ver.file_name.as_deref().unwrap_or("file");
95 files.push((key.clone(), format!("projects/{}/{}/v{}-{}", slug, title, ver.version_number, fname)));
96 }
97 }
98
99 // Insertions are user-scoped (not project-scoped), so only include
100 // them when exporting all content (no project_id filter).
101 if query.project_id.is_none() {
102 let insertions = db::content_insertions::list_insertions(&state.db, user.id).await?;
103 for ins in &insertions {
104 let ext = extension_from_key(&ins.storage_key);
105 let title = sanitize_filename(&ins.title);
106 files.push((ins.storage_key.clone(), format!("insertions/{}.{}", title, ext)));
107 }
108 }
109
110 if files.is_empty() {
111 if is_htmx {
112 return Ok(export_error_html("No content files to export."));
113 }
114 return Err(AppError::BadRequest("No content files to export.".to_string()));
115 }
116
117 // Write ZIP to a temporary file, downloading files one at a time.
118 // Peak memory is O(largest_single_file) — the ZIP itself lives on disk.
119 let s3_clone = s3.clone();
120 let username = user.username.to_string();
121
122 let tmp_dir = tempfile::tempdir()
123 .context("create temp dir for export")?;
124 let zip_path = tmp_dir.path().join("export.zip");
125
126 {
127 let zip_file = std::fs::File::create(&zip_path)
128 .context("create export zip file")?;
129 let mut zip = zip::ZipWriter::new(std::io::BufWriter::new(zip_file));
130 let options = SimpleFileOptions::default()
131 .compression_method(zip::CompressionMethod::Stored);
132
133 let mut manifest: Vec<(String, i64)> = Vec::new();
134 let mut total_size: u64 = 0;
135 const MAX_TOTAL_SIZE: u64 = 2 * 1024 * 1024 * 1024; // 2 GB
136 const MAX_FILE_SIZE: u64 = 500 * 1024 * 1024; // 500 MB per file
137 let mut skipped: Vec<String> = Vec::new();
138
139 for (s3_key, zip_path_entry) in &files {
140 // Per-file size pre-check BEFORE downloading so a single 20 GB
141 // video can't blow the heap before the post-download total
142 // check fires. We HEAD the object first; if it's over the
143 // per-file cap, skip it with a clear message in the manifest.
144 // The total-size guard below still catches the 2 GB aggregate.
145 if let Ok(Some(size)) = s3_clone.object_size(s3_key).await {
146 if size as u64 > MAX_FILE_SIZE {
147 skipped.push(format!(
148 "{} (exceeds 500 MB per-file export cap)",
149 zip_path_entry
150 ));
151 continue;
152 }
153 if total_size + size as u64 > MAX_TOTAL_SIZE {
154 let msg = "Content export exceeds 2 GB limit. Try exporting a single project instead.";
155 if is_htmx {
156 return Ok(export_error_html(msg));
157 }
158 return Err(AppError::BadRequest(msg.to_string()));
159 }
160 }
161
162 match s3_clone.download_object(s3_key).await {
163 Ok(data) => {
164 total_size += data.len() as u64;
165 if total_size > MAX_TOTAL_SIZE {
166 let msg = "Content export exceeds 2 GB limit. Try exporting a single project instead.";
167 if is_htmx {
168 return Ok(export_error_html(msg));
169 }
170 return Err(AppError::BadRequest(msg.to_string()));
171 }
172 let file_size = data.len() as i64;
173 zip.start_file(zip_path_entry, options)
174 .context("zip start file")?;
175 zip.write_all(&data)
176 .context("zip write")?;
177 manifest.push((zip_path_entry.clone(), file_size));
178 // data dropped here -- only one file in RAM at a time
179 }
180 Err(e) => {
181 tracing::warn!("Failed to download S3 key {}: {}", s3_key, e);
182 skipped.push(zip_path_entry.clone());
183 }
184 }
185 }
186
187 if manifest.is_empty() {
188 let msg = "Could not download any files from storage. Please try again later.";
189 if is_htmx {
190 return Ok(export_error_html(msg));
191 }
192 return Err(AppError::Storage(msg.to_string()));
193 }
194
195 // Build README.txt as the last ZIP entry
196 let now = chrono::Utc::now();
197 let mut readme = format!(
198 "Makenot.work Content Export\n\
199 Creator: {}\n\
200 Exported: {}\n\
201 Files: {}\n\n\
202 Manifest:\n",
203 username,
204 now.format("%Y-%m-%d %H:%M:%S UTC"),
205 manifest.len(),
206 );
207 for (path, size) in &manifest {
208 readme.push_str(&format!(" {} ({})\n", path, crate::helpers::format_file_size(*size)));
209 }
210 if !skipped.is_empty() {
211 readme.push_str(&format!("\nSkipped ({} files could not be downloaded):\n", skipped.len()));
212 for path in &skipped {
213 readme.push_str(&format!(" {}\n", path));
214 }
215 }
216 readme.push_str("\nNote: Git repositories are not included in this export.\n");
217 readme.push_str("Clone them separately: git clone https://makenot.work/source/<username>/<repo>.git\n");
218 zip.start_file("README.txt", options)
219 .context("zip start readme")?;
220 zip.write_all(readme.as_bytes())
221 .context("zip write readme")?;
222
223 zip.finish()
224 .context("zip finish")?;
225 }
226
227 // Upload ZIP to S3 via multipart upload (streams from disk in 10 MB parts)
228 let timestamp = chrono::Utc::now().format("%Y%m%d-%H%M%S");
229 let export_key = format!("{}/exports/content-{}.zip", user.id, timestamp);
230 if let Err(e) = s3.upload_multipart(&export_key, "application/zip", &zip_path).await {
231 tracing::error!(error = ?e, "Failed to upload content export ZIP to S3");
232 if is_htmx {
233 return Ok(export_error_html("Failed to prepare download. Please try again."));
234 }
235 return Err(e);
236 }
237
238 // Generate presigned download URL (1 hour)
239 let download_url = match s3.presign_download(&export_key, Some(3600)).await {
240 Ok(url) => url,
241 Err(e) => {
242 tracing::error!(error = ?e, "Failed to generate presigned URL for content export");
243 if is_htmx {
244 return Ok(export_error_html("Export created but download link failed. Please try again."));
245 }
246 return Err(e);
247 }
248 };
249
250 if is_htmx {
251 return Ok(ExportContentReadyTemplate { download_url }.into_response());
252 }
253
254 // Direct API call: redirect to presigned URL
255 Response::builder()
256 .status(303)
257 .header("Location", &download_url)
258 .body("".into())
259 .context("build export redirect response")
260 }
261
262 /// Extract file extension from an S3 key (e.g. "user/item/audio/track.mp3" -> "mp3").
263 fn extension_from_key(key: &str) -> &str {
264 key.rsplit('.').next().unwrap_or("bin")
265 }
266
267 /// Sanitize a title for use as a filename in the ZIP archive.
268 fn sanitize_filename(name: &str) -> String {
269 name.chars()
270 .map(|c| if c.is_alphanumeric() || c == '-' || c == '_' || c == ' ' { c } else { '_' })
271 .collect::<String>()
272 .trim()
273 .to_string()
274 }
275
276 #[cfg(test)]
277 mod tests {
278 use super::*;
279
280 #[test]
281 fn extension_from_key_mp3() {
282 assert_eq!(extension_from_key("user/item/audio/track.mp3"), "mp3");
283 }
284
285 #[test]
286 fn extension_from_key_nested_path() {
287 assert_eq!(extension_from_key("a/b/c/file.tar.gz"), "gz");
288 }
289
290 #[test]
291 fn extension_from_key_no_dot_returns_whole_segment() {
292 // rsplit('.').next() returns the whole string when no dot is present
293 assert_eq!(extension_from_key("user/item/audio/noext"), "user/item/audio/noext");
294 }
295
296 #[test]
297 fn extension_from_key_empty_returns_bin() {
298 // rsplit('.').next() on "" returns Some(""), which unwrap_or("bin") keeps as ""
299 assert_eq!(extension_from_key(""), "");
300 }
301
302 #[test]
303 fn extension_from_key_dot_only() {
304 assert_eq!(extension_from_key("file."), "");
305 }
306
307 #[test]
308 fn sanitize_filename_passthrough() {
309 assert_eq!(sanitize_filename("My Track"), "My Track");
310 }
311
312 #[test]
313 fn sanitize_filename_special_chars() {
314 assert_eq!(sanitize_filename("hello/world:2"), "hello_world_2");
315 }
316
317 #[test]
318 fn sanitize_filename_preserves_hyphens_underscores() {
319 assert_eq!(sanitize_filename("my-file_name"), "my-file_name");
320 }
321
322 #[test]
323 fn sanitize_filename_trims_whitespace() {
324 assert_eq!(sanitize_filename(" padded "), "padded");
325 }
326
327 #[test]
328 fn sanitize_filename_empty() {
329 assert_eq!(sanitize_filename(""), "");
330 }
331
332 #[test]
333 fn sanitize_filename_all_special() {
334 assert_eq!(sanitize_filename("@#$%"), "____");
335 }
336 }
337