Skip to main content

max / makenotwork

42.9 KB · 1120 lines History Blame Raw
1 //! S3-compatible storage client for Hetzner Object Storage
2 //!
3 //! Provides presigned URL generation for client-direct uploads/downloads.
4 //! Delegates S3 operations to the shared `s3_storage` crate.
5 //!
6 //! See also: `/docs/guide/audio`, `/docs/guide/video`, `/docs/guide/software`
7
8 use std::str::FromStr;
9
10 use crate::config::StorageConfig;
11 use crate::db::{ItemId, ProjectId, UserId};
12 use crate::error::{AppError, Result, ResultExt};
13
14 /// Allowed audio file extensions and their MIME types
15 const ALLOWED_AUDIO_TYPES: &[(&str, &str)] = &[
16 ("mp3", "audio/mpeg"),
17 ("wav", "audio/wav"),
18 ("m4a", "audio/mp4"),
19 ("ogg", "audio/ogg"),
20 ("flac", "audio/flac"),
21 ("aac", "audio/aac"),
22 ];
23
24 /// Allowed image file extensions and their MIME types
25 const ALLOWED_IMAGE_TYPES: &[(&str, &str)] = &[
26 ("jpg", "image/jpeg"),
27 ("jpeg", "image/jpeg"),
28 ("png", "image/png"),
29 ("webp", "image/webp"),
30 ("gif", "image/gif"),
31 ];
32
33 /// Allowed video file extensions and their MIME types
34 const ALLOWED_VIDEO_TYPES: &[(&str, &str)] = &[
35 ("mp4", "video/mp4"),
36 ("webm", "video/webm"),
37 ("mov", "video/quicktime"),
38 ];
39
40 /// MIME types accepted for video uploads
41 const ALLOWED_VIDEO_MIMES: &[&str] = &[
42 "video/mp4",
43 "video/webm",
44 "video/quicktime",
45 ];
46
47 /// Allowed download file extensions and their MIME types
48 /// Browsers are inconsistent about MIME types for binary downloads,
49 /// so we accept several common types for each extension.
50 const ALLOWED_DOWNLOAD_TYPES: &[(&str, &str)] = &[
51 ("zip", "application/zip"),
52 ("dmg", "application/x-apple-diskimage"),
53 ("exe", "application/octet-stream"),
54 ("appimage", "application/octet-stream"),
55 ("deb", "application/octet-stream"),
56 ("clap", "application/octet-stream"),
57 ("vst3", "application/octet-stream"),
58 ];
59
60 /// MIME types accepted for download uploads (browsers vary widely)
61 const ALLOWED_DOWNLOAD_MIMES: &[&str] = &[
62 "application/octet-stream",
63 "application/zip",
64 "application/x-zip-compressed",
65 "application/x-apple-diskimage",
66 "application/x-diskcopy",
67 "application/x-msi",
68 "application/x-ole-storage",
69 "application/gzip",
70 "application/x-tar",
71 "application/x-gtar",
72 "application/x-compressed",
73 "application/x-executable",
74 "application/x-deb",
75 "application/vnd.debian.binary-package",
76 ];
77
78 /// Allowed download file extensions (checked separately from MIME)
79 const ALLOWED_DOWNLOAD_EXTENSIONS: &[&str] = &[
80 "zip", "dmg", "exe", "msi", "appimage", "deb", "tar.gz", "clap", "vst3",
81 ];
82
83 /// Maximum file sizes in bytes
84 const MAX_AUDIO_SIZE: u64 = 500 * 1024 * 1024; // 500 MB
85 const MAX_IMAGE_SIZE: u64 = 10 * 1024 * 1024; // 10 MB
86 const MAX_DOWNLOAD_SIZE: u64 = 500 * 1024 * 1024; // 500 MB
87 const MAX_INSERTION_SIZE: u64 = 500 * 1024 * 1024; // 500 MB
88 const MAX_VIDEO_SIZE: u64 = 20 * 1024 * 1024 * 1024; // 20 GB
89 const MAX_MEDIA_IMAGE_SIZE: u64 = 10 * 1024 * 1024; // 10 MB
90 const MAX_MEDIA_VIDEO_SIZE: u64 = 20 * 1024 * 1024 * 1024; // 20 GB
91
92 /// Default presigned URL expiration.
93 /// 1 hour balances usability (large uploads over slow connections) against
94 /// security (limiting the window for URL leakage). Overridable per-call.
95 const PRESIGN_EXPIRY_SECS: u64 = 3600;
96
97 /// File type categories for upload
98 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
99 pub enum FileType {
100 Audio,
101 Cover,
102 Download,
103 Insertion,
104 Video,
105 /// Media library image (user-scoped, for inline markdown content).
106 MediaImage,
107 /// Media library video (user-scoped, for inline markdown content).
108 MediaVideo,
109 }
110
111 /// How the generic `/api/upload/confirm` handler confirms a file type onto an
112 /// `items` row. Returned by [`FileType::generic_item_confirm`], whose `match`
113 /// is exhaustive — adding a `FileType` variant fails the build until its
114 /// posture is declared here, so a new type can't silently fall into the wrong
115 /// column set (the `Cover` branch that wrote `cover_s3_key` but never
116 /// `cover_image_url`, leaving an invisible cover — Run #13).
117 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
118 pub enum GenericItemConfirm {
119 /// Confirmable by the generic handler: write these two columns on the item.
120 /// Only types fully described by `(s3_key, size)` belong here — anything
121 /// that needs an extra column (e.g. a CDN render URL) must use a dedicated
122 /// route instead.
123 Columns { s3_key: &'static str, size: &'static str },
124 /// Not confirmable by the generic handler — use this dedicated route. The
125 /// handler rejects the request (after cleaning up the staged object) so a
126 /// misrouted confirm never half-writes the row.
127 UseRoute(&'static str),
128 }
129
130 impl FileType {
131 /// Declare, exhaustively, how `/api/upload/confirm` treats this type.
132 /// See [`GenericItemConfirm`] for why this is the single source of truth.
133 pub fn generic_item_confirm(self) -> GenericItemConfirm {
134 match self {
135 FileType::Audio => GenericItemConfirm::Columns {
136 s3_key: "audio_s3_key",
137 size: "audio_file_size_bytes",
138 },
139 FileType::Video => GenericItemConfirm::Columns {
140 s3_key: "video_s3_key",
141 size: "video_file_size_bytes",
142 },
143 // Cover IS confirmable, but it must also set `cover_image_url` (the
144 // CDN render source). The generic two-column writer can't, so covers
145 // go through the dedicated route that writes all three atomically.
146 FileType::Cover => GenericItemConfirm::UseRoute("/api/items/image/confirm"),
147 FileType::Download => {
148 GenericItemConfirm::UseRoute("/api/versions/{version_id}/upload/*")
149 }
150 FileType::Insertion => {
151 GenericItemConfirm::UseRoute("/api/users/me/insertions/*")
152 }
153 FileType::MediaImage | FileType::MediaVideo => {
154 GenericItemConfirm::UseRoute("/api/media/*")
155 }
156 }
157 }
158
159 pub fn as_str(&self) -> &'static str {
160 match self {
161 FileType::Audio => "audio",
162 FileType::Cover => "cover",
163 FileType::Download => "download",
164 FileType::Insertion => "insertion",
165 FileType::Video => "video",
166 FileType::MediaImage => "media_image",
167 FileType::MediaVideo => "media_video",
168 }
169 }
170
171 pub fn max_size(&self) -> u64 {
172 match self {
173 FileType::Audio => MAX_AUDIO_SIZE,
174 FileType::Cover => MAX_IMAGE_SIZE,
175 FileType::Download => MAX_DOWNLOAD_SIZE,
176 FileType::Insertion => MAX_INSERTION_SIZE,
177 FileType::Video => MAX_VIDEO_SIZE,
178 FileType::MediaImage => MAX_MEDIA_IMAGE_SIZE,
179 FileType::MediaVideo => MAX_MEDIA_VIDEO_SIZE,
180 }
181 }
182
183 pub fn allowed_types(&self) -> &'static [(&'static str, &'static str)] {
184 match self {
185 FileType::Audio | FileType::Insertion => ALLOWED_AUDIO_TYPES,
186 FileType::Cover | FileType::MediaImage => ALLOWED_IMAGE_TYPES,
187 FileType::Download => ALLOWED_DOWNLOAD_TYPES,
188 FileType::Video | FileType::MediaVideo => ALLOWED_VIDEO_TYPES,
189 }
190 }
191 }
192
193 impl FromStr for FileType {
194 type Err = String;
195
196 fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
197 match s.to_lowercase().as_str() {
198 "audio" => Ok(FileType::Audio),
199 "cover" | "image" => Ok(FileType::Cover),
200 "download" => Ok(FileType::Download),
201 "insertion" => Ok(FileType::Insertion),
202 "video" => Ok(FileType::Video),
203 "media_image" => Ok(FileType::MediaImage),
204 "media_video" => Ok(FileType::MediaVideo),
205 _ => Err(format!("Invalid file type: {}", s)),
206 }
207 }
208 }
209
210 /// Cache-Control value for immutable content (builds, audio, covers).
211 /// One year with immutable directive — Cloudflare and browsers cache indefinitely.
212 pub const CACHE_CONTROL_IMMUTABLE: &str = "public, max-age=31536000, immutable";
213
214 /// Abstract storage backend — implemented by `S3Client` (production) and
215 /// `InMemoryStorage` (tests). Routes access storage through this trait.
216 #[async_trait::async_trait]
217 pub trait StorageBackend: Send + Sync {
218 /// Generate a presigned upload URL. `max_bytes`, when set, is signed into
219 /// the URL as `Content-Length` so S3 itself enforces the size cap at the
220 /// protocol level (prevents oversized PUTs from burning bandwidth before
221 /// hitting the post-PUT delete-and-charge fallback).
222 async fn presign_upload(&self, s3_key: &str, content_type: &str, expiry_secs: Option<u64>, cache_control: Option<&str>, max_bytes: Option<i64>) -> Result<String>;
223 async fn presign_download(&self, s3_key: &str, expiry_secs: Option<u64>) -> Result<String>;
224 async fn object_exists(&self, s3_key: &str) -> Result<bool>;
225 async fn object_size(&self, s3_key: &str) -> Result<Option<i64>>;
226 async fn download_object(&self, s3_key: &str) -> Result<Vec<u8>>;
227 /// Stream the object body without buffering the whole payload. Callers
228 /// drive the stream to disk (scanner spool) or to a layer that consumes
229 /// chunks directly.
230 async fn download_stream(&self, s3_key: &str) -> Result<s3_storage::ByteStream>;
231 async fn upload_object(&self, s3_key: &str, content_type: &str, data: Vec<u8>, cache_control: Option<&str>) -> Result<()>;
232 async fn delete_object(&self, s3_key: &str) -> Result<()>;
233 /// Delete a batch of objects in a single S3 `DeleteObjects` request
234 /// (up to 1000 keys/call). Default loops `delete_object` so test backends
235 /// don't have to implement it, but production should override.
236 async fn delete_objects(&self, keys: &[String]) -> Result<()> {
237 for k in keys {
238 if let Err(e) = self.delete_object(k).await {
239 tracing::warn!(key = %k, error = ?e, "delete_objects: per-key delete failed");
240 }
241 }
242 Ok(())
243 }
244 /// Delete all objects under a key prefix. Default logs a warning (no-op).
245 async fn delete_prefix(&self, _prefix: &str) -> Result<()> {
246 tracing::warn!("delete_prefix called on a storage backend that does not implement it");
247 Ok(())
248 }
249 /// Upload a file via S3 multipart upload. Default falls back to single upload.
250 async fn upload_multipart(&self, s3_key: &str, content_type: &str, file_path: &std::path::Path) -> Result<()> {
251 let data = tokio::fs::read(file_path)
252 .await
253 .context("read multipart upload source file")?;
254 self.upload_object(s3_key, content_type, data, None).await
255 }
256 async fn check_connectivity(&self) -> std::result::Result<(), String>;
257 fn bucket(&self) -> &str;
258 }
259
260 /// S3 client wrapper for presigned URL operations.
261 /// Delegates S3 operations to `s3_storage::S3Client`.
262 #[derive(Clone)]
263 pub struct S3Client {
264 inner: s3_storage::S3Client,
265 }
266
267 impl S3Client {
268 /// Create a new S3 client from storage configuration.
269 ///
270 /// Configures CORS on the bucket at startup so browser PUT uploads to
271 /// presigned URLs work without manual bucket configuration.
272 pub async fn new(config: &StorageConfig, host_url: &str) -> Result<Self> {
273 let s3_config = s3_storage::S3Config {
274 endpoint: config.endpoint.clone(),
275 bucket: config.bucket.clone(),
276 access_key: config.access_key.clone(),
277 secret_key: config.secret_key.clone(),
278 region: config.region.clone(),
279 };
280
281 let inner = s3_storage::S3Client::new(&s3_config)
282 .await
283 .map_err(AppError::Storage)?;
284
285 inner.configure_cors(host_url).await;
286
287 Ok(S3Client { inner })
288 }
289
290 /// Generate a consistent S3 key for an object
291 /// Format: {user_id}/{item_id}/{file_type}/{filename}
292 pub fn generate_key(
293 user_id: UserId,
294 item_id: ItemId,
295 file_type: FileType,
296 filename: &str,
297 ) -> String {
298 let safe_filename = sanitize_filename(filename);
299 format!(
300 "{}/{}/{}/{}",
301 user_id,
302 item_id,
303 file_type.as_str(),
304 safe_filename
305 )
306 }
307
308 /// Generate an S3 key for a reusable insertion clip (not tied to any item).
309 /// Format: {user_id}/insertions/{filename}
310 pub fn generate_insertion_key(user_id: UserId, filename: &str) -> String {
311 let safe_filename = sanitize_filename(filename);
312 format!("{}/insertions/{}", user_id, safe_filename)
313 }
314
315 /// Generate an S3 key for a media library file.
316 /// Format: `{user_id}/media/{folder}/{filename}` (or `{user_id}/media/{filename}` for root folder).
317 pub fn generate_media_key(user_id: UserId, folder: &str, filename: &str) -> String {
318 let safe_filename = sanitize_filename(filename);
319 let safe_folder = sanitize_folder(folder);
320 if safe_folder.is_empty() {
321 format!("{}/media/{}", user_id, safe_filename)
322 } else {
323 format!("{}/media/{}/{}", user_id, safe_folder, safe_filename)
324 }
325 }
326
327 /// Generate an S3 key for a project image (logo/avatar).
328 /// Format: projects/{project_id}/image/{sanitized_filename}
329 pub fn generate_project_image_key(project_id: ProjectId, filename: &str) -> String {
330 let safe_filename = sanitize_filename(filename);
331 format!("projects/{}/image/{}", project_id, safe_filename)
332 }
333
334 /// Generate an S3 key for an item gallery image. A per-image uuid segment
335 /// keeps multiple gallery uploads from colliding (unlike the single cover,
336 /// which has a fixed `cover/` path).
337 /// Format: {user_id}/{item_id}/gallery/{image_uuid}/{sanitized_filename}
338 pub fn generate_item_gallery_key(
339 user_id: UserId,
340 item_id: ItemId,
341 image_uuid: uuid::Uuid,
342 filename: &str,
343 ) -> String {
344 let safe_filename = sanitize_filename(filename);
345 format!("{}/{}/gallery/{}/{}", user_id, item_id, image_uuid, safe_filename)
346 }
347
348 /// Generate an S3 key for a project gallery image.
349 /// Format: projects/{project_id}/gallery/{image_uuid}/{sanitized_filename}
350 pub fn generate_project_gallery_key(
351 project_id: ProjectId,
352 image_uuid: uuid::Uuid,
353 filename: &str,
354 ) -> String {
355 let safe_filename = sanitize_filename(filename);
356 format!("projects/{}/gallery/{}/{}", project_id, image_uuid, safe_filename)
357 }
358
359 /// Validate content type for the given file type
360 pub fn validate_content_type(file_type: FileType, content_type: &str) -> Result<()> {
361 let is_valid = if file_type == FileType::Download {
362 ALLOWED_DOWNLOAD_MIMES.contains(&content_type)
363 } else if file_type == FileType::Video {
364 ALLOWED_VIDEO_MIMES.contains(&content_type)
365 } else {
366 let allowed = file_type.allowed_types();
367 allowed.iter().any(|(_, mime)| *mime == content_type)
368 };
369
370 if !is_valid {
371 let allowed_list = if file_type == FileType::Download {
372 ALLOWED_DOWNLOAD_MIMES.join(", ")
373 } else if file_type == FileType::Video {
374 ALLOWED_VIDEO_MIMES.join(", ")
375 } else {
376 let allowed = file_type.allowed_types();
377 allowed.iter().map(|(_, m)| *m).collect::<Vec<_>>().join(", ")
378 };
379 return Err(AppError::InvalidFileType(format!(
380 "Content type '{}' not allowed. Allowed types: {}",
381 content_type,
382 allowed_list
383 )));
384 }
385
386 Ok(())
387 }
388
389 /// Validate file extension for the given file type
390 pub fn validate_extension(file_type: FileType, filename: &str) -> Result<()> {
391 if file_type == FileType::Download {
392 let lower = filename.to_lowercase();
393 let is_valid = ALLOWED_DOWNLOAD_EXTENSIONS.iter().any(|ext| lower.ends_with(&format!(".{}", ext)));
394 if !is_valid {
395 return Err(AppError::InvalidFileType(format!(
396 "File extension not allowed. Allowed extensions: {}",
397 ALLOWED_DOWNLOAD_EXTENSIONS.join(", ")
398 )));
399 }
400 return Ok(());
401 }
402
403 let extension = filename
404 .rsplit('.')
405 .next()
406 .map(|s| s.to_lowercase())
407 .unwrap_or_default();
408
409 let allowed = file_type.allowed_types();
410 let is_valid = allowed.iter().any(|(ext, _)| *ext == extension);
411
412 if !is_valid {
413 let allowed_exts: Vec<&str> = allowed.iter().map(|(e, _)| *e).collect();
414 return Err(AppError::InvalidFileType(format!(
415 "File extension '.{}' not allowed. Allowed extensions: {}",
416 extension,
417 allowed_exts.join(", ")
418 )));
419 }
420
421 Ok(())
422 }
423
424 /// Generate a presigned URL for uploading a file. `max_bytes`, when set,
425 /// binds `Content-Length` into the signature — S3 will reject any PUT
426 /// whose actual body length differs from `max_bytes`.
427 pub async fn presign_upload(
428 &self,
429 s3_key: &str,
430 content_type: &str,
431 expiry_secs: Option<u64>,
432 cache_control: Option<&str>,
433 max_bytes: Option<i64>,
434 ) -> Result<String> {
435 self.inner
436 .presign_upload(s3_key, content_type, expiry_secs.unwrap_or(PRESIGN_EXPIRY_SECS), cache_control, max_bytes)
437 .await
438 .map_err(AppError::Storage)
439 }
440
441 /// Generate a presigned URL for downloading/streaming a file
442 pub async fn presign_download(
443 &self,
444 s3_key: &str,
445 expiry_secs: Option<u64>,
446 ) -> Result<String> {
447 self.inner
448 .presign_download(s3_key, expiry_secs.unwrap_or(PRESIGN_EXPIRY_SECS))
449 .await
450 .map_err(AppError::Storage)
451 }
452
453 /// Check if an object exists in S3
454 pub async fn object_exists(&self, s3_key: &str) -> Result<bool> {
455 self.inner.object_exists(s3_key).await.map_err(AppError::Storage)
456 }
457
458 /// Get the size of an object in S3 (bytes), or None if not found.
459 pub async fn object_size(&self, s3_key: &str) -> Result<Option<i64>> {
460 self.inner.object_size(s3_key).await.map_err(AppError::Storage)
461 }
462
463 /// Download an object's bytes from S3
464 pub async fn download_object(&self, s3_key: &str) -> Result<Vec<u8>> {
465 self.inner
466 .download(s3_key)
467 .await
468 .map(|(bytes, _content_type)| bytes)
469 .map_err(AppError::Storage)
470 }
471
472 /// Stream an object's body from S3 without buffering. See trait docs.
473 pub async fn download_stream(&self, s3_key: &str) -> Result<s3_storage::ByteStream> {
474 self.inner
475 .download_stream(s3_key)
476 .await
477 .map_err(AppError::Storage)
478 }
479
480 /// Upload an object to S3 from bytes
481 pub async fn upload_object(
482 &self,
483 s3_key: &str,
484 content_type: &str,
485 data: Vec<u8>,
486 cache_control: Option<&str>,
487 ) -> Result<()> {
488 self.inner
489 .upload(s3_key, content_type, data, cache_control)
490 .await
491 .map_err(AppError::Storage)
492 }
493
494 /// Delete an object from S3
495 pub async fn delete_object(&self, s3_key: &str) -> Result<()> {
496 self.inner.delete(s3_key).await.map_err(AppError::Storage)
497 }
498
499 /// Batched S3 delete (`DeleteObjects`, up to 1000 keys per call).
500 /// Chunks larger slices into 1000-key batches and logs per-key failures
501 /// without bubbling — the pending_s3_deletions queue is the safety net.
502 pub async fn delete_objects(&self, keys: &[String]) -> Result<()> {
503 if keys.is_empty() {
504 return Ok(());
505 }
506 for chunk in keys.chunks(1000) {
507 match self.inner.delete_objects(chunk).await {
508 Ok(failures) => {
509 for (k, msg) in failures {
510 tracing::warn!(key = %k, error = %msg, "S3 delete_objects: key-level failure");
511 }
512 }
513 Err(e) => return Err(AppError::Storage(e)),
514 }
515 }
516 Ok(())
517 }
518
519 /// Upload a file to S3 using multipart upload (10 MB parts).
520 pub async fn upload_multipart(&self, s3_key: &str, content_type: &str, file_path: &std::path::Path) -> Result<()> {
521 self.inner
522 .upload_multipart(s3_key, content_type, file_path, None)
523 .await
524 .map_err(AppError::Storage)
525 }
526
527 /// Lightweight connectivity check — issues a list with max_keys(0).
528 pub async fn check_connectivity(&self) -> std::result::Result<(), String> {
529 self.inner.check_connectivity().await
530 }
531 }
532
533 /// Sanitize a filename: keep only alphanumeric, dots, dashes, and underscores.
534 /// Prevents path traversal, shell injection, and S3 key encoding issues.
535 /// Falls back to "file" if the sanitized result has no basename (only extension or empty).
536 ///
537 /// **By design**: the sanitizer keeps `.`/`-`/`_` and strips everything else,
538 /// so e.g. `"../etc/passwd"` collapses to `"..etcpasswd"` — preserved as a
539 /// literal filename, not as a directory traversal. The unit test pins this
540 /// behavior: we don't reject names containing `..`, we just guarantee the
541 /// output has no path separators. S3 keys are namespaced by user/item ID
542 /// upstream, so a flat literal here can't escape the user's prefix.
543 fn sanitize_filename(filename: &str) -> String {
544 let sanitized: String = filename
545 .chars()
546 .filter(|c| c.is_alphanumeric() || *c == '.' || *c == '-' || *c == '_')
547 .collect();
548 // Ensure the result has a non-empty basename (not just ".ext" or empty)
549 let stem = std::path::Path::new(&sanitized)
550 .file_stem()
551 .and_then(|s| s.to_str())
552 .unwrap_or("");
553 if stem.is_empty() {
554 let ext = std::path::Path::new(&sanitized)
555 .extension()
556 .and_then(|s| s.to_str())
557 .unwrap_or("");
558 if ext.is_empty() {
559 "file".to_string()
560 } else {
561 format!("file.{ext}")
562 }
563 } else {
564 sanitized
565 }
566 }
567
568 /// Sanitize a folder name: keep only alphanumeric, dashes, and underscores.
569 /// Rejects path traversal (`..`) and slashes. Returns empty string for root folder.
570 pub fn sanitize_folder(folder: &str) -> String {
571 let trimmed = folder.trim();
572 if trimmed.is_empty() {
573 return String::new();
574 }
575 // Reject any path traversal
576 if trimmed.contains("..") {
577 return String::new();
578 }
579 trimmed
580 .chars()
581 .filter(|c| c.is_alphanumeric() || *c == '-' || *c == '_')
582 .collect()
583 }
584
585 /// Extract the S3 key from a CDN or presigned URL.
586 ///
587 /// Accepts two URL shapes:
588 /// - **CDN**: `https://cdn.example.com/{s3_key}` — caller supplies the
589 /// CDN base; the function strips it verbatim.
590 /// - **Path-style S3**: `https://{host}/{bucket}/{s3_key}?...` — caller
591 /// supplies the bucket name; the function strips host + bucket prefix.
592 ///
593 /// Returns `None` if neither prefix matches. Query strings (presigned URL
594 /// signatures) are stripped before returning.
595 ///
596 /// **Why explicit prefixes**: the prior implementation used
597 /// `find("projects/")` as a heuristic, which would silently mis-key any URL
598 /// whose path happened to contain the literal substring (e.g. a key with a
599 /// `projects/` suffix inside a user folder). Passing the known CDN base and
600 /// bucket eliminates the heuristic entirely.
601 pub fn extract_s3_key_from_url(
602 url: &str,
603 cdn_base: Option<&str>,
604 bucket: Option<&str>,
605 s3_endpoint: Option<&str>,
606 ) -> Option<String> {
607 let no_query = url.split('?').next()?;
608
609 // Try CDN-base prefix first.
610 if let Some(base) = cdn_base {
611 let base = base.trim_end_matches('/');
612 if let Some(rest) = no_query.strip_prefix(base)
613 && let Some(key) = rest.strip_prefix('/')
614 && !key.is_empty()
615 {
616 return Some(key.to_string());
617 }
618 }
619
620 // Path-style S3: must match the configured `{endpoint}/{bucket}/` exactly.
621 // Without the endpoint pin, the prior implementation accepted any
622 // `https://{any-host}/{bucket}/{key}` — so an attacker-controlled URL like
623 // `https://attacker.example/my-bucket/poisoned` would extract a real-looking
624 // key and direct downstream code at attacker-chosen storage paths.
625 if let (Some(bucket), Some(endpoint)) = (bucket, s3_endpoint) {
626 let endpoint = endpoint.trim_end_matches('/');
627 let prefix = format!("{endpoint}/{bucket}/");
628 if let Some(key) = no_query.strip_prefix(&prefix)
629 && !key.is_empty()
630 {
631 return Some(key.to_string());
632 }
633 }
634
635 None
636 }
637
638 /// Build a permanent URL for a project image.
639 /// CDN configured: permanent CDN URL. No CDN: 24-hour presigned S3 URL.
640 pub async fn build_project_image_url(
641 s3: &dyn StorageBackend,
642 cdn_base_url: Option<&str>,
643 s3_key: &str,
644 ) -> Result<String> {
645 if let Some(cdn_base) = cdn_base_url {
646 return Ok(format!("{}/{}", cdn_base, s3_key));
647 }
648 s3.presign_download(s3_key, Some(86400)).await
649 }
650
651 #[async_trait::async_trait]
652 impl StorageBackend for S3Client {
653 async fn presign_upload(&self, s3_key: &str, content_type: &str, expiry_secs: Option<u64>, cache_control: Option<&str>, max_bytes: Option<i64>) -> Result<String> {
654 self.presign_upload(s3_key, content_type, expiry_secs, cache_control, max_bytes).await
655 }
656
657 async fn presign_download(&self, s3_key: &str, expiry_secs: Option<u64>) -> Result<String> {
658 self.presign_download(s3_key, expiry_secs).await
659 }
660
661 async fn object_exists(&self, s3_key: &str) -> Result<bool> {
662 self.object_exists(s3_key).await
663 }
664
665 async fn object_size(&self, s3_key: &str) -> Result<Option<i64>> {
666 self.object_size(s3_key).await
667 }
668
669 async fn download_object(&self, s3_key: &str) -> Result<Vec<u8>> {
670 self.download_object(s3_key).await
671 }
672
673 async fn download_stream(&self, s3_key: &str) -> Result<s3_storage::ByteStream> {
674 self.download_stream(s3_key).await
675 }
676
677 async fn upload_object(&self, s3_key: &str, content_type: &str, data: Vec<u8>, cache_control: Option<&str>) -> Result<()> {
678 self.upload_object(s3_key, content_type, data, cache_control).await
679 }
680
681 async fn delete_object(&self, s3_key: &str) -> Result<()> {
682 self.delete_object(s3_key).await
683 }
684
685 async fn delete_objects(&self, keys: &[String]) -> Result<()> {
686 self.delete_objects(keys).await
687 }
688
689 async fn delete_prefix(&self, prefix: &str) -> Result<()> {
690 self.inner.delete_prefix(prefix).await
691 .map_err(AppError::Storage)
692 }
693
694 async fn upload_multipart(&self, s3_key: &str, content_type: &str, file_path: &std::path::Path) -> Result<()> {
695 self.upload_multipart(s3_key, content_type, file_path).await
696 }
697
698 async fn check_connectivity(&self) -> std::result::Result<(), String> {
699 self.check_connectivity().await
700 }
701
702 fn bucket(&self) -> &str {
703 self.inner.bucket()
704 }
705 }
706
707 #[cfg(test)]
708 mod tests {
709 use super::*;
710
711 #[test]
712 fn extract_key_cdn_form() {
713 let key = extract_s3_key_from_url(
714 "https://cdn.makenot.work/projects/abc/image/cover.png",
715 Some("https://cdn.makenot.work"),
716 None,
717 None,
718 );
719 assert_eq!(key.as_deref(), Some("projects/abc/image/cover.png"));
720 }
721
722 #[test]
723 fn extract_key_cdn_with_trailing_slash_in_base() {
724 let key = extract_s3_key_from_url(
725 "https://cdn.makenot.work/foo/bar",
726 Some("https://cdn.makenot.work/"),
727 None,
728 None,
729 );
730 assert_eq!(key.as_deref(), Some("foo/bar"));
731 }
732
733 #[test]
734 fn extract_key_strips_query_string() {
735 let key = extract_s3_key_from_url(
736 "https://cdn.makenot.work/foo/bar?X-Amz-Signature=zzz",
737 Some("https://cdn.makenot.work"),
738 None,
739 None,
740 );
741 assert_eq!(key.as_deref(), Some("foo/bar"));
742 }
743
744 #[test]
745 fn extract_key_path_style_s3() {
746 let key = extract_s3_key_from_url(
747 "https://fsn1.your-objectstorage.com/my-bucket/u/123/image/cover.png?X-Amz=...",
748 None,
749 Some("my-bucket"),
750 Some("https://fsn1.your-objectstorage.com"),
751 );
752 assert_eq!(key.as_deref(), Some("u/123/image/cover.png"));
753 }
754
755 #[test]
756 fn extract_key_path_style_rejects_attacker_host() {
757 // Attacker-controlled host with the legitimate bucket name in the
758 // path must NOT be accepted. The endpoint pin closes the gap.
759 let key = extract_s3_key_from_url(
760 "https://attacker.example/my-bucket/poisoned",
761 None,
762 Some("my-bucket"),
763 Some("https://fsn1.your-objectstorage.com"),
764 );
765 assert_eq!(key, None);
766 }
767
768 #[test]
769 fn extract_key_path_style_requires_endpoint() {
770 // Without the endpoint, the path-style branch must not fire — bucket
771 // name alone is not enough to identify a trustworthy host.
772 let key = extract_s3_key_from_url(
773 "https://fsn1.your-objectstorage.com/my-bucket/u/123/key",
774 None,
775 Some("my-bucket"),
776 None,
777 );
778 assert_eq!(key, None);
779 }
780
781 #[test]
782 fn extract_key_returns_none_when_no_prefix_matches() {
783 // Neither the CDN base nor the bucket name is present in the URL.
784 let key = extract_s3_key_from_url(
785 "https://random.example.com/foo/bar",
786 Some("https://cdn.makenot.work"),
787 Some("my-bucket"),
788 Some("https://fsn1.your-objectstorage.com"),
789 );
790 assert_eq!(key, None);
791 }
792
793 #[test]
794 fn extract_key_does_not_misparse_keys_containing_projects_substring() {
795 // Regression: the old heuristic would have returned just
796 // "projects/x" from this URL, dropping the user-scoped prefix.
797 let key = extract_s3_key_from_url(
798 "https://cdn.makenot.work/u/me/projects/x",
799 Some("https://cdn.makenot.work"),
800 None,
801 None,
802 );
803 assert_eq!(key.as_deref(), Some("u/me/projects/x"));
804 }
805
806 #[test]
807 fn test_generate_key() {
808 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
809 let item_id: ItemId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
810
811 let key = S3Client::generate_key(user_id, item_id, FileType::Audio, "episode.mp3");
812 assert_eq!(
813 key,
814 "11111111-1111-1111-1111-111111111111/22222222-2222-2222-2222-222222222222/audio/episode.mp3"
815 );
816 }
817
818 #[test]
819 fn test_generate_key_sanitizes_filename() {
820 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
821 let item_id: ItemId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
822
823 let key = S3Client::generate_key(user_id, item_id, FileType::Audio, "my file (1).mp3");
824 assert!(key.ends_with("/myfile1.mp3"));
825 }
826
827 #[test]
828 fn test_validate_content_type() {
829 assert!(S3Client::validate_content_type(FileType::Audio, "audio/mpeg").is_ok());
830 assert!(S3Client::validate_content_type(FileType::Audio, "audio/wav").is_ok());
831 assert!(S3Client::validate_content_type(FileType::Audio, "image/png").is_err());
832
833 assert!(S3Client::validate_content_type(FileType::Cover, "image/png").is_ok());
834 assert!(S3Client::validate_content_type(FileType::Cover, "image/jpeg").is_ok());
835 assert!(S3Client::validate_content_type(FileType::Cover, "audio/mpeg").is_err());
836 }
837
838 #[test]
839 fn test_validate_extension() {
840 assert!(S3Client::validate_extension(FileType::Audio, "episode.mp3").is_ok());
841 assert!(S3Client::validate_extension(FileType::Audio, "episode.MP3").is_ok());
842 assert!(S3Client::validate_extension(FileType::Audio, "episode.png").is_err());
843
844 assert!(S3Client::validate_extension(FileType::Cover, "cover.jpg").is_ok());
845 assert!(S3Client::validate_extension(FileType::Cover, "cover.webp").is_ok());
846 assert!(S3Client::validate_extension(FileType::Cover, "cover.mp3").is_err());
847 }
848
849 #[test]
850 fn test_file_type_from_str() {
851 assert_eq!(FileType::from_str("audio"), Ok(FileType::Audio));
852 assert_eq!(FileType::from_str("AUDIO"), Ok(FileType::Audio));
853 assert_eq!(FileType::from_str("cover"), Ok(FileType::Cover));
854 assert_eq!(FileType::from_str("image"), Ok(FileType::Cover));
855 assert!(FileType::from_str("invalid").is_err());
856 }
857
858 #[test]
859 fn file_type_as_str() {
860 assert_eq!(FileType::Audio.as_str(), "audio");
861 assert_eq!(FileType::Cover.as_str(), "cover");
862 }
863
864 #[test]
865 fn file_type_max_size() {
866 assert_eq!(FileType::Audio.max_size(), 500 * 1024 * 1024);
867 assert_eq!(FileType::Cover.max_size(), 10 * 1024 * 1024);
868 }
869
870 #[test]
871 fn file_type_allowed_types_audio() {
872 let types = FileType::Audio.allowed_types();
873 let exts: Vec<&str> = types.iter().map(|(e, _)| *e).collect();
874 assert!(exts.contains(&"mp3"));
875 assert!(exts.contains(&"wav"));
876 assert!(exts.contains(&"flac"));
877 assert!(!exts.contains(&"png"));
878 }
879
880 #[test]
881 fn file_type_allowed_types_cover() {
882 let types = FileType::Cover.allowed_types();
883 let exts: Vec<&str> = types.iter().map(|(e, _)| *e).collect();
884 assert!(exts.contains(&"jpg"));
885 assert!(exts.contains(&"png"));
886 assert!(exts.contains(&"webp"));
887 assert!(!exts.contains(&"mp3"));
888 }
889
890 #[test]
891 fn generate_key_strips_path_traversal() {
892 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
893 let item_id: ItemId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
894
895 let key = S3Client::generate_key(user_id, item_id, FileType::Audio, "../../etc/passwd");
896 // Slashes are stripped, dots kept: "../../etc/passwd" -> "....etcpasswd"
897 assert!(key.ends_with("/audio/....etcpasswd"));
898 }
899
900 #[test]
901 fn generate_key_empty_filename_gets_fallback() {
902 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
903 let item_id: ItemId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
904
905 let key = S3Client::generate_key(user_id, item_id, FileType::Cover, "");
906 assert!(key.ends_with("/cover/file"), "expected fallback name 'file', got: {}", key);
907 }
908
909 #[test]
910 fn validate_extension_no_extension() {
911 assert!(S3Client::validate_extension(FileType::Audio, "noext").is_err());
912 }
913
914 #[test]
915 fn validate_extension_double_dot() {
916 assert!(S3Client::validate_extension(FileType::Audio, "file.backup.mp3").is_ok());
917 }
918
919 #[test]
920 fn validate_content_type_empty() {
921 assert!(S3Client::validate_content_type(FileType::Audio, "").is_err());
922 }
923
924 #[test]
925 fn file_type_insertion_from_str() {
926 assert_eq!(FileType::from_str("insertion"), Ok(FileType::Insertion));
927 assert_eq!(FileType::from_str("INSERTION"), Ok(FileType::Insertion));
928 }
929
930 #[test]
931 fn file_type_insertion_as_str() {
932 assert_eq!(FileType::Insertion.as_str(), "insertion");
933 }
934
935 #[test]
936 fn file_type_insertion_max_size() {
937 assert_eq!(FileType::Insertion.max_size(), 500 * 1024 * 1024);
938 }
939
940 #[test]
941 fn validate_insertion_content_types() {
942 assert!(S3Client::validate_content_type(FileType::Insertion, "audio/mpeg").is_ok());
943 assert!(S3Client::validate_content_type(FileType::Insertion, "audio/wav").is_ok());
944 assert!(S3Client::validate_content_type(FileType::Insertion, "audio/flac").is_ok());
945 assert!(S3Client::validate_content_type(FileType::Insertion, "image/png").is_err());
946 }
947
948 #[test]
949 fn validate_insertion_extensions() {
950 assert!(S3Client::validate_extension(FileType::Insertion, "intro.mp3").is_ok());
951 assert!(S3Client::validate_extension(FileType::Insertion, "sponsor.wav").is_ok());
952 assert!(S3Client::validate_extension(FileType::Insertion, "outro.flac").is_ok());
953 assert!(S3Client::validate_extension(FileType::Insertion, "clip.png").is_err());
954 }
955
956 #[test]
957 fn generate_insertion_key_format() {
958 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
959 let key = S3Client::generate_insertion_key(user_id, "intro.mp3");
960 assert_eq!(key, "11111111-1111-1111-1111-111111111111/insertions/intro.mp3");
961 }
962
963 #[test]
964 fn generate_insertion_key_sanitizes() {
965 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
966 let key = S3Client::generate_insertion_key(user_id, "my sponsor read (v2).mp3");
967 assert_eq!(key, "11111111-1111-1111-1111-111111111111/insertions/mysponsorreadv2.mp3");
968 }
969
970 #[test]
971 fn generate_key_cover_type() {
972 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
973 let item_id: ItemId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
974
975 let key = S3Client::generate_key(user_id, item_id, FileType::Cover, "art.png");
976 assert!(key.contains("/cover/"));
977 assert!(key.ends_with("art.png"));
978 }
979
980 // FileType::Download tests
981
982 #[test]
983 fn file_type_download_from_str() {
984 assert_eq!(FileType::from_str("download"), Ok(FileType::Download));
985 assert_eq!(FileType::from_str("DOWNLOAD"), Ok(FileType::Download));
986 }
987
988 #[test]
989 fn file_type_download_as_str() {
990 assert_eq!(FileType::Download.as_str(), "download");
991 }
992
993 #[test]
994 fn file_type_download_max_size() {
995 assert_eq!(FileType::Download.max_size(), 500 * 1024 * 1024);
996 }
997
998 #[test]
999 fn validate_download_content_types() {
1000 assert!(S3Client::validate_content_type(FileType::Download, "application/octet-stream").is_ok());
1001 assert!(S3Client::validate_content_type(FileType::Download, "application/zip").is_ok());
1002 assert!(S3Client::validate_content_type(FileType::Download, "application/x-apple-diskimage").is_ok());
1003 assert!(S3Client::validate_content_type(FileType::Download, "application/gzip").is_ok());
1004 assert!(S3Client::validate_content_type(FileType::Download, "application/x-tar").is_ok());
1005 // Reject clearly wrong types
1006 assert!(S3Client::validate_content_type(FileType::Download, "text/html").is_err());
1007 assert!(S3Client::validate_content_type(FileType::Download, "image/png").is_err());
1008 }
1009
1010 #[test]
1011 fn validate_download_extensions() {
1012 assert!(S3Client::validate_extension(FileType::Download, "app.zip").is_ok());
1013 assert!(S3Client::validate_extension(FileType::Download, "app.dmg").is_ok());
1014 assert!(S3Client::validate_extension(FileType::Download, "app.exe").is_ok());
1015 assert!(S3Client::validate_extension(FileType::Download, "app.appimage").is_ok());
1016 assert!(S3Client::validate_extension(FileType::Download, "app.deb").is_ok());
1017 assert!(S3Client::validate_extension(FileType::Download, "app.tar.gz").is_ok());
1018 assert!(S3Client::validate_extension(FileType::Download, "app.clap").is_ok());
1019 assert!(S3Client::validate_extension(FileType::Download, "app.vst3").is_ok());
1020 assert!(S3Client::validate_extension(FileType::Download, "App.ZIP").is_ok());
1021 // Reject invalid extensions
1022 assert!(S3Client::validate_extension(FileType::Download, "app.mp3").is_err());
1023 assert!(S3Client::validate_extension(FileType::Download, "app.txt").is_err());
1024 }
1025
1026 #[test]
1027 fn generate_key_download_type() {
1028 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
1029 let item_id: ItemId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
1030
1031 let key = S3Client::generate_key(user_id, item_id, FileType::Download, "plugin-v1.0.zip");
1032 assert!(key.contains("/download/"));
1033 assert!(key.ends_with("plugin-v1.0.zip"));
1034 }
1035
1036 // CDN tests
1037
1038 #[test]
1039 fn cache_control_immutable_format() {
1040 assert!(CACHE_CONTROL_IMMUTABLE.contains("public"));
1041 assert!(CACHE_CONTROL_IMMUTABLE.contains("max-age=31536000"));
1042 assert!(CACHE_CONTROL_IMMUTABLE.contains("immutable"));
1043 }
1044
1045 #[test]
1046 fn generate_project_image_key_format() {
1047 let project_id: ProjectId = "33333333-3333-3333-3333-333333333333".parse().unwrap();
1048 let key = S3Client::generate_project_image_key(project_id, "logo.png");
1049 assert_eq!(key, "projects/33333333-3333-3333-3333-333333333333/image/logo.png");
1050 }
1051
1052 #[test]
1053 fn generate_project_image_key_sanitizes() {
1054 let project_id: ProjectId = "33333333-3333-3333-3333-333333333333".parse().unwrap();
1055 let key = S3Client::generate_project_image_key(project_id, "my logo (v2).png");
1056 assert_eq!(key, "projects/33333333-3333-3333-3333-333333333333/image/mylogov2.png");
1057 }
1058
1059 #[test]
1060 fn cdn_url_from_s3_key() {
1061 let cdn_base = "https://cdn.makenot.work";
1062 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
1063 let item_id: ItemId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
1064 let key = S3Client::generate_key(user_id, item_id, FileType::Audio, "episode.mp3");
1065 let cdn_url = format!("{}/{}", cdn_base, key);
1066 assert_eq!(
1067 cdn_url,
1068 "https://cdn.makenot.work/11111111-1111-1111-1111-111111111111/22222222-2222-2222-2222-222222222222/audio/episode.mp3"
1069 );
1070 }
1071
1072 // FileType::Video tests
1073
1074 #[test]
1075 fn file_type_video_from_str() {
1076 assert_eq!(FileType::from_str("video"), Ok(FileType::Video));
1077 assert_eq!(FileType::from_str("VIDEO"), Ok(FileType::Video));
1078 }
1079
1080 #[test]
1081 fn file_type_video_as_str() {
1082 assert_eq!(FileType::Video.as_str(), "video");
1083 }
1084
1085 #[test]
1086 fn file_type_video_max_size() {
1087 assert_eq!(FileType::Video.max_size(), 20 * 1024 * 1024 * 1024);
1088 }
1089
1090 #[test]
1091 fn validate_video_content_types() {
1092 assert!(S3Client::validate_content_type(FileType::Video, "video/mp4").is_ok());
1093 assert!(S3Client::validate_content_type(FileType::Video, "video/webm").is_ok());
1094 assert!(S3Client::validate_content_type(FileType::Video, "video/quicktime").is_ok());
1095 assert!(S3Client::validate_content_type(FileType::Video, "audio/mpeg").is_err());
1096 assert!(S3Client::validate_content_type(FileType::Video, "application/octet-stream").is_err());
1097 assert!(S3Client::validate_content_type(FileType::Video, "text/html").is_err());
1098 }
1099
1100 #[test]
1101 fn validate_video_extensions() {
1102 assert!(S3Client::validate_extension(FileType::Video, "clip.mp4").is_ok());
1103 assert!(S3Client::validate_extension(FileType::Video, "clip.webm").is_ok());
1104 assert!(S3Client::validate_extension(FileType::Video, "clip.mov").is_ok());
1105 assert!(S3Client::validate_extension(FileType::Video, "Clip.MP4").is_ok());
1106 assert!(S3Client::validate_extension(FileType::Video, "clip.avi").is_err());
1107 assert!(S3Client::validate_extension(FileType::Video, "clip.mp3").is_err());
1108 }
1109
1110 #[test]
1111 fn generate_key_video_type() {
1112 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
1113 let item_id: ItemId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
1114
1115 let key = S3Client::generate_key(user_id, item_id, FileType::Video, "tutorial.mp4");
1116 assert!(key.contains("/video/"));
1117 assert!(key.ends_with("tutorial.mp4"));
1118 }
1119 }
1120