Skip to main content

max / makenotwork

Add copy_object and content-addressed key foundations for scan-then-promote First phases of the content-addressed scan-integrity fix (presign uploads to an unserved staging key, scan it, copy to {user_id}/c/{sha256}.{ext} only on a Clean verdict, so served bytes are provably the scanned bytes). Adds the server-side copy_object primitive to the s3-storage lib, the StorageBackend trait, S3Client, and the in-memory test backend, plus generate_staging_key / content_key / extension helpers with unit tests. Additive and inert: no upload path uses them yet (Phase 2 wires Versions). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-06 13:17 UTC
Commit: f3a624122bf69500cde73ea579dd2dae5d390bb1
Parent: d70ea9a
3 files changed, +152 insertions, -0 deletions
M server/src/storage.rs +119
@@ -466,6 +466,13 @@ pub trait StorageBackend: Send + Sync {
466 466 /// silently defeats streaming, so a future backend that forgot to override
467 467 /// it would quietly lose multipart. Every backend must declare its strategy.
468 468 async fn upload_multipart(&self, s3_key: &S3Key, content_type: &str, file_path: &std::path::Path) -> Result<()>;
469 + /// Server-side copy `src_key` to `dst_key` within this backend's bucket
470 + /// (no bytes transit the process). The scan-then-promote primitive: a Clean
471 + /// staging object is copied to the served key the client holds no presign
472 + /// for, so served bytes are provably the scanned bytes (Run #24 Storage
473 + /// HIGH). Required (not defaulted): a silent no-op default would make a
474 + /// promote "succeed" while the served key stays empty.
475 + async fn copy_object(&self, src_key: &S3Key, dst_key: &S3Key) -> Result<()>;
469 476 async fn check_connectivity(&self) -> std::result::Result<(), String>;
470 477 fn bucket(&self) -> &str;
471 478 }
@@ -518,6 +525,29 @@ impl S3Client {
518 525 ))
519 526 }
520 527
528 + /// Staging key for scan-then-promote: browser uploads presign to this key,
529 + /// which is NEVER served. After a Clean scan the worker copies the object to
530 + /// its content-addressed [`content_key`](Self::content_key) and deletes the
531 + /// staging object. The upload's extension is carried in the staging key so
532 + /// the worker can build the content key without re-reading the entity row.
533 + /// Format: `staging/{uuid}.{ext}`. The random uuid means a replayed presigned
534 + /// PUT can only re-write the (unserved, post-scan-deleted) staging object,
535 + /// never the served content key (ultra-fuzz Run #24 Storage HIGH).
536 + pub fn generate_staging_key(filename: &str) -> S3Key {
537 + S3Key(format!("staging/{}.{}", uuid::Uuid::new_v4(), extension_for(filename)))
538 + }
539 +
540 + /// Content-addressed served key: `{user_id}/c/{sha256}.{ext}`. The object's
541 + /// name *is* its content hash, so the served bytes are provably the bytes
542 + /// that were scanned — a swapped object would hash to a different key. The
543 + /// key is per-owner (`user_id`) namespaced, so identical bytes uploaded by
544 + /// different creators do NOT collapse to one shared object (no cross-tenant
545 + /// existence oracle). The `c` marker segment cannot collide with the legacy
546 + /// `{user_id}/{item_id}/...` layout because `c` is not a UUID.
547 + pub fn content_key(user_id: UserId, sha256: &str, ext: &str) -> S3Key {
548 + S3Key(format!("{user_id}/c/{sha256}.{ext}"))
549 + }
550 +
521 551 /// Generate an S3 key for a version download file. The version's own id is
522 552 /// woven into the path so two versions of the same item that share a
523 553 /// filename (e.g. a creator who ships every release as `plugin.zip`) never
@@ -793,6 +823,15 @@ impl S3Client {
793 823 self.inner.delete(s3_key.as_str()).await.map_err(AppError::Storage)
794 824 }
795 825
826 + /// Server-side copy within the bucket. See the [`StorageBackend::copy_object`]
827 + /// trait method for the scan-then-promote rationale.
828 + pub async fn copy_object(&self, src_key: &S3Key, dst_key: &S3Key) -> Result<()> {
829 + self.inner
830 + .copy_object(src_key.as_str(), dst_key.as_str())
831 + .await
832 + .map_err(AppError::Storage)
833 + }
834 +
796 835 /// Batched S3 delete (`DeleteObjects`, up to 1000 keys per call).
797 836 /// Chunks larger slices into 1000-key batches and logs per-key failures
798 837 /// without bubbling — the pending_s3_deletions queue is the safety net.
@@ -851,6 +890,38 @@ impl S3Client {
851 890 /// `pub(crate)` so confirm handlers store a filename that matches the tail of
852 891 /// the key `generate_media_key` produced, rather than re-deriving a weaker
853 892 /// filter that drops the empty-basename fallback (Run #18 Storage B9).
893 + /// The lowercased ASCII-alphanumeric file extension for a staging/content key,
894 + /// or `"bin"` when the filename has none. Bounded to 16 chars so a crafted
895 + /// filename can't bloat the key. Content keys carry an extension purely so
896 + /// CDN-served objects keep a sensible suffix (content-type sniffing, browser
897 + /// "save as"); the hash is the identity, the extension is cosmetic.
898 + pub(crate) fn extension_for(filename: &str) -> String {
899 + let ext: String = std::path::Path::new(filename)
900 + .extension()
901 + .and_then(|s| s.to_str())
902 + .unwrap_or("")
903 + .chars()
904 + .filter(|c| c.is_ascii_alphanumeric())
905 + .map(|c| c.to_ascii_lowercase())
906 + .take(16)
907 + .collect();
908 + if ext.is_empty() { "bin".to_string() } else { ext }
909 + }
910 +
911 + /// The extension segment of a key's basename (the text after the last `.`), or
912 + /// `"bin"`. Lets the scan worker carry a `staging/{uuid}.{ext}` object's
913 + /// extension onto its content key without re-reading the entity row.
914 + // Consumed by the scan worker's copy-on-clean step (Phase 2 of the
915 + // content-addressed scan-integrity change); tested now as a Phase 1 foundation.
916 + #[allow(dead_code)]
917 + pub(crate) fn key_extension(key: &str) -> &str {
918 + key.rsplit('/')
919 + .next()
920 + .and_then(|base| base.rsplit_once('.').map(|(_, ext)| ext))
921 + .filter(|ext| !ext.is_empty())
922 + .unwrap_or("bin")
923 + }
924 +
854 925 pub(crate) fn sanitize_filename(filename: &str) -> String {
855 926 let sanitized: String = filename
856 927 .chars()
@@ -1006,6 +1077,11 @@ impl StorageBackend for S3Client {
1006 1077 self.delete_objects(keys).await
1007 1078 }
1008 1079
1080 + async fn copy_object(&self, src_key: &S3Key, dst_key: &S3Key) -> Result<()> {
1081 + // Inherent method; delegate.
1082 + self.copy_object(src_key, dst_key).await
1083 + }
1084 +
1009 1085 async fn delete_prefix(&self, _auth: &S3DeleteAuthority, prefix: &str) -> Result<()> {
1010 1086 self.inner.delete_prefix(prefix).await
1011 1087 .map_err(AppError::Storage)
@@ -1362,6 +1438,49 @@ mod tests {
1362 1438 }
1363 1439
1364 1440 #[test]
1441 + fn extension_for_cases() {
1442 + assert_eq!(extension_for("plugin.zip"), "zip");
1443 + assert_eq!(extension_for("LOUD.WAV"), "wav"); // lowercased
1444 + assert_eq!(extension_for("archive.tar.gz"), "gz"); // last segment only
1445 + assert_eq!(extension_for("noextension"), "bin"); // fallback
1446 + assert_eq!(extension_for("trailing."), "bin"); // empty ext → fallback
1447 + assert_eq!(extension_for("weird.z!p"), "zp"); // non-alnum stripped
1448 + }
1449 +
1450 + #[test]
1451 + fn key_extension_cases() {
1452 + assert_eq!(key_extension("staging/2f9a.zip"), "zip");
1453 + assert_eq!(key_extension("uid/c/abcd1234.mp3"), "mp3");
1454 + assert_eq!(key_extension("staging/no-dot-basename"), "bin");
1455 + assert_eq!(key_extension("dir.with.dot/basename"), "bin"); // dot in dir, not basename
1456 + }
1457 +
1458 + #[test]
1459 + fn staging_key_is_unserved_and_carries_extension() {
1460 + let key = S3Client::generate_staging_key("release.zip");
1461 + assert!(key.as_str().starts_with("staging/"), "staging key: {key}");
1462 + assert_eq!(key_extension(key.as_str()), "zip");
1463 + // Two calls never collide (random uuid), so a replayed PUT can't target
1464 + // another upload's staging object.
1465 + let key2 = S3Client::generate_staging_key("release.zip");
1466 + assert_ne!(key, key2);
1467 + }
1468 +
1469 + #[test]
1470 + fn content_key_is_hash_addressed_and_owner_namespaced() {
1471 + let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
1472 + let sha = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
1473 + let key = S3Client::content_key(user_id, sha, "zip");
1474 + assert_eq!(
1475 + key,
1476 + "11111111-1111-1111-1111-111111111111/c/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.zip"
1477 + );
1478 + // Same bytes, different owner → different key (no cross-tenant sharing).
1479 + let other: UserId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
1480 + assert_ne!(S3Client::content_key(other, sha, "zip"), key);
1481 + }
1482 +
1483 + #[test]
1365 1484 fn generate_key_cover_type() {
1366 1485 let user_id: UserId = "11111111-1111-1111-1111-111111111111".parse().unwrap();
1367 1486 let item_id: ItemId = "22222222-2222-2222-2222-222222222222".parse().unwrap();
M server/tests/harness/storage.rs +10
@@ -97,6 +97,16 @@ impl StorageBackend for InMemoryStorage {
97 97 Ok(())
98 98 }
99 99
100 + async fn copy_object(&self, src_key: &S3Key, dst_key: &S3Key) -> Result<()> {
101 + let mut objects = self.objects.lock().unwrap();
102 + let bytes = objects
103 + .get(src_key.as_str())
104 + .cloned()
105 + .ok_or_else(|| AppError::Storage(format!("copy source not found: {}", src_key)))?;
106 + objects.insert(dst_key.as_str().to_string(), bytes);
107 + Ok(())
108 + }
109 +
100 110 async fn check_connectivity(&self) -> std::result::Result<(), String> {
101 111 Ok(())
102 112 }
M shared/s3-storage/src/lib.rs +23
@@ -210,6 +210,29 @@ impl S3Client {
210 210 Ok(())
211 211 }
212 212
213 + /// Server-side copy an object from `src_key` to `dst_key` within this
214 + /// bucket. No bytes transit the caller — S3 performs the copy internally.
215 + ///
216 + /// This is the primitive behind scan-then-promote: an upload is scanned at a
217 + /// staging key the client can overwrite, and only a Clean object is copied to
218 + /// the served key (which the client holds no presign for), so the served
219 + /// bytes are provably the scanned bytes (ultra-fuzz Run #24 Storage HIGH).
220 + ///
221 + /// `CopySource` is `{bucket}/{key}`. Keys in this system are sanitized to
222 + /// `[A-Za-z0-9._/-]` (see the server's `sanitize_filename`), none of which
223 + /// require percent-encoding, so the source is formed directly.
224 + pub async fn copy_object(&self, src_key: &str, dst_key: &str) -> Result<(), String> {
225 + self.client
226 + .copy_object()
227 + .bucket(&self.bucket)
228 + .copy_source(format!("{}/{}", self.bucket, src_key))
229 + .key(dst_key)
230 + .send()
231 + .await
232 + .map_err(|e| format!("S3 copy_object {src_key} -> {dst_key} failed: {e}"))?;
233 + Ok(())
234 + }
235 +
213 236 /// Delete a batch of objects in a single S3 `DeleteObjects` request.
214 237 ///
215 238 /// S3 accepts up to 1000 keys per call; the caller is responsible for