max / makenotwork

server: add download_stream to StorageBackend

Author: Max J. <87768334+MaxJMath@users.noreply.github.com> · 2026-05-27 14:52 UTC

Commit: 625a832196bb849ca60ed54f5a643d257ab4aaf9

Parent: a259ef2

3 files changed, +29 insertions, -0 deletions

M server/src/storage.rs +16

			@@ -177,6 +177,10 @@ pub trait StorageBackend: Send + Sync {
177	177		async fn object_exists(&self, s3_key: &str) -> Result<bool>;
178	178		async fn object_size(&self, s3_key: &str) -> Result<Option<i64>>;
179	179		async fn download_object(&self, s3_key: &str) -> Result<Vec<u8>>;
	180	+	/// Stream the object body without buffering the whole payload. Callers
	181	+	/// drive the stream to disk (scanner spool) or to a layer that consumes
	182	+	/// chunks directly.
	183	+	async fn download_stream(&self, s3_key: &str) -> Result<s3_storage::ByteStream>;
180	184		async fn upload_object(&self, s3_key: &str, content_type: &str, data: Vec<u8>, cache_control: Option<&str>) -> Result<()>;
181	185		async fn delete_object(&self, s3_key: &str) -> Result<()>;
182	186		/// Delete a batch of objects in a single S3 `DeleteObjects` request
			@@ -393,6 +397,14 @@ impl S3Client {
393	397		.map_err(AppError::Storage)
394	398		}
395	399
	400	+	/// Stream an object's body from S3 without buffering. See trait docs.
	401	+	pub async fn download_stream(&self, s3_key: &str) -> Result<s3_storage::ByteStream> {
	402	+	self.inner
	403	+	.download_stream(s3_key)
	404	+	.await
	405	+	.map_err(AppError::Storage)
	406	+	}
	407	+
396	408		/// Upload an object to S3 from bytes
397	409		pub async fn upload_object(
398	410		&self,
			@@ -587,6 +599,10 @@ impl StorageBackend for S3Client {
587	599		self.download_object(s3_key).await
588	600		}
589	601
	602	+	async fn download_stream(&self, s3_key: &str) -> Result<s3_storage::ByteStream> {
	603	+	self.download_stream(s3_key).await
	604	+	}
	605	+
590	606		async fn upload_object(&self, s3_key: &str, content_type: &str, data: Vec<u8>, cache_control: Option<&str>) -> Result<()> {
591	607		self.upload_object(s3_key, content_type, data, cache_control).await
592	608		}

M server/tests/harness/storage.rs +11

			@@ -64,6 +64,17 @@ impl StorageBackend for InMemoryStorage {
64	64		.ok_or_else(\|\| AppError::Storage(format!("Object not found: {}", s3_key)))
65	65		}
66	66
	67	+	async fn download_stream(&self, s3_key: &str) -> Result<s3_storage::ByteStream> {
	68	+	let bytes = self
	69	+	.objects
	70	+	.lock()
	71	+	.unwrap()
	72	+	.get(s3_key)
	73	+	.cloned()
	74	+	.ok_or_else(\|\| AppError::Storage(format!("Object not found: {}", s3_key)))?;
	75	+	Ok(s3_storage::ByteStream::from(bytes))
	76	+	}
	77	+
67	78		async fn upload_object(&self, s3_key: &str, _content_type: &str, data: Vec<u8>, _cache_control: Option<&str>) -> Result<()> {
68	79		self.objects.lock().unwrap().insert(s3_key.to_string(), data);
69	80		Ok(())

M shared/s3-storage/src/lib.rs +2

			@@ -14,6 +14,8 @@ use aws_sdk_s3::types::{
14	14		use aws_sdk_s3::Client;
15	15		use std::time::Duration;
16	16
	17	+	pub use aws_sdk_s3::primitives::ByteStream;
	18	+
17	19		/// S3 connection configuration.
18	20		#[derive(Debug, Clone)]
19	21		pub struct S3Config {