Skip to main content

max / audiofiles

Upload samples through the streaming blob path Samples routinely run to hundreds of MB, and the old path read each one whole into memory before sealing it — and could not upload anything above the server's one-shot PUT ceiling at all. blob_upload_streaming holds one part at a time and has no such ceiling. Dedup moves server-side into the same call, ahead of any file read, so an unchanged sample still costs one round trip and no disk I/O. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-21 15:52 UTC
Commit: 5d25e59d7d64bfb1a98be8c2b57d4f99fa29fa04
Parent: 760f6e5
1 file changed, +13 insertions, -19 deletions
M crates/audiofiles-sync/src/service/upload.rs +13 -19
@@ -3,7 +3,7 @@
3 3 use std::path::Path;
4 4
5 5 use rusqlite::Connection;
6 - use synckit_client::{ChangeEntry, ChangeOp, DeviceId, SyncKitClient};
6 + use synckit_client::{BlobUploadOutcome, ChangeEntry, ChangeOp, DeviceId, SyncKitClient};
7 7 use uuid::Uuid;
8 8
9 9 use tracing::instrument;
@@ -90,29 +90,23 @@ pub async fn upload_pending_blobs(
90 90 continue;
91 91 }
92 92
93 - let resp = client
94 - .blob_upload_url(hash, *size)
93 + // Streams the file a part at a time, so peak memory is one part rather
94 + // than the whole sample — audio files routinely run to hundreds of MB,
95 + // and this is the only path the server accepts above its one-shot PUT
96 + // ceiling. Dedup happens server-side before any read, so an unchanged
97 + // sample costs one round trip.
98 + let outcome = client
99 + .blob_upload_streaming(hash, &blob_path)
95 100 .await
96 101 .map_err(|e| SyncError::Client(e.to_string()))?;
97 102
98 - if resp.already_exists {
99 - uploaded += 1;
100 - continue;
103 + if outcome == BlobUploadOutcome::Uploaded {
104 + client
105 + .blob_confirm(hash, *size)
106 + .await
107 + .map_err(|e| SyncError::Client(e.to_string()))?;
101 108 }
102 109
103 - let data = std::fs::read(&blob_path)
104 - .map_err(SyncError::Io)?;
105 -
106 - client
107 - .blob_upload(hash, &resp.upload_url, data)
108 - .await
109 - .map_err(|e| SyncError::Client(e.to_string()))?;
110 -
111 - client
112 - .blob_confirm(hash, *size)
113 - .await
114 - .map_err(|e| SyncError::Client(e.to_string()))?;
115 -
116 110 uploaded += 1;
117 111 }
118 112