Skip to main content

max / makenotwork

server: add multipart upload session endpoints for the CLI Add start/parts/complete/abort under /api/internal/upload/multipart/. These sit on the internal (CLI/desktop) surface only: a browser keeps the one-shot presigned PUT and its 5 GiB ceiling, because a tab cannot resume a multi-hour transfer. Large files belong on a client that can. The endpoints replace the transport only. complete finalizes the staging object and the client then calls the existing /api/internal/upload/confirm, which reads the authoritative size from S3 and applies every size, tier, scan, and commit rule unchanged. Split validate_declared_upload_size into a transport-agnostic validate_declared_upload_size_limits (positive, per-type cap, tier per-file cap) plus the single-PUT ceiling. The multipart path uses the former, so it can carry files above 5 GiB while still enforcing every limit that describes the file rather than how it is carried. start also checks the tier cap up front so an over-cap file is refused before it stages any parts. Part URLs are minted in bounded windows (100) rather than all at once, each signed with its exact Content-Length. Ownership of a staging key is proved via the pending_uploads row start recorded, the same proof confirm_upload uses, so one creator cannot drive parts into another's in-flight session. 1887 lib + 1174 integration tests green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-21 04:01 UTC
Commit: 405e0fa534f3aa9fbb5ad00d088f3946d548c06b
Parent: debcfb6
3 files changed, +375 insertions, -11 deletions
M server/src/routes/api/internal/mod.rs +4
@@ -35,6 +35,10 @@ pub(super) fn internal_routes() -> CsrfRouter<AppState> {
35 35 .route("/api/internal/creator/items", post_csrf_skip(INTERNAL_SKIP, items::create_item))
36 36 .route("/api/internal/upload/presign", post_csrf_skip(INTERNAL_SKIP, uploads::presign_upload))
37 37 .route("/api/internal/upload/confirm", post_csrf_skip(INTERNAL_SKIP, uploads::confirm_upload))
38 + .route("/api/internal/upload/multipart/start", post_csrf_skip(INTERNAL_SKIP, uploads::multipart_start))
39 + .route("/api/internal/upload/multipart/parts", post_csrf_skip(INTERNAL_SKIP, uploads::multipart_parts))
40 + .route("/api/internal/upload/multipart/complete", post_csrf_skip(INTERNAL_SKIP, uploads::multipart_complete))
41 + .route("/api/internal/upload/multipart/abort", post_csrf_skip(INTERNAL_SKIP, uploads::multipart_abort))
38 42 .route_get("/api/internal/creator/storage", get(uploads::creator_storage))
39 43 .route_get("/api/internal/creator/items/{id}", get(items::get_item))
40 44 .route("/api/internal/creator/items/{id}", put_csrf_skip(INTERNAL_SKIP, items::update_item))
M server/src/routes/api/internal/uploads.rs +283
@@ -94,6 +94,289 @@ pub(super) async fn presign_upload(
94 94 }))
95 95 }
96 96
97 + // ── Multipart upload session (CLI / desktop, large files) ──
98 + //
99 + // The chunked counterpart to `presign_upload`, and deliberately only on the
100 + // internal (CLI/desktop) surface: a browser keeps the one-shot presigned PUT and
101 + // its 5 GiB ceiling, because a tab cannot resume a multi-hour transfer. These
102 + // three endpoints replace the *transport* only — the client finishes by calling
103 + // the existing `/api/internal/upload/confirm`, which reads the authoritative
104 + // object size from S3 and does all the size/tier/scan/DB work unchanged.
105 +
106 + /// Largest window of presigned part URLs one `parts` call will mint. A 20 GB
107 + /// object is ~1250 parts at the auto-chosen part size; handing out every URL at
108 + /// once would mint thousands of credentials with a 1-hour life for an upload
109 + /// that may never happen, so the client pulls them in windows as it progresses.
110 + const MULTIPART_PART_URL_WINDOW: u32 = 100;
111 +
112 + #[derive(Deserialize)]
113 + pub(super) struct MultipartStartRequest {
114 + item_id: ItemId,
115 + file_type: String,
116 + file_name: String,
117 + content_type: String,
118 + file_size_bytes: i64,
119 + }
120 +
121 + #[derive(Serialize)]
122 + struct MultipartStartResponse {
123 + upload_id: String,
124 + s3_key: String,
125 + part_size: usize,
126 + part_count: u32,
127 + expires_in: u64,
128 + }
129 +
130 + /// POST /api/internal/upload/multipart/start
131 + ///
132 + /// Open a multipart upload session and return the part geometry the client
133 + /// uploads against. Mirrors `presign_upload`'s pre-checks (ownership, type,
134 + /// quota) and additionally validates the declared size, since a multipart
135 + /// session stages real S3 state that a rejected upload would orphan.
136 + #[tracing::instrument(skip_all, name = "internal::multipart_start")]
137 + pub(super) async fn multipart_start(
138 + State(db): State<PgPool>,
139 + State(storage): State<AppStorage>,
140 + actor: InternalActor,
141 + _auth: ServiceAuth,
142 + Json(req): Json<MultipartStartRequest>,
143 + ) -> Result<impl IntoResponse> {
144 + let s3 = storage.require_s3()?;
145 +
146 + let file_type = FileType::from_str(&req.file_type)
147 + .map_err(|_| AppError::BadRequest(format!("Invalid file type: {}", req.file_type)))?;
148 +
149 + S3Client::validate_content_type(file_type, &req.content_type)?;
150 + S3Client::validate_extension(file_type, &req.file_name)?;
151 +
152 + let owner = db::items::get_item_owner(&db, req.item_id)
153 + .await?
154 + .ok_or(AppError::NotFound)?;
155 + if owner != actor.user_id() {
156 + return Err(AppError::Forbidden);
157 + }
158 +
159 + db::creator_tiers::check_presign_allowed(&db, actor.user_id(), file_type).await?;
160 +
161 + // Every size limit that describes the file, but NOT the single-PUT ceiling —
162 + // chunking is precisely what lets this path exceed it. The tier's per-file
163 + // cap is enforced here so an over-cap file is refused before it stages parts;
164 + // confirm re-checks against the real object size regardless.
165 + let tier_cap = db::creator_tiers::get_active_creator_tier(&db, actor.user_id())
166 + .await?
167 + .map(|t| t.max_file_bytes() as u64);
168 + crate::routes::storage::validate_declared_upload_size_limits(
169 + Some(req.file_size_bytes),
170 + file_type,
171 + tier_cap,
172 + )?;
173 +
174 + // Part geometry is pure arithmetic over the declared size, so the client can
175 + // derive identical boundaries without a round trip.
176 + let plan = s3_storage::MultipartPlan::auto(req.file_size_bytes.max(0) as u64)
177 + .map_err(AppError::BadRequest)?;
178 +
179 + let s3_key = S3Client::generate_staging_key(&req.file_name);
180 + db::pending_uploads::record_pending_upload(&db, actor.user_id(), &s3_key, "main").await?;
181 +
182 + let upload_id = s3.create_multipart_upload(&s3_key, &req.content_type).await?;
183 +
184 + tracing::info!(
185 + user = %actor.user_id(), item = %req.item_id, s3_key = %s3_key,
186 + size = req.file_size_bytes, parts = plan.part_count,
187 + "CLI multipart upload started"
188 + );
189 +
190 + Ok(Json(MultipartStartResponse {
191 + upload_id,
192 + s3_key: s3_key.into_string(),
193 + part_size: plan.part_size,
194 + part_count: plan.part_count,
195 + expires_in: 3600,
196 + }))
197 + }
198 +
199 + #[derive(Deserialize)]
200 + pub(super) struct MultipartPartsRequest {
201 + s3_key: String,
202 + upload_id: String,
203 + /// Declared total size, so each part URL can be signed with its exact
204 + /// `Content-Length`. The plan is deterministic in this value, so it must
205 + /// match the one passed to `start` or the signed lengths will not line up.
206 + file_size_bytes: i64,
207 + first_part: u32,
208 + count: u32,
209 + }
210 +
211 + #[derive(Serialize)]
212 + struct MultipartPartUrl {
213 + part_number: i32,
214 + content_length: u64,
215 + url: String,
216 + }
217 +
218 + #[derive(Serialize)]
219 + struct MultipartPartsResponse {
220 + parts: Vec<MultipartPartUrl>,
221 + expires_in: u64,
222 + }
223 +
224 + /// POST /api/internal/upload/multipart/parts
225 + ///
226 + /// Mint a bounded window of presigned `UploadPart` URLs. Each carries its exact
227 + /// signed `Content-Length`, the same defense-in-depth the single-PUT presign
228 + /// applies.
229 + #[tracing::instrument(skip_all, name = "internal::multipart_parts")]
230 + pub(super) async fn multipart_parts(
231 + State(db): State<PgPool>,
232 + State(storage): State<AppStorage>,
233 + actor: InternalActor,
234 + _auth: ServiceAuth,
235 + Json(req): Json<MultipartPartsRequest>,
236 + ) -> Result<impl IntoResponse> {
237 + let s3 = storage.require_s3()?;
238 + let s3_key = authorize_multipart_key(&db, actor.user_id(), &req.s3_key).await?;
239 +
240 + let plan = s3_storage::MultipartPlan::auto(req.file_size_bytes.max(0) as u64)
241 + .map_err(AppError::BadRequest)?;
242 +
243 + if req.count == 0 || req.count > MULTIPART_PART_URL_WINDOW {
244 + return Err(AppError::BadRequest(format!(
245 + "count must be between 1 and {MULTIPART_PART_URL_WINDOW}"
246 + )));
247 + }
248 + if req.first_part == 0 || req.first_part > plan.part_count {
249 + return Err(AppError::BadRequest(format!(
250 + "first_part must be between 1 and {}",
251 + plan.part_count
252 + )));
253 + }
254 +
255 + let expires_in = 3600u64;
256 + let last = (req.first_part + req.count - 1).min(plan.part_count);
257 + let mut parts = Vec::with_capacity((last - req.first_part + 1) as usize);
258 + for part_number in req.first_part..=last {
259 + let content_length = plan.part_len(part_number);
260 + let url = s3
261 + .presign_upload_part(
262 + &s3_key,
263 + &req.upload_id,
264 + part_number as i32,
265 + Some(expires_in),
266 + Some(content_length as i64),
267 + )
268 + .await?;
269 + parts.push(MultipartPartUrl {
270 + part_number: part_number as i32,
271 + content_length,
272 + url,
273 + });
274 + }
275 +
276 + Ok(Json(MultipartPartsResponse { parts, expires_in }))
277 + }
278 +
279 + #[derive(Deserialize)]
280 + pub(super) struct MultipartCompletedPart {
281 + part_number: i32,
282 + etag: String,
283 + }
284 +
285 + #[derive(Deserialize)]
286 + pub(super) struct MultipartCompleteRequest {
287 + s3_key: String,
288 + upload_id: String,
289 + parts: Vec<MultipartCompletedPart>,
290 + }
291 +
292 + #[derive(Serialize)]
293 + struct MultipartCompleteResponse {
294 + success: bool,
295 + s3_key: String,
296 + }
297 +
298 + /// POST /api/internal/upload/multipart/complete
299 + ///
300 + /// Assemble the uploaded parts into the staging object. This finalizes the
301 + /// transport only; the client then calls `/api/internal/upload/confirm`, which
302 + /// applies every size/tier/scan/commit rule against the real object.
303 + #[tracing::instrument(skip_all, name = "internal::multipart_complete")]
304 + pub(super) async fn multipart_complete(
305 + State(db): State<PgPool>,
306 + State(storage): State<AppStorage>,
307 + actor: InternalActor,
308 + _auth: ServiceAuth,
309 + Json(req): Json<MultipartCompleteRequest>,
310 + ) -> Result<impl IntoResponse> {
311 + let s3 = storage.require_s3()?;
312 + let s3_key = authorize_multipart_key(&db, actor.user_id(), &req.s3_key).await?;
313 +
314 + let parts: Vec<(i32, String)> = req
315 + .parts
316 + .into_iter()
317 + .map(|p| (p.part_number, p.etag))
318 + .collect();
319 +
320 + s3.complete_multipart_upload(&s3_key, &req.upload_id, &parts)
321 + .await?;
322 +
323 + tracing::info!(
324 + user = %actor.user_id(), s3_key = %s3_key, parts = parts.len(),
325 + "CLI multipart upload completed"
326 + );
327 +
328 + Ok(Json(MultipartCompleteResponse {
329 + success: true,
330 + s3_key: s3_key.into_string(),
331 + }))
332 + }
333 +
334 + #[derive(Deserialize)]
335 + pub(super) struct MultipartAbortRequest {
336 + s3_key: String,
337 + upload_id: String,
338 + }
339 +
340 + /// POST /api/internal/upload/multipart/abort
341 + ///
342 + /// Release the parts of an abandoned session (client cancel). Incomplete
343 + /// multipart uploads bill for their parts until aborted, so the client cleaning
344 + /// up on cancel is the cheapest fix; the pending-upload reaper is the backstop
345 + /// for clients that vanish.
346 + #[tracing::instrument(skip_all, name = "internal::multipart_abort")]
347 + pub(super) async fn multipart_abort(
348 + State(db): State<PgPool>,
349 + State(storage): State<AppStorage>,
350 + actor: InternalActor,
351 + _auth: ServiceAuth,
352 + Json(req): Json<MultipartAbortRequest>,
353 + ) -> Result<impl IntoResponse> {
354 + let s3 = storage.require_s3()?;
355 + let s3_key = authorize_multipart_key(&db, actor.user_id(), &req.s3_key).await?;
356 +
357 + s3.abort_multipart_upload(&s3_key, &req.upload_id).await?;
358 +
359 + tracing::info!(user = %actor.user_id(), s3_key = %s3_key, "CLI multipart upload aborted");
360 + Ok(Json(InternalConfirmResponse { success: true }))
361 + }
362 +
363 + /// Bind a caller-supplied staging key to the caller.
364 + ///
365 + /// A `staging/{uuid}` key carries no user in its path, so ownership comes from
366 + /// the `pending_uploads` row `start` recorded — the same proof `confirm_upload`
367 + /// uses. Without this, any authenticated creator could drive parts into another
368 + /// creator's in-flight session.
369 + async fn authorize_multipart_key(
370 + db: &PgPool,
371 + user_id: crate::db::UserId,
372 + s3_key: &str,
373 + ) -> Result<crate::storage::S3Key> {
374 + if !db::pending_uploads::is_owned(db, user_id, s3_key, "main").await? {
375 + return Err(AppError::BadRequest("Invalid upload key".to_string()));
376 + }
377 + Ok(crate::storage::S3Key::from_stored(s3_key))
378 + }
379 +
97 380 // ── Confirm upload (for CLI upload pipeline) ──
98 381
99 382 #[derive(Deserialize)]
M server/src/routes/storage/mod.rs +88 -11
@@ -40,6 +40,36 @@ pub(crate) fn validate_declared_upload_size(
40 40 file_type: FileType,
41 41 max_file_bytes: Option<u64>,
42 42 ) -> Result<()> {
43 + validate_declared_upload_size_limits(size, file_type, max_file_bytes)?;
44 + let Some(size) = size else {
45 + return Ok(());
46 + };
47 + // Browser transport ceiling: every caller of THIS function issues a single
48 + // presigned PUT, which S3/Ceph rejects above 5 GiB. The tier cap can be
49 + // higher (BigFiles/Everything allow 20 GB) — those files upload through the
50 + // CLI/desktop clients, which chunk them — so point a too-big browser upload
51 + // there instead of handing out a presigned URL that would fail at S3.
52 + // Deliberately NOT in `validate_declared_upload_size_limits`: it is a
53 + // property of the one-shot transport, not of the file, and the multipart
54 + // path is exactly what this message points people toward.
55 + if size as u64 > constants::S3_SINGLE_PUT_MAX_BYTES {
56 + let limit_gb = constants::S3_SINGLE_PUT_MAX_BYTES / (1024 * 1024 * 1024);
57 + return Err(AppError::FileTooLarge(format!(
58 + "Files larger than {limit_gb} GB must be uploaded with the makenot.work CLI or desktop app."
59 + )));
60 + }
61 + Ok(())
62 + }
63 +
64 + /// The size limits that hold for *any* transport: positive, the per-file-type
65 + /// cap, and the tier's per-file cap. The single-PUT ceiling is excluded, so the
66 + /// multipart (CLI/desktop) path can accept files above it while still enforcing
67 + /// every limit that describes the file rather than how it is carried.
68 + pub(crate) fn validate_declared_upload_size_limits(
69 + size: Option<i64>,
70 + file_type: FileType,
71 + max_file_bytes: Option<u64>,
72 + ) -> Result<()> {
43 73 let Some(size) = size else {
44 74 return Ok(());
45 75 };
@@ -67,17 +97,6 @@ pub(crate) fn validate_declared_upload_size(
67 97 limit_mb
68 98 )));
69 99 }
70 - // Browser transport ceiling: every caller of this function issues a single
71 - // presigned PUT, which S3/Ceph rejects above 5 GiB. The tier cap can be
72 - // higher (BigFiles/Everything allow 20 GB) — those files upload through the
73 - // CLI/desktop clients, which chunk them — so point a too-big browser upload
74 - // there instead of handing out a presigned URL that would fail at S3.
75 - if size as u64 > constants::S3_SINGLE_PUT_MAX_BYTES {
76 - let limit_gb = constants::S3_SINGLE_PUT_MAX_BYTES / (1024 * 1024 * 1024);
77 - return Err(AppError::FileTooLarge(format!(
78 - "Files larger than {limit_gb} GB must be uploaded with the makenot.work CLI or desktop app."
79 - )));
80 - }
81 100 Ok(())
82 101 }
83 102
@@ -538,3 +557,61 @@ async fn enqueue_scan_for(
538 557
539 558 Ok(db::FileScanStatus::Pending)
540 559 }
560 +
561 + #[cfg(test)]
562 + mod tests {
563 + use super::*;
564 +
565 + const GIB: i64 = 1024 * 1024 * 1024;
566 +
567 + #[test]
568 + fn browser_validator_refuses_above_the_single_put_ceiling() {
569 + // 10 GiB video: within the 20 GB per-type cap, but a browser issues one
570 + // presigned PUT and S3 rejects that above 5 GiB.
571 + let err = validate_declared_upload_size(Some(10 * GIB), FileType::Video, None)
572 + .expect_err("browser upload above 5 GiB must be refused");
573 + assert!(
574 + matches!(err, AppError::FileTooLarge(ref m) if m.contains("CLI or desktop app")),
575 + "expected a pointer to the CLI, got: {err:?}"
576 + );
577 + }
578 +
579 + #[test]
580 + fn multipart_validator_allows_above_the_single_put_ceiling() {
581 + // The same file over the chunked CLI path is fine — exceeding the
582 + // one-shot ceiling is the entire point of multipart. If this ever starts
583 + // failing, the 5 GiB-to-20 GB tier band is unreachable again.
584 + validate_declared_upload_size_limits(Some(10 * GIB), FileType::Video, None)
585 + .expect("multipart upload above 5 GiB must be allowed");
586 + }
587 +
588 + #[test]
589 + fn multipart_validator_still_enforces_the_per_type_cap() {
590 + // Skipping the transport ceiling must not skip the limits that describe
591 + // the file itself: 25 GiB is past FileType::Video's 20 GB cap.
592 + assert!(validate_declared_upload_size_limits(Some(25 * GIB), FileType::Video, None).is_err());
593 + }
594 +
595 + #[test]
596 + fn multipart_validator_still_enforces_the_tier_cap() {
597 + // A 10 GiB file under a 6 GiB tier cap is refused before it stages parts.
598 + assert!(
599 + validate_declared_upload_size_limits(Some(10 * GIB), FileType::Video, Some(6 * GIB as u64))
600 + .is_err()
601 + );
602 + }
603 +
604 + #[test]
605 + fn both_validators_reject_non_positive_sizes() {
606 + assert!(validate_declared_upload_size(Some(0), FileType::Video, None).is_err());
607 + assert!(validate_declared_upload_size_limits(Some(0), FileType::Video, None).is_err());
608 + assert!(validate_declared_upload_size_limits(Some(-1), FileType::Video, None).is_err());
609 + }
610 +
611 + #[test]
612 + fn absent_size_is_accepted_by_both() {
613 + // No declared size: the confirm-time HEAD is the bound.
614 + validate_declared_upload_size(None, FileType::Video, None).unwrap();
615 + validate_declared_upload_size_limits(None, FileType::Video, None).unwrap();
616 + }
617 + }