Skip to main content

max / makenotwork

8.0 KB · 228 lines History Blame Raw
1 //! Negative paths: what the server does when object storage fails.
2 //!
3 //! Split out of the former `failure_paths.rs` on 2026-08-05, which reached 868
4 //! lines and tripped the oversized-module ratchet in `tests/test_hygiene.rs`.
5 //! The split is by the dependency that fails, which is also how you look these
6 //! up: anything about S3 being down lands here.
7 //!
8 //! These tests exist because the mocks used to be infallible, so the retry and
9 //! compensation machinery the server carries had no test that could reach it.
10 //! Retry logic no test can enter is worse than none, because it reads as
11 //! handled. Each test installs a failure policy on a mock (see
12 //! `harness::faults`) and asserts the compensating behaviour, not just that the
13 //! request failed.
14 //!
15 //! Rationale: wiki `testing-posture`, the "absent oracle" section.
16
17 use crate::harness::TestHarness;
18 use crate::harness::faults::storage_unavailable;
19 use makenotwork::db;
20 use makenotwork::storage::StorageBackend;
21
22 // The durable S3 deletion queue
23
24 /// Count rows still queued for deletion of `key`.
25 async fn queued_deletions(h: &TestHarness, key: &str) -> i64 {
26 sqlx::query_scalar("SELECT COUNT(*) FROM pending_s3_deletions WHERE s3_key = $1")
27 .bind(key)
28 .fetch_one(&h.db)
29 .await
30 .unwrap()
31 }
32
33 /// A delete that fails must leave the row queued. Dequeuing it would orphan the
34 /// S3 object with no durable record, which is the leak the queue exists to
35 /// prevent.
36 #[tokio::test]
37 async fn s3_delete_failure_keeps_the_row_queued_for_retry() {
38 let h = TestHarness::with_storage().await;
39 let storage = h.storage.clone().expect("with_storage provides a backend");
40 let key = "test/orphan-retry.bin";
41
42 storage.put(key, b"payload".to_vec());
43 db::pending_s3_deletions::enqueue_deletions(
44 &h.db,
45 &[(key.to_string(), "main".to_string())],
46 "test_failure_path",
47 )
48 .await
49 .unwrap();
50 assert_eq!(queued_deletions(&h, key).await, 1, "row starts queued");
51
52 storage
53 .faults()
54 .fail_always("delete_object", storage_unavailable);
55 let deleted = h.drain_s3_deletions().await;
56
57 assert_eq!(deleted, 0, "a failing backend deletes nothing");
58 assert_eq!(
59 queued_deletions(&h, key).await,
60 1,
61 "the row must survive a failed delete, dropping it would orphan the object"
62 );
63 assert!(
64 storage.object_exists(key).await.unwrap(),
65 "the object is still there, which is why the row must be"
66 );
67 assert_eq!(
68 storage.faults().calls("delete_object"),
69 1,
70 "the drain attempted the delete exactly once"
71 );
72 }
73
74 /// The point of keeping the row: a later drain finishes the job. This is the
75 /// whole contract of the durable queue and nothing asserted it before.
76 #[tokio::test]
77 async fn s3_delete_queue_recovers_when_the_backend_comes_back() {
78 let h = TestHarness::with_storage().await;
79 let storage = h.storage.clone().expect("with_storage provides a backend");
80 let key = "test/orphan-recovers.bin";
81
82 storage.put(key, b"payload".to_vec());
83 db::pending_s3_deletions::enqueue_deletions(
84 &h.db,
85 &[(key.to_string(), "main".to_string())],
86 "test_failure_path",
87 )
88 .await
89 .unwrap();
90
91 // Down for the first attempt, up for the second.
92 storage
93 .faults()
94 .fail_until("delete_object", 2, storage_unavailable);
95
96 assert_eq!(h.drain_s3_deletions().await, 0, "first drain fails");
97 assert_eq!(queued_deletions(&h, key).await, 1, "still queued");
98
99 assert_eq!(h.drain_s3_deletions().await, 1, "second drain succeeds");
100 assert_eq!(
101 queued_deletions(&h, key).await,
102 0,
103 "a completed delete is dequeued"
104 );
105 assert!(
106 !storage.object_exists(key).await.unwrap(),
107 "the object is gone"
108 );
109 }
110
111 // The orphaned-upload reaper
112
113 /// Insert a pending upload that is already old enough for the reaper, with the
114 /// object present in storage. Returns the key.
115 async fn stale_pending_upload(h: &TestHarness, user_id: db::UserId, key: &str) -> String {
116 h.storage.as_ref().unwrap().put(key, b"orphan".to_vec());
117 sqlx::query(
118 "INSERT INTO pending_uploads (user_id, s3_key, bucket, created_at)
119 VALUES ($1, $2, 'main', NOW() - INTERVAL '48 hours')",
120 )
121 .bind(user_id)
122 .bind(key)
123 .execute(&h.db)
124 .await
125 .unwrap();
126 key.to_string()
127 }
128
129 async fn pending_upload_rows(h: &TestHarness, key: &str) -> i64 {
130 sqlx::query_scalar("SELECT COUNT(*) FROM pending_uploads WHERE s3_key = $1")
131 .bind(key)
132 .fetch_one(&h.db)
133 .await
134 .unwrap()
135 }
136
137 /// The happy path, asserted here so the failure path below is a contrast rather
138 /// than the only thing observed: a reaped orphan is deleted, its tracking row is
139 /// cleared, and nothing is handed to the durable queue.
140 #[tokio::test]
141 async fn the_reaper_deletes_an_orphan_and_clears_its_row() {
142 let mut h = TestHarness::with_storage().await;
143 let user_id = h.signup("reap1", "reap1@test.com", "pass1234").await;
144 let key = stale_pending_upload(&h, user_id, "staging/reaped.bin").await;
145 let storage = h.storage.clone().unwrap();
146
147 h.run_orphan_upload_reaper().await;
148
149 assert!(
150 !storage.object_exists(&key).await.unwrap(),
151 "the orphan object is deleted"
152 );
153 assert_eq!(
154 pending_upload_rows(&h, &key).await,
155 0,
156 "tracking row cleared"
157 );
158 assert_eq!(
159 queued_deletions(&h, &key).await,
160 0,
161 "a successful delete must not also enqueue, that would double-handle the key"
162 );
163 }
164
165 /// A transient S3 failure must hand the key to the durable deletion queue
166 /// BEFORE the tracking row is cleared. Clearing the row on a transient failure
167 /// dropped the only record of the object and leaked it permanently (Run #2
168 /// Storage SERIOUS). The fix has been in the tree unobserved since; this is the
169 /// test that enters it.
170 #[tokio::test]
171 async fn a_transient_delete_failure_hands_the_orphan_to_the_durable_queue() {
172 let mut h = TestHarness::with_storage().await;
173 let user_id = h.signup("reap2", "reap2@test.com", "pass1234").await;
174 let key = stale_pending_upload(&h, user_id, "staging/handed-off.bin").await;
175 let storage = h.storage.clone().unwrap();
176
177 storage
178 .faults()
179 .fail_always("delete_object", storage_unavailable);
180 h.run_orphan_upload_reaper().await;
181
182 assert!(
183 storage.object_exists(&key).await.unwrap(),
184 "the delete failed, so the object is still there"
185 );
186 assert_eq!(
187 queued_deletions(&h, &key).await,
188 1,
189 "the key must be queued for retry; without this the object leaks"
190 );
191 assert_eq!(
192 pending_upload_rows(&h, &key).await,
193 0,
194 "the tracking row is cleared only because the durable queue now owns the key"
195 );
196
197 // The handoff is worth nothing if the queue cannot then finish the job.
198 storage.faults().clear("delete_object");
199 assert_eq!(h.drain_s3_deletions().await, 1, "the retry completes it");
200 assert!(!storage.object_exists(&key).await.unwrap(), "object gone");
201 }
202
203 /// Aborting orphaned multipart sessions is documented best-effort: it must not
204 /// block the object delete. A failing abort that stranded the delete would leave
205 /// the orphan in place every tick forever, and the tracking row with it.
206 #[tokio::test]
207 async fn a_failed_multipart_abort_does_not_block_the_orphan_delete() {
208 let mut h = TestHarness::with_storage().await;
209 let user_id = h.signup("reap3", "reap3@test.com", "pass1234").await;
210 let key = stale_pending_upload(&h, user_id, "staging/abort-fails.bin").await;
211 let storage = h.storage.clone().unwrap();
212
213 storage
214 .faults()
215 .fail_always("list_multipart_uploads_for_key", storage_unavailable);
216 h.run_orphan_upload_reaper().await;
217
218 assert!(
219 !storage.object_exists(&key).await.unwrap(),
220 "a failed abort is best-effort and must not stop the delete"
221 );
222 assert_eq!(
223 pending_upload_rows(&h, &key).await,
224 0,
225 "and the tracking row is still cleared"
226 );
227 }
228