Skip to main content

max / makenotwork

7.7 KB · 224 lines History Blame Raw
1 //! Negative paths: what the server does when object storage fails.
2 //!
3 //! Negative paths are split by the dependency that fails, which is also how you
4 //! look these up: anything about S3 being down lands here.
5 //!
6 //! These tests exist because an infallible mock leaves the retry and
7 //! compensation machinery the server carries with no test that can reach it.
8 //! Retry logic no test can enter is worse than none, because it reads as
9 //! handled. Each test installs a failure policy on a mock (see
10 //! `harness::faults`) and asserts the compensating behaviour, not just that the
11 //! request failed.
12 //!
13 //! Rationale: wiki `testing-posture`, the "absent oracle" section.
14
15 use crate::harness::TestHarness;
16 use crate::harness::faults::storage_unavailable;
17 use makenotwork::db;
18 use makenotwork::storage::StorageBackend;
19
20 // The durable S3 deletion queue
21
22 /// Count rows still queued for deletion of `key`.
23 async fn queued_deletions(h: &TestHarness, key: &str) -> i64 {
24 sqlx::query_scalar("SELECT COUNT(*) FROM pending_s3_deletions WHERE s3_key = $1")
25 .bind(key)
26 .fetch_one(&h.db)
27 .await
28 .unwrap()
29 }
30
31 /// A delete that fails must leave the row queued. Dequeuing it would orphan the
32 /// S3 object with no durable record, which is the leak the queue exists to
33 /// prevent.
34 #[tokio::test]
35 async fn s3_delete_failure_keeps_the_row_queued_for_retry() {
36 let h = TestHarness::with_storage().await;
37 let storage = h.storage.clone().expect("with_storage provides a backend");
38 let key = "test/orphan-retry.bin";
39
40 storage.put(key, b"payload".to_vec());
41 db::pending_s3_deletions::enqueue_deletions(
42 &h.db,
43 &[(key.to_string(), "main".to_string())],
44 "test_failure_path",
45 )
46 .await
47 .unwrap();
48 assert_eq!(queued_deletions(&h, key).await, 1, "row starts queued");
49
50 storage
51 .faults()
52 .fail_always("delete_object", storage_unavailable);
53 let deleted = h.drain_s3_deletions().await;
54
55 assert_eq!(deleted, 0, "a failing backend deletes nothing");
56 assert_eq!(
57 queued_deletions(&h, key).await,
58 1,
59 "the row must survive a failed delete, dropping it would orphan the object"
60 );
61 assert!(
62 storage.object_exists(key).await.unwrap(),
63 "the object is still there, which is why the row must be"
64 );
65 assert_eq!(
66 storage.faults().calls("delete_object"),
67 1,
68 "the drain attempted the delete exactly once"
69 );
70 }
71
72 /// The point of keeping the row: a later drain finishes the job. This is the
73 /// whole contract of the durable queue and nothing asserted it before.
74 #[tokio::test]
75 async fn s3_delete_queue_recovers_when_the_backend_comes_back() {
76 let h = TestHarness::with_storage().await;
77 let storage = h.storage.clone().expect("with_storage provides a backend");
78 let key = "test/orphan-recovers.bin";
79
80 storage.put(key, b"payload".to_vec());
81 db::pending_s3_deletions::enqueue_deletions(
82 &h.db,
83 &[(key.to_string(), "main".to_string())],
84 "test_failure_path",
85 )
86 .await
87 .unwrap();
88
89 // Down for the first attempt, up for the second.
90 storage
91 .faults()
92 .fail_until("delete_object", 2, storage_unavailable);
93
94 assert_eq!(h.drain_s3_deletions().await, 0, "first drain fails");
95 assert_eq!(queued_deletions(&h, key).await, 1, "still queued");
96
97 assert_eq!(h.drain_s3_deletions().await, 1, "second drain succeeds");
98 assert_eq!(
99 queued_deletions(&h, key).await,
100 0,
101 "a completed delete is dequeued"
102 );
103 assert!(
104 !storage.object_exists(key).await.unwrap(),
105 "the object is gone"
106 );
107 }
108
109 // The orphaned-upload reaper
110
111 /// Insert a pending upload that is already old enough for the reaper, with the
112 /// object present in storage. Returns the key.
113 async fn stale_pending_upload(h: &TestHarness, user_id: db::UserId, key: &str) -> String {
114 h.storage.as_ref().unwrap().put(key, b"orphan".to_vec());
115 sqlx::query(
116 "INSERT INTO pending_uploads (user_id, s3_key, bucket, created_at)
117 VALUES ($1, $2, 'main', NOW() - INTERVAL '48 hours')",
118 )
119 .bind(user_id)
120 .bind(key)
121 .execute(&h.db)
122 .await
123 .unwrap();
124 key.to_string()
125 }
126
127 async fn pending_upload_rows(h: &TestHarness, key: &str) -> i64 {
128 sqlx::query_scalar("SELECT COUNT(*) FROM pending_uploads WHERE s3_key = $1")
129 .bind(key)
130 .fetch_one(&h.db)
131 .await
132 .unwrap()
133 }
134
135 /// The happy path, asserted here so the failure path below is a contrast rather
136 /// than the only thing observed: a reaped orphan is deleted, its tracking row is
137 /// cleared, and nothing is handed to the durable queue.
138 #[tokio::test]
139 async fn the_reaper_deletes_an_orphan_and_clears_its_row() {
140 let mut h = TestHarness::with_storage().await;
141 let user_id = h.signup("reap1", "reap1@test.com", "pass1234").await;
142 let key = stale_pending_upload(&h, user_id, "staging/reaped.bin").await;
143 let storage = h.storage.clone().unwrap();
144
145 h.run_orphan_upload_reaper().await;
146
147 assert!(
148 !storage.object_exists(&key).await.unwrap(),
149 "the orphan object is deleted"
150 );
151 assert_eq!(
152 pending_upload_rows(&h, &key).await,
153 0,
154 "tracking row cleared"
155 );
156 assert_eq!(
157 queued_deletions(&h, &key).await,
158 0,
159 "a successful delete must not also enqueue, that would double-handle the key"
160 );
161 }
162
163 /// A transient S3 failure must hand the key to the durable deletion queue
164 /// BEFORE the tracking row is cleared. Clearing the row on a transient failure
165 /// drops the only record of the object and leaks it permanently.
166 #[tokio::test]
167 async fn a_transient_delete_failure_hands_the_orphan_to_the_durable_queue() {
168 let mut h = TestHarness::with_storage().await;
169 let user_id = h.signup("reap2", "reap2@test.com", "pass1234").await;
170 let key = stale_pending_upload(&h, user_id, "staging/handed-off.bin").await;
171 let storage = h.storage.clone().unwrap();
172
173 storage
174 .faults()
175 .fail_always("delete_object", storage_unavailable);
176 h.run_orphan_upload_reaper().await;
177
178 assert!(
179 storage.object_exists(&key).await.unwrap(),
180 "the delete failed, so the object is still there"
181 );
182 assert_eq!(
183 queued_deletions(&h, &key).await,
184 1,
185 "the key must be queued for retry; without this the object leaks"
186 );
187 assert_eq!(
188 pending_upload_rows(&h, &key).await,
189 0,
190 "the tracking row is cleared only because the durable queue now owns the key"
191 );
192
193 // The handoff is worth nothing if the queue cannot then finish the job.
194 storage.faults().clear("delete_object");
195 assert_eq!(h.drain_s3_deletions().await, 1, "the retry completes it");
196 assert!(!storage.object_exists(&key).await.unwrap(), "object gone");
197 }
198
199 /// Aborting orphaned multipart sessions is documented best-effort: it must not
200 /// block the object delete. A failing abort that stranded the delete would leave
201 /// the orphan in place every tick forever, and the tracking row with it.
202 #[tokio::test]
203 async fn a_failed_multipart_abort_does_not_block_the_orphan_delete() {
204 let mut h = TestHarness::with_storage().await;
205 let user_id = h.signup("reap3", "reap3@test.com", "pass1234").await;
206 let key = stale_pending_upload(&h, user_id, "staging/abort-fails.bin").await;
207 let storage = h.storage.clone().unwrap();
208
209 storage
210 .faults()
211 .fail_always("list_multipart_uploads_for_key", storage_unavailable);
212 h.run_orphan_upload_reaper().await;
213
214 assert!(
215 !storage.object_exists(&key).await.unwrap(),
216 "a failed abort is best-effort and must not stop the delete"
217 );
218 assert_eq!(
219 pending_upload_rows(&h, &key).await,
220 0,
221 "and the tracking row is still cleared"
222 );
223 }
224