Skip to main content

max / makenotwork

Make the test mocks fallible, and reach the failure machinery The mocks could not fail. InMemoryStorage errored only on genuine domain conditions, and the payment and email mocks had no error path at all, so nothing in 1,226 integration tests could enter the retry and compensation machinery the server carries. Retry logic no test can reach is worse than none, because it reads as handled. Adds tests/harness/faults.rs: one failure policy shared by the three mocks, keyed by trait-method name, with fail_always, fail_nth and fail_until plus a call counter. A mock with no policy installed behaves exactly as before, so existing suites are untouched. Five negative paths that could not previously be written: - a failed S3 delete leaves the row queued, and a later drain finishes the job, which is the whole contract of the durable deletion queue - a Stripe outage at checkout releases the promo-code reservation, so an outage does not burn uses off a creator's code, and the retry then succeeds on a max_uses=1 code - a failed buyer receipt still sends the seller notification and leaves the transaction completed Phase 1 of wiki testing-posture, the "absent oracle" section.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-08-05 03:08 UTC
Signed with PGP, not checked
Commit: d758d8b5748f7318c818d6bb1961e8d01b6a766b
Parent: 052f553
7 files changed, +727 insertions, -1 deletion
M server/tests/harness/email.rs +15
@@ -3,6 +3,7 @@
3 3 //! Records all sent emails so tests can assert on recipients, subjects, and bodies
4 4 //! without hitting any external service.
5 5
6 + use super::faults::Faults;
6 7 use makenotwork::email::EmailTransport;
7 8 use makenotwork::error::Result;
8 9 use std::sync::Mutex;
@@ -21,6 +22,8 @@
21 22 /// In-memory email transport that records all sent emails.
22 23 pub(crate) struct MockEmailTransport {
23 24 sent: Mutex<Vec<SentEmail>>,
25 + /// Injected send failures. Empty by default.
26 + faults: Faults,
24 27 }
25 28
26 29 #[allow(dead_code)]
@@ -28,9 +31,17 @@
28 31 pub(crate) fn new() -> Self {
29 32 MockEmailTransport {
30 33 sent: Mutex::new(Vec::new()),
34 + faults: Faults::new(),
31 35 }
32 36 }
33 37
38 + /// The failure policy. All four send methods share the operation name
39 + /// `send_email`, because a caller picks one by what it needs to include and
40 + /// a transport outage takes out all of them together.
41 + pub(crate) fn faults(&self) -> &Faults {
42 + &self.faults
43 + }
44 +
34 45 /// Return all emails sent so far.
35 46 pub(crate) fn sent(&self) -> Vec<SentEmail> {
36 47 self.sent.lock().unwrap().clone()
@@ -61,6 +72,7 @@
61 72 #[async_trait::async_trait]
62 73 impl EmailTransport for MockEmailTransport {
63 74 async fn send_email(&self, to: &str, subject: &str, body: &str) -> Result<()> {
75 + self.faults.check("send_email")?;
64 76 self.sent.lock().unwrap().push(SentEmail {
65 77 to: to.to_string(),
66 78 subject: subject.to_string(),
@@ -78,6 +90,7 @@
78 90 body: &str,
79 91 unsub_url: Option<&str>,
80 92 ) -> Result<()> {
93 + self.faults.check("send_email")?;
81 94 self.sent.lock().unwrap().push(SentEmail {
82 95 to: to.to_string(),
83 96 subject: subject.to_string(),
@@ -96,6 +109,7 @@
96 109 _extra_headers: &[(&str, String)],
97 110 unsub_url: Option<&str>,
98 111 ) -> Result<()> {
112 + self.faults.check("send_email")?;
99 113 self.sent.lock().unwrap().push(SentEmail {
100 114 to: to.to_string(),
101 115 subject: subject.to_string(),
@@ -113,6 +127,7 @@
113 127 body: &str,
114 128 unsub_url: Option<&str>,
115 129 ) -> Result<()> {
130 + self.faults.check("send_email")?;
116 131 self.sent.lock().unwrap().push(SentEmail {
117 132 to: to.to_string(),
118 133 subject: subject.to_string(),
M server/tests/harness/mod.rs +1
@@ -3,6 +3,7 @@
3 3 pub(crate) mod client;
4 4 pub(crate) mod db;
5 5 pub(crate) mod email;
6 + pub(crate) mod faults;
6 7 pub(crate) mod gitfixture;
7 8 pub(crate) mod seed;
8 9 pub(crate) mod storage;
M server/tests/harness/storage.rs +35 -1
@@ -1,5 +1,6 @@
1 1 //! In-memory storage backend for integration tests.
2 2
3 + use super::faults::Faults;
3 4 use makenotwork::error::{AppError, Result};
4 5 use makenotwork::storage::{S3DeleteAuthority, S3Key, StorageBackend};
5 6 use std::collections::HashMap;
@@ -14,6 +15,9 @@
14 15 /// unaborted session is exactly the leak this models.
15 16 multipart: Mutex<HashMap<String, String>>,
16 17 bucket: String,
18 + /// Injected transport failures. Empty by default, so a backend with no
19 + /// policy installed behaves exactly as it did before this existed.
20 + faults: Faults,
17 21 }
18 22
19 23 #[allow(dead_code)]
@@ -23,9 +27,16 @@
23 27 objects: Mutex::new(HashMap::new()),
24 28 multipart: Mutex::new(HashMap::new()),
25 29 bucket: "test-bucket".to_string(),
30 + faults: Faults::new(),
26 31 }
27 32 }
28 33
34 + /// The failure policy. Install rules on it to reach the retry and
35 + /// compensation paths that no test could otherwise enter.
36 + pub(crate) fn faults(&self) -> &Faults {
37 + &self.faults
38 + }
39 +
29 40 /// Number of multipart sessions still open. Zero after a clean reap.
30 41 pub(crate) fn open_multipart_count(&self) -> usize {
31 42 self.multipart.lock().unwrap().len()
@@ -67,10 +78,12 @@
67 78 _cache_control: Option<&str>,
68 79 _max_bytes: Option<i64>,
69 80 ) -> Result<String> {
81 + self.faults.check("presign_upload")?;
70 82 Ok(format!("http://test-storage/{s3_key}"))
71 83 }
72 84
73 85 async fn presign_download(&self, s3_key: &S3Key, _expiry_secs: Option<u64>) -> Result<String> {
86 + self.faults.check("presign_download")?;
74 87 if self.objects.lock().unwrap().contains_key(s3_key.as_str()) {
75 88 Ok(format!("http://test-storage/{s3_key}"))
76 89 } else {
@@ -79,10 +92,12 @@
79 92 }
80 93
81 94 async fn object_exists(&self, s3_key: &str) -> Result<bool> {
95 + self.faults.check("object_exists")?;
82 96 Ok(self.objects.lock().unwrap().contains_key(s3_key))
83 97 }
84 98
85 99 async fn object_size(&self, s3_key: &str) -> Result<Option<i64>> {
100 + self.faults.check("object_size")?;
86 101 Ok(self
87 102 .objects
88 103 .lock()
@@ -91,7 +106,12 @@
91 106 .map(|v| v.len() as i64))
92 107 }
93 108
109 + // `download_object_buf` and the two `copy_object_*` wrappers delegate to the
110 + // methods below them, so a policy on the delegate is what fires and the call
111 + // count is recorded once. Name the underlying operation in a rule, not the
112 + // wrapper.
94 113 async fn download_object(&self, s3_key: &str) -> Result<Vec<u8>> {
114 + self.faults.check("download_object")?;
95 115 self.objects
96 116 .lock()
97 117 .unwrap()
@@ -105,6 +125,7 @@
105 125 }
106 126
107 127 async fn download_stream(&self, s3_key: &str) -> Result<s3_storage::ByteStream> {
128 + self.faults.check("download_stream")?;
108 129 let bytes = self
109 130 .objects
110 131 .lock()
@@ -122,6 +143,7 @@
122 143 data: Vec<u8>,
123 144 _cache_control: Option<&str>,
124 145 ) -> Result<()> {
146 + self.faults.check("upload_object")?;
125 147 self.objects
126 148 .lock()
127 149 .unwrap()
@@ -135,6 +157,7 @@
135 157 _content_type: &str,
136 158 file_path: &std::path::Path,
137 159 ) -> Result<()> {
160 + self.faults.check("upload_multipart")?;
138 161 let data = tokio::fs::read(file_path)
139 162 .await
140 163 .map_err(|e| AppError::Storage(format!("read multipart source: {e}")))?;
@@ -146,11 +169,13 @@
146 169 }
147 170
148 171 async fn delete_object(&self, _auth: &S3DeleteAuthority, s3_key: &S3Key) -> Result<()> {
172 + self.faults.check("delete_object")?;
149 173 self.objects.lock().unwrap().remove(s3_key.as_str());
150 174 Ok(())
151 175 }
152 176
153 177 async fn copy_object(&self, src_key: &S3Key, dst_key: &S3Key) -> Result<()> {
178 + self.faults.check("copy_object")?;
154 179 let mut objects = self.objects.lock().unwrap();
155 180 let bytes = objects
156 181 .get(src_key.as_str())
@@ -195,6 +220,7 @@
195 220 // confirm path, exactly as they do for `presign_upload` + confirm.
196 221
197 222 async fn create_multipart_upload(&self, s3_key: &S3Key, _content_type: &str) -> Result<String> {
223 + self.faults.check("create_multipart_upload")?;
198 224 let upload_id = format!("test-upload-id/{s3_key}");
199 225 self.put_open_multipart(&upload_id, s3_key.as_str());
200 226 Ok(upload_id)
@@ -209,6 +235,7 @@
209 235 _max_bytes: Option<i64>,
210 236 checksum_sha256: Option<&str>,
211 237 ) -> Result<String> {
238 + self.faults.check("presign_upload_part")?;
212 239 // Mirror the production range check so a bad part number fails in tests
213 240 // the same way it would against S3.
214 241 if !(1..=s3_storage::MULTIPART_MAX_PARTS as i32).contains(&part_number) {
@@ -233,6 +260,7 @@
233 260 upload_id: &str,
234 261 parts: &[(i32, String)],
235 262 ) -> Result<()> {
263 + self.faults.check("complete_multipart_upload")?;
236 264 if parts.is_empty() {
237 265 return Err(AppError::Storage(
238 266 "cannot complete a multipart upload with no parts".to_string(),
@@ -244,12 +272,14 @@
244 272 }
245 273
246 274 async fn abort_multipart_upload(&self, _s3_key: &S3Key, upload_id: &str) -> Result<()> {
275 + self.faults.check("abort_multipart_upload")?;
247 276 // Idempotent: aborting an unknown session is fine.
248 277 self.multipart.lock().unwrap().remove(upload_id);
249 278 Ok(())
250 279 }
251 280
252 281 async fn list_multipart_uploads_for_key(&self, s3_key: &str) -> Result<Vec<String>> {
282 + self.faults.check("list_multipart_uploads_for_key")?;
253 283 Ok(self
254 284 .multipart
255 285 .lock()
@@ -261,7 +291,11 @@
261 291 }
262 292
263 293 async fn check_connectivity(&self) -> std::result::Result<(), String> {
264 - Ok(())
294 + // Health checks report a string, not an `AppError`, so the injected
295 + // error is rendered rather than propagated.
296 + self.faults
297 + .check("check_connectivity")
298 + .map_err(|e| e.to_string())
265 299 }
266 300
267 301 fn bucket(&self) -> &str {
M server/tests/harness/stripe.rs +44
@@ -1,5 +1,6 @@
1 1 //! Stripe test helpers, webhook signature computation and mock payment provider.
2 2
3 + use super::faults::Faults;
3 4 use hmac::{Hmac, KeyInit, Mac};
4 5 use sha2::Sha256;
5 6 use std::sync::Mutex;
@@ -84,6 +85,9 @@
84 85 /// Platform-credit reversals requested, in call order. Lets refund tests
85 86 /// assert a settled credit was clawed back.
86 87 reversals: Mutex<Vec<MockReversal>>,
88 + /// Injected Stripe failures. Empty by default. Operations are named for the
89 + /// trait method, so a rule reads as the Stripe call it breaks.
90 + faults: Faults,
87 91 }
88 92
89 93 /// A line-scoped refund captured by the mock.
@@ -123,9 +127,17 @@
123 127 refunds: Mutex::new(Vec::new()),
124 128 transfers: Mutex::new(Vec::new()),
125 129 reversals: Mutex::new(Vec::new()),
130 + faults: Faults::new(),
126 131 }
127 132 }
128 133
134 + /// The failure policy. Install rules on it to reach the compensation paths,
135 + /// `db/pending_refunds.rs` above all, that a provider which never fails
136 + /// leaves unobserved.
137 + pub(crate) fn faults(&self) -> &Faults {
138 + &self.faults
139 + }
140 +
129 141 /// All line-scoped refunds requested so far.
130 142 pub(crate) fn refunds(&self) -> Vec<MockRefund> {
131 143 self.refunds.lock().unwrap().clone()
@@ -170,6 +182,7 @@
170 182 &self,
171 183 _params: &CheckoutParams<'_>,
172 184 ) -> Result<CheckoutResult> {
185 + self.faults.check("create_checkout_session")?;
173 186 Ok(self.next_session())
174 187 }
175 188
@@ -177,6 +190,7 @@
177 190 &self,
178 191 _params: &makenotwork::payments::GuestCheckoutParams<'_>,
179 192 ) -> Result<CheckoutResult> {
193 + self.faults.check("create_guest_checkout_session")?;
180 194 Ok(self.next_session())
181 195 }
182 196
@@ -184,6 +198,7 @@
184 198 &self,
185 199 _params: &SubscriptionCheckoutParams<'_>,
186 200 ) -> Result<CheckoutResult> {
201 + self.faults.check("create_subscription_checkout_session")?;
187 202 Ok(self.next_session())
188 203 }
189 204
@@ -191,6 +206,7 @@
191 206 &self,
192 207 _params: &TipCheckoutParams<'_>,
193 208 ) -> Result<CheckoutResult> {
209 + self.faults.check("create_tip_checkout_session")?;
194 210 Ok(self.next_session())
195 211 }
196 212
@@ -201,6 +217,7 @@
201 217 _success_url: &str,
202 218 _cancel_url: &str,
203 219 ) -> Result<CheckoutResult> {
220 + self.faults.check("create_fan_plus_checkout_session")?;
204 221 Ok(self.next_session())
205 222 }
206 223
@@ -213,6 +230,7 @@
213 230 _cancel_url: &str,
214 231 trial_days: Option<i32>,
215 232 ) -> Result<CheckoutResult> {
233 + self.faults.check("create_creator_tier_checkout_session")?;
216 234 self.creator_tier_trial_days
217 235 .lock()
218 236 .unwrap()
@@ -224,6 +242,7 @@
224 242 &self,
225 243 _params: &makenotwork::payments::CartCheckoutParams<'_>,
226 244 ) -> Result<CheckoutResult> {
245 + self.faults.check("create_cart_checkout_session")?;
227 246 Ok(self.next_session())
228 247 }
229 248
@@ -231,6 +250,7 @@
231 250 &self,
232 251 _email: &str,
233 252 ) -> Result<makenotwork::db::StripeAccountId> {
253 + self.faults.check("create_connect_account")?;
234 254 Ok(makenotwork::db::StripeAccountId::from_trusted(
235 255 "acct_test_mock".to_string(),
236 256 ))
@@ -242,10 +262,12 @@
242 262 _return_url: &str,
243 263 _refresh_url: &str,
244 264 ) -> Result<String> {
265 + self.faults.check("create_account_link")?;
245 266 Ok("https://connect.stripe.com/test/onboarding".to_string())
246 267 }
247 268
248 269 async fn fetch_account(&self, account_id: &str) -> Result<AccountUpdate> {
270 + self.faults.check("fetch_account")?;
249 271 Ok(AccountUpdate {
250 272 account_id: account_id.to_string(),
251 273 charges_enabled: true,
@@ -261,10 +283,12 @@
261 283 _tier_description: Option<&str>,
262 284 _price_cents: i64,
263 285 ) -> Result<(String, String)> {
286 + self.faults.check("create_subscription_product_and_price")?;
264 287 Ok(("prod_test_mock".to_string(), "price_test_mock".to_string()))
265 288 }
266 289
267 290 async fn get_balance(&self, _account_id: &str) -> Result<BalanceSummary> {
291 + self.faults.check("get_balance")?;
268 292 Ok(BalanceSummary {
269 293 available_cents: 0,
270 294 pending_cents: 0,
@@ -276,12 +300,14 @@
276 300 payload: &str,
277 301 signature: &str,
278 302 ) -> Result<makenotwork::payments::UntypedEvent> {
303 + self.faults.check("verify_webhook")?;
279 304 makenotwork::payments::verify_signature(payload, signature, TEST_WEBHOOK_SECRET)
280 305 .map_err(AppError::BadRequest)?;
281 306 makenotwork::payments::UntypedEvent::from_payload(payload)
282 307 }
283 308
284 309 fn verify_webhook_v2(&self, payload: &str, signature: &str) -> Result<serde_json::Value> {
310 + self.faults.check("verify_webhook_v2")?;
285 311 makenotwork::payments::verify_signature(payload, signature, TEST_WEBHOOK_SECRET_V2)
286 312 .map_err(AppError::BadRequest)?;
287 313 serde_json::from_str(payload)
@@ -293,6 +319,7 @@
293 319 _stripe_sub_id: &str,
294 320 _connected_account_id: &str,
295 321 ) -> Result<()> {
322 + self.faults.check("pause_subscription")?;
296 323 Ok(())
297 324 }
298 325
@@ -301,6 +328,7 @@
301 328 _stripe_sub_id: &str,
302 329 _connected_account_id: &str,
303 330 ) -> Result<()> {
331 + self.faults.check("resume_subscription")?;
304 332 Ok(())
305 333 }
306 334
@@ -309,6 +337,7 @@
309 337 _stripe_sub_id: &str,
310 338 _connected_account_id: &str,
311 339 ) -> Result<()> {
340 + self.faults.check("cancel_subscription")?;
312 341 Ok(())
313 342 }
314 343
@@ -318,10 +347,12 @@
318 347 _connected_account_id: &str,
319 348 _cancel: bool,
320 349 ) -> Result<()> {
350 + self.faults.check("set_cancel_at_period_end")?;
321 351 Ok(())
322 352 }
323 353
324 354 async fn cancel_platform_subscription(&self, _stripe_sub_id: &str) -> Result<()> {
355 + self.faults.check("cancel_platform_subscription")?;
325 356 Ok(())
326 357 }
327 358
@@ -330,6 +361,7 @@
330 361 _stripe_sub_id: &str,
331 362 _cancel: bool,
332 363 ) -> Result<()> {
364 + self.faults.check("set_platform_cancel_at_period_end")?;
333 365 Ok(())
334 366 }
335 367
@@ -338,6 +370,7 @@
338 370 _stripe_customer_id: &str,
339 371 return_url: &str,
340 372 ) -> Result<String> {
373 + self.faults.check("create_billing_portal_session")?;
341 374 // Echo a deterministic URL so tests can assert the redirect target.
342 375 Ok(format!(
343 376 "https://billing.stripe.test/portal?return={}",
@@ -352,6 +385,7 @@
352 385 amount_cents: i64,
353 386 transaction_id: makenotwork::db::TransactionId,
354 387 ) -> Result<()> {
388 + self.faults.check("create_refund_for_transaction")?;
355 389 self.refunds.lock().unwrap().push(MockRefund {
356 390 payment_intent_id: payment_intent_id.to_string(),
357 391 amount_cents,
@@ -366,6 +400,7 @@
366 400 amount_cents: i64,
367 401 transaction_id: makenotwork::db::TransactionId,
368 402 ) -> Result<String> {
403 + self.faults.check("create_platform_credit_transfer")?;
369 404 self.transfers.lock().unwrap().push(MockTransfer {
370 405 connected_account_id: connected_account_id.to_string(),
371 406 amount_cents,
@@ -381,6 +416,7 @@
381 416 amount_cents: i64,
382 417 transaction_id: makenotwork::db::TransactionId,
383 418 ) -> Result<()> {
419 + self.faults.check("create_platform_credit_reversal")?;
384 420 self.reversals.lock().unwrap().push(MockReversal {
385 421 transfer_id: transfer_id.to_string(),
386 422 amount_cents,
@@ -396,6 +432,7 @@
396 432 _email: &str,
397 433 _app_name: &str,
398 434 ) -> Result<String> {
435 + self.faults.check("create_synckit_customer")?;
399 436 // Deterministic dummy; tests assert on shape, not content.
400 437 Ok("cus_test_synckit".to_string())
401 438 }
@@ -407,6 +444,7 @@
407 444 _app_name: &str,
408 445 _price_cents: i64,
409 446 ) -> Result<makenotwork::payments::SynckitSubResult> {
447 + self.faults.check("create_synckit_subscription")?;
410 448 let now = SystemTime::now()
411 449 .duration_since(UNIX_EPOCH)
412 450 .unwrap()
@@ -424,6 +462,7 @@
424 462 _new_price_cents: i64,
425 463 _app_name: &str,
426 464 ) -> Result<()> {
465 + self.faults.check("update_synckit_subscription_price")?;
427 466 Ok(())
428 467 }
429 468
@@ -434,6 +473,7 @@
434 473 _interval: makenotwork::payments::SyncBillingInterval,
435 474 _product_name: &str,
436 475 ) -> Result<()> {
476 + self.faults.check("update_synckit_app_sub_price")?;
437 477 Ok(())
438 478 }
439 479
@@ -441,10 +481,13 @@
441 481 &self,
442 482 _params: &makenotwork::payments::SynckitAppSubCheckoutParams<'_>,
443 483 ) -> Result<CheckoutResult> {
484 + self.faults
485 + .check("create_synckit_app_sub_checkout_session")?;
444 486 Ok(self.next_session())
445 487 }
446 488
447 489 async fn cancel_synckit_subscription(&self, _subscription_id: &str) -> Result<()> {
490 + self.faults.check("cancel_synckit_subscription")?;
448 491 Ok(())
449 492 }
450 493
@@ -453,6 +496,7 @@
453 496 _customer_id: &str,
454 497 return_url: &str,
455 498 ) -> Result<String> {
499 + self.faults.check("create_synckit_billing_portal")?;
456 500 Ok(format!(
457 501 "https://billing.stripe.test/portal?return={}",
458 502 urlencoding::encode(return_url)
M server/tests/workflows/mod.rs +1
@@ -50,6 +50,7 @@
50 50 mod embeds;
51 51 mod enum_drift;
52 52 mod exports;
53 + mod failure_paths;
53 54 mod fan_plus;
54 55 mod fingerprinting;
55 56 mod follows;
A server/tests/harness/faults.rs +230
@@ -1,0 +1,230 @@
1 + //! Injectable failure policy for the test mocks.
2 + //!
3 + //! The mocks were infallible: `InMemoryStorage` errored only on genuine domain
4 + //! conditions (object missing, bad part number), and the payment and email
5 + //! mocks had no error path at all. Meanwhile the server carries real failure
6 + //! machinery, `db/pending_refunds.rs`, `db/pending_uploads.rs`,
7 + //! `db/scan_jobs.rs` retry, `storage.rs` retry, and the scheduler's S3 deletion
8 + //! drain. None of it was reachable from a test, which reads as handled while
9 + //! being unobserved.
10 + //!
11 + //! A mock holds one `Faults` and calls `check()` at the top of each trait
12 + //! method. A test with no policy installed sees exactly the old behaviour, so
13 + //! this is additive and the existing suites are untouched.
14 + //!
15 + //! Method and rationale: wiki `testing-posture`, the "absent oracle" section.
16 +
17 + use makenotwork::error::{AppError, Result};
18 + use std::collections::HashMap;
19 + use std::sync::Mutex;
20 +
21 + /// When a rule fires. Call numbers are 1-based.
22 + #[derive(Debug, Clone, Copy)]
23 + enum Trigger {
24 + /// Every call. A backend that is simply down.
25 + Always,
26 + /// The Nth call only. Isolates one step of a multi-call flow, so a test can
27 + /// say which call failed rather than that something did.
28 + Nth(u32),
29 + /// Calls 1 through N-1, so the Nth call is the first to succeed. This is the
30 + /// shape the retry paths exist to serve: down, then recovered.
31 + Until(u32),
32 + }
33 +
34 + /// Builds the error to return. `AppError` is not `Clone` (it wraps `sqlx::Error`
35 + /// and `anyhow::Error`), so a rule stores a factory rather than a value.
36 + type ErrorFactory = Box<dyn Fn() -> AppError + Send + Sync>;
37 +
38 + struct Rule {
39 + trigger: Trigger,
40 + error: ErrorFactory,
41 + }
42 +
43 + /// Failure policy for one mock.
44 + ///
45 + /// Operations are named by the trait method they guard, so a rule reads as the
46 + /// call it breaks. An unregistered operation never fails.
47 + #[derive(Default)]
48 + pub(crate) struct Faults {
49 + rules: Mutex<HashMap<&'static str, Rule>>,
50 + calls: Mutex<HashMap<&'static str, u32>>,
51 + }
52 +
53 + #[allow(dead_code)]
54 + impl Faults {
55 + pub(crate) fn new() -> Self {
56 + Self::default()
57 + }
58 +
59 + /// Fail every call to `op`.
60 + pub(crate) fn fail_always(
61 + &self,
62 + op: &'static str,
63 + error: impl Fn() -> AppError + Send + Sync + 'static,
64 + ) {
65 + self.install(op, Trigger::Always, error);
66 + }
67 +
68 + /// Fail the `n`th call to `op` and no other.
69 + pub(crate) fn fail_nth(
70 + &self,
71 + op: &'static str,
72 + n: u32,
73 + error: impl Fn() -> AppError + Send + Sync + 'static,
74 + ) {
75 + self.install(op, Trigger::Nth(n), error);
76 + }
77 +
78 + /// Fail calls to `op` until the `n`th, which is the first to succeed.
79 + ///
80 + /// `fail_until(op, 3, ..)` fails calls 1 and 2. A retry path with two
81 + /// retries budgeted therefore succeeds, and one with fewer does not, which
82 + /// is the assertion worth making.
83 + pub(crate) fn fail_until(
84 + &self,
85 + op: &'static str,
86 + n: u32,
87 + error: impl Fn() -> AppError + Send + Sync + 'static,
88 + ) {
89 + self.install(op, Trigger::Until(n), error);
90 + }
91 +
92 + fn install(
93 + &self,
94 + op: &'static str,
95 + trigger: Trigger,
96 + error: impl Fn() -> AppError + Send + Sync + 'static,
97 + ) {
98 + self.rules.lock().unwrap().insert(
99 + op,
100 + Rule {
101 + trigger,
102 + error: Box::new(error),
103 + },
104 + );
105 + }
106 +
107 + /// Drop the rule for `op`, leaving its call count intact.
108 + pub(crate) fn clear(&self, op: &'static str) {
109 + self.rules.lock().unwrap().remove(op);
110 + }
111 +
112 + /// Drop every rule. Call counts survive, so a test can install a policy,
113 + /// clear it, and still assert how many times the failing call was made.
114 + pub(crate) fn clear_all(&self) {
115 + self.rules.lock().unwrap().clear();
116 + }
117 +
118 + /// How many times `op` has been called, whether it failed or not. This is
119 + /// how a test counts retries.
120 + pub(crate) fn calls(&self, op: &str) -> u32 {
121 + self.calls.lock().unwrap().get(op).copied().unwrap_or(0)
122 + }
123 +
124 + /// Reset every call count.
125 + pub(crate) fn reset_calls(&self) {
126 + self.calls.lock().unwrap().clear();
127 + }
128 +
129 + /// Record a call to `op` and return its configured error, if this call is
130 + /// one the policy fails. Mocks call this at the top of each trait method.
131 + pub(crate) fn check(&self, op: &'static str) -> Result<()> {
132 + let n = {
133 + let mut calls = self.calls.lock().unwrap();
134 + let entry = calls.entry(op).or_insert(0);
135 + *entry += 1;
136 + *entry
137 + };
138 +
139 + let rules = self.rules.lock().unwrap();
140 + let Some(rule) = rules.get(op) else {
141 + return Ok(());
142 + };
143 +
144 + let fires = match rule.trigger {
145 + Trigger::Always => true,
146 + Trigger::Nth(target) => n == target,
147 + Trigger::Until(target) => n < target,
148 + };
149 +
150 + if fires { Err((rule.error)()) } else { Ok(()) }
151 + }
152 + }
153 +
154 + /// The error a storage backend returns when the service is unreachable, as
155 + /// opposed to the object being absent. Retry paths key on this distinction.
156 + #[allow(dead_code)]
157 + pub(crate) fn storage_unavailable() -> AppError {
158 + AppError::Storage("test-injected: S3 unavailable".to_string())
159 + }
160 +
161 + /// The error the payment provider returns when Stripe is unreachable.
162 + #[allow(dead_code)]
163 + pub(crate) fn stripe_unavailable() -> AppError {
164 + AppError::ServiceUnavailable("test-injected: Stripe unavailable".to_string())
165 + }
166 +
167 + /// The error the email transport returns when the sending service rejects.
168 + #[allow(dead_code)]
169 + pub(crate) fn email_unavailable() -> AppError {
170 + AppError::ServiceUnavailable("test-injected: email transport unavailable".to_string())
171 + }
172 +
173 + #[cfg(test)]
174 + mod tests {
175 + use super::*;
176 +
177 + #[test]
178 + fn no_rule_never_fails() {
179 + let faults = Faults::new();
180 + assert!(faults.check("op").is_ok());
181 + assert!(faults.check("op").is_ok());
182 + assert_eq!(faults.calls("op"), 2);
183 + }
184 +
185 + #[test]
186 + fn always_fails_every_call() {
187 + let faults = Faults::new();
188 + faults.fail_always("op", storage_unavailable);
189 + assert!(faults.check("op").is_err());
190 + assert!(faults.check("op").is_err());
191 + }
192 +
193 + #[test]
194 + fn nth_fails_only_that_call() {
195 + let faults = Faults::new();
196 + faults.fail_nth("op", 2, storage_unavailable);
197 + assert!(faults.check("op").is_ok(), "call 1 succeeds");
198 + assert!(faults.check("op").is_err(), "call 2 fails");
199 + assert!(faults.check("op").is_ok(), "call 3 succeeds");
200 + }
201 +
202 + #[test]
203 + fn until_fails_up_to_that_call() {
204 + let faults = Faults::new();
205 + faults.fail_until("op", 3, storage_unavailable);
206 + assert!(faults.check("op").is_err(), "call 1 fails");
207 + assert!(faults.check("op").is_err(), "call 2 fails");
208 + assert!(faults.check("op").is_ok(), "call 3 is the first to succeed");
209 + assert!(faults.check("op").is_ok());
210 + }
211 +
212 + #[test]
213 + fn rules_are_scoped_to_one_operation() {
214 + let faults = Faults::new();
215 + faults.fail_always("broken", storage_unavailable);
216 + assert!(faults.check("broken").is_err());
217 + assert!(faults.check("fine").is_ok());
218 + }
219 +
220 + #[test]
221 + fn counts_survive_clearing_the_rule() {
222 + let faults = Faults::new();
223 + faults.fail_always("op", storage_unavailable);
224 + let _ = faults.check("op");
225 + let _ = faults.check("op");
226 + faults.clear_all();
227 + assert!(faults.check("op").is_ok());
228 + assert_eq!(faults.calls("op"), 3, "counts every call, failed or not");
229 + }
230 + }
A server/tests/workflows/failure_paths.rs +401
@@ -1,0 +1,401 @@
1 + //! Negative paths: what the server does when a dependency fails.
2 + //!
3 + //! These tests exist because the mocks used to be infallible, so the retry and
4 + //! compensation machinery the server carries had no test that could reach it.
5 + //! Retry logic no test can enter is worse than none, because it reads as
6 + //! handled. Each test here installs a failure policy on a mock (see
7 + //! `harness::faults`) and asserts the compensating behaviour, not just that the
8 + //! request failed.
9 + //!
10 + //! Rationale: wiki `testing-posture`, the "absent oracle" section.
11 +
12 + use crate::harness::TestHarness;
13 + use crate::harness::faults::{email_unavailable, storage_unavailable, stripe_unavailable};
14 + use makenotwork::db;
15 + use makenotwork::storage::StorageBackend;
16 + use serde_json::Value;
17 + use std::collections::HashMap;
18 +
19 + // The durable S3 deletion queue
20 +
21 + /// Count rows still queued for deletion of `key`.
22 + async fn queued_deletions(h: &TestHarness, key: &str) -> i64 {
23 + sqlx::query_scalar("SELECT COUNT(*) FROM pending_s3_deletions WHERE s3_key = $1")
24 + .bind(key)
25 + .fetch_one(&h.db)
26 + .await
27 + .unwrap()
28 + }
29 +
30 + /// A delete that fails must leave the row queued. Dequeuing it would orphan the
31 + /// S3 object with no durable record, which is the leak the queue exists to
32 + /// prevent.
33 + #[tokio::test]
34 + async fn s3_delete_failure_keeps_the_row_queued_for_retry() {
35 + let h = TestHarness::with_storage().await;
36 + let storage = h.storage.clone().expect("with_storage provides a backend");
37 + let key = "test/orphan-retry.bin";
38 +
39 + storage.put(key, b"payload".to_vec());
40 + db::pending_s3_deletions::enqueue_deletions(
41 + &h.db,
42 + &[(key.to_string(), "main".to_string())],
43 + "test_failure_path",
44 + )
45 + .await
46 + .unwrap();
47 + assert_eq!(queued_deletions(&h, key).await, 1, "row starts queued");
48 +
49 + storage
50 + .faults()
51 + .fail_always("delete_object", storage_unavailable);
52 + let deleted = h.drain_s3_deletions().await;
53 +
54 + assert_eq!(deleted, 0, "a failing backend deletes nothing");
55 + assert_eq!(
56 + queued_deletions(&h, key).await,
57 + 1,
58 + "the row must survive a failed delete, dropping it would orphan the object"
59 + );
60 + assert!(
61 + storage.object_exists(key).await.unwrap(),
62 + "the object is still there, which is why the row must be"
63 + );
64 + assert_eq!(
65 + storage.faults().calls("delete_object"),
66 + 1,
67 + "the drain attempted the delete exactly once"
68 + );
69 + }
70 +
71 + /// The point of keeping the row: a later drain finishes the job. This is the
72 + /// whole contract of the durable queue and nothing asserted it before.
73 + #[tokio::test]
74 + async fn s3_delete_queue_recovers_when_the_backend_comes_back() {
75 + let h = TestHarness::with_storage().await;
76 + let storage = h.storage.clone().expect("with_storage provides a backend");
77 + let key = "test/orphan-recovers.bin";
78 +
79 + storage.put(key, b"payload".to_vec());
80 + db::pending_s3_deletions::enqueue_deletions(
81 + &h.db,
82 + &[(key.to_string(), "main".to_string())],
83 + "test_failure_path",
84 + )
85 + .await
86 + .unwrap();
87 +
88 + // Down for the first attempt, up for the second.
89 + storage
90 + .faults()
91 + .fail_until("delete_object", 2, storage_unavailable);
92 +
93 + assert_eq!(h.drain_s3_deletions().await, 0, "first drain fails");
94 + assert_eq!(queued_deletions(&h, key).await, 1, "still queued");
95 +
96 + assert_eq!(h.drain_s3_deletions().await, 1, "second drain succeeds");
97 + assert_eq!(
98 + queued_deletions(&h, key).await,
99 + 0,
100 + "a completed delete is dequeued"
101 + );
102 + assert!(
103 + !storage.object_exists(key).await.unwrap(),
104 + "the object is gone"
105 + );
106 + }
107 +
108 + // Checkout compensation when Stripe is down
109 +
110 + /// Create a creator with Stripe connected and a published paid item, logged in
111 + /// as the creator afterwards. Mirrors the helper in `promo_codes_checkout`.
112 + async fn setup_paid_item(h: &mut TestHarness, price_cents: i32) -> (db::UserId, String) {
113 + let seller_id = h.signup("fpseller", "fpseller@test.com", "pass1234").await;
114 + h.grant_creator(seller_id).await;
115 +
116 + sqlx::query("UPDATE users SET stripe_account_id = 'acct_mock_fpseller', stripe_charges_enabled = true WHERE id = $1")
117 + .bind(seller_id)
118 + .execute(&h.db)
119 + .await
120 + .unwrap();
121 +
122 + h.client.post_form("/logout", "").await;
123 + h.login("fpseller", "pass1234").await;
124 +
125 + let resp = h
126 + .client
127 + .post_form("/api/projects", "slug=fpshop&title=FP+Shop")
128 + .await;
129 + let project: Value = resp.json();
130 + let project_id = project["id"].as_str().unwrap().to_string();
131 +
132 + let resp = h
133 + .client
134 + .post_form(
135 + &format!("/api/projects/{project_id}/items"),
136 + &format!("title=FP+Track&price_cents={price_cents}&item_type=audio"),
137 + )
138 + .await;
139 + let item: Value = resp.json();
140 + let item_id = item["id"].as_str().unwrap().to_string();
141 +
142 + h.client
143 + .put_form(&format!("/api/projects/{project_id}"), "is_public=true")
144 + .await;
145 + h.client
146 + .put_form(&format!("/api/items/{item_id}"), "is_public=true")
147 + .await;
148 +
149 + (seller_id, item_id)
150 + }
151 +
152 + /// A promo code is reserved (its `use_count` incremented) before the Stripe
153 + /// call, so a Stripe failure has to release it. Without that, every outage
154 + /// burns uses off a creator's code and the last buyers are told it is exhausted
155 + /// when it never was. `routes/stripe/checkout/item.rs` compensates for this and
156 + /// no test could reach the branch.
157 + #[tokio::test]
158 + async fn stripe_failure_at_checkout_releases_the_promo_reservation() {
159 + let mut h = TestHarness::with_mocks().await;
160 + let (seller_id, item_id) = setup_paid_item(&mut h, 1000).await;
161 +
162 + sqlx::query(
163 + "INSERT INTO promo_codes (creator_id, code, code_purpose, discount_type, discount_value, min_price_cents, max_uses) \
164 + VALUES ($1, 'OUTAGE', 'discount', 'percentage', 25, 0, 5)",
165 + )
166 + .bind(seller_id)
167 + .execute(&h.db)
168 + .await
169 + .unwrap();
170 +
171 + h.client.post_form("/logout", "").await;
172 + let _buyer_id = h.signup("fpbuyer", "fpbuyer@test.com", "pass1234").await;
173 +
174 + h.mock_stripe
175 + .as_ref()
176 + .expect("with_mocks provides a payment provider")
177 + .faults()
178 + .fail_always("create_checkout_session", stripe_unavailable);
179 +
180 + let resp = h
181 + .client
182 + .post_form(
183 + &format!("/stripe/checkout/{item_id}"),
184 + "share_contact=false&promo_code=OUTAGE",
185 + )
186 + .await;
187 + assert_eq!(
188 + resp.status.as_u16(),
189 + 500,
190 + "a Stripe outage is not the buyer's fault, got {}",
191 + resp.status
192 + );
193 +
194 + let use_count: i32 = sqlx::query_scalar(
195 + "SELECT use_count FROM promo_codes WHERE creator_id = $1 AND upper(code) = 'OUTAGE'",
196 + )
197 + .bind(seller_id)
198 + .fetch_one(&h.db)
199 + .await
200 + .unwrap();
201 + assert_eq!(
202 + use_count, 0,
203 + "the reservation must be released when Stripe fails, or an outage burns the code"
204 + );
205 +
206 + let pending: i64 = sqlx::query_scalar(
207 + "SELECT COUNT(*) FROM transactions WHERE item_id = $1::uuid AND status = 'pending'",
208 + )
209 + .bind(&item_id)
210 + .fetch_one(&h.db)
211 + .await
212 + .unwrap();
213 + assert_eq!(
214 + pending, 0,
215 + "no session means no transaction, a pending row here would block the buyer's retry"
216 + );
217 + }
218 +
219 + /// The recovery half: once Stripe is back, the same buyer and the same code go
220 + /// through. This is what proves the release above actually restored the code
221 + /// rather than merely decrementing a counter.
222 + #[tokio::test]
223 + async fn checkout_succeeds_on_retry_after_a_stripe_outage() {
224 + let mut h = TestHarness::with_mocks().await;
225 + let (seller_id, item_id) = setup_paid_item(&mut h, 1000).await;
226 +
227 + sqlx::query(
228 + "INSERT INTO promo_codes (creator_id, code, code_purpose, discount_type, discount_value, min_price_cents, max_uses) \
229 + VALUES ($1, 'RETRY', 'discount', 'percentage', 25, 0, 1)",
230 + )
231 + .bind(seller_id)
232 + .execute(&h.db)
233 + .await
234 + .unwrap();
235 +
236 + h.client.post_form("/logout", "").await;
237 + let _buyer_id = h.signup("fpbuyer2", "fpbuyer2@test.com", "pass1234").await;
238 +
239 + let stripe = h
240 + .mock_stripe
241 + .clone()
242 + .expect("with_mocks provides a payment provider");
243 +
244 + // max_uses is 1, so a burned reservation makes the retry below impossible.
245 + stripe
246 + .faults()
247 + .fail_until("create_checkout_session", 2, stripe_unavailable);
248 +
249 + let failed = h
250 + .client
251 + .post_form(
252 + &format!("/stripe/checkout/{item_id}"),
253 + "share_contact=false&promo_code=RETRY",
254 + )
255 + .await;
256 + assert_eq!(failed.status.as_u16(), 500, "first attempt fails");
257 +
258 + let resp = h
259 + .client
260 + .post_form(
261 + &format!("/stripe/checkout/{item_id}"),
262 + "share_contact=false&promo_code=RETRY",
263 + )
264 + .await;
265 + assert_eq!(
266 + resp.status.as_u16(),
267 + 303,
268 + "second attempt redirects to the Stripe session, got {} {}",
269 + resp.status,
270 + resp.text
271 + );
272 +
273 + let amount: i32 = sqlx::query_scalar(
274 + "SELECT amount_cents FROM transactions WHERE item_id = $1::uuid AND status = 'pending'",
275 + )
276 + .bind(&item_id)
277 + .fetch_one(&h.db)
278 + .await
279 + .unwrap();
280 + assert_eq!(amount, 750, "the discount still applied on the retry");
281 +
282 + assert_eq!(
283 + stripe.faults().calls("create_checkout_session"),
284 + 2,
285 + "exactly two Stripe attempts, the route does not retry internally"
286 + );
287 + }
288 +
289 + // Email is best-effort, and has to actually be best-effort
290 +
291 + /// A failed buyer receipt must not swallow the seller's sale notification. The
292 + /// two sends are guarded separately in `checkout_helpers.rs` precisely so one
293 + /// bad address or one transport hiccup does not take out the other, and with an
294 + /// infallible transport nothing checked that they really are independent.
295 + ///
296 + /// The purchase itself is settled before either email is queued, so this also
297 + /// asserts the money outcome is untouched by an email outage.
298 + #[tokio::test]
299 + async fn a_failed_buyer_receipt_still_notifies_the_seller() {
300 + let mut h = TestHarness::with_mocks().await;
301 + let (seller_id, item_id) = setup_paid_item(&mut h, 500).await;
302 +
303 + sqlx::query("UPDATE users SET notify_sale = true WHERE id = $1")
304 + .bind(seller_id)
305 + .execute(&h.db)
306 + .await
307 + .unwrap();
308 +
309 + h.client.post_form("/logout", "").await;
310 + let buyer_id = h.signup("fpbuyer3", "fpbuyer3@test.com", "pass1234").await;
311 +
312 + let session_id = "cs_failure_path_email";
313 + sqlx::query(
314 + r"INSERT INTO transactions
315 + (buyer_id, seller_id, item_id, amount_cents, status,
316 + stripe_checkout_session_id, item_title, seller_username)
317 + VALUES ($1, $2, $3::uuid, 500, 'pending', $4, 'FP Track', 'fpseller')",
318 + )
319 + .bind(buyer_id)
320 + .bind(seller_id)
321 + .bind(&item_id)
322 + .bind(session_id)
323 + .execute(&h.db)
324 + .await
325 + .unwrap();
326 +
327 + // Signup already sent this buyer mail, so clear the log and the call counts
328 + // before arming the policy. After this the only sends are the webhook's two,
329 + // and the buyer receipt is call 1.
330 + let email = h
331 + .mock_email
332 + .clone()
333 + .expect("with_mocks provides an email transport");
334 + email.clear();
335 + email.faults().reset_calls();
336 + email.faults().fail_nth("send_email", 1, email_unavailable);
337 +
338 + let mut meta = HashMap::new();
339 + meta.insert("buyer_id".to_string(), buyer_id.to_string());
340 + meta.insert("seller_id".to_string(), seller_id.to_string());
341 + meta.insert("item_id".to_string(), item_id.clone());
342 + let session = serde_json::json!({
343 + "id": session_id,
344 + "object": "checkout_session",
345 + "mode": "payment",
346 + "metadata": meta,
347 + "payment_intent": "pi_failure_path_email",
348 + });
349 +
350 + let payload = serde_json::json!({
351 + "id": "evt_failure_path_email",
352 + "type": "checkout.session.completed",
353 + "data": { "object": session },
354 + })
355 + .to_string();
356 + let signature = crate::harness::stripe::sign_webhook_payload(
357 + &payload,
358 + crate::harness::stripe::TEST_WEBHOOK_SECRET,
359 + );
360 + let resp = h
361 + .client
362 + .request_with_headers(
363 + "POST",
364 + "/stripe/webhook",
365 + Some(&payload),
366 + &[
367 + ("stripe-signature", &signature),
368 + ("content-type", "application/json"),
369 + ],
370 + )
371 + .await;
372 + assert_eq!(resp.status.as_u16(), 200, "webhook accepted: {}", resp.text);
373 +
374 + // Fire-and-forget email tasks, same wait the sibling email test uses.
375 + tokio::time::sleep(std::time::Duration::from_millis(200)).await;
376 +
377 + assert_eq!(
378 + email.sent_to("fpbuyer3@test.com").len(),
379 + 0,
380 + "the buyer receipt was the injected failure"
381 + );
382 + let seller_emails = email.sent_to("fpseller@test.com");
383 + assert!(
384 + seller_emails
385 + .iter()
386 + .any(|e| e.subject.to_lowercase().contains("sale")),
387 + "the sale notification must still go out, got: {:?}",
388 + seller_emails.iter().map(|e| &e.subject).collect::<Vec<_>>()
389 + );
390 +
391 + let status: String =
392 + sqlx::query_scalar("SELECT status FROM transactions WHERE stripe_checkout_session_id = $1")
393 + .bind(session_id)
394 + .fetch_one(&h.db)
395 + .await
396 + .unwrap();
397 + assert_eq!(
398 + status, "completed",
399 + "an email outage must not touch the purchase"
400 + );
401 + }