Skip to main content

max / makenotwork

7.4 KB · 231 lines History Blame Raw
1 //! Injectable failure policy for the test mocks.
2 //!
3 //! The mocks were infallible: `InMemoryStorage` errored only on genuine domain
4 //! conditions (object missing, bad part number), and the payment and email
5 //! mocks had no error path at all. Meanwhile the server carries real failure
6 //! machinery, `db/pending_refunds.rs`, `db/pending_uploads.rs`,
7 //! `db/scan_jobs.rs` retry, `storage.rs` retry, and the scheduler's S3 deletion
8 //! drain. None of it was reachable from a test, which reads as handled while
9 //! being unobserved.
10 //!
11 //! A mock holds one `Faults` and calls `check()` at the top of each trait
12 //! method. A test with no policy installed sees exactly the old behaviour, so
13 //! this is additive and the existing suites are untouched.
14 //!
15 //! Method and rationale: wiki `testing-posture`, the "absent oracle" section.
16
17 use makenotwork::error::{AppError, Result};
18 use std::collections::HashMap;
19 use std::sync::Mutex;
20
21 /// When a rule fires. Call numbers are 1-based.
22 #[derive(Debug, Clone, Copy)]
23 enum Trigger {
24 /// Every call. A backend that is simply down.
25 Always,
26 /// The Nth call only. Isolates one step of a multi-call flow, so a test can
27 /// say which call failed rather than that something did.
28 Nth(u32),
29 /// Calls 1 through N-1, so the Nth call is the first to succeed. This is the
30 /// shape the retry paths exist to serve: down, then recovered.
31 Until(u32),
32 }
33
34 /// Builds the error to return. `AppError` is not `Clone` (it wraps `sqlx::Error`
35 /// and `anyhow::Error`), so a rule stores a factory rather than a value.
36 type ErrorFactory = Box<dyn Fn() -> AppError + Send + Sync>;
37
38 struct Rule {
39 trigger: Trigger,
40 error: ErrorFactory,
41 }
42
43 /// Failure policy for one mock.
44 ///
45 /// Operations are named by the trait method they guard, so a rule reads as the
46 /// call it breaks. An unregistered operation never fails.
47 #[derive(Default)]
48 pub(crate) struct Faults {
49 rules: Mutex<HashMap<&'static str, Rule>>,
50 calls: Mutex<HashMap<&'static str, u32>>,
51 }
52
53 #[allow(dead_code)]
54 impl Faults {
55 pub(crate) fn new() -> Self {
56 Self::default()
57 }
58
59 /// Fail every call to `op`.
60 pub(crate) fn fail_always(
61 &self,
62 op: &'static str,
63 error: impl Fn() -> AppError + Send + Sync + 'static,
64 ) {
65 self.install(op, Trigger::Always, error);
66 }
67
68 /// Fail the `n`th call to `op` and no other.
69 pub(crate) fn fail_nth(
70 &self,
71 op: &'static str,
72 n: u32,
73 error: impl Fn() -> AppError + Send + Sync + 'static,
74 ) {
75 self.install(op, Trigger::Nth(n), error);
76 }
77
78 /// Fail calls to `op` until the `n`th, which is the first to succeed.
79 ///
80 /// `fail_until(op, 3, ..)` fails calls 1 and 2. A retry path with two
81 /// retries budgeted therefore succeeds, and one with fewer does not, which
82 /// is the assertion worth making.
83 pub(crate) fn fail_until(
84 &self,
85 op: &'static str,
86 n: u32,
87 error: impl Fn() -> AppError + Send + Sync + 'static,
88 ) {
89 self.install(op, Trigger::Until(n), error);
90 }
91
92 fn install(
93 &self,
94 op: &'static str,
95 trigger: Trigger,
96 error: impl Fn() -> AppError + Send + Sync + 'static,
97 ) {
98 self.rules.lock().unwrap().insert(
99 op,
100 Rule {
101 trigger,
102 error: Box::new(error),
103 },
104 );
105 }
106
107 /// Drop the rule for `op`, leaving its call count intact.
108 pub(crate) fn clear(&self, op: &'static str) {
109 self.rules.lock().unwrap().remove(op);
110 }
111
112 /// Drop every rule. Call counts survive, so a test can install a policy,
113 /// clear it, and still assert how many times the failing call was made.
114 pub(crate) fn clear_all(&self) {
115 self.rules.lock().unwrap().clear();
116 }
117
118 /// How many times `op` has been called, whether it failed or not. This is
119 /// how a test counts retries.
120 pub(crate) fn calls(&self, op: &str) -> u32 {
121 self.calls.lock().unwrap().get(op).copied().unwrap_or(0)
122 }
123
124 /// Reset every call count.
125 pub(crate) fn reset_calls(&self) {
126 self.calls.lock().unwrap().clear();
127 }
128
129 /// Record a call to `op` and return its configured error, if this call is
130 /// one the policy fails. Mocks call this at the top of each trait method.
131 pub(crate) fn check(&self, op: &'static str) -> Result<()> {
132 let n = {
133 let mut calls = self.calls.lock().unwrap();
134 let entry = calls.entry(op).or_insert(0);
135 *entry += 1;
136 *entry
137 };
138
139 let rules = self.rules.lock().unwrap();
140 let Some(rule) = rules.get(op) else {
141 return Ok(());
142 };
143
144 let fires = match rule.trigger {
145 Trigger::Always => true,
146 Trigger::Nth(target) => n == target,
147 Trigger::Until(target) => n < target,
148 };
149
150 if fires { Err((rule.error)()) } else { Ok(()) }
151 }
152 }
153
154 /// The error a storage backend returns when the service is unreachable, as
155 /// opposed to the object being absent. Retry paths key on this distinction.
156 #[allow(dead_code)]
157 pub(crate) fn storage_unavailable() -> AppError {
158 AppError::Storage("test-injected: S3 unavailable".to_string())
159 }
160
161 /// The error the payment provider returns when Stripe is unreachable.
162 #[allow(dead_code)]
163 pub(crate) fn stripe_unavailable() -> AppError {
164 AppError::ServiceUnavailable("test-injected: Stripe unavailable".to_string())
165 }
166
167 /// The error the email transport returns when the sending service rejects.
168 #[allow(dead_code)]
169 pub(crate) fn email_unavailable() -> AppError {
170 AppError::ServiceUnavailable("test-injected: email transport unavailable".to_string())
171 }
172
173 #[cfg(test)]
174 mod tests {
175 use super::*;
176
177 #[test]
178 fn no_rule_never_fails() {
179 let faults = Faults::new();
180 assert!(faults.check("op").is_ok());
181 assert!(faults.check("op").is_ok());
182 assert_eq!(faults.calls("op"), 2);
183 }
184
185 #[test]
186 fn always_fails_every_call() {
187 let faults = Faults::new();
188 faults.fail_always("op", storage_unavailable);
189 assert!(faults.check("op").is_err());
190 assert!(faults.check("op").is_err());
191 }
192
193 #[test]
194 fn nth_fails_only_that_call() {
195 let faults = Faults::new();
196 faults.fail_nth("op", 2, storage_unavailable);
197 assert!(faults.check("op").is_ok(), "call 1 succeeds");
198 assert!(faults.check("op").is_err(), "call 2 fails");
199 assert!(faults.check("op").is_ok(), "call 3 succeeds");
200 }
201
202 #[test]
203 fn until_fails_up_to_that_call() {
204 let faults = Faults::new();
205 faults.fail_until("op", 3, storage_unavailable);
206 assert!(faults.check("op").is_err(), "call 1 fails");
207 assert!(faults.check("op").is_err(), "call 2 fails");
208 assert!(faults.check("op").is_ok(), "call 3 is the first to succeed");
209 assert!(faults.check("op").is_ok());
210 }
211
212 #[test]
213 fn rules_are_scoped_to_one_operation() {
214 let faults = Faults::new();
215 faults.fail_always("broken", storage_unavailable);
216 assert!(faults.check("broken").is_err());
217 assert!(faults.check("fine").is_ok());
218 }
219
220 #[test]
221 fn counts_survive_clearing_the_rule() {
222 let faults = Faults::new();
223 faults.fail_always("op", storage_unavailable);
224 let _ = faults.check("op");
225 let _ = faults.check("op");
226 faults.clear_all();
227 assert!(faults.check("op").is_ok());
228 assert_eq!(faults.calls("op"), 3, "counts every call, failed or not");
229 }
230 }
231