//! Injectable failure policy for the test mocks. //! //! The mocks were infallible: `InMemoryStorage` errored only on genuine domain //! conditions (object missing, bad part number), and the payment and email //! mocks had no error path at all. Meanwhile the server carries real failure //! machinery, `db/pending_refunds.rs`, `db/pending_uploads.rs`, //! `db/scan_jobs.rs` retry, `storage.rs` retry, and the scheduler's S3 deletion //! drain. None of it was reachable from a test, which reads as handled while //! being unobserved. //! //! A mock holds one `Faults` and calls `check()` at the top of each trait //! method. A test with no policy installed sees exactly the old behaviour, so //! this is additive and the existing suites are untouched. //! //! Method and rationale: wiki `testing-posture`, the "absent oracle" section. use makenotwork::error::{AppError, Result}; use std::collections::HashMap; use std::sync::Mutex; /// When a rule fires. Call numbers are 1-based. #[derive(Debug, Clone, Copy)] enum Trigger { /// Every call. A backend that is simply down. Always, /// The Nth call only. Isolates one step of a multi-call flow, so a test can /// say which call failed rather than that something did. Nth(u32), /// Calls 1 through N-1, so the Nth call is the first to succeed. This is the /// shape the retry paths exist to serve: down, then recovered. Until(u32), } /// Builds the error to return. `AppError` is not `Clone` (it wraps `sqlx::Error` /// and `anyhow::Error`), so a rule stores a factory rather than a value. type ErrorFactory = Box AppError + Send + Sync>; struct Rule { trigger: Trigger, error: ErrorFactory, } /// Failure policy for one mock. /// /// Operations are named by the trait method they guard, so a rule reads as the /// call it breaks. An unregistered operation never fails. #[derive(Default)] pub(crate) struct Faults { rules: Mutex>, calls: Mutex>, } #[allow(dead_code)] impl Faults { pub(crate) fn new() -> Self { Self::default() } /// Fail every call to `op`. pub(crate) fn fail_always( &self, op: &'static str, error: impl Fn() -> AppError + Send + Sync + 'static, ) { self.install(op, Trigger::Always, error); } /// Fail the `n`th call to `op` and no other. pub(crate) fn fail_nth( &self, op: &'static str, n: u32, error: impl Fn() -> AppError + Send + Sync + 'static, ) { self.install(op, Trigger::Nth(n), error); } /// Fail calls to `op` until the `n`th, which is the first to succeed. /// /// `fail_until(op, 3, ..)` fails calls 1 and 2. A retry path with two /// retries budgeted therefore succeeds, and one with fewer does not, which /// is the assertion worth making. pub(crate) fn fail_until( &self, op: &'static str, n: u32, error: impl Fn() -> AppError + Send + Sync + 'static, ) { self.install(op, Trigger::Until(n), error); } fn install( &self, op: &'static str, trigger: Trigger, error: impl Fn() -> AppError + Send + Sync + 'static, ) { self.rules.lock().unwrap().insert( op, Rule { trigger, error: Box::new(error), }, ); } /// Drop the rule for `op`, leaving its call count intact. pub(crate) fn clear(&self, op: &'static str) { self.rules.lock().unwrap().remove(op); } /// Drop every rule. Call counts survive, so a test can install a policy, /// clear it, and still assert how many times the failing call was made. pub(crate) fn clear_all(&self) { self.rules.lock().unwrap().clear(); } /// How many times `op` has been called, whether it failed or not. This is /// how a test counts retries. pub(crate) fn calls(&self, op: &str) -> u32 { self.calls.lock().unwrap().get(op).copied().unwrap_or(0) } /// Reset every call count. pub(crate) fn reset_calls(&self) { self.calls.lock().unwrap().clear(); } /// Record a call to `op` and return its configured error, if this call is /// one the policy fails. Mocks call this at the top of each trait method. pub(crate) fn check(&self, op: &'static str) -> Result<()> { let n = { let mut calls = self.calls.lock().unwrap(); let entry = calls.entry(op).or_insert(0); *entry += 1; *entry }; let rules = self.rules.lock().unwrap(); let Some(rule) = rules.get(op) else { return Ok(()); }; let fires = match rule.trigger { Trigger::Always => true, Trigger::Nth(target) => n == target, Trigger::Until(target) => n < target, }; if fires { Err((rule.error)()) } else { Ok(()) } } } /// The error a storage backend returns when the service is unreachable, as /// opposed to the object being absent. Retry paths key on this distinction. #[allow(dead_code)] pub(crate) fn storage_unavailable() -> AppError { AppError::Storage("test-injected: S3 unavailable".to_string()) } /// The error the payment provider returns when Stripe is unreachable. #[allow(dead_code)] pub(crate) fn stripe_unavailable() -> AppError { AppError::ServiceUnavailable("test-injected: Stripe unavailable".to_string()) } /// The error the email transport returns when the sending service rejects. #[allow(dead_code)] pub(crate) fn email_unavailable() -> AppError { AppError::ServiceUnavailable("test-injected: email transport unavailable".to_string()) } #[cfg(test)] mod tests { use super::*; #[test] fn no_rule_never_fails() { let faults = Faults::new(); assert!(faults.check("op").is_ok()); assert!(faults.check("op").is_ok()); assert_eq!(faults.calls("op"), 2); } #[test] fn always_fails_every_call() { let faults = Faults::new(); faults.fail_always("op", storage_unavailable); assert!(faults.check("op").is_err()); assert!(faults.check("op").is_err()); } #[test] fn nth_fails_only_that_call() { let faults = Faults::new(); faults.fail_nth("op", 2, storage_unavailable); assert!(faults.check("op").is_ok(), "call 1 succeeds"); assert!(faults.check("op").is_err(), "call 2 fails"); assert!(faults.check("op").is_ok(), "call 3 succeeds"); } #[test] fn until_fails_up_to_that_call() { let faults = Faults::new(); faults.fail_until("op", 3, storage_unavailable); assert!(faults.check("op").is_err(), "call 1 fails"); assert!(faults.check("op").is_err(), "call 2 fails"); assert!(faults.check("op").is_ok(), "call 3 is the first to succeed"); assert!(faults.check("op").is_ok()); } #[test] fn rules_are_scoped_to_one_operation() { let faults = Faults::new(); faults.fail_always("broken", storage_unavailable); assert!(faults.check("broken").is_err()); assert!(faults.check("fine").is_ok()); } #[test] fn counts_survive_clearing_the_rule() { let faults = Faults::new(); faults.fail_always("op", storage_unavailable); let _ = faults.check("op"); let _ = faults.check("op"); faults.clear_all(); assert!(faults.check("op").is_ok()); assert_eq!(faults.calls("op"), 3, "counts every call, failed or not"); } }