//! Webhook retry with exponential backoff and stale refund escalation. use crate::db; use crate::AppState; /// Maximum webhook retry attempts before marking as dead letter. const WEBHOOK_MAX_RETRIES: i32 = 5; /// Determine whether a webhook retry attempt should be treated as a dead letter. pub(super) fn is_webhook_dead(attempt: i32) -> bool { attempt >= WEBHOOK_MAX_RETRIES } /// Retry failed webhook events with exponential backoff. pub(super) async fn retry_failed_webhooks(state: &AppState) { let events = match db::webhook_events::get_retryable_events(&state.db).await { Ok(e) if e.is_empty() => return, Ok(e) => e, Err(e) => { tracing::error!(error = ?e, "failed to fetch retryable webhook events"); return; } }; if state.stripe.is_none() { return; } for event in events { let attempt = event.attempts + 1; tracing::info!( event_id = %event.id, source = %event.source, event_type = %event.event_type, attempt = attempt, "retrying webhook event" ); // Retry re-runs the full event handler. All handlers must be idempotent // (use ON CONFLICT / WHERE status='pending' guards) since steps completed // before the original failure are not rolled back. let result = if event.source == "stripe" { match crate::payments::UntypedEvent::from_payload(&event.payload) { Ok(parsed) => { let crate::payments::UntypedEvent { id, type_, data_object } = parsed; crate::routes::stripe::process_webhook_event(state, &type_, &id, data_object).await } Err(e) => Err(e), } } else { Err(crate::error::AppError::BadRequest(format!("Unknown webhook source: {}", event.source))) }; match result { Ok(()) => { tracing::info!(event_id = %event.id, "webhook retry succeeded"); if let Err(e) = db::webhook_events::mark_processed(&state.db, event.id).await { tracing::error!(error = ?e, "failed to mark webhook event as processed"); } } Err(e) => { let is_dead = is_webhook_dead(attempt); tracing::warn!( event_id = %event.id, attempt = attempt, error = ?e, dead = is_dead, "webhook retry failed" ); if let Err(e) = db::webhook_events::schedule_retry( &state.db, event.id, attempt, &format!("{:?}", e), ).await { tracing::error!(error = ?e, "failed to schedule webhook retry"); } if is_dead && let Some(ref wam) = state.wam { let title = format!( "Dead webhook: {} ({})", event.event_type, event.id ); let body = format!( "Webhook event exhausted all {} retry attempts.\n\ Source: {}\nType: {}\nLast error: {:?}", attempt, event.source, event.event_type, e, ); wam.create_ticket( &title, Some(&body), "high", "webhook-dead-letter", Some(&event.id.to_string()), ) .await; } } } } } /// Alert the admin about pending refunds that have gone unmatched for >24 hours. pub(super) async fn escalate_stale_refunds(state: &AppState) { let stale = match db::pending_refunds::get_stale_refunds( &state.db, chrono::Duration::hours(24), ) .await { Ok(s) if s.is_empty() => return, Ok(s) => s, Err(e) => { tracing::error!(error = ?e, "failed to query stale pending refunds"); return; } }; let alert_email = std::env::var("ALERT_EMAIL").ok(); for refund in &stale { // Mark escalated FIRST to prevent duplicate alerts on retry if let Err(e) = db::pending_refunds::mark_escalated(&state.db, refund.id).await { tracing::error!(error = ?e, "failed to mark pending refund as escalated, skipping alerts"); continue; } tracing::error!( payment_intent_id = %refund.payment_intent_id, amount = refund.amount.as_i64(), amount_refunded = refund.amount_refunded.as_i64(), created_at = %refund.created_at, "STALE PENDING REFUND: unmatched for >24h, needs manual investigation" ); if let Some(ref to) = alert_email { let subject = format!( "Unmatched refund: {} ({}c refunded)", refund.payment_intent_id, refund.amount_refunded ); let body = format!( "A charge.refunded webhook for payment intent {} has been pending for >24 hours \ with no matching completed transaction.\n\n\ Amount: {}c\nAmount refunded: {}c\nReceived: {}\n\n\ This likely means the checkout.session.completed webhook was lost. \ Check the Stripe dashboard and reconcile manually.", refund.payment_intent_id, refund.amount, refund.amount_refunded, refund.created_at, ); if let Err(e) = state.email.send_alert(to, &subject, &body).await { tracing::error!(error = ?e, "failed to send stale refund alert email"); } } if let Some(ref wam) = state.wam { let title = format!( "Unmatched refund: {} ({}c)", refund.payment_intent_id, refund.amount_refunded ); let body = format!( "charge.refunded webhook pending >24h with no matching completed transaction.\n\ Amount: {}c\nRefunded: {}c\nReceived: {}\n\ Check Stripe dashboard and reconcile manually.", refund.amount, refund.amount_refunded, refund.created_at, ); wam.create_ticket( &title, Some(&body), "critical", "refund-escalation", Some(&refund.payment_intent_id), ) .await; } } } #[cfg(test)] mod tests { use super::*; #[test] fn webhook_not_dead_under_threshold() { assert!(!is_webhook_dead(1)); assert!(!is_webhook_dead(4)); } #[test] fn webhook_dead_at_threshold() { assert!(is_webhook_dead(5)); } #[test] fn webhook_dead_above_threshold() { assert!(is_webhook_dead(6)); assert!(is_webhook_dead(100)); } }