Skip to main content

max / makenotwork

Add WAM (Whack-a-Mole) ticket manager with 25 proactive ticket sources New internal tool: ratatui TUI + HTTP API for operational ticket management. SQLite-backed, tailnet-secured (no auth layer needed). WAM features (v0.1.0): - CLI: create, list, show, resolve, close (git-style prefix matching) - TUI: priority-colored list, detail view, inline status changes, search, filter by status/priority/source, create popup - HTTP API: POST/GET/PATCH /tickets (axum, for programmatic integration) MNW server integration (13 ticket sources): - User-facing: license key gen failed, Fan+ credit failed, file quarantined, subscription payment failed, Stripe Connect degraded, build failed - Infrastructure: health status change, DB pool pressure - Periodic: sales count drift, stale subscriptions, email bounce spike - Existing: refund escalation, webhook dead letter PoM integration (12 ticket sources): - All non-recovery alerts migrated from email to WAM tickets - Health, TLS, peer, route, DNS, WHOIS, CORS, latency, backup, monitoring - Recovery alerts remain email-only Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Author: Max J. <87768334+MaxJMath@users.noreply.github.com> · 2026-04-25 20:07 UTC
Commit: 5b85d927947d9438adc614e3ff817554d2798aeb
Parent: 1be62a4
23 files changed, +2339 insertions, -15 deletions
M pom/src/alerts.rs +60 -14
@@ -16,6 +16,7 @@ pub struct Alerter {
16 16 client: reqwest::Client,
17 17 pool: SqlitePool,
18 18 instance_name: String,
19 + wam_url: Option<String>,
19 20 }
20 21
21 22 impl Alerter {
@@ -24,7 +25,8 @@ impl Alerter {
24 25 .timeout(std::time::Duration::from_secs(10))
25 26 .build()
26 27 .unwrap_or_default();
27 - Self { config, client, pool, instance_name }
28 + let wam_url = config.wam_url.clone();
29 + Self { config, client, pool, instance_name, wam_url }
28 30 }
29 31
30 32 #[instrument(skip_all)]
@@ -56,7 +58,12 @@ impl Alerter {
56 58 }
57 59 body.push_str("\n- PoM");
58 60
59 - self.send_email(&subject, &body).await;
61 + let priority = match to_status {
62 + "error" | "unreachable" => "critical",
63 + "degraded" => "high",
64 + _ => "medium",
65 + };
66 + self.wam_ticket(&subject, &body, priority, "pom-health", Some(target)).await;
60 67 self.record_alert(&alert_key, AlertCategory::Health, Some(from_status), Some(to_status), error).await;
61 68 }
62 69
@@ -111,7 +118,8 @@ impl Alerter {
111 118 chrono::Utc::now().to_rfc3339(),
112 119 );
113 120
114 - self.send_email(&subject, &body).await;
121 + let priority = if days_remaining <= 3 { "critical" } else if days_remaining <= 7 { "high" } else { "medium" };
122 + self.wam_ticket(&subject, &body, priority, "pom-tls", Some(&format!("{target}:{host}"))).await;
115 123 self.record_alert(&alert_key, AlertCategory::TlsExpiry, None, None, None).await;
116 124 }
117 125
@@ -140,7 +148,7 @@ impl Alerter {
140 148 chrono::Utc::now().to_rfc3339(),
141 149 );
142 150
143 - self.send_email(&subject, &body).await;
151 + self.wam_ticket(&subject, &body, "high", "pom-tls", Some(&format!("{target}:{host}"))).await;
144 152 self.record_alert(&alert_key, AlertCategory::TlsError, None, None, Some(error)).await;
145 153 }
146 154
@@ -193,7 +201,7 @@ impl Alerter {
193 201 chrono::Utc::now().to_rfc3339(),
194 202 );
195 203
196 - self.send_email(&subject, &body).await;
204 + self.wam_ticket(&subject, &body, "high", "pom-peer", Some(peer_name)).await;
197 205 self.record_alert(&alert_key, AlertCategory::PeerMissing, None, None, None).await;
198 206 }
199 207
@@ -245,7 +253,7 @@ impl Alerter {
245 253 chrono::Utc::now().to_rfc3339(),
246 254 );
247 255
248 - self.send_email(&subject, &body).await;
256 + self.wam_ticket(&subject, &body, "high", "pom-routes", Some(target)).await;
249 257 self.record_alert(&alert_key, AlertCategory::RouteFailure, None, None, None).await;
250 258 }
251 259
@@ -313,7 +321,7 @@ impl Alerter {
313 321 chrono::Utc::now().to_rfc3339(),
314 322 );
315 323
316 - self.send_email(&subject, &body).await;
324 + self.wam_ticket(&subject, &body, "high", "pom-dns", Some(target)).await;
317 325 self.record_alert(&alert_key, AlertCategory::DnsMismatch, None, None, None).await;
318 326 }
319 327
@@ -366,7 +374,8 @@ impl Alerter {
366 374 chrono::Utc::now().to_rfc3339(),
367 375 );
368 376
369 - self.send_email(&subject, &body).await;
377 + let priority = if days_remaining <= 7 { "critical" } else if days_remaining <= 14 { "high" } else { "medium" };
378 + self.wam_ticket(&subject, &body, priority, "pom-whois", Some(&format!("{target}:{domain}"))).await;
370 379 self.record_alert(&alert_key, AlertCategory::WhoisExpiry, None, None, None).await;
371 380 }
372 381
@@ -396,7 +405,7 @@ impl Alerter {
396 405 chrono::Utc::now().to_rfc3339(),
397 406 );
398 407
399 - self.send_email(&subject, &body).await;
408 + self.wam_ticket(&subject, &body, "high", "pom-whois", Some(&format!("{target}:{domain}"))).await;
400 409 self.record_alert(&alert_key, AlertCategory::WhoisError, None, None, Some(error)).await;
401 410 }
402 411
@@ -437,7 +446,7 @@ impl Alerter {
437 446 chrono::Utc::now().to_rfc3339(),
438 447 );
439 448
440 - self.send_email(&subject, &body).await;
449 + self.wam_ticket(&subject, &body, "high", "pom-cors", Some(target)).await;
441 450 self.record_alert(&alert_key, AlertCategory::CorsFailure, None, None, None).await;
442 451 }
443 452
@@ -488,7 +497,7 @@ impl Alerter {
488 497 chrono::Utc::now().to_rfc3339(),
489 498 );
490 499
491 - self.send_email(&subject, &body).await;
500 + self.wam_ticket(&subject, &body, "medium", "pom-latency", Some(target)).await;
492 501 self.record_alert(&alert_key, AlertCategory::LatencyDrift, None, None, Some(drift_message)).await;
493 502 }
494 503
@@ -539,7 +548,7 @@ impl Alerter {
539 548 chrono::Utc::now().to_rfc3339(),
540 549 );
541 550
542 - self.send_email(&subject, &body).await;
551 + self.wam_ticket(&subject, &body, "medium", "pom-test-duration", Some(target)).await;
543 552 self.record_alert(&alert_key, AlertCategory::TestDurationDrift, None, None, Some(drift_message)).await;
544 553 }
545 554
@@ -578,7 +587,8 @@ impl Alerter {
578 587 chrono::Utc::now().to_rfc3339(),
579 588 );
580 589
581 - self.send_email(&subject, &body).await;
590 + let priority = if status == "missing" { "critical" } else { "high" };
591 + self.wam_ticket(&subject, &body, priority, "pom-backup", Some(&format!("{target}:{database}"))).await;
582 592 self.record_alert(&alert_key, AlertCategory::BackupStale, None, Some(status), None).await;
583 593 }
584 594
@@ -628,7 +638,7 @@ impl Alerter {
628 638 chrono::Utc::now().to_rfc3339(),
629 639 );
630 640
631 - self.send_email(&subject, &body).await;
641 + self.wam_ticket(&subject, &body, "critical", "pom-monitoring", Some("self")).await;
632 642 self.record_alert(alert_key, AlertCategory::MonitoringOffline, None, None, None).await;
633 643 }
634 644
@@ -721,6 +731,41 @@ impl Alerter {
721 731 warn!("failed to record alert: {e}");
722 732 }
723 733 }
734 +
735 + /// Create a WAM ticket (best-effort, fire-and-forget).
736 + async fn wam_ticket(
737 + &self,
738 + title: &str,
739 + body: &str,
740 + priority: &str,
741 + source: &str,
742 + source_ref: Option<&str>,
743 + ) {
744 + let Some(ref base_url) = self.wam_url else { return };
745 + let url = format!("{base_url}/tickets");
746 +
747 + let mut payload = serde_json::json!({
748 + "title": title,
749 + "body": body,
750 + "priority": priority,
751 + "source": source,
752 + });
753 + if let Some(r) = source_ref {
754 + payload["source_ref"] = serde_json::json!(r);
755 + }
756 +
757 + match self.client.post(&url).json(&payload).send().await {
758 + Ok(resp) if resp.status().is_success() => {
759 + info!("WAM ticket created: {title}");
760 + }
761 + Ok(resp) => {
762 + warn!("WAM ticket creation returned {}: {title}", resp.status());
763 + }
764 + Err(e) => {
765 + warn!("WAM unreachable: {e}");
766 + }
767 + }
768 + }
724 769 }
725 770
726 771 #[cfg(test)]
@@ -733,6 +778,7 @@ mod tests {
733 778 to: "test@example.com".to_string(),
734 779 from: "PoM Alerts <pom-alerts@makenot.work>".to_string(),
735 780 cooldown_secs: 300,
781 + wam_url: None,
736 782 };
737 783 Alerter::new(config, pool, "test-instance".to_string())
738 784 }
@@ -38,6 +38,8 @@ pub struct AlertConfig {
38 38 /// Minimum seconds between repeated alerts for the same target.
39 39 #[serde(default = "default_cooldown_secs")]
40 40 pub cooldown_secs: u64,
41 + /// WAM ticket manager URL (tailnet). When set, alerts also create WAM tickets.
42 + pub wam_url: Option<String>,
41 43 }
42 44
43 45 #[derive(Debug, Clone, Default, Deserialize)]
@@ -1926,6 +1926,7 @@ async fn alert_cooldown_key_matches_across_send_and_check() {
1926 1926 to: "test@example.com".to_string(),
1927 1927 from: "PoM Alerts <pom@test.com>".to_string(),
1928 1928 cooldown_secs: 300,
1929 + wam_url: None,
1929 1930 };
1930 1931 let alerter = pom::alerts::Alerter::new(config, pool.clone(), "test".to_string());
1931 1932
@@ -427,6 +427,7 @@ mod tests {
427 427 postmark_inbound_webhook_token: None,
428 428 internal_shared_secret: None,
429 429 cli_service_token: None,
430 + wam_url: None,
430 431 };
431 432 assert!(require_admin(&user, &config).is_ok());
432 433 }
@@ -487,6 +488,7 @@ mod tests {
487 488 postmark_inbound_webhook_token: None,
488 489 internal_shared_secret: None,
489 490 cli_service_token: None,
491 + wam_url: None,
490 492 };
491 493 assert!(require_admin(&user, &config).is_err());
492 494 }
@@ -246,6 +246,18 @@ async fn run_build(state: &AppState, build: &DbBuild, config: &DbBuildConfig) {
246 246 tracing::error!(build_id = %build.id, status = %status, error = ?e, "failed to update final build status");
247 247 }
248 248
249 + if status == BuildStatus::Failed {
250 + if let Some(ref wam) = state.wam {
251 + let title = format!("Build failed: {} v{}", build.tag, build.version);
252 + let body = format!(
253 + "Build {} for {} v{} failed.\n\nError: {}",
254 + build.id, build.tag, build.version,
255 + first_error.as_deref().unwrap_or("unknown"),
256 + );
257 + wam.create_ticket(&title, Some(&body), "high", "build-failed", Some(&build.id.to_string())).await;
258 + }
259 + }
260 +
249 261 tracing::info!(
250 262 build_id = %build.id,
251 263 version = %build.version,
@@ -64,6 +64,9 @@ pub struct Config {
64 64 /// Bearer token for authenticating CLI SSH server → MNW internal API calls.
65 65 /// When unset, internal API endpoints return 503.
66 66 pub cli_service_token: Option<String>,
67 + /// Base URL of the WAM ticket manager (e.g., "http://100.x.x.x:7890").
68 + /// When set, operational events create WAM tickets for human triage.
69 + pub wam_url: Option<String>,
67 70 }
68 71
69 72 /// S3-compatible storage configuration (Hetzner Object Storage)
@@ -187,6 +190,9 @@ impl Config {
187 190 // CLI service token for SSH server → internal API authentication
188 191 let cli_service_token = std::env::var("CLI_SERVICE_TOKEN").ok();
189 192
193 + // WAM ticket manager URL (tailnet, e.g. "http://100.x.x.x:7890")
194 + let wam_url = std::env::var("WAM_URL").ok();
195 +
190 196 Ok(Config {
191 197 host,
192 198 port,
@@ -213,6 +219,7 @@ impl Config {
213 219 postmark_inbound_webhook_token,
214 220 internal_shared_secret,
215 221 cli_service_token,
222 + wam_url,
216 223 })
217 224 }
218 225
@@ -483,6 +490,7 @@ mod tests {
483 490 postmark_inbound_webhook_token: None,
484 491 internal_shared_secret: None,
485 492 cli_service_token: None,
493 + wam_url: None,
486 494 };
487 495 let addr = config.socket_addr();
488 496 assert_eq!(addr.port(), 8080);
@@ -16,6 +16,7 @@ pub mod markdown;
16 16 pub mod metrics;
17 17 pub mod monitor;
18 18 pub mod mt_client;
19 + pub mod wam_client;
19 20 pub mod payments;
20 21 pub mod pricing;
21 22 pub mod scheduler;
@@ -76,6 +77,8 @@ pub struct AppState {
76 77 pub session_cache: Arc<DashMap<UserSessionId, Instant>>,
77 78 /// HTTP client for the Multithreaded internal API (community/thread provisioning).
78 79 pub mt_client: Option<mt_client::MtClient>,
80 + /// HTTP client for the WAM ticket manager (operational alerts).
81 + pub wam: Option<wam_client::WamClient>,
79 82 /// Cache of verified custom domains → user IDs (populated on startup, updated on verify/delete).
80 83 pub domain_cache: Arc<DashMap<String, db::UserId>>,
81 84 /// Unix timestamp when the server will restart (0 = no restart pending).
@@ -218,6 +218,12 @@ async fn main() {
218 218 }
219 219 };
220 220
221 + // WAM ticket manager client (tailnet-only, for operational alerts)
222 + let wam = config.wam_url.as_ref().map(|url| {
223 + tracing::info!(url = %url, "WAM integration enabled");
224 + makenotwork::wam_client::WamClient::new(url.clone())
225 + });
226 +
221 227 // Warm custom domain cache
222 228 let domain_cache = std::sync::Arc::new(dashmap::DashMap::new());
223 229 match makenotwork::db::custom_domains::get_all_verified_domains(&db).await {
@@ -252,6 +258,7 @@ async fn main() {
252 258 start_instant,
253 259 session_cache: std::sync::Arc::new(dashmap::DashMap::new()),
254 260 mt_client,
261 + wam,
255 262 domain_cache,
256 263 restart_at: std::sync::Arc::new(std::sync::atomic::AtomicI64::new(0)),
257 264 sync_notify: std::sync::Arc::new(dashmap::DashMap::new()),
@@ -173,9 +173,40 @@ pub fn spawn_monitor(
173 173 }
174 174 }
175 175
176 + // Create WAM ticket on degradation/error transitions
177 + if snap.status != MonitorStatus::Operational {
178 + if let Some(ref wam) = state.wam {
179 + let priority = match snap.status {
180 + MonitorStatus::Error => "critical",
181 + MonitorStatus::Degraded => "high",
182 + MonitorStatus::Operational => unreachable!(),
183 + };
184 + let title = format!("Health status: {}", snap.status.as_str());
185 + let body = format!(
186 + "db: {}\ns3: {}\nsessions: {}\ncheck_ms: {}",
187 + snap.db_ok, snap.s3_ok, snap.sessions_ok, snap.check_duration_ms,
188 + );
189 + wam.create_ticket(&title, Some(&body), priority, "health-status-change", None).await;
190 + }
191 + }
192 +
176 193 previous_status = Some(snap.status);
177 194 }
178 195
196 + // DB pool pressure check (>80% active connections)
197 + {
198 + let pool_size = state.db.size();
199 + let pool_idle = state.db.num_idle() as u32;
200 + let active = pool_size.saturating_sub(pool_idle);
201 + if pool_size > 0 && active * 100 / pool_size > 80 {
202 + tracing::warn!(pool_size, active, idle = pool_idle, "DB pool pressure >80%");
203 + if let Some(ref wam) = state.wam {
204 + let title = format!("DB pool pressure: {active}/{pool_size} active");
205 + wam.create_ticket(&title, None, "high", "db-pool-pressure", None).await;
206 + }
207 + }
208 + }
209 +
179 210 // Persist snapshot (best-effort)
180 211 if let Err(e) = db::monitor::insert_health_history(
181 212 &state.db,
@@ -160,5 +160,17 @@ async fn handle_account_updated(
160 160 ).await
161 161 .with_context(|| format!("update Stripe status for account {}", update.account_id))?;
162 162
163 + // Alert if charges or payouts became disabled (creator can't receive payments)
164 + if !update.charges_enabled || !update.payouts_enabled {
165 + if let Some(ref wam) = state.wam {
166 + let title = format!("Stripe Connect degraded: {}", update.account_id);
167 + let body = format!(
168 + "charges_enabled: {}\npayouts_enabled: {}\ndetails_submitted: {}",
169 + update.charges_enabled, update.payouts_enabled, update.details_submitted,
170 + );
171 + wam.create_ticket(&title, Some(&body), "high", "stripe-connect-degraded", Some(&update.account_id)).await;
172 + }
173 + }
174 +
163 175 Ok(())
164 176 }
@@ -392,15 +392,135 @@ pub fn spawn_scheduler(
392 392 // Retry failed webhook events
393 393 retry_failed_webhooks(&state).await;
394 394
395 - // Weekly storage drift correction
395 + // Escalate stale pending refunds (unmatched for >24 hours)
396 + escalate_stale_refunds(&state).await;
397 +
398 + // Weekly storage drift correction + integrity checks
396 399 if tick_count.is_multiple_of(DRIFT_CORRECTION_INTERVAL) {
397 400 recalculate_all_storage_used(&state).await;
401 + check_sales_count_drift(&state).await;
402 + }
403 +
404 + // Daily checks (every 1440 ticks at 60s interval)
405 + if tick_count.is_multiple_of(1440) {
406 + check_stale_subscriptions(&state).await;
407 + check_email_bounce_spike(&state).await;
398 408 }
399 409 }
400 410 })
401 411 }
402 412
403 413 // ============================================================================
414 + // Periodic integrity checks
415 + // ============================================================================
416 +
417 + /// Detect items where denormalized sales_count has drifted from actual transaction count.
418 + async fn check_sales_count_drift(state: &AppState) {
419 + let rows = match sqlx::query_as::<_, (db::ItemId, i32, i64)>(
420 + r#"
421 + SELECT i.id, i.sales_count, COUNT(t.id)
422 + FROM items i
423 + LEFT JOIN transactions t ON t.item_id = i.id AND t.status = 'completed'
424 + GROUP BY i.id
425 + HAVING i.sales_count != COUNT(t.id)
426 + LIMIT 50
427 + "#,
428 + )
429 + .fetch_all(&state.db)
430 + .await
431 + {
432 + Ok(r) if r.is_empty() => return,
433 + Ok(r) => r,
434 + Err(e) => {
435 + tracing::error!(error = ?e, "sales count drift check failed");
436 + return;
437 + }
438 + };
439 +
440 + tracing::warn!(count = rows.len(), "sales count drift detected");
441 +
442 + if let Some(ref wam) = state.wam {
443 + let items: Vec<String> = rows
444 + .iter()
445 + .map(|(id, cached, actual)| format!(" {id}: cached={cached}, actual={actual}"))
446 + .collect();
447 + let body = format!("Items with drifted sales_count:\n{}", items.join("\n"));
448 + wam.create_ticket(
449 + &format!("Sales count drift: {} items", rows.len()),
450 + Some(&body),
451 + "medium",
452 + "sales-count-drift",
453 + None,
454 + )
455 + .await;
456 + }
457 + }
458 +
459 + /// Find subscriptions stuck in past_due for >7 days (possible missed webhook).
460 + async fn check_stale_subscriptions(state: &AppState) {
461 + let count: i64 = match sqlx::query_scalar(
462 + r#"
463 + SELECT COUNT(*) FROM (
464 + SELECT 1 FROM creator_subscriptions WHERE status = 'past_due' AND updated_at < NOW() - INTERVAL '7 days'
465 + UNION ALL
466 + SELECT 1 FROM subscriptions WHERE status = 'past_due' AND updated_at < NOW() - INTERVAL '7 days'
467 + ) stale
468 + "#,
469 + )
470 + .fetch_one(&state.db)
471 + .await
472 + {
473 + Ok(c) => c,
474 + Err(e) => {
475 + tracing::error!(error = ?e, "stale subscription check failed");
476 + return;
477 + }
478 + };
479 +
480 + if count > 0 {
481 + tracing::warn!(count, "stale past_due subscriptions detected");
482 + if let Some(ref wam) = state.wam {
483 + wam.create_ticket(
484 + &format!("{count} subscriptions past_due >7 days"),
485 + Some("Subscriptions stuck in past_due for over 7 days. A Stripe webhook may have been missed. Check the Stripe dashboard."),
486 + "medium",
487 + "subscription-stale-past-due",
488 + None,
489 + ).await;
490 + }
491 + }
492 + }
493 +
494 + /// Detect email bounce/complaint spikes (>10 suppressions in 24h).
495 + async fn check_email_bounce_spike(state: &AppState) {
496 + let count: i64 = match sqlx::query_scalar(
497 + "SELECT COUNT(*) FROM email_suppressions WHERE created_at > NOW() - INTERVAL '24 hours'",
498 + )
499 + .fetch_one(&state.db)
500 + .await
501 + {
502 + Ok(c) => c,
503 + Err(e) => {
504 + tracing::error!(error = ?e, "email bounce spike check failed");
505 + return;
506 + }
507 + };
508 +
509 + if count > 10 {
510 + tracing::warn!(count, "email bounce/complaint spike");
511 + if let Some(ref wam) = state.wam {
512 + wam.create_ticket(
513 + &format!("Email bounce spike: {count} suppressions in 24h"),
514 + Some("Elevated bounce/complaint rate may indicate a deliverability problem. Check Postmark dashboard."),
515 + "high",
516 + "email-bounce-spike",
517 + None,
518 + ).await;
519 + }
520 + }
521 + }
522 +
523 + // ============================================================================
404 524 // Webhook retry
405 525 // ============================================================================
406 526
@@ -451,8 +571,10 @@ async fn retry_failed_webhooks(state: &AppState) {
451 571 }
452 572 }
453 573 Err(e) => {
574 + let is_dead = attempt >= 5;
454 575 tracing::warn!(
455 576 event_id = %event.id, attempt = attempt, error = ?e,
577 + dead = is_dead,
456 578 "webhook retry failed"
457 579 );
458 580 if let Err(e) = db::webhook_events::schedule_retry(
@@ -460,8 +582,115 @@ async fn retry_failed_webhooks(state: &AppState) {
460 582 ).await {
461 583 tracing::error!(error = ?e, "failed to schedule webhook retry");
462 584 }
585 +
586 + // Create WAM ticket when retries are exhausted (dead letter)
587 + if is_dead {
588 + if let Some(ref wam) = state.wam {
589 + let title = format!(
590 + "Dead webhook: {} ({})",
591 + event.event_type, event.id
592 + );
593 + let body = format!(
594 + "Webhook event exhausted all {} retry attempts.\n\
595 + Source: {}\nType: {}\nLast error: {:?}",
596 + attempt, event.source, event.event_type, e,
597 + );
598 + wam.create_ticket(
599 + &title,
600 + Some(&body),
601 + "high",
602 + "webhook-dead-letter",
603 + Some(&event.id.to_string()),
604 + )
605 + .await;
606 + }
607 + }
608 + }
609 + }
610 + }
611 + }
612 +
613 + // ============================================================================
614 + // Pending refund escalation
615 + // ============================================================================
616 +
617 + /// Alert the admin about pending refunds that have gone unmatched for >24 hours.
618 + ///
619 + /// These represent charge.refunded webhooks that arrived before their matching
620 + /// checkout.session.completed and never got resolved. Likely indicates a lost
621 + /// payment webhook that needs manual investigation.
622 + async fn escalate_stale_refunds(state: &AppState) {
623 + let stale = match db::pending_refunds::get_stale_refunds(
624 + &state.db,
625 + chrono::Duration::hours(24),
626 + )
627 + .await
628 + {
629 + Ok(s) if s.is_empty() => return,
630 + Ok(s) => s,
631 + Err(e) => {
632 + tracing::error!(error = ?e, "failed to query stale pending refunds");
633 + return;
634 + }
635 + };
636 +
637 + let alert_email = std::env::var("ALERT_EMAIL").ok();
638 +
639 + for refund in &stale {
640 + tracing::error!(
641 + payment_intent_id = %refund.payment_intent_id,
642 + amount = refund.amount,
643 + amount_refunded = refund.amount_refunded,
644 + created_at = %refund.created_at,
645 + "STALE PENDING REFUND: unmatched for >24h, needs manual investigation"
646 + );
647 +
648 + if let Some(ref to) = alert_email {
649 + let subject = format!(
650 + "Unmatched refund: {} ({}c refunded)",
651 + refund.payment_intent_id, refund.amount_refunded
652 + );
653 + let body = format!(
654 + "A charge.refunded webhook for payment intent {} has been pending for >24 hours \
655 + with no matching completed transaction.\n\n\
656 + Amount: {}c\nAmount refunded: {}c\nReceived: {}\n\n\
657 + This likely means the checkout.session.completed webhook was lost. \
658 + Check the Stripe dashboard and reconcile manually.",
659 + refund.payment_intent_id,
660 + refund.amount,
661 + refund.amount_refunded,
662 + refund.created_at,
663 + );
664 + if let Err(e) = state.email.send_alert(to, &subject, &body).await {
665 + tracing::error!(error = ?e, "failed to send stale refund alert email");
463 666 }
464 667 }
668 +
669 + // Create WAM ticket alongside the alert email
670 + if let Some(ref wam) = state.wam {
671 + let title = format!(
672 + "Unmatched refund: {} ({}c)",
673 + refund.payment_intent_id, refund.amount_refunded
674 + );
675 + let body = format!(
676 + "charge.refunded webhook pending >24h with no matching completed transaction.\n\
677 + Amount: {}c\nRefunded: {}c\nReceived: {}\n\
678 + Check Stripe dashboard and reconcile manually.",
679 + refund.amount, refund.amount_refunded, refund.created_at,
680 + );
681 + wam.create_ticket(
682 + &title,
683 + Some(&body),
684 + "critical",
685 + "refund-escalation",
686 + Some(&refund.payment_intent_id),
687 + )
688 + .await;
689 + }
690 +
691 + if let Err(e) = db::pending_refunds::mark_escalated(&state.db, refund.id).await {
692 + tracing::error!(error = ?e, "failed to mark pending refund as escalated");
693 + }
465 694 }
466 695 }
467 696
@@ -0,0 +1,71 @@
1 + //! HTTP client for the WAM (Whack-a-Mole) ticket manager.
2 + //!
3 + //! WAM runs on the tailnet — no auth required. The MNW server creates tickets
4 + //! for operational events that need human attention (stale refunds, dead
5 + //! webhooks, etc.).
6 +
7 + use serde::Serialize;
8 +
9 + /// WAM ticket creation client.
10 + #[derive(Clone)]
11 + pub struct WamClient {
12 + http: reqwest::Client,
13 + base_url: String,
14 + }
15 +
16 + #[derive(Serialize)]
17 + struct CreateTicketRequest<'a> {
18 + title: &'a str,
19 + #[serde(skip_serializing_if = "Option::is_none")]
20 + body: Option<&'a str>,
21 + priority: &'a str,
22 + source: &'a str,
23 + #[serde(skip_serializing_if = "Option::is_none")]
24 + source_ref: Option<&'a str>,
25 + }
26 +
27 + impl WamClient {
28 + pub fn new(base_url: String) -> Self {
29 + let http = reqwest::Client::builder()
30 + .timeout(std::time::Duration::from_secs(5))
31 + .connect_timeout(std::time::Duration::from_secs(3))
32 + .build()
33 + .expect("WAM HTTP client");
34 + Self { http, base_url }
35 + }
36 +
37 + /// Create a ticket in WAM. Errors are logged but never propagated — WAM
38 + /// is a best-effort notification channel, not a critical path.
39 + pub async fn create_ticket(
40 + &self,
41 + title: &str,
42 + body: Option<&str>,
43 + priority: &str,
44 + source: &str,
45 + source_ref: Option<&str>,
46 + ) {
47 + let url = format!("{}/tickets", self.base_url);
48 + let req = CreateTicketRequest {
49 + title,
50 + body,
51 + priority,
52 + source,
53 + source_ref,
54 + };
55 +
56 + match self.http.post(&url).json(&req).send().await {
57 + Ok(resp) if resp.status().is_success() => {
58 + tracing::info!(title, source, "WAM ticket created");
59 + }
60 + Ok(resp) => {
61 + tracing::warn!(
62 + status = %resp.status(), title, source,
63 + "WAM ticket creation returned non-success"
64 + );
65 + }
66 + Err(e) => {
67 + tracing::warn!(error = %e, title, source, "WAM unreachable");
68 + }
69 + }
70 + }
71 + }
@@ -308,6 +308,7 @@ impl TestHarness {
308 308 mt_client: opts.mt_base_url.zip(opts.internal_shared_secret).map(
309 309 |(url, secret)| makenotwork::mt_client::MtClient::new(url, secret),
310 310 ),
311 + wam: None,
311 312 domain_cache: Arc::new(dashmap::DashMap::new()),
312 313 restart_at: Arc::new(std::sync::atomic::AtomicI64::new(0)),
313 314 sync_notify: Arc::new(dashmap::DashMap::new()),
@@ -112,6 +112,7 @@ pub async fn run(config: LoadConfig) {
112 112 start_instant: Instant::now(),
113 113 session_cache: Arc::new(dashmap::DashMap::new()),
114 114 mt_client: None,
115 + wam: None,
115 116 domain_cache: Arc::new(dashmap::DashMap::new()),
116 117 restart_at: Arc::new(std::sync::atomic::AtomicI64::new(0)),
117 118 sync_notify: Arc::new(dashmap::DashMap::new()),
@@ -0,0 +1,1842 @@
1 + # This file is automatically @generated by Cargo.
2 + # It is not intended for manual editing.
3 + version = 4
4 +
5 + [[package]]
6 + name = "addr2line"
7 + version = "0.25.1"
8 + source = "registry+https://github.com/rust-lang/crates.io-index"
9 + checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
10 + dependencies = [
11 + "gimli",
12 + ]
13 +
14 + [[package]]
15 + name = "adler2"
16 + version = "2.0.1"
17 + source = "registry+https://github.com/rust-lang/crates.io-index"
18 + checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
19 +
20 + [[package]]
21 + name = "allocator-api2"
22 + version = "0.2.21"
23 + source = "registry+https://github.com/rust-lang/crates.io-index"
24 + checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
25 +
26 + [[package]]
27 + name = "android_system_properties"
28 + version = "0.1.5"
29 + source = "registry+https://github.com/rust-lang/crates.io-index"
30 + checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
31 + dependencies = [
32 + "libc",
33 + ]
34 +
35 + [[package]]
36 + name = "anstream"
37 + version = "1.0.0"
38 + source = "registry+https://github.com/rust-lang/crates.io-index"
39 + checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
40 + dependencies = [
41 + "anstyle",
42 + "anstyle-parse",
43 + "anstyle-query",
44 + "anstyle-wincon",
45 + "colorchoice",
46 + "is_terminal_polyfill",
47 + "utf8parse",
48 + ]
49 +
50 + [[package]]
51 + name = "anstyle"
52 + version = "1.0.14"
53 + source = "registry+https://github.com/rust-lang/crates.io-index"
54 + checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
55 +
56 + [[package]]
57 + name = "anstyle-parse"
58 + version = "1.0.0"
59 + source = "registry+https://github.com/rust-lang/crates.io-index"
60 + checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
61 + dependencies = [
62 + "utf8parse",
63 + ]
64 +
65 + [[package]]
66 + name = "anstyle-query"
67 + version = "1.1.5"
68 + source = "registry+https://github.com/rust-lang/crates.io-index"
69 + checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
70 + dependencies = [
71 + "windows-sys 0.61.2",
72 + ]
73 +
74 + [[package]]
75 + name = "anstyle-wincon"
76 + version = "3.0.11"
77 + source = "registry+https://github.com/rust-lang/crates.io-index"
78 + checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
79 + dependencies = [
80 + "anstyle",
81 + "once_cell_polyfill",
82 + "windows-sys 0.61.2",
83 + ]
84 +
85 + [[package]]
86 + name = "anyhow"
87 + version = "1.0.102"
88 + source = "registry+https://github.com/rust-lang/crates.io-index"
89 + checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
90 +
91 + [[package]]
92 + name = "atomic-waker"
93 + version = "1.1.2"
94 + source = "registry+https://github.com/rust-lang/crates.io-index"
95 + checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
96 +
97 + [[package]]
98 + name = "autocfg"
99 + version = "1.5.0"
100 + source = "registry+https://github.com/rust-lang/crates.io-index"
101 + checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
102 +
103 + [[package]]
104 + name = "axum"
105 + version = "0.8.9"
106 + source = "registry+https://github.com/rust-lang/crates.io-index"
107 + checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
108 + dependencies = [
109 + "axum-core",
110 + "bytes",
111 + "form_urlencoded",
112 + "futures-util",
113 + "http",
114 + "http-body",
115 + "http-body-util",
116 + "hyper",
117 + "hyper-util",
118 + "itoa",
119 + "matchit",
120 + "memchr",
121 + "mime",
122 + "percent-encoding",
123 + "pin-project-lite",
124 + "serde_core",
125 + "serde_json",
126 + "serde_path_to_error",
127 + "serde_urlencoded",
128 + "sync_wrapper",
129 + "tokio",
130 + "tower",
131 + "tower-layer",
132 + "tower-service",
133 + "tracing",
134 + ]
135 +
136 + [[package]]
137 + name = "axum-core"
138 + version = "0.5.6"
139 + source = "registry+https://github.com/rust-lang/crates.io-index"
140 + checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
141 + dependencies = [
142 + "bytes",
143 + "futures-core",
144 + "http",
145 + "http-body",
146 + "http-body-util",
147 + "mime",
148 + "pin-project-lite",
149 + "sync_wrapper",
150 + "tower-layer",
151 + "tower-service",
152 + "tracing",
153 + ]
154 +
155 + [[package]]
156 + name = "backtrace"
157 + version = "0.3.76"
158 + source = "registry+https://github.com/rust-lang/crates.io-index"
159 + checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
160 + dependencies = [
161 + "addr2line",
162 + "cfg-if",
163 + "libc",
164 + "miniz_oxide",
165 + "object",
166 + "rustc-demangle",
167 + "windows-link",
168 + ]
169 +
170 + [[package]]
171 + name = "bitflags"
172 + version = "2.11.1"
173 + source = "registry+https://github.com/rust-lang/crates.io-index"
174 + checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
175 +
176 + [[package]]
177 + name = "bumpalo"
178 + version = "3.20.2"
179 + source = "registry+https://github.com/rust-lang/crates.io-index"
180 + checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
181 +
182 + [[package]]
183 + name = "bytes"
184 + version = "1.11.1"
185 + source = "registry+https://github.com/rust-lang/crates.io-index"
186 + checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
187 +
188 + [[package]]
189 + name = "cassowary"
190 + version = "0.3.0"
191 + source = "registry+https://github.com/rust-lang/crates.io-index"
192 + checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
193 +
194 + [[package]]
195 + name = "castaway"
196 + version = "0.2.4"
197 + source = "registry+https://github.com/rust-lang/crates.io-index"
198 + checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
199 + dependencies = [
200 + "rustversion",
201 + ]
202 +
203 + [[package]]
204 + name = "cc"
205 + version = "1.2.61"
206 + source = "registry+https://github.com/rust-lang/crates.io-index"
207 + checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d"
208 + dependencies = [
209 + "find-msvc-tools",
210 + "shlex",
211 + ]
212 +
213 + [[package]]
214 + name = "cfg-if"
215 + version = "1.0.4"
216 + source = "registry+https://github.com/rust-lang/crates.io-index"
217 + checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
218 +
219 + [[package]]
220 + name = "chrono"
221 + version = "0.4.44"
222 + source = "registry+https://github.com/rust-lang/crates.io-index"
223 + checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
224 + dependencies = [
225 + "iana-time-zone",
226 + "js-sys",
227 + "num-traits",
228 + "serde",
229 + "wasm-bindgen",
230 + "windows-link",
231 + ]
232 +
233 + [[package]]
234 + name = "clap"
235 + version = "4.6.1"
236 + source = "registry+https://github.com/rust-lang/crates.io-index"
237 + checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
238 + dependencies = [
239 + "clap_builder",
240 + "clap_derive",
241 + ]
242 +
243 + [[package]]
244 + name = "clap_builder"
245 + version = "4.6.0"
246 + source = "registry+https://github.com/rust-lang/crates.io-index"
247 + checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
248 + dependencies = [
249 + "anstream",
250 + "anstyle",
251 + "clap_lex",
252 + "strsim",
253 + ]
254 +
255 + [[package]]
256 + name = "clap_derive"
257 + version = "4.6.1"
258 + source = "registry+https://github.com/rust-lang/crates.io-index"
259 + checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
260 + dependencies = [
261 + "heck",
262 + "proc-macro2",
263 + "quote",
264 + "syn",
265 + ]
266 +
267 + [[package]]
268 + name = "clap_lex"
269 + version = "1.1.0"
270 + source = "registry+https://github.com/rust-lang/crates.io-index"
271 + checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
272 +
273 + [[package]]
274 + name = "color-eyre"
275 + version = "0.6.5"
276 + source = "registry+https://github.com/rust-lang/crates.io-index"
277 + checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d"
278 + dependencies = [
279 + "backtrace",
280 + "color-spantrace",
281 + "eyre",
282 + "indenter",
283 + "once_cell",
284 + "owo-colors",
285 + "tracing-error",
286 + ]
287 +
288 + [[package]]
289 + name = "color-spantrace"
290 + version = "0.3.0"
291 + source = "registry+https://github.com/rust-lang/crates.io-index"
292 + checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427"
293 + dependencies = [
294 + "once_cell",
295 + "owo-colors",
296 + "tracing-core",
297 + "tracing-error",
298 + ]
299 +
300 + [[package]]
301 + name = "colorchoice"
302 + version = "1.0.5"
303 + source = "registry+https://github.com/rust-lang/crates.io-index"
304 + checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
305 +
306 + [[package]]
307 + name = "compact_str"
308 + version = "0.8.1"
309 + source = "registry+https://github.com/rust-lang/crates.io-index"
310 + checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32"
311 + dependencies = [
312 + "castaway",
313 + "cfg-if",
314 + "itoa",
315 + "rustversion",
316 + "ryu",
317 + "static_assertions",
318 + ]
319 +
320 + [[package]]
321 + name = "core-foundation-sys"
322 + version = "0.8.7"
323 + source = "registry+https://github.com/rust-lang/crates.io-index"
324 + checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
325 +
326 + [[package]]
327 + name = "crossterm"
328 + version = "0.28.1"
329 + source = "registry+https://github.com/rust-lang/crates.io-index"
330 + checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
331 + dependencies = [
332 + "bitflags",
333 + "crossterm_winapi",
334 + "mio",
335 + "parking_lot",
336 + "rustix",
337 + "signal-hook",
338 + "signal-hook-mio",
339 + "winapi",
340 + ]
341 +
342 + [[package]]
343 + name = "crossterm_winapi"
344 + version = "0.9.1"
345 + source = "registry+https://github.com/rust-lang/crates.io-index"
346 + checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
347 + dependencies = [
348 + "winapi",
349 + ]
350 +
351 + [[package]]
352 + name = "darling"
353 + version = "0.23.0"
354 + source = "registry+https://github.com/rust-lang/crates.io-index"
355 + checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
356 + dependencies = [
357 + "darling_core",
358 + "darling_macro",
359 + ]
360 +
361 + [[package]]
362 + name = "darling_core"
363 + version = "0.23.0"
364 + source = "registry+https://github.com/rust-lang/crates.io-index"
365 + checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
366 + dependencies = [
367 + "ident_case",
368 + "proc-macro2",
369 + "quote",
370 + "strsim",
371 + "syn",
372 + ]
373 +
374 + [[package]]
375 + name = "darling_macro"
376 + version = "0.23.0"
377 + source = "registry+https://github.com/rust-lang/crates.io-index"
378 + checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
379 + dependencies = [
380 + "darling_core",
381 + "quote",
382 + "syn",
383 + ]
384 +
385 + [[package]]
386 + name = "directories"
387 + version = "6.0.0"
388 + source = "registry+https://github.com/rust-lang/crates.io-index"
389 + checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d"
390 + dependencies = [
391 + "dirs-sys",
392 + ]
393 +
394 + [[package]]
395 + name = "dirs-sys"
396 + version = "0.5.0"
397 + source = "registry+https://github.com/rust-lang/crates.io-index"
398 + checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
399 + dependencies = [
400 + "libc",
401 + "option-ext",
402 + "redox_users",
403 + "windows-sys 0.61.2",
404 + ]
405 +
406 + [[package]]
407 + name = "either"
408 + version = "1.15.0"
409 + source = "registry+https://github.com/rust-lang/crates.io-index"
410 + checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
411 +
412 + [[package]]
413 + name = "equivalent"
414 + version = "1.0.2"
415 + source = "registry+https://github.com/rust-lang/crates.io-index"
416 + checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
417 +
418 + [[package]]
419 + name = "errno"
420 + version = "0.3.14"
421 + source = "registry+https://github.com/rust-lang/crates.io-index"
422 + checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
423 + dependencies = [
424 + "libc",
425 + "windows-sys 0.61.2",
426 + ]
427 +
428 + [[package]]
429 + name = "eyre"
430 + version = "0.6.12"
431 + source = "registry+https://github.com/rust-lang/crates.io-index"
432 + checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"
433 + dependencies = [
434 + "indenter",
435 + "once_cell",
436 + ]
437 +
438 + [[package]]
439 + name = "fallible-iterator"
440 + version = "0.3.0"
441 + source = "registry+https://github.com/rust-lang/crates.io-index"
442 + checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
443 +
444 + [[package]]
445 + name = "fallible-streaming-iterator"
446 + version = "0.1.9"
447 + source = "registry+https://github.com/rust-lang/crates.io-index"
448 + checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
449 +
450 + [[package]]
451 + name = "find-msvc-tools"
452 + version = "0.1.9"
453 + source = "registry+https://github.com/rust-lang/crates.io-index"
454 + checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
455 +
456 + [[package]]
457 + name = "foldhash"
458 + version = "0.1.5"
459 + source = "registry+https://github.com/rust-lang/crates.io-index"
460 + checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
461 +
462 + [[package]]
463 + name = "form_urlencoded"
464 + version = "1.2.2"
465 + source = "registry+https://github.com/rust-lang/crates.io-index"
466 + checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
467 + dependencies = [
468 + "percent-encoding",
469 + ]
470 +
471 + [[package]]
472 + name = "futures-channel"
473 + version = "0.3.32"
474 + source = "registry+https://github.com/rust-lang/crates.io-index"
475 + checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
476 + dependencies = [
477 + "futures-core",
478 + ]
479 +
480 + [[package]]
481 + name = "futures-core"
482 + version = "0.3.32"
483 + source = "registry+https://github.com/rust-lang/crates.io-index"
484 + checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
485 +
486 + [[package]]
487 + name = "futures-task"
488 + version = "0.3.32"
489 + source = "registry+https://github.com/rust-lang/crates.io-index"
490 + checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
491 +
492 + [[package]]
493 + name = "futures-util"
494 + version = "0.3.32"
495 + source = "registry+https://github.com/rust-lang/crates.io-index"
496 + checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
497 + dependencies = [
498 + "futures-core",
499 + "futures-task",
500 + "pin-project-lite",
Lines truncated
@@ -0,0 +1,23 @@
1 + [package]
2 + name = "wam"
3 + version = "0.1.0"
4 + edition = "2024"
5 + license-file = "../LICENSE"
6 +
7 + [[bin]]
8 + name = "wam"
9 + path = "src/main.rs"
10 +
11 + [dependencies]
12 + axum = "0.8"
13 + chrono = { version = "0.4", features = ["serde"] }
14 + clap = { version = "4", features = ["derive"] }
15 + color-eyre = "0.6"
16 + crossterm = "0.28"
17 + directories = "6"
18 + ratatui = "0.29"
19 + rusqlite = { version = "0.34", features = ["bundled"] }
20 + serde = { version = "1", features = ["derive"] }
21 + serde_json = "1"
22 + tokio = { version = "1", features = ["macros", "rt-multi-thread", "net"] }
23 + uuid = { version = "1", features = ["v4"] }
@@ -0,0 +1,101 @@
1 + # WAM (Whack-a-Mole) TODO
2 +
3 + ## Status
4 + v0.1.0 complete. Phase 1 done. ~800 lines, 5 tests.
5 +
6 + ---
7 +
8 + ## Overview
9 + WAM (Whack-a-Mole) is a ratatui-based CLI ticket manager for internal MNW operations. Other MNW services (server, PoM, mnw-cli) can create tickets programmatically via HTTP API or CLI, and an operator manages them through the TUI. Security: HTTP API binds on tailnet only -- tailnet membership is the auth boundary.
10 +
11 + ---
12 +
13 + ## Phase 1: Core (v0.1.0) -- DONE
14 +
15 + ### Data Layer
16 + - [x] SQLite database with WAL mode, inline migration
17 + - [x] Ticket model: id (UUID), title, body, priority, status, source, source_ref, created_at, updated_at, resolved_at
18 + - [x] CRUD: create, get (prefix match), list (with filters), update status
19 + - [x] DB path: `~/.local/share/wam/wam.db` (via `directories` crate)
20 + - [x] 5 unit tests (in-memory SQLite)
21 +
22 + ### CLI
23 + - [x] `wam` -- launch TUI (default command)
24 + - [x] `wam create --title "..." [--body "..."] [--priority high] [--source manual] [--source-ref X]`
25 + - [x] `wam list [--status open] [--priority high] [--source X]`
26 + - [x] `wam show <id>` -- git-style prefix matching
27 + - [x] `wam resolve <id>`
28 + - [x] `wam close <id>`
29 +
30 + ### TUI
31 + - [x] List view: table sorted by priority desc, then date desc
32 + - [x] Color coding by priority (low=DarkGray, medium=White, high=Yellow, critical=Red)
33 + - [x] Status indicators (* open, > in-progress, v resolved, x closed)
34 + - [x] Detail view: full body, all metadata
35 + - [x] Inline status changes (o/i/r/c keys)
36 + - [x] Create new ticket (n key, popup with title input)
37 + - [x] Filter toggles: f (status), p (priority), s (source -- cycles through known sources)
38 + - [x] Search (/ key, filters by title)
39 + - [x] q to quit, Esc to go back
40 +
41 + ---
42 +
43 + ## Phase 2: Integration API (v0.2.0) -- HTTP API DONE, integration pending
44 +
45 + ### HTTP Listener -- DONE
46 + - [x] `wam serve [--port 7890]` -- axum HTTP server, binds 0.0.0.0
47 + - [x] `POST /tickets` -- create ticket (JSON body: title, body, priority, source, source_ref)
48 + - [x] `GET /tickets` -- list tickets (query params: status, priority, source, search)
49 + - [x] `GET /tickets/:id` -- get single ticket (prefix match)
50 + - [x] `PATCH /tickets/:id` -- update ticket status
51 + - [x] Security: tailnet-only (no auth layer needed -- tailnet membership is the trust boundary)
52 + - [x] Shared state via Arc<Mutex<Connection>>
53 +
54 + ### MNW Server Integration -- DONE
55 + - [x] `wam_client.rs` in MNW server -- fire-and-forget reqwest POST, 5s timeout
56 + - [x] `WAM_URL` env var in Config, `WamClient` in AppState (optional)
57 + - [x] Refund escalation: creates critical ticket alongside alert email (`source: refund-escalation`)
58 + - [x] Webhook dead letter: creates high-priority ticket when retries exhausted (`source: webhook-dead-letter`)
59 + - [x] 6 user-facing proactive tickets (license key, Fan+ credit, quarantine, payment failed, Stripe Connect, build failed)
60 + - [x] 2 infrastructure tickets (health status, DB pool pressure)
61 + - [x] 3 periodic integrity checks (sales count drift, stale subscriptions, bounce spike)
62 +
63 + ### PoM Integration -- DONE
64 + - [x] `wam_url` field in AlertConfig, WAM client in Alerter
65 + - [x] All non-recovery alerts now create WAM tickets instead of emails
66 + - [x] Recovery alerts remain email-only (transient good news)
67 + - [x] 12 alert types wired: health, TLS expiry, TLS error, peer missing, route failure, DNS mismatch, WHOIS expiry, WHOIS error, CORS failure, latency drift, test duration drift, backup stale, monitoring offline
68 +
69 + ---
70 +
71 + ## Phase 3: Polish (v0.3.0)
72 +
73 + ### TUI Enhancements
74 + - [ ] Edit ticket title/body inline
75 + - [ ] Ticket comments/notes (appended log of updates)
76 + - [ ] Bulk operations (select multiple, resolve all)
77 + - [ ] Configurable keybindings
78 +
79 + ### Operations
80 + - [ ] `wam stats` — summary (open by priority, by source, avg resolution time)
81 + - [ ] `wam export` — dump tickets as JSON/CSV
82 + - [ ] `wam prune --older-than 90d --status closed` — clean up old tickets
83 + - [ ] Notification sound/desktop notification on new critical ticket (when TUI is open)
84 +
85 + ---
86 +
87 + ## Future
88 + - [ ] Standalone distribution (outside MNW monorepo) if it proves generally useful
89 + - [ ] Paid hosted version with team features, SLA tracking, escalation chains
90 + - [ ] Webhook-based integrations (Slack, Discord, email)
91 + - [ ] Multi-user with conflict resolution (SQLite WAL mode already supports concurrent readers)
92 +
93 + ---
94 +
95 + ## Key Paths
96 + ```
97 + MNW/wam/
98 + Cargo.toml
99 + src/main.rs
100 + docs/todo.md
101 + ```
@@ -0,0 +1,168 @@
1 + //! HTTP API for programmatic ticket management.
2 + //!
3 + //! Designed to run on the tailnet -- tailnet membership is the auth boundary,
4 + //! so no HMAC or token auth is needed.
5 +
6 + use std::sync::{Arc, Mutex};
7 +
8 + use axum::{
9 + Json, Router,
10 + extract::{Path, Query, State},
11 + http::StatusCode,
12 + response::IntoResponse,
13 + routing::{get, patch, post},
14 + };
15 + use rusqlite::Connection;
16 + use serde::Deserialize;
17 +
18 + use crate::db::{self, ListFilter};
19 + use crate::types::{NewTicket, Priority, Status};
20 +
21 + /// Shared state: SQLite connection behind a mutex.
22 + pub type Db = Arc<Mutex<Connection>>;
23 +
24 + /// Build the axum router.
25 + pub fn router(conn: Connection) -> Router {
26 + let db: Db = Arc::new(Mutex::new(conn));
27 + Router::new()
28 + .route("/tickets", post(create_ticket))
29 + .route("/tickets", get(list_tickets))
30 + .route("/tickets/{id}", get(get_ticket))
31 + .route("/tickets/{id}", patch(update_ticket))
32 + .with_state(db)
33 + }
34 +
35 + /// Start the HTTP server on the given port.
36 + pub async fn serve(conn: Connection, port: u16) -> color_eyre::eyre::Result<()> {
37 + let app = router(conn);
38 + let addr = format!("0.0.0.0:{port}");
39 + let listener = tokio::net::TcpListener::bind(&addr).await?;
40 + eprintln!("wam serving on {addr}");
41 + axum::serve(listener, app).await?;
42 + Ok(())
43 + }
44 +
45 + // -- Handlers -----------------------------------------------------------------
46 +
47 + /// POST /tickets
48 + async fn create_ticket(
49 + State(db): State<Db>,
50 + Json(new): Json<NewTicket>,
51 + ) -> impl IntoResponse {
52 + let conn = db.lock().unwrap();
53 + match db::create_ticket(&conn, &new) {
54 + Ok(ticket) => (StatusCode::CREATED, Json(serde_json::json!(ticket))).into_response(),
55 + Err(e) => (
56 + StatusCode::INTERNAL_SERVER_ERROR,
57 + Json(serde_json::json!({"error": e.to_string()})),
58 + )
59 + .into_response(),
60 + }
61 + }
62 +
63 + /// Query params for GET /tickets.
64 + #[derive(Debug, Deserialize, Default)]
65 + pub struct ListQuery {
66 + pub status: Option<String>,
67 + pub priority: Option<String>,
68 + pub source: Option<String>,
69 + pub search: Option<String>,
70 + }
71 +
72 + /// GET /tickets
73 + async fn list_tickets(
74 + State(db): State<Db>,
75 + Query(q): Query<ListQuery>,
76 + ) -> impl IntoResponse {
77 + let status = q.status.as_deref().and_then(|s| s.parse::<Status>().ok());
78 + let priority = q.priority.as_deref().and_then(|s| s.parse::<Priority>().ok());
79 +
80 + let conn = db.lock().unwrap();
81 + let filter = ListFilter {
82 + status,
83 + priority,
84 + source: q.source.as_deref(),
85 + search: q.search.as_deref(),
86 + };
87 + match db::list_tickets(&conn, &filter) {
88 + Ok(tickets) => Json(serde_json::json!({"data": tickets, "count": tickets.len()})).into_response(),
89 + Err(e) => (
90 + StatusCode::INTERNAL_SERVER_ERROR,
91 + Json(serde_json::json!({"error": e.to_string()})),
92 + )
93 + .into_response(),
94 + }
95 + }
96 +
97 + /// GET /tickets/:id
98 + async fn get_ticket(
99 + State(db): State<Db>,
100 + Path(id): Path<String>,
101 + ) -> impl IntoResponse {
102 + let conn = db.lock().unwrap();
103 + match db::get_ticket(&conn, &id) {
104 + Ok(ticket) => Json(serde_json::json!(ticket)).into_response(),
105 + Err(_) => (
106 + StatusCode::NOT_FOUND,
107 + Json(serde_json::json!({"error": "ticket not found"})),
108 + )
109 + .into_response(),
110 + }
111 + }
112 +
113 + /// PATCH /tickets/:id body.
114 + #[derive(Debug, Deserialize)]
115 + pub struct UpdateBody {
116 + pub status: Option<String>,
117 + }
118 +
119 + /// PATCH /tickets/:id
120 + async fn update_ticket(
121 + State(db): State<Db>,
122 + Path(id): Path<String>,
123 + Json(body): Json<UpdateBody>,
124 + ) -> impl IntoResponse {
125 + let conn = db.lock().unwrap();
126 +
127 + // Resolve prefix to full ID first
128 + let ticket = match db::get_ticket(&conn, &id) {
129 + Ok(t) => t,
130 + Err(_) => {
131 + return (
132 + StatusCode::NOT_FOUND,
133 + Json(serde_json::json!({"error": "ticket not found"})),
134 + )
135 + .into_response();
136 + }
137 + };
138 +
139 + if let Some(status_str) = body.status {
140 + let status = match status_str.parse::<Status>() {
141 + Ok(s) => s,
142 + Err(_) => {
143 + return (
144 + StatusCode::BAD_REQUEST,
145 + Json(serde_json::json!({"error": format!("invalid status: {status_str}")})),
146 + )
147 + .into_response();
148 + }
149 + };
150 + if let Err(e) = db::update_status(&conn, &ticket.id, status) {
151 + return (
152 + StatusCode::INTERNAL_SERVER_ERROR,
153 + Json(serde_json::json!({"error": e.to_string()})),
154 + )
155 + .into_response();
156 + }
157 + }
158 +
159 + // Return updated ticket
160 + match db::get_ticket(&conn, &ticket.id) {
161 + Ok(t) => Json(serde_json::json!(t)).into_response(),
162 + Err(e) => (
163 + StatusCode::INTERNAL_SERVER_ERROR,
164 + Json(serde_json::json!({"error": e.to_string()})),
165 + )
166 + .into_response(),
167 + }
168 + }
@@ -0,0 +1,67 @@
1 + //! CLI argument parsing via clap.
2 +
3 + use clap::{Parser, Subcommand};
4 +
5 + use crate::types::{Priority, Status};
6 +
7 + #[derive(Parser)]
8 + #[command(name = "wam", about = "Whack-a-Mole -- ticket manager")]
9 + pub struct Cli {
10 + #[command(subcommand)]
11 + pub command: Option<Command>,
12 + }
13 +
14 + #[derive(Subcommand)]
15 + pub enum Command {
16 + /// Create a new ticket
17 + Create {
18 + /// Ticket title
19 + #[arg(short, long)]
20 + title: String,
21 + /// Ticket body / description
22 + #[arg(short, long)]
23 + body: Option<String>,
24 + /// Priority (low, medium, high, critical)
25 + #[arg(short, long, default_value = "medium")]
26 + priority: Priority,
27 + /// Source system (e.g. "refund-escalation", "pom")
28 + #[arg(short, long, default_value = "manual")]
29 + source: String,
30 + /// Reference ID in the source system
31 + #[arg(long)]
32 + source_ref: Option<String>,
33 + },
34 + /// List tickets
35 + List {
36 + /// Filter by status
37 + #[arg(short, long)]
38 + status: Option<Status>,
39 + /// Filter by priority
40 + #[arg(short, long)]
41 + priority: Option<Priority>,
42 + /// Filter by source
43 + #[arg(long)]
44 + source: Option<String>,
45 + },
46 + /// Show ticket details
47 + Show {
48 + /// Ticket ID (or unique prefix)
49 + id: String,
50 + },
51 + /// Mark a ticket as resolved
52 + Resolve {
53 + /// Ticket ID (or unique prefix)
54 + id: String,
55 + },
56 + /// Mark a ticket as closed
57 + Close {
58 + /// Ticket ID (or unique prefix)
59 + id: String,
60 + },
61 + /// Start the HTTP API server (tailnet-only, no auth required)
62 + Serve {
63 + /// Port to listen on
64 + #[arg(short, long, default_value = "7890")]
65 + port: u16,
66 + },
67 + }
A wam/src/db.rs +278
@@ -0,0 +1,278 @@
1 + //! SQLite data layer for tickets.
2 +
3 + use chrono::{DateTime, Utc};
4 + use color_eyre::eyre::{Result, WrapErr, eyre};
5 + use rusqlite::{Connection, Row, params};
6 +
7 + use crate::types::{NewTicket, Priority, Status, Ticket};
8 +
9 + /// Open (or create) the WAM database and run migrations.
10 + pub fn open_db() -> Result<Connection> {
11 + let dirs = directories::ProjectDirs::from("work", "makenot", "wam")
12 + .ok_or_else(|| eyre!("cannot determine data directory"))?;
13 + let data_dir = dirs.data_dir();
14 + std::fs::create_dir_all(data_dir)
15 + .wrap_err_with(|| format!("create data dir: {}", data_dir.display()))?;
16 +
17 + let db_path = data_dir.join("wam.db");
18 + let conn = Connection::open(&db_path)
19 + .wrap_err_with(|| format!("open database: {}", db_path.display()))?;
20 +
21 + conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;")?;
22 + migrate(&conn)?;
23 + Ok(conn)
24 + }
25 +
26 + /// Open an in-memory database for testing.
27 + #[cfg(test)]
28 + pub fn open_memory() -> Result<Connection> {
29 + let conn = Connection::open_in_memory()?;
30 + migrate(&conn)?;
31 + Ok(conn)
32 + }
33 +
34 + fn migrate(conn: &Connection) -> Result<()> {
35 + conn.execute_batch(
36 + "CREATE TABLE IF NOT EXISTS tickets (
37 + id TEXT PRIMARY KEY,
38 + title TEXT NOT NULL,
39 + body TEXT,
40 + priority TEXT NOT NULL DEFAULT 'medium',
41 + status TEXT NOT NULL DEFAULT 'open',
42 + source TEXT,
43 + source_ref TEXT,
44 + created_at TEXT NOT NULL,
45 + updated_at TEXT NOT NULL,
46 + resolved_at TEXT
47 + )",
48 + )?;
49 + Ok(())
50 + }
51 +
52 + fn row_to_ticket(row: &Row) -> rusqlite::Result<Ticket> {
53 + let priority_str: String = row.get("priority")?;
54 + let status_str: String = row.get("status")?;
55 + let created_str: String = row.get("created_at")?;
56 + let updated_str: String = row.get("updated_at")?;
57 + let resolved_str: Option<String> = row.get("resolved_at")?;
58 +
59 + Ok(Ticket {
60 + id: row.get("id")?,
61 + title: row.get("title")?,
62 + body: row.get("body")?,
63 + priority: priority_str.parse().unwrap_or(Priority::Medium),
64 + status: status_str.parse().unwrap_or(Status::Open),
65 + source: row.get("source")?,
66 + source_ref: row.get("source_ref")?,
67 + created_at: DateTime::parse_from_rfc3339(&created_str)
68 + .map(|dt| dt.with_timezone(&Utc))
69 + .unwrap_or_else(|_| Utc::now()),
70 + updated_at: DateTime::parse_from_rfc3339(&updated_str)
71 + .map(|dt| dt.with_timezone(&Utc))
72 + .unwrap_or_else(|_| Utc::now()),
73 + resolved_at: resolved_str.and_then(|s| {
74 + DateTime::parse_from_rfc3339(&s)
75 + .map(|dt| dt.with_timezone(&Utc))
76 + .ok()
77 + }),
78 + })
79 + }
80 +
81 + /// Create a new ticket. Returns the created ticket.
82 + pub fn create_ticket(conn: &Connection, new: &NewTicket) -> Result<Ticket> {
83 + let id = uuid::Uuid::new_v4().to_string();
84 + let now = Utc::now().to_rfc3339();
85 +
86 + conn.execute(
87 + "INSERT INTO tickets (id, title, body, priority, status, source, source_ref, created_at, updated_at)
88 + VALUES (?1, ?2, ?3, ?4, 'open', ?5, ?6, ?7, ?7)",
89 + params![
90 + id,
91 + new.title,
92 + new.body,
93 + new.priority.to_string(),
94 + new.source,
95 + new.source_ref,
96 + now,
97 + ],
98 + )?;
99 +
100 + get_ticket(conn, &id)
101 + }
102 +
103 + /// Get a ticket by exact ID or unique prefix match.
104 + pub fn get_ticket(conn: &Connection, id_prefix: &str) -> Result<Ticket> {
105 + let mut stmt = conn.prepare(
106 + "SELECT * FROM tickets WHERE id LIKE ?1 || '%'",
107 + )?;
108 + let tickets: Vec<Ticket> = stmt
109 + .query_map(params![id_prefix], row_to_ticket)?
110 + .collect::<rusqlite::Result<Vec<_>>>()?;
111 +
112 + match tickets.len() {
113 + 0 => Err(eyre!("no ticket matching '{id_prefix}'")),
114 + 1 => Ok(tickets.into_iter().next().unwrap()),
115 + n => Err(eyre!("ambiguous prefix '{id_prefix}' matches {n} tickets")),
116 + }
117 + }
118 +
119 + /// Filter criteria for listing tickets.
120 + #[derive(Default)]
121 + pub struct ListFilter<'a> {
122 + pub status: Option<Status>,
123 + pub priority: Option<Priority>,
124 + pub source: Option<&'a str>,
125 + pub search: Option<&'a str>,
126 + }
127 +
128 + /// List tickets with optional filters, ordered by priority (desc) then date (desc).
129 + pub fn list_tickets(conn: &Connection, filter: &ListFilter) -> Result<Vec<Ticket>> {
130 + let mut sql = String::from("SELECT * FROM tickets WHERE 1=1");
131 + let mut bind_values: Vec<String> = Vec::new();
132 +
133 + if let Some(status) = filter.status {
134 + bind_values.push(status.to_string());
135 + sql.push_str(&format!(" AND status = ?{}", bind_values.len()));
136 + }
137 + if let Some(priority) = filter.priority {
138 + bind_values.push(priority.to_string());
139 + sql.push_str(&format!(" AND priority = ?{}", bind_values.len()));
140 + }
141 + if let Some(source) = filter.source {
142 + bind_values.push(source.to_string());
143 + sql.push_str(&format!(" AND source = ?{}", bind_values.len()));
144 + }
145 + if let Some(search) = filter.search {
146 + bind_values.push(format!("%{search}%"));
147 + sql.push_str(&format!(" AND title LIKE ?{}", bind_values.len()));
148 + }
149 +
150 + sql.push_str(
151 + " ORDER BY CASE priority
152 + WHEN 'critical' THEN 3
153 + WHEN 'high' THEN 2
154 + WHEN 'medium' THEN 1
155 + WHEN 'low' THEN 0
156 + ELSE 1
157 + END DESC, created_at DESC",
158 + );
159 +
160 + let mut stmt = conn.prepare(&sql)?;
161 + let params_refs: Vec<&dyn rusqlite::types::ToSql> =
162 + bind_values.iter().map(|v| v as &dyn rusqlite::types::ToSql).collect();
163 + let tickets = stmt
164 + .query_map(params_refs.as_slice(), row_to_ticket)?
165 + .collect::<rusqlite::Result<Vec<_>>>()?;
166 +
167 + Ok(tickets)
168 + }
169 +
170 + /// Update a ticket's status. Sets resolved_at when moving to Resolved.
171 + pub fn update_status(conn: &Connection, id: &str, status: Status) -> Result<()> {
172 + let now = Utc::now().to_rfc3339();
173 + let resolved_at = if status == Status::Resolved {
174 + Some(now.clone())
175 + } else {
176 + None
177 + };
178 +
179 + let rows = conn.execute(
180 + "UPDATE tickets SET status = ?1, updated_at = ?2, resolved_at = ?3 WHERE id = ?4",
181 + params![status.to_string(), now, resolved_at, id],
182 + )?;
183 +
184 + if rows == 0 {
185 + return Err(eyre!("no ticket with id '{id}'"));
186 + }
187 + Ok(())
188 + }
189 +
190 + #[cfg(test)]
191 + mod tests {
192 + use super::*;
193 +
194 + fn test_new_ticket(title: &str) -> NewTicket {
195 + NewTicket {
196 + title: title.to_string(),
197 + body: None,
198 + priority: Priority::Medium,
199 + source: Some("test".to_string()),
200 + source_ref: None,
201 + }
202 + }
203 +
204 + #[test]
205 + fn create_and_get() {
206 + let conn = open_memory().unwrap();
207 + let t = create_ticket(&conn, &test_new_ticket("fix the thing")).unwrap();
208 + assert_eq!(t.title, "fix the thing");
209 + assert_eq!(t.status, Status::Open);
210 +
211 + let fetched = get_ticket(&conn, &t.id).unwrap();
212 + assert_eq!(fetched.id, t.id);
213 + }
214 +
215 + #[test]
216 + fn prefix_match() {
217 + let conn = open_memory().unwrap();
218 + let t = create_ticket(&conn, &test_new_ticket("test")).unwrap();
219 + let fetched = get_ticket(&conn, &t.id[..8]).unwrap();
220 + assert_eq!(fetched.id, t.id);
221 + }
222 +
223 + #[test]
224 + fn list_with_filter() {
225 + let conn = open_memory().unwrap();
226 + create_ticket(&conn, &NewTicket {
227 + title: "urgent".into(),
228 + body: None,
229 + priority: Priority::Critical,
230 + source: Some("pom".into()),
231 + source_ref: None,
232 + }).unwrap();
233 + create_ticket(&conn, &test_new_ticket("normal")).unwrap();
234 +
235 + let all = list_tickets(&conn, &ListFilter::default()).unwrap();
236 + assert_eq!(all.len(), 2);
237 + // Critical should sort first
238 + assert_eq!(all[0].title, "urgent");
239 +
240 + let critical_only = list_tickets(&conn, &ListFilter {
241 + priority: Some(Priority::Critical),
242 + ..Default::default()
243 + }).unwrap();
244 + assert_eq!(critical_only.len(), 1);
245 +
246 + let pom_only = list_tickets(&conn, &ListFilter {
247 + source: Some("pom"),
248 + ..Default::default()
249 + }).unwrap();
250 + assert_eq!(pom_only.len(), 1);
251 + }
252 +
253 + #[test]
254 + fn update_status_sets_resolved_at() {
255 + let conn = open_memory().unwrap();
256 + let t = create_ticket(&conn, &test_new_ticket("resolve me")).unwrap();
257 + assert!(t.resolved_at.is_none());
258 +
259 + update_status(&conn, &t.id, Status::Resolved).unwrap();
260 + let updated = get_ticket(&conn, &t.id).unwrap();
261 + assert_eq!(updated.status, Status::Resolved);
262 + assert!(updated.resolved_at.is_some());
263 + }
264 +
265 + #[test]
266 + fn search_filter() {
267 + let conn = open_memory().unwrap();
268 + create_ticket(&conn, &test_new_ticket("refund issue")).unwrap();
269 + create_ticket(&conn, &test_new_ticket("build failure")).unwrap();
270 +
271 + let results = list_tickets(&conn, &ListFilter {
272 + search: Some("refund"),
273 + ..Default::default()
274 + }).unwrap();
275 + assert_eq!(results.len(), 1);
276 + assert_eq!(results[0].title, "refund issue");
277 + }
278 + }