Add WAM (Whack-a-Mole) ticket manager with 25 proactive ticket sources New internal tool: ratatui TUI + HTTP API for operational ticket management. SQLite-backed, tailnet-secured (no auth layer needed). WAM features (v0.1.0): - CLI: create, list, show, resolve, close (git-style prefix matching) - TUI: priority-colored list, detail view, inline status changes, search, filter by status/priority/source, create popup - HTTP API: POST/GET/PATCH /tickets (axum, for programmatic integration) MNW server integration (13 ticket sources): - User-facing: license key gen failed, Fan+ credit failed, file quarantined, subscription payment failed, Stripe Connect degraded, build failed - Infrastructure: health status change, DB pool pressure - Periodic: sales count drift, stale subscriptions, email bounce spike - Existing: refund escalation, webhook dead letter PoM integration (12 ticket sources): - All non-recovery alerts migrated from email to WAM tickets - Health, TLS, peer, route, DNS, WHOIS, CORS, latency, backup, monitoring - Recovery alerts remain email-only
- Co-Authored-By
Author: Max J. <87768334+MaxJMath@users.noreply.github.com> - 2026-04-25 20:07 UTC
Commit:
5b85d927947d9438adc614e3ff817554d2798aebParent:
23 files changed,
+3681 insertions,
-15 deletions
OldNewLine
@@ -16,6 +16,7 @@
16
16
client: reqwest::Client,17
17
pool: SqlitePool,18
18
instance_name: String,19
wam_url: Option<String>,19
20
}20
21
21
22
impl Alerter {@@ -24,7 +25,8 @@
24
25
.timeout(std::time::Duration::from_secs(10))25
26
.build()26
27
.unwrap_or_default();27
Self { config, client, pool, instance_name }28
let wam_url = config.wam_url.clone();29
Self { config, client, pool, instance_name, wam_url }28
30
}29
31
30
32
#[instrument(skip_all)]@@ -56,7 +58,12 @@
56
58
}57
59
body.push_str("\n- PoM");58
60
59
self.send_email(&subject, &body).await;61
let priority = match to_status {62
"error" | "unreachable" => "critical",63
"degraded" => "high",64
_ => "medium",65
};66
self.wam_ticket(&subject, &body, priority, "pom-health", Some(target)).await;60
67
self.record_alert(&alert_key, AlertCategory::Health, Some(from_status), Some(to_status), error).await;61
68
}62
69
@@ -111,7 +118,8 @@
111
118
chrono::Utc::now().to_rfc3339(),112
119
);113
120
114
self.send_email(&subject, &body).await;121
let priority = if days_remaining <= 3 { "critical" } else if days_remaining <= 7 { "high" } else { "medium" };122
self.wam_ticket(&subject, &body, priority, "pom-tls", Some(&format!("{target}:{host}"))).await;115
123
self.record_alert(&alert_key, AlertCategory::TlsExpiry, None, None, None).await;116
124
}117
125
@@ -140,7 +148,7 @@
140
148
chrono::Utc::now().to_rfc3339(),141
149
);142
150
143
self.send_email(&subject, &body).await;151
self.wam_ticket(&subject, &body, "high", "pom-tls", Some(&format!("{target}:{host}"))).await;144
152
self.record_alert(&alert_key, AlertCategory::TlsError, None, None, Some(error)).await;145
153
}146
154
@@ -193,7 +201,7 @@
193
201
chrono::Utc::now().to_rfc3339(),194
202
);195
203
196
self.send_email(&subject, &body).await;204
self.wam_ticket(&subject, &body, "high", "pom-peer", Some(peer_name)).await;197
205
self.record_alert(&alert_key, AlertCategory::PeerMissing, None, None, None).await;198
206
}199
207
@@ -245,7 +253,7 @@
245
253
chrono::Utc::now().to_rfc3339(),246
254
);247
255
248
self.send_email(&subject, &body).await;256
self.wam_ticket(&subject, &body, "high", "pom-routes", Some(target)).await;249
257
self.record_alert(&alert_key, AlertCategory::RouteFailure, None, None, None).await;250
258
}251
259
@@ -313,7 +321,7 @@
313
321
chrono::Utc::now().to_rfc3339(),314
322
);315
323
316
self.send_email(&subject, &body).await;324
self.wam_ticket(&subject, &body, "high", "pom-dns", Some(target)).await;317
325
self.record_alert(&alert_key, AlertCategory::DnsMismatch, None, None, None).await;318
326
}319
327
@@ -366,7 +374,8 @@
366
374
chrono::Utc::now().to_rfc3339(),367
375
);368
376
369
self.send_email(&subject, &body).await;377
let priority = if days_remaining <= 7 { "critical" } else if days_remaining <= 14 { "high" } else { "medium" };378
self.wam_ticket(&subject, &body, priority, "pom-whois", Some(&format!("{target}:{domain}"))).await;370
379
self.record_alert(&alert_key, AlertCategory::WhoisExpiry, None, None, None).await;371
380
}372
381
@@ -396,7 +405,7 @@
396
405
chrono::Utc::now().to_rfc3339(),397
406
);398
407
399
self.send_email(&subject, &body).await;408
self.wam_ticket(&subject, &body, "high", "pom-whois", Some(&format!("{target}:{domain}"))).await;400
409
self.record_alert(&alert_key, AlertCategory::WhoisError, None, None, Some(error)).await;401
410
}402
411
@@ -437,7 +446,7 @@
437
446
chrono::Utc::now().to_rfc3339(),438
447
);439
448
440
self.send_email(&subject, &body).await;449
self.wam_ticket(&subject, &body, "high", "pom-cors", Some(target)).await;441
450
self.record_alert(&alert_key, AlertCategory::CorsFailure, None, None, None).await;442
451
}443
452
@@ -488,7 +497,7 @@
488
497
chrono::Utc::now().to_rfc3339(),489
498
);490
499
491
self.send_email(&subject, &body).await;500
self.wam_ticket(&subject, &body, "medium", "pom-latency", Some(target)).await;492
501
self.record_alert(&alert_key, AlertCategory::LatencyDrift, None, None, Some(drift_message)).await;493
502
}494
503
@@ -539,7 +548,7 @@
539
548
chrono::Utc::now().to_rfc3339(),540
549
);541
550
542
self.send_email(&subject, &body).await;551
self.wam_ticket(&subject, &body, "medium", "pom-test-duration", Some(target)).await;543
552
self.record_alert(&alert_key, AlertCategory::TestDurationDrift, None, None, Some(drift_message)).await;544
553
}545
554
@@ -578,7 +587,8 @@
578
587
chrono::Utc::now().to_rfc3339(),579
588
);580
589
581
self.send_email(&subject, &body).await;590
let priority = if status == "missing" { "critical" } else { "high" };591
self.wam_ticket(&subject, &body, priority, "pom-backup", Some(&format!("{target}:{database}"))).await;582
592
self.record_alert(&alert_key, AlertCategory::BackupStale, None, Some(status), None).await;583
593
}584
594
@@ -628,7 +638,7 @@
628
638
chrono::Utc::now().to_rfc3339(),629
639
);630
640
631
self.send_email(&subject, &body).await;641
self.wam_ticket(&subject, &body, "critical", "pom-monitoring", Some("self")).await;632
642
self.record_alert(alert_key, AlertCategory::MonitoringOffline, None, None, None).await;633
643
}634
644
@@ -721,6 +731,41 @@
721
731
warn!("failed to record alert: {e}");722
732
}723
733
}734
735
/// Create a WAM ticket (best-effort, fire-and-forget).736
async fn wam_ticket(737
&self,738
title: &str,739
body: &str,740
priority: &str,741
source: &str,742
source_ref: Option<&str>,743
) {744
let Some(ref base_url) = self.wam_url else { return };745
let url = format!("{base_url}/tickets");746
747
let mut payload = serde_json::json!({748
"title": title,749
"body": body,750
"priority": priority,751
"source": source,752
});753
if let Some(r) = source_ref {754
payload["source_ref"] = serde_json::json!(r);755
}756
757
match self.client.post(&url).json(&payload).send().await {758
Ok(resp) if resp.status().is_success() => {759
info!("WAM ticket created: {title}");760
}761
Ok(resp) => {762
warn!("WAM ticket creation returned {}: {title}", resp.status());763
}764
Err(e) => {765
warn!("WAM unreachable: {e}");766
}767
}768
}724
769
}725
770
726
771
#[cfg(test)]@@ -733,6 +778,7 @@
733
778
to: "test@example.com".to_string(),734
779
from: "PoM Alerts <pom-alerts@makenot.work>".to_string(),735
780
cooldown_secs: 300,781
wam_url: None,736
782
};737
783
Alerter::new(config, pool, "test-instance".to_string())738
784
}OldNewLine
@@ -38,6 +38,8 @@
38
38
/// Minimum seconds between repeated alerts for the same target.39
39
#[serde(default = "default_cooldown_secs")]40
40
pub cooldown_secs: u64,41
/// WAM ticket manager URL (tailnet). When set, alerts also create WAM tickets.42
pub wam_url: Option<String>,41
43
}42
44
43
45
#[derive(Debug, Clone, Default, Deserialize)]OldNewLine
@@ -1926,6 +1926,7 @@
1926
1926
to: "test@example.com".to_string(),1927
1927
from: "PoM Alerts <pom@test.com>".to_string(),1928
1928
cooldown_secs: 300,1929
wam_url: None,1929
1930
};1930
1931
let alerter = pom::alerts::Alerter::new(config, pool.clone(), "test".to_string());1931
1932
OldNewLine
@@ -427,6 +427,7 @@
427
427
postmark_inbound_webhook_token: None,428
428
internal_shared_secret: None,429
429
cli_service_token: None,430
wam_url: None,430
431
};431
432
assert!(require_admin(&user, &config).is_ok());432
433
}@@ -487,6 +488,7 @@
487
488
postmark_inbound_webhook_token: None,488
489
internal_shared_secret: None,489
490
cli_service_token: None,491
wam_url: None,490
492
};491
493
assert!(require_admin(&user, &config).is_err());492
494
}OldNewLine
@@ -246,6 +246,18 @@
246
246
tracing::error!(build_id = %build.id, status = %status, error = ?e, "failed to update final build status");247
247
}248
248
249
if status == BuildStatus::Failed {250
if let Some(ref wam) = state.wam {251
let title = format!("Build failed: {} v{}", build.tag, build.version);252
let body = format!(253
"Build {} for {} v{} failed.\n\nError: {}",254
build.id, build.tag, build.version,255
first_error.as_deref().unwrap_or("unknown"),256
);257
wam.create_ticket(&title, Some(&body), "high", "build-failed", Some(&build.id.to_string())).await;258
}259
}260
249
261
tracing::info!(250
262
build_id = %build.id,251
263
version = %build.version,OldNewLine
@@ -64,6 +64,9 @@
64
64
/// Bearer token for authenticating CLI SSH server → MNW internal API calls.65
65
/// When unset, internal API endpoints return 503.66
66
pub cli_service_token: Option<String>,67
/// Base URL of the WAM ticket manager (e.g., "http://100.x.x.x:7890").68
/// When set, operational events create WAM tickets for human triage.69
pub wam_url: Option<String>,67
70
}68
71
69
72
/// S3-compatible storage configuration (Hetzner Object Storage)@@ -187,6 +190,9 @@
187
190
// CLI service token for SSH server → internal API authentication188
191
let cli_service_token = std::env::var("CLI_SERVICE_TOKEN").ok();189
192
193
// WAM ticket manager URL (tailnet, e.g. "http://100.x.x.x:7890")194
let wam_url = std::env::var("WAM_URL").ok();195
190
196
Ok(Config {191
197
host,192
198
port,@@ -213,6 +219,7 @@
213
219
postmark_inbound_webhook_token,214
220
internal_shared_secret,215
221
cli_service_token,222
wam_url,216
223
})217
224
}218
225
@@ -483,6 +490,7 @@
483
490
postmark_inbound_webhook_token: None,484
491
internal_shared_secret: None,485
492
cli_service_token: None,493
wam_url: None,486
494
};487
495
let addr = config.socket_addr();488
496
assert_eq!(addr.port(), 8080);OldNewLine
@@ -16,6 +16,7 @@
16
16
pub mod metrics;17
17
pub mod monitor;18
18
pub mod mt_client;19
pub mod wam_client;19
20
pub mod payments;20
21
pub mod pricing;21
22
pub mod scheduler;@@ -76,6 +77,8 @@
76
77
pub session_cache: Arc<DashMap<UserSessionId, Instant>>,77
78
/// HTTP client for the Multithreaded internal API (community/thread provisioning).78
79
pub mt_client: Option<mt_client::MtClient>,80
/// HTTP client for the WAM ticket manager (operational alerts).81
pub wam: Option<wam_client::WamClient>,79
82
/// Cache of verified custom domains → user IDs (populated on startup, updated on verify/delete).80
83
pub domain_cache: Arc<DashMap<String, db::UserId>>,81
84
/// Unix timestamp when the server will restart (0 = no restart pending).OldNewLine
@@ -218,6 +218,12 @@
218
218
}219
219
};220
220
221
// WAM ticket manager client (tailnet-only, for operational alerts)222
let wam = config.wam_url.as_ref().map(|url| {223
tracing::info!(url = %url, "WAM integration enabled");224
makenotwork::wam_client::WamClient::new(url.clone())225
});226
221
227
// Warm custom domain cache222
228
let domain_cache = std::sync::Arc::new(dashmap::DashMap::new());223
229
match makenotwork::db::custom_domains::get_all_verified_domains(&db).await {@@ -252,6 +258,7 @@
252
258
start_instant,253
259
session_cache: std::sync::Arc::new(dashmap::DashMap::new()),254
260
mt_client,261
wam,255
262
domain_cache,256
263
restart_at: std::sync::Arc::new(std::sync::atomic::AtomicI64::new(0)),257
264
sync_notify: std::sync::Arc::new(dashmap::DashMap::new()),OldNewLine
@@ -173,9 +173,40 @@
173
173
}174
174
}175
175
176
// Create WAM ticket on degradation/error transitions177
if snap.status != MonitorStatus::Operational {178
if let Some(ref wam) = state.wam {179
let priority = match snap.status {180
MonitorStatus::Error => "critical",181
MonitorStatus::Degraded => "high",182
MonitorStatus::Operational => unreachable!(),183
};184
let title = format!("Health status: {}", snap.status.as_str());185
let body = format!(186
"db: {}\ns3: {}\nsessions: {}\ncheck_ms: {}",187
snap.db_ok, snap.s3_ok, snap.sessions_ok, snap.check_duration_ms,188
);189
wam.create_ticket(&title, Some(&body), priority, "health-status-change", None).await;190
}191
}192
176
193
previous_status = Some(snap.status);177
194
}178
195
196
// DB pool pressure check (>80% active connections)197
{198
let pool_size = state.db.size();199
let pool_idle = state.db.num_idle() as u32;200
let active = pool_size.saturating_sub(pool_idle);201
if pool_size > 0 && active * 100 / pool_size > 80 {202
tracing::warn!(pool_size, active, idle = pool_idle, "DB pool pressure >80%");203
if let Some(ref wam) = state.wam {204
let title = format!("DB pool pressure: {active}/{pool_size} active");205
wam.create_ticket(&title, None, "high", "db-pool-pressure", None).await;206
}207
}208
}209
179
210
// Persist snapshot (best-effort)180
211
if let Err(e) = db::monitor::insert_health_history(181
212
&state.db,OldNewLine
@@ -392,14 +392,134 @@
392
392
// Retry failed webhook events393
393
retry_failed_webhooks(&state).await;394
394
395
// Weekly storage drift correction395
// Escalate stale pending refunds (unmatched for >24 hours)396
escalate_stale_refunds(&state).await;397
398
// Weekly storage drift correction + integrity checks396
399
if tick_count.is_multiple_of(DRIFT_CORRECTION_INTERVAL) {397
400
recalculate_all_storage_used(&state).await;401
check_sales_count_drift(&state).await;402
}403
404
// Daily checks (every 1440 ticks at 60s interval)405
if tick_count.is_multiple_of(1440) {406
check_stale_subscriptions(&state).await;407
check_email_bounce_spike(&state).await;398
408
}399
409
}400
410
})401
411
}402
412
413
// ============================================================================414
// Periodic integrity checks415
// ============================================================================416
417
/// Detect items where denormalized sales_count has drifted from actual transaction count.418
async fn check_sales_count_drift(state: &AppState) {419
let rows = match sqlx::query_as::<_, (db::ItemId, i32, i64)>(420
r#"421
SELECT i.id, i.sales_count, COUNT(t.id)422
FROM items i423
LEFT JOIN transactions t ON t.item_id = i.id AND t.status = 'completed'424
GROUP BY i.id425
HAVING i.sales_count != COUNT(t.id)426
LIMIT 50427
"#,428
)429
.fetch_all(&state.db)430
.await431
{432
Ok(r) if r.is_empty() => return,433
Ok(r) => r,434
Err(e) => {435
tracing::error!(error = ?e, "sales count drift check failed");436
return;437
}438
};439
440
tracing::warn!(count = rows.len(), "sales count drift detected");441
442
if let Some(ref wam) = state.wam {443
let items: Vec<String> = rows444
.iter()445
.map(|(id, cached, actual)| format!(" {id}: cached={cached}, actual={actual}"))446
.collect();447
let body = format!("Items with drifted sales_count:\n{}", items.join("\n"));448
wam.create_ticket(449
&format!("Sales count drift: {} items", rows.len()),450
Some(&body),451
"medium",452
"sales-count-drift",453
None,454
)455
.await;456
}457
}458
459
/// Find subscriptions stuck in past_due for >7 days (possible missed webhook).460
async fn check_stale_subscriptions(state: &AppState) {461
let count: i64 = match sqlx::query_scalar(462
r#"463
SELECT COUNT(*) FROM (464
SELECT 1 FROM creator_subscriptions WHERE status = 'past_due' AND updated_at < NOW() - INTERVAL '7 days'465
UNION ALL466
SELECT 1 FROM subscriptions WHERE status = 'past_due' AND updated_at < NOW() - INTERVAL '7 days'467
) stale468
"#,469
)470
.fetch_one(&state.db)471
.await472
{473
Ok(c) => c,474
Err(e) => {475
tracing::error!(error = ?e, "stale subscription check failed");476
return;477
}478
};479
480
if count > 0 {481
tracing::warn!(count, "stale past_due subscriptions detected");482
if let Some(ref wam) = state.wam {483
wam.create_ticket(484
&format!("{count} subscriptions past_due >7 days"),485
Some("Subscriptions stuck in past_due for over 7 days. A Stripe webhook may have been missed. Check the Stripe dashboard."),486
"medium",487
"subscription-stale-past-due",488
None,489
).await;490
}491
}492
}493
494
/// Detect email bounce/complaint spikes (>10 suppressions in 24h).495
async fn check_email_bounce_spike(state: &AppState) {496
let count: i64 = match sqlx::query_scalar(497
"SELECT COUNT(*) FROM email_suppressions WHERE created_at > NOW() - INTERVAL '24 hours'",498
)499
.fetch_one(&state.db)500
.await501
{502
Ok(c) => c,503
Err(e) => {504
tracing::error!(error = ?e, "email bounce spike check failed");505
return;506
}507
};508
509
if count > 10 {510
tracing::warn!(count, "email bounce/complaint spike");511
if let Some(ref wam) = state.wam {512
wam.create_ticket(513
&format!("Email bounce spike: {count} suppressions in 24h"),514
Some("Elevated bounce/complaint rate may indicate a deliverability problem. Check Postmark dashboard."),515
"high",516
"email-bounce-spike",517
None,518
).await;519
}520
}521
}522
403
523
// ============================================================================404
524
// Webhook retry405
525
// ============================================================================@@ -451,8 +571,10 @@
451
571
}452
572
}453
573
Err(e) => {574
let is_dead = attempt >= 5;454
575
tracing::warn!(455
576
event_id = %event.id, attempt = attempt, error = ?e,577
dead = is_dead,456
578
"webhook retry failed"457
579
);458
580
if let Err(e) = db::webhook_events::schedule_retry(@@ -460,11 +582,118 @@
460
582
).await {461
583
tracing::error!(error = ?e, "failed to schedule webhook retry");462
584
}585
586
// Create WAM ticket when retries are exhausted (dead letter)587
if is_dead {588
if let Some(ref wam) = state.wam {589
let title = format!(590
"Dead webhook: {} ({})",591
event.event_type, event.id592
);593
let body = format!(594
"Webhook event exhausted all {} retry attempts.\n\595
Source: {}\nType: {}\nLast error: {:?}",596
attempt, event.source, event.event_type, e,597
);598
wam.create_ticket(599
&title,600
Some(&body),601
"high",602
"webhook-dead-letter",603
Some(&event.id.to_string()),604
)605
.await;606
}607
}463
608
}464
609
}465
610
}466
611
}467
612
613
// ============================================================================614
// Pending refund escalation615
// ============================================================================616
617
/// Alert the admin about pending refunds that have gone unmatched for >24 hours.618
///619
/// These represent charge.refunded webhooks that arrived before their matching620
/// checkout.session.completed and never got resolved. Likely indicates a lost621
/// payment webhook that needs manual investigation.622
async fn escalate_stale_refunds(state: &AppState) {623
let stale = match db::pending_refunds::get_stale_refunds(624
&state.db,625
chrono::Duration::hours(24),626
)627
.await628
{629
Ok(s) if s.is_empty() => return,630
Ok(s) => s,631
Err(e) => {632
tracing::error!(error = ?e, "failed to query stale pending refunds");633
return;634
}635
};636
637
let alert_email = std::env::var("ALERT_EMAIL").ok();638
639
for refund in &stale {640
tracing::error!(641
payment_intent_id = %refund.payment_intent_id,642
amount = refund.amount,643
amount_refunded = refund.amount_refunded,644
created_at = %refund.created_at,645
"STALE PENDING REFUND: unmatched for >24h, needs manual investigation"646
);647
648
if let Some(ref to) = alert_email {649
let subject = format!(650
"Unmatched refund: {} ({}c refunded)",651
refund.payment_intent_id, refund.amount_refunded652
);653
let body = format!(654
"A charge.refunded webhook for payment intent {} has been pending for >24 hours \655
with no matching completed transaction.\n\n\656
Amount: {}c\nAmount refunded: {}c\nReceived: {}\n\n\657
This likely means the checkout.session.completed webhook was lost. \658
Check the Stripe dashboard and reconcile manually.",659
refund.payment_intent_id,660
refund.amount,661
refund.amount_refunded,662
refund.created_at,663
);664
if let Err(e) = state.email.send_alert(to, &subject, &body).await {665
tracing::error!(error = ?e, "failed to send stale refund alert email");666
}667
}668
669
// Create WAM ticket alongside the alert email670
if let Some(ref wam) = state.wam {671
let title = format!(672
"Unmatched refund: {} ({}c)",673
refund.payment_intent_id, refund.amount_refunded674
);675
let body = format!(676
"charge.refunded webhook pending >24h with no matching completed transaction.\n\677
Amount: {}c\nRefunded: {}c\nReceived: {}\n\678
Check Stripe dashboard and reconcile manually.",679
refund.amount, refund.amount_refunded, refund.created_at,680
);681
wam.create_ticket(682
&title,683
Some(&body),684
"critical",685
"refund-escalation",686
Some(&refund.payment_intent_id),687
)688
.await;689
}690
691
if let Err(e) = db::pending_refunds::mark_escalated(&state.db, refund.id).await {692
tracing::error!(error = ?e, "failed to mark pending refund as escalated");693
}694
}695
}696
468
697
// ============================================================================469
698
// MT thread provisioning helpers470
699
// ============================================================================