Skip to main content

max / makenotwork

pom: deliver failure alerts by email when wam_url is unset Failure alerts route to WAM, recoveries to email. wam_url is an Option with no default, so a missing config line silently disabled the ENTIRE down-alert channel while recovery emails kept firing — 'recovered' for outages that were never reported (fuzz-2026-07-06 CRITICAL #2). A monitoring tool must never swallow a down-alert: wam_ticket now falls back to email when wam_url is unset (and on WAM delivery failure), and both send paths return whether delivery succeeded so the retry layer can act on it. Warn once at startup when wam_url is absent so the fallback is visible. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-07 15:47 UTC
Commit: 97d708f84b0a153d712cde22352e5000348a65d8
Parent: 723d84a
1 file changed, +47 insertions, -7 deletions
M pom/src/alerts.rs +47 -7
@@ -60,6 +60,12 @@ impl Alerter {
60 60 .build()
61 61 .unwrap_or_default();
62 62 let wam_url = config.wam_url.clone();
63 + if wam_url.is_none() {
64 + warn!(
65 + "alerts: wam_url not configured — failure alerts will be delivered by email \
66 + instead of WAM tickets (set alerts.wam_url to route them to WAM)"
67 + );
68 + }
63 69 Self { config, client, pool, instance_name, wam_url }
64 70 }
65 71
@@ -755,11 +761,13 @@ impl Alerter {
755 761 elapsed.num_seconds() < self.config.cooldown_secs as i64
756 762 }
757 763
758 - async fn send_email(&self, subject: &str, body: &str) {
764 + /// Send an email via Postmark. Returns `true` if the message was accepted
765 + /// (or in dev mode where there is no token — the operator sees it logged).
766 + async fn send_email(&self, subject: &str, body: &str) -> bool {
759 767 let Some(ref token) = self.config.postmark_token else {
760 768 info!("[dev] alert: {subject}");
761 769 info!("[dev] {body}");
762 - return;
770 + return true;
763 771 };
764 772
765 773 let payload = serde_json::json!({
@@ -783,17 +791,21 @@ impl Alerter {
783 791 match tokio::time::timeout(std::time::Duration::from_secs(30), send_fut).await {
784 792 Ok(Ok(resp)) if resp.status().is_success() => {
785 793 info!("alert sent: {subject}");
794 + true
786 795 }
787 796 Ok(Ok(resp)) => {
788 797 let status = resp.status();
789 798 let text = resp.text().await.unwrap_or_default();
790 799 warn!("postmark error ({status}): {text}");
800 + false
791 801 }
792 802 Ok(Err(e)) => {
793 803 warn!("failed to send alert: {e}");
804 + false
794 805 }
795 806 Err(_) => {
796 807 warn!("alert send timed out after 30s: {subject}");
808 + false
797 809 }
798 810 }
799 811 }
@@ -812,7 +824,15 @@ impl Alerter {
812 824 }
813 825 }
814 826
815 - /// Create a WAM ticket (best-effort, fire-and-forget).
827 + /// Create a WAM ticket for a failure alert. Returns `true` if the alert was
828 + /// delivered by *some* channel.
829 + ///
830 + /// If `wam_url` is unset the failure alert would otherwise be dropped on the
831 + /// floor — failures route to WAM, recoveries to email, so a missing `wam_url`
832 + /// silently disabled the entire down-alert channel while recovery emails kept
833 + /// firing (fuzz-2026-07-06 CRITICAL #2). Fall back to email instead: a
834 + /// monitoring tool must never silently swallow a down-alert. WAM delivery
835 + /// failures also fall back to email so a WAM outage can't blind the operator.
816 836 async fn wam_ticket(
817 837 &self,
818 838 title: &str,
@@ -820,8 +840,12 @@ impl Alerter {
820 840 priority: &str,
821 841 source: &str,
822 842 source_ref: Option<&str>,
823 - ) {
824 - let Some(ref base_url) = self.wam_url else { return };
843 + ) -> bool {
844 + let Some(ref base_url) = self.wam_url else {
845 + // No WAM configured: deliver the failure alert by email instead of
846 + // dropping it.
847 + return self.send_email(title, body).await;
848 + };
825 849 let url = format!("{base_url}/tickets");
826 850
827 851 let mut payload = serde_json::json!({
@@ -837,12 +861,15 @@ impl Alerter {
837 861 match self.client.post(&url).json(&payload).send().await {
838 862 Ok(resp) if resp.status().is_success() => {
839 863 info!("WAM ticket created: {title}");
864 + true
840 865 }
841 866 Ok(resp) => {
842 - warn!("WAM ticket creation returned {}: {title}", resp.status());
867 + warn!("WAM ticket creation returned {}: {title}; falling back to email", resp.status());
868 + self.send_email(title, body).await
843 869 }
844 870 Err(e) => {
845 - warn!("WAM unreachable: {e}");
871 + warn!("WAM unreachable: {e}; falling back to email");
872 + self.send_email(title, body).await
846 873 }
847 874 }
848 875 }
@@ -881,6 +908,19 @@ mod tests {
881 908 }
882 909
883 910 #[tokio::test]
911 + async fn wam_ticket_falls_back_to_email_when_wam_url_unset() {
912 + // CRITICAL #2: with no wam_url, a failure alert must NOT be silently
913 + // dropped — it falls back to email. test_alerter has wam_url: None and
914 + // postmark_token: None (dev mode), so the email path reports delivered.
915 + let pool = db::connect_in_memory().await.unwrap();
916 + let alerter = test_alerter(pool);
917 + assert!(
918 + alerter.wam_ticket("subj", "body", "high", "pom-test", None).await,
919 + "a failure alert with no wam_url must be delivered by email, not dropped"
920 + );
921 + }
922 +
923 + #[tokio::test]
884 924 async fn cooldown_does_not_affect_other_targets() {
885 925 let pool = db::connect_in_memory().await.unwrap();
886 926 let alerter = test_alerter(pool.clone());