Skip to main content

max / makenotwork

6.1 KB · 181 lines History Blame Raw
1 //! Alerts that repeat weekly until someone confirms they read them.
2 //!
3 //! The row is opened by whoever noticed the thing (see
4 //! [`crate::db::acknowledgements`]); every send comes from here, including the
5 //! first. One send path means the opening caller cannot compose a different
6 //! first message from the one the repeats use, and a caller that forgets to
7 //! send at all still gets the alert delivered.
8 //!
9 //! Runs every tick rather than daily. The query is one bounded indexed scan
10 //! that returns nothing almost always, and the alternative is up to a day
11 //! between something going wrong and the first person hearing about it. The
12 //! weekly cadence is in the `WHERE`, not in how often the job wakes.
13
14 use crate::AppState;
15 use crate::constants::{
16 ACKNOWLEDGEMENT_ESCALATE_AFTER, ACKNOWLEDGEMENT_REPEAT_DAYS, ACKNOWLEDGEMENTS_PER_TICK,
17 };
18 use crate::db::acknowledgements::{self, PendingAcknowledgement, detail_copy};
19
20 /// One pass: fetch the due alerts and hand each to the background pool.
21 ///
22 /// Like the SyncKit warning job, the tick itself does only bounded DB work. A
23 /// slow mail provider times N due alerts must never block the other periodic
24 /// jobs, so nothing here awaits a send.
25 #[tracing::instrument(skip_all)]
26 pub(super) async fn send_due_acknowledgements(state: &AppState) {
27 let due = match acknowledgements::due(
28 &state.db,
29 ACKNOWLEDGEMENT_REPEAT_DAYS,
30 ACKNOWLEDGEMENTS_PER_TICK,
31 )
32 .await
33 {
34 Ok(d) => d,
35 Err(e) => {
36 tracing::error!(error = ?e, "acknowledgements: due query failed");
37 return;
38 }
39 };
40
41 if due.is_empty() {
42 return;
43 }
44 tracing::info!(count = due.len(), "acknowledgements: enqueuing");
45
46 for alert in due {
47 let state = state.clone();
48 state.clone().bg.spawn("acknowledgement-nag", async move {
49 process(&state, alert).await;
50 });
51 }
52 }
53
54 /// Send one alert, or escalate it if it has gone unanswered long enough.
55 ///
56 /// Escalation is checked before sending, so the count is the number of messages
57 /// that went unanswered rather than that number plus one.
58 async fn process(state: &AppState, alert: PendingAcknowledgement) {
59 if alert.notify_count >= ACKNOWLEDGEMENT_ESCALATE_AFTER {
60 escalate(state, &alert).await;
61 return;
62 }
63
64 let url = format!(
65 "{}/acknowledge/{}",
66 state.config.host_url,
67 alert.token(&state.config.signing_secret)
68 );
69 let detail = detail_copy(alert.kind, &alert.details);
70
71 if let Err(e) = state
72 .email
73 .send_acknowledgement_required(
74 &alert.email,
75 alert.display_name.as_deref(),
76 alert.kind.title(),
77 &detail,
78 &url,
79 alert.notify_count,
80 )
81 .await
82 {
83 // No stamp on failure, so this re-sends next pass rather than burning a
84 // message towards escalation on a provider outage.
85 tracing::error!(
86 error = ?e,
87 alert_id = %alert.id,
88 kind = %alert.kind,
89 "acknowledgements: send failed",
90 );
91 return;
92 }
93
94 if let Err(e) = acknowledgements::mark_notified(&state.db, alert.id).await {
95 // Logged rather than retried: the cost is one duplicate message next
96 // pass, which is much better than the alternative failure (stamping a
97 // send that did not happen and letting the row go quiet).
98 tracing::error!(
99 error = ?e,
100 alert_id = %alert.id,
101 "acknowledgements: stamp failed (the message will repeat next pass)",
102 );
103 }
104 }
105
106 /// Stop sending and ask a person to make contact.
107 ///
108 /// `escalate` reports whether this call was the one that moved the row, so
109 /// exactly one ticket is filed however many passes race here.
110 async fn escalate(state: &AppState, alert: &PendingAcknowledgement) {
111 match acknowledgements::escalate(&state.db, alert.id).await {
112 Ok(false) => return,
113 Ok(true) => {}
114 Err(e) => {
115 tracing::error!(error = ?e, alert_id = %alert.id, "acknowledgements: escalation failed");
116 return;
117 }
118 }
119
120 tracing::warn!(
121 alert_id = %alert.id,
122 user_id = %alert.user_id,
123 kind = %alert.kind,
124 sends = alert.notify_count,
125 "acknowledgement unanswered after every message; handing to a person",
126 );
127
128 let Some(wam) = state.wam.as_ref() else {
129 return;
130 };
131 let title = format!("Unacknowledged alert: {}", alert.kind);
132 let body = format!(
133 "{sends} messages went to {email} about this and none were confirmed.\n\n\
134 {detail}\n\n\
135 Automatic sending has stopped. Someone needs to contact them directly.",
136 sends = alert.notify_count,
137 email = alert.email,
138 detail = detail_copy(alert.kind, &alert.details),
139 );
140 wam.create_ticket(
141 &title,
142 Some(&body),
143 "high",
144 "acknowledgement-unanswered",
145 Some(&alert.user_id.to_string()),
146 )
147 .await;
148 }
149
150 #[cfg(test)]
151 mod tests {
152 /// The same constructive seal the SyncKit warning job carries: the tick must
153 /// never await a mail send, or one slow provider stalls every other periodic
154 /// job behind it.
155 #[test]
156 fn tick_never_sends_inline() {
157 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
158 .join("src/scheduler/acknowledgements.rs");
159 let src = std::fs::read_to_string(&path).expect("read acknowledgements source");
160
161 let start = src
162 .find("async fn send_due_acknowledgements")
163 .expect("tick fn present");
164 let rest = &src[start..];
165 let end = rest[1..]
166 .find("\nasync fn ")
167 .or_else(|| rest[1..].find("\nfn "))
168 .map_or(rest.len(), |i| i + 1);
169 let tick_body = &rest[..end];
170
171 assert!(
172 !tick_body.contains("send_acknowledgement_required"),
173 "the tick must not send inline; enqueue onto state.bg and send in process()",
174 );
175 assert!(
176 tick_body.contains("bg.spawn"),
177 "the tick must dispatch onto the bounded background pool",
178 );
179 }
180 }
181