//! Alerts that repeat weekly until someone confirms they read them. //! //! The row is opened by whoever noticed the thing (see //! [`crate::db::acknowledgements`]); every send comes from here, including the //! first. One send path means the opening caller cannot compose a different //! first message from the one the repeats use, and a caller that forgets to //! send at all still gets the alert delivered. //! //! Runs every tick rather than daily. The query is one bounded indexed scan //! that returns nothing almost always, and the alternative is up to a day //! between something going wrong and the first person hearing about it. The //! weekly cadence is in the `WHERE`, not in how often the job wakes. use crate::AppState; use crate::constants::{ ACKNOWLEDGEMENT_ESCALATE_AFTER, ACKNOWLEDGEMENT_REPEAT_DAYS, ACKNOWLEDGEMENTS_PER_TICK, }; use crate::db::acknowledgements::{self, PendingAcknowledgement, detail_copy}; /// One pass: fetch the due alerts and hand each to the background pool. /// /// Like the SyncKit warning job, the tick itself does only bounded DB work. A /// slow mail provider times N due alerts must never block the other periodic /// jobs, so nothing here awaits a send. #[tracing::instrument(skip_all)] pub(super) async fn send_due_acknowledgements(state: &AppState) { let due = match acknowledgements::due( &state.db, ACKNOWLEDGEMENT_REPEAT_DAYS, ACKNOWLEDGEMENTS_PER_TICK, ) .await { Ok(d) => d, Err(e) => { tracing::error!(error = ?e, "acknowledgements: due query failed"); return; } }; if due.is_empty() { return; } tracing::info!(count = due.len(), "acknowledgements: enqueuing"); for alert in due { let state = state.clone(); state.clone().bg.spawn("acknowledgement-nag", async move { process(&state, alert).await; }); } } /// Send one alert, or escalate it if it has gone unanswered long enough. /// /// Escalation is checked before sending, so the count is the number of messages /// that went unanswered rather than that number plus one. async fn process(state: &AppState, alert: PendingAcknowledgement) { if alert.notify_count >= ACKNOWLEDGEMENT_ESCALATE_AFTER { escalate(state, &alert).await; return; } let url = format!( "{}/acknowledge/{}", state.config.host_url, alert.token(&state.config.signing_secret) ); let detail = detail_copy(alert.kind, &alert.details); if let Err(e) = state .email .send_acknowledgement_required( &alert.email, alert.display_name.as_deref(), alert.kind.title(), &detail, &url, alert.notify_count, ) .await { // No stamp on failure, so this re-sends next pass rather than burning a // message towards escalation on a provider outage. tracing::error!( error = ?e, alert_id = %alert.id, kind = %alert.kind, "acknowledgements: send failed", ); return; } if let Err(e) = acknowledgements::mark_notified(&state.db, alert.id).await { // Logged rather than retried: the cost is one duplicate message next // pass, which is much better than the alternative failure (stamping a // send that did not happen and letting the row go quiet). tracing::error!( error = ?e, alert_id = %alert.id, "acknowledgements: stamp failed (the message will repeat next pass)", ); } } /// Stop sending and ask a person to make contact. /// /// `escalate` reports whether this call was the one that moved the row, so /// exactly one ticket is filed however many passes race here. async fn escalate(state: &AppState, alert: &PendingAcknowledgement) { match acknowledgements::escalate(&state.db, alert.id).await { Ok(false) => return, Ok(true) => {} Err(e) => { tracing::error!(error = ?e, alert_id = %alert.id, "acknowledgements: escalation failed"); return; } } tracing::warn!( alert_id = %alert.id, user_id = %alert.user_id, kind = %alert.kind, sends = alert.notify_count, "acknowledgement unanswered after every message; handing to a person", ); let Some(wam) = state.wam.as_ref() else { return; }; let title = format!("Unacknowledged alert: {}", alert.kind); let body = format!( "{sends} messages went to {email} about this and none were confirmed.\n\n\ {detail}\n\n\ Automatic sending has stopped. Someone needs to contact them directly.", sends = alert.notify_count, email = alert.email, detail = detail_copy(alert.kind, &alert.details), ); wam.create_ticket( &title, Some(&body), "high", "acknowledgement-unanswered", Some(&alert.user_id.to_string()), ) .await; } #[cfg(test)] mod tests { /// The same constructive seal the SyncKit warning job carries: the tick must /// never await a mail send, or one slow provider stalls every other periodic /// job behind it. #[test] fn tick_never_sends_inline() { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("src/scheduler/acknowledgements.rs"); let src = std::fs::read_to_string(&path).expect("read acknowledgements source"); let start = src .find("async fn send_due_acknowledgements") .expect("tick fn present"); let rest = &src[start..]; let end = rest[1..] .find("\nasync fn ") .or_else(|| rest[1..].find("\nfn ")) .map_or(rest.len(), |i| i + 1); let tick_body = &rest[..end]; assert!( !tick_body.contains("send_acknowledgement_required"), "the tick must not send inline; enqueue onto state.bg and send in process()", ); assert!( tick_body.contains("bg.spawn"), "the tick must dispatch onto the bounded background pool", ); } }