Skip to main content

max / makenotwork

10.1 KB · 326 lines History Blame Raw
1 //! Background test-suite task.
2 //!
3 //! Schedules the runs the status readout grades: without this task a target
4 //! with a `[tests]` block sits at `pending` forever.
5 //!
6 //! The sweep interval is not the test cadence. Each tick asks
7 //! [`drift::compute_test_staleness`] whether a target's last run is still good
8 //! (never run / older than `staleness_days` / deployed version changed since);
9 //! only a stale target actually pays for a run. Reusing the same predicate the
10 //! status readout uses is deliberate: the scheduler and the dashboard cannot
11 //! disagree about what "stale" means.
12
13 use tokio::task::JoinHandle;
14 use tracing::{error, info};
15
16 use pom::alerts::Alerter;
17 use pom::checks::{drift, ssh};
18 use pom::config::{Config, TestsConfig};
19 use pom::db;
20 use pom::types::TestRun;
21
22 use super::{CheckInterval, configured_targets};
23
24 pub(crate) fn spawn_test_tasks(
25 config: &Config,
26 pool: &sqlx::SqlitePool,
27 cancel: &tokio_util::sync::CancellationToken,
28 alerter: Option<&Alerter>,
29 ) -> Vec<JoinHandle<()>> {
30 let sweep_secs = config.serve.test_sweep_interval_secs;
31 let mut handles = Vec::new();
32
33 for (name, target_config) in configured_targets(config) {
34 let Some(tests_config) = target_config.tests else {
35 continue;
36 };
37 let label = target_config.label.clone();
38 let pool = pool.clone();
39 let alerter = alerter.cloned();
40 let cancel = cancel.clone();
41
42 info!(
43 "{name}: test staleness sweep every {sweep_secs}s (threshold={}d, command={})",
44 tests_config.staleness_days, tests_config.command
45 );
46
47 handles.push(tokio::spawn(async move {
48 let mut ticks = CheckInterval::new(sweep_secs, cancel);
49
50 while ticks.next().await {
51 let Some(reason) = staleness_reason(&pool, &name, &tests_config).await else {
52 continue;
53 };
54
55 info!("{name}: running tests ({reason})");
56 let run = ssh::run_tests(&name, &tests_config, None).await;
57 let previous_passed = db::get_latest_test_run(&pool, &name)
58 .await
59 .ok()
60 .flatten()
61 .map(|r| r.passed);
62
63 if let Err(e) = store_run(&pool, &name, &run).await {
64 error!("{name}: failed to store test run: {e}");
65 }
66
67 info!(
68 "{}: tests {} ({} passed, {} failed, {}s)",
69 name,
70 if run.passed { "passed" } else { "FAILED" },
71 run.summary.total_passed.unwrap_or(-1),
72 run.summary.total_failed.unwrap_or(-1),
73 run.duration_secs.unwrap_or(-1),
74 );
75
76 if let Some(ref alerter) = alerter {
77 alert_on_transition(alerter, &name, &label, &run, previous_passed).await;
78 }
79 }
80 }));
81 }
82
83 handles
84 }
85
86 /// Why this target's tests need running, or `None` if the last run still counts.
87 async fn staleness_reason(
88 pool: &sqlx::SqlitePool,
89 name: &str,
90 tests_config: &TestsConfig,
91 ) -> Option<String> {
92 let current_version = db::get_health_history(pool, Some(name), 1)
93 .await
94 .unwrap_or_default()
95 .first()
96 .and_then(|s| s.details.as_ref())
97 .and_then(|d| d.version.clone());
98
99 let latest_test = db::get_latest_test_run(pool, name).await.unwrap_or(None);
100
101 let tested_version = match latest_test {
102 Some(ref test) => db::get_version_at_time(pool, name, &test.started_at)
103 .await
104 .unwrap_or(None),
105 None => None,
106 };
107
108 let staleness = drift::compute_test_staleness(
109 current_version.as_deref(),
110 tested_version.as_deref(),
111 latest_test.as_ref().map(|t| t.started_at.as_str()),
112 tests_config.staleness_days,
113 );
114
115 staleness
116 .stale
117 .then(|| staleness.reason.unwrap_or_else(|| "stale".to_string()))
118 }
119
120 /// Persist a run and its per-test details. Details are best-effort: losing them
121 /// costs regression detection on the next run, not the run record itself.
122 async fn store_run(
123 pool: &sqlx::SqlitePool,
124 name: &str,
125 run: &TestRun,
126 ) -> Result<(), pom::error::PomError> {
127 let run_id = db::insert_test_run(pool, run).await?;
128
129 if !run.summary.details.is_empty()
130 && let Err(e) = db::insert_test_details(pool, run_id, &run.summary.details).await
131 {
132 error!("{name}: failed to store test details: {e}");
133 }
134
135 Ok(())
136 }
137
138 /// What a sweep's result is worth telling someone about.
139 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
140 enum Transition {
141 Broke,
142 Recovered,
143 }
144
145 /// Alert on pass/fail transitions only, so a suite that stays red does not page
146 /// once per sweep. A first-ever run that fails counts as broken: there is no
147 /// prior state, and silence would be indistinguishable from passing.
148 fn transition(previous_passed: Option<bool>, now_passed: bool) -> Option<Transition> {
149 match (previous_passed.unwrap_or(true), now_passed) {
150 (true, false) => Some(Transition::Broke),
151 (false, true) => Some(Transition::Recovered),
152 _ => None,
153 }
154 }
155
156 async fn alert_on_transition(
157 alerter: &Alerter,
158 name: &str,
159 label: &str,
160 run: &TestRun,
161 previous_passed: Option<bool>,
162 ) {
163 match transition(previous_passed, run.passed) {
164 Some(Transition::Broke) => {
165 let detail = failure_detail(run);
166 alerter
167 .send_test_failure_alert(
168 name,
169 label,
170 run.summary.total_failed,
171 run.exit_code.map(i64::from),
172 &detail,
173 )
174 .await;
175 }
176 Some(Transition::Recovered) => alerter.send_test_recovery(name, label).await,
177 None => {}
178 }
179 }
180
181 /// A short, alert-sized reason for a red suite: the failing test names when the
182 /// output parsed, otherwise the tail of the raw output, which is where an SSH
183 /// or missing-checkout error lands.
184 fn failure_detail(run: &TestRun) -> String {
185 let failed: Vec<&str> = run
186 .summary
187 .details
188 .iter()
189 .filter(|d| !d.passed)
190 .map(|d| d.test_name.as_str())
191 .take(10)
192 .collect();
193
194 if !failed.is_empty() {
195 return failed.join(", ");
196 }
197
198 let tail: Vec<&str> = run
199 .raw_output
200 .lines()
201 .filter(|l| !l.trim().is_empty())
202 .rev()
203 .take(5)
204 .collect();
205
206 if tail.is_empty() {
207 "no output captured".to_string()
208 } else {
209 tail.into_iter().rev().collect::<Vec<_>>().join(" | ")
210 }
211 }
212
213 #[cfg(test)]
214 mod tests {
215 use super::*;
216 use pom::types::{TestDetail, TestSummary};
217
218 fn run_with(passed: bool, details: Vec<TestDetail>, raw_output: &str) -> TestRun {
219 TestRun {
220 id: None,
221 target: "mnw".to_string(),
222 started_at: "2026-08-06T00:00:00Z".to_string(),
223 finished_at: None,
224 duration_secs: Some(1),
225 exit_code: Some(if passed { 0 } else { 101 }),
226 passed,
227 summary: TestSummary {
228 steps: vec![],
229 total_passed: None,
230 total_failed: None,
231 details,
232 },
233 raw_output: raw_output.to_string(),
234 filter: None,
235 }
236 }
237
238 fn detail(name: &str, passed: bool) -> TestDetail {
239 TestDetail {
240 test_name: name.to_string(),
241 passed,
242 }
243 }
244
245 #[test]
246 fn failure_detail_prefers_failing_test_names() {
247 let run = run_with(
248 false,
249 vec![
250 detail("checks::whois::ok", true),
251 detail("checks::rdap::broken", false),
252 detail("db::also_broken", false),
253 ],
254 "irrelevant output",
255 );
256 assert_eq!(
257 failure_detail(&run),
258 "checks::rdap::broken, db::also_broken"
259 );
260 }
261
262 #[test]
263 fn failure_detail_caps_the_name_list() {
264 let details: Vec<TestDetail> = (0..20).map(|i| detail(&format!("t{i}"), false)).collect();
265 let run = run_with(false, details, "");
266 assert_eq!(failure_detail(&run).split(", ").count(), 10);
267 }
268
269 #[test]
270 fn failure_detail_falls_back_to_output_tail_in_order() {
271 // The suite never ran, so there are no test names. This is the shape an
272 // SSH refusal or a missing checkout produces, and it is the case that
273 // matters most: the target is unverified, not merely broken.
274 let run = run_with(
275 false,
276 vec![],
277 "ssh: connect to host 100.106.221.39 port 22: Connection refused\n\
278 exit status 255\n",
279 );
280 let detail = failure_detail(&run);
281 assert!(detail.starts_with("ssh: connect to host"), "got {detail}");
282 assert!(detail.ends_with("exit status 255"), "got {detail}");
283 }
284
285 #[test]
286 fn failure_detail_handles_no_output() {
287 assert_eq!(
288 failure_detail(&run_with(false, vec![], " \n\n")),
289 "no output captured"
290 );
291 }
292
293 #[test]
294 fn transition_first_ever_run_failing_is_a_break() {
295 // `None` means nothing has run before. Treating it as previously-passing
296 // is what makes the first red run alert instead of being swallowed as
297 // no-change, which is exactly the state every astra target is in today.
298 assert_eq!(transition(None, false), Some(Transition::Broke));
299 }
300
301 #[test]
302 fn transition_first_ever_run_passing_is_silent() {
303 assert_eq!(transition(None, true), None);
304 }
305
306 #[test]
307 fn transition_still_red_does_not_repage() {
308 assert_eq!(transition(Some(false), false), None);
309 }
310
311 #[test]
312 fn transition_still_green_is_silent() {
313 assert_eq!(transition(Some(true), true), None);
314 }
315
316 #[test]
317 fn transition_red_to_green_recovers() {
318 assert_eq!(transition(Some(false), true), Some(Transition::Recovered));
319 }
320
321 #[test]
322 fn transition_green_to_red_breaks() {
323 assert_eq!(transition(Some(true), false), Some(Transition::Broke));
324 }
325 }
326