Skip to main content

max / makenotwork

10.2 KB · 327 lines History Blame Raw
1 //! Background test-suite task.
2 //!
3 //! Every target with a `[tests]` block used to sit at `pending` forever: the
4 //! serve loop had no test task, so `staleness_days` was only ever read by the
5 //! status readout to grade a run that nothing scheduled. This closes that loop.
6 //!
7 //! The sweep interval is not the test cadence. Each tick asks
8 //! [`drift::compute_test_staleness`] whether a target's last run is still good
9 //! (never run / older than `staleness_days` / deployed version changed since);
10 //! only a stale target actually pays for a run. Reusing the same predicate the
11 //! status readout uses is deliberate: the scheduler and the dashboard cannot
12 //! disagree about what "stale" means.
13
14 use tokio::task::JoinHandle;
15 use tracing::{error, info};
16
17 use pom::alerts::Alerter;
18 use pom::checks::{drift, ssh};
19 use pom::config::{Config, TestsConfig};
20 use pom::db;
21 use pom::types::TestRun;
22
23 use super::{CheckInterval, configured_targets};
24
25 pub(crate) fn spawn_test_tasks(
26 config: &Config,
27 pool: &sqlx::SqlitePool,
28 cancel: &tokio_util::sync::CancellationToken,
29 alerter: Option<&Alerter>,
30 ) -> Vec<JoinHandle<()>> {
31 let sweep_secs = config.serve.test_sweep_interval_secs;
32 let mut handles = Vec::new();
33
34 for (name, target_config) in configured_targets(config) {
35 let Some(tests_config) = target_config.tests else {
36 continue;
37 };
38 let label = target_config.label.clone();
39 let pool = pool.clone();
40 let alerter = alerter.cloned();
41 let cancel = cancel.clone();
42
43 info!(
44 "{name}: test staleness sweep every {sweep_secs}s (threshold={}d, command={})",
45 tests_config.staleness_days, tests_config.command
46 );
47
48 handles.push(tokio::spawn(async move {
49 let mut ticks = CheckInterval::new(sweep_secs, cancel);
50
51 while ticks.next().await {
52 let Some(reason) = staleness_reason(&pool, &name, &tests_config).await else {
53 continue;
54 };
55
56 info!("{name}: running tests ({reason})");
57 let run = ssh::run_tests(&name, &tests_config, None).await;
58 let previous_passed = db::get_latest_test_run(&pool, &name)
59 .await
60 .ok()
61 .flatten()
62 .map(|r| r.passed);
63
64 if let Err(e) = store_run(&pool, &name, &run).await {
65 error!("{name}: failed to store test run: {e}");
66 }
67
68 info!(
69 "{}: tests {} ({} passed, {} failed, {}s)",
70 name,
71 if run.passed { "passed" } else { "FAILED" },
72 run.summary.total_passed.unwrap_or(-1),
73 run.summary.total_failed.unwrap_or(-1),
74 run.duration_secs.unwrap_or(-1),
75 );
76
77 if let Some(ref alerter) = alerter {
78 alert_on_transition(alerter, &name, &label, &run, previous_passed).await;
79 }
80 }
81 }));
82 }
83
84 handles
85 }
86
87 /// Why this target's tests need running, or `None` if the last run still counts.
88 async fn staleness_reason(
89 pool: &sqlx::SqlitePool,
90 name: &str,
91 tests_config: &TestsConfig,
92 ) -> Option<String> {
93 let current_version = db::get_health_history(pool, Some(name), 1)
94 .await
95 .unwrap_or_default()
96 .first()
97 .and_then(|s| s.details.as_ref())
98 .and_then(|d| d.version.clone());
99
100 let latest_test = db::get_latest_test_run(pool, name).await.unwrap_or(None);
101
102 let tested_version = match latest_test {
103 Some(ref test) => db::get_version_at_time(pool, name, &test.started_at)
104 .await
105 .unwrap_or(None),
106 None => None,
107 };
108
109 let staleness = drift::compute_test_staleness(
110 current_version.as_deref(),
111 tested_version.as_deref(),
112 latest_test.as_ref().map(|t| t.started_at.as_str()),
113 tests_config.staleness_days,
114 );
115
116 staleness
117 .stale
118 .then(|| staleness.reason.unwrap_or_else(|| "stale".to_string()))
119 }
120
121 /// Persist a run and its per-test details. Details are best-effort: losing them
122 /// costs regression detection on the next run, not the run record itself.
123 async fn store_run(
124 pool: &sqlx::SqlitePool,
125 name: &str,
126 run: &TestRun,
127 ) -> Result<(), pom::error::PomError> {
128 let run_id = db::insert_test_run(pool, run).await?;
129
130 if !run.summary.details.is_empty()
131 && let Err(e) = db::insert_test_details(pool, run_id, &run.summary.details).await
132 {
133 error!("{name}: failed to store test details: {e}");
134 }
135
136 Ok(())
137 }
138
139 /// What a sweep's result is worth telling someone about.
140 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
141 enum Transition {
142 Broke,
143 Recovered,
144 }
145
146 /// Alert on pass/fail transitions only, so a suite that stays red does not page
147 /// once per sweep. A first-ever run that fails counts as broken: there is no
148 /// prior state, and silence would be indistinguishable from passing.
149 fn transition(previous_passed: Option<bool>, now_passed: bool) -> Option<Transition> {
150 match (previous_passed.unwrap_or(true), now_passed) {
151 (true, false) => Some(Transition::Broke),
152 (false, true) => Some(Transition::Recovered),
153 _ => None,
154 }
155 }
156
157 async fn alert_on_transition(
158 alerter: &Alerter,
159 name: &str,
160 label: &str,
161 run: &TestRun,
162 previous_passed: Option<bool>,
163 ) {
164 match transition(previous_passed, run.passed) {
165 Some(Transition::Broke) => {
166 let detail = failure_detail(run);
167 alerter
168 .send_test_failure_alert(
169 name,
170 label,
171 run.summary.total_failed,
172 run.exit_code.map(i64::from),
173 &detail,
174 )
175 .await;
176 }
177 Some(Transition::Recovered) => alerter.send_test_recovery(name, label).await,
178 None => {}
179 }
180 }
181
182 /// A short, alert-sized reason for a red suite: the failing test names when the
183 /// output parsed, otherwise the tail of the raw output, which is where an SSH
184 /// or missing-checkout error lands.
185 fn failure_detail(run: &TestRun) -> String {
186 let failed: Vec<&str> = run
187 .summary
188 .details
189 .iter()
190 .filter(|d| !d.passed)
191 .map(|d| d.test_name.as_str())
192 .take(10)
193 .collect();
194
195 if !failed.is_empty() {
196 return failed.join(", ");
197 }
198
199 let tail: Vec<&str> = run
200 .raw_output
201 .lines()
202 .filter(|l| !l.trim().is_empty())
203 .rev()
204 .take(5)
205 .collect();
206
207 if tail.is_empty() {
208 "no output captured".to_string()
209 } else {
210 tail.into_iter().rev().collect::<Vec<_>>().join(" | ")
211 }
212 }
213
214 #[cfg(test)]
215 mod tests {
216 use super::*;
217 use pom::types::{TestDetail, TestSummary};
218
219 fn run_with(passed: bool, details: Vec<TestDetail>, raw_output: &str) -> TestRun {
220 TestRun {
221 id: None,
222 target: "mnw".to_string(),
223 started_at: "2026-08-06T00:00:00Z".to_string(),
224 finished_at: None,
225 duration_secs: Some(1),
226 exit_code: Some(if passed { 0 } else { 101 }),
227 passed,
228 summary: TestSummary {
229 steps: vec![],
230 total_passed: None,
231 total_failed: None,
232 details,
233 },
234 raw_output: raw_output.to_string(),
235 filter: None,
236 }
237 }
238
239 fn detail(name: &str, passed: bool) -> TestDetail {
240 TestDetail {
241 test_name: name.to_string(),
242 passed,
243 }
244 }
245
246 #[test]
247 fn failure_detail_prefers_failing_test_names() {
248 let run = run_with(
249 false,
250 vec![
251 detail("checks::whois::ok", true),
252 detail("checks::rdap::broken", false),
253 detail("db::also_broken", false),
254 ],
255 "irrelevant output",
256 );
257 assert_eq!(
258 failure_detail(&run),
259 "checks::rdap::broken, db::also_broken"
260 );
261 }
262
263 #[test]
264 fn failure_detail_caps_the_name_list() {
265 let details: Vec<TestDetail> = (0..20).map(|i| detail(&format!("t{i}"), false)).collect();
266 let run = run_with(false, details, "");
267 assert_eq!(failure_detail(&run).split(", ").count(), 10);
268 }
269
270 #[test]
271 fn failure_detail_falls_back_to_output_tail_in_order() {
272 // The suite never ran, so there are no test names. This is the shape an
273 // SSH refusal or a missing checkout produces, and it is the case that
274 // matters most: the target is unverified, not merely broken.
275 let run = run_with(
276 false,
277 vec![],
278 "ssh: connect to host 100.106.221.39 port 22: Connection refused\n\
279 exit status 255\n",
280 );
281 let detail = failure_detail(&run);
282 assert!(detail.starts_with("ssh: connect to host"), "got {detail}");
283 assert!(detail.ends_with("exit status 255"), "got {detail}");
284 }
285
286 #[test]
287 fn failure_detail_handles_no_output() {
288 assert_eq!(
289 failure_detail(&run_with(false, vec![], " \n\n")),
290 "no output captured"
291 );
292 }
293
294 #[test]
295 fn transition_first_ever_run_failing_is_a_break() {
296 // `None` means nothing has run before. Treating it as previously-passing
297 // is what makes the first red run alert instead of being swallowed as
298 // no-change, which is exactly the state every astra target is in today.
299 assert_eq!(transition(None, false), Some(Transition::Broke));
300 }
301
302 #[test]
303 fn transition_first_ever_run_passing_is_silent() {
304 assert_eq!(transition(None, true), None);
305 }
306
307 #[test]
308 fn transition_still_red_does_not_repage() {
309 assert_eq!(transition(Some(false), false), None);
310 }
311
312 #[test]
313 fn transition_still_green_is_silent() {
314 assert_eq!(transition(Some(true), true), None);
315 }
316
317 #[test]
318 fn transition_red_to_green_recovers() {
319 assert_eq!(transition(Some(false), true), Some(Transition::Recovered));
320 }
321
322 #[test]
323 fn transition_green_to_red_breaks() {
324 assert_eq!(transition(Some(true), false), Some(Transition::Broke));
325 }
326 }
327