//! Background test-suite task. //! //! Schedules the runs the status readout grades: without this task a target //! with a `[tests]` block sits at `pending` forever. //! //! The sweep interval is not the test cadence. Each tick asks //! [`drift::compute_test_staleness`] whether a target's last run is still good //! (never run / older than `staleness_days` / deployed version changed since); //! only a stale target actually pays for a run. Reusing the same predicate the //! status readout uses is deliberate: the scheduler and the dashboard cannot //! disagree about what "stale" means. use tokio::task::JoinHandle; use tracing::{error, info}; use pom::alerts::Alerter; use pom::checks::{drift, ssh}; use pom::config::{Config, TestsConfig}; use pom::db; use pom::types::TestRun; use super::{CheckInterval, configured_targets}; pub(crate) fn spawn_test_tasks( config: &Config, pool: &sqlx::SqlitePool, cancel: &tokio_util::sync::CancellationToken, alerter: Option<&Alerter>, ) -> Vec> { let sweep_secs = config.serve.test_sweep_interval_secs; let mut handles = Vec::new(); for (name, target_config) in configured_targets(config) { let Some(tests_config) = target_config.tests else { continue; }; let label = target_config.label.clone(); let pool = pool.clone(); let alerter = alerter.cloned(); let cancel = cancel.clone(); info!( "{name}: test staleness sweep every {sweep_secs}s (threshold={}d, command={})", tests_config.staleness_days, tests_config.command ); handles.push(tokio::spawn(async move { let mut ticks = CheckInterval::new(sweep_secs, cancel); while ticks.next().await { let Some(reason) = staleness_reason(&pool, &name, &tests_config).await else { continue; }; info!("{name}: running tests ({reason})"); let run = ssh::run_tests(&name, &tests_config, None).await; let previous_passed = db::get_latest_test_run(&pool, &name) .await .ok() .flatten() .map(|r| r.passed); if let Err(e) = store_run(&pool, &name, &run).await { error!("{name}: failed to store test run: {e}"); } info!( "{}: tests {} ({} passed, {} failed, {}s)", name, if run.passed { "passed" } else { "FAILED" }, run.summary.total_passed.unwrap_or(-1), run.summary.total_failed.unwrap_or(-1), run.duration_secs.unwrap_or(-1), ); if let Some(ref alerter) = alerter { alert_on_transition(alerter, &name, &label, &run, previous_passed).await; } } })); } handles } /// Why this target's tests need running, or `None` if the last run still counts. async fn staleness_reason( pool: &sqlx::SqlitePool, name: &str, tests_config: &TestsConfig, ) -> Option { let current_version = db::get_health_history(pool, Some(name), 1) .await .unwrap_or_default() .first() .and_then(|s| s.details.as_ref()) .and_then(|d| d.version.clone()); let latest_test = db::get_latest_test_run(pool, name).await.unwrap_or(None); let tested_version = match latest_test { Some(ref test) => db::get_version_at_time(pool, name, &test.started_at) .await .unwrap_or(None), None => None, }; let staleness = drift::compute_test_staleness( current_version.as_deref(), tested_version.as_deref(), latest_test.as_ref().map(|t| t.started_at.as_str()), tests_config.staleness_days, ); staleness .stale .then(|| staleness.reason.unwrap_or_else(|| "stale".to_string())) } /// Persist a run and its per-test details. Details are best-effort: losing them /// costs regression detection on the next run, not the run record itself. async fn store_run( pool: &sqlx::SqlitePool, name: &str, run: &TestRun, ) -> Result<(), pom::error::PomError> { let run_id = db::insert_test_run(pool, run).await?; if !run.summary.details.is_empty() && let Err(e) = db::insert_test_details(pool, run_id, &run.summary.details).await { error!("{name}: failed to store test details: {e}"); } Ok(()) } /// What a sweep's result is worth telling someone about. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Transition { Broke, Recovered, } /// Alert on pass/fail transitions only, so a suite that stays red does not page /// once per sweep. A first-ever run that fails counts as broken: there is no /// prior state, and silence would be indistinguishable from passing. fn transition(previous_passed: Option, now_passed: bool) -> Option { match (previous_passed.unwrap_or(true), now_passed) { (true, false) => Some(Transition::Broke), (false, true) => Some(Transition::Recovered), _ => None, } } async fn alert_on_transition( alerter: &Alerter, name: &str, label: &str, run: &TestRun, previous_passed: Option, ) { match transition(previous_passed, run.passed) { Some(Transition::Broke) => { let detail = failure_detail(run); alerter .send_test_failure_alert( name, label, run.summary.total_failed, run.exit_code.map(i64::from), &detail, ) .await; } Some(Transition::Recovered) => alerter.send_test_recovery(name, label).await, None => {} } } /// A short, alert-sized reason for a red suite: the failing test names when the /// output parsed, otherwise the tail of the raw output, which is where an SSH /// or missing-checkout error lands. fn failure_detail(run: &TestRun) -> String { let failed: Vec<&str> = run .summary .details .iter() .filter(|d| !d.passed) .map(|d| d.test_name.as_str()) .take(10) .collect(); if !failed.is_empty() { return failed.join(", "); } let tail: Vec<&str> = run .raw_output .lines() .filter(|l| !l.trim().is_empty()) .rev() .take(5) .collect(); if tail.is_empty() { "no output captured".to_string() } else { tail.into_iter().rev().collect::>().join(" | ") } } #[cfg(test)] mod tests { use super::*; use pom::types::{TestDetail, TestSummary}; fn run_with(passed: bool, details: Vec, raw_output: &str) -> TestRun { TestRun { id: None, target: "mnw".to_string(), started_at: "2026-08-06T00:00:00Z".to_string(), finished_at: None, duration_secs: Some(1), exit_code: Some(if passed { 0 } else { 101 }), passed, summary: TestSummary { steps: vec![], total_passed: None, total_failed: None, details, }, raw_output: raw_output.to_string(), filter: None, } } fn detail(name: &str, passed: bool) -> TestDetail { TestDetail { test_name: name.to_string(), passed, } } #[test] fn failure_detail_prefers_failing_test_names() { let run = run_with( false, vec![ detail("checks::whois::ok", true), detail("checks::rdap::broken", false), detail("db::also_broken", false), ], "irrelevant output", ); assert_eq!( failure_detail(&run), "checks::rdap::broken, db::also_broken" ); } #[test] fn failure_detail_caps_the_name_list() { let details: Vec = (0..20).map(|i| detail(&format!("t{i}"), false)).collect(); let run = run_with(false, details, ""); assert_eq!(failure_detail(&run).split(", ").count(), 10); } #[test] fn failure_detail_falls_back_to_output_tail_in_order() { // The suite never ran, so there are no test names. This is the shape an // SSH refusal or a missing checkout produces, and it is the case that // matters most: the target is unverified, not merely broken. let run = run_with( false, vec![], "ssh: connect to host 100.106.221.39 port 22: Connection refused\n\ exit status 255\n", ); let detail = failure_detail(&run); assert!(detail.starts_with("ssh: connect to host"), "got {detail}"); assert!(detail.ends_with("exit status 255"), "got {detail}"); } #[test] fn failure_detail_handles_no_output() { assert_eq!( failure_detail(&run_with(false, vec![], " \n\n")), "no output captured" ); } #[test] fn transition_first_ever_run_failing_is_a_break() { // `None` means nothing has run before. Treating it as previously-passing // is what makes the first red run alert instead of being swallowed as // no-change, which is exactly the state every astra target is in today. assert_eq!(transition(None, false), Some(Transition::Broke)); } #[test] fn transition_first_ever_run_passing_is_silent() { assert_eq!(transition(None, true), None); } #[test] fn transition_still_red_does_not_repage() { assert_eq!(transition(Some(false), false), None); } #[test] fn transition_still_green_is_silent() { assert_eq!(transition(Some(true), true), None); } #[test] fn transition_red_to_green_recovers() { assert_eq!(transition(Some(false), true), Some(Transition::Recovered)); } #[test] fn transition_green_to_red_breaks() { assert_eq!(transition(Some(true), false), Some(Transition::Broke)); } }