Skip to main content

max / makenotwork

pom: schedule test suites, and check domains over RDAP Three things kept astra's dashboard yellow. No test task existed. serve.rs spawned eleven check tasks and none of them ran a suite, so staleness_days was only ever read by the status readout to grade a run nothing scheduled. Every target with a [tests] block sat at pending forever, and mnw-cli read ok only because it is the one target with no [tests] block at all. Add spawn_test_tasks: a cheap hourly sweep that asks compute_test_staleness, the same predicate the readout uses, and runs a suite only when it answers stale. Alerts fire on pass/fail transitions, with a never-run target counted as previously passing so a first red run is not swallowed as no-change. The SSH hop was not optional. TestsConfig.ssh was required, so a host whose runner is itself had to name its own address, which needs a regular sshd listening. A Tailscale-SSH host has none, and tailscaled does not intercept a node connecting to itself, so the hop failed with Connection refused. Make ssh an Option and run the command locally when it is absent. whois.nic.google is NXDOMAIN. Google Registry retired WHOIS for .app and .dev in favour of RDAP, which left htpy permanently degraded on a domain that is fine. Add checks::rdap and try it first for every domain: bases come from IANA's bootstrap, cached a day, with a table covering the monitored TLDs so an IANA outage does not blind the check. WHOIS stays as the fallback for .io and .me, which have no RDAP service, and loses its two dead entries.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-08-06 20:44 UTC
Signed with PGP, not checked
Commit: 7dfb5d834c1735cf628c20d8f3c7b091bad5f454
Parent: 49c8766
11 files changed, +1096 insertions, -69 deletions
M pom/src/config.rs +24 -4
@@ -140,6 +140,11 @@
140 140 /// Seconds between WHOIS domain expiry checks.
141 141 #[serde(default = "default_whois_check_interval")]
142 142 pub whois_check_interval_secs: u64,
143 + /// Seconds between staleness sweeps of the configured test suites. This is
144 + /// not how often tests run: the sweep runs a suite only when its last run is
145 + /// older than that target's `tests.staleness_days`, or when it has never run.
146 + #[serde(default = "default_test_sweep_interval")]
147 + pub test_sweep_interval_secs: u64,
143 148 /// Bearer token required for API access. If set, all /api/* requests must
144 149 /// include `Authorization: Bearer <token>`. Can also be set via POM_API_TOKEN env var.
145 150 pub api_token: Option<String>,
@@ -166,6 +171,7 @@
166 171 .field("dns_check_interval_secs", &self.dns_check_interval_secs)
167 172 .field("cors_check_interval_secs", &self.cors_check_interval_secs)
168 173 .field("whois_check_interval_secs", &self.whois_check_interval_secs)
174 + .field("test_sweep_interval_secs", &self.test_sweep_interval_secs)
169 175 .field("api_token", &self.api_token.as_ref().map(|_| "***"))
170 176 .field("dashboard", &self.dashboard)
171 177 .field("confirmations", &self.confirmations)
@@ -189,6 +195,7 @@
189 195 dns_check_interval_secs: 3600,
190 196 cors_check_interval_secs: 3600,
191 197 whois_check_interval_secs: 86400,
198 + test_sweep_interval_secs: default_test_sweep_interval(),
192 199 api_token: None,
193 200 dashboard: false,
194 201 confirmations: default_confirmations(),
@@ -226,6 +233,12 @@
226 233 86400
227 234 }
228 235
236 + fn default_test_sweep_interval() -> u64 {
237 + // 1 hour. The sweep is cheap (one DB read per target); the suite it may
238 + // trigger is not, which is why `staleness_days` gates the run itself.
239 + 3600
240 + }
241 +
229 242 fn default_serve_interval() -> u64 {
230 243 // 5 minutes: frequent enough to catch outages within an SLA window,
231 244 // infrequent enough to avoid noise
@@ -617,9 +630,16 @@
617 630
618 631 #[derive(Debug, Clone, Deserialize)]
619 632 pub struct TestsConfig {
620 - /// SSH host alias (from ~/.ssh/config) for the test runner machine.
621 - pub ssh: String,
622 - /// Shell command to execute on the remote host to run tests.
633 + /// SSH host alias (from `~/.ssh/config`) for the test runner machine.
634 + ///
635 + /// Omit when the runner is the host PoM runs on: the command is then
636 + /// executed locally. Pointing this at the local machine's own address
637 + /// instead only works if a regular sshd is listening, which is not a given
638 + /// on a Tailscale-SSH host (tailscaled does not intercept a node
639 + /// connecting to itself).
640 + #[serde(default)]
641 + pub ssh: Option<String>,
642 + /// Shell command to run the tests, remotely or locally per `ssh`.
623 643 pub command: String,
624 644 /// Maximum seconds to wait for the test command before killing it.
625 645 #[serde(default = "default_test_timeout")]
@@ -812,7 +832,7 @@
812 832 let mnw = config.get_target("mnw").unwrap();
813 833 assert_eq!(mnw.label, "MakeNotWork");
814 834 assert_eq!(mnw.health.as_ref().unwrap().timeout_secs, 5);
815 - assert_eq!(mnw.tests.as_ref().unwrap().ssh, "hetzner");
835 + assert_eq!(mnw.tests.as_ref().unwrap().ssh.as_deref(), Some("hetzner"));
816 836
817 837 let astra = config.peers.get("astra").unwrap();
818 838 assert_eq!(astra.address, "100.0.0.1:9100");
M pom/src/types.rs +6
@@ -25,6 +25,8 @@
25 25 LatencyDrift,
26 26 LatencyRecovery,
27 27 TestDurationDrift,
28 + TestFailure,
29 + TestRecovery,
28 30 CorsFailure,
29 31 CorsRecovery,
30 32 BackupStale,
@@ -58,6 +60,8 @@
58 60 Self::LatencyDrift => write!(f, "latency_drift"),
59 61 Self::LatencyRecovery => write!(f, "latency_recovery"),
60 62 Self::TestDurationDrift => write!(f, "test_duration_drift"),
63 + Self::TestFailure => write!(f, "test_failure"),
64 + Self::TestRecovery => write!(f, "test_recovery"),
61 65 Self::CorsFailure => write!(f, "cors_failure"),
62 66 Self::CorsRecovery => write!(f, "cors_recovery"),
63 67 Self::BackupStale => write!(f, "backup_stale"),
@@ -94,6 +98,8 @@
94 98 "latency_drift" => Ok(Self::LatencyDrift),
95 99 "latency_recovery" => Ok(Self::LatencyRecovery),
96 100 "test_duration_drift" => Ok(Self::TestDurationDrift),
101 + "test_failure" => Ok(Self::TestFailure),
102 + "test_recovery" => Ok(Self::TestRecovery),
97 103 "cors_failure" => Ok(Self::CorsFailure),
98 104 "cors_recovery" => Ok(Self::CorsRecovery),
99 105 "backup_stale" => Ok(Self::BackupStale),
M pom/src/alerts/mod.rs +8 -2
@@ -22,6 +22,7 @@
22 22 mod scan;
23 23 mod systemd;
24 24 mod test_duration;
25 + mod test_suite;
25 26 mod tls;
26 27 mod whois;
27 28
@@ -87,10 +88,15 @@
87 88 DnsMismatch, DnsRecovery, Health, LatencyDrift, LatencyRecovery, MonitoringOffline,
88 89 MonitoringRecovery, PeerMissing, PeerRecovery, Recovery, RouteFailure, RouteRecovery,
89 90 ScanPipelineDegraded, ScanPipelineRecovery, SystemdFailure, SystemdRecovery,
90 - TestDurationDrift, TlsError, TlsExpiry, TlsRecovery, WhoisError, WhoisExpiry,
91 + TestDurationDrift, TestFailure, TestRecovery, TlsError, TlsExpiry, TlsRecovery, WhoisError,
92 + WhoisExpiry,
91 93 };
92 94 match category {
93 - Health | Recovery => "health",
95 + // A red test suite folds onto "health": it is a statement about the
96 + // target being sick, and MNW has no `test` AlertKind (an unknown kind
97 + // is a 422 from the ingest endpoint). The sub-condition rides in the
98 + // dedup key, the same way `Tls` already folds three PoM categories.
99 + Health | Recovery | TestFailure | TestRecovery => "health",
94 100 // The host CA bundle folds onto the "tls" domain: it is a trust-anchor
95 101 // problem, and every symptom it produces is a failed handshake. Adding a
96 102 // domain would need a matching MNW `AlertKind` or the ingest 422s.
M pom/src/checks/mod.rs +5 -3
@@ -1,6 +1,7 @@
1 - //! Check implementations, one module per probe kind: health, TLS, DNS, WHOIS,
2 - //! routes, CORS, backups, SSH, port scans, local systemd units, the local CA
3 - //! bundle, the SyncKit field-version readout, plus latency-drift analysis.
1 + //! Check implementations, one module per probe kind: health, TLS, DNS, domain
2 + //! registration (RDAP with a WHOIS fallback), routes, CORS, backups, SSH, port
3 + //! scans, local systemd units, the local CA bundle, the SyncKit field-version
4 + //! readout, plus latency-drift analysis.
4 5
5 6 pub mod backup;
6 7 pub mod ca_bundle;
@@ -9,6 +10,7 @@
9 10 pub mod drift;
10 11 pub mod http;
11 12 pub mod parse;
13 + pub mod rdap;
12 14 pub mod routes;
13 15 pub mod scan_pipeline;
14 16 pub mod ssh;
M pom/src/checks/ssh.rs +84 -12
@@ -16,6 +16,36 @@
16 16 .all(|c| c.is_alphanumeric() || c == '_' || c == ':' || c == '-')
17 17 }
18 18
19 + /// Build the process that runs the suite: an `ssh` invocation when the target
20 + /// names a runner host, a local shell otherwise.
21 + ///
22 + /// Local execution exists because the runner is often the machine PoM already
23 + /// runs on. Routing that through `ssh` to its own address needs a regular sshd
24 + /// listening, and a Tailscale-SSH host has none: tailscaled does not intercept
25 + /// a node connecting to itself, so the hop fails with `Connection refused`
26 + /// while every other SSH path into the box keeps working.
27 + fn build_command(config: &TestsConfig, cmd_str: &str) -> Command {
28 + match config.ssh.as_deref() {
29 + Some(host) => {
30 + let mut command = Command::new("ssh");
31 + command
32 + .arg("-o")
33 + .arg("BatchMode=yes")
34 + .arg("-o")
35 + .arg(format!("ConnectTimeout={}", config.timeout_secs))
36 + .arg(host)
37 + .arg("--")
38 + .arg(cmd_str);
39 + command
40 + }
41 + None => {
42 + let mut command = Command::new("sh");
43 + command.arg("-c").arg(cmd_str);
44 + command
45 + }
46 + }
47 + }
48 +
19 49 #[instrument(skip_all)]
20 50 pub async fn run_tests(target_name: &str, config: &TestsConfig, filter: Option<&str>) -> TestRun {
21 51 let started_at = chrono::Utc::now().to_rfc3339();
@@ -60,16 +90,8 @@
60 90 // handshake, not a connected-then-hung remote command (fuzz-2026-07-06). Wrap
61 91 // the whole run in a total timeout and `kill_on_drop` so the ssh child is
62 92 // reaped when it elapses, honoring the field's contract.
63 - let mut command = Command::new("ssh");
64 - command
65 - .arg("-o")
66 - .arg("BatchMode=yes")
67 - .arg("-o")
68 - .arg(format!("ConnectTimeout={}", config.timeout_secs))
69 - .arg(&config.ssh)
70 - .arg("--")
71 - .arg(&cmd_str)
72 - .kill_on_drop(true);
93 + let mut command = build_command(config, &cmd_str);
94 + command.kill_on_drop(true);
73 95
74 96 let total_timeout = std::time::Duration::from_secs(config.timeout_secs.max(1));
75 97 let result = tokio::time::timeout(total_timeout, command.output()).await;
@@ -93,7 +115,7 @@
93 115 details: vec![],
94 116 },
95 117 raw_output: format!(
96 - "SSH test command timed out after {}s (killed)",
118 + "test command timed out after {}s (killed)",
97 119 config.timeout_secs
98 120 ),
99 121 filter: filter.map(String::from),
@@ -134,7 +156,10 @@
134 156 total_failed: None,
135 157 details: vec![],
136 158 },
137 - raw_output: format!("SSH connection failed: {e}"),
159 + raw_output: match config.ssh.as_deref() {
160 + Some(host) => format!("SSH connection to {host} failed: {e}"),
161 + None => format!("local test command failed to spawn: {e}"),
162 + },
138 163 filter: filter.map(String::from),
139 164 },
140 165 }
@@ -144,6 +169,53 @@
144 169 mod tests {
145 170 use super::*;
146 171
172 + fn config_with_ssh(ssh: Option<&str>) -> TestsConfig {
173 + TestsConfig {
174 + ssh: ssh.map(String::from),
175 + command: "cargo test".to_string(),
176 + timeout_secs: 42,
177 + staleness_days: 7,
178 + }
179 + }
180 +
181 + /// The program plus its args, which is what distinguishes the two paths.
182 + fn command_line(command: &Command) -> Vec<String> {
183 + let std = command.as_std();
184 + std::iter::once(std.get_program())
185 + .chain(std.get_args())
186 + .map(|s| s.to_string_lossy().into_owned())
187 + .collect()
188 + }
189 +
190 + #[test]
191 + fn build_command_uses_ssh_when_a_host_is_named() {
192 + let config = config_with_ssh(Some("astra"));
193 + let line = command_line(&build_command(&config, "cargo test"));
194 + assert_eq!(line[0], "ssh");
195 + assert!(line.contains(&"astra".to_string()));
196 + assert!(line.contains(&"BatchMode=yes".to_string()));
197 + assert!(line.contains(&"ConnectTimeout=42".to_string()));
198 + assert_eq!(line.last().unwrap(), "cargo test");
199 + }
200 +
201 + #[test]
202 + fn build_command_runs_locally_when_no_host_is_named() {
203 + // Omitting `ssh` must not degrade into an SSH call to localhost: a
204 + // Tailscale-SSH host has no sshd, so that hop is refused outright.
205 + let config = config_with_ssh(None);
206 + let line = command_line(&build_command(&config, "cargo test"));
207 + assert_eq!(line, vec!["sh", "-c", "cargo test"]);
208 + }
209 +
210 + #[test]
211 + fn build_command_passes_the_whole_command_as_one_argument() {
212 + // The command is a shell string, `cd x && cargo test` among them. Split
213 + // on whitespace it would run `cd` with the rest as arguments.
214 + let config = config_with_ssh(None);
215 + let line = command_line(&build_command(&config, "cd /srv/app && cargo test"));
216 + assert_eq!(line.last().unwrap(), "cd /srv/app && cargo test");
217 + }
218 +
147 219 #[test]
148 220 fn validate_test_filter_valid_simple() {
149 221 assert!(validate_test_filter("foo"));
M pom/src/checks/whois.rs +91 -48
@@ -1,4 +1,8 @@
1 - //! WHOIS domain expiry checking, raw TCP to WHOIS servers.
1 + //! Domain registration checking: RDAP first, raw port-43 WHOIS as fallback.
2 + //!
3 + //! RDAP is the protocol registries are actually maintaining, so it is tried
4 + //! first for every domain (see [`super::rdap`]). WHOIS remains for the TLDs
5 + //! with no RDAP service, `.io` and `.me` among them.
2 6
3 7 use tokio::io::{AsyncReadExt, AsyncWriteExt};
4 8 use tokio::net::TcpStream;
@@ -7,60 +11,97 @@
7 11 use crate::config::WhoisConfig;
8 12 use crate::types::WhoisResult;
9 13
10 - /// Query WHOIS for domain registration info.
14 + use super::rdap;
15 +
16 + /// Query domain registration info, over RDAP where the TLD supports it.
11 17 #[instrument(skip_all)]
12 18 pub async fn check_whois(target: &str, config: &WhoisConfig) -> WhoisResult {
13 - let now = chrono::Utc::now().to_rfc3339();
19 + match rdap::lookup(&config.domain).await {
20 + Ok(Some(domain)) => {
21 + return build_result(
22 + target,
23 + config,
24 + domain.registrar,
25 + domain.expiry_date,
26 + domain.nameservers,
27 + );
28 + }
29 + Ok(None) => {}
30 + Err(e) => {
31 + // An RDAP service exists but did not answer. WHOIS is worth a try
32 + // before calling the domain unverifiable.
33 + tracing::warn!(
34 + "{target}: RDAP lookup for {} failed, falling back to WHOIS: {e}",
35 + config.domain
36 + );
37 + }
38 + }
14 39
15 40 let Some(server) = whois_server_for_tld(&config.domain) else {
16 - return WhoisResult {
17 - target: target.to_string(),
18 - domain: config.domain.clone(),
19 - registrar: None,
20 - expiry_date: None,
21 - days_remaining: None,
22 - nameservers: vec![],
23 - checked_at: now,
24 - error: Some(format!(
25 - "no WHOIS server known for TLD of {}",
41 + return error_result(
42 + target,
43 + config,
44 + format!(
45 + "no RDAP service or WHOIS server known for {}",
26 46 config.domain
27 - )),
28 - };
47 + ),
48 + );
29 49 };
30 50
31 51 match query_whois(server, &config.domain).await {
32 52 Ok(response) => {
33 53 let parsed = parse_whois_response(&response);
34 - let days_remaining = parsed
35 - .expiry_date
36 - .as_deref()
37 - .and_then(compute_days_remaining);
38 -
39 - WhoisResult {
40 - target: target.to_string(),
41 - domain: config.domain.clone(),
42 - registrar: parsed.registrar,
43 - expiry_date: parsed.expiry_date,
44 - days_remaining,
45 - nameservers: parsed.nameservers,
46 - checked_at: now,
47 - error: None,
48 - }
54 + build_result(
55 + target,
56 + config,
57 + parsed.registrar,
58 + parsed.expiry_date,
59 + parsed.nameservers,
60 + )
49 61 }
50 - Err(e) => WhoisResult {
51 - target: target.to_string(),
52 - domain: config.domain.clone(),
53 - registrar: None,
54 - expiry_date: None,
55 - days_remaining: None,
56 - nameservers: vec![],
57 - checked_at: now,
58 - error: Some(e),
59 - },
62 + Err(e) => error_result(target, config, e),
60 63 }
61 64 }
62 65
63 - /// Determine the WHOIS server for a domain based on its TLD.
66 + fn build_result(
67 + target: &str,
68 + config: &WhoisConfig,
69 + registrar: Option<String>,
70 + expiry_date: Option<String>,
71 + nameservers: Vec<String>,
72 + ) -> WhoisResult {
73 + let days_remaining = expiry_date.as_deref().and_then(compute_days_remaining);
74 +
75 + WhoisResult {
76 + target: target.to_string(),
77 + domain: config.domain.clone(),
78 + registrar,
79 + expiry_date,
80 + days_remaining,
81 + nameservers,
82 + checked_at: chrono::Utc::now().to_rfc3339(),
83 + error: None,
84 + }
85 + }
86 +
87 + fn error_result(target: &str, config: &WhoisConfig, error: String) -> WhoisResult {
88 + WhoisResult {
89 + target: target.to_string(),
90 + domain: config.domain.clone(),
91 + registrar: None,
92 + expiry_date: None,
93 + days_remaining: None,
94 + nameservers: vec![],
95 + checked_at: chrono::Utc::now().to_rfc3339(),
96 + error: Some(error),
97 + }
98 + }
99 +
100 + /// Determine the port-43 WHOIS server for a domain based on its TLD.
101 + ///
102 + /// Only TLDs whose WHOIS server is still answering belong here. `.app` and
103 + /// `.dev` used to map to `whois.nic.google`, which Google Registry retired in
104 + /// favour of RDAP; the hostname is now NXDOMAIN, so those two are RDAP-only.
64 105 pub fn whois_server_for_tld(domain: &str) -> Option<&'static str> {
65 106 let tld = domain.rsplit('.').next()?;
66 107 match tld {
@@ -68,8 +109,6 @@
68 109 "net" => Some("whois.verisign-grs.com"),
69 110 "org" => Some("whois.pir.org"),
70 111 "work" => Some("whois.nic.work"),
71 - "app" => Some("whois.nic.google"),
72 - "dev" => Some("whois.nic.google"),
73 112 "io" => Some("whois.nic.io"),
74 113 "me" => Some("whois.nic.me"),
75 114 "info" => Some("whois.afilias.net"),
@@ -198,10 +237,6 @@
198 237 #[test]
199 238 fn whois_server_known_tlds() {
200 239 assert_eq!(whois_server_for_tld("example.work"), Some("whois.nic.work"));
201 - assert_eq!(
202 - whois_server_for_tld("example.app"),
203 - Some("whois.nic.google")
204 - );
205 240 assert_eq!(
206 241 whois_server_for_tld("example.com"),
207 242 Some("whois.verisign-grs.com")
@@ -218,6 +253,15 @@
218 253 assert_eq!(whois_server_for_tld("example.xyz"), None);
219 254 }
220 255
256 + #[test]
257 + fn whois_server_omits_retired_google_registry() {
258 + // `whois.nic.google` is NXDOMAIN since Google Registry moved to RDAP.
259 + // Mapping .app/.dev to it produced a permanent degraded state on the
260 + // htpy target; these two must stay RDAP-only.
261 + assert_eq!(whois_server_for_tld("htpy.app"), None);
262 + assert_eq!(whois_server_for_tld("example.dev"), None);
263 + }
264 +
221 265 #[test]
222 266 fn parse_whois_verisign_response() {
223 267 let response = r"
@@ -332,7 +376,6 @@
332 376 fn whois_server_all_known_tlds() {
333 377 // Pins every match arm, a mutation swapping or removing an arm would
334 378 // surface as Some→None or wrong host.
335 - assert_eq!(whois_server_for_tld("x.dev"), Some("whois.nic.google"));
336 379 assert_eq!(whois_server_for_tld("x.io"), Some("whois.nic.io"));
337 380 assert_eq!(whois_server_for_tld("x.me"), Some("whois.nic.me"));
338 381 assert_eq!(whois_server_for_tld("x.info"), Some("whois.afilias.net"));
M pom/src/cli/serve.rs +6
@@ -121,6 +121,12 @@
121 121 &token,
122 122 alerter.as_ref(),
123 123 ));
124 + handles.extend(tasks::spawn_test_tasks(
125 + config,
126 + pool,
127 + &token,
128 + alerter.as_ref(),
129 + ));
124 130 // No alerter: the fleet readout has nothing to alert on. See the task module.
125 131 handles.extend(tasks::spawn_synckit_fleet_tasks(config, pool, &token));
126 132 handles.push(tasks::spawn_prune_task(pool, prune_days, &token));
M pom/src/cli/tasks/mod.rs +2
@@ -11,6 +11,7 @@
11 11 mod scan_pipeline;
12 12 mod synckit_fleet;
13 13 mod systemd;
14 + mod test_suite;
14 15 mod tls;
15 16 mod whois;
16 17
@@ -89,5 +90,6 @@
89 90 pub(crate) use scan_pipeline::spawn_scan_pipeline_tasks;
90 91 pub(crate) use synckit_fleet::spawn_synckit_fleet_tasks;
91 92 pub(crate) use systemd::spawn_systemd_tasks;
93 + pub(crate) use test_suite::spawn_test_tasks;
92 94 pub(crate) use tls::spawn_tls_tasks;
93 95 pub(crate) use whois::spawn_whois_tasks;
A pom/src/alerts/test_suite.rs +89
@@ -1,0 +1,89 @@
1 + //! Test-suite alert/recovery messages. Domain half of the Alerter split;
2 + //! shared dispatch/cooldown plumbing lives in the parent module.
3 + //!
4 + //! Distinct from [`super::test_duration`], which fires on a suite that still
5 + //! passes but has slowed down. These fire on pass/fail transitions.
6 +
7 + use tracing::instrument;
8 +
9 + use super::{AlertCategory, AlertMeta, Alerter};
10 +
11 + impl Alerter {
12 + #[instrument(skip_all)]
13 + pub async fn send_test_failure_alert(
14 + &self,
15 + target: &str,
16 + label: &str,
17 + failed: Option<i64>,
18 + exit_code: Option<i64>,
19 + detail: &str,
20 + ) {
21 + let alert_key = format!("tests:{target}");
22 + let subject = match failed {
23 + Some(n) => format!("[PoM] {label}: {n} test(s) failing"),
24 + None => format!("[PoM] {label}: test suite failed to complete"),
25 + };
26 + let body = format!(
27 + "Target: {label} ({target})\n\
28 + Failed: {}\n\
29 + Exit code: {}\n\
30 + Detail: {detail}\n\
31 + Instance: {}\n\
32 + Time: {}\n\n\
33 + - PoM",
34 + failed.map_or_else(|| "unknown".to_string(), |n| n.to_string()),
35 + exit_code.map_or_else(|| "none".to_string(), |c| c.to_string()),
36 + self.instance_name,
37 + chrono::Utc::now().to_rfc3339(),
38 + );
39 +
40 + // A suite that never produced a count did not run at all (SSH refused,
41 + // missing checkout, timeout), which is worse than a known count of
42 + // failures: it means the target is unverified rather than broken.
43 + let priority = if failed.is_some() { "high" } else { "critical" };
44 +
45 + self.fire_failure(
46 + &subject,
47 + &body,
48 + priority,
49 + "pom-tests",
50 + Some(target),
51 + AlertMeta {
52 + key: &alert_key,
53 + category: AlertCategory::TestFailure,
54 + from: None,
55 + to: None,
56 + error: Some(detail),
57 + },
58 + )
59 + .await;
60 + }
61 +
62 + #[instrument(skip_all)]
63 + pub async fn send_test_recovery(&self, target: &str, label: &str) {
64 + let alert_key = format!("tests:{target}");
65 + let subject = format!("[PoM] {label}: test suite passing again");
66 + let body = format!(
67 + "Target: {label} ({target})\n\
68 + The test suite passed on its latest scheduled run.\n\
69 + Instance: {}\n\
70 + Time: {}\n\n\
71 + - PoM",
72 + self.instance_name,
73 + chrono::Utc::now().to_rfc3339(),
74 + );
75 +
76 + self.fire_recovery(
77 + &subject,
78 + &body,
79 + AlertMeta {
80 + key: &alert_key,
81 + category: AlertCategory::TestRecovery,
82 + from: None,
83 + to: None,
84 + error: None,
85 + },
86 + )
87 + .await;
88 + }
89 + }
A pom/src/checks/rdap.rs +455
@@ -1,0 +1,455 @@
1 + //! RDAP domain lookups, the successor protocol to WHOIS.
2 + //!
3 + //! Registries are retiring port-43 WHOIS one TLD at a time. Google Registry
4 + //! already has: `whois.nic.google` is NXDOMAIN, which took `.app` and `.dev`
5 + //! down with it. RDAP is the replacement, and unlike WHOIS it has a registry
6 + //! of registries, so the server for a TLD is discoverable rather than
7 + //! hardcoded.
8 + //!
9 + //! The base URL for a TLD comes from IANA's bootstrap file, cached in process
10 + //! for [`BOOTSTRAP_TTL`]. [`FALLBACK_BASES`] covers the TLDs we actually
11 + //! monitor so a bootstrap outage does not blind the check. A TLD in neither
12 + //! place has no RDAP path and falls back to WHOIS (see [`super::whois`]).
13 +
14 + use std::collections::HashMap;
15 + use std::sync::{Arc, RwLock};
16 + use std::time::{Duration, Instant};
17 +
18 + /// IANA's TLD-to-RDAP-service map.
19 + const BOOTSTRAP_URL: &str = "https://data.iana.org/rdap/dns.json";
20 +
21 + /// How long a fetched bootstrap stays good. The file changes when a registry
22 + /// joins or moves, which is a matter of months, so a day is generous.
23 + const BOOTSTRAP_TTL: Duration = Duration::from_hours(24);
24 +
25 + /// Bases for the TLDs under monitoring, used when the bootstrap is
26 + /// unreachable. Values match IANA's file as of 2026-08-06.
27 + const FALLBACK_BASES: &[(&str, &str)] = &[
28 + ("app", "https://pubapi.registry.google/rdap/"),
29 + ("dev", "https://pubapi.registry.google/rdap/"),
30 + ("work", "https://rdap.nic.work/"),
31 + ("com", "https://rdap.verisign.com/com/v1/"),
32 + ("net", "https://rdap.verisign.com/net/v1/"),
33 + ("org", "https://rdap.publicinterestregistry.org/rdap/"),
34 + ("info", "https://rdap.identitydigital.services/rdap/"),
35 + ];
36 +
37 + static BOOTSTRAP_CACHE: RwLock<Option<CachedBootstrap>> = RwLock::new(None);
38 +
39 + struct CachedBootstrap {
40 + fetched_at: Instant,
41 + bases: Arc<HashMap<String, String>>,
42 + }
43 +
44 + /// Domain facts an RDAP response carries, in the same shape the WHOIS parser
45 + /// produces so both paths feed one result type.
46 + pub struct RdapDomain {
47 + pub registrar: Option<String>,
48 + pub expiry_date: Option<String>,
49 + pub nameservers: Vec<String>,
50 + }
51 +
52 + /// Look up a domain over RDAP. `Ok(None)` means no RDAP service is known for
53 + /// the TLD, which is a signal to try WHOIS, not a failure.
54 + pub async fn lookup(domain: &str) -> Result<Option<RdapDomain>, String> {
55 + let Some(tld) = domain.rsplit('.').next().filter(|t| !t.is_empty()) else {
56 + return Ok(None);
57 + };
58 + let Some(base) = base_for_tld(tld).await else {
59 + return Ok(None);
60 + };
61 + query(&base, domain).await.map(Some)
62 + }
63 +
64 + /// Resolve a TLD to its RDAP base URL, preferring the IANA bootstrap and
65 + /// falling back to the built-in table.
66 + async fn base_for_tld(tld: &str) -> Option<String> {
67 + let tld = tld.to_lowercase();
68 +
69 + if let Some(bases) = bootstrap().await
70 + && let Some(base) = bases.get(&tld)
71 + {
72 + return Some(base.clone());
73 + }
74 +
75 + FALLBACK_BASES
76 + .iter()
77 + .find(|(t, _)| *t == tld)
78 + .map(|(_, base)| (*base).to_string())
79 + }
80 +
81 + /// The cached bootstrap map, refetched once past [`BOOTSTRAP_TTL`]. A failed
82 + /// fetch returns the stale map if there is one, since a map from last week
83 + /// beats no map at all.
84 + async fn bootstrap() -> Option<Arc<HashMap<String, String>>> {
85 + if let Ok(guard) = BOOTSTRAP_CACHE.read()
86 + && let Some(cached) = guard.as_ref()
87 + && cached.fetched_at.elapsed() < BOOTSTRAP_TTL
88 + {
89 + return Some(Arc::clone(&cached.bases));
90 + }
91 +
92 + match fetch_bootstrap().await {
93 + Ok(bases) => {
94 + let bases = Arc::new(bases);
95 + if let Ok(mut guard) = BOOTSTRAP_CACHE.write() {
96 + *guard = Some(CachedBootstrap {
97 + fetched_at: Instant::now(),
98 + bases: Arc::clone(&bases),
99 + });
100 + }
101 + Some(bases)
102 + }
103 + Err(e) => {
104 + tracing::warn!("RDAP bootstrap fetch failed, using fallback table: {e}");
105 + BOOTSTRAP_CACHE
106 + .read()
107 + .ok()
108 + .and_then(|guard| guard.as_ref().map(|c| Arc::clone(&c.bases)))
109 + }
110 + }
111 + }
112 +
113 + async fn fetch_bootstrap() -> Result<HashMap<String, String>, String> {
114 + let client = crate::tls::https_client_builder()
115 + .timeout(Duration::from_secs(15))
116 + .build()
117 + .map_err(|e| format!("RDAP client build failed: {e}"))?;
118 +
119 + let body = client
120 + .get(BOOTSTRAP_URL)
121 + .send()
122 + .await
123 + .map_err(|e| format!("RDAP bootstrap request failed: {e}"))?
124 + .error_for_status()
125 + .map_err(|e| format!("RDAP bootstrap returned {e}"))?
126 + .text()
127 + .await
128 + .map_err(|e| format!("RDAP bootstrap body read failed: {e}"))?;
129 +
130 + parse_bootstrap(&body)
131 + }
132 +
133 + /// Flatten the bootstrap's `services` array into a TLD-to-base map.
134 + ///
135 + /// Each service is `[[tld, ...], [url, ...]]`; the first HTTPS URL wins, and
136 + /// a service with no HTTPS URL is skipped rather than downgraded.
137 + pub fn parse_bootstrap(body: &str) -> Result<HashMap<String, String>, String> {
138 + let doc: serde_json::Value =
139 + serde_json::from_str(body).map_err(|e| format!("RDAP bootstrap not valid JSON: {e}"))?;
140 +
141 + let services = doc
142 + .get("services")
143 + .and_then(|s| s.as_array())
144 + .ok_or_else(|| "RDAP bootstrap has no services array".to_string())?;
145 +
146 + let mut bases = HashMap::new();
147 + for service in services {
148 + let Some(entry) = service.as_array() else {
149 + continue;
150 + };
151 + let (Some(tlds), Some(urls)) = (
152 + entry.first().and_then(|v| v.as_array()),
153 + entry.get(1).and_then(|v| v.as_array()),
154 + ) else {
155 + continue;
156 + };
157 +
158 + let Some(url) = urls
159 + .iter()
160 + .filter_map(|u| u.as_str())
161 + .find(|u| u.starts_with("https://"))
162 + else {
163 + continue;
164 + };
165 +
166 + for tld in tlds.iter().filter_map(|t| t.as_str()) {
167 + bases.insert(tld.to_lowercase(), url.to_string());
168 + }
169 + }
170 +
171 + if bases.is_empty() {
172 + return Err("RDAP bootstrap contained no usable services".to_string());
173 + }
174 + Ok(bases)
175 + }
176 +
177 + async fn query(base: &str, domain: &str) -> Result<RdapDomain, String> {
178 + let url = format!("{}domain/{domain}", ensure_trailing_slash(base));
179 +
180 + let client = crate::tls::https_client_builder()
181 + .timeout(Duration::from_secs(10))
182 + .build()
183 + .map_err(|e| format!("RDAP client build failed: {e}"))?;
184 +
185 + let response = client
186 + .get(&url)
187 + .header("Accept", "application/rdap+json")
188 + .send()
189 + .await
190 + .map_err(|e| format!("RDAP request to {url} failed: {e}"))?;
191 +
192 + let status = response.status();
193 + if !status.is_success() {
194 + return Err(format!("RDAP query to {url} returned HTTP {status}"));
195 + }
196 +
197 + let body = response
198 + .text()
199 + .await
200 + .map_err(|e| format!("RDAP body read failed: {e}"))?;
201 +
202 + parse_domain(&body)
203 + }
204 +
205 + fn ensure_trailing_slash(base: &str) -> String {
206 + if base.ends_with('/') {
207 + base.to_string()
208 + } else {
209 + format!("{base}/")
210 + }
211 + }
212 +
213 + /// Pull registrar, expiry and nameservers out of an RDAP domain object.
214 + pub fn parse_domain(body: &str) -> Result<RdapDomain, String> {
215 + let doc: serde_json::Value =
216 + serde_json::from_str(body).map_err(|e| format!("RDAP response not valid JSON: {e}"))?;
217 +
218 + let expiry_date = doc
219 + .get("events")
220 + .and_then(|e| e.as_array())
221 + .and_then(|events| {
222 + events
223 + .iter()
224 + .find(|e| e.get("eventAction").and_then(|a| a.as_str()) == Some("expiration"))
225 + })
226 + .and_then(|e| e.get("eventDate"))
227 + .and_then(|d| d.as_str())
228 + .map(str::to_string);
229 +
230 + let registrar = doc
231 + .get("entities")
232 + .and_then(|e| e.as_array())
233 + .and_then(|entities| {
234 + entities.iter().find(|e| {
235 + e.get("roles")
236 + .and_then(|r| r.as_array())
237 + .is_some_and(|roles| roles.iter().any(|r| r.as_str() == Some("registrar")))
238 + })
239 + })
240 + .and_then(vcard_full_name);
241 +
242 + let mut nameservers = Vec::new();
243 + if let Some(list) = doc.get("nameservers").and_then(|n| n.as_array()) {
244 + for ns in list {
245 + let Some(name) = ns.get("ldhName").and_then(|n| n.as_str()) else {
246 + continue;
247 + };
248 + let name = name.trim_end_matches('.').to_lowercase();
249 + if !name.is_empty() && !nameservers.contains(&name) {
250 + nameservers.push(name);
251 + }
252 + }
253 + }
254 +
255 + Ok(RdapDomain {
256 + registrar,
257 + expiry_date,
258 + nameservers,
259 + })
260 + }
261 +
262 + /// Extract the `fn` (formatted name) property from an entity's jCard.
263 + ///
264 + /// jCard is `["vcard", [[name, params, type, value], ...]]`, so the value
265 + /// sits at index 3 of the property whose index 0 is `"fn"`.
266 + fn vcard_full_name(entity: &serde_json::Value) -> Option<String> {
267 + let properties = entity.get("vcardArray")?.as_array()?.get(1)?.as_array()?;
268 +
269 + properties
270 + .iter()
271 + .filter_map(|p| p.as_array())
272 + .find(|p| p.first().and_then(|k| k.as_str()) == Some("fn"))
273 + .and_then(|p| p.get(3))
274 + .and_then(|v| v.as_str())
275 + .filter(|v| !v.is_empty())
276 + .map(str::to_string)
277 + }
278 +
279 + #[cfg(test)]
280 + mod tests {
281 + use super::*;
282 +
283 + const HTPY_RDAP: &str = r#"{
284 + "objectClassName": "domain",
285 + "ldhName": "htpy.app",
286 + "events": [
287 + {"eventAction": "registration", "eventDate": "2026-03-11T19:13:33.195Z"},
288 + {"eventAction": "expiration", "eventDate": "2027-03-11T19:13:33.195Z"},
289 + {"eventAction": "last changed", "eventDate": "2026-03-16T19:13:33.195Z"}
290 + ],
291 + "entities": [
292 + {
293 + "roles": ["registrar"],
294 + "vcardArray": ["vcard", [
295 + ["version", {}, "text", "4.0"],
296 + ["fn", {}, "text", "CloudFlare, Inc."]
297 + ]]
298 + }
299 + ],
300 + "nameservers": [
301 + {"ldhName": "PAM.NS.CLOUDFLARE.COM."},
302 + {"ldhName": "simon.ns.cloudflare.com"}
303 + ]
304 + }"#;
305 +
306 + #[test]
307 + fn parse_domain_extracts_expiry_registrar_nameservers() {
308 + let parsed = parse_domain(HTPY_RDAP).unwrap();
309 + assert_eq!(
310 + parsed.expiry_date.as_deref(),
311 + Some("2027-03-11T19:13:33.195Z")
312 + );
313 + assert_eq!(parsed.registrar.as_deref(), Some("CloudFlare, Inc."));
314 + assert_eq!(
315 + parsed.nameservers,
316 + vec![
317 + "pam.ns.cloudflare.com".to_string(),
318 + "simon.ns.cloudflare.com".to_string()
319 + ]
320 + );
321 + }
322 +
323 + #[test]
324 + fn parse_domain_ignores_non_expiration_events() {
325 + // Pins the eventAction filter: registration comes first in the array,
326 + // so a missing filter would return the wrong date.
327 + let parsed = parse_domain(HTPY_RDAP).unwrap();
328 + assert_ne!(
329 + parsed.expiry_date.as_deref(),
330 + Some("2026-03-11T19:13:33.195Z")
331 + );
332 + }
333 +
334 + #[test]
335 + fn parse_domain_skips_entities_without_registrar_role() {
336 + let body = r#"{
337 + "entities": [
338 + {"roles": ["technical"], "vcardArray": ["vcard", [["fn", {}, "text", "Tech Co"]]]},
339 + {"roles": ["registrar"], "vcardArray": ["vcard", [["fn", {}, "text", "Real Registrar"]]]}
340 + ]
341 + }"#;
342 + let parsed = parse_domain(body).unwrap();
343 + assert_eq!(parsed.registrar.as_deref(), Some("Real Registrar"));
344 + }
345 +
346 + #[test]
347 + fn parse_domain_tolerates_missing_sections() {
348 + let parsed = parse_domain(r#"{"objectClassName": "domain"}"#).unwrap();
349 + assert!(parsed.expiry_date.is_none());
350 + assert!(parsed.registrar.is_none());
351 + assert!(parsed.nameservers.is_empty());
352 + }
353 +
354 + #[test]
355 + fn parse_domain_deduplicates_nameservers() {
356 + let body =
357 + r#"{"nameservers": [{"ldhName": "ns1.example.com."}, {"ldhName": "NS1.example.com"}]}"#;
358 + let parsed = parse_domain(body).unwrap();
359 + assert_eq!(parsed.nameservers, vec!["ns1.example.com".to_string()]);
360 + }
361 +
362 + #[test]
363 + fn parse_domain_rejects_non_json() {
364 + assert!(parse_domain("not json at all").is_err());
365 + }
366 +
367 + #[test]
368 + fn parse_bootstrap_flattens_services() {
369 + let body = r#"{
370 + "version": "1.0",
371 + "services": [
372 + [["app", "dev"], ["https://pubapi.registry.google/rdap/"]],
373 + [["work"], ["https://rdap.nic.work/"]]
374 + ]
375 + }"#;
376 + let bases = parse_bootstrap(body).unwrap();
377 + assert_eq!(
378 + bases.get("app").map(String::as_str),
379 + Some("https://pubapi.registry.google/rdap/")
380 + );
381 + assert_eq!(
382 + bases.get("dev").map(String::as_str),
383 + Some("https://pubapi.registry.google/rdap/")
384 + );
385 + assert_eq!(
386 + bases.get("work").map(String::as_str),
387 + Some("https://rdap.nic.work/")
388 + );
389 + }
390 +
391 + #[test]
392 + fn parse_bootstrap_prefers_https() {
393 + let body = r#"{"services": [[["test"], ["http://insecure.example/", "https://secure.example/"]]]}"#;
394 + let bases = parse_bootstrap(body).unwrap();
395 + assert_eq!(
396 + bases.get("test").map(String::as_str),
397 + Some("https://secure.example/")
398 + );
399 + }
400 +
401 + #[test]
402 + fn parse_bootstrap_skips_http_only_services() {
403 + let body = r#"{
404 + "services": [
405 + [["insecure"], ["http://only.example/"]],
406 + [["fine"], ["https://ok.example/"]]
407 + ]
408 + }"#;
409 + let bases = parse_bootstrap(body).unwrap();
410 + assert!(!bases.contains_key("insecure"));
411 + assert!(bases.contains_key("fine"));
412 + }
413 +
414 + #[test]
415 + fn parse_bootstrap_lowercases_tlds() {
416 + let body = r#"{"services": [[["APP"], ["https://example/"]]]}"#;
417 + let bases = parse_bootstrap(body).unwrap();
418 + assert!(bases.contains_key("app"));
419 + }
420 +
421 + #[test]
422 + fn parse_bootstrap_rejects_missing_services() {
423 + assert!(parse_bootstrap(r#"{"version": "1.0"}"#).is_err());
424 + }
425 +
426 + #[test]
427 + fn parse_bootstrap_rejects_empty_result() {
428 + assert!(parse_bootstrap(r#"{"services": []}"#).is_err());
429 + }
430 +
431 + #[test]
432 + fn fallback_covers_monitored_tlds() {
433 + // The point of the fallback table: an IANA outage must not blind the
434 + // check for a TLD we actually watch.
435 + for tld in ["app", "dev", "work", "com", "net", "org", "info"] {
436 + assert!(
437 + FALLBACK_BASES.iter().any(|(t, _)| *t == tld),
438 + "no fallback RDAP base for .{tld}"
439 + );
440 + }
441 + }
442 +
443 + #[test]
444 + fn fallback_bases_are_https() {
445 + for (tld, base) in FALLBACK_BASES {
446 + assert!(base.starts_with("https://"), ".{tld} fallback is not HTTPS");
447 + }
448 + }
449 +
450 + #[test]
451 + fn ensure_trailing_slash_is_idempotent() {
452 + assert_eq!(ensure_trailing_slash("https://x/rdap/"), "https://x/rdap/");
453 + assert_eq!(ensure_trailing_slash("https://x/rdap"), "https://x/rdap/");
454 + }
455 + }
A pom/src/cli/tasks/test_suite.rs +326
@@ -1,0 +1,326 @@
1 + //! Background test-suite task.
2 + //!
3 + //! Every target with a `[tests]` block used to sit at `pending` forever: the
4 + //! serve loop had no test task, so `staleness_days` was only ever read by the
5 + //! status readout to grade a run that nothing scheduled. This closes that loop.
6 + //!
7 + //! The sweep interval is not the test cadence. Each tick asks
8 + //! [`drift::compute_test_staleness`] whether a target's last run is still good
9 + //! (never run / older than `staleness_days` / deployed version changed since);
10 + //! only a stale target actually pays for a run. Reusing the same predicate the
11 + //! status readout uses is deliberate: the scheduler and the dashboard cannot
12 + //! disagree about what "stale" means.
13 +
14 + use tokio::task::JoinHandle;
15 + use tracing::{error, info};
16 +
17 + use pom::alerts::Alerter;
18 + use pom::checks::{drift, ssh};
19 + use pom::config::{Config, TestsConfig};
20 + use pom::db;
21 + use pom::types::TestRun;
22 +
23 + use super::{CheckInterval, configured_targets};
24 +
25 + pub(crate) fn spawn_test_tasks(
26 + config: &Config,
27 + pool: &sqlx::SqlitePool,
28 + cancel: &tokio_util::sync::CancellationToken,
29 + alerter: Option<&Alerter>,
30 + ) -> Vec<JoinHandle<()>> {
31 + let sweep_secs = config.serve.test_sweep_interval_secs;
32 + let mut handles = Vec::new();
33 +
34 + for (name, target_config) in configured_targets(config) {
35 + let Some(tests_config) = target_config.tests else {
36 + continue;
37 + };
38 + let label = target_config.label.clone();
39 + let pool = pool.clone();
40 + let alerter = alerter.cloned();
41 + let cancel = cancel.clone();
42 +
43 + info!(
44 + "{name}: test staleness sweep every {sweep_secs}s (threshold={}d, command={})",
45 + tests_config.staleness_days, tests_config.command
46 + );
47 +
48 + handles.push(tokio::spawn(async move {
49 + let mut ticks = CheckInterval::new(sweep_secs, cancel);
50 +
51 + while ticks.next().await {
52 + let Some(reason) = staleness_reason(&pool, &name, &tests_config).await else {
53 + continue;
54 + };
55 +
56 + info!("{name}: running tests ({reason})");
57 + let run = ssh::run_tests(&name, &tests_config, None).await;
58 + let previous_passed = db::get_latest_test_run(&pool, &name)
59 + .await
60 + .ok()
61 + .flatten()
62 + .map(|r| r.passed);
63 +
64 + if let Err(e) = store_run(&pool, &name, &run).await {
65 + error!("{name}: failed to store test run: {e}");
66 + }
67 +
68 + info!(
69 + "{}: tests {} ({} passed, {} failed, {}s)",
70 + name,
71 + if run.passed { "passed" } else { "FAILED" },
72 + run.summary.total_passed.unwrap_or(-1),
73 + run.summary.total_failed.unwrap_or(-1),
74 + run.duration_secs.unwrap_or(-1),
75 + );
76 +
77 + if let Some(ref alerter) = alerter {
78 + alert_on_transition(alerter, &name, &label, &run, previous_passed).await;
79 + }
80 + }
81 + }));
82 + }
83 +
84 + handles
85 + }
86 +
87 + /// Why this target's tests need running, or `None` if the last run still counts.
88 + async fn staleness_reason(
89 + pool: &sqlx::SqlitePool,
90 + name: &str,
91 + tests_config: &TestsConfig,
92 + ) -> Option<String> {
93 + let current_version = db::get_health_history(pool, Some(name), 1)
94 + .await
95 + .unwrap_or_default()
96 + .first()
97 + .and_then(|s| s.details.as_ref())
98 + .and_then(|d| d.version.clone());
99 +
100 + let latest_test = db::get_latest_test_run(pool, name).await.unwrap_or(None);
101 +
102 + let tested_version = match latest_test {
103 + Some(ref test) => db::get_version_at_time(pool, name, &test.started_at)
104 + .await
105 + .unwrap_or(None),
106 + None => None,
107 + };
108 +
109 + let staleness = drift::compute_test_staleness(
110 + current_version.as_deref(),
111 + tested_version.as_deref(),
112 + latest_test.as_ref().map(|t| t.started_at.as_str()),
113 + tests_config.staleness_days,
114 + );
115 +
116 + staleness
117 + .stale
118 + .then(|| staleness.reason.unwrap_or_else(|| "stale".to_string()))
119 + }
120 +
121 + /// Persist a run and its per-test details. Details are best-effort: losing them
122 + /// costs regression detection on the next run, not the run record itself.
123 + async fn store_run(
124 + pool: &sqlx::SqlitePool,
125 + name: &str,
126 + run: &TestRun,
127 + ) -> Result<(), pom::error::PomError> {
128 + let run_id = db::insert_test_run(pool, run).await?;
129 +
130 + if !run.summary.details.is_empty()
131 + && let Err(e) = db::insert_test_details(pool, run_id, &run.summary.details).await
132 + {
133 + error!("{name}: failed to store test details: {e}");
134 + }
135 +
136 + Ok(())
137 + }
138 +
139 + /// What a sweep's result is worth telling someone about.
140 + #[derive(Debug, Clone, Copy, PartialEq, Eq)]
141 + enum Transition {
142 + Broke,
143 + Recovered,
144 + }
145 +
146 + /// Alert on pass/fail transitions only, so a suite that stays red does not page
147 + /// once per sweep. A first-ever run that fails counts as broken: there is no
148 + /// prior state, and silence would be indistinguishable from passing.
149 + fn transition(previous_passed: Option<bool>, now_passed: bool) -> Option<Transition> {
150 + match (previous_passed.unwrap_or(true), now_passed) {
151 + (true, false) => Some(Transition::Broke),
152 + (false, true) => Some(Transition::Recovered),
153 + _ => None,
154 + }
155 + }
156 +
157 + async fn alert_on_transition(
158 + alerter: &Alerter,
159 + name: &str,
160 + label: &str,
161 + run: &TestRun,
162 + previous_passed: Option<bool>,
163 + ) {
164 + match transition(previous_passed, run.passed) {
165 + Some(Transition::Broke) => {
166 + let detail = failure_detail(run);
167 + alerter
168 + .send_test_failure_alert(
169 + name,
170 + label,
171 + run.summary.total_failed,
172 + run.exit_code.map(i64::from),
173 + &detail,
174 + )
175 + .await;
176 + }
177 + Some(Transition::Recovered) => alerter.send_test_recovery(name, label).await,
178 + None => {}
179 + }
180 + }
181 +
182 + /// A short, alert-sized reason for a red suite: the failing test names when the
183 + /// output parsed, otherwise the tail of the raw output, which is where an SSH
184 + /// or missing-checkout error lands.
185 + fn failure_detail(run: &TestRun) -> String {
186 + let failed: Vec<&str> = run
187 + .summary
188 + .details
189 + .iter()
190 + .filter(|d| !d.passed)
191 + .map(|d| d.test_name.as_str())
192 + .take(10)
193 + .collect();
194 +
195 + if !failed.is_empty() {
196 + return failed.join(", ");
197 + }
198 +
199 + let tail: Vec<&str> = run
200 + .raw_output
201 + .lines()
202 + .filter(|l| !l.trim().is_empty())
203 + .rev()
204 + .take(5)
205 + .collect();
206 +
207 + if tail.is_empty() {
208 + "no output captured".to_string()
209 + } else {
210 + tail.into_iter().rev().collect::<Vec<_>>().join(" | ")
211 + }
212 + }
213 +
214 + #[cfg(test)]
215 + mod tests {
216 + use super::*;
217 + use pom::types::{TestDetail, TestSummary};
218 +
219 + fn run_with(passed: bool, details: Vec<TestDetail>, raw_output: &str) -> TestRun {
220 + TestRun {
221 + id: None,
222 + target: "mnw".to_string(),
223 + started_at: "2026-08-06T00:00:00Z".to_string(),
224 + finished_at: None,
225 + duration_secs: Some(1),
226 + exit_code: Some(if passed { 0 } else { 101 }),
227 + passed,
228 + summary: TestSummary {
229 + steps: vec![],
230 + total_passed: None,
231 + total_failed: None,
232 + details,
233 + },
234 + raw_output: raw_output.to_string(),
235 + filter: None,
236 + }
237 + }
238 +
239 + fn detail(name: &str, passed: bool) -> TestDetail {
240 + TestDetail {
241 + test_name: name.to_string(),
242 + passed,
243 + }
244 + }
245 +
246 + #[test]
247 + fn failure_detail_prefers_failing_test_names() {
248 + let run = run_with(
249 + false,
250 + vec![
251 + detail("checks::whois::ok", true),
252 + detail("checks::rdap::broken", false),
253 + detail("db::also_broken", false),
254 + ],
255 + "irrelevant output",
256 + );
257 + assert_eq!(
258 + failure_detail(&run),
259 + "checks::rdap::broken, db::also_broken"
260 + );
261 + }
262 +
263 + #[test]
264 + fn failure_detail_caps_the_name_list() {
265 + let details: Vec<TestDetail> = (0..20).map(|i| detail(&format!("t{i}"), false)).collect();
266 + let run = run_with(false, details, "");
267 + assert_eq!(failure_detail(&run).split(", ").count(), 10);
268 + }
269 +
270 + #[test]
271 + fn failure_detail_falls_back_to_output_tail_in_order() {
272 + // The suite never ran, so there are no test names. This is the shape an
273 + // SSH refusal or a missing checkout produces, and it is the case that
274 + // matters most: the target is unverified, not merely broken.
275 + let run = run_with(
276 + false,
277 + vec![],
278 + "ssh: connect to host 100.106.221.39 port 22: Connection refused\n\
279 + exit status 255\n",
280 + );
281 + let detail = failure_detail(&run);
282 + assert!(detail.starts_with("ssh: connect to host"), "got {detail}");
283 + assert!(detail.ends_with("exit status 255"), "got {detail}");
284 + }
285 +
286 + #[test]
287 + fn failure_detail_handles_no_output() {
288 + assert_eq!(
289 + failure_detail(&run_with(false, vec![], " \n\n")),
290 + "no output captured"
291 + );
292 + }
293 +
294 + #[test]
295 + fn transition_first_ever_run_failing_is_a_break() {
296 + // `None` means nothing has run before. Treating it as previously-passing
297 + // is what makes the first red run alert instead of being swallowed as
298 + // no-change, which is exactly the state every astra target is in today.
299 + assert_eq!(transition(None, false), Some(Transition::Broke));
300 + }
301 +
302 + #[test]
303 + fn transition_first_ever_run_passing_is_silent() {
304 + assert_eq!(transition(None, true), None);
305 + }
306 +
307 + #[test]
308 + fn transition_still_red_does_not_repage() {
309 + assert_eq!(transition(Some(false), false), None);
310 + }
311 +
312 + #[test]
313 + fn transition_still_green_is_silent() {
314 + assert_eq!(transition(Some(true), true), None);
315 + }
316 +
317 + #[test]
318 + fn transition_red_to_green_recovers() {
319 + assert_eq!(transition(Some(false), true), Some(Transition::Recovered));
320 + }
321 +
322 + #[test]
323 + fn transition_green_to_red_breaks() {
324 + assert_eq!(transition(Some(true), false), Some(Transition::Broke));
325 + }
326 + }