Skip to main content

max / pom

Split cli.rs into cli/ directory module, add dashboard, test and monitoring enhancements (v0.2.6) Break 1035-line cli.rs into cli/{mod,serve,status,incident}.rs and cli/tasks/{health,tls,routes,dns,whois,prune,meta_alert}.rs. Each background task loop is now a standalone spawn function. Also includes test duration drift detection, SSH test detail parsing, WHOIS monitoring, dashboard improvements, and expanded integration test coverage.
Co-Authored-By
Claude Opus 4.6 <noreply@anthropic.com>
Author: Max J. <87768334+MaxJMath@users.noreply.github.com> · 2026-03-17 02:07 UTC
Commit: 3e1314507e60b08463ae0223fa996c279cac3bd3
Parent: c0f3d59
29 files changed, +2190 insertions, -953 deletions
M Cargo.lock +2 -1
@@ -1665,7 +1665,7 @@
1665 1665
1666 1666 [[package]]
1667 1667 name = "pom"
1668 - version = "0.2.4"
1668 + version = "0.2.5"
1669 1669 dependencies = [
1670 1670 "axum",
1671 1671 "chrono",
@@ -1682,6 +1682,7 @@
1682 1682 "serde",
1683 1683 "serde_json",
1684 1684 "sqlx",
1685 + "subtle",
1685 1686 "thiserror 2.0.18",
1686 1687 "tokio",
1687 1688 "tokio-rustls",
M Cargo.toml +4 -1
@@ -1,6 +1,6 @@
1 1 [package]
2 2 name = "pom"
3 - version = "0.2.5"
3 + version = "0.2.6"
4 4 edition = "2024"
5 5 license-file = "LICENSE"
6 6
@@ -39,6 +39,9 @@
39 39 # Config
40 40 toml = "0.8"
41 41
42 + # Constant-time comparison
43 + subtle = "2"
44 +
42 45 # Errors
43 46 thiserror = "2"
44 47
M deploy/pom-astra.toml +6
@@ -69,6 +69,12 @@
69 69 [targets.mt.tls]
70 70 host = "forums.makenot.work"
71 71
72 + [targets.mt.tests]
73 + ssh = "max@100.106.221.39"
74 + command = "cd /home/max/staging/multithreaded && cargo test --workspace 2>&1"
75 + timeout_secs = 300
76 + staleness_days = 7
77 +
72 78 [targets.htpy]
73 79 label = "htpy.app"
74 80
M deploy/pom-hetzner.toml +6
@@ -69,6 +69,12 @@
69 69 [targets.mt.tls]
70 70 host = "forums.makenot.work"
71 71
72 + [targets.mt.tests]
73 + ssh = "max@100.106.221.39"
74 + command = "cd /home/max/staging/multithreaded && cargo test --workspace 2>&1"
75 + timeout_secs = 300
76 + staleness_days = 7
77 +
72 78 [targets.htpy]
73 79 label = "htpy.app"
74 80
M src/alerts.rs +150
@@ -450,6 +450,77 @@
450 450 self.record_alert(&alert_key, "latency_recovery", None, None, None).await;
451 451 }
452 452
453 + #[instrument(skip_all)]
454 + pub async fn send_test_duration_drift_alert(
455 + &self,
456 + target: &str,
457 + label: &str,
458 + drift_message: &str,
459 + ) {
460 + let alert_key = format!("test_duration:{target}");
461 + if self.is_within_cooldown(&alert_key).await {
462 + info!("alert cooldown active for {alert_key}, skipping");
463 + return;
464 + }
465 +
466 + let subject = format!("[PoM] {target}: test duration drift detected");
467 + let body = format!(
468 + "Target: {label} ({target})\n\
469 + {drift_message}\n\
470 + Instance: {}\n\
471 + Time: {}\n\n\
472 + - PoM",
473 + self.instance_name,
474 + chrono::Utc::now().to_rfc3339(),
475 + );
476 +
477 + self.send_email(&subject, &body).await;
478 + self.record_alert(&alert_key, "test_duration_drift", None, None, Some(drift_message)).await;
479 + }
480 +
481 + /// All monitored targets are unreachable — likely a network issue with PoM itself.
482 + #[instrument(skip_all)]
483 + pub async fn send_monitoring_offline_alert(&self, target_count: usize) {
484 + let alert_key = "monitoring:self";
485 + if self.is_within_cooldown(alert_key).await {
486 + info!("alert cooldown active for {alert_key}, skipping");
487 + return;
488 + }
489 +
490 + let subject = format!("[PoM] all {target_count} targets unreachable");
491 + let body = format!(
492 + "All {target_count} monitored targets are non-operational.\n\
493 + This likely indicates a network issue with the PoM instance itself,\n\
494 + not an actual outage of all targets.\n\n\
495 + Instance: {}\n\
496 + Time: {}\n\n\
497 + - PoM",
498 + self.instance_name,
499 + chrono::Utc::now().to_rfc3339(),
500 + );
501 +
502 + self.send_email(&subject, &body).await;
503 + self.record_alert(alert_key, "monitoring_offline", None, None, None).await;
504 + }
505 +
506 + /// At least one target is reachable again after a monitoring-offline event.
507 + #[instrument(skip_all)]
508 + pub async fn send_monitoring_recovery(&self) {
509 + let alert_key = "monitoring:self";
510 + let subject = "[PoM] monitoring recovered".to_string();
511 + let body = format!(
512 + "At least one target is reachable again.\n\
513 + Instance: {}\n\
514 + Time: {}\n\n\
515 + - PoM",
516 + self.instance_name,
517 + chrono::Utc::now().to_rfc3339(),
518 + );
519 +
520 + self.send_email(&subject, &body).await;
521 + self.record_alert(alert_key, "monitoring_recovery", None, None, None).await;
522 + }
523 +
453 524 async fn is_within_cooldown(&self, target: &str) -> bool {
454 525 let latest = match db::get_latest_alert_for_target(&self.pool, target).await {
455 526 Ok(Some(row)) => row,
@@ -662,4 +733,83 @@
662 733 // Different target should NOT be in cooldown
663 734 assert!(!alerter.is_within_cooldown("health:other.com").await);
664 735 }
736 +
737 + #[tokio::test]
738 + async fn tls_expiry_alert_cooldown_key() {
739 + let pool = db::connect_in_memory().await.unwrap();
740 + let alerter = test_alerter(pool.clone());
741 +
742 + assert!(!alerter.is_within_cooldown("tls:mnw").await);
743 + alerter.send_tls_expiry_alert("mnw", "makenot.work", 10, "2026-04-01T00:00:00Z").await;
744 + assert!(alerter.is_within_cooldown("tls:mnw").await);
745 + }
746 +
747 + #[tokio::test]
748 + async fn tls_error_alert_cooldown_key() {
749 + let pool = db::connect_in_memory().await.unwrap();
750 + let alerter = test_alerter(pool.clone());
751 +
752 + assert!(!alerter.is_within_cooldown("tls:mnw").await);
753 + alerter.send_tls_error_alert("mnw", "makenot.work", "certificate expired").await;
754 + assert!(alerter.is_within_cooldown("tls:mnw").await);
755 + }
756 +
757 + #[tokio::test]
758 + async fn latency_drift_alert_cooldown_key() {
759 + let pool = db::connect_in_memory().await.unwrap();
760 + let alerter = test_alerter(pool.clone());
761 +
762 + assert!(!alerter.is_within_cooldown("latency:mnw").await);
763 + alerter.send_latency_drift_alert("mnw", "MakeNotWork", "avg 500ms, baseline 100ms").await;
764 + assert!(alerter.is_within_cooldown("latency:mnw").await);
765 + }
766 +
767 + #[tokio::test]
768 + async fn test_duration_drift_alert_cooldown_key() {
769 + let pool = db::connect_in_memory().await.unwrap();
770 + let alerter = test_alerter(pool.clone());
771 +
772 + assert!(!alerter.is_within_cooldown("test_duration:mnw").await);
773 + alerter.send_test_duration_drift_alert("mnw", "MakeNotWork", "drift: 120s vs 60s baseline").await;
774 + assert!(alerter.is_within_cooldown("test_duration:mnw").await);
775 + }
776 +
777 + #[tokio::test]
778 + async fn monitoring_offline_alert_cooldown_key() {
779 + let pool = db::connect_in_memory().await.unwrap();
780 + let alerter = test_alerter(pool.clone());
781 +
782 + assert!(!alerter.is_within_cooldown("monitoring:self").await);
783 + alerter.send_monitoring_offline_alert(3).await;
784 + assert!(alerter.is_within_cooldown("monitoring:self").await);
785 + }
786 +
787 + #[tokio::test]
788 + async fn route_recovery_does_not_start_cooldown() {
789 + let pool = db::connect_in_memory().await.unwrap();
790 + let alerter = test_alerter(pool.clone());
791 +
792 + alerter.send_route_recovery_alert("mnw", "MakeNotWork", &["/health".to_string()]).await;
793 + // Recovery alerts are excluded from cooldown lookups, so sending a recovery
794 + // should NOT put the key into cooldown.
795 + assert!(!alerter.is_within_cooldown("route:mnw").await);
796 + }
797 +
798 + #[tokio::test]
799 + async fn dns_recovery_does_not_start_cooldown() {
800 + let pool = db::connect_in_memory().await.unwrap();
801 + let alerter = test_alerter(pool.clone());
802 +
803 + alerter.send_dns_recovery_alert("mnw", "MakeNotWork").await;
804 + assert!(!alerter.is_within_cooldown("dns:mnw").await);
805 + }
806 +
807 + #[tokio::test]
808 + async fn tls_recovery_does_not_start_cooldown() {
809 + let pool = db::connect_in_memory().await.unwrap();
810 + let alerter = test_alerter(pool.clone());
811 +
812 + alerter.send_tls_recovery("mnw", "MakeNotWork", 90).await;
813 + assert!(!alerter.is_within_cooldown("tls:mnw").await);
814 + }
665 815 }
M src/api.rs +18 -2
@@ -12,7 +12,7 @@
12 12 use axum::{Json, Router};
13 13 use serde::Serialize;
14 14
15 - use crate::checks::http::compute_test_staleness;
15 + use crate::checks::http::{compute_test_staleness, detect_test_duration_drift};
16 16 use crate::config::Config;
17 17 use crate::db;
18 18 use crate::peer::SharedMeshState;
@@ -92,7 +92,9 @@
92 92 match auth_header {
93 93 Some(header) if header.starts_with("Bearer ") => {
94 94 let token = &header[7..];
95 - if token == expected {
95 + // Constant-time comparison to prevent timing side-channels
96 + use subtle::ConstantTimeEq;
97 + if token.as_bytes().ct_eq(expected.as_bytes()).into() {
96 98 Ok(next.run(req).await)
97 99 } else {
98 100 Err((StatusCode::UNAUTHORIZED, Json(serde_json::json!({
@@ -192,6 +194,9 @@
192 194 /// Latest WHOIS check result. Omitted if no WHOIS monitoring is configured.
193 195 #[serde(skip_serializing_if = "Option::is_none")]
194 196 whois: Option<db::WhoisCheckRow>,
197 + /// Test duration drift warning. Omitted if no drift detected or no test config.
198 + #[serde(skip_serializing_if = "Option::is_none")]
199 + test_duration_drift: Option<String>,
195 200 }
196 201
197 202 #[derive(Serialize)]
@@ -310,6 +315,16 @@
310 315 None
311 316 };
312 317
318 + // Compute test duration drift for targets with test config
319 + let test_duration_drift = if config.get_target(name).and_then(|t| t.tests.as_ref()).is_some() {
320 + let durations = db::get_test_durations(pool, name, 13)
321 + .await
322 + .unwrap_or_default();
323 + detect_test_duration_drift(&durations, 10, 3, 1.5)
324 + } else {
325 + None
326 + };
327 +
313 328 let current_incident = db::get_open_incident(pool, name)
314 329 .await
315 330 .unwrap_or(None);
@@ -360,6 +375,7 @@
360 375 latency_24h,
361 376 tls,
362 377 test_staleness,
378 + test_duration_drift,
363 379 current_incident,
364 380 incidents,
365 381 route_status,
M src/config.rs +19 -9
@@ -312,18 +312,28 @@
312 312 let mut config: Config = toml::from_str(&contents)?;
313 313
314 314 // Allow postmark_token from environment variable (preferred over config file)
315 - if let Some(ref mut alerts) = config.alerts {
316 - if alerts.postmark_token.is_none() {
317 - if let Ok(token) = std::env::var("POM_POSTMARK_TOKEN") {
318 - alerts.postmark_token = Some(token);
319 - }
320 - }
315 + if let Some(ref mut alerts) = config.alerts
316 + && alerts.postmark_token.is_none()
317 + && let Ok(token) = std::env::var("POM_POSTMARK_TOKEN")
318 + {
319 + alerts.postmark_token = Some(token);
321 320 }
322 321
323 322 // Allow api_token from environment variable (preferred over config file)
324 - if config.serve.api_token.is_none() {
325 - if let Ok(token) = std::env::var("POM_API_TOKEN") {
326 - config.serve.api_token = Some(token);
323 + if config.serve.api_token.is_none()
324 + && let Ok(token) = std::env::var("POM_API_TOKEN")
325 + {
326 + config.serve.api_token = Some(token);
327 + }
328 +
329 + // Validate no duplicate target labels
330 + let mut seen_labels = std::collections::HashSet::new();
331 + for (name, target) in &config.targets {
332 + if !seen_labels.insert(target.label.to_lowercase()) {
333 + return Err(PomError::Config(format!(
334 + "duplicate target label: \"{}\" (on target \"{name}\")",
335 + target.label
336 + )));
327 337 }
328 338 }
329 339
M src/dashboard.rs +34 -1
@@ -15,7 +15,7 @@
15 15 }
16 16
17 17 /// Escape a string for safe embedding in a JS string literal.
18 - fn escape_js(s: &str) -> String {
18 + pub(crate) fn escape_js(s: &str) -> String {
19 19 s.replace('\\', "\\\\").replace('"', "\\\"")
20 20 }
21 21
@@ -253,6 +253,9 @@
253 253 if (t.test_staleness && t.test_staleness.stale) {
254 254 html += '<div class="alert-bar stale-bar">Tests stale: ' + esc(t.test_staleness.reason) + '</div>';
255 255 }
256 + if (t.test_duration_drift) {
257 + html += '<div class="alert-bar stale-bar">' + esc(t.test_duration_drift) + '</div>';
258 + }
256 259
257 260 html += '</div>';
258 261 return html;
@@ -404,3 +407,33 @@
404 407 refresh();
405 408 timer = setInterval(tick, 1000);
406 409 "##;
410 +
411 + #[cfg(test)]
412 + mod tests {
413 + use super::*;
414 +
415 + #[test]
416 + fn escape_js_backslash() {
417 + assert_eq!(escape_js(r"a\b"), r"a\\b");
418 + }
419 +
420 + #[test]
421 + fn escape_js_double_quote() {
422 + assert_eq!(escape_js(r#"a"b"#), r#"a\"b"#);
423 + }
424 +
425 + #[test]
426 + fn escape_js_both() {
427 + assert_eq!(escape_js(r#"a\"b"#), r#"a\\\"b"#);
428 + }
429 +
430 + #[test]
431 + fn escape_js_clean_string() {
432 + assert_eq!(escape_js("hello"), "hello");
433 + }
434 +
435 + #[test]
436 + fn escape_js_empty() {
437 + assert_eq!(escape_js(""), "");
438 + }
439 + }
M src/db.rs +100 -2
@@ -11,7 +11,7 @@
11 11 use tracing::{info, instrument};
12 12
13 13 use crate::error::Result;
14 - use crate::types::{DnsCheckResult, HealthDetails, HealthSnapshot, HealthStatus, TestRun, TestSummary, TlsStatus, WhoisResult};
14 + use crate::types::{DnsCheckResult, HealthDetails, HealthSnapshot, HealthStatus, TestDetail, TestRun, TestSummary, TlsStatus, WhoisResult};
15 15
16 16 /// Each migration is a (version, description, SQL) tuple. Versions start at 1.
17 17 /// The SQL may contain multiple statements separated by semicolons.
@@ -136,6 +136,17 @@
136 136 );
137 137 CREATE INDEX idx_whois_checks_target ON whois_checks(target, id DESC);
138 138 "#),
139 + (7, "add test_details table", r#"
140 + CREATE TABLE IF NOT EXISTS test_details (
141 + id INTEGER PRIMARY KEY AUTOINCREMENT,
142 + run_id INTEGER NOT NULL REFERENCES test_runs(id) ON DELETE CASCADE,
143 + test_name TEXT NOT NULL,
144 + passed INTEGER NOT NULL,
145 + duration_ms INTEGER
146 + );
147 + CREATE INDEX IF NOT EXISTS idx_test_details_run_id ON test_details(run_id);
148 + CREATE INDEX IF NOT EXISTS idx_test_details_name ON test_details(test_name, run_id DESC);
149 + "#),
139 150 ];
140 151
141 152 #[instrument(skip_all)]
@@ -413,6 +424,83 @@
413 424 Ok(row.map(|r| r.into_test_run()))
414 425 }
415 426
427 + // --- Test detail queries ---
428 +
429 + /// Insert per-test results for a given test run.
430 + #[instrument(skip_all)]
431 + pub async fn insert_test_details(
432 + pool: &SqlitePool,
433 + run_id: i64,
434 + details: &[TestDetail],
435 + ) -> Result<()> {
436 + for detail in details {
437 + sqlx::query(
438 + "INSERT INTO test_details (run_id, test_name, passed) VALUES (?, ?, ?)",
439 + )
440 + .bind(run_id)
441 + .bind(&detail.test_name)
442 + .bind(detail.passed)
443 + .execute(pool)
444 + .await?;
445 + }
446 + Ok(())
447 + }
448 +
449 + /// Find tests that passed in the previous run but failed in this one (regressions).
450 + #[instrument(skip_all)]
451 + pub async fn get_test_regressions(
452 + pool: &SqlitePool,
453 + target: &str,
454 + current_run_id: i64,
455 + ) -> Result<Vec<String>> {
456 + // Find the run immediately before this one for the same target
457 + let prev_run = sqlx::query_as::<_, (i64,)>(
458 + "SELECT id FROM test_runs
459 + WHERE target = ? AND id < ?
460 + ORDER BY id DESC LIMIT 1",
461 + )
462 + .bind(target)
463 + .bind(current_run_id)
464 + .fetch_optional(pool)
465 + .await?;
466 +
467 + let Some((prev_id,)) = prev_run else {
468 + return Ok(vec![]);
469 + };
470 +
471 + // Tests that passed in prev run but failed in current run
472 + let rows = sqlx::query_as::<_, (String,)>(
473 + "SELECT curr.test_name FROM test_details curr
474 + INNER JOIN test_details prev ON prev.test_name = curr.test_name AND prev.run_id = ?
475 + WHERE curr.run_id = ? AND curr.passed = 0 AND prev.passed = 1",
476 + )
477 + .bind(prev_id)
478 + .bind(current_run_id)
479 + .fetch_all(pool)
480 + .await?;
481 +
482 + Ok(rows.into_iter().map(|r| r.0).collect())
483 + }
484 +
485 + /// Get test duration history for a target (most recent first).
486 + #[instrument(skip_all)]
487 + pub async fn get_test_durations(
488 + pool: &SqlitePool,
489 + target: &str,
490 + limit: i64,
491 + ) -> Result<Vec<(String, i64)>> {
492 + let rows = sqlx::query_as::<_, (String, i64)>(
493 + "SELECT started_at, duration_secs FROM test_runs
494 + WHERE target = ? AND duration_secs IS NOT NULL
495 + ORDER BY id DESC LIMIT ?",
496 + )
497 + .bind(target)
498 + .bind(limit)
499 + .fetch_all(pool)
500 + .await?;
501 + Ok(rows)
502 + }
503 +
416 504 /// Get the version from the health check closest to (but before) a given timestamp.
417 505 #[instrument(skip_all)]
418 506 pub async fn get_version_at_time(
@@ -869,6 +957,7 @@
869 957 pub struct PruneResult {
870 958 pub health: u64,
871 959 pub tests: u64,
960 + pub test_details: u64,
872 961 pub heartbeats: u64,
873 962 pub alerts: u64,
874 963 pub tls: u64,
@@ -888,7 +977,7 @@
888 977 // Guard: days <= 0 would set cutoff to now (or the future), deleting
889 978 // everything. Treat this as a no-op instead.
890 979 if days <= 0 {
891 - return Ok(PruneResult { health: 0, tests: 0, heartbeats: 0, alerts: 0, tls: 0, incidents: 0, routes: 0, dns: 0, whois: 0 });
980 + return Ok(PruneResult { health: 0, tests: 0, test_details: 0, heartbeats: 0, alerts: 0, tls: 0, incidents: 0, routes: 0, dns: 0, whois: 0 });
892 981 }
893 982
894 983 let cutoff = chrono::Utc::now() - chrono::Duration::days(days);
@@ -904,6 +993,13 @@
904 993 .execute(pool)
905 994 .await?;
906 995
996 + // Prune orphaned test_details (run was deleted above).
997 + let test_details_result = sqlx::query(
998 + "DELETE FROM test_details WHERE run_id NOT IN (SELECT id FROM test_runs)",
999 + )
1000 + .execute(pool)
1001 + .await?;
1002 +
907 1003 let peer_hb_result = sqlx::query("DELETE FROM peer_heartbeats WHERE checked_at < ?")
908 1004 .bind(&cutoff_str)
909 1005 .execute(pool)
@@ -942,6 +1038,7 @@
942 1038 Ok(PruneResult {
943 1039 health: health_result.rows_affected(),
944 1040 tests: test_result.rows_affected(),
1041 + test_details: test_details_result.rows_affected(),
945 1042 heartbeats: peer_hb_result.rows_affected(),
946 1043 alerts: alerts_result.rows_affected(),
947 1044 tls: tls_result.rows_affected(),
@@ -1113,6 +1210,7 @@
1113 1210 steps: vec![],
1114 1211 total_passed: None,
1115 1212 total_failed: None,
1213 + details: vec![],
1116 1214 });
1117 1215
1118 1216 TestRun {
M src/display.rs +38 -6
@@ -221,6 +221,30 @@
221 221 out
222 222 }
223 223
224 + /// Format regression warnings for CLI display.
225 + pub fn format_regressions(regressions: &[String]) -> String {
226 + let mut out = String::new();
227 + writeln!(out, "\nREGRESSIONS (passed last run, failed now):").unwrap();
228 + for name in regressions {
229 + writeln!(out, " {name}").unwrap();
230 + }
231 + out
232 + }
233 +
234 + /// Format test duration trend for CLI display.
235 + pub fn format_test_duration_trend(durations: &[(String, i64)], drift: Option<&str>) -> String {
236 + let mut out = String::new();
237 + if !durations.is_empty() {
238 + write!(out, " Duration trend (last {}): ", durations.len()).unwrap();
239 + let strs: Vec<String> = durations.iter().map(|(_, d)| format!("{d}s")).collect();
240 + writeln!(out, "{}", strs.join(", ")).unwrap();
241 + }
242 + if let Some(msg) = drift {
243 + writeln!(out, " DRIFT: {msg}").unwrap();
244 + }
245 + out
246 + }
247 +
224 248 /// Format DNS check results and WHOIS results for CLI display.
225 249 pub fn format_dns_results(dns_results: &[DnsCheckResult], whois_results: &[WhoisResult]) -> String {
226 250 let mut out = String::new();
@@ -262,8 +286,8 @@
262 286 /// Format prune results for CLI display.
263 287 pub fn format_prune(result: &PruneResult, days: i64) -> String {
264 288 format!(
265 - "Pruned {} health checks, {} test runs, {} peer heartbeats, {} alerts, {} TLS checks, {} incidents, {} route checks, {} DNS checks, {} WHOIS checks older than {} days.\n",
266 - result.health, result.tests, result.heartbeats, result.alerts, result.tls,
289 + "Pruned {} health checks, {} test runs, {} test details, {} peer heartbeats, {} alerts, {} TLS checks, {} incidents, {} route checks, {} DNS checks, {} WHOIS checks older than {} days.\n",
290 + result.health, result.tests, result.test_details, result.heartbeats, result.alerts, result.tls,
267 291 result.incidents, result.routes, result.dns, result.whois, days
268 292 )
269 293 }
@@ -464,6 +488,7 @@
464 488 ],
465 489 total_passed: Some(759),
466 490 total_failed: Some(0),
491 + details: vec![],
467 492 },
468 493 raw_output: String::new(),
469 494 filter: None,
@@ -494,6 +519,7 @@
494 519 ],
495 520 total_passed: Some(750),
496 521 total_failed: Some(9),
522 + details: vec![],
497 523 },
498 524 raw_output: "thread 'test_foo' panicked at 'assertion failed'".to_string(),
499 525 filter: None,
@@ -521,6 +547,7 @@
521 547 steps: vec![],
522 548 total_passed: None,
523 549 total_failed: None,
550 + details: vec![],
524 551 },
525 552 raw_output: String::new(),
526 553 filter: None,
@@ -561,6 +588,7 @@
561 588 steps: vec![],
562 589 total_passed: Some(100),
563 590 total_failed: Some(0),
591 + details: vec![],
564 592 },
565 593 raw_output: String::new(),
566 594 filter: None,
@@ -610,6 +638,7 @@
610 638 steps: vec![],
611 639 total_passed: Some(80),
612 640 total_failed: Some(5),
641 + details: vec![],
613 642 },
614 643 raw_output: String::new(),
615 644 filter: None,
@@ -787,6 +816,7 @@
787 816 steps: vec![],
788 817 total_passed: Some(810),
789 818 total_failed: Some(0),
819 + details: vec![],
790 820 },
791 821 raw_output: String::new(),
792 822 filter: None,
@@ -803,6 +833,7 @@
803 833 steps: vec![],
804 834 total_passed: None,
805 835 total_failed: None,
836 + details: vec![],
806 837 },
807 838 raw_output: String::new(),
808 839 filter: None,
@@ -828,6 +859,7 @@
828 859 steps: vec![],
829 860 total_passed: None,
830 861 total_failed: None,
862 + details: vec![],
831 863 },
832 864 raw_output: String::new(),
833 865 filter: None,
@@ -843,24 +875,24 @@
843 875 #[test]
844 876 fn prune_formatting() {
845 877 let result = PruneResult {
846 - health: 5, tests: 3, heartbeats: 10, alerts: 2, tls: 1,
878 + health: 5, tests: 3, test_details: 15, heartbeats: 10, alerts: 2, tls: 1,
847 879 incidents: 4, routes: 0, dns: 8, whois: 2,
848 880 };
849 881 let out = format_prune(&result, 30);
850 882 assert_eq!(
851 883 out,
852 - "Pruned 5 health checks, 3 test runs, 10 peer heartbeats, 2 alerts, 1 TLS checks, 4 incidents, 0 route checks, 8 DNS checks, 2 WHOIS checks older than 30 days.\n"
884 + "Pruned 5 health checks, 3 test runs, 15 test details, 10 peer heartbeats, 2 alerts, 1 TLS checks, 4 incidents, 0 route checks, 8 DNS checks, 2 WHOIS checks older than 30 days.\n"
853 885 );
854 886 }
855 887
856 888 #[test]
857 889 fn prune_zero_records() {
858 890 let result = PruneResult {
859 - health: 0, tests: 0, heartbeats: 0, alerts: 0, tls: 0,
891 + health: 0, tests: 0, test_details: 0, heartbeats: 0, alerts: 0, tls: 0,
860 892 incidents: 0, routes: 0, dns: 0, whois: 0,
861 893 };
862 894 let out = format_prune(&result, 7);
863 - assert!(out.contains("Pruned 0 health checks, 0 test runs, 0 peer heartbeats, 0 alerts, 0 TLS checks, 0 incidents, 0 route checks, 0 DNS checks, 0 WHOIS checks older than 7 days."));
895 + assert!(out.contains("Pruned 0 health checks, 0 test runs, 0 test details, 0 peer heartbeats, 0 alerts, 0 TLS checks, 0 incidents, 0 route checks, 0 DNS checks, 0 WHOIS checks older than 7 days."));
864 896 }
865 897
866 898 // --- format_mesh ---
M src/types.rs +12 -1
@@ -2,7 +2,7 @@
2 2
3 3 use serde::{Deserialize, Serialize};
4 4
5 - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
5 + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
6 6 #[serde(rename_all = "lowercase")]
7 7 pub enum HealthStatus {
8 8 /// All checks passed — endpoint responded with expected status and content.
@@ -113,6 +113,17 @@
113 113 pub total_passed: Option<i64>,
114 114 /// Total number of failing tests, if parseable from the output.
115 115 pub total_failed: Option<i64>,
116 + /// Per-test results parsed from individual `test <name> ... ok/FAILED` lines.
117 + #[serde(default)]
118 + pub details: Vec<TestDetail>,
119 + }
120 +
121 + #[derive(Debug, Clone, Serialize, Deserialize)]
122 + pub struct TestDetail {
123 + /// Fully qualified test name (e.g. "workflows::endorsements::toggle").
124 + pub test_name: String,
125 + /// Whether this individual test passed.
126 + pub passed: bool,
116 127 }
117 128
118 129 #[derive(Debug, Clone, Serialize, Deserialize)]
M tests/integration.rs +353 -20
@@ -106,6 +106,7 @@
106 106 ],
107 107 total_passed: Some(759),
108 108 total_failed: Some(0),
109 + details: vec![],
109 110 },
110 111 raw_output: "test output here".to_string(),
111 112 filter: None,
@@ -136,7 +137,7 @@
136 137 duration_secs: None,
137 138 exit_code: None,
138 139 passed: true,
139 - summary: TestSummary { steps: vec![], total_passed: None, total_failed: None },
140 + summary: TestSummary { steps: vec![], total_passed: None, total_failed: None, details: vec![] },
140 141 raw_output: String::new(),
141 142 filter: None,
142 143 };
@@ -372,10 +373,10 @@
372 373
373 374 #[tokio::test]
374 375 async fn migration_fresh_db_reaches_latest_version() {
375 - // A fresh in-memory DB should run all migrations and reach version 5.
376 + // A fresh in-memory DB should run all migrations and reach the latest version.
376 377 let pool = db::connect_in_memory().await.unwrap();
377 378 let version = db::get_schema_version(&pool).await.unwrap();
378 - assert_eq!(version, 6);
379 + assert_eq!(version, 7);
379 380
380 381 // Verify the schema_version table has entries for each migration
381 382 let rows = sqlx::query_as::<_, (i64, String)>(
@@ -384,7 +385,7 @@
384 385 .fetch_all(&pool)
385 386 .await
386 387 .unwrap();
387 - assert_eq!(rows.len(), 6);
388 + assert_eq!(rows.len(), 7);
388 389 assert_eq!(rows[0].0, 1);
389 390 assert_eq!(rows[0].1, "initial schema");
390 391 assert_eq!(rows[1].0, 2);
@@ -397,6 +398,8 @@
397 398 assert_eq!(rows[4].1, "add route_checks table");
398 399 assert_eq!(rows[5].0, 6);
399 400 assert_eq!(rows[5].1, "add dns_checks and whois_checks tables");
401 + assert_eq!(rows[6].0, 7);
402 + assert_eq!(rows[6].1, "add test_details table");
400 403
401 404 // Verify actual tables were created by inserting data
402 405 let snapshot = HealthSnapshot {
@@ -416,18 +419,18 @@
416 419 async fn migration_already_current_is_idempotent() {
417 420 // Running migrations on an already-migrated DB should be a no-op.
418 421 let pool = db::connect_in_memory().await.unwrap();
419 - assert_eq!(db::get_schema_version(&pool).await.unwrap(), 6);
422 + assert_eq!(db::get_schema_version(&pool).await.unwrap(), 7);
420 423
421 424 // Run migrations again
422 425 db::run_migrations(&pool).await.unwrap();
423 - assert_eq!(db::get_schema_version(&pool).await.unwrap(), 6);
426 + assert_eq!(db::get_schema_version(&pool).await.unwrap(), 7);
424 427
425 - // schema_version should still have exactly six entries (not duplicated)
428 + // schema_version should still have exactly seven entries (not duplicated)
426 429 let count = sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM schema_version")
427 430 .fetch_one(&pool)
428 431 .await
429 432 .unwrap();
430 - assert_eq!(count.0, 6);
433 + assert_eq!(count.0, 7);
431 434 }
432 435
433 436 #[tokio::test]
@@ -487,7 +490,7 @@
487 490 db::run_migrations(&pool).await.unwrap();
488 491
489 492 // Version should be 6 (stamped v1 + ran v2 + ran v3 + ran v4 + ran v5 + ran v6)
490 - assert_eq!(db::get_schema_version(&pool).await.unwrap(), 6);
493 + assert_eq!(db::get_schema_version(&pool).await.unwrap(), 7);
491 494
492 495 // Description should indicate pre-existing
493 496 let row = sqlx::query_as::<_, (String,)>(
@@ -544,6 +547,7 @@
544 547 steps: vec![StepResult { name: "cargo test".to_string(), passed: true }],
545 548 total_passed: Some(100),
546 549 total_failed: Some(0),
550 + details: vec![],
547 551 },
548 552 raw_output: "all good".to_string(),
549 553 filter: None,
@@ -687,7 +691,7 @@
687 691 duration_secs: None,
688 692 exit_code: None,
689 693 passed: true,
690 - summary: TestSummary { steps: vec![], total_passed: None, total_failed: None },
694 + summary: TestSummary { steps: vec![], total_passed: None, total_failed: None, details: vec![] },
691 695 raw_output: "HUGE OUTPUT THAT SHOULD NOT APPEAR".to_string(),
692 696 filter: None,
693 697 };
@@ -728,7 +732,7 @@
728 732 duration_secs: None,
729 733 exit_code: None,
730 734 passed: true,
731 - summary: TestSummary { steps: vec![], total_passed: None, total_failed: None },
735 + summary: TestSummary { steps: vec![], total_passed: None, total_failed: None, details: vec![] },
732 736 raw_output: "running 42 tests\ntest result: ok".to_string(),
733 737 filter: None,
734 738 };
@@ -787,7 +791,7 @@
787 791 async fn migration_v2_creates_alerts_table() {
788 792 let pool = db::connect_in_memory().await.unwrap();
789 793 let version = db::get_schema_version(&pool).await.unwrap();
790 - assert_eq!(version, 6);
794 + assert_eq!(version, 7);
791 795
792 796 // Verify alerts table exists by inserting
793 797 let id = db::insert_alert(&pool, "mnw", "health", Some("operational"), Some("error"), None)
@@ -855,7 +859,7 @@
855 859 async fn migration_v3_creates_tls_checks_table() {
856 860 let pool = db::connect_in_memory().await.unwrap();
857 861 let version = db::get_schema_version(&pool).await.unwrap();
858 - assert_eq!(version, 6);
862 + assert_eq!(version, 7);
859 863
860 864 // Verify tls_checks table exists by inserting
861 865 let status = pom::types::TlsStatus {
@@ -1070,7 +1074,7 @@
1070 1074 async fn migration_v4_creates_incidents_table() {
1071 1075 let pool = db::connect_in_memory().await.unwrap();
1072 1076 let version = db::get_schema_version(&pool).await.unwrap();
1073 - assert_eq!(version, 6);
1077 + assert_eq!(version, 7);
1074 1078
1075 1079 // Verify incidents table exists by inserting
1076 1080 let id = db::insert_incident(&pool, "mnw", "operational", "degraded")
@@ -1199,7 +1203,7 @@
1199 1203 async fn migration_v5_creates_route_checks_table() {
1200 1204 let pool = db::connect_in_memory().await.unwrap();
1201 1205 let version = db::get_schema_version(&pool).await.unwrap();
1202 - assert_eq!(version, 6);
1206 + assert_eq!(version, 7);
1203 1207
1204 1208 // Verify route_checks table exists by inserting
1205 1209 let result = pom::checks::routes::RouteCheckResult {
@@ -1410,7 +1414,7 @@
1410 1414 assert_eq!(json["target"], "mnw");
1411 1415 assert_eq!(json["window_hours"], 24);
1412 1416 assert_eq!(json["bucket_minutes"], 60);
1413 - assert!(json["buckets"].as_array().unwrap().len() > 0);
1417 + assert!(!json["buckets"].as_array().unwrap().is_empty());
1414 1418 assert!(json["overall"].is_object());
1415 1419 }
1416 1420
@@ -1583,7 +1587,7 @@
1583 1587 duration_secs: Some(60),
1584 1588 exit_code: Some(0),
1585 1589 passed: true,
1586 - summary: TestSummary { steps: vec![], total_passed: Some(100), total_failed: Some(0) },
1590 + summary: TestSummary { steps: vec![], total_passed: Some(100), total_failed: Some(0), details: vec![] },
1587 1591 raw_output: String::new(),
1588 1592 filter: None,
1589 1593 };
@@ -1647,7 +1651,7 @@
1647 1651 duration_secs: Some(60),
1648 1652 exit_code: Some(0),
1649 1653 passed: true,
1650 - summary: TestSummary { steps: vec![], total_passed: Some(100), total_failed: Some(0) },
1654 + summary: TestSummary { steps: vec![], total_passed: Some(100), total_failed: Some(0), details: vec![] },
1651 1655 raw_output: String::new(),
1652 1656 filter: None,
1653 1657 };
@@ -1710,7 +1714,7 @@
1710 1714 duration_secs: Some(60),
1711 1715 exit_code: Some(0),
1712 1716 passed: true,
1713 - summary: TestSummary { steps: vec![], total_passed: Some(100), total_failed: Some(0) },
1717 + summary: TestSummary { steps: vec![], total_passed: Some(100), total_failed: Some(0), details: vec![] },
1714 1718 raw_output: String::new(),
1715 1719 filter: None,
1716 1720 };
@@ -2193,7 +2197,7 @@
2193 2197 async fn migration_v6_creates_dns_and_whois_tables() {
2194 2198 let pool = db::connect_in_memory().await.unwrap();
2195 2199 let version = db::get_schema_version(&pool).await.unwrap();
2196 - assert_eq!(version, 6);
2200 + assert_eq!(version, 7);
2197 2201
2198 2202 // Verify dns_checks table exists
2199 2203 let dns_result = DnsCheckResult {
@@ -2716,3 +2720,460 @@
2716 2720 assert_eq!(status, 200);
2717 2721 assert!(body.contains("HAS_MESH = false"), "should set HAS_MESH to false");
2718 2722 }
2723 +
2724 + // --- Per-test detail tracking ---
2725 +
2726 + #[tokio::test]
2727 + async fn insert_and_query_test_details() {
2728 + let pool = db::connect_in_memory().await.unwrap();
2729 +
2730 + let run = TestRun {
2731 + id: None,
2732 + target: "mnw".to_string(),
2733 + started_at: "2026-03-16T00:00:00Z".to_string(),
2734 + finished_at: Some("2026-03-16T00:02:00Z".to_string()),
2735 + duration_secs: Some(120),
2736 + exit_code: Some(0),
2737 + passed: true,
2738 + summary: TestSummary {
2739 + steps: vec![],
2740 + total_passed: Some(3),
2741 + total_failed: Some(0),
2742 + details: vec![
2743 + TestDetail { test_name: "foo::bar".to_string(), passed: true },
2744 + TestDetail { test_name: "foo::baz".to_string(), passed: true },
2745 + TestDetail { test_name: "foo::qux".to_string(), passed: true },
2746 + ],
2747 + },
2748 + raw_output: String::new(),
2749 + filter: None,
2750 + };
2751 +
2752 + let run_id = db::insert_test_run(&pool, &run).await.unwrap();
2753 + db::insert_test_details(&pool, run_id, &run.summary.details).await.unwrap();
2754 +
2755 + // Verify via regression detection (no previous run = no regressions)
2756 + let regressions = db::get_test_regressions(&pool, "mnw", run_id).await.unwrap();
2757 + assert!(regressions.is_empty());
2758 + }
2759 +
2760 + #[tokio::test]
2761 + async fn regression_detection_finds_newly_failing_tests() {
2762 + let pool = db::connect_in_memory().await.unwrap();
2763 +
2764 + // First run: all pass
2765 + let run1 = TestRun {
2766 + id: None,
2767 + target: "mnw".to_string(),
2768 + started_at: "2026-03-16T00:00:00Z".to_string(),
2769 + finished_at: Some("2026-03-16T00:02:00Z".to_string()),
2770 + duration_secs: Some(120),
2771 + exit_code: Some(0),
2772 + passed: true,
2773 + summary: TestSummary {
2774 + steps: vec![],
2775 + total_passed: Some(3),
2776 + total_failed: Some(0),
2777 + details: vec![
2778 + TestDetail { test_name: "foo::bar".to_string(), passed: true },
2779 + TestDetail { test_name: "foo::baz".to_string(), passed: true },
2780 + TestDetail { test_name: "foo::qux".to_string(), passed: true },
2781 + ],
2782 + },
2783 + raw_output: String::new(),
2784 + filter: None,
2785 + };
2786 +
2787 + let run1_id = db::insert_test_run(&pool, &run1).await.unwrap();
2788 + db::insert_test_details(&pool, run1_id, &run1.summary.details).await.unwrap();
2789 +
2790 + // Second run: foo::baz fails
2791 + let run2 = TestRun {
2792 + id: None,
2793 + target: "mnw".to_string(),
2794 + started_at: "2026-03-16T00:05:00Z".to_string(),
2795 + finished_at: Some("2026-03-16T00:07:00Z".to_string()),
2796 + duration_secs: Some(120),
2797 + exit_code: Some(1),
2798 + passed: false,
2799 + summary: TestSummary {
2800 + steps: vec![],
2801 + total_passed: Some(2),
2802 + total_failed: Some(1),
2803 + details: vec![
2804 + TestDetail { test_name: "foo::bar".to_string(), passed: true },
2805 + TestDetail { test_name: "foo::baz".to_string(), passed: false },
2806 + TestDetail { test_name: "foo::qux".to_string(), passed: true },
2807 + ],
2808 + },
2809 + raw_output: String::new(),
2810 + filter: None,
2811 + };
2812 +
2813 + let run2_id = db::insert_test_run(&pool, &run2).await.unwrap();
2814 + db::insert_test_details(&pool, run2_id, &run2.summary.details).await.unwrap();
2815 +
2816 + let regressions = db::get_test_regressions(&pool, "mnw", run2_id).await.unwrap();
2817 + assert_eq!(regressions.len(), 1);
2818 + assert_eq!(regressions[0], "foo::baz");
2819 + }
2820 +
2821 + #[tokio::test]
2822 + async fn regression_ignores_already_failing_tests() {
2823 + let pool = db::connect_in_memory().await.unwrap();
2824 +
2825 + // Both runs: foo::baz fails
2826 + for (i, ts) in ["00:00:00", "00:05:00"].iter().enumerate() {
2827 + let run = TestRun {
2828 + id: None,
2829 + target: "mnw".to_string(),
2830 + started_at: format!("2026-03-16T{ts}Z"),
2831 + finished_at: None,
2832 + duration_secs: Some(120),
2833 + exit_code: Some(1),
2834 + passed: false,
2835 + summary: TestSummary {
2836 + steps: vec![],
2837 + total_passed: Some(2),
2838 + total_failed: Some(1),
2839 + details: vec![
2840 + TestDetail { test_name: "foo::bar".to_string(), passed: true },
2841 + TestDetail { test_name: "foo::baz".to_string(), passed: false },
2842 + ],
2843 + },
2844 + raw_output: String::new(),
2845 + filter: None,
2846 + };
2847 +
2848 + let run_id = db::insert_test_run(&pool, &run).await.unwrap();
2849 + db::insert_test_details(&pool, run_id, &run.summary.details).await.unwrap();
2850 +
2851 + if i == 1 {
2852 + // Second run — baz was already failing, not a regression
2853 + let regressions = db::get_test_regressions(&pool, "mnw", run_id).await.unwrap();
2854 + assert!(regressions.is_empty());
2855 + }
2856 + }
2857 + }
2858 +
2859 + // --- Test duration drift detection ---
2860 +
2861 + #[tokio::test]
2862 + async fn test_duration_drift_detected() {
2863 + use pom::checks::http::detect_test_duration_drift;
2864 +
2865 + // 10 baseline runs at 60s, 3 recent runs at 120s (2x baseline > 1.5x threshold)
2866 + let mut durations: Vec<(String, i64)> = Vec::new();
2867 + // Most recent first
2868 + for i in 0..3 {
2869 + durations.push((format!("2026-03-16T00:{:02}:00Z", 12 - i), 120));
2870 + }
2871 + for i in 0..10 {
2872 + durations.push((format!("2026-03-16T00:{:02}:00Z", 9 - i), 60));
2873 + }
2874 +
2875 + let drift = detect_test_duration_drift(&durations, 10, 3, 1.5);
2876 + assert!(drift.is_some());
2877 + let msg = drift.unwrap();
2878 + assert!(msg.contains("drift"), "drift message: {msg}");
2879 + }
2880 +
2881 + #[tokio::test]
2882 + async fn test_duration_no_drift_when_stable() {
2883 + use pom::checks::http::detect_test_duration_drift;
2884 +
2885 + // All runs at ~60s
2886 + let mut durations: Vec<(String, i64)> = Vec::new();
2887 + for i in 0..13 {
2888 + durations.push((format!("2026-03-16T00:{:02}:00Z", 12 - i), 60));
2889 + }
2890 +
2891 + let drift = detect_test_duration_drift(&durations, 10, 3, 1.5);
2892 + assert!(drift.is_none());
2893 + }
2894 +
2895 + #[tokio::test]
2896 + async fn test_duration_drift_not_enough_data() {
2897 + use pom::checks::http::detect_test_duration_drift;
2898 +
2899 + // Only 5 runs (need 13 for baseline 10 + recent 3)
2900 + let durations: Vec<(String, i64)> = (0..5)
2901 + .map(|i| (format!("2026-03-16T00:{:02}:00Z", i), 120))
2902 + .collect();
2903 +
2904 + let drift = detect_test_duration_drift(&durations, 10, 3, 1.5);
2905 + assert!(drift.is_none());
2906 + }
2907 +
2908 + #[tokio::test]
2909 + async fn get_test_durations_returns_ordered() {
2910 + let pool = db::connect_in_memory().await.unwrap();
2911 +
2912 + for (i, secs) in [60, 80, 100].iter().enumerate() {
2913 + let run = TestRun {
2914 + id: None,
2915 + target: "mnw".to_string(),
2916 + started_at: format!("2026-03-16T00:{:02}:00Z", i),
2917 + finished_at: None,
2918 + duration_secs: Some(*secs),
2919 + exit_code: Some(0),
2920 + passed: true,
2921 + summary: TestSummary { steps: vec![], total_passed: None, total_failed: None, details: vec![] },
2922 + raw_output: String::new(),
2923 + filter: None,
2924 + };
2925 + db::insert_test_run(&pool, &run).await.unwrap();
2926 + }
2927 +
2928 + let durations = db::get_test_durations(&pool, "mnw", 10).await.unwrap();
2929 + assert_eq!(durations.len(), 3);
2930 + // Most recent first
2931 + assert_eq!(durations[0].1, 100);
2932 + assert_eq!(durations[2].1, 60);
2933 + }
2934 +
2935 + // --- Uptime percent tests ---
2936 +
2937 + #[tokio::test]
2938 + async fn uptime_percent_all_operational() {
2939 + let pool = db::connect_in_memory().await.unwrap();
2940 +
2941 + for i in 0..10 {
2942 + let snapshot = HealthSnapshot {
2943 + id: None,
2944 + target: "mnw".to_string(),
2945 + status: HealthStatus::Operational,
2946 + checked_at: (chrono::Utc::now() - chrono::Duration::hours(i)).to_rfc3339(),
2947 + response_time_ms: 100,
2948 + details: None,
2949 + error: None,
2950 + };
2951 + db::insert_health_check(&pool, &snapshot).await.unwrap();
2952 + }
2953 +
2954 + let pct = db::get_uptime_percent(&pool, "mnw", 24).await.unwrap();
2955 + assert_eq!(pct, Some(100.0));
2956 + }
2957 +
2958 + #[tokio::test]
2959 + async fn uptime_percent_mixed() {
2960 + let pool = db::connect_in_memory().await.unwrap();
2961 +
2962 + // 8 operational
2963 + for i in 0..8 {
2964 + let snapshot = HealthSnapshot {
2965 + id: None,
2966 + target: "mnw".to_string(),
2967 + status: HealthStatus::Operational,
2968 + checked_at: (chrono::Utc::now() - chrono::Duration::hours(i)).to_rfc3339(),
2969 + response_time_ms: 100,
2970 + details: None,
2971 + error: None,
2972 + };
2973 + db::insert_health_check(&pool, &snapshot).await.unwrap();
2974 + }
2975 +
2976 + // 2 error
2977 + for i in 8..10 {
2978 + let snapshot = HealthSnapshot {
2979 + id: None,
2980 + target: "mnw".to_string(),
2981 + status: HealthStatus::Error,
2982 + checked_at: (chrono::Utc::now() - chrono::Duration::hours(i)).to_rfc3339(),
2983 + response_time_ms: 0,
2984 + details: None,
2985 + error: Some("down".to_string()),
2986 + };
2987 + db::insert_health_check(&pool, &snapshot).await.unwrap();
2988 + }
2989 +
2990 + let pct = db::get_uptime_percent(&pool, "mnw", 24).await.unwrap();
2991 + assert!(pct.is_some());
2992 + let p = pct.unwrap();
2993 + assert!((p - 80.0).abs() < 0.01, "expected ~80.0, got {p}");
2994 + }
2995 +
2996 + #[tokio::test]
2997 + async fn uptime_percent_no_data() {
2998 + let pool = db::connect_in_memory().await.unwrap();
2999 +
3000 + let pct = db::get_uptime_percent(&pool, "mnw", 24).await.unwrap();
3001 + assert_eq!(pct, None);
3002 + }
3003 +
3004 + #[tokio::test]
3005 + async fn uptime_percent_only_old_data() {
3006 + let pool = db::connect_in_memory().await.unwrap();
3007 +
3008 + // Insert checks from 48 hours ago
3009 + for i in 0..5 {
3010 + let snapshot = HealthSnapshot {
3011 + id: None,
3012 + target: "mnw".to_string(),
3013 + status: HealthStatus::Operational,
3014 + checked_at: (chrono::Utc::now() - chrono::Duration::hours(48 + i)).to_rfc3339(),
3015 + response_time_ms: 100,
3016 + details: None,
3017 + error: None,
3018 + };
3019 + db::insert_health_check(&pool, &snapshot).await.unwrap();
3020 + }
3021 +
3022 + // Query last 24h — should find nothing
3023 + let pct = db::get_uptime_percent(&pool, "mnw", 24).await.unwrap();
3024 + assert_eq!(pct, None);
3025 + }
3026 +
3027 + // --- Prune test_details orphan cleanup ---
3028 +
3029 + #[tokio::test]
3030 + async fn prune_cascades_test_details_with_deleted_run() {
3031 + let pool = db::connect_in_memory().await.unwrap();
3032 +
3033 + // Insert an old test run with details
3034 + let old_time = (chrono::Utc::now() - chrono::Duration::days(60)).to_rfc3339();
3035 + let run = TestRun {
3036 + id: None,
3037 + target: "mnw".to_string(),
3038 + started_at: old_time,
3039 + finished_at: None,
3040 + duration_secs: Some(60),
3041 + exit_code: Some(0),
3042 + passed: true,
3043 + summary: TestSummary {
3044 + steps: vec![],
3045 + total_passed: Some(2),
3046 + total_failed: Some(0),
3047 + details: vec![
3048 + TestDetail { test_name: "test_a".to_string(), passed: true },
3049 + TestDetail { test_name: "test_b".to_string(), passed: true },
3050 + ],
3051 + },
Lines truncated
M src/checks/http.rs +55 -14
@@ -28,20 +28,20 @@
28 28 let status_code = response.status().as_u16();
29 29
30 30 // Reject responses that declare a content-length exceeding our limit.
31 - if let Some(len) = response.content_length() {
32 - if len > MAX_RESPONSE_BYTES {
33 - return HealthSnapshot {
34 - id: None,
35 - target: target_name.to_string(),
36 - status: HealthStatus::Degraded,
37 - checked_at,
38 - response_time_ms,
39 - details: None,
40 - error: Some(format!(
41 - "Response body too large: {len} bytes (limit: {MAX_RESPONSE_BYTES} bytes)"
42 - )),
43 - };
44 - }
31 + if let Some(len) = response.content_length()
32 + && len > MAX_RESPONSE_BYTES
33 + {
34 + return HealthSnapshot {
35 + id: None,
36 + target: target_name.to_string(),
37 + status: HealthStatus::Degraded,
38 + checked_at,
39 + response_time_ms,
40 + details: None,
41 + error: Some(format!(
42 + "Response body too large: {len} bytes (limit: {MAX_RESPONSE_BYTES} bytes)"
43 + )),
44 + };
45 45 }
46 46
47 47 // Read body with size cap (handles chunked/streaming responses without content-length).
@@ -241,6 +241,47 @@
241 241 }
242 242 }
243 243
244 + /// Detect sustained test duration drift by checking if all recent durations
245 + /// exceed the baseline average by the given threshold multiplier.
246 + ///
247 + /// Returns a description string if drift is detected, `None` otherwise.
248 + /// Requires at least `baseline_count` samples for the baseline window.
249 + pub fn detect_test_duration_drift(
250 + durations: &[(String, i64)],
251 + baseline_count: usize,
252 + recent_count: usize,
253 + threshold: f64,
254 + ) -> Option<String> {
255 + if durations.len() < baseline_count + recent_count {
256 + return None;
257 + }
258 +
259 + // durations are ordered most recent first from get_test_durations
260 + let recent = &durations[..recent_count];
261 + let baseline = &durations[recent_count..];
262 +
263 + if baseline.is_empty() {
264 + return None;
265 + }
266 +
267 + let baseline_avg = baseline.iter().map(|(_, d)| *d).sum::<i64>() as f64 / baseline.len() as f64;
268 + let drift_threshold = baseline_avg * threshold;
269 +
270 + let all_over = recent.iter().all(|(_, d)| *d as f64 > drift_threshold);
271 + if all_over {
272 + let recent_avg = recent.iter().map(|(_, d)| *d).sum::<i64>() as f64 / recent.len() as f64;
273 + Some(format!(
274 + "test duration drift: last {} runs avg {:.0}s (baseline avg {:.0}s, threshold {:.0}s)",
275 + recent_count,
276 + recent_avg,
277 + baseline_avg,
278 + drift_threshold,
279 + ))
280 + } else {
281 + None
282 + }
283 + }
284 +
244 285 /// Compute test staleness from version and timing data.
245 286 ///
246 287 /// A target's tests are considered stale when:
M src/checks/parse.rs +96 -1
@@ -1,10 +1,11 @@
1 - use crate::types::{StepResult, TestSummary};
1 + use crate::types::{StepResult, TestDetail, TestSummary};
2 2
3 3 /// Parse run-ci.sh output into a structured TestSummary.
4 4 ///
5 5 /// Looks for:
6 6 /// - `PASS <step name>` / `FAIL <step name>` lines from the CI summary
7 7 /// - `test result: ok. N passed; M failed` lines from cargo test
8 + /// - `test <name> ... ok` / `test <name> ... FAILED` individual test lines
8 9 pub fn parse_ci_output(output: &str) -> TestSummary {
9 10 let mut steps = Vec::new();
10 11 let mut total_passed: i64 = 0;
@@ -38,13 +39,46 @@
38 39 }
39 40 }
40 41
42 + let details = parse_individual_tests(output);
43 +
41 44 TestSummary {
42 45 steps,
43 46 total_passed: if found_test_results { Some(total_passed) } else { None },
44 47 total_failed: if found_test_results { Some(total_failed) } else { None },
48 + details,
45 49 }
46 50 }
47 51
52 + /// Parse individual test result lines from cargo test output.
53 + ///
54 + /// Matches lines like:
55 + /// - `test workflows::endorsements::toggle ... ok`
56 + /// - `test markdown::tests::bold_and_italic ... FAILED`
57 + pub fn parse_individual_tests(output: &str) -> Vec<TestDetail> {
58 + let mut details = Vec::new();
59 +
60 + for line in output.lines() {
61 + let trimmed = line.trim();
62 +
63 + // Match: "test <name> ... ok" or "test <name> ... FAILED"
64 + if let Some(rest) = trimmed.strip_prefix("test ") {
65 + if let Some(name) = rest.strip_suffix(" ... ok") {
66 + details.push(TestDetail {
67 + test_name: name.to_string(),
68 + passed: true,
69 + });
70 + } else if let Some(name) = rest.strip_suffix(" ... FAILED") {
71 + details.push(TestDetail {
72 + test_name: name.to_string(),
73 + passed: false,
74 + });
75 + }
76 + }
77 + }
78 +
79 + details
80 + }
81 +
48 82 fn parse_test_result_line(line: &str) -> Option<(i64, i64)> {
49 83 // "test result: ok. 42 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out"
50 84 let mut passed = 0i64;
@@ -136,6 +170,67 @@
136 170 assert_eq!(summary.steps[1].name, "cargo test --lib");
137 171 }
138 172
173 + #[test]
174 + fn parse_individual_tests_ok_and_failed() {
175 + let output = r#"
176 + running 4 tests
177 + test workflows::endorsements::toggle_endorsement_removes ... ok
178 + test workflows::endorsements::self_endorse_rejected ... ok
179 + test markdown::tests::bold_and_italic ... ok
180 + test workflows::endorsements::endorse_post_happy_path ... FAILED
181 +
182 + test result: FAILED. 3 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out
183 + "#;
184 + let details = parse_individual_tests(output);
185 + assert_eq!(details.len(), 4);
186 + assert!(details[0].passed);
187 + assert_eq!(details[0].test_name, "workflows::endorsements::toggle_endorsement_removes");
188 + assert!(details[1].passed);
189 + assert!(!details[3].passed);
190 + assert_eq!(details[3].test_name, "workflows::endorsements::endorse_post_happy_path");
191 + }
192 +
193 + #[test]
194 + fn parse_individual_tests_empty_when_no_individual_lines() {
195 + let output = r#"
196 + ========================================
197 + CI Summary
198 + ========================================
199 +
200 + PASS cargo check
201 + PASS cargo test --lib
202 + "#;
203 + let details = parse_individual_tests(output);
204 + assert!(details.is_empty());
205 + }
206 +
207 + #[test]
208 + fn parse_ci_output_includes_details() {
209 + let output = r#"
210 + ========================================
211 + cargo test --lib
212 + ========================================
213 +
214 + running 2 tests
215 + test foo::bar ... ok
216 + test foo::baz ... FAILED
217 +
218 + test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out
219 +
220 + ========================================
221 + CI Summary
222 + ========================================
223 +
224 + FAIL cargo test --lib
225 + "#;
226 + let summary = parse_ci_output(output);
227 + assert_eq!(summary.details.len(), 2);
228 + assert!(summary.details[0].passed);
229 + assert!(!summary.details[1].passed);
230 + assert_eq!(summary.total_passed, Some(1));
231 + assert_eq!(summary.total_failed, Some(1));
232 + }
233 +
139 234 #[test]
140 235 fn parse_test_result_line_ok() {
141 236 let line = "test result: ok. 42 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out";
M src/checks/ssh.rs +91 -23
@@ -5,6 +5,12 @@
5 5 use crate::config::TestsConfig;
6 6 use crate::types::{TestRun, TestSummary};
7 7
8 + /// Returns `true` if every character in `filter` is in `[a-zA-Z0-9_:-]`.
9 + /// An empty string is considered valid (no characters to reject).
10 + pub fn validate_test_filter(filter: &str) -> bool {
11 + filter.chars().all(|c| c.is_alphanumeric() || c == '_' || c == ':' || c == '-')
12 + }
13 +
8 14 #[instrument(skip_all)]
9 15 pub async fn run_tests(
10 16 target_name: &str,
@@ -16,29 +22,30 @@
16 22
17 23 // Validate filter characters before appending to SSH command.
18 24 // Only allow alphanumeric, underscore, colon, dash — covers all valid Rust test filter patterns.
19 - if let Some(f) = filter {
20 - if !f.chars().all(|c| c.is_alphanumeric() || c == '_' || c == ':' || c == '-') {
21 - let finished_at = chrono::Utc::now().to_rfc3339();
22 - let duration_secs = start.elapsed().as_secs() as i64;
23 - return TestRun {
24 - id: None,
25 - target: target_name.to_string(),
26 - started_at,
27 - finished_at: Some(finished_at),
28 - duration_secs: Some(duration_secs),
29 - exit_code: None,
30 - passed: false,
31 - summary: TestSummary {
32 - steps: vec![],
33 - total_passed: None,
34 - total_failed: None,
35 - },
36 - raw_output: format!(
37 - "Invalid filter: contains characters outside [a-zA-Z0-9_:-]. Got: {f}"
38 - ),
39 - filter: Some(f.to_string()),
40 - };
41 - }
25 + if let Some(f) = filter
26 + && !validate_test_filter(f)
27 + {
28 + let finished_at = chrono::Utc::now().to_rfc3339();
29 + let duration_secs = start.elapsed().as_secs() as i64;
30 + return TestRun {
31 + id: None,
32 + target: target_name.to_string(),
33 + started_at,
34 + finished_at: Some(finished_at),
35 + duration_secs: Some(duration_secs),
36 + exit_code: None,
37 + passed: false,
38 + summary: TestSummary {
39 + steps: vec![],
40 + total_passed: None,
41 + total_failed: None,
42 + details: vec![],
43 + },
44 + raw_output: format!(
45 + "Invalid filter: contains characters outside [a-zA-Z0-9_:-]. Got: {f}"
46 + ),
47 + filter: Some(f.to_string()),
48 + };
42 49 }
43 50
44 51 let mut cmd_str = config.command.clone();
@@ -96,9 +103,70 @@
96 103 steps: vec![],
97 104 total_passed: None,
98 105 total_failed: None,
106 + details: vec![],
99 107 },
100 108 raw_output: format!("SSH connection failed: {e}"),
101 109 filter: filter.map(String::from),
102 110 },
103 111 }
104 112 }
113 +
114 + #[cfg(test)]
115 + mod tests {
116 + use super::*;
117 +
118 + #[test]
119 + fn validate_test_filter_valid_simple() {
120 + assert!(validate_test_filter("foo"));
121 + }
122 +
123 + #[test]
124 + fn validate_test_filter_valid_module_path() {
125 + assert!(validate_test_filter("foo::bar"));
126 + }
127 +
128 + #[test]
129 + fn validate_test_filter_valid_underscore() {
130 + assert!(validate_test_filter("foo_bar"));
131 + }
132 +
133 + #[test]
134 + fn validate_test_filter_valid_dash() {
135 + assert!(validate_test_filter("foo-bar"));
136 + }
137 +
138 + #[test]
139 + fn validate_test_filter_valid_alphanumeric() {
140 + assert!(validate_test_filter("a123"));
141 + }
142 +
143 + #[test]
144 + fn validate_test_filter_empty_is_valid() {
145 + assert!(validate_test_filter(""));
146 + }
147 +
148 + #[test]
149 + fn validate_test_filter_rejects_semicolon() {
150 + assert!(!validate_test_filter("foo;rm"));
151 + }
152 +
153 + #[test]
154 + fn validate_test_filter_rejects_ampersand() {
155 + assert!(!validate_test_filter("foo && bar"));
156 + }
157 +
158 + #[test]
159 + fn validate_test_filter_rejects_pipe() {
160 + assert!(!validate_test_filter("foo|bar"));
161 + }
162 +
163 + #[test]
164 + fn validate_test_filter_rejects_subshell() {
165 + assert!(!validate_test_filter("$(cmd)"));
166 + }
167 +
168 + #[test]
169 + fn validate_test_filter_rejects_space() {
170 + assert!(!validate_test_filter("foo bar"));
171 + }
172 + }
M src/tools/tests.rs +10 -1
D src/cli.rs -871
A src/cli/incident.rs +92
A src/cli/mod.rs +262
A src/cli/serve.rs +145
A src/cli/status.rs +121
A src/cli/tasks/dns.rs +85
A src/cli/tasks/health.rs +141
A src/cli/tasks/meta_alert.rs +64
A src/cli/tasks/mod.rs +15
A src/cli/tasks/prune.rs +30
A src/cli/tasks/routes.rs +95
A src/cli/tasks/tls.rs +84
A src/cli/tasks/whois.rs +62