Skip to main content

max / makenotwork

pom: add systemd local-daemon health check Watch local daemons for liveness and crash-loops, and sweep the host for any unit in the failed state. Nothing watched the daemons that watch everything else: bentod crash-looped 13,836 times reading as `activating` the whole time (a crash-loop never settles to `failed`, so an is-active check misses it), and sandod-backup-fetch sat `failed` for four days behind a nightly timer, both silently. The probe runs against the local host via `systemctl show` / `systemctl list-units --failed`, covering the system bus and the `--user` bus for units like bentod. A climbing NRestarts reads as a crash-loop; a watched unit that is down or unloaded is a failure; a host-wide failed unit is degraded. classify_unit/overall_status are pure and unit-tested against both incident shapes. Fully wired end to end following the scan_pipeline pattern: config ([targets.<host>.systemd]), check, SystemdCheckResult, db migration 12 + accessors, alerts (folded onto the "monitoring" MNW domain), serve task, /status.json surfacing, and retention pruning.
Co-Authored-By
Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-25 15:29 UTC
Signed with PGP, not checked
Commit: 395c6546df59f6af9dbdeb22f3e8bde8018cfefd
Parent: b131027
18 files changed, +949 insertions, -24 deletions
M pom/src/api.rs +12
@@ -619,6 +619,17 @@
619 619 error: s.error,
620 620 });
621 621
622 + let systemd = db::get_latest_systemd_check(&state.pool, &name)
623 + .await
624 + .ok()
625 + .flatten()
626 + .map(|s| crate::status::SystemdView {
627 + issues: s.issue_list(),
628 + status: s.status,
629 + checked_at: s.checked_at,
630 + error: s.error,
631 + });
632 +
622 633 // Tests: the latest run plus PoM's staleness verdict, sourced exactly as
623 634 // build_target_status does (version at test time vs current version).
624 635 let tests = if let Some(tests_config) = &target_config.tests {
@@ -704,6 +715,7 @@
704 715 whois,
705 716 backups,
706 717 scan_pipeline,
718 + systemd,
707 719 tests,
708 720 dns,
709 721 cors,
M pom/src/config.rs +114
@@ -264,6 +264,57 @@
264 264 /// Scan-pipeline health check against `<base_url>/admin/uploads/health.json`.
265 265 /// `None` disables.
266 266 pub scan_pipeline: Option<ScanPipelineConfig>,
267 + /// Local systemd daemon liveness / crash-loop / failed-unit check. `None`
268 + /// disables. Probes the host PoM runs on, not a remote target.
269 + pub systemd: Option<SystemdConfig>,
270 + }
271 +
272 + /// Local systemd unit monitoring for a host target. Watches named daemons for
273 + /// liveness and crash-loops, and optionally sweeps the host for any failed unit.
274 + #[derive(Debug, Clone, Deserialize)]
275 + pub struct SystemdConfig {
276 + /// Units to watch for liveness (e.g. sandod, bentod, wam, pom).
277 + #[serde(default)]
278 + pub units: Vec<SystemdUnit>,
279 + /// Also alert on any unit in the failed state on the host, including
280 + /// oneshot/timer-driven units a liveness watch would never enumerate
281 + /// (the sandod-backup-fetch class). Defaults to true.
282 + #[serde(default = "default_systemd_check_failed")]
283 + pub check_failed: bool,
284 + /// `NRestarts` at or above this reads as a crash-loop even while the unit
285 + /// still shows `activating`. Defaults to 5.
286 + #[serde(default = "default_systemd_restart_threshold")]
287 + pub restart_threshold: i64,
288 + /// Seconds between checks. Defaults to 60.
289 + #[serde(default = "default_systemd_interval")]
290 + pub interval_secs: u64,
291 + }
292 +
293 + /// One watched systemd unit.
294 + #[derive(Debug, Clone, Deserialize)]
295 + pub struct SystemdUnit {
296 + /// Unit name including its suffix (e.g. "sandod.service").
297 + pub name: String,
298 + /// Whether the unit lives on the `--user` bus rather than the system bus.
299 + /// bentod runs under `systemd --user`, so this must be set for it.
300 + #[serde(default)]
301 + pub user: bool,
302 + }
303 +
304 + fn default_systemd_check_failed() -> bool {
305 + true
306 + }
307 +
308 + fn default_systemd_restart_threshold() -> i64 {
309 + // 5 automatic restarts: a healthy long-lived daemon does not flap this much,
310 + // and it is well below the thousands bentod racked up while going unnoticed.
311 + 5
312 + }
313 +
314 + fn default_systemd_interval() -> u64 {
315 + // 1 minute: a crash-loop should surface fast, and a local systemctl probe is
316 + // cheap.
317 + 60
267 318 }
268 319
269 320 #[derive(Debug, Clone, Deserialize)]
@@ -1100,6 +1151,69 @@
1100 1151 assert!(config.get_target("mnw").unwrap().whois.is_none());
1101 1152 }
1102 1153
1154 + #[test]
1155 + fn config_with_systemd() {
1156 + let toml = r#"
1157 + [targets.fw13]
1158 + label = "fw13 daemons"
1159 +
1160 + [targets.fw13.systemd]
1161 + restart_threshold = 3
1162 + interval_secs = 30
1163 + check_failed = false
1164 +
1165 + [[targets.fw13.systemd.units]]
1166 + name = "sandod.service"
1167 +
1168 + [[targets.fw13.systemd.units]]
1169 + name = "bentod.service"
1170 + user = true
1171 + "#;
1172 + let config: Config = toml::from_str(toml).unwrap();
1173 + let sd = config.get_target("fw13").unwrap().systemd.as_ref().unwrap();
1174 + assert_eq!(sd.restart_threshold, 3);
1175 + assert_eq!(sd.interval_secs, 30);
1176 + assert!(!sd.check_failed);
1177 + assert_eq!(sd.units.len(), 2);
1178 + assert_eq!(sd.units[0].name, "sandod.service");
1179 + assert!(!sd.units[0].user, "system bus by default");
1180 + assert_eq!(sd.units[1].name, "bentod.service");
1181 + assert!(sd.units[1].user, "bentod is on the --user bus");
1182 + }
1183 +
1184 + #[test]
1185 + fn config_systemd_defaults() {
1186 + let toml = r#"
1187 + [targets.fw13]
1188 + label = "fw13 daemons"
1189 + [targets.fw13.systemd]
1190 + [[targets.fw13.systemd.units]]
1191 + name = "pom.service"
1192 + "#;
1193 + let config: Config = toml::from_str(toml).unwrap();
1194 + let sd = config.get_target("fw13").unwrap().systemd.as_ref().unwrap();
1195 + assert!(sd.check_failed, "failed-unit sweep on by default");
1196 + assert_eq!(sd.restart_threshold, 5);
1197 + assert_eq!(sd.interval_secs, 60);
1198 + }
1199 +
1200 + #[test]
1201 + fn config_without_systemd() {
1202 + let toml = r#"
1203 + [targets.mnw]
1204 + label = "MakeNotWork"
1205 + "#;
1206 + let config: Config = toml::from_str(toml).unwrap();
1207 + assert!(config.get_target("mnw").unwrap().systemd.is_none());
1208 + }
1209 +
1210 + #[test]
1211 + fn defaults_systemd() {
1212 + assert!(default_systemd_check_failed(), "failed sweep on by default");
1213 + assert_eq!(default_systemd_restart_threshold(), 5);
1214 + assert_eq!(default_systemd_interval(), 60, "1-minute liveness cadence");
1215 + }
1216 +
1103 1217 #[test]
1104 1218 fn config_dashboard_default_false() {
1105 1219 let config: Config = toml::from_str("").unwrap();
M pom/src/display.rs +6 -3
@@ -379,7 +379,7 @@
379 379 /// Format prune results for CLI display.
380 380 pub fn format_prune(result: &PruneResult, days: i64) -> String {
381 381 format!(
382 - "Pruned {} health checks, {} test runs, {} test details, {} peer heartbeats, {} alerts, {} TLS checks, {} incidents, {} route checks, {} DNS checks, {} WHOIS checks, {} backup checks older than {} days.\n",
382 + "Pruned {} health checks, {} test runs, {} test details, {} peer heartbeats, {} alerts, {} TLS checks, {} incidents, {} route checks, {} DNS checks, {} WHOIS checks, {} backup checks, {} systemd checks older than {} days.\n",
383 383 result.health,
384 384 result.tests,
385 385 result.test_details,
@@ -391,6 +391,7 @@
391 391 result.dns,
392 392 result.whois,
393 393 result.backups,
394 + result.systemd,
394 395 days
395 396 )
396 397 }
@@ -1175,11 +1176,12 @@
1175 1176 dns: 8,
1176 1177 whois: 2,
1177 1178 backups: 1,
1179 + systemd: 6,
1178 1180 };
1179 1181 let out = format_prune(&result, 30);
1180 1182 assert_eq!(
1181 1183 out,
1182 - "Pruned 5 health checks, 3 test runs, 15 test details, 10 peer heartbeats, 2 alerts, 1 TLS checks, 4 incidents, 0 route checks, 8 DNS checks, 2 WHOIS checks, 1 backup checks older than 30 days.\n"
1184 + "Pruned 5 health checks, 3 test runs, 15 test details, 10 peer heartbeats, 2 alerts, 1 TLS checks, 4 incidents, 0 route checks, 8 DNS checks, 2 WHOIS checks, 1 backup checks, 6 systemd checks older than 30 days.\n"
1183 1185 );
1184 1186 }
1185 1187
@@ -1197,9 +1199,10 @@
1197 1199 dns: 0,
1198 1200 whois: 0,
1199 1201 backups: 0,
1202 + systemd: 0,
1200 1203 };
1201 1204 let out = format_prune(&result, 7);
1202 - assert!(out.contains("Pruned 0 health checks, 0 test runs, 0 test details, 0 peer heartbeats, 0 alerts, 0 TLS checks, 0 incidents, 0 route checks, 0 DNS checks, 0 WHOIS checks, 0 backup checks older than 7 days."));
1205 + assert!(out.contains("Pruned 0 health checks, 0 test runs, 0 test details, 0 peer heartbeats, 0 alerts, 0 TLS checks, 0 incidents, 0 route checks, 0 DNS checks, 0 WHOIS checks, 0 backup checks, 0 systemd checks older than 7 days."));
1203 1206 }
1204 1207
1205 1208 // format_mesh
M pom/src/status.rs +42
@@ -78,6 +78,8 @@
78 78 pub backups: Vec<BackupView>,
79 79 /// Latest scan-pipeline check. `None` if not monitored for this target.
80 80 pub scan_pipeline: Option<ScanView>,
81 + /// Latest local systemd daemon-health check. `None` if not monitored.
82 + pub systemd: Option<SystemdView>,
81 83 /// Latest test run and PoM's staleness verdict. `None` if the target has no
82 84 /// test config.
83 85 pub tests: Option<TestsView>,
@@ -132,6 +134,16 @@
132 134 pub error: Option<String>,
133 135 }
134 136
137 + pub(crate) struct SystemdView {
138 + /// One of "operational", "degraded", "down".
139 + pub status: String,
140 + /// Unhealthy-unit / failed-sweep lines, the why behind a non-operational
141 + /// status.
142 + pub issues: Vec<String>,
143 + pub checked_at: String,
144 + pub error: Option<String>,
145 + }
146 +
135 147 pub(crate) struct TestsView {
136 148 /// Whether any run has ever been recorded. Separates "no evidence"
137 149 /// (pending) from a run that failed.
@@ -206,6 +218,9 @@
206 218 if let Some(scan) = &target.scan_pipeline {
207 219 conditions.push(scan_condition(scan));
208 220 }
221 + if let Some(sd) = &target.systemd {
222 + conditions.push(systemd_condition(sd));
223 + }
209 224 if let Some(tests) = &target.tests {
210 225 conditions.push(tests_condition(tests));
211 226 }
@@ -422,6 +437,32 @@
422 437 }
423 438 }
424 439
440 + /// Local systemd daemon health as a condition. A down watched unit is `failed`
441 + /// (red, a daemon that watches the platform is itself dead); a crash-loop or a
442 + /// host-wide failed unit is `degraded` (yellow, look at this). A probe failure
443 + /// (no systemd, no `--user` bus) is `failed` and names the error.
444 + fn systemd_condition(sd: &SystemdView) -> Condition {
445 + let status = match sd.status.as_str() {
446 + "operational" => Status::Ok,
447 + "degraded" => Status::Degraded,
448 + "down" => Status::Failed,
449 + _ => Status::Unknown,
450 + };
451 + let detail = if let Some(error) = &sd.error {
452 + format!("probe error: {error}")
453 + } else if !sd.issues.is_empty() {
454 + sd.issues.join("; ")
455 + } else {
456 + "all watched daemons healthy".into()
457 + };
458 + Condition {
459 + condition_type: "systemd".into(),
460 + status,
461 + since: parse_instant(&sd.checked_at),
462 + detail: Some(detail),
463 + }
464 + }
465 +
425 466 /// Test state as a condition. A never-run target is `pending`: evidence of
426 467 /// nothing, like one never health-checked. A failing or stale run is `degraded`,
427 468 /// not `failed`: a red suite is a real regression signal, but the running
@@ -633,6 +674,7 @@
633 674 whois: None,
634 675 backups: Vec::new(),
635 676 scan_pipeline: None,
677 + systemd: None,
636 678 tests: None,
637 679 dns: None,
638 680 cors: None,
M pom/src/types.rs +47
@@ -31,6 +31,8 @@
31 31 BackupRecovery,
32 32 ScanPipelineDegraded,
33 33 ScanPipelineRecovery,
34 + SystemdFailure,
35 + SystemdRecovery,
34 36 MonitoringOffline,
35 37 MonitoringRecovery,
36 38 }
@@ -60,6 +62,8 @@
60 62 Self::BackupRecovery => write!(f, "backup_recovery"),
61 63 Self::ScanPipelineDegraded => write!(f, "scan_pipeline_degraded"),
62 64 Self::ScanPipelineRecovery => write!(f, "scan_pipeline_recovery"),
65 + Self::SystemdFailure => write!(f, "systemd_failure"),
66 + Self::SystemdRecovery => write!(f, "systemd_recovery"),
63 67 Self::MonitoringOffline => write!(f, "monitoring_offline"),
64 68 Self::MonitoringRecovery => write!(f, "monitoring_recovery"),
65 69 }
@@ -92,6 +96,8 @@
92 96 "backup_recovery" => Ok(Self::BackupRecovery),
93 97 "scan_pipeline_degraded" => Ok(Self::ScanPipelineDegraded),
94 98 "scan_pipeline_recovery" => Ok(Self::ScanPipelineRecovery),
99 + "systemd_failure" => Ok(Self::SystemdFailure),
100 + "systemd_recovery" => Ok(Self::SystemdRecovery),
95 101 "monitoring_offline" => Ok(Self::MonitoringOffline),
96 102 "monitoring_recovery" => Ok(Self::MonitoringRecovery),
97 103 other => Err(format!("unknown alert category: {other}")),
@@ -466,6 +472,43 @@
466 472 pub error: Option<String>,
467 473 }
468 474
475 + #[derive(Debug, Clone, Serialize, Deserialize)]
476 + pub struct SystemdUnitSnapshot {
477 + /// Unit name (e.g. "sandod.service").
478 + pub name: String,
479 + /// Which bus the unit lives on: "system" or "user".
480 + pub scope: String,
481 + /// systemd `ActiveState` (active, inactive, failed, activating, ...).
482 + pub active_state: String,
483 + /// systemd `SubState` (running, dead, auto-restart, ...).
484 + pub sub_state: String,
485 + /// Automatic restart count since the unit started. A climbing value is the
486 + /// signal a crash-loop leaves while `ActiveState` stays `activating`.
487 + pub n_restarts: i64,
488 + /// Derived status: "active", "crash-loop", "down", or "not-loaded".
489 + pub status: String,
490 + }
491 +
492 + #[derive(Debug, Clone, Serialize, Deserialize)]
493 + pub struct SystemdCheckResult {
494 + /// Config key identifying the monitored host target.
495 + pub target: String,
496 + /// Overall status: "operational", "degraded", or "down".
497 + pub status: String,
498 + /// Per-watched-unit snapshots.
499 + pub units: Vec<SystemdUnitSnapshot>,
500 + /// Any unit on the host in the failed state, from the `systemctl --failed`
501 + /// sweep. Empty when the sweep is disabled or nothing is failed.
502 + pub failed_units: Vec<String>,
503 + /// Aggregated issues (one short line per unhealthy unit or failed sweep hit).
504 + pub issues: Vec<String>,
505 + /// When this check was performed, in RFC 3339 format (UTC).
506 + pub checked_at: String,
507 + /// Error message if one or more `systemctl` invocations failed. `None` on a
508 + /// clean probe even when `issues` is non-empty (a down unit is not an error).
509 + pub error: Option<String>,
510 + }
511 +
469 512 #[derive(Debug, Clone, Serialize, Deserialize)]
470 513 pub struct BackupCheckResult {
471 514 /// Config key identifying the monitored target.
@@ -717,6 +760,10 @@
717 760 AlertCategory::CorsRecovery,
718 761 AlertCategory::BackupStale,
719 762 AlertCategory::BackupRecovery,
763 + AlertCategory::ScanPipelineDegraded,
764 + AlertCategory::ScanPipelineRecovery,
765 + AlertCategory::SystemdFailure,
766 + AlertCategory::SystemdRecovery,
720 767 AlertCategory::MonitoringOffline,
721 768 AlertCategory::MonitoringRecovery,
722 769 ] {
M pom/tests/integration.rs +15 -13
@@ -412,7 +412,7 @@
412 412 // A fresh in-memory DB should run all migrations and reach the latest version.
413 413 let pool = db::connect_in_memory().await.unwrap();
414 414 let version = db::get_schema_version(&pool).await.unwrap();
415 - assert_eq!(version, 11);
415 + assert_eq!(version, 12);
416 416
417 417 // Verify the schema_version table has entries for each migration
418 418 let rows = sqlx::query_as::<_, (i64, String)>(
@@ -421,7 +421,7 @@
421 421 .fetch_all(&pool)
422 422 .await
423 423 .unwrap();
424 - assert_eq!(rows.len(), 11);
424 + assert_eq!(rows.len(), 12);
425 425 assert_eq!(rows[0].0, 1);
426 426 assert_eq!(rows[0].1, "initial schema");
427 427 assert_eq!(rows[1].0, 2);
@@ -444,6 +444,8 @@
444 444 assert_eq!(rows[9].1, "add pending_alerts retry queue");
445 445 assert_eq!(rows[10].0, 11);
446 446 assert_eq!(rows[10].1, "add scan_pipeline_checks table");
447 + assert_eq!(rows[11].0, 12);
448 + assert_eq!(rows[11].1, "add systemd_checks table");
447 449
448 450 // Verify actual tables were created by inserting data
449 451 let snapshot = HealthSnapshot {
@@ -463,18 +465,18 @@
463 465 async fn migration_already_current_is_idempotent() {
464 466 // Running migrations on an already-migrated DB should be a no-op.
465 467 let pool = db::connect_in_memory().await.unwrap();
466 - assert_eq!(db::get_schema_version(&pool).await.unwrap(), 11);
468 + assert_eq!(db::get_schema_version(&pool).await.unwrap(), 12);
467 469
468 470 // Run migrations again
469 471 db::run_migrations(&pool).await.unwrap();
470 - assert_eq!(db::get_schema_version(&pool).await.unwrap(), 11);
472 + assert_eq!(db::get_schema_version(&pool).await.unwrap(), 12);
471 473
472 - // schema_version should still have exactly eleven entries (not duplicated)
474 + // schema_version should still have exactly twelve entries (not duplicated)
473 475 let count = sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM schema_version")
474 476 .fetch_one(&pool)
475 477 .await
476 478 .unwrap();
477 - assert_eq!(count.0, 11);
479 + assert_eq!(count.0, 12);
478 480 }
479 481
480 482 #[tokio::test]
@@ -533,8 +535,8 @@
533 535 // Now run migrations, should detect existing tables, stamp as v1, then run v2+v3+v4+v5+v6
534 536 db::run_migrations(&pool).await.unwrap();
535 537
536 - // Version should be 11 (stamped v1 + ran v2..v11)
537 - assert_eq!(db::get_schema_version(&pool).await.unwrap(), 11);
538 + // Version should be 12 (stamped v1 + ran v2..v12)
539 + assert_eq!(db::get_schema_version(&pool).await.unwrap(), 12);
538 540
539 541 // Description should indicate pre-existing
540 542 let row =
@@ -854,7 +856,7 @@
854 856 async fn migration_v2_creates_alerts_table() {
855 857 let pool = db::connect_in_memory().await.unwrap();
856 858 let version = db::get_schema_version(&pool).await.unwrap();
857 - assert_eq!(version, 11);
859 + assert_eq!(version, 12);
858 860
859 861 // Verify alerts table exists by inserting
860 862 let id = db::insert_alert(
@@ -940,7 +942,7 @@
940 942 async fn migration_v3_creates_tls_checks_table() {
941 943 let pool = db::connect_in_memory().await.unwrap();
942 944 let version = db::get_schema_version(&pool).await.unwrap();
943 - assert_eq!(version, 11);
945 + assert_eq!(version, 12);
944 946
945 947 // Verify tls_checks table exists by inserting
946 948 let status = pom::types::TlsStatus {
@@ -1178,7 +1180,7 @@
1178 1180 async fn migration_v4_creates_incidents_table() {
1179 1181 let pool = db::connect_in_memory().await.unwrap();
1180 1182 let version = db::get_schema_version(&pool).await.unwrap();
1181 - assert_eq!(version, 11);
1183 + assert_eq!(version, 12);
1182 1184
1183 1185 // Verify incidents table exists by inserting
1184 1186 let id = db::insert_incident(&pool, "mnw", "operational", "degraded")
@@ -1318,7 +1320,7 @@
1318 1320 async fn migration_v5_creates_route_checks_table() {
1319 1321 let pool = db::connect_in_memory().await.unwrap();
1320 1322 let version = db::get_schema_version(&pool).await.unwrap();
1321 - assert_eq!(version, 11);
1323 + assert_eq!(version, 12);
1322 1324
1323 1325 // Verify route_checks table exists by inserting
1324 1326 let result = pom::checks::routes::RouteCheckResult {
@@ -2460,7 +2462,7 @@
2460 2462 async fn migration_v6_creates_dns_and_whois_tables() {
2461 2463 let pool = db::connect_in_memory().await.unwrap();
2462 2464 let version = db::get_schema_version(&pool).await.unwrap();
2463 - assert_eq!(version, 11);
2465 + assert_eq!(version, 12);
2464 2466
2465 2467 // Verify dns_checks table exists
2466 2468 let dns_result = DnsCheckResult {
M pom/src/alerts/mod.rs +7 -3
@@ -19,6 +19,7 @@
19 19 mod peer;
20 20 mod route;
21 21 mod scan;
22 + mod systemd;
22 23 mod test_duration;
23 24 mod tls;
24 25 mod whois;
@@ -84,8 +85,8 @@
84 85 BackupRecovery, BackupStale, CorsFailure, CorsRecovery, DnsMismatch, DnsRecovery, Health,
85 86 LatencyDrift, LatencyRecovery, MonitoringOffline, MonitoringRecovery, PeerMissing,
86 87 PeerRecovery, Recovery, RouteFailure, RouteRecovery, ScanPipelineDegraded,
87 - ScanPipelineRecovery, TestDurationDrift, TlsError, TlsExpiry, TlsRecovery, WhoisError,
88 - WhoisExpiry,
88 + ScanPipelineRecovery, SystemdFailure, SystemdRecovery, TestDurationDrift, TlsError,
89 + TlsExpiry, TlsRecovery, WhoisError, WhoisExpiry,
89 90 };
90 91 match category {
91 92 Health | Recovery => "health",
@@ -98,7 +99,10 @@
98 99 PeerMissing | PeerRecovery => "peer",
99 100 RouteFailure | RouteRecovery => "route",
100 101 ScanPipelineDegraded | ScanPipelineRecovery => "scan",
101 - MonitoringOffline | MonitoringRecovery => "monitoring",
102 + // Daemon liveness folds onto the "monitoring" domain rather than adding a
103 + // new MNW `AlertKind` (an unknown kind is a 422 from the ingest endpoint).
104 + // It sits with MonitoringOffline: both are "the ops plane itself is sick".
105 + SystemdFailure | SystemdRecovery | MonitoringOffline | MonitoringRecovery => "monitoring",
102 106 }
103 107 }
104 108
M pom/src/checks/mod.rs +3 -1
@@ -1,5 +1,6 @@
1 1 //! Check implementations, one module per probe kind: health, TLS, DNS, WHOIS,
2 - //! routes, CORS, backups, SSH, port scans, plus latency-drift analysis.
2 + //! routes, CORS, backups, SSH, port scans, local systemd units, plus
3 + //! latency-drift analysis.
3 4
4 5 pub mod backup;
5 6 pub mod cors;
@@ -11,5 +12,6 @@
11 12 pub mod scan_pipeline;
12 13 pub mod ssh;
13 14 pub mod ssh_banner;
15 + pub mod systemd;
14 16 pub mod tls;
15 17 pub mod whois;
M pom/src/cli/serve.rs +6
@@ -113,6 +113,12 @@
113 113 &token,
114 114 alerter.as_ref(),
115 115 ));
116 + handles.extend(tasks::spawn_systemd_tasks(
117 + config,
118 + pool,
119 + &token,
120 + alerter.as_ref(),
121 + ));
116 122 handles.push(tasks::spawn_prune_task(pool, prune_days, &token));
117 123
118 124 // Spawn peer heartbeat tasks
M pom/src/db/maintenance.rs +8
@@ -17,6 +17,7 @@
17 17 pub dns: u64,
18 18 pub whois: u64,
19 19 pub backups: u64,
20 + pub systemd: u64,
20 21 }
21 22
22 23 /// Delete records older than `days` from all tables.
@@ -38,6 +39,7 @@
38 39 dns: 0,
39 40 whois: 0,
40 41 backups: 0,
42 + systemd: 0,
41 43 });
42 44 }
43 45
@@ -114,6 +116,11 @@
114 116 .execute(pool)
115 117 .await?;
116 118
119 + let systemd_result = sqlx::query("DELETE FROM systemd_checks WHERE checked_at < ?")
120 + .bind(&cutoff_str)
121 + .execute(pool)
122 + .await?;
123 +
117 124 Ok(PruneResult {
118 125 health: health_result.rows_affected(),
119 126 tests: test_result.rows_affected(),
@@ -126,5 +133,6 @@
126 133 dns: dns_result.rows_affected(),
127 134 whois: whois_result.rows_affected(),
128 135 backups: backups_result.rows_affected(),
136 + systemd: systemd_result.rows_affected(),
129 137 })
130 138 }
M pom/src/db/migrations.rs +17
@@ -245,6 +245,23 @@
245 245 CREATE INDEX IF NOT EXISTS idx_scan_pipeline_checks_target ON scan_pipeline_checks(target, id DESC);
246 246 ",
247 247 ),
248 + (
249 + 12,
250 + "add systemd_checks table",
251 + r"
252 + CREATE TABLE IF NOT EXISTS systemd_checks (
253 + id INTEGER PRIMARY KEY AUTOINCREMENT,
254 + target TEXT NOT NULL,
255 + status TEXT NOT NULL,
256 + units TEXT NOT NULL, -- JSON array of unit snapshots
257 + failed_units TEXT NOT NULL, -- JSON array of host-wide failed unit names
258 + issues TEXT NOT NULL, -- JSON array of fired-threshold lines
259 + checked_at TEXT NOT NULL,
260 + error TEXT
261 + );
262 + CREATE INDEX IF NOT EXISTS idx_systemd_checks_target ON systemd_checks(target, id DESC);
263 + ",
264 + ),
248 265 ];
249 266
250 267 #[instrument(skip_all)]
M pom/src/db/mod.rs +4 -2
@@ -12,8 +12,8 @@
12 12 use crate::error::Result;
13 13 use crate::types::{
14 14 BackupCheckResult, CorsCheckResult, DnsCheckResult, HealthDetails, HealthSnapshot,
15 - HealthStatus, ScanPipelineCheckResult, TestDetail, TestRun, TestRunId, TestSummary, TlsStatus,
16 - WhoisResult,
15 + HealthStatus, ScanPipelineCheckResult, SystemdCheckResult, SystemdUnitSnapshot, TestDetail,
16 + TestRun, TestRunId, TestSummary, TlsStatus, WhoisResult,
17 17 };
18 18
19 19 mod alerts;
@@ -27,6 +27,7 @@
27 27 mod peers;
28 28 mod routes;
29 29 mod scan_pipeline;
30 + mod systemd;
30 31 mod test_runs;
31 32 mod tls;
32 33 mod whois;
@@ -42,6 +43,7 @@
42 43 pub use peers::*;
43 44 pub use routes::*;
44 45 pub use scan_pipeline::*;
46 + pub use systemd::*;
45 47 pub use test_runs::*;
46 48 pub use tls::*;
47 49 pub use whois::*;
M pom/src/cli/tasks/mod.rs +2
M pom/src/cli/tasks/prune.rs +3 -2
A pom/src/alerts/systemd.rs +85
A pom/src/checks/systemd.rs +398
A pom/src/cli/tasks/systemd.rs +103
A pom/src/db/systemd.rs +77