Skip to main content

max / makenotwork

13.8 KB · 399 lines History Blame Raw
1 //! Local systemd unit health check. Watches named daemons for liveness and
2 //! crash-loops, and optionally flags any unit on the host sitting in the failed
3 //! state.
4 //!
5 //! This closes a whole class of silent failure. bentod crash-looped 13,836 times
6 //! over ~20h reading as `activating` the entire time (a crash-loop never settles
7 //! to `failed`, so an is-active check alone misses it); sandod-backup-fetch sat
8 //! `failed` for four days behind a nightly timer with nothing watching. Nothing
9 //! watched the daemons that watch everything else. A liveness probe that treats a
10 //! climbing `NRestarts` as unhealthy catches the first, and a `systemctl
11 //! --failed` sweep catches the second.
12 //!
13 //! The probe runs against the local host via `systemctl show` / `systemctl
14 //! list-units --failed`, covering both the system bus and, for user-scoped units
15 //! like bentod, the `--user` bus (the audit's "user-bus-capable probe").
16
17 use tokio::process::Command;
18 use tracing::instrument;
19
20 use crate::config::SystemdUnit;
21 use crate::types::{SystemdCheckResult, SystemdUnitSnapshot};
22
23 /// Per-unit health derived from one `systemctl show` reading.
24 ///
25 /// Pure so it is testable without a live `systemctl`. Returns the unit status
26 /// and, when not healthy, one issue line naming why.
27 ///
28 /// A crash-loop is judged by `NRestarts` rather than `ActiveState`: a flapping
29 /// unit reads as `activating` (auto-restart) forever, never `failed`, so the
30 /// restart count is the only single-reading signal that catches it. `not-found`
31 /// (the unit file is missing or mistyped) is distinguished from a loaded unit
32 /// that is merely down.
33 pub fn classify_unit(
34 active_state: &str,
35 load_state: &str,
36 n_restarts: i64,
37 restart_threshold: i64,
38 ) -> (&'static str, Option<String>) {
39 if load_state == "not-found" {
40 return (
41 "not-loaded",
42 Some("unit not loaded (LoadState=not-found)".into()),
43 );
44 }
45 if n_restarts >= restart_threshold {
46 return (
47 "crash-loop",
48 Some(format!(
49 "{n_restarts} restarts (>= {restart_threshold}), unit is flapping"
50 )),
51 );
52 }
53 if active_state == "active" {
54 return ("active", None);
55 }
56 ("down", Some(format!("ActiveState={active_state}")))
57 }
58
59 /// Aggregate per-unit snapshots and the host-wide failed-unit sweep into one
60 /// overall status. A watched unit that is down or not loaded is the worst
61 /// signal (red); a crash-looping watched unit or any host-wide failed unit is
62 /// `degraded` (yellow, look at this).
63 fn overall_status(units: &[SystemdUnitSnapshot], failed_units: &[String]) -> &'static str {
64 if units
65 .iter()
66 .any(|u| u.status == "down" || u.status == "not-loaded")
67 {
68 "down"
69 } else if !failed_units.is_empty() || units.iter().any(|u| u.status == "crash-loop") {
70 "degraded"
71 } else {
72 "operational"
73 }
74 }
75
76 /// Parse a `systemctl show` key=value block into the four properties we read.
77 /// A missing property reads as an empty string (or 0 for `NRestarts`), which
78 /// [`classify_unit`] treats as not-active, the safe direction.
79 fn parse_show(output: &str) -> (String, String, String, i64) {
80 let mut active_state = String::new();
81 let mut sub_state = String::new();
82 let mut load_state = String::new();
83 let mut n_restarts = 0i64;
84 for line in output.lines() {
85 let Some((key, value)) = line.split_once('=') else {
86 continue;
87 };
88 match key {
89 "ActiveState" => active_state = value.to_string(),
90 "SubState" => sub_state = value.to_string(),
91 "LoadState" => load_state = value.to_string(),
92 "NRestarts" => n_restarts = value.trim().parse().unwrap_or(0),
93 _ => {}
94 }
95 }
96 (active_state, sub_state, load_state, n_restarts)
97 }
98
99 /// The first whitespace token of each non-empty line, `list-units --plain
100 /// --no-legend` puts the unit name first. Used to read the failed-unit sweep.
101 fn parse_failed_units(output: &str) -> Vec<String> {
102 output
103 .lines()
104 .filter_map(|line| line.split_whitespace().next())
105 .filter(|tok| !tok.is_empty())
106 .map(str::to_string)
107 .collect()
108 }
109
110 async fn systemctl_show(user: bool, unit: &str) -> Result<(String, String, String, i64), String> {
111 let mut cmd = Command::new("systemctl");
112 if user {
113 cmd.arg("--user");
114 }
115 cmd.args([
116 "show",
117 unit,
118 "--property=ActiveState,SubState,LoadState,NRestarts",
119 "--no-pager",
120 ]);
121 let output = cmd.output().await.map_err(|e| e.to_string())?;
122 // `systemctl show` exits 0 even for an unknown unit (it reports
123 // LoadState=not-found), so a non-zero exit is a real invocation failure
124 // (no systemd, no --user bus) rather than a missing unit.
125 if !output.status.success() {
126 let stderr = String::from_utf8_lossy(&output.stderr);
127 return Err(format!(
128 "systemctl show exited {}: {}",
129 output.status.code().unwrap_or(-1),
130 stderr.trim()
131 ));
132 }
133 Ok(parse_show(&String::from_utf8_lossy(&output.stdout)))
134 }
135
136 async fn systemctl_failed(user: bool) -> Result<Vec<String>, String> {
137 let mut cmd = Command::new("systemctl");
138 if user {
139 cmd.arg("--user");
140 }
141 cmd.args([
142 "list-units",
143 "--failed",
144 "--no-legend",
145 "--plain",
146 "--no-pager",
147 ]);
148 let output = cmd.output().await.map_err(|e| e.to_string())?;
149 if !output.status.success() {
150 let stderr = String::from_utf8_lossy(&output.stderr);
151 return Err(format!(
152 "systemctl list-units --failed exited {}: {}",
153 output.status.code().unwrap_or(-1),
154 stderr.trim()
155 ));
156 }
157 Ok(parse_failed_units(&String::from_utf8_lossy(&output.stdout)))
158 }
159
160 /// Probe the configured units on the local host and, when `check_failed` is set,
161 /// sweep for any unit in the failed state on each bus the watched units span.
162 ///
163 /// A per-unit probe failure marks that unit `down` with the error as its issue
164 /// rather than aborting the whole check, so one broken invocation cannot blind
165 /// the rest. A failed sweep is recorded in `error` but does not by itself change
166 /// the status, the watched units still report.
167 #[instrument(skip_all)]
168 pub async fn check_systemd(
169 target_name: &str,
170 units: &[SystemdUnit],
171 check_failed: bool,
172 restart_threshold: i64,
173 ) -> SystemdCheckResult {
174 let checked_at = chrono::Utc::now().to_rfc3339();
175 let mut snapshots: Vec<SystemdUnitSnapshot> = Vec::new();
176 let mut issues: Vec<String> = Vec::new();
177 let mut errors: Vec<String> = Vec::new();
178
179 for unit in units {
180 let scope = if unit.user { "user" } else { "system" };
181 match systemctl_show(unit.user, &unit.name).await {
182 Ok((active_state, sub_state, load_state, n_restarts)) => {
183 let (status, issue) =
184 classify_unit(&active_state, &load_state, n_restarts, restart_threshold);
185 if let Some(msg) = issue {
186 issues.push(format!("{} ({scope}): {msg}", unit.name));
187 }
188 snapshots.push(SystemdUnitSnapshot {
189 name: unit.name.clone(),
190 scope: scope.to_string(),
191 active_state,
192 sub_state,
193 n_restarts,
194 status: status.to_string(),
195 });
196 }
197 Err(e) => {
198 issues.push(format!("{} ({scope}): probe failed: {e}", unit.name));
199 errors.push(format!("{}: {e}", unit.name));
200 snapshots.push(SystemdUnitSnapshot {
201 name: unit.name.clone(),
202 scope: scope.to_string(),
203 active_state: "unknown".to_string(),
204 sub_state: String::new(),
205 n_restarts: 0,
206 status: "down".to_string(),
207 });
208 }
209 }
210 }
211
212 let mut failed_units: Vec<String> = Vec::new();
213 if check_failed {
214 // Sweep every bus the watched units span (system, and --user if any
215 // watched unit is user-scoped), so a failed oneshot/timer on the same
216 // bus as bentod is not missed just because it was not enumerated.
217 let want_user = units.iter().any(|u| u.user);
218 let mut scopes = vec![false];
219 if want_user {
220 scopes.push(true);
221 }
222 for user in scopes {
223 match systemctl_failed(user).await {
224 Ok(mut names) => failed_units.append(&mut names),
225 Err(e) => errors.push(format!("failed-unit sweep ({user}): {e}")),
226 }
227 }
228 failed_units.sort();
229 failed_units.dedup();
230 for name in &failed_units {
231 issues.push(format!("host unit failed: {name}"));
232 }
233 }
234
235 let status = overall_status(&snapshots, &failed_units);
236
237 SystemdCheckResult {
238 target: target_name.to_string(),
239 status: status.to_string(),
240 units: snapshots,
241 failed_units,
242 issues,
243 checked_at,
244 error: (!errors.is_empty()).then(|| errors.join("; ")),
245 }
246 }
247
248 #[cfg(test)]
249 mod tests {
250 use super::*;
251
252 fn snap(name: &str, status: &str) -> SystemdUnitSnapshot {
253 SystemdUnitSnapshot {
254 name: name.to_string(),
255 scope: "system".to_string(),
256 active_state: "active".to_string(),
257 sub_state: "running".to_string(),
258 n_restarts: 0,
259 status: status.to_string(),
260 }
261 }
262
263 #[test]
264 fn active_unit_is_healthy() {
265 let (status, issue) = classify_unit("active", "loaded", 0, 5);
266 assert_eq!(status, "active");
267 assert!(issue.is_none());
268 }
269
270 #[test]
271 fn inactive_unit_is_down() {
272 let (status, issue) = classify_unit("inactive", "loaded", 0, 5);
273 assert_eq!(status, "down");
274 assert!(issue.unwrap().contains("ActiveState=inactive"));
275 }
276
277 #[test]
278 fn failed_unit_is_down() {
279 let (status, _) = classify_unit("failed", "loaded", 0, 5);
280 assert_eq!(status, "down");
281 }
282
283 #[test]
284 fn crash_loop_while_activating_is_caught() {
285 // The bentod case: a flapping unit reads `activating`, never `failed`,
286 // but NRestarts climbs. Restart count is what catches it.
287 let (status, issue) = classify_unit("activating", "loaded", 42, 5);
288 assert_eq!(status, "crash-loop");
289 assert!(issue.unwrap().contains("42 restarts"));
290 }
291
292 #[test]
293 fn crash_loop_while_active_is_still_flagged() {
294 // A unit can be `active` right now yet have restarted many times.
295 let (status, _) = classify_unit("active", "loaded", 9, 5);
296 assert_eq!(status, "crash-loop");
297 }
298
299 #[test]
300 fn restart_threshold_boundary_is_inclusive() {
301 // NRestarts == threshold must trip (pins `>=` vs `>`).
302 assert_eq!(classify_unit("active", "loaded", 5, 5).0, "crash-loop");
303 assert_eq!(classify_unit("active", "loaded", 4, 5).0, "active");
304 }
305
306 #[test]
307 fn missing_unit_file_is_not_loaded() {
308 let (status, issue) = classify_unit("inactive", "not-found", 0, 5);
309 assert_eq!(status, "not-loaded");
310 assert!(issue.unwrap().contains("not-found"));
311 }
312
313 #[test]
314 fn all_healthy_is_operational() {
315 let units = vec![
316 snap("sandod.service", "active"),
317 snap("wam.service", "active"),
318 ];
319 assert_eq!(overall_status(&units, &[]), "operational");
320 }
321
322 #[test]
323 fn a_down_watched_unit_makes_the_host_down() {
324 let units = vec![snap("sandod.service", "down")];
325 assert_eq!(overall_status(&units, &[]), "down");
326 }
327
328 #[test]
329 fn a_not_loaded_watched_unit_makes_the_host_down() {
330 let units = vec![snap("bentod.service", "not-loaded")];
331 assert_eq!(overall_status(&units, &[]), "down");
332 }
333
334 #[test]
335 fn a_crash_loop_alone_is_degraded_not_down() {
336 let units = vec![snap("bentod.service", "crash-loop")];
337 assert_eq!(overall_status(&units, &[]), "degraded");
338 }
339
340 #[test]
341 fn a_host_wide_failed_unit_is_degraded() {
342 // sandod-backup-fetch: failed on the host but not a watched daemon.
343 let units = vec![snap("sandod.service", "active")];
344 assert_eq!(
345 overall_status(&units, &["sandod-backup-fetch.service".to_string()]),
346 "degraded"
347 );
348 }
349
350 #[test]
351 fn a_down_watched_unit_beats_a_failed_sweep() {
352 let units = vec![snap("sandod.service", "down")];
353 assert_eq!(
354 overall_status(&units, &["other.service".to_string()]),
355 "down"
356 );
357 }
358
359 #[test]
360 fn parse_show_reads_the_four_properties() {
361 let out = "ActiveState=active\nSubState=running\nLoadState=loaded\nNRestarts=3\n";
362 let (active, sub, load, n) = parse_show(out);
363 assert_eq!(active, "active");
364 assert_eq!(sub, "running");
365 assert_eq!(load, "loaded");
366 assert_eq!(n, 3);
367 }
368
369 #[test]
370 fn parse_show_tolerates_missing_and_extra_keys() {
371 let out = "Id=foo.service\nActiveState=failed\nUnrelated=x\n";
372 let (active, sub, load, n) = parse_show(out);
373 assert_eq!(active, "failed");
374 assert!(sub.is_empty());
375 assert!(load.is_empty());
376 assert_eq!(n, 0, "absent NRestarts defaults to 0");
377 }
378
379 #[test]
380 fn parse_failed_units_takes_the_first_column() {
381 let out = " sandod-backup-fetch.service loaded failed failed Sando backup puller\n\
382 foo.timer loaded failed failed A timer\n";
383 let names = parse_failed_units(out);
384 assert_eq!(
385 names,
386 vec![
387 "sandod-backup-fetch.service".to_string(),
388 "foo.timer".to_string()
389 ]
390 );
391 }
392
393 #[test]
394 fn parse_failed_units_empty_is_empty() {
395 assert!(parse_failed_units("").is_empty());
396 assert!(parse_failed_units("\n \n").is_empty());
397 }
398 }
399