Skip to main content

max / makenotwork

13.1 KB · 395 lines History Blame Raw
1 //! Local CA-bundle freshness check. Reads the trust-anchor package's installed
2 //! version against what apt would install now, how long ago the package lists
3 //! were last refreshed, and how many certificates the concatenated bundle
4 //! actually holds.
5 //!
6 //! This is the leading indicator. The trailing one already exists: `checks::tls`
7 //! validates every target against the host trust store as well as the bundled
8 //! web PKI, so a bundle that has gone thin surfaces as a chain the host rejects.
9 //! By then multithreaded is already failing, because it takes its outbound trust
10 //! anchors from this bundle on all three of its outbound paths (OAuth to the MNW
11 //! server, link previews, S3) and neither reqwest nor aws-smithy-http-client
12 //! offers bundled roots to fall back on.
13 //!
14 //! Three signals, and the second is the one that is easy to leave out:
15 //!
16 //! 1. Installed versus candidate version of the package.
17 //! 2. Age of the last successful package-list update. Without it, "installed
18 //! equals candidate" stays true forever the moment `apt-daily.timer` stops
19 //! running, and the check reports a green it has no evidence for.
20 //! 3. Certificate count against a floor, which catches a truncated or emptied
21 //! bundle that both version numbers call fine.
22 //!
23 //! The probe runs against the local host via `apt-cache policy`, which works
24 //! unprivileged and inside pom.service's own sandbox (verified on prod-1 under
25 //! `ProtectSystem=strict`, `ProtectHome`, `PrivateTmp`).
26
27 use std::path::Path;
28
29 use tokio::process::Command;
30 use tracing::instrument;
31
32 use crate::config::CaBundleConfig;
33 use crate::types::CaBundleCheckResult;
34
35 /// Pull the installed and candidate versions out of `apt-cache policy <pkg>`.
36 ///
37 /// Pure so it is testable without apt. The two lines look like:
38 ///
39 /// ```text
40 /// Installed: 20260601~24.04.1
41 /// Candidate: 20260601~24.04.1
42 /// ```
43 ///
44 /// `(none)` is apt's way of saying not installed, and is mapped to `None` rather
45 /// than carried through as a version string that would never compare equal.
46 pub fn parse_apt_policy(output: &str) -> (Option<String>, Option<String>) {
47 let mut installed = None;
48 let mut candidate = None;
49 for line in output.lines() {
50 let line = line.trim();
51 if let Some(value) = line.strip_prefix("Installed:") {
52 installed = normalize_version(value);
53 } else if let Some(value) = line.strip_prefix("Candidate:") {
54 candidate = normalize_version(value);
55 }
56 }
57 (installed, candidate)
58 }
59
60 fn normalize_version(value: &str) -> Option<String> {
61 let value = value.trim();
62 if value.is_empty() || value == "(none)" {
63 None
64 } else {
65 Some(value.to_string())
66 }
67 }
68
69 /// Count `BEGIN CERTIFICATE` markers in a concatenated PEM bundle.
70 pub fn count_certificates(bundle: &str) -> i64 {
71 bundle
72 .lines()
73 .filter(|line| line.trim_start().starts_with("-----BEGIN CERTIFICATE-----"))
74 .count() as i64
75 }
76
77 /// Derive the overall status and the issue lines from the readings.
78 ///
79 /// Pure, and the whole judgement lives here so the rules are testable without a
80 /// host to read. Ordering is by severity: a bundle that cannot hold a working
81 /// trust store outranks one that is merely behind, which outranks not knowing.
82 ///
83 /// `lists_age_hours` of `None` means the stamp file was absent. That is treated
84 /// as not knowing rather than as fine: a host with no record of a successful
85 /// package-list update cannot support any claim about the candidate version.
86 pub fn classify(
87 package: &str,
88 installed: Option<&str>,
89 candidate: Option<&str>,
90 cert_count: Option<i64>,
91 min_certs: usize,
92 lists_age_hours: Option<i64>,
93 lists_max_age_hours: i64,
94 ) -> (&'static str, Vec<String>) {
95 let mut issues = Vec::new();
96 let mut status = "ok";
97
98 if let Some(count) = cert_count
99 && count < min_certs as i64
100 {
101 issues.push(format!(
102 "bundle holds {count} certificates, below the floor of {min_certs}"
103 ));
104 status = "thin";
105 }
106
107 match (installed, candidate) {
108 (Some(installed), Some(candidate)) if installed != candidate => {
109 issues.push(format!(
110 "{package} {installed} installed, {candidate} available"
111 ));
112 if status == "ok" {
113 status = "stale";
114 }
115 }
116 (None, _) => {
117 issues.push(format!("{package} is not installed"));
118 if status == "ok" {
119 status = "thin";
120 }
121 }
122 _ => {}
123 }
124
125 match lists_age_hours {
126 Some(age) if age > lists_max_age_hours => {
127 issues.push(format!(
128 "package lists last updated {age}h ago, over the {lists_max_age_hours}h limit, so the available version is not evidence of anything"
129 ));
130 if status == "ok" {
131 status = "unknown";
132 }
133 }
134 None => {
135 issues.push(
136 "no record of a successful package-list update, so the available version is not evidence of anything"
137 .to_string(),
138 );
139 if status == "ok" {
140 status = "unknown";
141 }
142 }
143 _ => {}
144 }
145
146 (status, issues)
147 }
148
149 async fn apt_policy(package: &str) -> Result<String, String> {
150 let output = Command::new("apt-cache")
151 .args(["policy", package])
152 .output()
153 .await
154 .map_err(|e| format!("apt-cache policy failed to run: {e}"))?;
155 if !output.status.success() {
156 return Err(format!(
157 "apt-cache policy exited {}: {}",
158 output.status.code().unwrap_or(-1),
159 String::from_utf8_lossy(&output.stderr).trim()
160 ));
161 }
162 Ok(String::from_utf8_lossy(&output.stdout).into_owned())
163 }
164
165 /// Age in whole hours of a stamp file's mtime. `None` when the file is absent
166 /// or its mtime cannot be read.
167 fn stamp_age_hours(path: &Path) -> Option<i64> {
168 let modified = std::fs::metadata(path).ok()?.modified().ok()?;
169 let modified = chrono::DateTime::<chrono::Utc>::from(modified);
170 Some(
171 chrono::Utc::now()
172 .signed_duration_since(modified)
173 .num_hours(),
174 )
175 }
176
177 /// Probe the local host's trust-anchor package and bundle.
178 ///
179 /// A failure to run `apt-cache` is an `error` and stops there: without the
180 /// version readings the other two signals cannot be assembled into a verdict,
181 /// and reporting a partial one would understate the situation. A bundle file
182 /// that cannot be read is not fatal, it just leaves `cert_count` unset.
183 #[instrument(skip_all)]
184 pub async fn check_ca_bundle(target_name: &str, config: &CaBundleConfig) -> CaBundleCheckResult {
185 let checked_at = chrono::Utc::now().to_rfc3339();
186
187 let policy = match apt_policy(&config.package).await {
188 Ok(output) => output,
189 Err(e) => {
190 return CaBundleCheckResult {
191 target: target_name.to_string(),
192 status: "error".to_string(),
193 package: config.package.clone(),
194 installed: None,
195 candidate: None,
196 cert_count: None,
197 lists_age_hours: None,
198 issues: Vec::new(),
199 checked_at,
200 error: Some(e),
201 };
202 }
203 };
204
205 let (installed, candidate) = parse_apt_policy(&policy);
206 let cert_count = std::fs::read_to_string(&config.bundle_path)
207 .ok()
208 .as_deref()
209 .map(count_certificates);
210 let lists_age_hours = stamp_age_hours(&config.update_stamp);
211
212 let (status, issues) = classify(
213 &config.package,
214 installed.as_deref(),
215 candidate.as_deref(),
216 cert_count,
217 config.min_certs,
218 lists_age_hours,
219 config.update_stamp_max_age_hours,
220 );
221
222 CaBundleCheckResult {
223 target: target_name.to_string(),
224 status: status.to_string(),
225 package: config.package.clone(),
226 installed,
227 candidate,
228 cert_count,
229 lists_age_hours,
230 issues,
231 checked_at,
232 error: None,
233 }
234 }
235
236 #[cfg(test)]
237 mod tests {
238 use super::*;
239
240 const POLICY: &str = "ca-certificates:
241 Installed: 20260601~24.04.1
242 Candidate: 20260601~24.04.1
243 Version table:
244 *** 20260601~24.04.1 500
245 500 https://mirror.hetzner.com/ubuntu/packages noble-updates/main amd64 Packages
246 100 /var/lib/dpkg/status
247 ";
248
249 #[test]
250 fn reads_both_versions_off_apt_policy() {
251 let (installed, candidate) = parse_apt_policy(POLICY);
252 assert_eq!(installed.as_deref(), Some("20260601~24.04.1"));
253 assert_eq!(candidate.as_deref(), Some("20260601~24.04.1"));
254 }
255
256 #[test]
257 fn an_uninstalled_package_reads_as_absent_not_as_a_version() {
258 let (installed, candidate) =
259 parse_apt_policy("ca-certificates:\n Installed: (none)\n Candidate: 20260601\n");
260 assert_eq!(installed, None);
261 assert_eq!(candidate.as_deref(), Some("20260601"));
262 }
263
264 #[test]
265 fn counts_only_certificate_markers() {
266 let bundle = "# comment\n-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----\n\
267 -----BEGIN CERTIFICATE-----\nBBBB\n-----END CERTIFICATE-----\n";
268 assert_eq!(count_certificates(bundle), 2);
269 assert_eq!(count_certificates(""), 0);
270 }
271
272 #[test]
273 fn a_current_host_with_fresh_lists_is_ok() {
274 let (status, issues) = classify(
275 "ca-certificates",
276 Some("20260601~24.04.1"),
277 Some("20260601~24.04.1"),
278 Some(121),
279 80,
280 Some(19),
281 48,
282 );
283 assert_eq!(status, "ok");
284 assert!(issues.is_empty());
285 }
286
287 #[test]
288 fn a_newer_candidate_is_stale_and_names_both_versions() {
289 let (status, issues) = classify(
290 "ca-certificates",
291 Some("20260601~24.04.1"),
292 Some("20261101~24.04.1"),
293 Some(121),
294 80,
295 Some(2),
296 48,
297 );
298 assert_eq!(status, "stale");
299 assert_eq!(issues.len(), 1);
300 assert!(issues[0].contains("20260601~24.04.1 installed"));
301 assert!(issues[0].contains("20261101~24.04.1 available"));
302 }
303
304 #[test]
305 fn matching_versions_prove_nothing_once_the_lists_have_gone_stale() {
306 // The failure this exists to stop: apt-daily.timer dies, the candidate
307 // freezes at whatever was last fetched, and installed == candidate reads
308 // green forever while the bundle quietly falls behind.
309 let (status, issues) = classify(
310 "ca-certificates",
311 Some("20260601~24.04.1"),
312 Some("20260601~24.04.1"),
313 Some(121),
314 80,
315 Some(400),
316 48,
317 );
318 assert_eq!(status, "unknown");
319 assert_eq!(issues.len(), 1);
320 assert!(issues[0].contains("400h ago"));
321 }
322
323 #[test]
324 fn a_missing_update_stamp_is_also_unknown_rather_than_fine() {
325 let (status, issues) = classify(
326 "ca-certificates",
327 Some("20260601~24.04.1"),
328 Some("20260601~24.04.1"),
329 Some(121),
330 80,
331 None,
332 48,
333 );
334 assert_eq!(status, "unknown");
335 assert!(issues[0].contains("no record of a successful package-list update"));
336 }
337
338 #[test]
339 fn a_truncated_bundle_outranks_a_stale_version() {
340 // Both are wrong at once. The one that means TLS is already broken wins
341 // the status line, and the other still gets its issue reported.
342 let (status, issues) = classify(
343 "ca-certificates",
344 Some("20260601~24.04.1"),
345 Some("20261101~24.04.1"),
346 Some(3),
347 80,
348 Some(2),
349 48,
350 );
351 assert_eq!(status, "thin");
352 assert_eq!(issues.len(), 2);
353 assert!(issues[0].contains("3 certificates"));
354 }
355
356 /// Runs the real probe against this host. Ignored by default because it
357 /// needs apt and reports whatever the machine happens to be, which is not a
358 /// property of the code. Run it by hand when changing the probe:
359 /// `cargo test --lib probes_this_host -- --ignored --nocapture`.
360 #[tokio::test]
361 #[ignore = "reads the live host's apt state"]
362 async fn probes_this_host() {
363 let config = CaBundleConfig {
364 package: "ca-certificates".into(),
365 bundle_path: "/etc/ssl/certs/ca-certificates.crt".into(),
366 min_certs: 80,
367 update_stamp: "/var/lib/apt/periodic/update-success-stamp".into(),
368 update_stamp_max_age_hours: 48,
369 interval_secs: 3600,
370 };
371 let result = check_ca_bundle("local", &config).await;
372 println!("{}", serde_json::to_string_pretty(&result).unwrap());
373 assert_ne!(
374 result.status, "error",
375 "probe could not run: {:?}",
376 result.error
377 );
378 }
379
380 #[test]
381 fn an_uninstalled_trust_package_is_thin() {
382 let (status, issues) = classify(
383 "ca-certificates",
384 None,
385 Some("20260601~24.04.1"),
386 None,
387 80,
388 Some(2),
389 48,
390 );
391 assert_eq!(status, "thin");
392 assert!(issues[0].contains("not installed"));
393 }
394 }
395