//! Local CA-bundle freshness check. Reads the trust-anchor package's installed //! version against what apt would install now, how long ago the package lists //! were last refreshed, and how many certificates the concatenated bundle //! actually holds. //! //! This is the leading indicator. The trailing one already exists: `checks::tls` //! validates every target against the host trust store as well as the bundled //! web PKI, so a bundle that has gone thin surfaces as a chain the host rejects. //! By then multithreaded is already failing, because it takes its outbound trust //! anchors from this bundle on all three of its outbound paths (OAuth to the MNW //! server, link previews, S3) and neither reqwest nor aws-smithy-http-client //! offers bundled roots to fall back on. //! //! Three signals, and the second is the one that is easy to leave out: //! //! 1. Installed versus candidate version of the package. //! 2. Age of the last successful package-list update. Without it, "installed //! equals candidate" stays true forever the moment `apt-daily.timer` stops //! running, and the check reports a green it has no evidence for. //! 3. Certificate count against a floor, which catches a truncated or emptied //! bundle that both version numbers call fine. //! //! The probe runs against the local host via `apt-cache policy`, which works //! unprivileged and inside pom.service's own sandbox (verified on prod-1 under //! `ProtectSystem=strict`, `ProtectHome`, `PrivateTmp`). use std::path::Path; use tokio::process::Command; use tracing::instrument; use crate::config::CaBundleConfig; use crate::types::CaBundleCheckResult; /// Pull the installed and candidate versions out of `apt-cache policy `. /// /// Pure so it is testable without apt. The two lines look like: /// /// ```text /// Installed: 20260601~24.04.1 /// Candidate: 20260601~24.04.1 /// ``` /// /// `(none)` is apt's way of saying not installed, and is mapped to `None` rather /// than carried through as a version string that would never compare equal. pub fn parse_apt_policy(output: &str) -> (Option, Option) { let mut installed = None; let mut candidate = None; for line in output.lines() { let line = line.trim(); if let Some(value) = line.strip_prefix("Installed:") { installed = normalize_version(value); } else if let Some(value) = line.strip_prefix("Candidate:") { candidate = normalize_version(value); } } (installed, candidate) } fn normalize_version(value: &str) -> Option { let value = value.trim(); if value.is_empty() || value == "(none)" { None } else { Some(value.to_string()) } } /// Count `BEGIN CERTIFICATE` markers in a concatenated PEM bundle. pub fn count_certificates(bundle: &str) -> i64 { bundle .lines() .filter(|line| line.trim_start().starts_with("-----BEGIN CERTIFICATE-----")) .count() as i64 } /// Derive the overall status and the issue lines from the readings. /// /// Pure, and the whole judgement lives here so the rules are testable without a /// host to read. Ordering is by severity: a bundle that cannot hold a working /// trust store outranks one that is merely behind, which outranks not knowing. /// /// `lists_age_hours` of `None` means the stamp file was absent. That is treated /// as not knowing rather than as fine: a host with no record of a successful /// package-list update cannot support any claim about the candidate version. pub fn classify( package: &str, installed: Option<&str>, candidate: Option<&str>, cert_count: Option, min_certs: usize, lists_age_hours: Option, lists_max_age_hours: i64, ) -> (&'static str, Vec) { let mut issues = Vec::new(); let mut status = "ok"; if let Some(count) = cert_count && count < min_certs as i64 { issues.push(format!( "bundle holds {count} certificates, below the floor of {min_certs}" )); status = "thin"; } match (installed, candidate) { (Some(installed), Some(candidate)) if installed != candidate => { issues.push(format!( "{package} {installed} installed, {candidate} available" )); if status == "ok" { status = "stale"; } } (None, _) => { issues.push(format!("{package} is not installed")); if status == "ok" { status = "thin"; } } _ => {} } match lists_age_hours { Some(age) if age > lists_max_age_hours => { issues.push(format!( "package lists last updated {age}h ago, over the {lists_max_age_hours}h limit, so the available version is not evidence of anything" )); if status == "ok" { status = "unknown"; } } None => { issues.push( "no record of a successful package-list update, so the available version is not evidence of anything" .to_string(), ); if status == "ok" { status = "unknown"; } } _ => {} } (status, issues) } async fn apt_policy(package: &str) -> Result { let output = Command::new("apt-cache") .args(["policy", package]) .output() .await .map_err(|e| format!("apt-cache policy failed to run: {e}"))?; if !output.status.success() { return Err(format!( "apt-cache policy exited {}: {}", output.status.code().unwrap_or(-1), String::from_utf8_lossy(&output.stderr).trim() )); } Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } /// Age in whole hours of a stamp file's mtime. `None` when the file is absent /// or its mtime cannot be read. fn stamp_age_hours(path: &Path) -> Option { let modified = std::fs::metadata(path).ok()?.modified().ok()?; let modified = chrono::DateTime::::from(modified); Some( chrono::Utc::now() .signed_duration_since(modified) .num_hours(), ) } /// Probe the local host's trust-anchor package and bundle. /// /// A failure to run `apt-cache` is an `error` and stops there: without the /// version readings the other two signals cannot be assembled into a verdict, /// and reporting a partial one would understate the situation. A bundle file /// that cannot be read is not fatal, it just leaves `cert_count` unset. #[instrument(skip_all)] pub async fn check_ca_bundle(target_name: &str, config: &CaBundleConfig) -> CaBundleCheckResult { let checked_at = chrono::Utc::now().to_rfc3339(); let policy = match apt_policy(&config.package).await { Ok(output) => output, Err(e) => { return CaBundleCheckResult { target: target_name.to_string(), status: "error".to_string(), package: config.package.clone(), installed: None, candidate: None, cert_count: None, lists_age_hours: None, issues: Vec::new(), checked_at, error: Some(e), }; } }; let (installed, candidate) = parse_apt_policy(&policy); let cert_count = std::fs::read_to_string(&config.bundle_path) .ok() .as_deref() .map(count_certificates); let lists_age_hours = stamp_age_hours(&config.update_stamp); let (status, issues) = classify( &config.package, installed.as_deref(), candidate.as_deref(), cert_count, config.min_certs, lists_age_hours, config.update_stamp_max_age_hours, ); CaBundleCheckResult { target: target_name.to_string(), status: status.to_string(), package: config.package.clone(), installed, candidate, cert_count, lists_age_hours, issues, checked_at, error: None, } } #[cfg(test)] mod tests { use super::*; const POLICY: &str = "ca-certificates: Installed: 20260601~24.04.1 Candidate: 20260601~24.04.1 Version table: *** 20260601~24.04.1 500 500 https://mirror.hetzner.com/ubuntu/packages noble-updates/main amd64 Packages 100 /var/lib/dpkg/status "; #[test] fn reads_both_versions_off_apt_policy() { let (installed, candidate) = parse_apt_policy(POLICY); assert_eq!(installed.as_deref(), Some("20260601~24.04.1")); assert_eq!(candidate.as_deref(), Some("20260601~24.04.1")); } #[test] fn an_uninstalled_package_reads_as_absent_not_as_a_version() { let (installed, candidate) = parse_apt_policy("ca-certificates:\n Installed: (none)\n Candidate: 20260601\n"); assert_eq!(installed, None); assert_eq!(candidate.as_deref(), Some("20260601")); } #[test] fn counts_only_certificate_markers() { let bundle = "# comment\n-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----\n\ -----BEGIN CERTIFICATE-----\nBBBB\n-----END CERTIFICATE-----\n"; assert_eq!(count_certificates(bundle), 2); assert_eq!(count_certificates(""), 0); } #[test] fn a_current_host_with_fresh_lists_is_ok() { let (status, issues) = classify( "ca-certificates", Some("20260601~24.04.1"), Some("20260601~24.04.1"), Some(121), 80, Some(19), 48, ); assert_eq!(status, "ok"); assert!(issues.is_empty()); } #[test] fn a_newer_candidate_is_stale_and_names_both_versions() { let (status, issues) = classify( "ca-certificates", Some("20260601~24.04.1"), Some("20261101~24.04.1"), Some(121), 80, Some(2), 48, ); assert_eq!(status, "stale"); assert_eq!(issues.len(), 1); assert!(issues[0].contains("20260601~24.04.1 installed")); assert!(issues[0].contains("20261101~24.04.1 available")); } #[test] fn matching_versions_prove_nothing_once_the_lists_have_gone_stale() { // The failure this exists to stop: apt-daily.timer dies, the candidate // freezes at whatever was last fetched, and installed == candidate reads // green forever while the bundle quietly falls behind. let (status, issues) = classify( "ca-certificates", Some("20260601~24.04.1"), Some("20260601~24.04.1"), Some(121), 80, Some(400), 48, ); assert_eq!(status, "unknown"); assert_eq!(issues.len(), 1); assert!(issues[0].contains("400h ago")); } #[test] fn a_missing_update_stamp_is_also_unknown_rather_than_fine() { let (status, issues) = classify( "ca-certificates", Some("20260601~24.04.1"), Some("20260601~24.04.1"), Some(121), 80, None, 48, ); assert_eq!(status, "unknown"); assert!(issues[0].contains("no record of a successful package-list update")); } #[test] fn a_truncated_bundle_outranks_a_stale_version() { // Both are wrong at once. The one that means TLS is already broken wins // the status line, and the other still gets its issue reported. let (status, issues) = classify( "ca-certificates", Some("20260601~24.04.1"), Some("20261101~24.04.1"), Some(3), 80, Some(2), 48, ); assert_eq!(status, "thin"); assert_eq!(issues.len(), 2); assert!(issues[0].contains("3 certificates")); } /// Runs the real probe against this host. Ignored by default because it /// needs apt and reports whatever the machine happens to be, which is not a /// property of the code. Run it by hand when changing the probe: /// `cargo test --lib probes_this_host -- --ignored --nocapture`. #[tokio::test] #[ignore = "reads the live host's apt state"] async fn probes_this_host() { let config = CaBundleConfig { package: "ca-certificates".into(), bundle_path: "/etc/ssl/certs/ca-certificates.crt".into(), min_certs: 80, update_stamp: "/var/lib/apt/periodic/update-success-stamp".into(), update_stamp_max_age_hours: 48, interval_secs: 3600, }; let result = check_ca_bundle("local", &config).await; println!("{}", serde_json::to_string_pretty(&result).unwrap()); assert_ne!( result.status, "error", "probe could not run: {:?}", result.error ); } #[test] fn an_uninstalled_trust_package_is_thin() { let (status, issues) = classify( "ca-certificates", None, Some("20260601~24.04.1"), None, 80, Some(2), 48, ); assert_eq!(status, "thin"); assert!(issues[0].contains("not installed")); } }