//! Local systemd unit health check. Watches named daemons for liveness and
//! crash-loops, and optionally flags any unit on the host sitting in the failed
//! state.
//!
//! This closes a whole class of silent failure. bentod crash-looped 13,836 times
//! over ~20h reading as `activating` the entire time (a crash-loop never settles
//! to `failed`, so an is-active check alone misses it); sandod-backup-fetch sat
//! `failed` for four days behind a nightly timer with nothing watching. Nothing
//! watched the daemons that watch everything else. A liveness probe that treats a
//! climbing `NRestarts` as unhealthy catches the first, and a `systemctl
//! --failed` sweep catches the second.
//!
//! The probe runs against the local host via `systemctl show` / `systemctl
//! list-units --failed`, covering both the system bus and, for user-scoped units
//! like bentod, the `--user` bus (the audit's "user-bus-capable probe").

use tokio::process::Command;
use tracing::instrument;

use crate::config::SystemdUnit;
use crate::types::{SystemdCheckResult, SystemdUnitSnapshot};

/// Per-unit health derived from one `systemctl show` reading.
///
/// Pure so it is testable without a live `systemctl`. Returns the unit status
/// and, when not healthy, one issue line naming why.
///
/// A crash-loop is judged by `NRestarts` rather than `ActiveState`: a flapping
/// unit reads as `activating` (auto-restart) forever, never `failed`, so the
/// restart count is the only single-reading signal that catches it. `not-found`
/// (the unit file is missing or mistyped) is distinguished from a loaded unit
/// that is merely down.
pub fn classify_unit(
    active_state: &str,
    load_state: &str,
    n_restarts: i64,
    restart_threshold: i64,
) -> (&'static str, Option<String>) {
    if load_state == "not-found" {
        return (
            "not-loaded",
            Some("unit not loaded (LoadState=not-found)".into()),
        );
    }
    if n_restarts >= restart_threshold {
        return (
            "crash-loop",
            Some(format!(
                "{n_restarts} restarts (>= {restart_threshold}), unit is flapping"
            )),
        );
    }
    if active_state == "active" {
        return ("active", None);
    }
    ("down", Some(format!("ActiveState={active_state}")))
}

/// Aggregate per-unit snapshots and the host-wide failed-unit sweep into one
/// overall status. A watched unit that is down or not loaded is the worst
/// signal (red); a crash-looping watched unit or any host-wide failed unit is
/// `degraded` (yellow, look at this).
fn overall_status(units: &[SystemdUnitSnapshot], failed_units: &[String]) -> &'static str {
    if units
        .iter()
        .any(|u| u.status == "down" || u.status == "not-loaded")
    {
        "down"
    } else if !failed_units.is_empty() || units.iter().any(|u| u.status == "crash-loop") {
        "degraded"
    } else {
        "operational"
    }
}

/// Parse a `systemctl show` key=value block into the four properties we read.
/// A missing property reads as an empty string (or 0 for `NRestarts`), which
/// [`classify_unit`] treats as not-active, the safe direction.
fn parse_show(output: &str) -> (String, String, String, i64) {
    let mut active_state = String::new();
    let mut sub_state = String::new();
    let mut load_state = String::new();
    let mut n_restarts = 0i64;
    for line in output.lines() {
        let Some((key, value)) = line.split_once('=') else {
            continue;
        };
        match key {
            "ActiveState" => active_state = value.to_string(),
            "SubState" => sub_state = value.to_string(),
            "LoadState" => load_state = value.to_string(),
            "NRestarts" => n_restarts = value.trim().parse().unwrap_or(0),
            _ => {}
        }
    }
    (active_state, sub_state, load_state, n_restarts)
}

/// The first whitespace token of each non-empty line, `list-units --plain
/// --no-legend` puts the unit name first. Used to read the failed-unit sweep.
fn parse_failed_units(output: &str) -> Vec<String> {
    output
        .lines()
        .filter_map(|line| line.split_whitespace().next())
        .filter(|tok| !tok.is_empty())
        .map(str::to_string)
        .collect()
}

async fn systemctl_show(user: bool, unit: &str) -> Result<(String, String, String, i64), String> {
    let mut cmd = Command::new("systemctl");
    if user {
        cmd.arg("--user");
    }
    cmd.args([
        "show",
        unit,
        "--property=ActiveState,SubState,LoadState,NRestarts",
        "--no-pager",
    ]);
    let output = cmd.output().await.map_err(|e| e.to_string())?;
    // `systemctl show` exits 0 even for an unknown unit (it reports
    // LoadState=not-found), so a non-zero exit is a real invocation failure
    // (no systemd, no --user bus) rather than a missing unit.
    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
        return Err(format!(
            "systemctl show exited {}: {}",
            output.status.code().unwrap_or(-1),
            stderr.trim()
        ));
    }
    Ok(parse_show(&String::from_utf8_lossy(&output.stdout)))
}

async fn systemctl_failed(user: bool) -> Result<Vec<String>, String> {
    let mut cmd = Command::new("systemctl");
    if user {
        cmd.arg("--user");
    }
    cmd.args([
        "list-units",
        "--failed",
        "--no-legend",
        "--plain",
        "--no-pager",
    ]);
    let output = cmd.output().await.map_err(|e| e.to_string())?;
    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
        return Err(format!(
            "systemctl list-units --failed exited {}: {}",
            output.status.code().unwrap_or(-1),
            stderr.trim()
        ));
    }
    Ok(parse_failed_units(&String::from_utf8_lossy(&output.stdout)))
}

/// Probe the configured units on the local host and, when `check_failed` is set,
/// sweep for any unit in the failed state on each bus the watched units span.
///
/// A per-unit probe failure marks that unit `down` with the error as its issue
/// rather than aborting the whole check, so one broken invocation cannot blind
/// the rest. A failed sweep is recorded in `error` but does not by itself change
/// the status, the watched units still report.
#[instrument(skip_all)]
pub async fn check_systemd(
    target_name: &str,
    units: &[SystemdUnit],
    check_failed: bool,
    restart_threshold: i64,
) -> SystemdCheckResult {
    let checked_at = chrono::Utc::now().to_rfc3339();
    let mut snapshots: Vec<SystemdUnitSnapshot> = Vec::new();
    let mut issues: Vec<String> = Vec::new();
    let mut errors: Vec<String> = Vec::new();

    for unit in units {
        let scope = if unit.user { "user" } else { "system" };
        match systemctl_show(unit.user, &unit.name).await {
            Ok((active_state, sub_state, load_state, n_restarts)) => {
                let (status, issue) =
                    classify_unit(&active_state, &load_state, n_restarts, restart_threshold);
                if let Some(msg) = issue {
                    issues.push(format!("{} ({scope}): {msg}", unit.name));
                }
                snapshots.push(SystemdUnitSnapshot {
                    name: unit.name.clone(),
                    scope: scope.to_string(),
                    active_state,
                    sub_state,
                    n_restarts,
                    status: status.to_string(),
                });
            }
            Err(e) => {
                issues.push(format!("{} ({scope}): probe failed: {e}", unit.name));
                errors.push(format!("{}: {e}", unit.name));
                snapshots.push(SystemdUnitSnapshot {
                    name: unit.name.clone(),
                    scope: scope.to_string(),
                    active_state: "unknown".to_string(),
                    sub_state: String::new(),
                    n_restarts: 0,
                    status: "down".to_string(),
                });
            }
        }
    }

    let mut failed_units: Vec<String> = Vec::new();
    if check_failed {
        // Sweep every bus the watched units span (system, and --user if any
        // watched unit is user-scoped), so a failed oneshot/timer on the same
        // bus as bentod is not missed just because it was not enumerated.
        let want_user = units.iter().any(|u| u.user);
        let mut scopes = vec![false];
        if want_user {
            scopes.push(true);
        }
        for user in scopes {
            match systemctl_failed(user).await {
                Ok(mut names) => failed_units.append(&mut names),
                Err(e) => errors.push(format!("failed-unit sweep ({user}): {e}")),
            }
        }
        failed_units.sort();
        failed_units.dedup();
        for name in &failed_units {
            issues.push(format!("host unit failed: {name}"));
        }
    }

    let status = overall_status(&snapshots, &failed_units);

    SystemdCheckResult {
        target: target_name.to_string(),
        status: status.to_string(),
        units: snapshots,
        failed_units,
        issues,
        checked_at,
        error: (!errors.is_empty()).then(|| errors.join("; ")),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn snap(name: &str, status: &str) -> SystemdUnitSnapshot {
        SystemdUnitSnapshot {
            name: name.to_string(),
            scope: "system".to_string(),
            active_state: "active".to_string(),
            sub_state: "running".to_string(),
            n_restarts: 0,
            status: status.to_string(),
        }
    }

    #[test]
    fn active_unit_is_healthy() {
        let (status, issue) = classify_unit("active", "loaded", 0, 5);
        assert_eq!(status, "active");
        assert!(issue.is_none());
    }

    #[test]
    fn inactive_unit_is_down() {
        let (status, issue) = classify_unit("inactive", "loaded", 0, 5);
        assert_eq!(status, "down");
        assert!(issue.unwrap().contains("ActiveState=inactive"));
    }

    #[test]
    fn failed_unit_is_down() {
        let (status, _) = classify_unit("failed", "loaded", 0, 5);
        assert_eq!(status, "down");
    }

    #[test]
    fn crash_loop_while_activating_is_caught() {
        // The bentod case: a flapping unit reads `activating`, never `failed`,
        // but NRestarts climbs. Restart count is what catches it.
        let (status, issue) = classify_unit("activating", "loaded", 42, 5);
        assert_eq!(status, "crash-loop");
        assert!(issue.unwrap().contains("42 restarts"));
    }

    #[test]
    fn crash_loop_while_active_is_still_flagged() {
        // A unit can be `active` right now yet have restarted many times.
        let (status, _) = classify_unit("active", "loaded", 9, 5);
        assert_eq!(status, "crash-loop");
    }

    #[test]
    fn restart_threshold_boundary_is_inclusive() {
        // NRestarts == threshold must trip (pins `>=` vs `>`).
        assert_eq!(classify_unit("active", "loaded", 5, 5).0, "crash-loop");
        assert_eq!(classify_unit("active", "loaded", 4, 5).0, "active");
    }

    #[test]
    fn missing_unit_file_is_not_loaded() {
        let (status, issue) = classify_unit("inactive", "not-found", 0, 5);
        assert_eq!(status, "not-loaded");
        assert!(issue.unwrap().contains("not-found"));
    }

    #[test]
    fn all_healthy_is_operational() {
        let units = vec![
            snap("sandod.service", "active"),
            snap("wam.service", "active"),
        ];
        assert_eq!(overall_status(&units, &[]), "operational");
    }

    #[test]
    fn a_down_watched_unit_makes_the_host_down() {
        let units = vec![snap("sandod.service", "down")];
        assert_eq!(overall_status(&units, &[]), "down");
    }

    #[test]
    fn a_not_loaded_watched_unit_makes_the_host_down() {
        let units = vec![snap("bentod.service", "not-loaded")];
        assert_eq!(overall_status(&units, &[]), "down");
    }

    #[test]
    fn a_crash_loop_alone_is_degraded_not_down() {
        let units = vec![snap("bentod.service", "crash-loop")];
        assert_eq!(overall_status(&units, &[]), "degraded");
    }

    #[test]
    fn a_host_wide_failed_unit_is_degraded() {
        // sandod-backup-fetch: failed on the host but not a watched daemon.
        let units = vec![snap("sandod.service", "active")];
        assert_eq!(
            overall_status(&units, &["sandod-backup-fetch.service".to_string()]),
            "degraded"
        );
    }

    #[test]
    fn a_down_watched_unit_beats_a_failed_sweep() {
        let units = vec![snap("sandod.service", "down")];
        assert_eq!(
            overall_status(&units, &["other.service".to_string()]),
            "down"
        );
    }

    #[test]
    fn parse_show_reads_the_four_properties() {
        let out = "ActiveState=active\nSubState=running\nLoadState=loaded\nNRestarts=3\n";
        let (active, sub, load, n) = parse_show(out);
        assert_eq!(active, "active");
        assert_eq!(sub, "running");
        assert_eq!(load, "loaded");
        assert_eq!(n, 3);
    }

    #[test]
    fn parse_show_tolerates_missing_and_extra_keys() {
        let out = "Id=foo.service\nActiveState=failed\nUnrelated=x\n";
        let (active, sub, load, n) = parse_show(out);
        assert_eq!(active, "failed");
        assert!(sub.is_empty());
        assert!(load.is_empty());
        assert_eq!(n, 0, "absent NRestarts defaults to 0");
    }

    #[test]
    fn parse_failed_units_takes_the_first_column() {
        let out = "  sandod-backup-fetch.service loaded failed failed Sando backup puller\n\
                   foo.timer loaded failed failed A timer\n";
        let names = parse_failed_units(out);
        assert_eq!(
            names,
            vec![
                "sandod-backup-fetch.service".to_string(),
                "foo.timer".to_string()
            ]
        );
    }

    #[test]
    fn parse_failed_units_empty_is_empty() {
        assert!(parse_failed_units("").is_empty());
        assert!(parse_failed_units("\n  \n").is_empty());
    }
}