//! Read-only MCP tools for orienting on what is live. //! //! These answer the question that otherwise costs an ssh and a curl: what is //! running where, what is wrong with it, and how far behind is it. Nothing here //! promotes, deploys, or closes an incident. Acting stays with Sando and the //! CLI, so a session can read production without being able to change it. //! //! # Local and remote read the same shape //! //! Every tool takes an optional `instance`. Omitted, it reads this machine's //! database directly, which works whether or not a `pom serve` daemon is up //! here. Named, it resolves a configured peer and reads that instance's HTTP //! API over the tailnet, which is the only way to see checks that are local to //! *that* host: systemd units and backup freshness on the production box are //! not observable from here at all. //! //! Both paths land on the same types, `ops_status::Payload` for the status //! tools and [`VersionRow`] for versions, so the formatting below is written //! once and never branches on where the data came from. use std::fmt::Write as _; use ops_status::{Node, Payload, Status}; use schemars::JsonSchema; use serde::Deserialize; use tracing::instrument; use crate::api; use crate::error::{PomError, Result}; use crate::types::VersionRow; use crate::versions; use super::PomServer; /// How long to wait on a peer before giving up. A session is waiting on this /// answer, so a hung instance should say so quickly rather than stall the turn. const REMOTE_TIMEOUT_SECS: u64 = 10; #[derive(Debug, Deserialize, JsonSchema)] pub struct InstanceParams { /// PoM instance to read: omit for this machine, or name a configured peer /// (e.g. the production instance) to read that host's view. pub instance: Option, } #[derive(Debug, Deserialize, JsonSchema)] pub struct TargetInstanceParams { /// Target name, as configured (e.g. "mnw"). pub target: String, /// PoM instance to read: omit for this machine, or name a configured peer. pub instance: Option, } #[derive(Debug, Deserialize, JsonSchema)] pub struct TrendsParams { /// Target name, as configured (e.g. "mnw"). pub target: String, /// Window to report, in hours (default 24). pub hours: Option, /// Width of each latency bucket, in minutes (default 60). pub bucket_minutes: Option, /// PoM instance to read: omit for this machine, or name a configured peer. pub instance: Option, } /// Which instance a tool call is reading. enum Source { /// This machine's database, read without going through HTTP. Local, /// A configured peer's API. Remote { name: String, base_url: String, token: Option, }, } impl PomServer { /// Resolve an `instance` parameter to the source to read. /// /// An unknown name lists what is configured rather than just refusing: the /// caller cannot see pom.toml, so the names are otherwise unguessable. fn source(&self, instance: Option<&str>) -> Result { match instance { None | Some("local" | "") => Ok(Source::Local), Some(name) => match self.config.peers.get(name) { Some(peer) => Ok(Source::Remote { name: name.to_string(), base_url: crate::peer::peer_base_url(&peer.address), token: peer.token.clone(), }), None => { let mut known: Vec<&str> = self.config.peers.keys().map(String::as_str).collect(); known.sort_unstable(); Err(PomError::Config(format!( "unknown instance: {name}. Configured peers: {}. Omit `instance` to read \ this machine.", if known.is_empty() { "none".to_string() } else { known.join(", ") } ))) } }, } } /// GET a path on a remote instance, with the peer's bearer token. async fn get_remote( &self, base_url: &str, token: Option<&str>, path: &str, ) -> Result { let url = format!("{base_url}{path}"); let client = crate::tls::https_client_builder() .timeout(std::time::Duration::from_secs(REMOTE_TIMEOUT_SECS)) .build()?; let mut request = client.get(&url); if let Some(token) = token { request = request.bearer_auth(token); } let response = request .send() .await .map_err(|e| PomError::Config(format!("could not reach {url}: {e}")))?; let status = response.status(); if !status.is_success() { // 401 here means the peer's token is wrong or missing in pom.toml, // which is a config problem on this side and worth naming as one. return Err(PomError::Config(format!( "{url} returned {status}{}", if status.as_u16() == 401 { " (check this peer's token in pom.toml)" } else { "" } ))); } Ok(response.json().await?) } /// The status payload for whichever instance was asked for. async fn payload_for(&self, instance: Option<&str>) -> Result<(String, Payload)> { match self.source(instance)? { Source::Local => Ok(( "local".to_string(), api::status_payload(&self.pool, &self.config).await, )), Source::Remote { name, base_url, token, } => { let value = self .get_remote(&base_url, token.as_deref(), "/status.json") .await?; let payload = serde_json::from_value(value)?; Ok((name, payload)) } } } #[instrument(skip_all)] pub async fn status_table_impl(&self, params: InstanceParams) -> Result { let (instance, payload) = self.payload_for(params.instance.as_deref()).await?; Ok(format_status_table(&instance, &payload)) } #[instrument(skip_all)] pub async fn target_status_impl(&self, params: TargetInstanceParams) -> Result { let (instance, payload) = self.payload_for(params.instance.as_deref()).await?; let wanted = format!("target:{}", params.target); let Some(node) = payload.nodes.iter().find(|n| n.id == wanted) else { let names: Vec<&str> = payload .nodes .iter() .filter_map(|n| n.id.strip_prefix("target:")) .collect(); return Ok(format!( "Unknown target: {} on instance {instance}. Known targets: {}", params.target, if names.is_empty() { "none".to_string() } else { names.join(", ") } )); }; Ok(format_target_detail(&instance, node)) } #[instrument(skip_all)] pub async fn incidents_impl(&self, params: InstanceParams) -> Result { let (instance, payload) = self.payload_for(params.instance.as_deref()).await?; Ok(format_incidents(&instance, &payload)) } #[instrument(skip_all)] pub async fn versions_impl(&self, params: InstanceParams) -> Result { let (instance, rows) = match self.source(params.instance.as_deref())? { Source::Local => ( "local".to_string(), versions::collect(&self.pool, &self.config).await?, ), Source::Remote { name, base_url, token, } => { let value = self .get_remote(&base_url, token.as_deref(), "/api/versions") .await?; let rows: Vec = serde_json::from_value(value)?; (name, rows) } }; let mut out = format!("# Versions on {instance}\n\n"); out.push_str(&crate::display::format_versions(&rows)); // The count is taken against a checkout on the instance being read, so // a remote answer says what is live there, not how far behind here. if instance != "local" { out.push_str( "\nBEHIND is measured against a checkout on that host, and is blank where it has \ no repo.\n", ); } Ok(out) } #[instrument(skip_all)] pub async fn trends_impl(&self, params: TrendsParams) -> Result { let hours = params.hours.unwrap_or(24); let bucket_minutes = params.bucket_minutes.unwrap_or(60); let (instance, trends) = match self.source(params.instance.as_deref())? { Source::Local => { if self.config.get_target(¶ms.target).is_none() { return Ok(format!("Unknown target: {}", params.target)); } ( "local".to_string(), api::build_trends(&self.pool, ¶ms.target, hours, bucket_minutes).await, ) } Source::Remote { name, base_url, token, } => { // The target name goes into a URL path. Config keys are plain // identifiers, so anything else is rejected here rather than // escaped: a `../` or a `?` would address a different endpoint. if !is_config_name(¶ms.target) { return Ok(format!("Unusable target name: {:?}", params.target)); } let path = format!( "/api/trends/{}?hours={hours}&bucket_minutes={bucket_minutes}", params.target ); let value = self.get_remote(&base_url, token.as_deref(), &path).await?; (name, serde_json::from_value(value)?) } }; Ok(format_trends(&instance, &trends)) } } /// Every target on one line, worst-first, with the reason it is not green. fn format_status_table(instance: &str, payload: &Payload) -> String { if payload.nodes.is_empty() { return format!("No targets configured on {instance}.\n"); } let mut nodes: Vec<&Node> = payload.nodes.iter().collect(); // Worst first: the point of the table is that the problem is on line one. nodes.sort_by(|a, b| b.status.cmp(&a.status).then(a.id.cmp(&b.id))); let rows: Vec<[String; 5]> = nodes .iter() .map(|n| { [ n.id.strip_prefix("target:").unwrap_or(&n.id).to_string(), status_word(n.status).to_string(), field_text(n, "version"), field_text(n, "checked"), worst_condition_summary(n), ] }) .collect(); const HEADERS: [&str; 5] = ["TARGET", "STATUS", "VERSION", "CHECKED", "WHY"]; let widths: Vec = (0..HEADERS.len()) .map(|i| { rows.iter() .map(|r| r[i].chars().count()) .chain(std::iter::once(HEADERS[i].len())) .max() .unwrap_or(0) }) .collect(); let mut out = format!( "# {} on {instance} ({} target{}, worst: {})\n\n", payload.source, nodes.len(), if nodes.len() == 1 { "" } else { "s" }, status_word(payload.worst_status()), ); write_row(&mut out, &HEADERS.map(String::from), &widths); for row in &rows { write_row(&mut out, row, &widths); } let _ = write!( out, "\nGenerated {}. Use target_status for the full condition list.\n", payload.generated_at.format("%Y-%m-%d %H:%M UTC") ); out } /// One target in full: its fields, then every condition with its detail. fn format_target_detail(instance: &str, node: &Node) -> String { let name = node.id.strip_prefix("target:").unwrap_or(&node.id); let mut out = format!( "# {name} ({}) on {instance}: {}\n\n", node.label, status_word(node.status) ); if !node.fields.is_empty() { for field in &node.fields { let _ = writeln!(out, "{}: {}", field.label, value_text(&field.value)); } out.push('\n'); } if node.conditions.is_empty() { out.push_str("No conditions recorded.\n"); return out; } // Worst first here too, so a failing check is never below three green ones. let mut conditions: Vec<&ops_status::Condition> = node.conditions.iter().collect(); conditions.sort_by_key(|c| std::cmp::Reverse(c.status)); for condition in conditions { let _ = write!( out, "[{}] {}", status_word(condition.status), condition.condition_type ); if let Some(since) = condition.since { let _ = write!(out, " (since {})", since.format("%Y-%m-%d %H:%M UTC")); } if let Some(detail) = &condition.detail { let _ = write!(out, ": {}", scrub(detail)); } out.push('\n'); } out } /// Open incidents across every target, and anything else that is not green. fn format_incidents(instance: &str, payload: &Payload) -> String { let mut incidents = Vec::new(); let mut other = Vec::new(); for node in &payload.nodes { let name = node.id.strip_prefix("target:").unwrap_or(&node.id); for condition in &node.conditions { if condition.status == Status::Ok || condition.status == Status::Pending { continue; } let line = format!( "[{}] {name} / {}{}{}", status_word(condition.status), condition.condition_type, condition .since .map(|s| format!(" since {}", s.format("%Y-%m-%d %H:%M UTC"))) .unwrap_or_default(), condition .detail .as_ref() .map(|d| format!(": {}", scrub(d))) .unwrap_or_default(), ); if condition.condition_type == "incident" { incidents.push(line); } else { other.push(line); } } } let mut out = format!("# Open incidents on {instance}\n\n"); if incidents.is_empty() { out.push_str("No open incidents.\n"); } else { for line in &incidents { let _ = writeln!(out, "{line}"); } } // A failing check that has not yet opened an incident is still the answer to // "is anything wrong", so it is reported rather than filtered out. if !other.is_empty() { out.push_str("\nOther checks not passing:\n"); for line in &other { let _ = writeln!(out, "{line}"); } } out } /// Latency over the window, against the 7-day baseline. fn format_trends(instance: &str, trends: &api::TrendResponse) -> String { let mut out = format!( "# {} latency on {instance}: last {}h, {}-minute buckets\n\n", trends.target, trends.window_hours, trends.bucket_minutes ); match &trends.overall { Some(o) => { let _ = writeln!( out, "Window: avg {:.0}ms, p95 {}ms, range {}-{}ms ({} samples)", o.avg_ms, o.p95_ms, o.min_ms, o.max_ms, o.sample_count ); } None => out.push_str("Window: no operational checks in this window.\n"), } if let Some(b) = &trends.baseline { let _ = writeln!( out, "7d baseline: avg {:.0}ms, p95 {}ms ({} samples)", b.avg_ms, b.p95_ms, b.sample_count ); } if trends.buckets.is_empty() { return out; } out.push_str("\nBUCKET AVG P95 N\n"); for bucket in &trends.buckets { let _ = writeln!( out, "{:<18} {:>4.0}ms {:>4}ms {:>4}", bucket.period_start.chars().take(16).collect::(), bucket.avg_ms, bucket.p95_ms, bucket.sample_count ); } out } /// The wire spelling of a status, which is also the shortest honest label. fn status_word(status: Status) -> &'static str { status.as_str() } /// A node field's value as one short string. fn value_text(value: &ops_status::Value) -> String { use ops_status::Value; match value { Value::Text { value } | Value::Ident { value, .. } | Value::Version { value } => { scrub(value) } Value::Path { value } => scrub(value), Value::Instant { value } => value.format("%Y-%m-%d %H:%M UTC").to_string(), Value::Duration { seconds } => format!("{seconds}s"), Value::Progress { value, unit, .. } | Value::Quantity { value, unit } => { format!("{value:.1}{}", unit.as_deref().unwrap_or("")) } Value::State { value } => status_word(*value).to_string(), Value::Link { url, text } => format!("{} <{}>", scrub(text.as_deref().unwrap_or("")), url), } } /// A named field's text, or `-` when the node does not carry it. fn field_text(node: &Node, label: &str) -> String { node.fields .iter() .find(|f| f.label == label) .map_or_else(|| "-".to_string(), |f| value_text(&f.value)) } /// The worst thing said about a node, in a few words. Green nodes get a dash. fn worst_condition_summary(node: &Node) -> String { let worst = node .conditions .iter() .filter(|c| c.status != Status::Ok) .max_by_key(|c| c.status); match worst { None => "-".to_string(), Some(c) => { let detail = c .detail .as_deref() .map(|d| format!(": {}", scrub(d))) .unwrap_or_default(); let line = format!("{}{detail}", c.condition_type); // One line per target is the whole point of this table, so a long // detail is cut rather than allowed to wrap. if line.chars().count() > 60 { format!("{}...", line.chars().take(57).collect::()) } else { line } } } } /// Whether a string is shaped like a config key, and so safe to put in a URL /// path segment without escaping. fn is_config_name(s: &str) -> bool { !s.is_empty() && s.len() <= 64 && s.chars() .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.') } /// Strip terminal control characters from values a monitored target chose. /// /// Same surface as the CLI display sink: these strings reach an operator's /// terminal through the session transcript. fn scrub(s: &str) -> String { s.chars().filter(|c| !c.is_control()).collect() } /// Write one table row, every column but the last padded to its width. fn write_row(out: &mut String, cells: &[String; 5], widths: &[usize]) { for (i, cell) in cells.iter().enumerate() { if i + 1 == cells.len() { let _ = writeln!(out, "{cell}"); } else { let pad = widths[i].saturating_sub(cell.chars().count()); let _ = write!(out, "{cell}{:pad$} ", "", pad = pad); } } }