max / makenotwork
- Co-Authored-By
- Claude Opus 5 (1M context) <noreply@anthropic.com>
7 files changed,
+947 insertions,
-45 deletions
| @@ -34,6 +34,19 @@ | |||
| 34 | 34 | pom tls | |
| 35 | 35 | ``` | |
| 36 | 36 | ||
| 37 | + | ### MCP tools | |
| 38 | + | ||
| 39 | + | In MCP mode, five read-only tools answer "what is live right now" without an | |
| 40 | + | ssh: `status_table` (every target on one line, worst first), `target_status` | |
| 41 | + | (one target's full condition list), `versions`, `incidents`, and `trends`. None | |
| 42 | + | of them can promote, deploy, or close anything. | |
| 43 | + | ||
| 44 | + | Each takes an optional `instance`. Omitted, it reads this machine's database | |
| 45 | + | directly and works whether or not a daemon is running here. Named, it reads that | |
| 46 | + | configured peer's HTTP API over the tailnet, which is the only way to see checks | |
| 47 | + | local to that host: systemd units and backup freshness on the production box are | |
| 48 | + | not observable from anywhere else. | |
| 49 | + | ||
| 37 | 50 | ## Configuration | |
| 38 | 51 | ||
| 39 | 52 | PoM reads `~/.config/pom/pom.toml`. The config defines: | |
| @@ -57,7 +70,7 @@ | |||
| 57 | 70 | | `db.rs` | SQLite persistence (incidents, history, trends) | | |
| 58 | 71 | | `api.rs` | Axum HTTP API (status, trends, mesh data) | | |
| 59 | 72 | | `alerts.rs` | Email alerts via Postmark API | | |
| 60 | - | | `tools/` | MCP tool definitions for Claude integration | | |
| 73 | + | | `tools/` | MCP tool definitions for Claude integration, including the read-only orientation set | | |
| 61 | 74 | | `versions.rs` | Live version per target and how far it is behind the local checkout | | |
| 62 | 75 | | `display.rs` | Terminal output formatting | | |
| 63 | 76 | | `error.rs` | Error types | |
| @@ -6,7 +6,9 @@ | |||
| 6 | 6 | ||
| 7 | 7 | 1. **CLI mode** (`pom health`, `pom test`, `pom status`, etc.): runs a single command and exits. Useful for ad-hoc checks and cron jobs. | |
| 8 | 8 | 2. **Serve mode** (`pom serve`): long-running daemon that spawns per-target health check loops, TLS check loops, peer heartbeat tasks, a daily prune task, and an HTTP API server. This is the production deployment mode. | |
| 9 | - | 3. **MCP server mode** (bare `pom` with no subcommand): launches an MCP server over stdio for Claude integration. Exposes health checks, test execution, history queries, and mesh status as MCP tools. | |
| 9 | + | 3. **MCP server mode** (bare `pom` with no subcommand): launches an MCP server over stdio for Claude integration. Exposes health checks, test execution, history queries, and mesh status as MCP tools, plus a read-only set (`status_table`, `target_status`, `versions`, `incidents`, `trends`) for orienting on what is live. | |
| 10 | + | ||
| 11 | + | The read-only tools take an optional `instance`. Omitted, they read this machine's database directly and need no running daemon. Named, they resolve a configured peer and read that instance's HTTP API, which is the only way to see checks that are local to that host: systemd units and backup freshness on the production box are not observable from anywhere else. Both paths produce the same types, so the formatting is written once. | |
| 10 | 12 | ||
| 11 | 13 | All three modes load the same TOML config and connect to the same SQLite database. | |
| 12 | 14 | ||
| @@ -32,6 +34,7 @@ | |||
| 32 | 34 | | `tools` | `src/tools/mod.rs` | MCP server definition (PomServer), tool registration via rmcp | | |
| 33 | 35 | | `tools::health` | `src/tools/health.rs` | MCP tool implementations for health checks, history, targets, mesh status | | |
| 34 | 36 | | `tools::tests` | `src/tools/tests.rs` | MCP tool implementations for test execution, history, raw output | | |
| 37 | + | | `tools::orient` | `src/tools/orient.rs` | Read-only MCP tools (status_table, target_status, versions, incidents, trends), local or against a peer | | |
| 35 | 38 | ||
| 36 | 39 | ## Data Flow | |
| 37 | 40 | ||
| @@ -152,6 +155,7 @@ | |||
| 152 | 155 | | `/api/status` | GET | JSON summary of all targets (latest health, uptime, latency, TLS, staleness, incidents) | | |
| 153 | 156 | | `/api/status/{target}` | GET | Same as above for a single target | | |
| 154 | 157 | | `/api/trends/{target}` | GET | Latency trend data with configurable window and bucket size (`?hours=24&bucket_minutes=60`) | | |
| 158 | + | | `/api/versions` | GET | What each target is running: version, git sha, when it was seen, commits behind this host's checkout | | |
| 155 | 159 | | `/api/peer/info` | GET | This instance's identity (id, name, version, targets, started_at) | | |
| 156 | 160 | | `/api/peer/status` | GET | This instance's full view: identity + target statuses + peer summaries | | |
| 157 | 161 | | `/api/mesh` | GET | Aggregated mesh view: self + each peer's cached status | |
| @@ -201,6 +201,7 @@ | |||
| 201 | 201 | .route("/api/status", get(status_all)) | |
| 202 | 202 | .route("/api/status/{target}", get(status_target)) | |
| 203 | 203 | .route("/api/trends/{target}", get(trends)) | |
| 204 | + | .route("/api/versions", get(versions)) | |
| 204 | 205 | .route("/api/peer/info", get(peer_info)) | |
| 205 | 206 | .route("/api/peer/status", get(peer_status)) | |
| 206 | 207 | .route("/api/mesh", get(mesh_view)) | |
| @@ -517,22 +518,35 @@ | |||
| 517 | 518 | /// cross-service payload the release viewer renders. See `crate::status`. | |
| 518 | 519 | #[instrument(skip_all)] | |
| 519 | 520 | async fn status_json(AxumState(state): AxumState<ApiState>) -> impl IntoResponse { | |
| 520 | - | let targets = build_status_view(&state).await; | |
| 521 | - | Json(crate::status::payload(&targets, chrono::Utc::now())) | |
| 521 | + | Json(status_payload(&state.pool, &state.config).await) | |
| 522 | + | } | |
| 523 | + | ||
| 524 | + | /// The `/status.json` payload, built straight from the database. | |
| 525 | + | /// | |
| 526 | + | /// Public because the MCP tools serve the same payload for the local instance | |
| 527 | + | /// without going through HTTP: a session asking what is live should not need a | |
| 528 | + | /// running `pom serve` on its own machine to read its own database. Reading a | |
| 529 | + | /// *remote* instance is what the HTTP path is for. | |
| 530 | + | pub async fn status_payload(pool: &sqlx::SqlitePool, config: &Config) -> ops_status::Payload { | |
| 531 | + | let targets = build_status_view(pool, config).await; | |
| 532 | + | crate::status::payload(&targets, chrono::Utc::now()) | |
| 522 | 533 | } | |
| 523 | 534 | ||
| 524 | 535 | /// Read each target's current signals out of the database into the pure view the | |
| 525 | 536 | /// payload mapping consumes. The DB lives here; `crate::status::payload` stays a | |
| 526 | 537 | /// pure function of `(targets, now)`. | |
| 527 | - | async fn build_status_view(state: &ApiState) -> Vec<crate::status::TargetView> { | |
| 538 | + | async fn build_status_view( | |
| 539 | + | pool: &sqlx::SqlitePool, | |
| 540 | + | config: &Config, | |
| 541 | + | ) -> Vec<crate::status::TargetView> { | |
| 528 | 542 | let mut targets = Vec::new(); | |
| 529 | 543 | ||
| 530 | - | for name in state.config.target_names() { | |
| 531 | - | let Some(target_config) = state.config.get_target(&name) else { | |
| 544 | + | for name in config.target_names() { | |
| 545 | + | let Some(target_config) = config.get_target(&name) else { | |
| 532 | 546 | continue; | |
| 533 | 547 | }; | |
| 534 | 548 | ||
| 535 | - | let health = db::get_latest_health(&state.pool, &name) | |
| 549 | + | let health = db::get_latest_health(pool, &name) | |
| 536 | 550 | .await | |
| 537 | 551 | .ok() | |
| 538 | 552 | .flatten() | |
| @@ -543,13 +557,13 @@ | |||
| 543 | 557 | error: s.error, | |
| 544 | 558 | }); | |
| 545 | 559 | ||
| 546 | - | let uptime_24h = db::get_uptime_percent(&state.pool, &name, 24) | |
| 560 | + | let uptime_24h = db::get_uptime_percent(pool, &name, 24) | |
| 547 | 561 | .await | |
| 548 | 562 | .unwrap_or(None); | |
| 549 | 563 | ||
| 550 | 564 | let latency_avg_ms = { | |
| 551 | 565 | let cutoff = (chrono::Utc::now() - chrono::Duration::hours(24)).to_rfc3339(); | |
| 552 | - | let times = db::get_response_times(&state.pool, &name, &cutoff) | |
| 566 | + | let times = db::get_response_times(pool, &name, &cutoff) | |
| 553 | 567 | .await | |
| 554 | 568 | .unwrap_or_default(); | |
| 555 | 569 | let operational: Vec<i64> = times | |
| @@ -560,7 +574,7 @@ | |||
| 560 | 574 | LatencyStats::from_times(&operational).map(|s| s.avg_ms) | |
| 561 | 575 | }; | |
| 562 | 576 | ||
| 563 | - | let tls = db::get_latest_tls_check(&state.pool, &name) | |
| 577 | + | let tls = db::get_latest_tls_check(pool, &name) | |
| 564 | 578 | .await | |
| 565 | 579 | .ok() | |
| 566 | 580 | .flatten() | |
| @@ -571,7 +585,7 @@ | |||
| 571 | 585 | error: r.error, | |
| 572 | 586 | }); | |
| 573 | 587 | ||
| 574 | - | let incident = db::get_open_incident(&state.pool, &name) | |
| 588 | + | let incident = db::get_open_incident(pool, &name) | |
| 575 | 589 | .await | |
| 576 | 590 | .ok() | |
| 577 | 591 | .flatten() | |
| @@ -581,7 +595,7 @@ | |||
| 581 | 595 | started_at: i.started_at, | |
| 582 | 596 | }); | |
| 583 | 597 | ||
| 584 | - | let whois = db::get_latest_whois_check(&state.pool, &name) | |
| 598 | + | let whois = db::get_latest_whois_check(pool, &name) | |
| 585 | 599 | .await | |
| 586 | 600 | .ok() | |
| 587 | 601 | .flatten() | |
| @@ -594,9 +608,7 @@ | |||
| 594 | 608 | let mut backups = Vec::new(); | |
| 595 | 609 | if let Some(backup_config) = &target_config.backups { | |
| 596 | 610 | for database in &backup_config.databases { | |
| 597 | - | if let Ok(Some(row)) = | |
| 598 | - | db::get_latest_backup_check(&state.pool, &name, database).await | |
| 599 | - | { | |
| 611 | + | if let Ok(Some(row)) = db::get_latest_backup_check(pool, &name, database).await { | |
| 600 | 612 | backups.push(crate::status::BackupView { | |
| 601 | 613 | database: row.database_name, | |
| 602 | 614 | status: row.status, | |
| @@ -608,7 +620,7 @@ | |||
| 608 | 620 | } | |
| 609 | 621 | } | |
| 610 | 622 | ||
| 611 | - | let scan_pipeline = db::get_latest_scan_pipeline_check(&state.pool, &name) | |
| 623 | + | let scan_pipeline = db::get_latest_scan_pipeline_check(pool, &name) | |
| 612 | 624 | .await | |
| 613 | 625 | .ok() | |
| 614 | 626 | .flatten() | |
| @@ -619,7 +631,7 @@ | |||
| 619 | 631 | error: s.error, | |
| 620 | 632 | }); | |
| 621 | 633 | ||
| 622 | - | let systemd = db::get_latest_systemd_check(&state.pool, &name) | |
| 634 | + | let systemd = db::get_latest_systemd_check(pool, &name) | |
| 623 | 635 | .await | |
| 624 | 636 | .ok() | |
| 625 | 637 | .flatten() | |
| @@ -633,13 +645,10 @@ | |||
| 633 | 645 | // Tests: the latest run plus PoM's staleness verdict, sourced exactly as | |
| 634 | 646 | // build_target_status does (version at test time vs current version). | |
| 635 | 647 | let tests = if let Some(tests_config) = &target_config.tests { | |
| 636 | - | let latest_test = db::get_latest_test_run(&state.pool, &name) | |
| 637 | - | .await | |
| 638 | - | .ok() | |
| 639 | - | .flatten(); | |
| 648 | + | let latest_test = db::get_latest_test_run(pool, &name).await.ok().flatten(); | |
| 640 | 649 | let current_version = health.as_ref().and_then(|h| h.version.clone()); | |
| 641 | 650 | let tested_version = if let Some(test) = &latest_test { | |
| 642 | - | db::get_version_at_time(&state.pool, &name, &test.started_at) | |
| 651 | + | db::get_version_at_time(pool, &name, &test.started_at) | |
| 643 | 652 | .await | |
| 644 | 653 | .ok() | |
| 645 | 654 | .flatten() | |
| @@ -668,7 +677,7 @@ | |||
| 668 | 677 | // DNS: one entry per monitored record. Absent config yields no rows and | |
| 669 | 678 | // therefore no condition. | |
| 670 | 679 | let dns = { | |
| 671 | - | let rows = db::get_latest_dns_checks(&state.pool, &name) | |
| 680 | + | let rows = db::get_latest_dns_checks(pool, &name) | |
| 672 | 681 | .await | |
| 673 | 682 | .unwrap_or_default(); | |
| 674 | 683 | (!rows.is_empty()).then(|| crate::status::DnsView { | |
| @@ -687,7 +696,7 @@ | |||
| 687 | 696 | ||
| 688 | 697 | // CORS: one entry per monitored URL. | |
| 689 | 698 | let cors = { | |
| 690 | - | let rows = db::get_latest_cors_checks(&state.pool, &name) | |
| 699 | + | let rows = db::get_latest_cors_checks(pool, &name) | |
| 691 | 700 | .await | |
| 692 | 701 | .unwrap_or_default(); | |
| 693 | 702 | (!rows.is_empty()).then(|| crate::status::CorsView { | |
| @@ -939,20 +948,20 @@ | |||
| 939 | 948 | ||
| 940 | 949 | // Trends endpoint | |
| 941 | 950 | ||
| 942 | - | #[derive(Serialize)] | |
| 943 | - | struct TrendResponse { | |
| 951 | + | #[derive(Serialize, serde::Deserialize)] | |
| 952 | + | pub struct TrendResponse { | |
| 944 | 953 | /// Target config name this trend data belongs to. | |
| 945 | - | target: String, | |
| 954 | + | pub target: String, | |
| 946 | 955 | /// Requested time window in hours (from query param, default 24). | |
| 947 | - | window_hours: u64, | |
| 956 | + | pub window_hours: u64, | |
| 948 | 957 | /// Requested bucket width in minutes (from query param, default 60). | |
| 949 | - | bucket_minutes: u64, | |
| 958 | + | pub bucket_minutes: u64, | |
| 950 | 959 | /// Per-bucket latency statistics within the requested window. | |
| 951 | - | buckets: Vec<LatencyBucket>, | |
| 960 | + | pub buckets: Vec<LatencyBucket>, | |
| 952 | 961 | /// Aggregate latency statistics across the entire requested window. | |
| 953 | - | overall: Option<LatencyStats>, | |
| 962 | + | pub overall: Option<LatencyStats>, | |
| 954 | 963 | /// 7-day baseline latency statistics for drift comparison. | |
| 955 | - | baseline: Option<LatencyStats>, | |
| 964 | + | pub baseline: Option<LatencyStats>, | |
| 956 | 965 | } | |
| 957 | 966 | ||
| 958 | 967 | /// `GET /api/trends/{target}?hours=24&bucket_minutes=60`: latency trend data. | |
| @@ -971,11 +980,30 @@ | |||
| 971 | 980 | )); | |
| 972 | 981 | }; | |
| 973 | 982 | ||
| 974 | - | let hours = params.hours.unwrap_or(24); | |
| 975 | - | let bucket_minutes = params.bucket_minutes.unwrap_or(60); | |
| 983 | + | let response = build_trends( | |
| 984 | + | &state.pool, | |
| 985 | + | &target, | |
| 986 | + | params.hours.unwrap_or(24), | |
| 987 | + | params.bucket_minutes.unwrap_or(60), | |
| 988 | + | ) | |
| 989 | + | .await; | |
| 976 | 990 | ||
| 991 | + | Ok(Json(response)) | |
| 992 | + | } | |
| 993 | + | ||
| 994 | + | /// The latency trend for one target: per-bucket stats within the window, the | |
| 995 | + | /// window aggregate, and a 7-day baseline to read it against. | |
| 996 | + | /// | |
| 997 | + | /// Public for the same reason as [`status_payload`]: the MCP tools serve the | |
| 998 | + | /// local instance's answer without requiring a running daemon on this machine. | |
| 999 | + | pub async fn build_trends( | |
| 1000 | + | pool: &sqlx::SqlitePool, | |
| 1001 | + | target: &str, | |
| 1002 | + | hours: u64, | |
| 1003 | + | bucket_minutes: u64, | |
| 1004 | + | ) -> TrendResponse { | |
| 977 | 1005 | let cutoff = (chrono::Utc::now() - chrono::Duration::hours(hours as i64)).to_rfc3339(); | |
| 978 | - | let times = db::get_response_times(&state.pool, &target, &cutoff) | |
| 1006 | + | let times = db::get_response_times(pool, target, &cutoff) | |
| 979 | 1007 | .await | |
| 980 | 1008 | .unwrap_or_default(); | |
| 981 | 1009 | ||
| @@ -992,7 +1020,7 @@ | |||
| 992 | 1020 | ||
| 993 | 1021 | // 7d baseline for reference | |
| 994 | 1022 | let baseline_cutoff = (chrono::Utc::now() - chrono::Duration::hours(168)).to_rfc3339(); | |
| 995 | - | let baseline_times = db::get_response_times(&state.pool, &target, &baseline_cutoff) | |
| 1023 | + | let baseline_times = db::get_response_times(pool, target, &baseline_cutoff) | |
| 996 | 1024 | .await | |
| 997 | 1025 | .unwrap_or_default(); | |
| 998 | 1026 | let baseline_operational: Vec<i64> = baseline_times | |
| @@ -1002,14 +1030,31 @@ | |||
| 1002 | 1030 | .collect(); | |
| 1003 | 1031 | let baseline = LatencyStats::from_times(&baseline_operational); | |
| 1004 | 1032 | ||
| 1005 | - | Ok(Json(TrendResponse { | |
| 1006 | - | target, | |
| 1033 | + | TrendResponse { | |
| 1034 | + | target: target.to_string(), | |
| 1007 | 1035 | window_hours: hours, | |
| 1008 | 1036 | bucket_minutes, | |
| 1009 | 1037 | buckets, | |
| 1010 | 1038 | overall, | |
| 1011 | 1039 | baseline, | |
| 1012 | - | })) | |
| 1040 | + | } | |
| 1041 | + | } | |
| 1042 | + | ||
| 1043 | + | /// `GET /api/versions`: what every target is running, and how far behind. | |
| 1044 | + | /// | |
| 1045 | + | /// The commits-behind column is measured against a checkout on *this* host, so | |
| 1046 | + | /// an instance without the repo serves the rest of the row and leaves that one | |
| 1047 | + | /// blank. Reading it from a peer therefore answers "what is live there", not | |
| 1048 | + | /// "how far behind is it here". | |
| 1049 | + | #[instrument(skip_all)] | |
| 1050 | + | async fn versions(AxumState(state): AxumState<ApiState>) -> impl IntoResponse { | |
| 1051 | + | match crate::versions::collect(&state.pool, &state.config).await { | |
| 1052 | + | Ok(rows) => Ok(Json(rows)), | |
| 1053 | + | Err(e) => Err(( | |
| 1054 | + | StatusCode::INTERNAL_SERVER_ERROR, | |
| 1055 | + | Json(serde_json::json!({ "error": e.to_string() })), | |
| 1056 | + | )), | |
| 1057 | + | } | |
| 1013 | 1058 | } | |
| 1014 | 1059 | ||
| 1015 | 1060 | #[derive(serde::Deserialize)] |
| @@ -47,7 +47,7 @@ | |||
| 47 | 47 | /// `https://host:port` when the hop crosses an untrusted network), and otherwise | |
| 48 | 48 | /// defaults to `http://`: the tailnet deployment where WireGuard already | |
| 49 | 49 | /// encrypts the transport, and pom's own API server binds plain HTTP. | |
| 50 | - | fn peer_base_url(address: &str) -> String { | |
| 50 | + | pub(crate) fn peer_base_url(address: &str) -> String { | |
| 51 | 51 | if address.contains("://") { | |
| 52 | 52 | address.trim_end_matches('/').to_string() | |
| 53 | 53 | } else { |
| @@ -4025,3 +4025,249 @@ | |||
| 4025 | 4025 | let err = result.unwrap_err().to_string(); | |
| 4026 | 4026 | assert!(err.contains("must be absolute"), "error: {err}"); | |
| 4027 | 4027 | } | |
| 4028 | + | ||
| 4029 | + | // Read-only orientation tools | |
| 4030 | + | ||
| 4031 | + | /// A server whose config also knows one peer, so instance resolution has | |
| 4032 | + | /// something to resolve and something to reject. | |
| 4033 | + | fn orient_server(pool: sqlx::SqlitePool) -> PomServer { | |
| 4034 | + | let config: pom::config::Config = toml::from_str( | |
| 4035 | + | r#" | |
| 4036 | + | [targets.mnw] | |
| 4037 | + | label = "MakeNotWork" | |
| 4038 | + | [targets.mnw.health] | |
| 4039 | + | url = "https://makenot.work/api/health" | |
| 4040 | + | ||
| 4041 | + | [peers.hetzner] | |
| 4042 | + | address = "100.64.0.1:9100" | |
| 4043 | + | token = "peer-token" | |
| 4044 | + | "#, | |
| 4045 | + | ) | |
| 4046 | + | .unwrap(); | |
| 4047 | + | PomServer::new(pool, config) | |
| 4048 | + | } | |
| 4049 | + | ||
| 4050 | + | fn instance_params(instance: Option<&str>) -> pom::tools::orient::InstanceParams { | |
| 4051 | + | serde_json::from_value(serde_json::json!({ "instance": instance })).unwrap() | |
| 4052 | + | } | |
| 4053 | + | ||
| 4054 | + | #[tokio::test] | |
| 4055 | + | async fn tool_status_table_reports_one_line_per_target() { | |
| 4056 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4057 | + | let server = orient_server(pool.clone()); | |
| 4058 | + | insert_version_health( | |
| 4059 | + | &pool, | |
| 4060 | + | "mnw", | |
| 4061 | + | Some("0.11.0"), | |
| 4062 | + | Some("6402bf4e"), | |
| 4063 | + | "2026-07-29T18:00:00Z", | |
| 4064 | + | ) | |
| 4065 | + | .await; | |
| 4066 | + | ||
| 4067 | + | let out = server | |
| 4068 | + | .status_table_impl(instance_params(None)) | |
| 4069 | + | .await | |
| 4070 | + | .unwrap(); | |
| 4071 | + | assert!(out.contains("TARGET"), "{out}"); | |
| 4072 | + | let row = out | |
| 4073 | + | .lines() | |
| 4074 | + | .find(|l| l.starts_with("mnw")) | |
| 4075 | + | .unwrap_or_else(|| panic!("no mnw row in:\n{out}")); | |
| 4076 | + | // "ok" is the shared status vocabulary of the payload, not PoM's own | |
| 4077 | + | // "operational": the table speaks the contract every source speaks. | |
| 4078 | + | assert!(row.contains("ok"), "row: {row}"); | |
| 4079 | + | assert!(row.contains("0.11.0"), "row: {row}"); | |
| 4080 | + | } | |
| 4081 | + | ||
| 4082 | + | #[tokio::test] | |
| 4083 | + | async fn tool_status_table_puts_the_worst_target_first() { | |
| 4084 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4085 | + | let config: pom::config::Config = toml::from_str( | |
| 4086 | + | r#" | |
| 4087 | + | [targets.aaa] | |
| 4088 | + | label = "Fine" | |
| 4089 | + | [targets.aaa.health] | |
| 4090 | + | url = "https://example.invalid/health" | |
| 4091 | + | [targets.zzz] | |
| 4092 | + | label = "Broken" | |
| 4093 | + | [targets.zzz.health] | |
| 4094 | + | url = "https://example.invalid/health" | |
| 4095 | + | "#, | |
| 4096 | + | ) | |
| 4097 | + | .unwrap(); | |
| 4098 | + | let server = PomServer::new(pool.clone(), config); | |
| 4099 | + | ||
| 4100 | + | insert_version_health(&pool, "aaa", Some("1.0.0"), None, "2026-07-29T18:00:00Z").await; | |
| 4101 | + | let broken = HealthSnapshot { | |
| 4102 | + | id: None, | |
| 4103 | + | target: "zzz".to_string(), | |
| 4104 | + | status: HealthStatus::Unreachable, | |
| 4105 | + | checked_at: "2026-07-29T18:00:00Z".to_string(), | |
| 4106 | + | response_time_ms: 0, | |
| 4107 | + | details: None, | |
| 4108 | + | error: Some("connection refused".to_string()), | |
| 4109 | + | }; | |
| 4110 | + | db::insert_health_check(&pool, &broken).await.unwrap(); | |
| 4111 | + | ||
| 4112 | + | let out = server | |
| 4113 | + | .status_table_impl(instance_params(None)) | |
| 4114 | + | .await | |
| 4115 | + | .unwrap(); | |
| 4116 | + | let first_target = out | |
| 4117 | + | .lines() | |
| 4118 | + | .find(|l| l.starts_with("aaa") || l.starts_with("zzz")) | |
| 4119 | + | .unwrap(); | |
| 4120 | + | assert!( | |
| 4121 | + | first_target.starts_with("zzz"), | |
| 4122 | + | "the broken target must head the table:\n{out}" | |
| 4123 | + | ); | |
| 4124 | + | assert!(first_target.contains("connection refused"), "{out}"); | |
| 4125 | + | } | |
| 4126 | + | ||
| 4127 | + | #[tokio::test] | |
| 4128 | + | async fn tool_target_status_lists_every_condition() { | |
| 4129 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4130 | + | let server = orient_server(pool.clone()); | |
| 4131 | + | insert_version_health(&pool, "mnw", Some("0.11.0"), None, "2026-07-29T18:00:00Z").await; | |
| 4132 | + | db::insert_incident(&pool, "mnw", "operational", "degraded") | |
| 4133 | + | .await | |
| 4134 | + | .unwrap(); | |
| 4135 | + | ||
| 4136 | + | let params: pom::tools::orient::TargetInstanceParams = | |
| 4137 | + | serde_json::from_value(serde_json::json!({ "target": "mnw" })).unwrap(); | |
| 4138 | + | let out = server.target_status_impl(params).await.unwrap(); | |
| 4139 | + | assert!(out.contains("health"), "{out}"); | |
| 4140 | + | assert!(out.contains("incident"), "{out}"); | |
| 4141 | + | assert!(out.contains("version: 0.11.0"), "{out}"); | |
| 4142 | + | } | |
| 4143 | + | ||
| 4144 | + | #[tokio::test] | |
| 4145 | + | async fn tool_target_status_names_the_known_targets_when_asked_for_a_stranger() { | |
| 4146 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4147 | + | let server = orient_server(pool); | |
| 4148 | + | ||
| 4149 | + | let params: pom::tools::orient::TargetInstanceParams = | |
| 4150 | + | serde_json::from_value(serde_json::json!({ "target": "nope" })).unwrap(); | |
| 4151 | + | let out = server.target_status_impl(params).await.unwrap(); | |
| 4152 | + | assert!(out.contains("Unknown target"), "{out}"); | |
| 4153 | + | assert!(out.contains("mnw"), "must list what it does know: {out}"); | |
| 4154 | + | } | |
| 4155 | + | ||
| 4156 | + | #[tokio::test] | |
| 4157 | + | async fn tool_incidents_reports_open_ones_and_other_failing_checks() { | |
| 4158 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4159 | + | let server = orient_server(pool.clone()); | |
| 4160 | + | ||
| 4161 | + | let out = server.incidents_impl(instance_params(None)).await.unwrap(); | |
| 4162 | + | assert!(out.contains("No open incidents"), "{out}"); | |
| 4163 | + | ||
| 4164 | + | db::insert_incident(&pool, "mnw", "operational", "error") | |
| 4165 | + | .await | |
| 4166 | + | .unwrap(); | |
| 4167 | + | let out = server.incidents_impl(instance_params(None)).await.unwrap(); | |
| 4168 | + | assert!(out.contains("mnw / incident"), "{out}"); | |
| 4169 | + | // The health condition is pending (no check recorded), which is not a | |
| 4170 | + | // failure and must not be reported as one. | |
| 4171 | + | assert!(!out.contains("mnw / health"), "{out}"); | |
| 4172 | + | } | |
| 4173 | + | ||
| 4174 | + | #[tokio::test] | |
| 4175 | + | async fn tool_versions_returns_the_roll_up() { | |
| 4176 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4177 | + | let server = orient_server(pool.clone()); | |
| 4178 | + | insert_version_health( | |
| 4179 | + | &pool, | |
| 4180 | + | "mnw", | |
| 4181 | + | Some("0.11.0"), | |
| 4182 | + | Some("6402bf4e"), | |
| 4183 | + | "2026-07-29T18:00:00Z", | |
| 4184 | + | ) | |
| 4185 | + | .await; | |
| 4186 | + | ||
| 4187 | + | let out = server.versions_impl(instance_params(None)).await.unwrap(); | |
| 4188 | + | assert!(out.contains("TARGET"), "{out}"); | |
| 4189 | + | assert!(out.contains("0.11.0") && out.contains("6402bf4e"), "{out}"); | |
| 4190 | + | } | |
| 4191 | + | ||
| 4192 | + | #[tokio::test] | |
| 4193 | + | async fn tool_trends_reports_the_window_and_baseline() { | |
| 4194 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4195 | + | let server = orient_server(pool.clone()); | |
| 4196 | + | for i in 0..3 { | |
| 4197 | + | let snapshot = HealthSnapshot { | |
| 4198 | + | id: None, | |
| 4199 | + | target: "mnw".to_string(), | |
| 4200 | + | status: HealthStatus::Operational, | |
| 4201 | + | checked_at: chrono::Utc::now().to_rfc3339(), | |
| 4202 | + | response_time_ms: 100 + i, | |
| 4203 | + | details: None, | |
| 4204 | + | error: None, | |
| 4205 | + | }; | |
| 4206 | + | db::insert_health_check(&pool, &snapshot).await.unwrap(); | |
| 4207 | + | } | |
| 4208 | + | ||
| 4209 | + | let params: pom::tools::orient::TrendsParams = | |
| 4210 | + | serde_json::from_value(serde_json::json!({ "target": "mnw" })).unwrap(); | |
| 4211 | + | let out = server.trends_impl(params).await.unwrap(); | |
| 4212 | + | assert!(out.contains("last 24h"), "{out}"); | |
| 4213 | + | assert!(out.contains("Window: avg"), "{out}"); | |
| 4214 | + | ||
| 4215 | + | let unknown: pom::tools::orient::TrendsParams = | |
| 4216 | + | serde_json::from_value(serde_json::json!({ "target": "nope" })).unwrap(); | |
| 4217 | + | let out = server.trends_impl(unknown).await.unwrap(); | |
| 4218 | + | assert!(out.contains("Unknown target"), "{out}"); | |
| 4219 | + | } | |
| 4220 | + | ||
| 4221 | + | #[tokio::test] | |
| 4222 | + | async fn tool_unknown_instance_names_the_configured_peers() { | |
| 4223 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4224 | + | let server = orient_server(pool); | |
| 4225 | + | ||
| 4226 | + | let err = server | |
| 4227 | + | .status_table_impl(instance_params(Some("mars"))) | |
| 4228 | + | .await | |
| 4229 | + | .unwrap_err() | |
| 4230 | + | .to_string(); | |
| 4231 | + | assert!(err.contains("unknown instance"), "{err}"); | |
| 4232 | + | assert!( | |
| 4233 | + | err.contains("hetzner"), | |
| 4234 | + | "must list the peers it knows: {err}" | |
| 4235 | + | ); | |
| 4236 | + | } | |
| 4237 | + | ||
| 4238 | + | #[tokio::test] | |
| 4239 | + | async fn tool_local_instance_needs_no_running_daemon() { | |
| 4240 | + | // The whole point of reading the pool directly: `pom serve` is not up in | |
| 4241 | + | // this test, and the local answer still comes back. | |
| 4242 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4243 | + | let server = orient_server(pool.clone()); | |
| 4244 | + | insert_version_health(&pool, "mnw", Some("0.11.0"), None, "2026-07-29T18:00:00Z").await; | |
| 4245 | + | ||
| 4246 | + | for instance in [None, Some("local")] { | |
| 4247 | + | let out = server | |
| 4248 | + | .status_table_impl(instance_params(instance)) | |
| 4249 | + | .await | |
| 4250 | + | .unwrap(); | |
| 4251 | + | assert!(out.contains("mnw"), "{out}"); | |
| 4252 | + | } | |
| 4253 | + | } | |
| 4254 | + | ||
| 4255 | + | #[tokio::test] | |
| 4256 | + | async fn api_versions_serves_the_roll_up() { | |
| 4257 | + | let pool = db::connect_in_memory().await.unwrap(); | |
| 4258 | + | let app = pom::api::router(pool.clone(), test_config(), None); | |
| 4259 | + | insert_version_health( | |
| 4260 | + | &pool, | |
| 4261 | + | "mnw", | |
| 4262 | + | Some("0.11.0"), | |
| 4263 | + | Some("6402bf4e"), | |
| 4264 | + | "2026-07-29T18:00:00Z", | |
| 4265 | + | ) | |
| 4266 | + | .await; | |
| 4267 | + | ||
| 4268 | + | let (status, json) = api_get(&app, "/api/versions").await; | |
| 4269 | + | assert_eq!(status, 200); | |
| 4270 | + | assert_eq!(json[0]["target"], "mnw"); | |
| 4271 | + | assert_eq!(json[0]["version"], "0.11.0"); | |
| 4272 | + | assert_eq!(json[0]["git_sha"], "6402bf4e"); | |
| 4273 | + | } |
| @@ -1,6 +1,7 @@ | |||
| 1 | 1 | //! MCP server surface, exposing PoM's health and test data as MCP tools. | |
| 2 | 2 | ||
| 3 | 3 | pub mod health; | |
| 4 | + | pub mod orient; | |
| 4 | 5 | pub mod tests; | |
| 5 | 6 | ||
| 6 | 7 | use rmcp::handler::server::router::tool::ToolRouter; | |
| @@ -118,6 +119,69 @@ | |||
| 118 | 119 | } | |
| 119 | 120 | } | |
| 120 | 121 | ||
| 122 | + | // The orientation tools below are read-only by design: they answer "what is | |
| 123 | + | // live and what is wrong with it" without being able to change any of it. | |
| 124 | + | // Promoting, deploying, and closing incidents stay with Sando and the CLI. | |
| 125 | + | ||
| 126 | + | #[tool( | |
| 127 | + | description = "One compact table of every monitored target: status, live version, when it was last checked, and the worst thing currently wrong with it. Start here when orienting on production. Omit instance to read this machine, or name a configured peer to read that host's view." | |
| 128 | + | )] | |
| 129 | + | pub async fn status_table( | |
| 130 | + | &self, | |
| 131 | + | Parameters(params): Parameters<orient::InstanceParams>, | |
| 132 | + | ) -> String { | |
| 133 | + | match self.status_table_impl(params).await { | |
| 134 | + | Ok(result) => result, | |
| 135 | + | Err(e) => format!("Error getting status table: {e}"), | |
| 136 | + | } | |
| 137 | + | } | |
| 138 | + | ||
| 139 | + | #[tool( | |
| 140 | + | description = "Everything known about one target: health, TLS, DNS, WHOIS, backups, systemd units, scan pipeline, tests, and any open incident, each with its status and why. Reads stored check results; does not probe the target." | |
| 141 | + | )] | |
| 142 | + | pub async fn target_status( | |
| 143 | + | &self, | |
| 144 | + | Parameters(params): Parameters<orient::TargetInstanceParams>, | |
| 145 | + | ) -> String { | |
| 146 | + | match self.target_status_impl(params).await { | |
| 147 | + | Ok(result) => result, | |
| 148 | + | Err(e) => format!("Error getting target status: {e}"), | |
| 149 | + | } | |
| 150 | + | } | |
| 151 | + | ||
| 152 | + | #[tool( | |
| 153 | + | description = "What version and commit each target is running, when it was last seen, and how many commits behind the local checkout it is. Answers 'is what I am looking at what is deployed'." | |
| 154 | + | )] | |
| 155 | + | pub async fn versions(&self, Parameters(params): Parameters<orient::InstanceParams>) -> String { | |
| 156 | + | match self.versions_impl(params).await { | |
| 157 | + | Ok(result) => result, | |
| 158 | + | Err(e) => format!("Error getting versions: {e}"), | |
| 159 | + | } | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | #[tool( | |
| 163 | + | description = "Open incidents across every target, plus any check that is currently not passing. Read-only: this reports incidents, it does not open or close them." | |
| 164 | + | )] | |
| 165 | + | pub async fn incidents( | |
| 166 | + | &self, | |
| 167 | + | Parameters(params): Parameters<orient::InstanceParams>, | |
| 168 | + | ) -> String { | |
| 169 | + | match self.incidents_impl(params).await { | |
| 170 | + | Ok(result) => result, | |
| 171 | + | Err(e) => format!("Error getting incidents: {e}"), | |
| 172 | + | } | |
| 173 | + | } | |
| 174 | + | ||
| 175 | + | #[tool( | |
| 176 | + | description = "Latency trend for one target: per-bucket stats over a window (default 24h, 60-minute buckets) against the 7-day baseline. Use when something feels slower than it was." | |
| 177 | + | )] | |
| 178 | + | pub async fn trends(&self, Parameters(params): Parameters<orient::TrendsParams>) -> String { | |
| 179 | + | match self.trends_impl(params).await { | |
| 180 | + | Ok(result) => result, | |
| 181 | + | Err(e) => format!("Error getting trends: {e}"), | |
| 182 | + | } | |
| 183 | + | } | |
| 184 | + | ||
| 121 | 185 | #[tool( | |
| 122 | 186 | description = "Get the peer mesh status showing all PoM instances, their connectivity, versions, and target health. Requires serve mode to be running." | |
| 123 | 187 | )] | |
| @@ -134,9 +198,14 @@ | |||
| 134 | 198 | fn get_info(&self) -> ServerInfo { | |
| 135 | 199 | ServerInfo::new(ServerCapabilities::builder().enable_tools().build()).with_instructions( | |
| 136 | 200 | "Peace of Mind (PoM) server for monitoring production health and running tests. \ | |
| 137 | - | Tools: get_status (dashboard), check_health (live health check), health_history, \ | |
| 138 | - | list_targets, run_tests (SSH test execution), test_history, last_test_output, \ | |
| 139 | - | get_mesh_status (peer mesh overview).", | |
| 201 | + | To orient on what is live, start with status_table, then target_status for one \ | |
| 202 | + | target, versions for what is deployed where, incidents for what is wrong, and \ | |
| 203 | + | trends for latency over time. Those five are read-only and take an optional \ | |
| 204 | + | `instance`: omitted they read this machine, named they read that configured peer, \ | |
| 205 | + | which is the only way to see checks local to that host (systemd, backups). \ | |
| 206 | + | Also: get_status (verbose per-target dump), check_health (probes the target live \ | |
| 207 | + | and records the result), health_history, list_targets, run_tests (SSH test \ | |
| 208 | + | execution), test_history, last_test_output, get_mesh_status (peer mesh overview).", | |
| 140 | 209 | ) | |
| 141 | 210 | } | |
| 142 | 211 | } | |
| @@ -151,7 +220,7 @@ | |||
| 151 | 220 | let tools = router.list_all(); | |
| 152 | 221 | let names: Vec<&str> = tools.iter().map(|t| t.name.as_ref()).collect(); | |
| 153 | 222 | eprintln!("registered tools: {names:?}"); | |
| 154 | - | assert_eq!(tools.len(), 8, "expected 8 tools, got {}", tools.len()); | |
| 223 | + | assert_eq!(tools.len(), 13, "expected 13 tools, got {}", tools.len()); | |
| 155 | 224 | for expected in [ | |
| 156 | 225 | "get_status", | |
| 157 | 226 | "check_health", | |
| @@ -161,6 +230,11 @@ | |||
| 161 | 230 | "test_history", | |
| 162 | 231 | "last_test_output", | |
| 163 | 232 | "get_mesh_status", | |
| 233 | + | "status_table", | |
| 234 | + | "target_status", | |
| 235 | + | "versions", | |
| 236 | + | "incidents", | |
| 237 | + | "trends", | |
| 164 | 238 | ] { | |
| 165 | 239 | let t = tools | |
| 166 | 240 | .iter() | |
| @@ -187,5 +261,25 @@ | |||
| 187 | 261 | "check_health schema missing 'target' property: {:?}", | |
| 188 | 262 | ch.input_schema | |
| 189 | 263 | ); | |
| 264 | + | ||
| 265 | + | // The read-only tools are only reachable against a remote instance if | |
| 266 | + | // the caller can see that the parameter exists. | |
| 267 | + | for expected in [ | |
| 268 | + | "status_table", | |
| 269 | + | "target_status", | |
| 270 | + | "versions", | |
| 271 | + | "incidents", | |
| 272 | + | "trends", | |
| 273 | + | ] { | |
| 274 | + | let t = tools.iter().find(|t| t.name == expected).unwrap(); | |
| 275 | + | assert!( | |
| 276 | + | t.input_schema | |
| 277 | + | .get("properties") | |
| 278 | + | .and_then(|p| p.get("instance")) | |
| 279 | + | .is_some(), | |
| 280 | + | "{expected} schema missing 'instance' property: {:?}", | |
| 281 | + | t.input_schema | |
| 282 | + | ); | |
| 283 | + | } | |
| 190 | 284 | } | |
| 191 | 285 | } |
| @@ -1,0 +1,558 @@ | |||
| 1 | + | //! Read-only MCP tools for orienting on what is live. | |
| 2 | + | //! | |
| 3 | + | //! These answer the question that otherwise costs an ssh and a curl: what is | |
| 4 | + | //! running where, what is wrong with it, and how far behind is it. Nothing here | |
| 5 | + | //! promotes, deploys, or closes an incident. Acting stays with Sando and the | |
| 6 | + | //! CLI, so a session can read production without being able to change it. | |
| 7 | + | //! | |
| 8 | + | //! # Local and remote read the same shape | |
| 9 | + | //! | |
| 10 | + | //! Every tool takes an optional `instance`. Omitted, it reads this machine's | |
| 11 | + | //! database directly, which works whether or not a `pom serve` daemon is up | |
| 12 | + | //! here. Named, it resolves a configured peer and reads that instance's HTTP | |
| 13 | + | //! API over the tailnet, which is the only way to see checks that are local to | |
| 14 | + | //! *that* host: systemd units and backup freshness on the production box are | |
| 15 | + | //! not observable from here at all. | |
| 16 | + | //! | |
| 17 | + | //! Both paths land on the same types, `ops_status::Payload` for the status | |
| 18 | + | //! tools and [`VersionRow`] for versions, so the formatting below is written | |
| 19 | + | //! once and never branches on where the data came from. | |
| 20 | + | ||
| 21 | + | use std::fmt::Write as _; | |
| 22 | + | ||
| 23 | + | use ops_status::{Node, Payload, Status}; | |
| 24 | + | use schemars::JsonSchema; | |
| 25 | + | use serde::Deserialize; | |
| 26 | + | use tracing::instrument; | |
| 27 | + | ||
| 28 | + | use crate::api; | |
| 29 | + | use crate::error::{PomError, Result}; | |
| 30 | + | use crate::types::VersionRow; | |
| 31 | + | use crate::versions; | |
| 32 | + | ||
| 33 | + | use super::PomServer; | |
| 34 | + | ||
| 35 | + | /// How long to wait on a peer before giving up. A session is waiting on this | |
| 36 | + | /// answer, so a hung instance should say so quickly rather than stall the turn. | |
| 37 | + | const REMOTE_TIMEOUT_SECS: u64 = 10; | |
| 38 | + | ||
| 39 | + | #[derive(Debug, Deserialize, JsonSchema)] | |
| 40 | + | pub struct InstanceParams { | |
| 41 | + | /// PoM instance to read: omit for this machine, or name a configured peer | |
| 42 | + | /// (e.g. the production instance) to read that host's view. | |
| 43 | + | pub instance: Option<String>, | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | #[derive(Debug, Deserialize, JsonSchema)] | |
| 47 | + | pub struct TargetInstanceParams { | |
| 48 | + | /// Target name, as configured (e.g. "mnw"). | |
| 49 | + | pub target: String, | |
| 50 | + | /// PoM instance to read: omit for this machine, or name a configured peer. | |
| 51 | + | pub instance: Option<String>, | |
| 52 | + | } | |
| 53 | + | ||
| 54 | + | #[derive(Debug, Deserialize, JsonSchema)] | |
| 55 | + | pub struct TrendsParams { | |
| 56 | + | /// Target name, as configured (e.g. "mnw"). | |
| 57 | + | pub target: String, | |
| 58 | + | /// Window to report, in hours (default 24). | |
| 59 | + | pub hours: Option<u64>, | |
| 60 | + | /// Width of each latency bucket, in minutes (default 60). | |
| 61 | + | pub bucket_minutes: Option<u64>, | |
| 62 | + | /// PoM instance to read: omit for this machine, or name a configured peer. | |
| 63 | + | pub instance: Option<String>, | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | /// Which instance a tool call is reading. | |
| 67 | + | enum Source { | |
| 68 | + | /// This machine's database, read without going through HTTP. | |
| 69 | + | Local, | |
| 70 | + | /// A configured peer's API. | |
| 71 | + | Remote { | |
| 72 | + | name: String, | |
| 73 | + | base_url: String, | |
| 74 | + | token: Option<String>, | |
| 75 | + | }, | |
| 76 | + | } | |
| 77 | + | ||
| 78 | + | impl PomServer { | |
| 79 | + | /// Resolve an `instance` parameter to the source to read. | |
| 80 | + | /// | |
| 81 | + | /// An unknown name lists what is configured rather than just refusing: the | |
| 82 | + | /// caller cannot see pom.toml, so the names are otherwise unguessable. | |
| 83 | + | fn source(&self, instance: Option<&str>) -> Result<Source> { | |
| 84 | + | match instance { | |
| 85 | + | None | Some("local" | "") => Ok(Source::Local), | |
| 86 | + | Some(name) => match self.config.peers.get(name) { | |
| 87 | + | Some(peer) => Ok(Source::Remote { | |
| 88 | + | name: name.to_string(), | |
| 89 | + | base_url: crate::peer::peer_base_url(&peer.address), | |
| 90 | + | token: peer.token.clone(), | |
| 91 | + | }), | |
| 92 | + | None => { | |
| 93 | + | let mut known: Vec<&str> = | |
| 94 | + | self.config.peers.keys().map(String::as_str).collect(); | |
| 95 | + | known.sort_unstable(); | |
| 96 | + | Err(PomError::Config(format!( | |
| 97 | + | "unknown instance: {name}. Configured peers: {}. Omit `instance` to read \ | |
| 98 | + | this machine.", | |
| 99 | + | if known.is_empty() { | |
| 100 | + | "none".to_string() | |
| 101 | + | } else { | |
| 102 | + | known.join(", ") | |
| 103 | + | } | |
| 104 | + | ))) | |
| 105 | + | } | |
| 106 | + | }, | |
| 107 | + | } | |
| 108 | + | } | |
| 109 | + | ||
| 110 | + | /// GET a path on a remote instance, with the peer's bearer token. | |
| 111 | + | async fn get_remote( | |
| 112 | + | &self, | |
| 113 | + | base_url: &str, | |
| 114 | + | token: Option<&str>, | |
| 115 | + | path: &str, | |
| 116 | + | ) -> Result<serde_json::Value> { | |
| 117 | + | let url = format!("{base_url}{path}"); | |
| 118 | + | let client = crate::tls::https_client_builder() | |
| 119 | + | .timeout(std::time::Duration::from_secs(REMOTE_TIMEOUT_SECS)) | |
| 120 | + | .build()?; | |
| 121 | + | ||
| 122 | + | let mut request = client.get(&url); | |
| 123 | + | if let Some(token) = token { | |
| 124 | + | request = request.bearer_auth(token); | |
| 125 | + | } | |
| 126 | + | ||
| 127 | + | let response = request | |
| 128 | + | .send() | |
| 129 | + | .await | |
| 130 | + | .map_err(|e| PomError::Config(format!("could not reach {url}: {e}")))?; | |
| 131 | + | ||
| 132 | + | let status = response.status(); | |
| 133 | + | if !status.is_success() { | |
| 134 | + | // 401 here means the peer's token is wrong or missing in pom.toml, | |
| 135 | + | // which is a config problem on this side and worth naming as one. | |
| 136 | + | return Err(PomError::Config(format!( | |
| 137 | + | "{url} returned {status}{}", | |
| 138 | + | if status.as_u16() == 401 { | |
| 139 | + | " (check this peer's token in pom.toml)" | |
| 140 | + | } else { | |
| 141 | + | "" | |
| 142 | + | } | |
| 143 | + | ))); | |
| 144 | + | } | |
| 145 | + | ||
| 146 | + | Ok(response.json().await?) | |
| 147 | + | } | |
| 148 | + | ||
| 149 | + | /// The status payload for whichever instance was asked for. | |
| 150 | + | async fn payload_for(&self, instance: Option<&str>) -> Result<(String, Payload)> { | |
| 151 | + | match self.source(instance)? { | |
| 152 | + | Source::Local => Ok(( | |
| 153 | + | "local".to_string(), | |
| 154 | + | api::status_payload(&self.pool, &self.config).await, | |
| 155 | + | )), | |
| 156 | + | Source::Remote { | |
| 157 | + | name, | |
| 158 | + | base_url, | |
| 159 | + | token, | |
| 160 | + | } => { | |
| 161 | + | let value = self | |
| 162 | + | .get_remote(&base_url, token.as_deref(), "/status.json") | |
| 163 | + | .await?; | |
| 164 | + | let payload = serde_json::from_value(value)?; | |
| 165 | + | Ok((name, payload)) | |
| 166 | + | } | |
| 167 | + | } | |
| 168 | + | } | |
| 169 | + | ||
| 170 | + | #[instrument(skip_all)] | |
| 171 | + | pub async fn status_table_impl(&self, params: InstanceParams) -> Result<String> { | |
| 172 | + | let (instance, payload) = self.payload_for(params.instance.as_deref()).await?; | |
| 173 | + | Ok(format_status_table(&instance, &payload)) | |
| 174 | + | } | |
| 175 | + | ||
| 176 | + | #[instrument(skip_all)] | |
| 177 | + | pub async fn target_status_impl(&self, params: TargetInstanceParams) -> Result<String> { | |
| 178 | + | let (instance, payload) = self.payload_for(params.instance.as_deref()).await?; | |
| 179 | + | let wanted = format!("target:{}", params.target); | |
| 180 | + | ||
| 181 | + | let Some(node) = payload.nodes.iter().find(|n| n.id == wanted) else { | |
| 182 | + | let names: Vec<&str> = payload | |
| 183 | + | .nodes | |
| 184 | + | .iter() | |
| 185 | + | .filter_map(|n| n.id.strip_prefix("target:")) | |
| 186 | + | .collect(); | |
| 187 | + | return Ok(format!( | |
| 188 | + | "Unknown target: {} on instance {instance}. Known targets: {}", | |
| 189 | + | params.target, | |
| 190 | + | if names.is_empty() { | |
| 191 | + | "none".to_string() | |
| 192 | + | } else { | |
| 193 | + | names.join(", ") | |
| 194 | + | } | |
| 195 | + | )); | |
| 196 | + | }; | |
| 197 | + | ||
| 198 | + | Ok(format_target_detail(&instance, node)) | |
| 199 | + | } | |
| 200 | + | ||
| 201 | + | #[instrument(skip_all)] | |
| 202 | + | pub async fn incidents_impl(&self, params: InstanceParams) -> Result<String> { | |
| 203 | + | let (instance, payload) = self.payload_for(params.instance.as_deref()).await?; | |
| 204 | + | Ok(format_incidents(&instance, &payload)) | |
| 205 | + | } | |
| 206 | + | ||
| 207 | + | #[instrument(skip_all)] | |
| 208 | + | pub async fn versions_impl(&self, params: InstanceParams) -> Result<String> { | |
| 209 | + | let (instance, rows) = match self.source(params.instance.as_deref())? { | |
| 210 | + | Source::Local => ( | |
| 211 | + | "local".to_string(), | |
| 212 | + | versions::collect(&self.pool, &self.config).await?, | |
| 213 | + | ), | |
| 214 | + | Source::Remote { | |
| 215 | + | name, | |
| 216 | + | base_url, | |
| 217 | + | token, | |
| 218 | + | } => { | |
| 219 | + | let value = self | |
| 220 | + | .get_remote(&base_url, token.as_deref(), "/api/versions") | |
| 221 | + | .await?; | |
| 222 | + | let rows: Vec<VersionRow> = serde_json::from_value(value)?; | |
| 223 | + | (name, rows) | |
| 224 | + | } | |
| 225 | + | }; | |
| 226 | + | ||
| 227 | + | let mut out = format!("# Versions on {instance}\n\n"); | |
| 228 | + | out.push_str(&crate::display::format_versions(&rows)); | |
| 229 | + | // The count is taken against a checkout on the instance being read, so | |
| 230 | + | // a remote answer says what is live there, not how far behind here. | |
| 231 | + | if instance != "local" { | |
| 232 | + | out.push_str( | |
| 233 | + | "\nBEHIND is measured against a checkout on that host, and is blank where it has \ | |
| 234 | + | no repo.\n", | |
| 235 | + | ); | |
| 236 | + | } | |
| 237 | + | Ok(out) | |
| 238 | + | } | |
| 239 | + | ||
| 240 | + | #[instrument(skip_all)] | |
| 241 | + | pub async fn trends_impl(&self, params: TrendsParams) -> Result<String> { | |
| 242 | + | let hours = params.hours.unwrap_or(24); | |
| 243 | + | let bucket_minutes = params.bucket_minutes.unwrap_or(60); | |
| 244 | + | ||
| 245 | + | let (instance, trends) = match self.source(params.instance.as_deref())? { | |
| 246 | + | Source::Local => { | |
| 247 | + | if self.config.get_target(¶ms.target).is_none() { | |
| 248 | + | return Ok(format!("Unknown target: {}", params.target)); | |
| 249 | + | } | |
| 250 | + | ( | |
| 251 | + | "local".to_string(), | |
| 252 | + | api::build_trends(&self.pool, ¶ms.target, hours, bucket_minutes).await, | |
| 253 | + | ) | |
| 254 | + | } | |
| 255 | + | Source::Remote { | |
| 256 | + | name, | |
| 257 | + | base_url, | |
| 258 | + | token, | |
| 259 | + | } => { | |
| 260 | + | // The target name goes into a URL path. Config keys are plain | |
| 261 | + | // identifiers, so anything else is rejected here rather than | |
| 262 | + | // escaped: a `../` or a `?` would address a different endpoint. | |
| 263 | + | if !is_config_name(¶ms.target) { | |
| 264 | + | return Ok(format!("Unusable target name: {:?}", params.target)); | |
| 265 | + | } | |
| 266 | + | let path = format!( | |
| 267 | + | "/api/trends/{}?hours={hours}&bucket_minutes={bucket_minutes}", | |
| 268 | + | params.target | |
| 269 | + | ); | |
| 270 | + | let value = self.get_remote(&base_url, token.as_deref(), &path).await?; | |
| 271 | + | (name, serde_json::from_value(value)?) | |
| 272 | + | } | |
| 273 | + | }; | |
| 274 | + | ||
| 275 | + | Ok(format_trends(&instance, &trends)) | |
| 276 | + | } | |
| 277 | + | } | |
| 278 | + | ||
| 279 | + | /// Every target on one line, worst-first, with the reason it is not green. | |
| 280 | + | fn format_status_table(instance: &str, payload: &Payload) -> String { | |
| 281 | + | if payload.nodes.is_empty() { | |
| 282 | + | return format!("No targets configured on {instance}.\n"); | |
| 283 | + | } | |
| 284 | + | ||
| 285 | + | let mut nodes: Vec<&Node> = payload.nodes.iter().collect(); | |
| 286 | + | // Worst first: the point of the table is that the problem is on line one. | |
| 287 | + | nodes.sort_by(|a, b| b.status.cmp(&a.status).then(a.id.cmp(&b.id))); | |
| 288 | + | ||
| 289 | + | let rows: Vec<[String; 5]> = nodes | |
| 290 | + | .iter() | |
| 291 | + | .map(|n| { | |
| 292 | + | [ | |
| 293 | + | n.id.strip_prefix("target:").unwrap_or(&n.id).to_string(), | |
| 294 | + | status_word(n.status).to_string(), | |
| 295 | + | field_text(n, "version"), | |
| 296 | + | field_text(n, "checked"), | |
| 297 | + | worst_condition_summary(n), | |
| 298 | + | ] | |
| 299 | + | }) | |
| 300 | + | .collect(); | |
| 301 | + | ||
| 302 | + | const HEADERS: [&str; 5] = ["TARGET", "STATUS", "VERSION", "CHECKED", "WHY"]; | |
| 303 | + | let widths: Vec<usize> = (0..HEADERS.len()) | |
| 304 | + | .map(|i| { | |
| 305 | + | rows.iter() | |
| 306 | + | .map(|r| r[i].chars().count()) | |
| 307 | + | .chain(std::iter::once(HEADERS[i].len())) | |
| 308 | + | .max() | |
| 309 | + | .unwrap_or(0) | |
| 310 | + | }) | |
| 311 | + | .collect(); | |
| 312 | + | ||
| 313 | + | let mut out = format!( | |
| 314 | + | "# {} on {instance} ({} target{}, worst: {})\n\n", | |
| 315 | + | payload.source, | |
| 316 | + | nodes.len(), | |
| 317 | + | if nodes.len() == 1 { "" } else { "s" }, | |
| 318 | + | status_word(payload.worst_status()), | |
| 319 | + | ); | |
| 320 | + | write_row(&mut out, &HEADERS.map(String::from), &widths); | |
| 321 | + | for row in &rows { | |
| 322 | + | write_row(&mut out, row, &widths); | |
| 323 | + | } | |
| 324 | + | let _ = write!( | |
| 325 | + | out, | |
| 326 | + | "\nGenerated {}. Use target_status for the full condition list.\n", | |
| 327 | + | payload.generated_at.format("%Y-%m-%d %H:%M UTC") | |
| 328 | + | ); | |
| 329 | + | out | |
| 330 | + | } | |
| 331 | + | ||
| 332 | + | /// One target in full: its fields, then every condition with its detail. | |
| 333 | + | fn format_target_detail(instance: &str, node: &Node) -> String { | |
| 334 | + | let name = node.id.strip_prefix("target:").unwrap_or(&node.id); | |
| 335 | + | let mut out = format!( | |
| 336 | + | "# {name} ({}) on {instance}: {}\n\n", | |
| 337 | + | node.label, | |
| 338 | + | status_word(node.status) | |
| 339 | + | ); | |
| 340 | + | ||
| 341 | + | if !node.fields.is_empty() { | |
| 342 | + | for field in &node.fields { | |
| 343 | + | let _ = writeln!(out, "{}: {}", field.label, value_text(&field.value)); | |
| 344 | + | } | |
| 345 | + | out.push('\n'); | |
| 346 | + | } | |
| 347 | + | ||
| 348 | + | if node.conditions.is_empty() { | |
| 349 | + | out.push_str("No conditions recorded.\n"); | |
| 350 | + | return out; | |
| 351 | + | } | |
| 352 | + | ||
| 353 | + | // Worst first here too, so a failing check is never below three green ones. | |
| 354 | + | let mut conditions: Vec<&ops_status::Condition> = node.conditions.iter().collect(); | |
| 355 | + | conditions.sort_by_key(|c| std::cmp::Reverse(c.status)); | |
| 356 | + | ||
| 357 | + | for condition in conditions { | |
| 358 | + | let _ = write!( | |
| 359 | + | out, | |
| 360 | + | "[{}] {}", | |
| 361 | + | status_word(condition.status), | |
| 362 | + | condition.condition_type | |
| 363 | + | ); | |
| 364 | + | if let Some(since) = condition.since { | |
| 365 | + | let _ = write!(out, " (since {})", since.format("%Y-%m-%d %H:%M UTC")); | |
| 366 | + | } | |
| 367 | + | if let Some(detail) = &condition.detail { | |
| 368 | + | let _ = write!(out, ": {}", scrub(detail)); | |
| 369 | + | } | |
| 370 | + | out.push('\n'); | |
| 371 | + | } | |
| 372 | + | out | |
| 373 | + | } | |
| 374 | + | ||
| 375 | + | /// Open incidents across every target, and anything else that is not green. | |
| 376 | + | fn format_incidents(instance: &str, payload: &Payload) -> String { | |
| 377 | + | let mut incidents = Vec::new(); | |
| 378 | + | let mut other = Vec::new(); | |
| 379 | + | ||
| 380 | + | for node in &payload.nodes { | |
| 381 | + | let name = node.id.strip_prefix("target:").unwrap_or(&node.id); | |
| 382 | + | for condition in &node.conditions { | |
| 383 | + | if condition.status == Status::Ok || condition.status == Status::Pending { | |
| 384 | + | continue; | |
| 385 | + | } | |
| 386 | + | let line = format!( | |
| 387 | + | "[{}] {name} / {}{}{}", | |
| 388 | + | status_word(condition.status), | |
| 389 | + | condition.condition_type, | |
| 390 | + | condition | |
| 391 | + | .since | |
| 392 | + | .map(|s| format!(" since {}", s.format("%Y-%m-%d %H:%M UTC"))) | |
| 393 | + | .unwrap_or_default(), | |
| 394 | + | condition | |
| 395 | + | .detail | |
| 396 | + | .as_ref() | |
| 397 | + | .map(|d| format!(": {}", scrub(d))) | |
| 398 | + | .unwrap_or_default(), | |
| 399 | + | ); | |
| 400 | + | if condition.condition_type == "incident" { | |
| 401 | + | incidents.push(line); | |
| 402 | + | } else { | |
| 403 | + | other.push(line); | |
| 404 | + | } | |
| 405 | + | } | |
| 406 | + | } | |
| 407 | + | ||
| 408 | + | let mut out = format!("# Open incidents on {instance}\n\n"); | |
| 409 | + | if incidents.is_empty() { | |
| 410 | + | out.push_str("No open incidents.\n"); | |
| 411 | + | } else { | |
| 412 | + | for line in &incidents { | |
| 413 | + | let _ = writeln!(out, "{line}"); | |
| 414 | + | } | |
| 415 | + | } | |
| 416 | + | ||
| 417 | + | // A failing check that has not yet opened an incident is still the answer to | |
| 418 | + | // "is anything wrong", so it is reported rather than filtered out. | |
| 419 | + | if !other.is_empty() { | |
| 420 | + | out.push_str("\nOther checks not passing:\n"); | |
| 421 | + | for line in &other { | |
| 422 | + | let _ = writeln!(out, "{line}"); | |
| 423 | + | } | |
| 424 | + | } | |
| 425 | + | out | |
| 426 | + | } | |
| 427 | + | ||
| 428 | + | /// Latency over the window, against the 7-day baseline. | |
| 429 | + | fn format_trends(instance: &str, trends: &api::TrendResponse) -> String { | |
| 430 | + | let mut out = format!( | |
| 431 | + | "# {} latency on {instance}: last {}h, {}-minute buckets\n\n", | |
| 432 | + | trends.target, trends.window_hours, trends.bucket_minutes | |
| 433 | + | ); | |
| 434 | + | ||
| 435 | + | match &trends.overall { | |
| 436 | + | Some(o) => { | |
| 437 | + | let _ = writeln!( | |
| 438 | + | out, | |
| 439 | + | "Window: avg {:.0}ms, p95 {}ms, range {}-{}ms ({} samples)", | |
| 440 | + | o.avg_ms, o.p95_ms, o.min_ms, o.max_ms, o.sample_count | |
| 441 | + | ); | |
| 442 | + | } | |
| 443 | + | None => out.push_str("Window: no operational checks in this window.\n"), | |
| 444 | + | } | |
| 445 | + | ||
| 446 | + | if let Some(b) = &trends.baseline { | |
| 447 | + | let _ = writeln!( | |
| 448 | + | out, | |
| 449 | + | "7d baseline: avg {:.0}ms, p95 {}ms ({} samples)", | |
| 450 | + | b.avg_ms, b.p95_ms, b.sample_count | |
| 451 | + | ); | |
| 452 | + | } | |
| 453 | + | ||
| 454 | + | if trends.buckets.is_empty() { | |
| 455 | + | return out; | |
| 456 | + | } | |
| 457 | + | ||
| 458 | + | out.push_str("\nBUCKET AVG P95 N\n"); | |
| 459 | + | for bucket in &trends.buckets { | |
| 460 | + | let _ = writeln!( | |
| 461 | + | out, | |
| 462 | + | "{:<18} {:>4.0}ms {:>4}ms {:>4}", | |
| 463 | + | bucket.period_start.chars().take(16).collect::<String>(), | |
| 464 | + | bucket.avg_ms, | |
| 465 | + | bucket.p95_ms, | |
| 466 | + | bucket.sample_count | |
| 467 | + | ); | |
| 468 | + | } | |
| 469 | + | out | |
| 470 | + | } | |
| 471 | + | ||
| 472 | + | /// The wire spelling of a status, which is also the shortest honest label. | |
| 473 | + | fn status_word(status: Status) -> &'static str { | |
| 474 | + | status.as_str() | |
| 475 | + | } | |
| 476 | + | ||
| 477 | + | /// A node field's value as one short string. | |
| 478 | + | fn value_text(value: &ops_status::Value) -> String { | |
| 479 | + | use ops_status::Value; | |
| 480 | + | match value { | |
| 481 | + | Value::Text { value } | Value::Ident { value, .. } | Value::Version { value } => { | |
| 482 | + | scrub(value) | |
| 483 | + | } | |
| 484 | + | Value::Path { value } => scrub(value), | |
| 485 | + | Value::Instant { value } => value.format("%Y-%m-%d %H:%M UTC").to_string(), | |
| 486 | + | Value::Duration { seconds } => format!("{seconds}s"), | |
| 487 | + | Value::Progress { value, unit, .. } | Value::Quantity { value, unit } => { | |
| 488 | + | format!("{value:.1}{}", unit.as_deref().unwrap_or("")) | |
| 489 | + | } | |
| 490 | + | Value::State { value } => status_word(*value).to_string(), | |
| 491 | + | Value::Link { url, text } => format!("{} <{}>", scrub(text.as_deref().unwrap_or("")), url), | |
| 492 | + | } | |
| 493 | + | } | |
| 494 | + | ||
| 495 | + | /// A named field's text, or `-` when the node does not carry it. | |
| 496 | + | fn field_text(node: &Node, label: &str) -> String { | |
| 497 | + | node.fields | |
| 498 | + | .iter() | |
| 499 | + | .find(|f| f.label == label) | |
| 500 | + | .map_or_else(|| "-".to_string(), |f| value_text(&f.value)) |
Lines truncated