Skip to main content

max / makenotwork

sando: remove the Prometheus exporter instead of deciding its auth posture The /metrics endpoint was left unauthenticated on the tailnet bind as a deliberate Prometheus-convention opt-out, and a prior audit flagged it as "decide the posture" -- a decision never recorded, so every audit re-raised it. Nothing scrapes it. There is no Prometheus config anywhere in ~/Code and pom does not consume it, so the exporter was carrying an open endpoint and a recorder for metrics no one reads. Rather than pick an auth story for a dead feature, the feature is gone: the /metrics route, the metrics module, the PrometheusHandle on AppState, the twelve emission call sites, and the metrics + metrics-exporter-prometheus dependencies. The router now has no unauthenticated routes at all, which is a simpler thing to reason about than one deliberate exception. The comment there and the API table in the README are updated to match. Deploy health is already observable through /state and the WS event stream, neither of which this touched.
Author: Max Johnson <me@maxj.phd> · 2026-07-21 21:58 UTC
Commit: 37f8bf5951a1d3e194d0796d6ffa556bcf46f244
Parent: 995309f
11 files changed, +6 insertions, -217 deletions
M sando/Cargo.lock -169
@@ -367,15 +367,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
367 367 checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
368 368
369 369 [[package]]
370 - name = "crossbeam-epoch"
371 - version = "0.9.20"
372 - source = "registry+https://github.com/rust-lang/crates.io-index"
373 - checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
374 - dependencies = [
375 - "crossbeam-utils",
376 - ]
377 -
378 - [[package]]
379 370 name = "crossbeam-queue"
380 371 version = "0.3.12"
381 372 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -574,17 +565,6 @@ dependencies = [
574 565 ]
575 566
576 567 [[package]]
577 - name = "evmap"
578 - version = "11.0.0"
579 - source = "registry+https://github.com/rust-lang/crates.io-index"
580 - checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8"
581 - dependencies = [
582 - "hashbag",
583 - "left-right",
584 - "smallvec",
585 - ]
586 -
587 - [[package]]
588 568 name = "fastrand"
589 569 version = "2.4.1"
590 570 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -706,21 +686,6 @@ dependencies = [
706 686 ]
707 687
708 688 [[package]]
709 - name = "generator"
710 - version = "0.8.9"
711 - source = "registry+https://github.com/rust-lang/crates.io-index"
712 - checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae"
713 - dependencies = [
714 - "cc",
715 - "cfg-if",
716 - "libc",
717 - "log",
718 - "rustversion",
719 - "windows-link",
720 - "windows-result",
721 - ]
722 -
723 - [[package]]
724 689 name = "generic-array"
725 690 version = "0.14.7"
726 691 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -776,12 +741,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
776 741 checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
777 742
778 743 [[package]]
779 - name = "hashbag"
780 - version = "0.1.13"
781 - source = "registry+https://github.com/rust-lang/crates.io-index"
782 - checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064"
783 -
784 - [[package]]
785 744 name = "hashbrown"
786 745 version = "0.15.5"
787 746 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1224,17 +1183,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1224 1183 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
1225 1184
1226 1185 [[package]]
1227 - name = "left-right"
1228 - version = "0.11.7"
1229 - source = "registry+https://github.com/rust-lang/crates.io-index"
1230 - checksum = "0f0c21e4c8ff95f487fb34e6f9182875f42c84cef966d29216bf115d9bba835a"
1231 - dependencies = [
1232 - "crossbeam-utils",
1233 - "loom",
1234 - "slab",
1235 - ]
1236 -
1237 - [[package]]
1238 1186 name = "libc"
1239 1187 version = "0.2.186"
1240 1188 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1285,19 +1233,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1285 1233 checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
1286 1234
1287 1235 [[package]]
1288 - name = "loom"
1289 - version = "0.7.2"
1290 - source = "registry+https://github.com/rust-lang/crates.io-index"
1291 - checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca"
1292 - dependencies = [
1293 - "cfg-if",
1294 - "generator",
1295 - "scoped-tls",
1296 - "tracing",
1297 - "tracing-subscriber",
1298 - ]
1299 -
1300 - [[package]]
1301 1236 name = "lru"
1302 1237 version = "0.12.5"
1303 1238 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1344,48 +1279,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1344 1279 checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
1345 1280
1346 1281 [[package]]
1347 - name = "metrics"
1348 - version = "0.24.6"
1349 - source = "registry+https://github.com/rust-lang/crates.io-index"
1350 - checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
1351 - dependencies = [
1352 - "portable-atomic",
1353 - "rapidhash",
1354 - ]
1355 -
1356 - [[package]]
1357 - name = "metrics-exporter-prometheus"
1358 - version = "0.18.3"
1359 - source = "registry+https://github.com/rust-lang/crates.io-index"
1360 - checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108"
1361 - dependencies = [
1362 - "base64",
1363 - "evmap",
1364 - "indexmap",
1365 - "metrics",
1366 - "metrics-util",
1367 - "quanta",
1368 - "thiserror 2.0.18",
1369 - ]
1370 -
1371 - [[package]]
1372 - name = "metrics-util"
1373 - version = "0.20.4"
1374 - source = "registry+https://github.com/rust-lang/crates.io-index"
1375 - checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13"
1376 - dependencies = [
1377 - "crossbeam-epoch",
1378 - "crossbeam-utils",
1379 - "hashbrown 0.16.1",
1380 - "metrics",
1381 - "quanta",
1382 - "rand 0.9.4",
1383 - "rand_xoshiro",
1384 - "rapidhash",
1385 - "sketches-ddsketch",
1386 - ]
1387 -
1388 - [[package]]
1389 1282 name = "mime"
1390 1283 version = "0.3.17"
1391 1284 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1514,12 +1407,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1514 1407 checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
1515 1408
1516 1409 [[package]]
1517 - name = "portable-atomic"
1518 - version = "1.13.1"
1519 - source = "registry+https://github.com/rust-lang/crates.io-index"
1520 - checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
1521 -
1522 - [[package]]
1523 1410 name = "potential_utf"
1524 1411 version = "0.1.5"
1525 1412 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1547,21 +1434,6 @@ dependencies = [
1547 1434 ]
1548 1435
1549 1436 [[package]]
1550 - name = "quanta"
1551 - version = "0.12.6"
1552 - source = "registry+https://github.com/rust-lang/crates.io-index"
1553 - checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
1554 - dependencies = [
1555 - "crossbeam-utils",
1556 - "libc",
1557 - "once_cell",
1558 - "raw-cpuid",
1559 - "wasi",
1560 - "web-sys",
1561 - "winapi",
1562 - ]
1563 -
1564 - [[package]]
1565 1437 name = "quinn"
1566 1438 version = "0.11.11"
1567 1439 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1715,24 +1587,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1715 1587 checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
1716 1588
1717 1589 [[package]]
1718 - name = "rand_xoshiro"
1719 - version = "0.7.0"
1720 - source = "registry+https://github.com/rust-lang/crates.io-index"
1721 - checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
1722 - dependencies = [
1723 - "rand_core 0.9.5",
1724 - ]
1725 -
1726 - [[package]]
1727 - name = "rapidhash"
1728 - version = "4.4.1"
1729 - source = "registry+https://github.com/rust-lang/crates.io-index"
1730 - checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59"
1731 - dependencies = [
1732 - "rustversion",
1733 - ]
1734 -
1735 - [[package]]
1736 1590 name = "ratatui"
1737 1591 version = "0.29.0"
1738 1592 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1754,15 +1608,6 @@ dependencies = [
1754 1608 ]
1755 1609
1756 1610 [[package]]
1757 - name = "raw-cpuid"
1758 - version = "11.6.0"
1759 - source = "registry+https://github.com/rust-lang/crates.io-index"
1760 - checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
1761 - dependencies = [
1762 - "bitflags",
1763 - ]
1764 -
1765 - [[package]]
1766 1611 name = "redox_syscall"
1767 1612 version = "0.5.18"
1768 1613 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1985,8 +1830,6 @@ dependencies = [
1985 1830 "axum",
1986 1831 "chrono",
1987 1832 "http-body-util",
1988 - "metrics",
1989 - "metrics-exporter-prometheus",
1990 1833 "ops-core",
1991 1834 "ops-exec",
1992 1835 "reqwest",
@@ -2030,12 +1873,6 @@ dependencies = [
2030 1873 ]
2031 1874
2032 1875 [[package]]
2033 - name = "scoped-tls"
2034 - version = "1.0.1"
2035 - source = "registry+https://github.com/rust-lang/crates.io-index"
2036 - checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
2037 -
2038 - [[package]]
2039 1876 name = "scopeguard"
2040 1877 version = "1.2.0"
2041 1878 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2256,12 +2093,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
2256 2093 checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
2257 2094
2258 2095 [[package]]
2259 - name = "sketches-ddsketch"
2260 - version = "0.3.1"
2261 - source = "registry+https://github.com/rust-lang/crates.io-index"
2262 - checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
2263 -
2264 - [[package]]
2265 2096 name = "slab"
2266 2097 version = "0.4.12"
2267 2098 source = "registry+https://github.com/rust-lang/crates.io-index"
M sando/README.md -1
@@ -210,7 +210,6 @@ green no-op.
210 210 | POST | `/rollback/{tier}` | — | Swap `current` symlink to `previous_version` on every node in the tier. One step only: `previous_version` is cleared afterwards, so a second `/rollback` returns 409 rather than rolling forward onto the version you just escaped. |
211 211 | POST | `/confirm/{tier}` | — | Insert a passing `manual_confirm` gate row for the tier's `current_version`. Replaces hand-SQL. |
212 212 | POST | `/backup/fetch` | — | Pull the prod backup. Supports `file://`, `rsync://`, `ssh://user@host[:port]/path`. |
213 - | GET | `/metrics` | — | Prometheus exposition |
214 213 | GET | `/events` | — | WebSocket stream of typed events (RebuildRequested, BuildStart/Ok/Failed, GateStart/Done, DeployStart/Ok/Failed, PromoteComplete, Rollback, BackupFetched, ManualConfirm, BuildAborted). |
215 214
216 215 ## TUI
M sando/daemon/Cargo.toml -2
@@ -20,8 +20,6 @@ toml = "1.1"
20 20 sqlx = { version = "0.9", features = ["runtime-tokio", "sqlite", "postgres", "chrono", "migrate"] }
21 21 tracing = "0.1.44"
22 22 tracing-subscriber = { version = "0.3.22", features = ["env-filter", "json"] }
23 - metrics = "0.24"
24 - metrics-exporter-prometheus = { version = "0.18.1", default-features = false }
25 23 anyhow = "1.0.102"
26 24 thiserror = "2.0.18"
27 25 chrono = { version = "0.4", features = ["serde"] }
M sando/daemon/src/build.rs -4
@@ -157,8 +157,6 @@ pub async fn run(
157 157 elapsed_s,
158 158 },
159 159 );
160 - metrics::counter!("sando_builds_total", "result" => "failed").increment(1);
161 - metrics::histogram!("sando_build_seconds").record(elapsed_s as f64);
162 160 // Settle the run with the headline compiler diagnostic (not the raw
163 161 // 4 KB tail) so `GET /runs/{id}` answers "why" without a journald dive.
164 162 let summary = crate::classify::classify_compile_error(&out.stdout, &out.stderr).summary();
@@ -179,8 +177,6 @@ pub async fn run(
179 177 elapsed_s,
180 178 },
181 179 );
182 - metrics::counter!("sando_builds_total", "result" => "ok").increment(1);
183 - metrics::histogram!("sando_build_seconds").record(elapsed_s as f64);
184 180
185 181 // Binaries land under `<target>/release/`; with a shared target dir that's
186 182 // not inside the worktree, so resolve it the same way cargo did above.
M sando/daemon/src/gates.rs -6
@@ -46,7 +46,6 @@ pub struct NodeProbe {
46 46 pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> {
47 47 let kind = gate.kind();
48 48 let started_at = Utc::now().to_rfc3339();
49 - let t0 = std::time::Instant::now();
50 49
51 50 let id: i64 = sqlx::query_scalar(
52 51 "INSERT INTO gate_runs (version, tier, gate_kind, started_at) VALUES (?, ?, ?, ?)
@@ -157,11 +156,6 @@ pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> {
157 156 },
158 157 );
159 158
160 - metrics::counter!("sando_gate_runs_total", "gate" => kind.as_str(), "status" => outcome.status_str())
161 - .increment(1);
162 - metrics::histogram!("sando_gate_seconds", "gate" => kind.as_str())
163 - .record(t0.elapsed().as_secs_f64());
164 -
165 159 Ok(outcome)
166 160 }
167 161
M sando/daemon/src/lib.rs -1
@@ -25,7 +25,6 @@ pub mod events;
25 25 pub mod gates;
26 26 pub mod git;
27 27 pub mod live_log;
28 - pub mod metrics;
29 28 pub mod outcome;
30 29 pub mod routes;
31 30 pub mod runs;
M sando/daemon/src/main.rs +1 -3
@@ -1,5 +1,5 @@
1 1 use anyhow::Result;
2 - use sando_daemon::{config, db, events, git, metrics, routes, runs, state, sync, topology};
2 + use sando_daemon::{config, db, events, git, routes, runs, state, sync, topology};
3 3 use std::net::SocketAddr;
4 4 use std::path::Path;
5 5 use std::sync::Arc;
@@ -99,7 +99,6 @@ async fn run() -> Result<()> {
99 99 sando_daemon::gates::preflight_scratch_privileges(scratch_url).await?;
100 100 }
101 101
102 - let prom = metrics::init();
103 102 let addr: SocketAddr = cfg.listen.parse()?;
104 103
105 104 // Deploy-API auth (CF2). Token comes from SANDO_API_TOKEN (systemd
@@ -129,7 +128,6 @@ async fn run() -> Result<()> {
129 128 pool,
130 129 topo,
131 130 cfg,
132 - prom,
133 131 active_build: Arc::new(tokio::sync::Mutex::new(None)),
134 132 deploy_lock: Arc::new(tokio::sync::Mutex::new(())),
135 133 events: events::channel(),
D sando/daemon/src/metrics.rs -12
@@ -1,12 +0,0 @@
1 - use axum::{extract::State, response::IntoResponse};
2 - use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
3 -
4 - pub fn init() -> PrometheusHandle {
5 - PrometheusBuilder::new()
6 - .install_recorder()
7 - .expect("install prometheus recorder")
8 - }
9 -
10 - pub async fn render(State(handle): State<PrometheusHandle>) -> impl IntoResponse {
11 - handle.render()
12 - }
M sando/daemon/src/routes/mod.rs +5 -14
@@ -11,15 +11,15 @@ mod promotion;
11 11 use promotion::{clear_partial, promote_inner, set_partial};
12 12
13 13 pub fn router(state: AppState) -> Router {
14 - let prom = state.prom.clone();
15 14 let token = state.api_token.clone();
16 15
17 16 // Every route requires the bearer token (when one is configured): the
18 17 // mutators carry prod-deploy authority (CF2), and the reads expose prod
19 18 // state — deployed versions, build SHAs, full gate logs, the live event
20 - // stream — that a tailnet peer should not get unauthenticated. `/metrics`
21 - // is the one deliberate opt-out (Prometheus convention, no secrets), added
22 - // outside the layered group below. The bearer-success path logs the caller
19 + // stream — that a tailnet peer should not get unauthenticated. There are no
20 + // unauthenticated routes: /metrics used to be the one opt-out, but the
21 + // Prometheus exporter was removed rather than leave an auth posture on it
22 + // that no scraper actually used. The bearer-success path logs the caller
23 23 // identity for mutating (POST) requests so a prod ship is attributable.
24 24 let protected = Router::new()
25 25 // mutators — prod-deploy authority
@@ -39,10 +39,7 @@ pub fn router(state: AppState) -> Router {
39 39 require_bearer(token.clone(), req, next)
40 40 }));
41 41
42 - Router::new()
43 - .merge(protected)
44 - .with_state(state)
45 - .route("/metrics", get(crate::metrics::render).with_state(prom))
42 + Router::new().merge(protected).with_state(state)
46 43 }
47 44
48 45 /// Bearer-token gate for the deploy mutators. When no token is configured the
@@ -383,7 +380,6 @@ async fn rollback(
383 380 to: previous.clone(),
384 381 },
385 382 );
386 - metrics::counter!("sando_rollbacks_total", "tier" => tier.to_string()).increment(1);
387 383
388 384 Ok(Json(serde_json::json!({
389 385 "tier": tier,
@@ -831,7 +827,6 @@ mod tests {
831 827 use axum::body::Body;
832 828 use axum::http::{Request, StatusCode};
833 829 use http_body_util::BodyExt;
834 - use metrics_exporter_prometheus::PrometheusBuilder;
835 830 use sqlx::SqlitePool;
836 831 use sqlx::sqlite::SqlitePoolOptions;
837 832 use std::path::PathBuf;
@@ -938,14 +933,12 @@ mod tests {
938 933 }
939 934 // Don't call install_recorder in tests — it touches a process-global
940 935 // and conflicts when tests run in parallel.
941 - let prom = PrometheusBuilder::new().build_recorder().handle();
942 936 let topo = test_topo();
943 937 let executors = Arc::new(crate::state::build_executors(&topo));
944 938 AppState {
945 939 pool,
946 940 topo: Arc::new(topo),
947 941 cfg: Arc::new(test_cfg()),
948 - prom,
949 942 active_build: Arc::new(tokio::sync::Mutex::new(None)),
950 943 deploy_lock: Arc::new(tokio::sync::Mutex::new(())),
951 944 events: crate::events::channel(),
@@ -1499,7 +1492,6 @@ mod tests {
1499 1492 .await
1500 1493 .unwrap();
1501 1494 }
1502 - let prom = PrometheusBuilder::new().build_recorder().handle();
1503 1495 let mut topo = test_topo();
1504 1496 topo.tiers[0].gates = vec![Gate::BootSmoke]; // host configures a gate...
1505 1497 let executors = Arc::new(crate::state::build_executors(&topo));
@@ -1507,7 +1499,6 @@ mod tests {
1507 1499 pool,
1508 1500 topo: Arc::new(topo),
1509 1501 cfg: Arc::new(test_cfg()),
1510 - prom,
1511 1502 active_build: Arc::new(tokio::sync::Mutex::new(None)),
1512 1503 deploy_lock: Arc::new(tokio::sync::Mutex::new(())),
1513 1504 events: crate::events::channel(),
M sando/daemon/src/routes/promotion.rs -3
@@ -370,7 +370,6 @@ pub(super) async fn promote_inner(
370 370 version: version.clone(),
371 371 },
372 372 );
373 - metrics::counter!("sando_promotes_total", "tier" => target.name.to_string()).increment(1);
374 373 tracing::info!(
375 374 version = %version, tier = %target.name,
376 375 hotfix = body.hotfix, reset_burn_in = body.reset_burn_in,
@@ -469,7 +468,6 @@ pub(super) async fn set_partial(s: &AppState, tier: &crate::domain::TierId, reas
469 468 tracing::error!(tier = %tier, reason, error = %e,
470 469 "failed to record tier partial state; the fleet may be inconsistent without a /state flag");
471 470 }
472 - metrics::gauge!("sando_tier_partial", "tier" => tier.to_string()).set(1.0);
473 471 }
474 472
475 473 /// Clear a tier's partial flag after a clean full promote or rollback. Errors are
@@ -484,7 +482,6 @@ pub(super) async fn clear_partial(s: &AppState, tier: &crate::domain::TierId) {
484 482 {
485 483 tracing::warn!(tier = %tier, error = %e, "failed to clear tier partial flag");
486 484 }
487 - metrics::gauge!("sando_tier_partial", "tier" => tier.to_string()).set(0.0);
488 485 }
489 486
490 487 /// Returns the kinds of `tier`'s *configured* gates that are not satisfied for
M sando/daemon/src/state.rs -2
@@ -2,7 +2,6 @@ use crate::config::Config;
2 2 use crate::domain::NodeId;
3 3 use crate::events::EventTx;
4 4 use crate::topology::{Node, Topology};
5 - use metrics_exporter_prometheus::PrometheusHandle;
6 5 use ops_exec::{CapabilitySet, Executor, LocalExec, SshExec};
7 6 use sqlx::SqlitePool;
8 7 use std::collections::HashMap;
@@ -27,7 +26,6 @@ pub struct AppState {
27 26 pub pool: SqlitePool,
28 27 pub topo: Arc<Topology>,
29 28 pub cfg: Arc<Config>,
30 - pub prom: PrometheusHandle,
31 29 /// Single-slot guard for the build pipeline. A new /rebuild aborts any
32 30 /// in-flight build (cargo + gates) so the latest push always wins. Carries
33 31 /// the run id alongside the handle so the aborting `/rebuild` can settle