max / makenotwork

Log the pom alert-queue and peer-identity writes that were discarded The pending-alert delete, the backoff bump and the peer identity write each swallowed their error with `let _ =`. A failed delete re-sends an alert the operator already received, and a failed backoff bump leaves a row to retry on the next tick; both are recoverable, which is why they were ignored, and both are invisible when they happen. A monitoring tool that silently drops its own bookkeeping errors is the wrong tool to learn that from a duplicate alert.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: Max Johnson <me@maxj.phd> · 2026-07-30 23:44 UTC

Signed with PGP, not checked

Commit: 58083c0c02081da9fdf1d9bd912e01a81bea4561

Parent: 5b9d14f

3 files changed, +62 insertions, -16 deletions

M pom/docs/architecture.md +20 -7

			@@ -17,20 +17,33 @@
17	17		\| Module \| File \| Role \|
18	18		\|--------\|------\|------\|
19	19		\| `main` \| `src/main.rs` \| Entry point: parses CLI args, dispatches to CLI handler or MCP server \|
20		-	\| `cli` \| `src/cli.rs` \| CLI command handlers (health, test, status, versions, history, prune, serve, mesh) \|
	20	+	\| `cli` \| `src/cli/` \| CLI command handlers. `mod.rs` dispatch, plus `serve.rs`, `status.rs`, `incident.rs`, `transition.rs`, and `tasks/` (one spawner per background loop: health, tls, dns, whois, routes, cors, backup, systemd, scan_pipeline, synckit_fleet, meta_alert, prune) \|
21	21		\| `config` \| `src/config.rs` \| TOML config loading, types for targets/peers/alerts/serve settings \|
22	22		\| `types` \| `src/types.rs` \| Shared domain types: HealthSnapshot, TestRun, TlsStatus, LatencyStats, TestStaleness \|
23		-	\| `db` \| `src/db.rs` \| SQLite schema (versioned migrations), all queries for health/tests/alerts/TLS/incidents/peers \|
	23	+	\| `db` \| `src/db/` \| SQLite persistence. `migrations.rs` holds the numbered schema migrations and pool construction; the rest is one query module per subject: health, test_runs, alerts, incidents, peers, tls, dns, whois, routes, cors, backup, systemd, scan_pipeline, synckit_fleet, maintenance \|
24	24		\| `api` \| `src/api.rs` \| Axum HTTP API: status, trends, peer info, mesh view, bearer token auth middleware \|
25		-	\| `alerts` \| `src/alerts.rs` \| Alerter struct: sends emails via Postmark on status transitions, with cooldown tracking \|
	25	+	\| `alerts` \| `src/alerts/` \| Alerter struct: sends emails via Postmark on status transitions, with cooldown tracking. One module per alert kind: health, tls, latency, offline, peer, dns, whois, route, cors, backup, systemd, scan, test_duration \|
26	26		\| `peer` \| `src/peer.rs` \| Peer mesh: identity management, heartbeat loops, grace period state machine, mesh state \|
	27	+	\| `status` \| `src/status.rs` \| PoM's projection onto the shared operator status payload \|
	28	+	\| `dashboard` \| `src/dashboard.rs` \| Optional HTML dashboard served at `GET /` \|
	29	+	\| `tls` \| `src/tls.rs` \| Shared TLS configuration for pom's outbound HTTP, used by every probe \|
27	30		\| `display` \| `src/display.rs` \| Pure formatting functions for CLI output (no I/O) \|
28	31		\| `versions` \| `src/versions.rs` \| The `pom versions` roll-up: live version/sha per target, plus commits behind the local checkout \|
29	32		\| `error` \| `src/error.rs` \| Typed error enum (PomError) wrapping IO, DB, HTTP, JSON, config errors \|
30		-	\| `checks::http` \| `src/checks/http.rs` \| HTTP health checker, response classification, expectation validation, latency drift detection, test staleness computation \|
31		-	\| `checks::tls` \| `src/checks/tls.rs` \| TLS certificate prober: TCP connect, TLS handshake, x509 leaf cert parsing \|
32		-	\| `checks::ssh` \| `src/checks/ssh.rs` \| Remote test runner: executes commands over SSH, captures output \|
	33	+	\| `checks::http` \| `src/checks/http.rs` \| HTTP health probe: issues the request, applies configured JSON expectations, classifies the response into a HealthStatus \|
	34	+	\| `checks::drift` \| `src/checks/drift.rs` \| Trend and staleness analysis over stored check history: latency drift, test duration, test staleness \|
	35	+	\| `checks::tls` \| `src/checks/tls.rs` \| TLS certificate prober: connect, inspect the leaf cert, track expiry \|
	36	+	\| `checks::ssh` \| `src/checks/ssh.rs` \| Remote test runner: validates the filter, runs the target's command over SSH, parses output into a TestRun \|
	37	+	\| `checks::ssh_banner` \| `src/checks/ssh_banner.rs` \| TCP connect and verify the SSH protocol banner \|
33	38		\| `checks::parse` \| `src/checks/parse.rs` \| CI output parser: extracts PASS/FAIL steps and cargo test counts \|
	39	+	\| `checks::dns` \| `src/checks/dns.rs` \| DNS record verification: resolves hostnames, compares against expected values \|
	40	+	\| `checks::whois` \| `src/checks/whois.rs` \| WHOIS domain expiry checking over raw TCP \|
	41	+	\| `checks::routes` \| `src/checks/routes.rs` \| Route accessibility: verifies expected pages are reachable \|
	42	+	\| `checks::cors` \| `src/checks/cors.rs` \| CORS preflight: sends OPTIONS and checks Access-Control headers \|
	43	+	\| `checks::backup` \| `src/checks/backup.rs` \| Local filesystem backup verification: scans for PostgreSQL backup files, checks recency \|
	44	+	\| `checks::systemd` \| `src/checks/systemd.rs` \| Local systemd unit health: liveness, crash-loops, optionally any failed unit on the host \|
	45	+	\| `checks::scan_pipeline` \| `src/checks/scan_pipeline.rs` \| Polls a makenotwork instance's upload scan health and applies the audit thresholds \|
	46	+	\| `checks::synckit_fleet` \| `src/checks/synckit_fleet.rs` \| Polls a makenotwork instance for which SyncKit SDK versions are syncing \|
34	47		\| `tools` \| `src/tools/mod.rs` \| MCP server definition (PomServer), tool registration via rmcp \|
35	48		\| `tools::health` \| `src/tools/health.rs` \| MCP tool implementations for health checks, history, targets, mesh status \|
36	49		\| `tools::tests` \| `src/tools/tests.rs` \| MCP tool implementations for test execution, history, raw output \|
			@@ -101,7 +114,7 @@
101	114
102	115		## Database Schema
103	116
104		-	SQLite with WAL journal mode. Schema is managed through numbered migrations (currently v1-v4).
	117	+	SQLite with WAL journal mode. Schema is managed through numbered migrations (currently v1-v13).
105	118
106	119		### Tables
107	120

M pom/src/peer.rs +18 -6

			@@ -370,13 +370,21 @@
370	370		};
371	371		// Lock dropped, DB writes and alerts happen without holding mesh lock
372	372
373		-	if let Some(id) = new_identity_id {
374		-	let _ = crate::db::store_peer_identity(pool, peer_name, &id).await;
	373	+	if let Some(id) = new_identity_id
	374	+	&& let Err(e) = crate::db::store_peer_identity(pool, peer_name, &id).await
	375	+	{
	376	+	tracing::warn!(peer = %peer_name, error = %e, "peer: identity store failed");
375	377		}
376		-	if let Some(id) = updated_identity_id {
377		-	let _ = crate::db::update_peer_identity(pool, peer_name, &id).await;
	378	+	if let Some(id) = updated_identity_id
	379	+	&& let Err(e) = crate::db::update_peer_identity(pool, peer_name, &id).await
	380	+	{
	381	+	tracing::warn!(peer = %peer_name, error = %e, "peer: identity update failed");
	382	+	}
	383	+	if let Err(e) =
	384	+	crate::db::insert_peer_heartbeat(pool, peer_name, "online", latency_ms as i64).await
	385	+	{
	386	+	tracing::warn!(peer = %peer_name, error = %e, "peer: heartbeat insert failed");
378	387		}
379		-	let _ = crate::db::insert_peer_heartbeat(pool, peer_name, "online", latency_ms as i64).await;
380	388
381	389		if let (Some(address), Some(alerter)) = (recovery_info, alerter) {
382	390		alerter.send_peer_recovery(peer_name, &address).await;
			@@ -443,7 +451,11 @@
443	451		// Lock dropped, DB write and alerts happen without holding mesh lock
444	452
445	453		let status_str = new_status.to_string();
446		-	let _ = crate::db::insert_peer_heartbeat(pool, peer_name, &status_str, latency_ms as i64).await;
	454	+	if let Err(e) =
	455	+	crate::db::insert_peer_heartbeat(pool, peer_name, &status_str, latency_ms as i64).await
	456	+	{
	457	+	tracing::warn!(peer = %peer_name, error = %e, "peer: heartbeat insert failed");
	458	+	}
447	459
448	460		if let (Some((address, failures)), Some(alerter)) = (alert_info, alerter) {
449	461		alerter

M pom/src/cli/serve.rs +24 -3

			@@ -157,7 +157,14 @@
157	157		let due = db::due_pending_alerts(&pool, 50).await.unwrap_or_default();
158	158		for p in due {
159	159		if retry_alerter.retry_pending(&p).await {
160		-	let _ = db::delete_pending_alert(&pool, p.id).await;
	160	+	if let Err(e) = db::delete_pending_alert(&pool, p.id).await {
	161	+	tracing::warn!(
	162	+	alert_id = p.id,
	163	+	alert_key = %p.alert_key,
	164	+	error = %e,
	165	+	"pending alert delete failed, alert will re-send"
	166	+	);
	167	+	}
161	168		info!("delivered previously-queued alert to {}", p.alert_key);
162	169		} else if p.attempts + 1 >= MAX_ATTEMPTS {
163	170		// Give up so the queue can't grow without bound; losing the
			@@ -167,14 +174,28 @@
167	174		p.alert_key,
168	175		p.attempts + 1
169	176		);
170		-	let _ = db::delete_pending_alert(&pool, p.id).await;
	177	+	if let Err(e) = db::delete_pending_alert(&pool, p.id).await {
	178	+	tracing::warn!(
	179	+	alert_id = p.id,
	180	+	alert_key = %p.alert_key,
	181	+	error = %e,
	182	+	"pending alert delete failed, alert will re-send"
	183	+	);
	184	+	}
171	185		} else {
172	186		// Exponential backoff, capped at 1h.
173	187		let backoff =
174	188		std::cmp::min(30 * 2i64.pow((p.attempts.min(7)) as u32), 3600);
175	189		let next =
176	190		(chrono::Utc::now() + chrono::Duration::seconds(backoff)).to_rfc3339();
177		-	let _ = db::bump_pending_alert(&pool, p.id, &next).await;
	191	+	if let Err(e) = db::bump_pending_alert(&pool, p.id, &next).await {
	192	+	tracing::warn!(
	193	+	alert_id = p.id,
	194	+	alert_key = %p.alert_key,
	195	+	error = %e,
	196	+	"pending alert backoff bump failed, row retries next tick"
	197	+	);
	198	+	}
178	199		}
179	200		}
180	201		}