max / makenotwork

Report the host trust store at boot and on /api/health mt reads its outbound TLS anchors from the host CA bundle, so a thin or unreadable store takes out the OAuth token exchange that logs users in, link previews, and S3 media at once. Nothing noticed: the box booted, /api/health passed on a database probe alone, and pages served, so the first symptom was a login outage with a certificate error in the journal. trust_store probes the same call the verifier will make on Linux (rustls_native_certs::load_native_certs, which is what reqwest's platform verifier and the AWS client behind s3-storage both use) once per process and caches it. main forces the probe at startup for the log line; the health handler reads the cached answer. Emptiness is the only failure, matching what the platform verifier accepts: it logs certificates it had to ignore and builds a verifier anyway. The new tls_trust_anchors field deliberately leaves status and the HTTP code alone. A box with no anchors still serves everything that does not leave it, and a bad bundle is usually a whole-fleet condition, so failing the load-balancer check would turn a login outage into a total one. PoM's assertion on the field is what gets it monitored, so the pom-hetzner config moves with it. Watching the host bundle for staleness over time is infra's half, filed separately.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: Max Johnson <me@maxj.phd> · 2026-08-05 18:27 UTC

Signed with PGP, not checked

Commit: 71221bcdf0a7d24397f1fa14f8336251561b0628

Parent: dc50bf6

9 files changed, +144 insertions, -16 deletions

M multithreaded/Cargo.lock +9 -8

			@@ -2263,6 +2263,7 @@
2263	2263		"rand 0.10.2",
2264	2264		"regex-lite",
2265	2265		"reqwest",
	2266	+	"rustls-native-certs",
2266	2267		"s3-storage",
2267	2268		"serde",
2268	2269		"serde_json",
			@@ -5013,14 +5014,6 @@
5013	5014		name = "supernote-push"
5014	5015		version = "0.1.0"
5015	5016
5016		-	[[patch.unused]]
5017		-	name = "kberg"
5018		-	version = "0.1.0"
5019		-
5020		-	[[patch.unused]]
5021		-	name = "painhours"
5022		-	version = "0.1.0"
5023		-
5024	5017		[[patch.unused]]
5025	5018		name = "synckit-client"
5026	5019		version = "0.6.0"
			@@ -5028,3 +5021,11 @@
5028	5021		[[patch.unused]]
5029	5022		name = "synckit-config"
5030	5023		version = "0.1.2"
	5024	+
	5025	+	[[patch.unused]]
	5026	+	name = "kberg"
	5027	+	version = "0.1.0"
	5028	+
	5029	+	[[patch.unused]]
	5030	+	name = "painhours"
	5031	+	version = "0.1.0"

M multithreaded/Cargo.toml +6

			@@ -30,6 +30,11 @@
30	30
31	31		# HTTP client / crypto
32	32		reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"] }
	33	+	# Reads the host trust store, which is what reqwest's platform verifier does on
	34	+	# Linux and what the AWS client behind s3-storage calls directly. Already in the
	35	+	# tree through both of those; declared here so trust_store can probe the same
	36	+	# store the outbound paths will use (deploy/README.md).
	37	+	rustls-native-certs = "0.8"
33	38		# Pinned to the exact major reqwest resolves so the SSRF pre-check parses the
34	39		# host with the same parser reqwest connects through (link_preview::validate_url).
35	40		url = "2"
			@@ -84,6 +89,7 @@
84	89		tower-sessions = { workspace = true }
85	90		tower-sessions-sqlx-store = { workspace = true }
86	91		reqwest = { workspace = true }
	92	+	rustls-native-certs = { workspace = true }
87	93		url = { workspace = true }
88	94		sha2 = { workspace = true }
89	95		base64 = { workspace = true }

M multithreaded/deploy/README.md +8 -3

			@@ -89,9 +89,14 @@
89	89		deploy problem. mt starts, `/api/health` passes because it only probes the
90	90		database, and pages serve. The first outbound TLS request is a login attempt,
91	91		because mt is an OAuth relying party, so the symptom is a login outage with a
92		-	certificate error buried in the journal on a node that reports healthy. Check
93		-	the bundle when provisioning a node, and keep `ca-certificates` on an update
94		-	path rather than assuming it stays current.
	92	+	certificate error buried in the journal on a node that reports healthy.
	93	+
	94	+	Two things now make that visible. mt probes the store at startup and logs an
	95	+	error when it comes back empty, and `/api/health` carries a `tls_trust_anchors`
	96	+	field that PoM asserts, so the condition surfaces as a degraded target rather
	97	+	than as a support ticket. Neither watches the host bundle for staleness over
	98	+	time; that is infra's job, and keeping `ca-certificates` on an update path is
	99	+	still a precondition rather than something mt arranges.
95	100
96	101		Rollback is the MNW server's rollback: the previous bundle still holds the
97	102		previous mt binary.

M multithreaded/src/lib.rs +1

			@@ -19,6 +19,7 @@
19	19		pub mod static_assets;
20	20		pub mod storage;
21	21		pub mod templates;
	22	+	pub mod trust_store;
22	23		pub mod trusted_proxy;
23	24
24	25		use config::Config;

M multithreaded/src/main.rs +6

			@@ -46,6 +46,12 @@
46	46		let config = Config::from_env();
47	47		multithreaded::error_page::init(config.mnw_base_url.clone());
48	48
	49	+	// Probe the host trust store now rather than on the first outbound request,
	50	+	// so a host with no usable CA anchors says so in the journal at boot and on
	51	+	// /api/health, instead of presenting as a login outage on a box that reports
	52	+	// healthy. The answer is cached, so the health handler does not repeat it.
	53	+	multithreaded::trust_store::anchors_ok();
	54	+
49	55		// Optional S3 storage for image uploads
50	56		let s3 = if let Some(ref s3_config) = config.s3 {
51	57		match multithreaded::storage::S3Storage::new(s3_config).await {

M pom/deploy/pom-hetzner.toml +8 -1

			@@ -121,7 +121,14 @@
121	121
122	122		[targets.mt.health.expect]
123	123		status_code = 200
124		-	json_fields = { "status" = "operational", "database" = "true" }
	124	+	# `tls_trust_anchors` is whether the host CA bundle gave mt any outbound TLS
	125	+	# anchors. mt ships none of its own, so a thin or stale bundle takes out the
	126	+	# OAuth token exchange that logs users in, while the box still boots, serves
	127	+	# pages, and answers 200 here. mt deliberately does not degrade `status` for it
	128	+	# (a bad bundle is a whole-fleet condition, and failing the load-balancer check
	129	+	# would turn a login outage into a total one), so this assertion is the only
	130	+	# thing that makes the condition visible before a user finds it.
	131	+	json_fields = { "status" = "operational", "database" = "true", "tls_trust_anchors" = "true" }
125	132
126	133		[targets.mt.tls]
127	134		host = "forums.makenot.work"

M multithreaded/src/routes/mod.rs +28 -4

			@@ -520,7 +520,10 @@
520	520		.await
521	521		.is_ok();
522	522
523		-	(health_status(db_ok), Json(health_body(db_ok)))
	523	+	(
	524	+	health_status(db_ok),
	525	+	Json(health_body(db_ok, crate::trust_store::anchors_ok())),
	526	+	)
524	527		}
525	528
526	529		/// Map DB reachability to the HTTP status. Pure so the status contract can be
			@@ -539,7 +542,7 @@
539	542		/// test in this module can exercise it directly. PoM polls this endpoint
540	543		/// and runs key-by-key assertions from `pom/deploy/pom-hetzner.toml`; the
541	544		/// guard test validates that every asserted path still resolves here.
542		-	fn health_body(db_ok: bool) -> serde_json::Value {
	545	+	fn health_body(db_ok: bool, trust_anchors_ok: bool) -> serde_json::Value {
543	546		let status = if db_ok { "operational" } else { "degraded" };
544	547		serde_json::json!({
545	548		"status": status,
			@@ -549,6 +552,13 @@
549	552		// same-semver redeploy, which `version` alone cannot distinguish.
550	553		"git_sha": option_env!("GIT_HASH").filter(\|h\| !h.is_empty()),
551	554		"database": db_ok,
	555	+	// Whether the host trust store yielded outbound TLS anchors
	556	+	// (`crate::trust_store`). Deliberately does not move `status` or the
	557	+	// HTTP code: mt still serves every page that does not leave the box,
	558	+	// and a bad CA bundle is usually a whole-fleet condition, so failing
	559	+	// the load-balancer check would turn a login outage into a total one.
	560	+	// PoM asserts this field, which is what gets it monitored.
	561	+	"tls_trust_anchors": trust_anchors_ok,
552	562		})
553	563		}
554	564
			@@ -594,7 +604,7 @@
594	604		#[test]
595	605		#[ignore = "cross-repo: run by sweep's pom-contract check, which materializes pom"]
596	606		fn pom_hetzner_health_expectations_resolve() {
597		-	let body = health_body(true);
	607	+	let body = health_body(true, true);
598	608		pom_contract::assert_health_expectations_resolve(
599	609		"../pom/deploy/pom-hetzner.toml",
600	610		"mt",
			@@ -608,7 +618,7 @@
608	618		/// than in PoM's exact-match `json_fields`.
609	619		#[test]
610	620		fn health_body_carries_version_and_git_sha_keys() {
611		-	let body = health_body(true);
	621	+	let body = health_body(true, true);
612	622		assert_eq!(body["version"], env!("CARGO_PKG_VERSION"));
613	623		assert!(
614	624		body.get("git_sha").is_some(),
			@@ -616,6 +626,20 @@
616	626		);
617	627		}
618	628
	629	+	/// The trust-anchor field is what PoM watches for a stale or thin host CA
	630	+	/// bundle, so lock both that it is reported and that it does not move
	631	+	/// `status`. A box with no anchors still serves every page that does not
	632	+	/// leave it; degrading the whole target would hide that distinction.
	633	+	#[test]
	634	+	fn health_body_reports_trust_anchors_without_moving_status() {
	635	+	assert_eq!(health_body(true, true)["tls_trust_anchors"], true);
	636	+
	637	+	let body = health_body(true, false);
	638	+	assert_eq!(body["tls_trust_anchors"], false);
	639	+	assert_eq!(body["status"], "operational");
	640	+	assert_eq!(health_status(true), StatusCode::OK);
	641	+	}
	642	+
619	643		/// A reachable DB is `200`; an unreachable DB is `503` so status-only probes
620	644		/// don't read a degraded box as healthy. PoM expects `200` for operational.
621	645		#[test]

M multithreaded/tests/workflows/health.rs +1

			@@ -8,5 +8,6 @@
8	8		let body: serde_json::Value = resp.json();
9	9		assert_eq!(body["status"], "operational");
10	10		assert_eq!(body["database"], true);
	11	+	assert_eq!(body["tls_trust_anchors"], true);
11	12		assert!(body["version"].as_str().is_some());
12	13		}

A multithreaded/src/trust_store.rs +77

		@@ -1,0 +1,77 @@
1	+	//! Whether this host has usable outbound TLS trust anchors.
2	+	//!
3	+	//! mt ships none of its own. `reqwest` verifies through
4	+	//! `rustls-platform-verifier`, which on Linux loads the host store with
5	+	//! `rustls_native_certs::load_native_certs`, and the AWS client used by
6	+	//! `s3-storage` calls the same crate directly. Neither offers bundled roots, so
7	+	//! a thin or unreadable store takes out all three outbound paths at once: the
8	+	//! OAuth token exchange that logs users in, link previews, and S3 media.
9	+	//!
10	+	//! Left alone, that failure is invisible until someone tries to log in, on a box
11	+	//! that boots fine and passes a database-only health check. Probing the same call
12	+	//! the verifier will make turns it into a boot-time log line and an
13	+	//! `/api/health` field instead. The precondition itself is documented in
14	+	//! `deploy/README.md`.
15	+
16	+	use std::sync::OnceLock;
17	+
18	+	static ANCHORS_OK: OnceLock<bool> = OnceLock::new();
19	+
20	+	/// Whether the host trust store yielded any usable anchors.
21	+	///
22	+	/// Probed once per process and cached, so `main` can force the boot-time log and
23	+	/// the health handler can read the answer on every poll without reloading the
24	+	/// store. Called from the health handler, which is why it is not `main`-only.
25	+	pub fn anchors_ok() -> bool {
26	+	*ANCHORS_OK.get_or_init(probe)
27	+	}
28	+
29	+	/// Load the host store and judge it the way the verifier does.
30	+	///
31	+	/// Emptiness is the only failure. A store that yields some anchors and some
32	+	/// parse errors is what the platform verifier itself accepts (it logs the
33	+	/// ignored certificates and builds anyway), so treating partial success as a
34	+	/// failure here would report a problem mt does not have.
35	+	fn probe() -> bool {
36	+	let result = rustls_native_certs::load_native_certs();
37	+
38	+	for error in &result.errors {
39	+	tracing::warn!("trust store read error: {error}");
40	+	}
41	+
42	+	if result.certs.is_empty() {
43	+	tracing::error!(
44	+	"no CA certificates loaded from the host trust store. Every outbound \
45	+	TLS request will fail, including the OAuth token exchange that logs \
46	+	users in. Install or repair ca-certificates on this host; see \
47	+	deploy/README.md."
48	+	);
49	+	false
50	+	} else {
51	+	tracing::info!(
52	+	anchors = result.certs.len(),
53	+	"loaded CA certificates from the host trust store"
54	+	);
55	+	true
56	+	}
57	+	}
58	+
59	+	#[cfg(test)]
60	+	mod tests {
61	+	use super::anchors_ok;
62	+
63	+	/// Any developer or CI machine has a trust store, so this asserts the probe
64	+	/// reads one rather than asserting a hardcoded answer. It would fail on a
65	+	/// host that cannot make an outbound HTTPS request at all, which is the
66	+	/// condition worth failing on.
67	+	#[test]
68	+	fn probe_finds_anchors_on_this_host() {
69	+	assert!(anchors_ok(), "no CA anchors loaded from the host store");
70	+	}
71	+
72	+	/// The answer is cached, so repeated reads must agree and must not reload.
73	+	#[test]
74	+	fn repeated_reads_agree() {
75	+	assert_eq!(anchors_ok(), anchors_ok());
76	+	}
77	+	}