max / makenotwork

pom: probe astra's health where pom actually listens, not node_exporter

Author: Max Johnson <me@maxj.phd> · 2026-08-16 01:40 UTC

Signed with PGP, not checked

Commit: c05f3b3f4e78e076f23e558c3dca4d6b22e99497

Parent: 1817149

1 file changed, +25 insertions, -9 deletions

M sando/pom-topology.toml +25 -9

			@@ -65,7 +65,15 @@
65	65		# Readiness on top of `systemctl is-active`: pom binds its API on the tailnet
66	66		# address, and the dashboard is on. A crash-looping binary satisfies is-active
67	67		# between restarts, which is the failure this closes.
68		-	health_url = "http://127.0.0.1:9100/api/health"
	68	+	#
	69	+	# The tailnet name, NOT 127.0.0.1, and the difference is the gate working or
	70	+	# only appearing to. The probe runs on the node, astra's pom binds
	71	+	# 100.106.221.39:9100 (its `listen`), and 127.0.0.1:9100 there is
	72	+	# prometheus-node-exporter, which answers 200 with its index page for ANY path.
	73	+	# So the loopback URL passed this gate whether or not pom was running at all.
	74	+	# Measured 2026-08-15, before the first promote to this node. The Hetzner
	75	+	# instance binds 0.0.0.0 and is not affected.
	76	+	health_url = "http://astra:9100/api/health"
69	77
70	78		# ---- hetzner: x86_64, watching production from production ----
71	79		[[tier]]
			@@ -79,14 +87,22 @@
79	87		[[tier.node]]
80	88		name = "hetzner"
81	89		platform = "linux/x86_64"
82		-	# Not Tailscale SSH, whatever this line used to say. sandod runs as the `sando`
83		-	# user on fw13 and that user's ~/.ssh/config maps `alpha-west-1` to port 2200,
84		-	# which is the box's own sshd; Tailscale SSH is 22 and never sees the
85		-	# connection. So this needs sando's public key in root's authorized_keys there,
86		-	# and it is not in it today: measured 2026-08-15, `Permission denied
87		-	# (publickey)`. mnw reaches the same machine as `makenotwork@alpha-west-1`, a
88		-	# service user with the key and a scoped systemctl grant, which is the house
89		-	# pattern if this one wants revisiting.
	90	+	# STILL root@, and it does not work: measured 2026-08-15, `Permission denied
	91	+	# (publickey)`. Not Tailscale SSH, whatever this line used to say. sandod runs
	92	+	# as the `sando` user on fw13 and that user's ~/.ssh/config maps `alpha-west-1`
	93	+	# to port 2200, which is the box's own sshd; Tailscale SSH is 22 and never sees
	94	+	# the connection, so reaching root this way would mean sando's key in root's
	95	+	# authorized_keys on a production box.
	96	+	#
	97	+	# DECIDED 2026-08-15 (Max): become `pom@alpha-west-1` instead, matching
	98	+	# `makenotwork@alpha-west-1` on the same machine — a service user holding
	99	+	# sando's key and a sudoers grant scoped to `systemctl reload-or-restart
	100	+	# pom.service`, and no root login for a deploy daemon anywhere.
	101	+	#
	102	+	# Left as root@ until that account exists, because the `pom` user there is a
	103	+	# nologin system account today and a target nothing can log into is worse than
	104	+	# one that is honestly wrong. The rename and the account land in one pass, with
	105	+	# `bootstrap-pom-node.sh DEPLOY_USER=pom` doing the node half.
90	106		ssh_target = "root@alpha-west-1"
91	107		release_root = "/opt/pom"
92	108		service_name = "pom.service"