| 1 |
# pom's tiers and nodes. Design: wiki [[sando-bento-boundary]]. |
| 2 |
# |
| 3 |
# No [repo] and no [[backup]]-driven migration gate. pom is intake-only: Bento |
| 4 |
# builds it natively on astra and on the Hetzner box, and hands Sando finished |
| 5 |
# bytes with an artifact record. Sando fetches no source for it, so there is no |
| 6 |
# bare repo on this host to name, and declaring one would be a claim /rebuild |
| 7 |
# would then act on. |
| 8 |
backup = [] |
| 9 |
|
| 10 |
# ---- host: where an accepted artifact is gated before it goes anywhere ---- |
| 11 |
# |
| 12 |
# Not a machine pom runs on. It is the tier every intake lands on, the same way |
| 13 |
# a Sando-built product's host tier is where its build lands: gates run here, and |
| 14 |
# a green run is what makes the bundle promotable. sandod's own box (fw13) is not |
| 15 |
# a pom node and never runs pom. |
| 16 |
# |
| 17 |
# The gate list is empty on purpose, and that is the boundary rather than an |
| 18 |
# omission. Everything Sando would have run here is artifact-scoped — cargo_test, |
| 19 |
# clippy, fmt, the audits — and artifact-scoped evidence belongs to the builder |
| 20 |
# under this split. Bento already ran them, and the record carries their verdicts |
| 21 |
# (`intake::accept` reports whether the builder's gates passed). Asking Sando to |
| 22 |
# re-run them would need a source checkout it does not have, and the gates say so |
| 23 |
# rather than passing on having run nothing. |
| 24 |
# |
| 25 |
# What is left for Sando is evidence about the artifact *in an environment*, and |
| 26 |
# every one of those lives on a tier with a node in it, below. |
| 27 |
[[tier]] |
| 28 |
name = "host" |
| 29 |
provisioned = true |
| 30 |
canary = "sequential" |
| 31 |
gates = [] |
| 32 |
|
| 33 |
# ---- astra: aarch64, and pom's own CI runner ---- |
| 34 |
# |
| 35 |
# First real tier because it is the machine that matters least if pom is down for |
| 36 |
# a moment: it watches, it does not serve anyone. The Hetzner instance keeps |
| 37 |
# watching while this one restarts. |
| 38 |
# |
| 39 |
# The bootstrap caveat is why the two are a ladder and not a fan-out: pom watches |
| 40 |
# its own deploy, so deploying pom restarts the watcher mid-operation. One |
| 41 |
# instance at a time, health read from the peer over the mesh, never both at once. |
| 42 |
[[tier]] |
| 43 |
name = "astra" |
| 44 |
provisioned = true |
| 45 |
canary = "sequential" |
| 46 |
# Guards astra -> hetzner. node_health proves the restarted instance is actually |
| 47 |
# serving; burn_in makes production wait on a real interval of the new binary |
| 48 |
# running somewhere; manual_confirm is the human sign-off before the box that |
| 49 |
# watches production takes it. |
| 50 |
gates = [ |
| 51 |
{ kind = "node_health" }, |
| 52 |
{ kind = "burn_in", hours = 24 }, |
| 53 |
{ kind = "manual_confirm" }, |
| 54 |
] |
| 55 |
[[tier.node]] |
| 56 |
name = "astra" |
| 57 |
platform = "linux/aarch64" |
| 58 |
# Tailnet name, reached over Tailscale SSH. The connection is made by the |
| 59 |
# `sando` user on fw13, so astra's host key has to be in /srv/sando/.ssh/ |
| 60 |
# known_hosts; without it the deploy fails at `Host key verification failed` |
| 61 |
# before it has done anything, which is where it stood on 2026-08-15. |
| 62 |
ssh_target = "max@astra" |
| 63 |
release_root = "/opt/pom" |
| 64 |
service_name = "pom.service" |
| 65 |
# Readiness on top of `systemctl is-active`: pom binds its API on the tailnet |
| 66 |
# address, and the dashboard is on. A crash-looping binary satisfies is-active |
| 67 |
# between restarts, which is the failure this closes. |
| 68 |
# |
| 69 |
# The tailnet name, NOT 127.0.0.1, and the difference is the gate working or |
| 70 |
# only appearing to. The probe runs on the node, astra's pom binds |
| 71 |
# 100.106.221.39:9100 (its `listen`), and 127.0.0.1:9100 there is |
| 72 |
# prometheus-node-exporter, which answers 200 with its index page for ANY path. |
| 73 |
# So the loopback URL passed this gate whether or not pom was running at all. |
| 74 |
# Measured 2026-08-15, before the first promote to this node. The Hetzner |
| 75 |
# instance binds 0.0.0.0 and is not affected. |
| 76 |
health_url = "http://astra:9100/api/health" |
| 77 |
|
| 78 |
# ---- hetzner: x86_64, watching production from production ---- |
| 79 |
[[tier]] |
| 80 |
name = "hetzner" |
| 81 |
provisioned = true |
| 82 |
canary = "sequential" |
| 83 |
gates = [ |
| 84 |
{ kind = "node_health" }, |
| 85 |
{ kind = "manual_confirm" }, |
| 86 |
] |
| 87 |
[[tier.node]] |
| 88 |
name = "hetzner" |
| 89 |
platform = "linux/x86_64" |
| 90 |
# `pom@alpha-west-1` as of 2026-08-16, and the account it names now exists. |
| 91 |
# This was `root@alpha-west-1`, which never worked: sandod runs as the `sando` |
| 92 |
# user on fw13 and that user's ~/.ssh/config maps `alpha-west-1` to port 2200, |
| 93 |
# the box's own sshd. Tailscale SSH is 22 and never saw the connection, so |
| 94 |
# reaching root that way would have meant sando's key in root's authorized_keys |
| 95 |
# on a production box. |
| 96 |
# |
| 97 |
# The account matches `makenotwork@alpha-west-1` on the same machine, which is |
| 98 |
# the house pattern: the service user the unit already runs as, holding sando's |
| 99 |
# key and a sudoers grant scoped to the three systemctl verbs sandod uses. It |
| 100 |
# was a nologin system account until the cutover gave it a home and a shell. |
| 101 |
# Sando's key is pinned there with `restrict,from="100.103.89.95"`, which is |
| 102 |
# tighter than makenotwork's bare key line and still allows the rsync leg. |
| 103 |
# |
| 104 |
# Node half done by `bootstrap-pom-node.sh DEPLOY_USER=pom`: /opt/pom owned by |
| 105 |
# pom, the running 0.4.1 seeded as releases/preexisting-0.4.1 with `current` |
| 106 |
# pointing at it, ExecStart moved by /etc/systemd/system/pom.service.d/ |
| 107 |
# 20-release-root.conf. The node came out of it on the version it went in on. |
| 108 |
ssh_target = "pom@alpha-west-1" |
| 109 |
release_root = "/opt/pom" |
| 110 |
service_name = "pom.service" |
| 111 |
# Loopback is honest here, unlike astra's: this instance binds 0.0.0.0, and |
| 112 |
# 127.0.0.1:9100 on the node answers with pom's own health JSON rather than |
| 113 |
# some other daemon's index page. Verified 2026-08-16, before the first promote. |
| 114 |
health_url = "http://127.0.0.1:9100/api/health" |
| 115 |
|