Skip to main content

max / makenotwork

8.6 KB · 187 lines History Blame Raw
1 #!/usr/bin/env bash
2 # Idempotent bootstrap for a pom node under Sando. Run on the node as root.
3 #
4 # pom used to be installed by Bento straight into /usr/local/bin/pom. Under the
5 # Sando/Bento boundary Bento builds and hands off, and Sando deploys the way it
6 # deploys everything else: a content-addressed bundle under <release_root>,
7 # an atomic `current` symlink, and `systemctl reload-or-restart` on the unit.
8 #
9 # /opt/pom/
10 # releases/<digest16>/{pom,MANIFEST}
11 # current -> releases/<digest16>
12 #
13 # Three things have to be true on the node before the first promote, and none of
14 # them were:
15 #
16 # 1. <release_root> exists and the user Sando SSHes in as can write it.
17 # sandod runs `mkdir -p <release_root>/releases/<id>` as that user, and /opt
18 # is root-owned.
19 # 2. That user may restart the unit: sandod runs
20 # `sudo /bin/systemctl reload-or-restart pom.service`.
21 # 3. The unit's ExecStart points at <release_root>/current/<bin>. This is the
22 # one that fails silently rather than loudly: a promote to a node whose unit
23 # still runs /usr/local/bin/pom rsyncs the bundle, swaps the symlink,
24 # restarts the service onto the OLD binary, and passes node_health. Sando
25 # reports the version shipped and nothing shipped.
26 #
27 # The ExecStart move is a drop-in, not an edit of pom.service. A deploy never
28 # rewrites the unit (deploy/README.md, "What a deploy does not touch"), and the
29 # same restraint applies here: the override is one file, it is legible next to
30 # the unit it changes, and removing it puts the node back.
31 #
32 # Ordering is why this is a script rather than a runbook. Repointing ExecStart
33 # before a bundle exists leaves the unit pointing at a dangling symlink until the
34 # first promote, and any restart in between (a reboot, Restart=on-failure) takes
35 # pom down for however long that is. So the currently-installed binary is seeded
36 # as the first release and `current` points at it BEFORE the unit moves. The node
37 # comes out of this on the same version it went in on, deployable.
38 #
39 # Usage:
40 # sudo DEPLOY_USER=max SANDO_PUBKEY="$(ssh fw13 'sudo cat /srv/sando/.ssh/id_ed25519.pub')" \
41 # ./bootstrap-pom-node.sh
42 #
43 # Required env:
44 # DEPLOY_USER — the user in the node's `ssh_target` in pom-topology.toml
45 # (`max` on astra, `root` on the Hetzner box).
46 #
47 # Optional env:
48 # SANDO_PUBKEY — sando's public key on the Sando host, appended to
49 # DEPLOY_USER's authorized_keys. Omit on a host reached by
50 # Tailscale SSH, which does not consult authorized_keys.
51 # RELEASE_ROOT — defaults to /opt/pom. Must match the node's `release_root`.
52 # BIN_NAME — defaults to pom. Must match sando-pom.toml's `bin_names[0]`.
53 # SERVICE_NAME — defaults to pom.service.
54 # CONFIG_PATH — defaults to /etc/pom/pom.toml. The unit's --config argument.
55 # HEALTH_URL — defaults to http://127.0.0.1:9100/api/health.
56 #
57 # What this does NOT do, and will not:
58 #
59 # Config. pom-astra.toml and pom-hetzner.toml differ per instance and the live
60 # copies have carried blocks this repo did not. Config is a deliberate act.
61 #
62 # Remove the old Bento install path. /usr/local/bin/pom, its `.prev`, the
63 # /usr/local/lib/bento installer and its sudoers grant are all left in place:
64 # until a promote has actually landed on this node, that binary is what the
65 # rollback is. Clearing them is a separate pass after the cutover, and the
66 # script prints the paths so they are not forgotten.
67
68 set -euo pipefail
69
70 if [[ $EUID -ne 0 ]]; then
71 echo "must run as root" >&2
72 exit 1
73 fi
74 if [[ -z "${DEPLOY_USER:-}" ]]; then
75 echo "DEPLOY_USER env var is required (the user in this node's ssh_target)" >&2
76 exit 1
77 fi
78 if ! id "$DEPLOY_USER" &>/dev/null; then
79 echo "no such user: $DEPLOY_USER" >&2
80 exit 1
81 fi
82
83 RELEASE_ROOT="${RELEASE_ROOT:-/opt/pom}"
84 BIN_NAME="${BIN_NAME:-pom}"
85 SERVICE_NAME="${SERVICE_NAME:-pom.service}"
86 CONFIG_PATH="${CONFIG_PATH:-/etc/pom/pom.toml}"
87 HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:9100/api/health}"
88 DROPIN_DIR="/etc/systemd/system/${SERVICE_NAME}.d"
89 DROPIN="${DROPIN_DIR}/20-release-root.conf"
90
91 log() { echo "[bootstrap-pom] $*"; }
92
93 log "1/6 release root at $RELEASE_ROOT, writable by $DEPLOY_USER"
94 install -d -o "$DEPLOY_USER" -g "$(id -gn "$DEPLOY_USER")" -m 0755 \
95 "$RELEASE_ROOT" "$RELEASE_ROOT/releases"
96
97 log "2/6 seeding the running binary as a release, so the unit never points at nothing"
98 if [[ -L "$RELEASE_ROOT/current" ]]; then
99 log " current -> $(readlink "$RELEASE_ROOT/current") (already seeded, left alone)"
100 else
101 # Whatever the unit runs today, resolved from its own ExecStart rather than
102 # assumed to be /usr/local/bin/pom.
103 running="$(systemctl show -p ExecStart --value "$SERVICE_NAME" \
104 | sed -n 's/.*path=\([^ ;]*\).*/\1/p' | head -1)"
105 if [[ -z "$running" || ! -x "$running" ]]; then
106 echo "cannot find the binary $SERVICE_NAME currently runs (ExecStart=$running)." >&2
107 echo "Seed $RELEASE_ROOT/current by hand, or promote before repointing the unit." >&2
108 exit 4
109 fi
110 ver="$("$running" --version 2>/dev/null | awk '{print $NF}')"
111 seed="$RELEASE_ROOT/releases/preexisting-${ver:-unknown}"
112 install -d -o "$DEPLOY_USER" -g "$(id -gn "$DEPLOY_USER")" -m 0755 "$seed"
113 install -o "$DEPLOY_USER" -g "$(id -gn "$DEPLOY_USER")" -m 0755 "$running" "$seed/$BIN_NAME"
114 # No MANIFEST: this bundle came from the node, not from an intake, and a
115 # hand-written one would claim an artifact record that does not exist.
116 # Sando skips verification for a bundle without one and says so in the log.
117 ln -sfn "releases/preexisting-${ver:-unknown}" "$RELEASE_ROOT/current.new"
118 mv -Tf "$RELEASE_ROOT/current.new" "$RELEASE_ROOT/current"
119 chown -h "$DEPLOY_USER:$(id -gn "$DEPLOY_USER")" "$RELEASE_ROOT/current"
120 log " seeded $running (${ver:-unknown}) -> $RELEASE_ROOT/current"
121 fi
122
123 log "3/6 sudo grant: $DEPLOY_USER may restart $SERVICE_NAME"
124 if [[ "$DEPLOY_USER" == "root" ]]; then
125 log " root needs no grant; skipping (sudo must still be installed)"
126 command -v sudo >/dev/null || { echo "sudo is not installed; sandod shells out to it" >&2; exit 5; }
127 else
128 cat > "/etc/sudoers.d/${DEPLOY_USER}-pom" <<EOF
129 $DEPLOY_USER ALL=(ALL) NOPASSWD: /bin/systemctl reload-or-restart $SERVICE_NAME, /bin/systemctl restart $SERVICE_NAME, /bin/systemctl status $SERVICE_NAME
130 EOF
131 chmod 0440 "/etc/sudoers.d/${DEPLOY_USER}-pom"
132 visudo -c -f "/etc/sudoers.d/${DEPLOY_USER}-pom" >/dev/null
133 fi
134
135 log "4/6 sando's key for $DEPLOY_USER"
136 if [[ -n "${SANDO_PUBKEY:-}" ]]; then
137 home="$(getent passwd "$DEPLOY_USER" | cut -d: -f6)"
138 install -d -o "$DEPLOY_USER" -g "$(id -gn "$DEPLOY_USER")" -m 0700 "$home/.ssh"
139 if ! grep -qF "$SANDO_PUBKEY" "$home/.ssh/authorized_keys" 2>/dev/null; then
140 echo "$SANDO_PUBKEY" >> "$home/.ssh/authorized_keys"
141 fi
142 chown "$DEPLOY_USER:$(id -gn "$DEPLOY_USER")" "$home/.ssh/authorized_keys"
143 chmod 0600 "$home/.ssh/authorized_keys"
144 else
145 log " SANDO_PUBKEY unset; assuming Tailscale SSH reaches this node"
146 fi
147
148 log "5/6 ExecStart -> $RELEASE_ROOT/current/$BIN_NAME (drop-in, unit untouched)"
149 install -d -m 0755 "$DROPIN_DIR"
150 cat > "$DROPIN" <<EOF
151 # Written by pom/deploy/bootstrap-pom-node.sh. Sando deploys pom as a
152 # content-addressed bundle and swaps <release_root>/current; the unit has to
153 # follow the symlink or a promote restarts the service onto the old binary and
154 # still reports success.
155 #
156 # A drop-in rather than an edit of pom.service, so the release path is one
157 # reversible file and the hardened unit stays the unit this repo ships.
158 [Service]
159 ExecStart=
160 ExecStart=$RELEASE_ROOT/current/$BIN_NAME serve --config $CONFIG_PATH
161 EOF
162 systemctl daemon-reload
163
164 log "6/6 restart and prove the node is serving from the new path"
165 systemctl reload-or-restart "$SERVICE_NAME"
166 for _ in $(seq 1 30); do
167 if curl -fsS --max-time 3 "$HEALTH_URL" >/dev/null 2>&1; then
168 served=1
169 break
170 fi
171 sleep 1
172 done
173 if [[ "${served:-0}" != 1 ]]; then
174 echo "$SERVICE_NAME did not answer $HEALTH_URL within 30s." >&2
175 echo "Roll back by removing $DROPIN and running: systemctl daemon-reload && systemctl restart $SERVICE_NAME" >&2
176 exit 6
177 fi
178
179 echo
180 log "Done. $SERVICE_NAME is running $("$RELEASE_ROOT/current/$BIN_NAME" --version 2>/dev/null)"
181 log "Sando can now promote to this node: POST /apps/pom/promote/<tier>"
182 echo
183 log "Left in place on purpose, until a promote has landed here:"
184 log " /usr/local/bin/$BIN_NAME (+ .prev) the old Bento install path, and today's rollback"
185 log " /usr/local/lib/bento/install-service.sh + /etc/sudoers.d/bento-deploy"
186 log "Clear both in a separate pass once this node has taken a Sando release."
187