Skip to main content

max / makenotwork

13.8 KB · 315 lines History Blame Raw
1 #!/usr/bin/env bash
2 # Idempotent bootstrap for a fresh Sando host (the machine running sandod).
3 #
4 # Captures the three PG footguns + system user + systemd unit + scratch DB +
5 # .ssh setup + known_hosts seeding that fw13 accumulated by hand over the
6 # 2026-06-02 hardening session. Re-run any time the sando host is rebuilt.
7 #
8 # Run as root on the new host. The script is safe to run repeatedly — every
9 # step checks current state and skips if already satisfied.
10 #
11 # What it does:
12 # 1. base packages + postgresql
13 # 2. `sando` system user (login shell, /srv/sando home)
14 # 3. /srv/sando dirs (state/, work/, releases/, logs/, backups/)
15 # 4. postgres role `sando` with CREATEDB
16 # 5. `sando_scratch` database owned by `sando`
17 # 6. ALTER SCHEMA public OWNER TO sando inside sando_scratch
18 # (must be set explicitly — PG15+ no longer grants public to db owner)
19 # 7. sando's ed25519 SSH key (generated if missing)
20 # 8. /srv/sando/.ssh/config — declares port 2200 for alpha-west-1
21 # 9. known_hosts seeded for tailnet targets (testnot, alpha-west-1, etc.)
22 # 10. /etc/sando/{sando-daemon.toml,sando.toml,sando.env}
23 # 11. /etc/systemd/system/sandod.service + sandod-backup-fetch.{service,timer}
24 # 12. /usr/local/bin/sandod (built from the local checkout if missing)
25 # 13. /srv/sando/mnw.git bare repo (initial; operator pushes the working tree)
26 #
27 # What this does NOT do (operator's job):
28 # - tailscale up (auth)
29 # - Authorize sando's pubkey on each deploy target's `deploy` user
30 # (bootstrap-node.sh on the target consumes $SANDO_PUBKEY)
31 # - Populate /etc/sando/sando.env with anything beyond SANDO_DAEMON if
32 # additional secrets are needed
33 # - Push the MNW working tree to /srv/sando/mnw.git (only needed for
34 # push-based hosts; pull-based hosts set [repo].upstream in sando.toml and
35 # sandod fetches on /rebuild)
36 #
37 # Note: the old `mnw_test_template` ownership footgun (a template left owned by
38 # a different login wedged the cargo_test gate) is handled by step 6b — the
39 # harness `SET ROLE mnw_test` so all test DBs share one owner that every member
40 # can drop. No more per-user ownership resets.
41
42 set -euo pipefail
43
44 if [[ $EUID -ne 0 ]]; then
45 echo "must run as root" >&2
46 exit 1
47 fi
48
49 # All paths the host should accept overrides for, with sane defaults that
50 # match the live fw13 install.
51 SANDO_USER="${SANDO_USER:-sando}"
52 SANDO_HOME="${SANDO_HOME:-/srv/sando}"
53 SANDO_DAEMON_URL="${SANDO_DAEMON_URL:-http://127.0.0.1:7766}"
54 INSTALL_POSTGRES="${INSTALL_POSTGRES:-1}"
55 BUILD_SANDOD="${BUILD_SANDOD:-1}"
56
57 # Tailnet targets to pre-seed in sando's known_hosts. Override SEED_HOSTS to
58 # add/remove. Each entry is "name[:port]"; port defaults to 22.
59 SEED_HOSTS="${SEED_HOSTS:-testnot alpha-west-1:2200}"
60
61 # Roles to add to the shared `mnw_test` role (owns all test databases). A pure
62 # sando host needs only `sando`; a shared dev box should also list the human
63 # role, e.g. TEST_DB_MEMBERS="sando max".
64 TEST_DB_MEMBERS="${TEST_DB_MEMBERS:-$SANDO_USER}"
65
66 # Resolve the script's directory so it can copy sibling unit/config files
67 # without depending on cwd. Layout: `<SANDO_REPO>/deploy/this-script.sh`,
68 # so SANDO_REPO is one level up.
69 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
70 SANDO_REPO="$(cd "$SCRIPT_DIR/.." && pwd)"
71
72 export DEBIAN_FRONTEND=noninteractive
73
74 log() { echo "[bootstrap-sandod] $*"; }
75
76 log "1/13 base packages"
77 apt-get update -qq
78 apt-get install -y -qq curl ca-certificates rsync openssh-client git build-essential pkg-config libssl-dev > /dev/null
79
80 if [[ "$INSTALL_POSTGRES" == "1" ]]; then
81 log "2/13 postgresql"
82 apt-get install -y -qq postgresql > /dev/null
83 else
84 log "2/13 skipping postgresql"
85 fi
86
87 log "3/13 sando system user (home: $SANDO_HOME)"
88 if ! id "$SANDO_USER" &>/dev/null; then
89 useradd -m -d "$SANDO_HOME" -s /bin/bash "$SANDO_USER"
90 fi
91 # Re-assert home dir + mode in case a prior partial run left it root-owned.
92 install -d -o "$SANDO_USER" -g "$SANDO_USER" -m 0750 "$SANDO_HOME"
93
94 log "4/13 /srv/sando subdirs"
95 # `staging` is where an artifact built elsewhere lands before intake proves it.
96 # The route resolves `release_root/staging` and refuses a bundle staged anywhere
97 # else, so a host without this directory answers every intake with an error
98 # about a missing path rather than about the artifact.
99 for sub in state work releases staging logs backups; do
100 install -d -o "$SANDO_USER" -g "$SANDO_USER" -m 0750 "$SANDO_HOME/$sub"
101 done
102
103 log "5/13 postgres role + scratch db"
104 # All postgres ops go through `sudo -u postgres` since the role/db live on the
105 # local cluster. Idempotency via CREATE … IF NOT EXISTS where supported, and
106 # DO blocks where it isn't (CREATE ROLE has no IF NOT EXISTS in older PG).
107 sudo -u postgres psql -v ON_ERROR_STOP=1 <<SQL
108 DO \$\$
109 BEGIN
110 IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '$SANDO_USER') THEN
111 EXECUTE format('CREATE ROLE %I LOGIN CREATEDB', '$SANDO_USER');
112 ELSE
113 EXECUTE format('ALTER ROLE %I CREATEDB', '$SANDO_USER');
114 END IF;
115 END
116 \$\$;
117 SQL
118
119 # CREATE DATABASE can't be inside a DO block, hence the separate guard.
120 # Only the primary scratch DB is created here. A `[[migration_check]]` with its
121 # own `scratch_db` (e.g. sando_scratch_mt) is created by the daemon at the start
122 # of that check — DROP + CREATE, so it is sando-owned and the PG15+ public-schema
123 # grants below are applied by reset_scratch rather than by hand. Adding a gated
124 # database must not depend on someone having remembered a step in this script.
125 if ! sudo -u postgres psql -tAc \
126 "SELECT 1 FROM pg_database WHERE datname = 'sando_scratch'" \
127 | grep -q '^1$'; then
128 sudo -u postgres createdb -O "$SANDO_USER" sando_scratch
129 fi
130
131 log "6/13 sando_scratch public schema owner = $SANDO_USER"
132 # PG15+ no longer grants public to the DB owner automatically. Without this,
133 # reset_scratch (sando/daemon/src/gates.rs::reset_scratch) silently fails
134 # every rebuild because the DROP SCHEMA public CASCADE happens but the
135 # CREATE SCHEMA public lands as postgres, not sando, owning it.
136 sudo -u postgres psql -v ON_ERROR_STOP=1 sando_scratch -c \
137 "ALTER SCHEMA public OWNER TO $SANDO_USER" >/dev/null
138
139 log "6b/13 shared mnw_test role (owns all test DBs; members can drop them)"
140 # The server integration harness (server/tests/harness/db.rs) does
141 # `SET ROLE mnw_test` before creating its template + per-test clones, so they
142 # are owned by this shared role no matter which login ran the suite. Every
143 # member can then drop any `mnw_test_*` DB — no superuser, no cross-user
144 # ownership wedge (the failure mode that stalled the 0.10.1 deploy).
145 sudo -u postgres psql -v ON_ERROR_STOP=1 <<SQL
146 DO \$\$
147 BEGIN
148 IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'mnw_test') THEN
149 CREATE ROLE mnw_test NOLOGIN CREATEDB;
150 ELSE
151 ALTER ROLE mnw_test NOLOGIN CREATEDB;
152 END IF;
153 END
154 \$\$;
155 SQL
156 for member in $TEST_DB_MEMBERS; do
157 if sudo -u postgres psql -tAc \
158 "SELECT 1 FROM pg_roles WHERE rolname = '$member'" | grep -q '^1$'; then
159 sudo -u postgres psql -v ON_ERROR_STOP=1 \
160 -c "GRANT mnw_test TO \"$member\"" >/dev/null
161 else
162 log " warn: role $member missing; skipping its mnw_test grant"
163 fi
164 done
165
166 log "7/13 sando ssh key (ed25519)"
167 install -d -o "$SANDO_USER" -g "$SANDO_USER" -m 0700 "$SANDO_HOME/.ssh"
168 if [[ ! -f "$SANDO_HOME/.ssh/id_ed25519" ]]; then
169 sudo -u "$SANDO_USER" ssh-keygen -t ed25519 -N "" \
170 -f "$SANDO_HOME/.ssh/id_ed25519" \
171 -C "sando@$(hostname -s)"
172 fi
173
174 log "8/13 /srv/sando/.ssh/config"
175 # Declare alpha-west-1 on port 2200 (prod sshd convention). bootstrap-node.sh
176 # on each deploy target accepts SANDO_PUBKEY so we don't need to manage the
177 # remote authorized_keys here.
178 ssh_config="$SANDO_HOME/.ssh/config"
179 if ! grep -q "^Host alpha-west-1" "$ssh_config" 2>/dev/null; then
180 cat >> "$ssh_config" <<'EOF'
181 Host alpha-west-1
182 Port 2200
183
184 EOF
185 fi
186 chown "$SANDO_USER:$SANDO_USER" "$ssh_config"
187 chmod 0600 "$ssh_config"
188
189 log "9/13 known_hosts seeding ($SEED_HOSTS)"
190 # Strict-host-key-check failures on first contact would block sandod's deploy
191 # step. Pre-seed each declared tier-node host. ssh-keyscan is idempotent
192 # (running it again just appends a duplicate; we de-dup via sort -u after).
193 known="$SANDO_HOME/.ssh/known_hosts"
194 touch "$known"
195 chown "$SANDO_USER:$SANDO_USER" "$known"
196 chmod 0600 "$known"
197 for entry in $SEED_HOSTS; do
198 host="${entry%%:*}"
199 port="${entry#*:}"
200 [[ "$port" == "$host" ]] && port=22
201 # `ssh-keyscan` returns the host keys without contacting the user; on
202 # unreachable hosts it logs a warning and exits 0. We tolerate that —
203 # operator can re-run after the target is up.
204 sudo -u "$SANDO_USER" ssh-keyscan -p "$port" -T 5 "$host" 2>/dev/null \
205 >> "$known" || log " warn: ssh-keyscan $host:$port returned nothing"
206 done
207 # De-dup in place. sort+mv keeps ownership/mode via install.
208 sudo -u "$SANDO_USER" sort -u "$known" -o "$known"
209
210 log "10/13 /etc/sando configs"
211 install -d -m 0755 /etc/sando
212 # sando-daemon.toml.example is the canonical production config (per the
213 # header comment). Install as-is; operator edits the listen address if
214 # binding to a non-fw13 tailnet IP.
215 install -m 0644 -o root -g root \
216 "$SCRIPT_DIR/sando-daemon.toml.example" \
217 /etc/sando/sando-daemon.toml
218 # The topology. Bootstrap is not the only writer any more: `sando-self-update.sh`
219 # reinstalls this file from the checked-out sha on every self-update, so the repo
220 # copy stays the deployed copy instead of separating from it between bootstraps.
221 # Do not hand-edit the installed file; `sando-config-drift.timer` below reports
222 # it when someone does.
223 install -m 0644 -o root -g root \
224 "$SANDO_REPO/sando.toml" \
225 /etc/sando/sando.toml
226 # sando.env carries operator settings AND the deploy-API bearer token consumed
227 # by sandod, the post-receive hook, and the backup-fetch timer. A token is
228 # generated per host so a fresh install is authenticated by default — required
229 # once the daemon binds a non-loopback address (it refuses to start otherwise).
230 # 0640 root:sando keeps it readable by the daemon + hook, not world.
231 #
232 # Ensure the file AND a token, separately. An earlier install (or an operator)
233 # may have created sando.env with only SANDO_DAEMON; without the per-key check
234 # the token would never be added and the daemon would crash-loop on a
235 # non-loopback bind ("SANDO_API_TOKEN is unset ... refusing to start").
236 if [[ ! -f /etc/sando/sando.env ]]; then
237 echo "SANDO_DAEMON=$SANDO_DAEMON_URL" > /etc/sando/sando.env
238 fi
239 if ! grep -q '^SANDO_API_TOKEN=' /etc/sando/sando.env; then
240 echo "SANDO_API_TOKEN=$(openssl rand -hex 32)" >> /etc/sando/sando.env
241 log " generated SANDO_API_TOKEN in /etc/sando/sando.env"
242 fi
243 chown root:"$SANDO_USER" /etc/sando/sando.env
244 chmod 0640 /etc/sando/sando.env
245
246 log "11/13 systemd units"
247 # The drift check reads the bare repo and the live topology, so it goes beside
248 # the units that run it rather than in /usr/local/bin with the daemon.
249 install -d -m 0755 /usr/local/lib/sando
250 install -m 0755 -o root -g root \
251 "$SCRIPT_DIR/check-topology-drift.sh" \
252 /usr/local/lib/sando/check-topology-drift.sh
253 install -m 0644 -o root -g root \
254 "$SCRIPT_DIR/sandod.service" \
255 /etc/systemd/system/sandod.service
256 install -m 0644 -o root -g root \
257 "$SCRIPT_DIR/sandod-backup-fetch.service" \
258 /etc/systemd/system/sandod-backup-fetch.service
259 install -m 0644 -o root -g root \
260 "$SCRIPT_DIR/sandod-backup-fetch.timer" \
261 /etc/systemd/system/sandod-backup-fetch.timer
262 install -m 0644 -o root -g root \
263 "$SCRIPT_DIR/sando-config-drift.service" \
264 /etc/systemd/system/sando-config-drift.service
265 install -m 0644 -o root -g root \
266 "$SCRIPT_DIR/sando-config-drift.timer" \
267 /etc/systemd/system/sando-config-drift.timer
268 systemctl daemon-reload
269
270 if [[ "$BUILD_SANDOD" == "1" ]]; then
271 log "12/13 sandod binary (cargo build --release -p sando-daemon → /usr/local/bin/sandod)"
272 daemon_dir="$SANDO_REPO/daemon"
273 if [[ ! -d "$daemon_dir" ]]; then
274 log " warn: cannot locate sando/daemon source at $daemon_dir; skipping build"
275 else
276 # Build from the workspace root so the binary lands in the shared
277 # sando/target; -p sando-daemon skips the TUI. Binary owned root, mode 755.
278 (cd "$SANDO_REPO" && cargo build --release --quiet -p sando-daemon)
279 install -m 0755 "$SANDO_REPO/target/release/sandod" /usr/local/bin/sandod
280 fi
281 else
282 log "12/13 skipping sandod build (BUILD_SANDOD=0)"
283 fi
284
285 log "13/13 bare mnw.git"
286 if [[ ! -d "$SANDO_HOME/mnw.git" ]]; then
287 sudo -u "$SANDO_USER" git init --bare --initial-branch=main "$SANDO_HOME/mnw.git" >/dev/null
288 fi
289 # No hook is installed here. sandod owns post-receive: ensure_bare_repo() writes
290 # it from the copy embedded in the binary (include_str! of hooks/post-receive)
291 # on every start, so the hook can never drift from the daemon that answers it.
292 #
293 # Bootstrap used to install its own second copy, deploy/post-receive, which had
294 # fallen behind: it sent no Authorization header, so against a token-configured
295 # daemon every push 401'd and no build triggered -- swallowed by the hook's own
296 # `|| echo 'sando: rebuild trigger failed'`. It self-healed on the next daemon
297 # start, which is precisely why nobody noticed. That copy is deleted.
298
299 # Enable services last so a partial bootstrap doesn't leave a service trying
300 # to start against an incomplete environment.
301 systemctl enable sandod.service >/dev/null 2>&1 || true
302 systemctl enable sandod-backup-fetch.timer >/dev/null 2>&1 || true
303 systemctl enable sando-config-drift.timer >/dev/null 2>&1 || true
304
305 echo
306 log "Done. Next steps for the operator:"
307 echo " - tailscale up (auth this node onto the tailnet)"
308 echo " - on each deploy target, run bootstrap-node.sh with:"
309 echo " SANDO_PUBKEY=\"\$(cat $SANDO_HOME/.ssh/id_ed25519.pub)\""
310 echo " - push the MNW working tree to $SANDO_HOME/mnw.git:"
311 echo " git remote add sando $SANDO_USER@<host>:$SANDO_HOME/mnw.git"
312 echo " git push sando main"
313 echo " - sudo systemctl start sandod"
314 echo " - sudo systemctl start sandod-backup-fetch.timer"
315