Skip to main content

max / makenotwork

12.2 KB · 288 lines History Blame Raw
1 #!/usr/bin/env bash
2 # Idempotent bootstrap for a fresh Sando host (the machine running sandod).
3 #
4 # Captures the three PG footguns + system user + systemd unit + scratch DB +
5 # .ssh setup + known_hosts seeding that fw13 accumulated by hand over the
6 # 2026-06-02 hardening session. Re-run any time the sando host is rebuilt.
7 #
8 # Run as root on the new host. The script is safe to run repeatedly — every
9 # step checks current state and skips if already satisfied.
10 #
11 # What it does:
12 # 1. base packages + postgresql
13 # 2. `sando` system user (login shell, /srv/sando home)
14 # 3. /srv/sando dirs (state/, work/, releases/, logs/, backups/)
15 # 4. postgres role `sando` with CREATEDB
16 # 5. `sando_scratch` database owned by `sando`
17 # 6. ALTER SCHEMA public OWNER TO sando inside sando_scratch
18 # (must be set explicitly — PG15+ no longer grants public to db owner)
19 # 7. sando's ed25519 SSH key (generated if missing)
20 # 8. /srv/sando/.ssh/config — declares port 2200 for alpha-west-1
21 # 9. known_hosts seeded for tailnet targets (testnot, alpha-west-1, etc.)
22 # 10. /etc/sando/{sando-daemon.toml,sando.toml,sando.env}
23 # 11. /etc/systemd/system/sandod.service + sandod-backup-fetch.{service,timer}
24 # 12. /usr/local/bin/sandod (built from the local checkout if missing)
25 # 13. /srv/sando/mnw.git bare repo (initial; operator pushes the working tree)
26 #
27 # What this does NOT do (operator's job):
28 # - tailscale up (auth)
29 # - Authorize sando's pubkey on each deploy target's `deploy` user
30 # (bootstrap-node.sh on the target consumes $SANDO_PUBKEY)
31 # - Populate /etc/sando/sando.env with anything beyond SANDO_DAEMON if
32 # additional secrets are needed
33 # - Push the MNW working tree to /srv/sando/mnw.git (only needed for
34 # push-based hosts; pull-based hosts set [repo].upstream in sando.toml and
35 # sandod fetches on /rebuild)
36 #
37 # Note: the old `mnw_test_template` ownership footgun (a template left owned by
38 # a different login wedged the cargo_test gate) is handled by step 6b — the
39 # harness `SET ROLE mnw_test` so all test DBs share one owner that every member
40 # can drop. No more per-user ownership resets.
41
42 set -euo pipefail
43
44 if [[ $EUID -ne 0 ]]; then
45 echo "must run as root" >&2
46 exit 1
47 fi
48
49 # All paths the host should accept overrides for, with sane defaults that
50 # match the live fw13 install.
51 SANDO_USER="${SANDO_USER:-sando}"
52 SANDO_HOME="${SANDO_HOME:-/srv/sando}"
53 SANDO_DAEMON_URL="${SANDO_DAEMON_URL:-http://127.0.0.1:7766}"
54 INSTALL_POSTGRES="${INSTALL_POSTGRES:-1}"
55 BUILD_SANDOD="${BUILD_SANDOD:-1}"
56
57 # Tailnet targets to pre-seed in sando's known_hosts. Override SEED_HOSTS to
58 # add/remove. Each entry is "name[:port]"; port defaults to 22.
59 SEED_HOSTS="${SEED_HOSTS:-testnot alpha-west-1:2200}"
60
61 # Roles to add to the shared `mnw_test` role (owns all test databases). A pure
62 # sando host needs only `sando`; a shared dev box should also list the human
63 # role, e.g. TEST_DB_MEMBERS="sando max".
64 TEST_DB_MEMBERS="${TEST_DB_MEMBERS:-$SANDO_USER}"
65
66 # Resolve the script's directory so it can copy sibling unit/config files
67 # without depending on cwd. Layout: `<SANDO_REPO>/deploy/this-script.sh`,
68 # so SANDO_REPO is one level up.
69 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
70 SANDO_REPO="$(cd "$SCRIPT_DIR/.." && pwd)"
71
72 export DEBIAN_FRONTEND=noninteractive
73
74 log() { echo "[bootstrap-sandod] $*"; }
75
76 log "1/13 base packages"
77 apt-get update -qq
78 apt-get install -y -qq curl ca-certificates rsync openssh-client git build-essential pkg-config libssl-dev > /dev/null
79
80 if [[ "$INSTALL_POSTGRES" == "1" ]]; then
81 log "2/13 postgresql"
82 apt-get install -y -qq postgresql > /dev/null
83 else
84 log "2/13 skipping postgresql"
85 fi
86
87 log "3/13 sando system user (home: $SANDO_HOME)"
88 if ! id "$SANDO_USER" &>/dev/null; then
89 useradd -m -d "$SANDO_HOME" -s /bin/bash "$SANDO_USER"
90 fi
91 # Re-assert home dir + mode in case a prior partial run left it root-owned.
92 install -d -o "$SANDO_USER" -g "$SANDO_USER" -m 0750 "$SANDO_HOME"
93
94 log "4/13 /srv/sando subdirs"
95 for sub in state work releases logs backups; do
96 install -d -o "$SANDO_USER" -g "$SANDO_USER" -m 0750 "$SANDO_HOME/$sub"
97 done
98
99 log "5/13 postgres role + scratch db"
100 # All postgres ops go through `sudo -u postgres` since the role/db live on the
101 # local cluster. Idempotency via CREATE … IF NOT EXISTS where supported, and
102 # DO blocks where it isn't (CREATE ROLE has no IF NOT EXISTS in older PG).
103 sudo -u postgres psql -v ON_ERROR_STOP=1 <<SQL
104 DO \$\$
105 BEGIN
106 IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '$SANDO_USER') THEN
107 EXECUTE format('CREATE ROLE %I LOGIN CREATEDB', '$SANDO_USER');
108 ELSE
109 EXECUTE format('ALTER ROLE %I CREATEDB', '$SANDO_USER');
110 END IF;
111 END
112 \$\$;
113 SQL
114
115 # CREATE DATABASE can't be inside a DO block, hence the separate guard.
116 if ! sudo -u postgres psql -tAc \
117 "SELECT 1 FROM pg_database WHERE datname = 'sando_scratch'" \
118 | grep -q '^1$'; then
119 sudo -u postgres createdb -O "$SANDO_USER" sando_scratch
120 fi
121
122 log "6/13 sando_scratch public schema owner = $SANDO_USER"
123 # PG15+ no longer grants public to the DB owner automatically. Without this,
124 # reset_scratch (sando/daemon/src/gates.rs::reset_scratch) silently fails
125 # every rebuild because the DROP SCHEMA public CASCADE happens but the
126 # CREATE SCHEMA public lands as postgres, not sando, owning it.
127 sudo -u postgres psql -v ON_ERROR_STOP=1 sando_scratch -c \
128 "ALTER SCHEMA public OWNER TO $SANDO_USER" >/dev/null
129
130 log "6b/13 shared mnw_test role (owns all test DBs; members can drop them)"
131 # The server integration harness (server/tests/harness/db.rs) does
132 # `SET ROLE mnw_test` before creating its template + per-test clones, so they
133 # are owned by this shared role no matter which login ran the suite. Every
134 # member can then drop any `mnw_test_*` DB — no superuser, no cross-user
135 # ownership wedge (the failure mode that stalled the 0.10.1 deploy).
136 sudo -u postgres psql -v ON_ERROR_STOP=1 <<SQL
137 DO \$\$
138 BEGIN
139 IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'mnw_test') THEN
140 CREATE ROLE mnw_test NOLOGIN CREATEDB;
141 ELSE
142 ALTER ROLE mnw_test NOLOGIN CREATEDB;
143 END IF;
144 END
145 \$\$;
146 SQL
147 for member in $TEST_DB_MEMBERS; do
148 if sudo -u postgres psql -tAc \
149 "SELECT 1 FROM pg_roles WHERE rolname = '$member'" | grep -q '^1$'; then
150 sudo -u postgres psql -v ON_ERROR_STOP=1 \
151 -c "GRANT mnw_test TO \"$member\"" >/dev/null
152 else
153 log " warn: role $member missing; skipping its mnw_test grant"
154 fi
155 done
156
157 log "7/13 sando ssh key (ed25519)"
158 install -d -o "$SANDO_USER" -g "$SANDO_USER" -m 0700 "$SANDO_HOME/.ssh"
159 if [[ ! -f "$SANDO_HOME/.ssh/id_ed25519" ]]; then
160 sudo -u "$SANDO_USER" ssh-keygen -t ed25519 -N "" \
161 -f "$SANDO_HOME/.ssh/id_ed25519" \
162 -C "sando@$(hostname -s)"
163 fi
164
165 log "8/13 /srv/sando/.ssh/config"
166 # Declare alpha-west-1 on port 2200 (prod sshd convention). bootstrap-node.sh
167 # on each deploy target accepts SANDO_PUBKEY so we don't need to manage the
168 # remote authorized_keys here.
169 ssh_config="$SANDO_HOME/.ssh/config"
170 if ! grep -q "^Host alpha-west-1" "$ssh_config" 2>/dev/null; then
171 cat >> "$ssh_config" <<'EOF'
172 Host alpha-west-1
173 Port 2200
174
175 EOF
176 fi
177 chown "$SANDO_USER:$SANDO_USER" "$ssh_config"
178 chmod 0600 "$ssh_config"
179
180 log "9/13 known_hosts seeding ($SEED_HOSTS)"
181 # Strict-host-key-check failures on first contact would block sandod's deploy
182 # step. Pre-seed each declared tier-node host. ssh-keyscan is idempotent
183 # (running it again just appends a duplicate; we de-dup via sort -u after).
184 known="$SANDO_HOME/.ssh/known_hosts"
185 touch "$known"
186 chown "$SANDO_USER:$SANDO_USER" "$known"
187 chmod 0600 "$known"
188 for entry in $SEED_HOSTS; do
189 host="${entry%%:*}"
190 port="${entry#*:}"
191 [[ "$port" == "$host" ]] && port=22
192 # `ssh-keyscan` returns the host keys without contacting the user; on
193 # unreachable hosts it logs a warning and exits 0. We tolerate that —
194 # operator can re-run after the target is up.
195 sudo -u "$SANDO_USER" ssh-keyscan -p "$port" -T 5 "$host" 2>/dev/null \
196 >> "$known" || log " warn: ssh-keyscan $host:$port returned nothing"
197 done
198 # De-dup in place. sort+mv keeps ownership/mode via install.
199 sudo -u "$SANDO_USER" sort -u "$known" -o "$known"
200
201 log "10/13 /etc/sando configs"
202 install -d -m 0755 /etc/sando
203 # sando-daemon.toml.example is the canonical production config (per the
204 # header comment). Install as-is; operator edits the listen address if
205 # binding to a non-fw13 tailnet IP.
206 install -m 0644 -o root -g root \
207 "$SCRIPT_DIR/sando-daemon.toml.example" \
208 /etc/sando/sando-daemon.toml
209 install -m 0644 -o root -g root \
210 "$SANDO_REPO/sando.toml" \
211 /etc/sando/sando.toml
212 # sando.env carries operator settings AND the deploy-API bearer token consumed
213 # by sandod, the post-receive hook, and the backup-fetch timer. A token is
214 # generated per host so a fresh install is authenticated by default — required
215 # once the daemon binds a non-loopback address (it refuses to start otherwise).
216 # 0640 root:sando keeps it readable by the daemon + hook, not world.
217 #
218 # Ensure the file AND a token, separately. An earlier install (or an operator)
219 # may have created sando.env with only SANDO_DAEMON; without the per-key check
220 # the token would never be added and the daemon would crash-loop on a
221 # non-loopback bind ("SANDO_API_TOKEN is unset ... refusing to start").
222 if [[ ! -f /etc/sando/sando.env ]]; then
223 echo "SANDO_DAEMON=$SANDO_DAEMON_URL" > /etc/sando/sando.env
224 fi
225 if ! grep -q '^SANDO_API_TOKEN=' /etc/sando/sando.env; then
226 echo "SANDO_API_TOKEN=$(openssl rand -hex 32)" >> /etc/sando/sando.env
227 log " generated SANDO_API_TOKEN in /etc/sando/sando.env"
228 fi
229 chown root:"$SANDO_USER" /etc/sando/sando.env
230 chmod 0640 /etc/sando/sando.env
231
232 log "11/13 systemd units"
233 install -m 0644 -o root -g root \
234 "$SCRIPT_DIR/sandod.service" \
235 /etc/systemd/system/sandod.service
236 install -m 0644 -o root -g root \
237 "$SCRIPT_DIR/sandod-backup-fetch.service" \
238 /etc/systemd/system/sandod-backup-fetch.service
239 install -m 0644 -o root -g root \
240 "$SCRIPT_DIR/sandod-backup-fetch.timer" \
241 /etc/systemd/system/sandod-backup-fetch.timer
242 systemctl daemon-reload
243
244 if [[ "$BUILD_SANDOD" == "1" ]]; then
245 log "12/13 sandod binary (cargo build --release -p sando-daemon → /usr/local/bin/sandod)"
246 daemon_dir="$SANDO_REPO/daemon"
247 if [[ ! -d "$daemon_dir" ]]; then
248 log " warn: cannot locate sando/daemon source at $daemon_dir; skipping build"
249 else
250 # Build from the workspace root so the binary lands in the shared
251 # sando/target; -p sando-daemon skips the TUI. Binary owned root, mode 755.
252 (cd "$SANDO_REPO" && cargo build --release --quiet -p sando-daemon)
253 install -m 0755 "$SANDO_REPO/target/release/sandod" /usr/local/bin/sandod
254 fi
255 else
256 log "12/13 skipping sandod build (BUILD_SANDOD=0)"
257 fi
258
259 log "13/13 bare mnw.git"
260 if [[ ! -d "$SANDO_HOME/mnw.git" ]]; then
261 sudo -u "$SANDO_USER" git init --bare --initial-branch=main "$SANDO_HOME/mnw.git" >/dev/null
262 fi
263 # No hook is installed here. sandod owns post-receive: ensure_bare_repo() writes
264 # it from the copy embedded in the binary (include_str! of hooks/post-receive)
265 # on every start, so the hook can never drift from the daemon that answers it.
266 #
267 # Bootstrap used to install its own second copy, deploy/post-receive, which had
268 # fallen behind: it sent no Authorization header, so against a token-configured
269 # daemon every push 401'd and no build triggered -- swallowed by the hook's own
270 # `|| echo 'sando: rebuild trigger failed'`. It self-healed on the next daemon
271 # start, which is precisely why nobody noticed. That copy is deleted.
272
273 # Enable services last so a partial bootstrap doesn't leave a service trying
274 # to start against an incomplete environment.
275 systemctl enable sandod.service >/dev/null 2>&1 || true
276 systemctl enable sandod-backup-fetch.timer >/dev/null 2>&1 || true
277
278 echo
279 log "Done. Next steps for the operator:"
280 echo " - tailscale up (auth this node onto the tailnet)"
281 echo " - on each deploy target, run bootstrap-node.sh with:"
282 echo " SANDO_PUBKEY=\"\$(cat $SANDO_HOME/.ssh/id_ed25519.pub)\""
283 echo " - push the MNW working tree to $SANDO_HOME/mnw.git:"
284 echo " git remote add sando $SANDO_USER@<host>:$SANDO_HOME/mnw.git"
285 echo " git push sando main"
286 echo " - sudo systemctl start sandod"
287 echo " - sudo systemctl start sandod-backup-fetch.timer"
288