max / makenotwork

13.9 KB · 336 lines History Blame Raw

1	#!/usr/bin/env bash
2	# Idempotent bootstrap for a fresh MNW node (tier A/B/C deploy target).
3	#
4	# Run on the new node as root. After this finishes, sandod on the Sando host
5	# can rsync + deploy to <ssh_target>:/opt/mnw/.
6	#
7	# Required env:
8	# SANDO_PUBKEY — sando user's public key on the Sando host. Get it via:
9	# `ssh fw13 'sudo cat /srv/sando/.ssh/id_ed25519.pub'`
10	#
11	# Optional env:
12	# DEPLOY_ROOT — defaults to /opt/mnw
13	# BIN_NAME — primary binary name (matches sando-daemon.toml's
14	# bin_names[0]). Defaults to "makenotwork".
15	# SERVICE_NAME — systemd unit name. Defaults to "makenotwork.service".
16	# SERVICE_USER — runtime user for the binary. Defaults to "deploy".
17	# GIT_REPOS_PATH — where the server keeps bare git repositories. Defaults
18	# to $STATE_DIR/git, which is what prod and testnot both
19	# run. It must match GIT_REPOS_PATH in the env file: the
20	# unit's sandbox makes everything else read-only, and the
21	# binary's own default is /opt/git, so a node that omits
22	# it from the env file writes somewhere the sandbox
23	# forbids.
24	# ENABLE_FIREWALL — "1" to set up UFW (22/80/443). Defaults to "1".
25	# INSTALL_CADDY — "1" to apt-install caddy (config is operator's job).
26	# Defaults to "1".
27	# INSTALL_POSTGRES — "1" to apt-install postgresql. Defaults to "1".
28	# INSTALL_TAILSCALE — "1" to apt-install tailscale (NOT authenticated;
29	# operator runs `tailscale up`). Defaults to "1".
30	#
31	# What this does NOT do (operator's job):
32	# - tailscale up (auth)
33	# - DNS records
34	# - Caddyfile content + Cloudflare origin certs + private keys
35	# - postgres role + db + .env / DATABASE_URL
36	# - any secrets
37
38	set -euo pipefail
39
40	if [[ $EUID -ne 0 ]]; then
41	echo "must run as root" >&2
42	exit 1
43	fi
44	if [[ -z "${SANDO_PUBKEY:-}" ]]; then
45	echo "SANDO_PUBKEY env var is required" >&2
46	exit 1
47	fi
48
49	DEPLOY_ROOT="${DEPLOY_ROOT:-/opt/mnw}"
50	# FHS-style sidecar paths the systemd unit references. Bootstrap creates the
51	# dirs but does not populate `ENV_FILE` — operator drops secrets in after the
52	# bootstrap finishes, before starting the service.
53	ETC_DIR="${ETC_DIR:-/etc/mnw}"
54	ENV_FILE="${ENV_FILE:-$ETC_DIR/makenotwork.env}"
55	STATE_DIR="${STATE_DIR:-/var/lib/mnw}"
56	GIT_REPOS_PATH="${GIT_REPOS_PATH:-$STATE_DIR/git}"
57	# The git user's home, where `mnw-admin rebuild-keys` writes
58	# `.ssh/authorized_keys`. The server reads it from `GIT_HOME` independently of
59	# `GIT_REPOS_PATH`, so it gets its own ReadWritePaths treatment below: a node
60	# where the two differ and one falls outside STATE_DIR would otherwise get an
61	# unwritable authorized_keys with no warning.
62	GIT_HOME="${GIT_HOME:-$GIT_REPOS_PATH}"
63	BIN_NAME="${BIN_NAME:-makenotwork}"
64	SERVICE_NAME="${SERVICE_NAME:-makenotwork.service}"
65	SERVICE_USER="${SERVICE_USER:-deploy}"
66	ENABLE_FIREWALL="${ENABLE_FIREWALL:-1}"
67	INSTALL_CADDY="${INSTALL_CADDY:-1}"
68	INSTALL_POSTGRES="${INSTALL_POSTGRES:-1}"
69	INSTALL_TAILSCALE="${INSTALL_TAILSCALE:-1}"
70
71	export DEBIAN_FRONTEND=noninteractive
72
73	log() { echo "[bootstrap] $*"; }
74
75	log "1/8 base packages"
76	apt-get update -qq
77	apt-get install -y -qq curl gnupg ca-certificates rsync ufw fail2ban > /dev/null
78
79	if [[ "$INSTALL_POSTGRES" == "1" ]]; then
80	log "2/8 postgresql"
81	apt-get install -y -qq postgresql > /dev/null
82	else
83	log "2/8 skipping postgresql"
84	fi
85
86	if [[ "$INSTALL_TAILSCALE" == "1" ]]; then
87	log "3/8 tailscale (not authenticating)"
88	if ! command -v tailscale >/dev/null; then
89	# Ubuntu codename. tailscale's repo is published per-codename;
90	# noble (24.04) keys work on 24.04+ derivatives.
91	codename=$(. /etc/os-release && echo "$VERSION_CODENAME")
92	curl -fsSL "https://pkgs.tailscale.com/stable/ubuntu/${codename}.noarmor.gpg" \
93	> /usr/share/keyrings/tailscale-archive-keyring.gpg
94	curl -fsSL "https://pkgs.tailscale.com/stable/ubuntu/${codename}.tailscale-keyring.list" \
95	> /etc/apt/sources.list.d/tailscale.list
96	apt-get update -qq
97	apt-get install -y -qq tailscale > /dev/null
98	systemctl enable --now tailscaled
99	fi
100	else
101	log "3/8 skipping tailscale"
102	fi
103
104	if [[ "$INSTALL_CADDY" == "1" ]]; then
105	log "4/8 caddy (no Caddyfile — operator's job)"
106	if ! command -v caddy >/dev/null; then
107	curl -fsSL https://dl.cloudsmith.io/public/caddy/stable/gpg.key \
108	\| gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
109	curl -fsSL https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt \
110	> /etc/apt/sources.list.d/caddy-stable.list
111	apt-get update -qq
112	apt-get install -y -qq caddy > /dev/null
113	fi
114	else
115	log "4/8 skipping caddy"
116	fi
117
118	log "5/8 deploy user + dirs"
119	if ! id "$SERVICE_USER" &>/dev/null; then
120	useradd -m -d "/home/$SERVICE_USER" -s /bin/bash "$SERVICE_USER"
121	fi
122	install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0700 "/home/$SERVICE_USER/.ssh"
123	if ! grep -qF "$SANDO_PUBKEY" "/home/$SERVICE_USER/.ssh/authorized_keys" 2>/dev/null; then
124	echo "$SANDO_PUBKEY" >> "/home/$SERVICE_USER/.ssh/authorized_keys"
125	fi
126	chown "$SERVICE_USER:$SERVICE_USER" "/home/$SERVICE_USER/.ssh/authorized_keys"
127	chmod 0600 "/home/$SERVICE_USER/.ssh/authorized_keys"
128	install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0755 "$DEPLOY_ROOT" "$DEPLOY_ROOT/releases"
129	# FHS sidecars: /etc/mnw owned root:service (so the service can read the env
130	# file but not edit it); /var/lib/mnw owned service:service for runtime
131	# state (backups, scan-spool, anything else the binary writes).
132	install -d -o root -g "$SERVICE_USER" -m 0750 "$ETC_DIR"
133	install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0750 "$STATE_DIR"
134
135	# If the git user exists (i.e. this host runs git SSH), grant it read access
136	# to the env file via ACL so mnw-admin git-auth can load DATABASE_URL. The git
137	# user is neither owner nor in the SERVICE_USER group, so without this the
138	# /etc/mnw/makenotwork.env is unreadable and every `git push` panics with
139	# "DATABASE_URL must be set". Conditional + idempotent.
140	if getent passwd git >/dev/null; then
141	setfacl -m u:git:x "$ETC_DIR"
142	if [ -f "$ENV_FILE" ]; then
143	setfacl -m u:git:r "$ENV_FILE"
144	fi
145	fi
146
147	# Bare git repositories. On a host that runs git SSH, two accounts write here:
148	# the web app creates the owner directory and the bare repo, and `git push`
149	# writes objects as the git user. Group-writable + setgid so whichever of them
150	# creates a directory, the other can still write inside it. Measured
151	# 2026-08-25: prod has this directory 0755 git:git, so the service cannot
152	# create a new owner directory at all and a creator's first repository fails
153	# there today — unrelated to the sandbox, which is why it is fixed here.
154	if getent passwd git >/dev/null; then
155	install -d -o "$SERVICE_USER" -g git -m 2775 "$GIT_REPOS_PATH"
156	else
157	install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0755 "$GIT_REPOS_PATH"
158	fi
159
160	log "6/8 sudoers (systemctl on $SERVICE_NAME for $SERVICE_USER)"
161	cat > "/etc/sudoers.d/${SERVICE_USER}-mnw" <<EOF
162	$SERVICE_USER ALL=(ALL) NOPASSWD: /bin/systemctl reload-or-restart $SERVICE_NAME, /bin/systemctl restart $SERVICE_NAME, /bin/systemctl status $SERVICE_NAME
163	EOF
164	chmod 0440 "/etc/sudoers.d/${SERVICE_USER}-mnw"
165	visudo -c -f "/etc/sudoers.d/${SERVICE_USER}-mnw" >/dev/null
166
167	log "7/8 systemd unit ($SERVICE_NAME) — points at $DEPLOY_ROOT/current/$BIN_NAME"
168	cat > "/etc/systemd/system/$SERVICE_NAME" <<EOF
169	[Unit]
170	Description=Makenotwork
171	Documentation=https://makenot.work/docs
172	After=network.target
173
174	[Service]
175	Type=simple
176	User=$SERVICE_USER
177	Group=$SERVICE_USER
178	WorkingDirectory=$DEPLOY_ROOT/current
179	ExecStart=$DEPLOY_ROOT/current/$BIN_NAME
180	# Secrets live outside the release dir so they survive deploys + rollbacks.
181	# Bootstrap creates ETC_DIR but not ENV_FILE — operator populates that.
182	EnvironmentFile=$ENV_FILE
183	# Runtime state (backups, spool, etc.) on FHS path; never inside the release
184	# dir or the deploy will erase it.
185	ReadWritePaths=$STATE_DIR$(
186	# Bare repositories are normally under STATE_DIR and covered by the line
187	# above. Emit a second path only when the operator has put them elsewhere,
188	# so a sandbox that lists only STATE_DIR cannot silently break repository
189	# creation.
190	emitted=""
191	# GIT_HOME defaults to GIT_REPOS_PATH, so dedupe rather than emit twice.
192	for extra in "$GIT_REPOS_PATH" "$GIT_HOME"; do
193	case "$extra/" in
194	"$STATE_DIR"/*) continue ;;
195	esac
196	case " $emitted " in
197	" $extra ") continue ;;
198	esac
199	emitted="$emitted $extra"
200	printf '\nReadWritePaths=%s' "$extra"
201	done
202	)
203	Restart=on-failure
204	RestartSec=30
205	# Exit 2 = migration failure (MNW server convention). Don't restart;
206	# operator must intervene before the next deploy.
207	RestartPreventExitStatus=2
208	# Scan-spool tempfiles for streaming large uploads through the malware
209	# pipeline. systemd creates the directory, chowns it to the service user and
210	# adds it to ReadWritePaths. The path is mirrored in
211	# \`makenotwork::constants::SCAN_SPOOL_DIR\`, so it is /var/lib/makenotwork
212	# rather than \$STATE_DIR and the two names are not interchangeable.
213	# The second entry is the drop-box for authorized_keys rebuild requests; see the
214	# .path unit below. Named explicitly rather than relying on the parent of the
215	# spool being writable. Both paths are mirrored in
216	# \`makenotwork::constants\` (\`SCAN_SPOOL_DIR\`, \`KEYS_REBUILD_MARKER\`).
217	StateDirectory=makenotwork/scan-spool makenotwork/keys
218	StateDirectoryMode=0700
219	# The ceilings server/docs/troubleshooting.md documents. Both are measured
220	# rather than chosen: prod's own cgroup reported 392M anonymous (unreclaimable)
221	# with a 403M peak on 2026-08-25, so the 512M this template used to write would
222	# have left ~110M of headroom and been an OOM kill on the first content export,
223	# which holds one file of up to 500M in memory. MemoryHigh throttles and
224	# reclaims before MemoryMax kills. The soft NOFILE the service actually gets
225	# without this line is 1024, not the 524288 that \`systemctl show\` reports
226	# (that is the hard limit); the server had 18 descriptors open when measured.
227	LimitNOFILE=65535
228	MemoryHigh=1G
229	MemoryMax=2G
230	# The sandbox. Verified 2026-08-25 by running a probe under exactly these
231	# options as the service user, on prod and on testnot: bare-repo creation, the
232	# scan spool, an export's private /tmp, clamd's unix socket, postgres over both
233	# TCP and its unix socket, and the yara rules all still work, while /opt, /etc
234	# and the rest of /var are read-only. ProtectSystem=strict is what makes the
235	# ReadWritePaths above load-bearing rather than decorative.
236	NoNewPrivileges=yes
237	ProtectSystem=strict
238	ProtectHome=yes
239	PrivateTmp=yes
240	PrivateDevices=yes
241	ProtectProc=invisible
242	ProtectKernelTunables=yes
243	ProtectKernelModules=yes
244	ProtectKernelLogs=yes
245	ProtectControlGroups=yes
246	ProtectClock=yes
247	ProtectHostname=yes
248	RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
249	RestrictNamespaces=yes
250	RestrictRealtime=yes
251	RestrictSUIDSGID=yes
252	LockPersonality=yes
253	MemoryDenyWriteExecute=yes
254	SystemCallArchitectures=native
255	SystemCallFilter=@system-service
256	SystemCallErrorNumber=EPERM
257	StandardOutput=journal
258	StandardError=journal
259	SyslogIdentifier=$BIN_NAME
260
261	[Install]
262	WantedBy=multi-user.target
263	EOF
264
265	# The authorized_keys rebuild, out of the service's process tree.
266	#
267	# sshd's StrictModes refuses an authorized_keys whose file or .ssh directory is
268	# group-writable, so the service user cannot be given write access to it, and
269	# the sandbox above implies NoNewPrivileges (PrivateDevices, ProtectClock,
270	# MemoryDenyWriteExecute, RestrictNamespaces, RestrictSUIDSGID, LockPersonality
271	# and SystemCallFilter each imply it, and NoNewPrivileges=no does not undo the
272	# implication), so sudo cannot raise privilege from inside the unit either.
273	# Measured 2026-09-01: sudo under any one of those options exits 1 with "the
274	# no new privileges flag is set". The service therefore writes a marker into
275	# its StateDirectory and this pair does the write as root.
276	#
277	# The marker path is mirrored in `makenotwork::constants::KEYS_REBUILD_MARKER`
278	# and is fixed at /var/lib/makenotwork, not $STATE_DIR, because systemd derives
279	# it from StateDirectory.
280	KEYS_MARKER="/var/lib/makenotwork/keys/rebuild"
281	cat > /etc/systemd/system/makenotwork-rebuild-keys.path <<EOF
282	[Unit]
283	Description=Watch for makenotwork authorized_keys rebuild requests
284
285	[Path]
286	PathExists=$KEYS_MARKER
287	PathModified=$KEYS_MARKER
288	Unit=makenotwork-rebuild-keys.service
289
290	[Install]
291	WantedBy=multi-user.target
292	EOF
293
294	# Deletes the marker before rebuilding, so a rebuild that outlives its request
295	# cannot re-trigger itself. A failed rebuild loses its marker and shows up as a
296	# failed unit rather than a retry loop; the server's own warn line covers the
297	# case where the request never lands.
298	cat > /etc/systemd/system/makenotwork-rebuild-keys.service <<EOF
299	[Unit]
300	Description=Rebuild the git user's authorized_keys from the database
301
302	[Service]
303	Type=oneshot
304	EnvironmentFile=$ENV_FILE
305	ExecStartPre=-/bin/rm -f $KEYS_MARKER
306	ExecStart=$DEPLOY_ROOT/current/mnw-admin rebuild-keys
307	StandardOutput=journal
308	StandardError=journal
309	SyslogIdentifier=makenotwork-rebuild-keys
310	EOF
311
312	systemctl daemon-reload
313	systemctl enable "$SERVICE_NAME" >/dev/null 2>&1 \|\| true
314	systemctl enable --now makenotwork-rebuild-keys.path >/dev/null 2>&1 \|\| true
315
316	if [[ "$ENABLE_FIREWALL" == "1" ]]; then
317	log "8/8 firewall (UFW: 22/80/443 in, all else deny)"
318	ufw --force reset > /dev/null
319	ufw default deny incoming > /dev/null
320	ufw default allow outgoing > /dev/null
321	ufw allow 22/tcp > /dev/null
322	ufw allow 80/tcp > /dev/null
323	ufw allow 443/tcp > /dev/null
324	ufw --force enable > /dev/null
325	else
326	log "8/8 skipping firewall"
327	fi
328
329	echo
330	log "Done. Next steps for the operator:"
331	echo " - tailscale up (auth this node to the tailnet)"
332	echo " - DNS A/AAAA records for the domain you'll serve"
333	echo " - Install /etc/caddy/Caddyfile + Cloudflare Origin CA cert + key"
334	echo " - postgres: create role+db, drop secrets into $ENV_FILE (chmod 0640, chown root:$SERVICE_USER)"
335	echo " - Run a sando deploy from the Sando host: POST /promote/<tier>"
336