Skip to main content

max / makenotwork

11.1 KB · 269 lines History Blame Raw
1 #!/usr/bin/env bash
2 # Idempotent bootstrap for a fresh MNW node (tier A/B/C deploy target).
3 #
4 # Run on the new node as root. After this finishes, sandod on the Sando host
5 # can rsync + deploy to <ssh_target>:/opt/mnw/.
6 #
7 # Required env:
8 # SANDO_PUBKEY — sando user's public key on the Sando host. Get it via:
9 # `ssh fw13 'sudo cat /srv/sando/.ssh/id_ed25519.pub'`
10 #
11 # Optional env:
12 # DEPLOY_ROOT — defaults to /opt/mnw
13 # BIN_NAME — primary binary name (matches sando-daemon.toml's
14 # bin_names[0]). Defaults to "makenotwork".
15 # SERVICE_NAME — systemd unit name. Defaults to "makenotwork.service".
16 # SERVICE_USER — runtime user for the binary. Defaults to "deploy".
17 # GIT_REPOS_PATH — where the server keeps bare git repositories. Defaults
18 # to $STATE_DIR/git, which is what prod and testnot both
19 # run. It must match GIT_REPOS_PATH in the env file: the
20 # unit's sandbox makes everything else read-only, and the
21 # binary's own default is /opt/git, so a node that omits
22 # it from the env file writes somewhere the sandbox
23 # forbids.
24 # ENABLE_FIREWALL — "1" to set up UFW (22/80/443). Defaults to "1".
25 # INSTALL_CADDY — "1" to apt-install caddy (config is operator's job).
26 # Defaults to "1".
27 # INSTALL_POSTGRES — "1" to apt-install postgresql. Defaults to "1".
28 # INSTALL_TAILSCALE — "1" to apt-install tailscale (NOT authenticated;
29 # operator runs `tailscale up`). Defaults to "1".
30 #
31 # What this does NOT do (operator's job):
32 # - tailscale up (auth)
33 # - DNS records
34 # - Caddyfile content + Cloudflare origin certs + private keys
35 # - postgres role + db + .env / DATABASE_URL
36 # - any secrets
37
38 set -euo pipefail
39
40 if [[ $EUID -ne 0 ]]; then
41 echo "must run as root" >&2
42 exit 1
43 fi
44 if [[ -z "${SANDO_PUBKEY:-}" ]]; then
45 echo "SANDO_PUBKEY env var is required" >&2
46 exit 1
47 fi
48
49 DEPLOY_ROOT="${DEPLOY_ROOT:-/opt/mnw}"
50 # FHS-style sidecar paths the systemd unit references. Bootstrap creates the
51 # dirs but does not populate `ENV_FILE` — operator drops secrets in after the
52 # bootstrap finishes, before starting the service.
53 ETC_DIR="${ETC_DIR:-/etc/mnw}"
54 ENV_FILE="${ENV_FILE:-$ETC_DIR/makenotwork.env}"
55 STATE_DIR="${STATE_DIR:-/var/lib/mnw}"
56 GIT_REPOS_PATH="${GIT_REPOS_PATH:-$STATE_DIR/git}"
57 BIN_NAME="${BIN_NAME:-makenotwork}"
58 SERVICE_NAME="${SERVICE_NAME:-makenotwork.service}"
59 SERVICE_USER="${SERVICE_USER:-deploy}"
60 ENABLE_FIREWALL="${ENABLE_FIREWALL:-1}"
61 INSTALL_CADDY="${INSTALL_CADDY:-1}"
62 INSTALL_POSTGRES="${INSTALL_POSTGRES:-1}"
63 INSTALL_TAILSCALE="${INSTALL_TAILSCALE:-1}"
64
65 export DEBIAN_FRONTEND=noninteractive
66
67 log() { echo "[bootstrap] $*"; }
68
69 log "1/8 base packages"
70 apt-get update -qq
71 apt-get install -y -qq curl gnupg ca-certificates rsync ufw fail2ban > /dev/null
72
73 if [[ "$INSTALL_POSTGRES" == "1" ]]; then
74 log "2/8 postgresql"
75 apt-get install -y -qq postgresql > /dev/null
76 else
77 log "2/8 skipping postgresql"
78 fi
79
80 if [[ "$INSTALL_TAILSCALE" == "1" ]]; then
81 log "3/8 tailscale (not authenticating)"
82 if ! command -v tailscale >/dev/null; then
83 # Ubuntu codename. tailscale's repo is published per-codename;
84 # noble (24.04) keys work on 24.04+ derivatives.
85 codename=$(. /etc/os-release && echo "$VERSION_CODENAME")
86 curl -fsSL "https://pkgs.tailscale.com/stable/ubuntu/${codename}.noarmor.gpg" \
87 > /usr/share/keyrings/tailscale-archive-keyring.gpg
88 curl -fsSL "https://pkgs.tailscale.com/stable/ubuntu/${codename}.tailscale-keyring.list" \
89 > /etc/apt/sources.list.d/tailscale.list
90 apt-get update -qq
91 apt-get install -y -qq tailscale > /dev/null
92 systemctl enable --now tailscaled
93 fi
94 else
95 log "3/8 skipping tailscale"
96 fi
97
98 if [[ "$INSTALL_CADDY" == "1" ]]; then
99 log "4/8 caddy (no Caddyfile — operator's job)"
100 if ! command -v caddy >/dev/null; then
101 curl -fsSL https://dl.cloudsmith.io/public/caddy/stable/gpg.key \
102 | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
103 curl -fsSL https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt \
104 > /etc/apt/sources.list.d/caddy-stable.list
105 apt-get update -qq
106 apt-get install -y -qq caddy > /dev/null
107 fi
108 else
109 log "4/8 skipping caddy"
110 fi
111
112 log "5/8 deploy user + dirs"
113 if ! id "$SERVICE_USER" &>/dev/null; then
114 useradd -m -d "/home/$SERVICE_USER" -s /bin/bash "$SERVICE_USER"
115 fi
116 install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0700 "/home/$SERVICE_USER/.ssh"
117 if ! grep -qF "$SANDO_PUBKEY" "/home/$SERVICE_USER/.ssh/authorized_keys" 2>/dev/null; then
118 echo "$SANDO_PUBKEY" >> "/home/$SERVICE_USER/.ssh/authorized_keys"
119 fi
120 chown "$SERVICE_USER:$SERVICE_USER" "/home/$SERVICE_USER/.ssh/authorized_keys"
121 chmod 0600 "/home/$SERVICE_USER/.ssh/authorized_keys"
122 install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0755 "$DEPLOY_ROOT" "$DEPLOY_ROOT/releases"
123 # FHS sidecars: /etc/mnw owned root:service (so the service can read the env
124 # file but not edit it); /var/lib/mnw owned service:service for runtime
125 # state (backups, scan-spool, anything else the binary writes).
126 install -d -o root -g "$SERVICE_USER" -m 0750 "$ETC_DIR"
127 install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0750 "$STATE_DIR"
128
129 # If the git user exists (i.e. this host runs git SSH), grant it read access
130 # to the env file via ACL so mnw-admin git-auth can load DATABASE_URL. The git
131 # user is neither owner nor in the SERVICE_USER group, so without this the
132 # /etc/mnw/makenotwork.env is unreadable and every `git push` panics with
133 # "DATABASE_URL must be set". Conditional + idempotent.
134 if getent passwd git >/dev/null; then
135 setfacl -m u:git:x "$ETC_DIR"
136 if [ -f "$ENV_FILE" ]; then
137 setfacl -m u:git:r "$ENV_FILE"
138 fi
139 fi
140
141 # Bare git repositories. On a host that runs git SSH, two accounts write here:
142 # the web app creates the owner directory and the bare repo, and `git push`
143 # writes objects as the git user. Group-writable + setgid so whichever of them
144 # creates a directory, the other can still write inside it. Measured
145 # 2026-08-25: prod has this directory 0755 git:git, so the service cannot
146 # create a new owner directory at all and a creator's first repository fails
147 # there today — unrelated to the sandbox, which is why it is fixed here.
148 if getent passwd git >/dev/null; then
149 install -d -o "$SERVICE_USER" -g git -m 2775 "$GIT_REPOS_PATH"
150 else
151 install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0755 "$GIT_REPOS_PATH"
152 fi
153
154 log "6/8 sudoers (systemctl on $SERVICE_NAME for $SERVICE_USER)"
155 cat > "/etc/sudoers.d/${SERVICE_USER}-mnw" <<EOF
156 $SERVICE_USER ALL=(ALL) NOPASSWD: /bin/systemctl reload-or-restart $SERVICE_NAME, /bin/systemctl restart $SERVICE_NAME, /bin/systemctl status $SERVICE_NAME
157 EOF
158 chmod 0440 "/etc/sudoers.d/${SERVICE_USER}-mnw"
159 visudo -c -f "/etc/sudoers.d/${SERVICE_USER}-mnw" >/dev/null
160
161 log "7/8 systemd unit ($SERVICE_NAME) — points at $DEPLOY_ROOT/current/$BIN_NAME"
162 cat > "/etc/systemd/system/$SERVICE_NAME" <<EOF
163 [Unit]
164 Description=Makenotwork
165 Documentation=https://makenot.work/docs
166 After=network.target
167
168 [Service]
169 Type=simple
170 User=$SERVICE_USER
171 Group=$SERVICE_USER
172 WorkingDirectory=$DEPLOY_ROOT/current
173 ExecStart=$DEPLOY_ROOT/current/$BIN_NAME
174 # Secrets live outside the release dir so they survive deploys + rollbacks.
175 # Bootstrap creates ETC_DIR but not ENV_FILE — operator populates that.
176 EnvironmentFile=$ENV_FILE
177 # Runtime state (backups, spool, etc.) on FHS path; never inside the release
178 # dir or the deploy will erase it.
179 ReadWritePaths=$STATE_DIR$(
180 # Bare repositories are normally under STATE_DIR and covered by the line
181 # above. Emit a second path only when the operator has put them elsewhere,
182 # so a sandbox that lists only STATE_DIR cannot silently break repository
183 # creation.
184 case "$GIT_REPOS_PATH/" in
185 "$STATE_DIR"/*) ;;
186 *) printf '\nReadWritePaths=%s' "$GIT_REPOS_PATH" ;;
187 esac
188 )
189 Restart=on-failure
190 RestartSec=30
191 # Exit 2 = migration failure (MNW server convention). Don't restart;
192 # operator must intervene before the next deploy.
193 RestartPreventExitStatus=2
194 # Scan-spool tempfiles for streaming large uploads through the malware
195 # pipeline. systemd creates the directory, chowns it to the service user and
196 # adds it to ReadWritePaths. The path is mirrored in
197 # \`makenotwork::constants::SCAN_SPOOL_DIR\`, so it is /var/lib/makenotwork
198 # rather than \$STATE_DIR and the two names are not interchangeable.
199 StateDirectory=makenotwork/scan-spool
200 StateDirectoryMode=0700
201 # The ceilings server/docs/troubleshooting.md documents. Both are measured
202 # rather than chosen: prod's own cgroup reported 392M anonymous (unreclaimable)
203 # with a 403M peak on 2026-08-25, so the 512M this template used to write would
204 # have left ~110M of headroom and been an OOM kill on the first content export,
205 # which holds one file of up to 500M in memory. MemoryHigh throttles and
206 # reclaims before MemoryMax kills. The soft NOFILE the service actually gets
207 # without this line is 1024, not the 524288 that \`systemctl show\` reports
208 # (that is the hard limit); the server had 18 descriptors open when measured.
209 LimitNOFILE=65535
210 MemoryHigh=1G
211 MemoryMax=2G
212 # The sandbox. Verified 2026-08-25 by running a probe under exactly these
213 # options as the service user, on prod and on testnot: bare-repo creation, the
214 # scan spool, an export's private /tmp, clamd's unix socket, postgres over both
215 # TCP and its unix socket, and the yara rules all still work, while /opt, /etc
216 # and the rest of /var are read-only. ProtectSystem=strict is what makes the
217 # ReadWritePaths above load-bearing rather than decorative.
218 NoNewPrivileges=yes
219 ProtectSystem=strict
220 ProtectHome=yes
221 PrivateTmp=yes
222 PrivateDevices=yes
223 ProtectProc=invisible
224 ProtectKernelTunables=yes
225 ProtectKernelModules=yes
226 ProtectKernelLogs=yes
227 ProtectControlGroups=yes
228 ProtectClock=yes
229 ProtectHostname=yes
230 RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
231 RestrictNamespaces=yes
232 RestrictRealtime=yes
233 RestrictSUIDSGID=yes
234 LockPersonality=yes
235 MemoryDenyWriteExecute=yes
236 SystemCallArchitectures=native
237 SystemCallFilter=@system-service
238 SystemCallErrorNumber=EPERM
239 StandardOutput=journal
240 StandardError=journal
241 SyslogIdentifier=$BIN_NAME
242
243 [Install]
244 WantedBy=multi-user.target
245 EOF
246 systemctl daemon-reload
247 systemctl enable "$SERVICE_NAME" >/dev/null 2>&1 || true
248
249 if [[ "$ENABLE_FIREWALL" == "1" ]]; then
250 log "8/8 firewall (UFW: 22/80/443 in, all else deny)"
251 ufw --force reset > /dev/null
252 ufw default deny incoming > /dev/null
253 ufw default allow outgoing > /dev/null
254 ufw allow 22/tcp > /dev/null
255 ufw allow 80/tcp > /dev/null
256 ufw allow 443/tcp > /dev/null
257 ufw --force enable > /dev/null
258 else
259 log "8/8 skipping firewall"
260 fi
261
262 echo
263 log "Done. Next steps for the operator:"
264 echo " - tailscale up (auth this node to the tailnet)"
265 echo " - DNS A/AAAA records for the domain you'll serve"
266 echo " - Install /etc/caddy/Caddyfile + Cloudflare Origin CA cert + key"
267 echo " - postgres: create role+db, drop secrets into $ENV_FILE (chmod 0640, chown root:$SERVICE_USER)"
268 echo " - Run a sando deploy from the Sando host: POST /promote/<tier>"
269