max / makenotwork

6.9 KB · 127 lines History Blame Raw

1	#!/usr/bin/env bash
2	# Rebuild and restart sandod itself to a target commit.
3	#
4	# Runs as ROOT, invoked by the oneshot unit `sando-update@<sha>.service` (the
5	# sha is the instance name, passed here as $1). sandod cannot do this itself:
6	# it runs User=sando with NoNewPrivileges + ProtectSystem=strict, so it can
7	# neither write /usr/local/bin/sandod nor restart its own service. sandod only
8	# triggers this unit (authorized for the sando user by a scoped polkit rule);
9	# the actual privileged work lives here.
10	#
11	# Build runs as the unprivileged build user (the sando user already carries a
12	# rustup toolchain at /srv/sando/.cargo/bin); only the install + restart run as
13	# root. The build uses a dedicated checkout, never the operator's dev tree.
14	#
15	# Config via environment (defaults shown), set in the unit or /etc/sando/sando.env:
16	# SANDO_SELF_UPDATE_DIR /srv/sando/self-update build checkout parent (build-user-owned)
17	# SANDO_UPSTREAM_URL /srv/sando/mnw.git source repo (see "Source" below)
18	# SANDO_BUILD_USER sando
19	# SANDO_BIN /usr/local/bin/sandod install destination
20	# SANDO_DEPLOY_BRANCH main the only branch a self-update sha may live on
21	# SANDO_DAEMON_CONFIG /etc/sando/sando-daemon.toml config the --check-config self-test loads
22	#
23	# Source: build from the LOCAL bare repo sandod already maintains
24	# (/srv/sando/mnw.git), not a remote fetch. Building from a remote
25	# (git@ssh.makenot.work) gave the `sando` user no git creds and broke the moment
26	# git hosting was down — it blocked the self-update during the 2026-07-09 deploy
27	# (postmortem #7). The local bare repo has no such external dependency.
28	#
29	# sandod fetches the deploy branch from the canonical remote into that bare repo
30	# at the top of its /self-update handler, so a sha pushed minutes ago is present
31	# here. It used to be reachable only if a server build had fetched since, which
32	# chained the controller's currency to the server's release cadence: shipping a
33	# controller fix meant cutting a server release first, and was impossible at all
34	# while the server was red. sandod remains the only writer of that repo, so the
35	# provenance seal below is unchanged.
36	#
37	# Provenance: sandod (bearer-gated) only triggers this unit with a hex sha; it
38	# does not prove the sha is a commit anyone intended to deploy. Without a check,
39	# the deploy token would be root code-exec on this host. So after fetch we REFUSE
40	# any sha that is not an ancestor of origin/$SANDO_DEPLOY_BRANCH: a feature-branch
41	# tip, an unknown sha, or a commit not on the deploy branch never reaches the
42	# build/install lines (exit 4). Only sandod writes the bare repo (fetching main
43	# from the authenticated upstream), so its main is a trustworthy provenance seal.
44	# A signed-tag check is the planned follow-up once release signing exists.
45	#
46	# Safety net: a clean build (no stale incremental cache) plus a --check-config
47	# self-test of the freshly built binary against the LIVE config gate the install.
48	# A stale incremental object once produced a sandod that could not parse its own
49	# node_health config and crash-looped (postmortem #6); either guard alone stops
50	# that binary from ever being installed.
51	set -euo pipefail
52
53	SHA="${1:-}"
54	if [[ ! "$SHA" =~ ^[0-9a-f]{7,40}$ ]]; then
55	echo "sando-self-update: refusing non-hex sha: '$SHA'" >&2
56	exit 2
57	fi
58
59	SELF_DIR="${SANDO_SELF_UPDATE_DIR:-/srv/sando/self-update}"
60	UPSTREAM_URL="${SANDO_UPSTREAM_URL:-/srv/sando/mnw.git}"
61	BUILD_USER="${SANDO_BUILD_USER:-sando}"
62	BIN="${SANDO_BIN:-/usr/local/bin/sandod}"
63	DEPLOY_BRANCH="${SANDO_DEPLOY_BRANCH:-main}"
64	DAEMON_CONFIG="${SANDO_DAEMON_CONFIG:-/etc/sando/sando-daemon.toml}"
65	REPO_DIR="$SELF_DIR/MNW"
66	BUILD_HOME="$(getent passwd "$BUILD_USER" \| cut -d: -f6)"
67
68	echo "sando-self-update: building sandod @ $SHA as $BUILD_USER (provenance: origin/$DEPLOY_BRANCH)"
69
70	# Fetch + provenance check + checkout + build, all as the unprivileged build
71	# user. The clone is created once; thereafter we just fetch the new sha. Detached
72	# checkout so the dedicated tree never carries a branch to drift.
73	install -d -o "$BUILD_USER" -g "$BUILD_USER" "$SELF_DIR"
74	runuser -u "$BUILD_USER" -- env \
75	HOME="$BUILD_HOME" \
76	PATH="$BUILD_HOME/.cargo/bin:/usr/local/bin:/usr/bin:/bin" \
77	bash -euo pipefail -c "
78	if [[ ! -d '$REPO_DIR/.git' ]]; then
79	git clone '$UPSTREAM_URL' '$REPO_DIR'
80	fi
81	cd '$REPO_DIR'
82	# Pin origin to the configured source every run, so switching
83	# SANDO_UPSTREAM_URL (e.g. remote -> local bare repo) takes effect on an
84	# already-cloned checkout instead of silently keeping the old remote.
85	git remote set-url origin '$UPSTREAM_URL'
86	git fetch --prune origin
87	# Provenance seal: the sha must be reachable from the deploy branch in the
88	# source repo. --is-ancestor exits 1 for a non-ancestor and >1 for a
89	# bad/unresolvable ref, so any non-deploy-branch sha is refused fail-closed
90	# before a single line is built or installed.
91	if ! git merge-base --is-ancestor '$SHA' 'origin/$DEPLOY_BRANCH'; then
92	echo \"sando-self-update: refusing sha '$SHA' — not an ancestor of origin/$DEPLOY_BRANCH\" >&2
93	exit 4
94	fi
95	git checkout --detach '$SHA'
96	cd sando
97	# Clean build: wipe the workspace target so no stale incremental object
98	# survives across shas. A reused pre-node_health Gate enum object once
99	# produced a sandod that crash-looped on the current config (postmortem
100	# #6). Self-updates are rare, so a full recompile is a cheap insurance.
101	cargo clean
102	cargo build --release --locked -p sando-daemon
103	"
104
105	# Workspace shares one target dir at sando/target; -p sando-daemon avoids
106	# compiling the TUI on the build host.
107	NEW_BIN="$REPO_DIR/sando/target/release/sandod"
108	[[ -x "$NEW_BIN" ]] \|\| { echo "sando-self-update: build produced no binary at $NEW_BIN" >&2; exit 3; }
109
110	# Self-test the fresh binary against the LIVE config BEFORE the swap: prove it can
111	# load + parse the exact daemon config + topology sandod will boot against. Run as
112	# the build user (not root) so the readability check matches the running daemon's
113	# identity. A binary that can't parse the current config (the postmortem #6
114	# brick) fails here and is never installed — sandod keeps running on the old one.
115	echo "sando-self-update: self-testing $NEW_BIN against $DAEMON_CONFIG"
116	if ! runuser -u "$BUILD_USER" -- env SANDO_CONFIG="$DAEMON_CONFIG" "$NEW_BIN" --check-config; then
117	echo "sando-self-update: new binary FAILED --check-config against $DAEMON_CONFIG; refusing to install (sandod left running on the current binary)" >&2
118	exit 5
119	fi
120
121	# Install + restart as root. install is atomic (writes a temp then renames), so
122	# a concurrent exec of $BIN never sees a half-written file.
123	echo "sando-self-update: installing $NEW_BIN -> $BIN and restarting sandod"
124	install -m 0755 "$NEW_BIN" "$BIN"
125	systemctl restart sandod
126	echo "sando-self-update: done ($SHA live)"
127