Skip to main content

max / makenotwork

sando: harden self-update against self-bricking Three fixes from the 2026-07-09 prod-deploy postmortem, where a stale incremental build produced a sandod that could not parse its own node_health config and crash-looped, and the updater's remote git fetch failed while git hosting was down. - --check-config mode (main.rs): load + parse the live daemon config + topology and run the build-host invariant, then exit; no DB, network, or bind. Exactly the parse the brick failed. The self-update runs it against the live config before the swap, so a binary that can't understand its config is never installed. - Clean build (cargo clean) per self-update, so no stale incremental object survives across shas. - Build from the local bare repo /srv/sando/mnw.git instead of a remote fetch, removing the external git-auth dependency; provenance seal against origin/main still holds (the bare repo's main is kept current by /rebuild's authenticated fetch). Self-update during an in-flight build/deploy is already refused in code; documented as deliberate operator discipline in deploy/README.md. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-10 13:36 UTC
Commit: 93f04931ee1ab3aaa37af44ccfba7f2c045afd60
Parent: 8edbd93
3 files changed, +135 insertions, -14 deletions
M sando/daemon/src/main.rs +65 -2
@@ -11,8 +11,48 @@ fn refuse_unauthenticated_bind(addr: &SocketAddr, has_token: bool) -> bool {
11 11 !has_token && !addr.ip().is_loopback()
12 12 }
13 13
14 + fn main() -> Result<()> {
15 + // `--check-config`: prove THIS binary can load and parse its live daemon
16 + // config + topology, then exit — no runtime, no DB, no network, no bind.
17 + // This is the exact parse a stale self-update build once failed: a sandod
18 + // missing the `node_health` Gate variant could not parse the node_health
19 + // topology and crash-looped on every restart (postmortem 2026-07-09, #6).
20 + // The self-update runs this against the live config *before* swapping the
21 + // new binary in, so a binary that can't understand its config never gets
22 + // installed. Pure by design: a green here means "this build understands its
23 + // config", isolated from any environmental (DB/host) failure.
24 + if std::env::args().skip(1).any(|a| a == "--check-config") {
25 + return check_config();
26 + }
27 + run()
28 + }
29 +
30 + /// Load + validate the live config chain the daemon would boot against, without
31 + /// starting anything. Mirrors the config/topology steps of `run` (below) so a
32 + /// pass here guarantees `run` gets past them. Errors carry anyhow's context
33 + /// chain and exit non-zero via `main`'s `Result`.
34 + fn check_config() -> Result<()> {
35 + let cfg = config::Config::load()?;
36 + let topo = validate_loaded(&cfg)?;
37 + println!(
38 + "sandod --check-config: OK — {} tier(s), topology {}",
39 + topo.tiers.len(),
40 + cfg.topology_path.display()
41 + );
42 + Ok(())
43 + }
44 +
45 + /// The parse + invariant work `check_config` does once the daemon config is in
46 + /// hand. Split out so it can be exercised against a fixture config without
47 + /// touching process env (`Config::load` reads `SANDO_CONFIG`/CWD).
48 + fn validate_loaded(cfg: &config::Config) -> Result<topology::Topology> {
49 + let topo = topology::Topology::load(&cfg.topology_path)?;
50 + topo.ensure_build_host_not_serving(&cfg.build_host)?;
51 + Ok(topo)
52 + }
53 +
14 54 #[tokio::main]
15 - async fn main() -> Result<()> {
55 + async fn run() -> Result<()> {
16 56 tracing_subscriber::fmt()
17 57 // stdout is block-buffered under systemd (no TTY) so events never
18 58 // reach journald until the buffer fills or the process exits. stderr
@@ -95,7 +135,7 @@ async fn main() -> Result<()> {
95 135
96 136 #[cfg(test)]
97 137 mod tests {
98 - use super::refuse_unauthenticated_bind;
138 + use super::{refuse_unauthenticated_bind, validate_loaded};
99 139 use std::net::SocketAddr;
100 140
101 141 fn addr(s: &str) -> SocketAddr {
@@ -103,6 +143,29 @@ mod tests {
103 143 }
104 144
105 145 #[test]
146 + fn check_config_accepts_the_shipped_topology() {
147 + // The `--check-config` path a self-update runs before installing a new
148 + // binary must pass against the topology this build ships with — the
149 + // regression guard for the 2026-07-09 brick, where a binary that could
150 + // not parse the live node_health config was installed and crash-looped.
151 + // (`Config::for_tests` is lib-test-only and unavailable here, so build
152 + // the config by deserializing, pointing it at the shipped sando.toml.)
153 + let topo_path =
154 + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../sando.toml");
155 + let raw = format!(
156 + "listen = \"127.0.0.1:7766\"\n\
157 + db_path = \"./sando.db\"\n\
158 + topology_path = {topo_path:?}\n\
159 + build_host = \"fw13\"\n\
160 + workdir = \"./work\"\n\
161 + release_root = \"./releases\"\n"
162 + );
163 + let cfg: super::config::Config =
164 + toml::from_str(&raw).expect("build fixture daemon config");
165 + validate_loaded(&cfg).expect("shipped config must pass --check-config");
166 + }
167 +
168 + #[test]
106 169 fn non_loopback_without_token_is_refused() {
107 170 // The exact CF2 posture: tailnet bind, no token.
108 171 assert!(refuse_unauthenticated_bind(&addr("100.103.89.95:7766"), false));
M sando/deploy/README.md +20
@@ -31,6 +31,26 @@ runs `sando-self-update.sh`: it builds `sando/daemon` **as the sando user** in a
31 31 dedicated checkout (`/srv/sando/self-update`, never the operator's dev tree),
32 32 then — as root — installs the binary and `systemctl restart sandod`.
33 33
34 + Three hardening properties gate that install (postmortem 2026-07-09 recovery):
35 +
36 + - **Source = the local bare repo** `/srv/sando/mnw.git`, not a remote fetch. The
37 + self-update sha was already deployed as a server release, so `/rebuild` has
38 + force-updated the bare repo's `main` to include it — the sha and a trusted
39 + provenance ref are both present locally. A remote fetch gave the `sando` user
40 + no git creds and broke the moment git hosting was down.
41 + - **Clean build** (`cargo clean` first) so no stale incremental object survives
42 + across shas. A reused pre-`node_health` `Gate` enum object once produced a
43 + binary that crash-looped on the current config.
44 + - **`--check-config` self-test before the swap.** The freshly built binary must
45 + load + parse the live daemon config + topology (`sandod --check-config`, run as
46 + the sando user against `/etc/sando/sando-daemon.toml`). A binary that can't
47 + understand the current config fails here and is never installed — `sandod`
48 + keeps running on the old one instead of crash-looping on the new.
49 +
50 + Trigger `/self-update` only when no MNW build/deploy is in flight — `sandod`
51 + rejects it in that window anyway (the restart would SIGKILL an in-flight deploy),
52 + but treat controller updates as a separate, deliberate op.
53 +
34 54 Verify a self-update landed by polling `/state`: the new `sandod_version` field
35 55 reports the running controller's package version (distinct from the tier
36 56 versions, which are the deployed product).
M sando/deploy/sando-self-update.sh +50 -12
@@ -13,19 +13,36 @@
13 13 # root. The build uses a dedicated checkout, never the operator's dev tree.
14 14 #
15 15 # Config via environment (defaults shown), set in the unit or /etc/sando/sando.env:
16 - # SANDO_SELF_UPDATE_DIR /srv/sando/self-update build checkout parent (build-user-owned)
17 - # SANDO_UPSTREAM_URL git@ssh.makenot.work:max/makenotwork.git
16 + # SANDO_SELF_UPDATE_DIR /srv/sando/self-update build checkout parent (build-user-owned)
17 + # SANDO_UPSTREAM_URL /srv/sando/mnw.git source repo (see "Source" below)
18 18 # SANDO_BUILD_USER sando
19 - # SANDO_BIN /usr/local/bin/sandod install destination
20 - # SANDO_DEPLOY_BRANCH main the only branch a self-update sha may live on
19 + # SANDO_BIN /usr/local/bin/sandod install destination
20 + # SANDO_DEPLOY_BRANCH main the only branch a self-update sha may live on
21 + # SANDO_DAEMON_CONFIG /etc/sando/sando-daemon.toml config the --check-config self-test loads
22 + #
23 + # Source: build from the LOCAL bare repo sandod already maintains
24 + # (/srv/sando/mnw.git), not a remote fetch. The self-update sha is one that was
25 + # already deployed as a server release, so `/rebuild` has force-updated
26 + # refs/heads/main in the bare repo to include it — the sha and a current, trusted
27 + # main are both present locally. Building from a remote (git@ssh.makenot.work)
28 + # gave the `sando` user no git creds and broke the moment git hosting was down —
29 + # it blocked the self-update during the 2026-07-09 deploy (postmortem #7). The
30 + # local bare repo has no such external dependency.
21 31 #
22 32 # Provenance: sandod (bearer-gated) only *triggers* this unit with a hex sha; it
23 33 # does not prove the sha is a commit anyone intended to deploy. Without a check,
24 - # the deploy token would be root code-exec on this host — push any branch to the
25 - # remote, self-update to its tip. So after fetch we REFUSE any sha that is not an
26 - # ancestor of origin/$SANDO_DEPLOY_BRANCH: a feature-branch tip, a force-pushed
27 - # commit, or an unknown sha never reaches the build/install lines (exit 4). A
28 - # signed-tag check is the planned follow-up once release signing exists.
34 + # the deploy token would be root code-exec on this host. So after fetch we REFUSE
35 + # any sha that is not an ancestor of origin/$SANDO_DEPLOY_BRANCH: a feature-branch
36 + # tip, an unknown sha, or a commit not on the deploy branch never reaches the
37 + # build/install lines (exit 4). Only sandod writes the bare repo (fetching main
38 + # from the authenticated upstream), so its main is a trustworthy provenance seal.
39 + # A signed-tag check is the planned follow-up once release signing exists.
40 + #
41 + # Safety net: a clean build (no stale incremental cache) plus a --check-config
42 + # self-test of the freshly built binary against the LIVE config gate the install.
43 + # A stale incremental object once produced a sandod that could not parse its own
44 + # node_health config and crash-looped (postmortem #6); either guard alone stops
45 + # that binary from ever being installed.
29 46 set -euo pipefail
30 47
31 48 SHA="${1:-}"
@@ -35,10 +52,11 @@ if [[ ! "$SHA" =~ ^[0-9a-f]{7,40}$ ]]; then
35 52 fi
36 53
37 54 SELF_DIR="${SANDO_SELF_UPDATE_DIR:-/srv/sando/self-update}"
38 - UPSTREAM_URL="${SANDO_UPSTREAM_URL:-git@ssh.makenot.work:max/makenotwork.git}"
55 + UPSTREAM_URL="${SANDO_UPSTREAM_URL:-/srv/sando/mnw.git}"
39 56 BUILD_USER="${SANDO_BUILD_USER:-sando}"
40 57 BIN="${SANDO_BIN:-/usr/local/bin/sandod}"
41 58 DEPLOY_BRANCH="${SANDO_DEPLOY_BRANCH:-main}"
59 + DAEMON_CONFIG="${SANDO_DAEMON_CONFIG:-/etc/sando/sando-daemon.toml}"
42 60 REPO_DIR="$SELF_DIR/MNW"
43 61 BUILD_HOME="$(getent passwd "$BUILD_USER" | cut -d: -f6)"
44 62
@@ -56,9 +74,13 @@ runuser -u "$BUILD_USER" -- env \
56 74 git clone '$UPSTREAM_URL' '$REPO_DIR'
57 75 fi
58 76 cd '$REPO_DIR'
77 + # Pin origin to the configured source every run, so switching
78 + # SANDO_UPSTREAM_URL (e.g. remote -> local bare repo) takes effect on an
79 + # already-cloned checkout instead of silently keeping the old remote.
80 + git remote set-url origin '$UPSTREAM_URL'
59 81 git fetch --prune origin
60 - # Provenance seal: the sha must be reachable from the deploy branch on the
61 - # canonical remote. --is-ancestor exits 1 for a non-ancestor and >1 for a
82 + # Provenance seal: the sha must be reachable from the deploy branch in the
83 + # source repo. --is-ancestor exits 1 for a non-ancestor and >1 for a
62 84 # bad/unresolvable ref, so any non-deploy-branch sha is refused fail-closed
63 85 # before a single line is built or installed.
64 86 if ! git merge-base --is-ancestor '$SHA' 'origin/$DEPLOY_BRANCH'; then
@@ -67,6 +89,11 @@ runuser -u "$BUILD_USER" -- env \
67 89 fi
68 90 git checkout --detach '$SHA'
69 91 cd sando
92 + # Clean build: wipe the workspace target so no stale incremental object
93 + # survives across shas. A reused pre-node_health Gate enum object once
94 + # produced a sandod that crash-looped on the current config (postmortem
95 + # #6). Self-updates are rare, so a full recompile is a cheap insurance.
96 + cargo clean
70 97 cargo build --release --locked -p sando-daemon
71 98 "
72 99
@@ -75,6 +102,17 @@ runuser -u "$BUILD_USER" -- env \
75 102 NEW_BIN="$REPO_DIR/sando/target/release/sandod"
76 103 [[ -x "$NEW_BIN" ]] || { echo "sando-self-update: build produced no binary at $NEW_BIN" >&2; exit 3; }
77 104
105 + # Self-test the fresh binary against the LIVE config BEFORE the swap: prove it can
106 + # load + parse the exact daemon config + topology sandod will boot against. Run as
107 + # the build user (not root) so the readability check matches the running daemon's
108 + # identity. A binary that can't parse the current config (the postmortem #6
109 + # brick) fails here and is never installed — sandod keeps running on the old one.
110 + echo "sando-self-update: self-testing $NEW_BIN against $DAEMON_CONFIG"
111 + if ! runuser -u "$BUILD_USER" -- env SANDO_CONFIG="$DAEMON_CONFIG" "$NEW_BIN" --check-config; then
112 + echo "sando-self-update: new binary FAILED --check-config against $DAEMON_CONFIG; refusing to install (sandod left running on the current binary)" >&2
113 + exit 5
114 + fi
115 +
78 116 # Install + restart as root. install is atomic (writes a temp then renames), so
79 117 # a concurrent exec of $BIN never sees a half-written file.
80 118 echo "sando-self-update: installing $NEW_BIN -> $BIN and restarting sandod"