# Sando topology config. # # Tiers run in declaration order. Each tier lists the gates that must pass to # unlock promotion *to* the next tier, the nodes it ships to, and the canary # policy for shipping within the tier. # # Day-one wiring: host (fw13, local) -> A (testnot.work) -> B (prod-1). C is # declared but not provisioned; adding the second prod node later is a config # edit (set provisioned = true, fill in [[tier.node]]). # # The first tier is "host" — it refers to whatever machine sandod runs on # (currently fw13). Renamed from the legacy "mm" name in Session 1 of # the sando bundle redesign. [repo] bare_path = "/srv/sando/mnw.git" branch = "main" # Canonical remote sandod fetches the deploy branch from before worktree-ing a # sha (pull-based /rebuild). Authorized via the `sando-deploy` read-only # service account on the makenot.work git server (fetch-only collaborator). # Leave unset for push-based hosts (commits arrive via the bare-repo hook). upstream = "git@ssh.makenot.work:max/makenotwork.git" # ---- auxiliary repos ---- # Extra repos fetched and checked out beside the per-sha worktree so a cross-repo # path dependency resolves at build time. The mnw-cli companion (built from the # MNW worktree) carries `synckit-client = { path = "../../synckit/synckit-client" }` # after synckit moved to its own repo; from //mnw-cli that resolves # to /synckit, so synckit must be checked out there. checkout_dir is a # path under the workdir, nesting allowed; the checkout is shared across shas and # refreshed to `branch` HEAD each build. See the maintainer wiki, sando-overview. [[aux_repo]] name = "synckit" bare_path = "/srv/sando/synckit.git" upstream = "git@ssh.makenot.work:max/synckit.git" branch = "main" checkout_dir = "synckit" # The server and multithreaded both carry # `docengine = { path = "../../Libraries/docengine" }` after docengine left # MNW/shared on 2026-07-30. From //server that resolves to # /Libraries/docengine, hence the nested checkout_dir. [[aux_repo]] name = "docengine" bare_path = "/srv/sando/docengine.git" upstream = "git@ssh.makenot.work:max/docengine.git" branch = "main" checkout_dir = "Libraries/docengine" # Prod-backup clones used by migration_dry_run on the Sando host, one per # database that has a [[migration_check]] in the daemon config. For localhost dev # a source can be a file:// path to a fixture dump. In prod we pull from # alpha-west-1 via a scoped `backup-puller` rrsync user, whose forced command is # `rrsync -ro /var/lib/mnw/backups` — so every path here is relative to that # directory, and the per-DB subdirs are what `server/deploy/backup-db.sh` writes. [[backup]] name = "server" source = "ssh://backup-puller@alpha-west-1:2200/makenotwork/latest.sql.gz" local_path = "/srv/sando/backups/latest.sql.gz" # multithreaded has its own database and applies its own migrations at boot # (multithreaded/src/main.rs, `sqlx::migrate!()`), so it needs its own dump to # dry-run against. Restoring the server's would prove nothing about it. [[backup]] name = "multithreaded" source = "ssh://backup-puller@alpha-west-1:2200/multithreaded/latest.sql.gz" local_path = "/srv/sando/backups/multithreaded-latest.sql.gz" # ---- host: fw13 local pre-staging gate ---- [[tier]] name = "host" provisioned = true canary = "sequential" # code_smoke runs FIRST: it boots the freshly-built binary against a throwaway # empty DB it migrates from scratch + seeds (the example catalog), then probes # /health. Fast + infra-light (no prod-dump restore), so a green here proves the # code is sound and isolates a later cargo_test / migration_dry_run red as an # environment problem rather than a code one. # # hardening_test is cargo_test's blind spot: cargo_test builds with # --features fast-tests, which relaxes the auth/sandbox rate limits and argon2 # cost AND #[ignore]s the whole rate-limiting suite, so the gate never touched # the auth hardening it exists to protect. hardening_test re-runs that suite # with no features, single-threaded, against production constants. It pays for # a second compile of the server's test binary; that is the cost of the # coverage. # # The lint + supply-chain gates. `-D warnings` used to be enforced in exactly # one place (server/deploy/run-ci.sh, which died with the astra pipeline) and # `cargo fmt --check` nowhere at all. fmt runs first: it needs no compilation, # so a formatting red comes back in seconds instead of after a full build. # cargo_audit and cargo_deny only run in crates carrying a triaged # .cargo/audit.toml / deny.toml -- four crates in this repo fail cargo audit # purely for lack of a reviewed advisory posture, and a permanently red gate # teaches everyone to ignore it. gates = [ { kind = "fmt" }, { kind = "code_smoke" }, { kind = "cargo_test" }, { kind = "hardening_test" }, { kind = "clippy" }, { kind = "cargo_audit" }, { kind = "cargo_deny" }, { kind = "migration_dry_run" }, { kind = "boot_smoke" }, ] # Host is the daemon's own machine (fw13); no remote node row. # ---- A: testnot.work staging ---- [[tier]] name = "a" provisioned = true canary = "sequential" # node_health (post-deploy) probes the deployed node over its executor — the # gate boot_smoke used to stand in for, but boot_smoke runs on the build host and # proves nothing about testnot-1. boot_smoke stays a host build-time gate only. # # A tier's gates guard promotion *out* of it, so THIS list is what stands between # testnot and production. manual_confirm lives here for that reason. It used to # sit only on tier b, where it guards b -> c — and c is not provisioned, so it # gated nothing: a prod ship was cleared by node_health + burn_in alone, and # hotfix: true skips burn_in, leaving one gate on a production deploy. # # hotfix does NOT skip manual_confirm (only burn_in), so every ship to prod-1 # now needs an explicit POST /confirm/a naming the version. That is the point: # there was no human sign-off anywhere on the path to production. gates = [ { kind = "node_health" }, { kind = "burn_in", hours = 48 }, { kind = "manual_confirm" }, ] [[tier.node]] name = "testnot-1" ssh_target = "deploy@testnot" # tailnet name; never the public IP / testnot.work hostname release_root = "/opt/mnw" service_name = "makenotwork.service" # Pre-swap config-drift guard: source this node's env and run the freshly-rsynced # binary in MNW_CHECK_CONFIG=1 mode BEFORE the symlink swap. A required var # missing here (how prod crash-looped on CDN_BASE_URL, postmortem 2026-07-09 #2) # fails the promote with the running service intact. The node runs 0.10.14, which # supports the mode; validated end-to-end 2026-07-10. config_check_env_file = "/etc/mnw/makenotwork.env" # Readiness probe on top of `systemctl is-active`. Without this, node_health # proves only that systemd thinks the unit is running — which a crash-looping # binary can satisfy between restarts. The node binds HOST=127.0.0.1 PORT=8080 # (/etc/mnw/makenotwork.env); verified serving 200 on 2026-07-21. health_url = "http://127.0.0.1:8080/health" # Companion: multithreaded. Re-enabled 2026-08-06, after the read-only demo was # provisioned on this box (GoingsOn multithreaded 34dc3ff6). It was removed in # 2fccc2d0 because testnot-1 had no mt at all, and a companion block for a node # that cannot install it fails the promote AFTER the symlink swap — the expensive # half of the failure. That is no longer the case; what is on the box now: # - postgres role + database `multithreaded`, peer auth over the unix socket # (DATABASE_URL=postgres:///multithreaded, matching prod) # - system user `multithreaded`, /opt/multithreaded, .env, and the unit from # multithreaded/deploy/multithreaded.service, enabled and running on # 127.0.0.1:3400, seeded via `--seed` # - /usr/local/lib/mnw/install-companion.sh + /etc/sudoers.d/mnw-companion # # The install path was exercised end to end before this line was uncommented, as # `deploy`, through the wrapper's own guards — a staged binary under # */releases/*/companions/* installed to /opt/multithreaded and restarted the # unit, exit 0. It is not inferred from the grant being present. # # mt is on testnot FIRST, unlike mnw-cli, because this is the tier where the 48h # burn-in runs: a forum that fails to boot on the new binary should fail here, # not on prod. See multithreaded/deploy/README.md. [[tier.node.companion]] name = "multithreaded" install_path = "/opt/multithreaded/multithreaded" service_name = "multithreaded.service" # ---- B: prod-1 ---- [[tier]] name = "b" provisioned = true canary = "sequential" # Guards b -> c. Kept for the day prod-2 is provisioned; it gates nothing today # (c has provisioned = false). The sign-off that matters for shipping to THIS # tier lives on tier a. gates = [ { kind = "node_health" }, { kind = "manual_confirm" }, ] [[tier.node]] name = "prod-1" # Tailnet name; port 2200 supplied via /srv/sando/.ssh/config Host block. # Service user is "makenotwork" (pre-existing on prod), not "deploy" — chose # not to chown 885M of backups + redo postgres peer auth for a cosmetic rename. ssh_target = "makenotwork@alpha-west-1" release_root = "/opt/mnw" service_name = "makenotwork.service" # Pre-swap config-drift guard (see testnot-1 above). This node is the one the # 0.10.14 deploy crash-looped on a missing CDN_BASE_URL — the exact miss this # gate closes. First real exercise is the next prod promote. config_check_env_file = "/etc/mnw/makenotwork.env" # Readiness probe (see testnot-1). The unit binds HOST=0.0.0.0 PORT=3000, so the # probe dials it on loopback; verified serving 200 on 2026-07-21. This is the # node whose crash-loop `systemctl is-active` alone would have missed. health_url = "http://127.0.0.1:3000/health" # Companion: install the mnw-cli built in this same promote (see [[companion]] in # sando-daemon.toml) after the server is up, and restart its unit. Closes the # drift that broke git hosting during the 0.10.14 deploy — mnw-cli now ships from # the same sha as the server. Needs the node-side wrapper + sudoers grant # (deploy/install-companion.sh, deploy/mnw-companion.sudoers). [[tier.node.companion]] name = "mnw-cli" install_path = "/opt/mnw-cli/mnw-cli" service_name = "mnw-cli.service" # Companion: multithreaded, same sha as the server, same host. This is what # retires multithreaded/deploy/deploy-hetzner.sh (cross-compiled on macOS, scp'd # to root@, 41 commits behind when found). mt now rides the whole ladder — native # build on fw13, the migration_dry_run gate against its own dump, the test and # lint gates, and the server's burn-in on testnot — instead of a laptop and an # scp. Design: wiki sando-mt-pom-pipelines. # # prod-1 is the only node that installs it, because it is the only node that runs # mt (see the note on testnot-1 above). It already has the wrapper and the # sudoers grant, from mnw-cli. [[tier.node.companion]] name = "multithreaded" install_path = "/opt/multithreaded/multithreaded" service_name = "multithreaded.service" # ---- C: prod-2 (declared, not yet provisioned) ---- [[tier]] name = "c" provisioned = false canary = "sequential" gates = [ { kind = "node_health" }, ] # [[tier.node]] entries to be added when the second prod node ships.