| 1 |
# Sando topology config. |
| 2 |
# |
| 3 |
# Tiers run in declaration order. Each tier lists the gates that must pass to |
| 4 |
# unlock promotion *to* the next tier, the nodes it ships to, and the canary |
| 5 |
# policy for shipping within the tier. |
| 6 |
# |
| 7 |
# Day-one wiring: host (fw13, local) -> A (testnot.work) -> B (prod-1). C is |
| 8 |
# declared but not provisioned; adding the second prod node later is a config |
| 9 |
# edit (set provisioned = true, fill in [[tier.node]]). |
| 10 |
# |
| 11 |
# THIS FILE IS DEPLOYED, NOT A DOCUMENT. `sando-self-update.sh` installs it over |
| 12 |
# /etc/sando/sando.toml on every controller self-update, and |
| 13 |
# `sando-config-drift.timer` reports it daily when the two separate. Change what |
| 14 |
# sandod reads by editing here and shipping; do not hand-edit the installed copy. |
| 15 |
# See sando/deploy/README.md, "The topology is deployed, not hand-maintained". |
| 16 |
# |
| 17 |
# The first tier is "host" — it refers to whatever machine sandod runs on |
| 18 |
# (currently fw13). Renamed from the legacy "mm" name in Session 1 of |
| 19 |
# the sando bundle redesign. |
| 20 |
|
| 21 |
[repo] |
| 22 |
bare_path = "/srv/sando/mnw.git" |
| 23 |
branch = "main" |
| 24 |
# Canonical remote sandod fetches the deploy branch from before worktree-ing a |
| 25 |
# sha (pull-based /rebuild). Authorized via the `sando-deploy` read-only |
| 26 |
# service account on the makenot.work git server (fetch-only collaborator). |
| 27 |
# Leave unset for push-based hosts (commits arrive via the bare-repo hook). |
| 28 |
upstream = "git@ssh.makenot.work:max/makenotwork.git" |
| 29 |
|
| 30 |
# ---- auxiliary repos ---- |
| 31 |
# Extra repos fetched and checked out beside the per-sha worktree so a cross-repo |
| 32 |
# path dependency resolves at build time. The mnw-cli companion (built from the |
| 33 |
# MNW worktree) carries `synckit-client = { path = "../../synckit/synckit-client" }` |
| 34 |
# after synckit moved to its own repo; from <workdir>/<sha>/mnw-cli that resolves |
| 35 |
# to <workdir>/synckit, so synckit must be checked out there. checkout_dir is a |
| 36 |
# path under the workdir, nesting allowed; the checkout is shared across shas and |
| 37 |
# refreshed to `branch` HEAD each build. See the maintainer wiki, sando-overview. |
| 38 |
[[aux_repo]] |
| 39 |
name = "synckit" |
| 40 |
bare_path = "/srv/sando/synckit.git" |
| 41 |
upstream = "git@ssh.makenot.work:max/synckit.git" |
| 42 |
branch = "main" |
| 43 |
checkout_dir = "synckit" |
| 44 |
|
| 45 |
# The server and multithreaded both carry |
| 46 |
# `docengine = { path = "../../Libraries/docengine" }` after docengine left |
| 47 |
# MNW/shared on 2026-07-30. From <workdir>/<sha>/server that resolves to |
| 48 |
# <workdir>/Libraries/docengine, hence the nested checkout_dir. |
| 49 |
[[aux_repo]] |
| 50 |
name = "docengine" |
| 51 |
bare_path = "/srv/sando/docengine.git" |
| 52 |
upstream = "git@ssh.makenot.work:max/docengine.git" |
| 53 |
branch = "main" |
| 54 |
checkout_dir = "Libraries/docengine" |
| 55 |
|
| 56 |
# Prod-backup clones used by migration_dry_run on the Sando host, one per |
| 57 |
# database that has a [[migration_check]] in the daemon config. For localhost dev |
| 58 |
# a source can be a file:// path to a fixture dump. In prod we pull from |
| 59 |
# alpha-west-1 via a scoped `backup-puller` rrsync user, whose forced command is |
| 60 |
# `rrsync -ro /var/lib/mnw/backups` — so every path here is relative to that |
| 61 |
# directory, and the per-DB subdirs are what `server/deploy/backup-db.sh` writes. |
| 62 |
[[backup]] |
| 63 |
name = "server" |
| 64 |
source = "ssh://backup-puller@alpha-west-1:2200/makenotwork/latest.sql.gz" |
| 65 |
local_path = "/srv/sando/backups/latest.sql.gz" |
| 66 |
|
| 67 |
# multithreaded has its own database and applies its own migrations at boot |
| 68 |
# (multithreaded/src/main.rs, `sqlx::migrate!()`), so it needs its own dump to |
| 69 |
# dry-run against. Restoring the server's would prove nothing about it. |
| 70 |
[[backup]] |
| 71 |
name = "multithreaded" |
| 72 |
source = "ssh://backup-puller@alpha-west-1:2200/multithreaded/latest.sql.gz" |
| 73 |
local_path = "/srv/sando/backups/multithreaded-latest.sql.gz" |
| 74 |
|
| 75 |
# ---- host: fw13 local pre-staging gate ---- |
| 76 |
[[tier]] |
| 77 |
name = "host" |
| 78 |
provisioned = true |
| 79 |
canary = "sequential" |
| 80 |
# code_smoke runs FIRST: it boots the freshly-built binary against a throwaway |
| 81 |
# empty DB it migrates from scratch + seeds (the example catalog), then probes |
| 82 |
# /health. Fast + infra-light (no prod-dump restore), so a green here proves the |
| 83 |
# code is sound and isolates a later cargo_test / migration_dry_run red as an |
| 84 |
# environment problem rather than a code one. |
| 85 |
# |
| 86 |
# hardening_test is cargo_test's blind spot: cargo_test builds with |
| 87 |
# --features fast-tests, which relaxes the auth/sandbox rate limits and argon2 |
| 88 |
# cost AND #[ignore]s the whole rate-limiting suite, so the gate never touched |
| 89 |
# the auth hardening it exists to protect. hardening_test re-runs that suite |
| 90 |
# with no features, single-threaded, against production constants. It pays for |
| 91 |
# a second compile of the server's test binary; that is the cost of the |
| 92 |
# coverage. |
| 93 |
# |
| 94 |
# The lint + supply-chain gates. `-D warnings` used to be enforced in exactly |
| 95 |
# one place (server/deploy/run-ci.sh, which died with the astra pipeline) and |
| 96 |
# `cargo fmt --check` nowhere at all. fmt runs first: it needs no compilation, |
| 97 |
# so a formatting red comes back in seconds instead of after a full build. |
| 98 |
# cargo_audit and cargo_deny only run in crates carrying a triaged |
| 99 |
# .cargo/audit.toml / deny.toml -- four crates in this repo fail cargo audit |
| 100 |
# purely for lack of a reviewed advisory posture, and a permanently red gate |
| 101 |
# teaches everyone to ignore it. |
| 102 |
gates = [ |
| 103 |
{ kind = "fmt" }, |
| 104 |
{ kind = "code_smoke" }, |
| 105 |
{ kind = "cargo_test" }, |
| 106 |
{ kind = "hardening_test" }, |
| 107 |
{ kind = "clippy" }, |
| 108 |
{ kind = "cargo_audit" }, |
| 109 |
{ kind = "cargo_deny" }, |
| 110 |
{ kind = "migration_dry_run" }, |
| 111 |
{ kind = "boot_smoke" }, |
| 112 |
] |
| 113 |
# Host is the daemon's own machine (fw13); no remote node row. |
| 114 |
|
| 115 |
# ---- A: testnot.work staging ---- |
| 116 |
[[tier]] |
| 117 |
name = "a" |
| 118 |
provisioned = true |
| 119 |
canary = "sequential" |
| 120 |
# node_health (post-deploy) probes the deployed node over its executor — the |
| 121 |
# gate boot_smoke used to stand in for, but boot_smoke runs on the build host and |
| 122 |
# proves nothing about testnot-1. boot_smoke stays a host build-time gate only. |
| 123 |
# |
| 124 |
# A tier's gates guard promotion *out* of it, so THIS list is what stands between |
| 125 |
# testnot and production. manual_confirm lives here for that reason. It used to |
| 126 |
# sit only on tier b, where it guards b -> c — and c is not provisioned, so it |
| 127 |
# gated nothing: a prod ship was cleared by node_health + burn_in alone, and |
| 128 |
# hotfix: true skips burn_in, leaving one gate on a production deploy. |
| 129 |
# |
| 130 |
# hotfix does NOT skip manual_confirm (only burn_in), so every ship to prod-1 |
| 131 |
# now needs an explicit POST /confirm/a naming the version. That is the point: |
| 132 |
# there was no human sign-off anywhere on the path to production. |
| 133 |
gates = [ |
| 134 |
{ kind = "node_health" }, |
| 135 |
{ kind = "burn_in", hours = 48 }, |
| 136 |
{ kind = "manual_confirm" }, |
| 137 |
] |
| 138 |
[[tier.node]] |
| 139 |
name = "testnot-1" |
| 140 |
ssh_target = "deploy@testnot" # tailnet name; never the public IP / testnot.work hostname |
| 141 |
release_root = "/opt/mnw" |
| 142 |
service_name = "makenotwork.service" |
| 143 |
# What this box IS, checked against it before anything is pushed (2026-08-25). |
| 144 |
# Note it is NOT the same base as prod-1 below: staging is two Ubuntu releases |
| 145 |
# and four glibc point versions ahead, so tier A does not rehearse tier B on the |
| 146 |
# one axis this field exists for. That is a real gap, tracked separately; the |
| 147 |
# declaration states it rather than hiding it. |
| 148 |
base_image = "ubuntu/26.04" |
| 149 |
libc = "2.43" |
| 150 |
# Pre-swap config-drift guard: source this node's env and run the freshly-rsynced |
| 151 |
# binary in MNW_CHECK_CONFIG=1 mode BEFORE the symlink swap. A required var |
| 152 |
# missing here (how prod crash-looped on CDN_BASE_URL, postmortem 2026-07-09 #2) |
| 153 |
# fails the promote with the running service intact. The node runs 0.10.14, which |
| 154 |
# supports the mode; validated end-to-end 2026-07-10. |
| 155 |
config_check_env_file = "/etc/mnw/makenotwork.env" |
| 156 |
# Readiness probe on top of `systemctl is-active`. Without this, node_health |
| 157 |
# proves only that systemd thinks the unit is running — which a crash-looping |
| 158 |
# binary can satisfy between restarts. The node binds HOST=127.0.0.1 PORT=8080 |
| 159 |
# (/etc/mnw/makenotwork.env); verified serving 200 on 2026-07-21. |
| 160 |
health_url = "http://127.0.0.1:8080/health" |
| 161 |
# Companion: multithreaded. Re-enabled 2026-08-06, after the read-only demo was |
| 162 |
# provisioned on this box (GoingsOn multithreaded 34dc3ff6). It was removed in |
| 163 |
# 2fccc2d0 because testnot-1 had no mt at all, and a companion block for a node |
| 164 |
# that cannot install it fails the promote AFTER the symlink swap — the expensive |
| 165 |
# half of the failure. That is no longer the case; what is on the box now: |
| 166 |
# - postgres role + database `multithreaded`, peer auth over the unix socket |
| 167 |
# (DATABASE_URL=postgres:///multithreaded, matching prod) |
| 168 |
# - system user `multithreaded`, /opt/multithreaded, .env, and the unit from |
| 169 |
# multithreaded/deploy/multithreaded.service, enabled and running on |
| 170 |
# 127.0.0.1:3400, seeded via `--seed` |
| 171 |
# - /usr/local/lib/mnw/install-companion.sh + /etc/sudoers.d/mnw-companion |
| 172 |
# |
| 173 |
# The install path was exercised end to end before this line was uncommented, as |
| 174 |
# `deploy`, through the wrapper's own guards — a staged binary under |
| 175 |
# */releases/*/companions/* installed to /opt/multithreaded and restarted the |
| 176 |
# unit, exit 0. It is not inferred from the grant being present. |
| 177 |
# |
| 178 |
# mt is on testnot FIRST, unlike mnw-cli, because this is the tier where the 48h |
| 179 |
# burn-in runs: a forum that fails to boot on the new binary should fail here, |
| 180 |
# not on prod. See multithreaded/deploy/README.md. |
| 181 |
[[tier.node.companion]] |
| 182 |
name = "multithreaded" |
| 183 |
install_path = "/opt/multithreaded/multithreaded" |
| 184 |
service_name = "multithreaded.service" |
| 185 |
|
| 186 |
# ---- B: prod-1 ---- |
| 187 |
[[tier]] |
| 188 |
name = "b" |
| 189 |
provisioned = true |
| 190 |
canary = "sequential" |
| 191 |
# Guards b -> c. Kept for the day prod-2 is provisioned; it gates nothing today |
| 192 |
# (c has provisioned = false). The sign-off that matters for shipping to THIS |
| 193 |
# tier lives on tier a. |
| 194 |
gates = [ |
| 195 |
{ kind = "node_health" }, |
| 196 |
{ kind = "manual_confirm" }, |
| 197 |
] |
| 198 |
[[tier.node]] |
| 199 |
name = "prod-1" |
| 200 |
# Tailnet name; port 2200 supplied via /srv/sando/.ssh/config Host block. |
| 201 |
# Service user is "makenotwork" (pre-existing on prod), not "deploy" — chose |
| 202 |
# not to chown 885M of backups + redo postgres peer auth for a cosmetic rename. |
| 203 |
ssh_target = "makenotwork@alpha-west-1" |
| 204 |
release_root = "/opt/mnw" |
| 205 |
service_name = "makenotwork.service" |
| 206 |
# Measured 2026-08-25. The margin here is zero: makenotwork, mnw-cli and pom all |
| 207 |
# carry a GLIBC_2.39 floor and this box has exactly 2.39, so one point release of |
| 208 |
# drift on the build host breaks it. Declaring the number is what lets that be |
| 209 |
# compared before a build rather than by the pre-swap `ldd` guard afterwards. |
| 210 |
base_image = "ubuntu/24.04" |
| 211 |
libc = "2.39" |
| 212 |
# Pre-swap config-drift guard (see testnot-1 above). This node is the one the |
| 213 |
# 0.10.14 deploy crash-looped on a missing CDN_BASE_URL — the exact miss this |
| 214 |
# gate closes. First real exercise is the next prod promote. |
| 215 |
config_check_env_file = "/etc/mnw/makenotwork.env" |
| 216 |
# Readiness probe (see testnot-1). The unit binds HOST=0.0.0.0 PORT=3000, so the |
| 217 |
# probe dials it on loopback; verified serving 200 on 2026-07-21. This is the |
| 218 |
# node whose crash-loop `systemctl is-active` alone would have missed. |
| 219 |
health_url = "http://127.0.0.1:3000/health" |
| 220 |
# Companion: install the mnw-cli built in this same promote (see [[companion]] in |
| 221 |
# sando-daemon.toml) after the server is up, and restart its unit. Closes the |
| 222 |
# drift that broke git hosting during the 0.10.14 deploy — mnw-cli now ships from |
| 223 |
# the same sha as the server. Needs the node-side wrapper + sudoers grant |
| 224 |
# (deploy/install-companion.sh, deploy/mnw-companion.sudoers). |
| 225 |
[[tier.node.companion]] |
| 226 |
name = "mnw-cli" |
| 227 |
install_path = "/opt/mnw-cli/mnw-cli" |
| 228 |
service_name = "mnw-cli.service" |
| 229 |
# Companion: multithreaded, same sha as the server, same host. This is what |
| 230 |
# retires multithreaded/deploy/deploy-hetzner.sh (cross-compiled on macOS, scp'd |
| 231 |
# to root@, 41 commits behind when found). mt now rides the whole ladder — native |
| 232 |
# build on fw13, the migration_dry_run gate against its own dump, the test and |
| 233 |
# lint gates, and the server's burn-in on testnot — instead of a laptop and an |
| 234 |
# scp. Design: wiki sando-mt-pom-pipelines. |
| 235 |
# |
| 236 |
# prod-1 is the only node that installs it, because it is the only node that runs |
| 237 |
# mt (see the note on testnot-1 above). It already has the wrapper and the |
| 238 |
# sudoers grant, from mnw-cli. |
| 239 |
[[tier.node.companion]] |
| 240 |
name = "multithreaded" |
| 241 |
install_path = "/opt/multithreaded/multithreaded" |
| 242 |
service_name = "multithreaded.service" |
| 243 |
|
| 244 |
# ---- C: prod-2 (declared, not yet provisioned) ---- |
| 245 |
[[tier]] |
| 246 |
name = "c" |
| 247 |
provisioned = false |
| 248 |
canary = "sequential" |
| 249 |
gates = [ |
| 250 |
{ kind = "node_health" }, |
| 251 |
] |
| 252 |
# [[tier.node]] entries to be added when the second prod node ships. |
| 253 |
|